From 3759b53e72f35dcbc349c4466ff7980d7ef6569f Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 17:41:27 +0800 Subject: [PATCH 01/27] kernel: reject whole compress batch when any summary fails length validation (#3003) --- kernel/package-lock.json | 2 +- kernel/package.json | 2 +- kernel/src/compress.ts | 52 +++++ kernel/tests/compress.test.ts | 15 +- .../issue3003-all-or-nothing-length.test.ts | 196 ++++++++++++++++++ kernel/tests/validation.test.ts | 9 +- package-lock.json | 2 +- 7 files changed, 269 insertions(+), 9 deletions(-) create mode 100644 kernel/tests/issue3003-all-or-nothing-length.test.ts diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 1956cd54ec..4c282670af 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.138", "lockfileVersion": 3, "requires": true, "packages": { diff --git a/kernel/package.json b/kernel/package.json index ed0ff46830..8ac891d931 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.138", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 7257814f02..6a9db1cd0b 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -600,6 +600,58 @@ export function createCore(ports: Ports = {}): CompressionCore { } } + // #3003: all-or-nothing summary-LENGTH pre-check. Length-class failures are + // pure functions of (spec.summary, spec.summaryMaxChars, cfg) — no state and + // no cross-range coverage — so they can be screened BEFORE any block commits. + // Screening them here turns the common PARTIAL outcome (one oversized summary + // rejects while sibling ranges land and open a seam) into a clean 0-commit + // reject: nothing lands, so the model's single retry pays ONE seam instead of + // two. Content-class failures (protected-zone / turn-integrity / no-new- + // messages) DO depend on state + cross-range coverage and stay per-range below. + // Scope is EXACTLY the fold-candidate set that mutates state in the loop — + // never every requested range, else a consumed range whose summary happens to + // be long would wrongly kill a batch whose real candidates all validate. + { + const lengthFailures: string[] = []; + for (const spec of input.ranges) { + if (skipSpecs.has(spec)) continue; + const resolution = classifications.get(spec); + if (resolution === undefined) continue; + const foldsFresh = resolution.status === "ok"; + const foldsRefold = + resolution.status === "consumed" && + refoldDecisions.get(spec)?.kind === "refold"; + if (!foldsFresh && !foldsRefold) continue; + try { + validateSummaryLength(spec, input.config.compress); + } catch (error) { + lengthFailures.push( + rangeError( + spec, + error instanceof Error ? error.message : String(error), + ), + ); + } + } + if (lengthFailures.length > 0) { + // Rejected batches carry their dead-ref tombstone side effect (#2362): + // the clone differs from input.state only in deadRefs before this point, + // so returning it is safe (same contract as the minCompressRange gate). + return { + state, + result: { + blocksCreated: 0, + tokensCompressed: 0, + errors: [ + `Batch rejected before folding any range — ${lengthFailures.length} of ${input.ranges.length} requested range(s) failed summary-length validation: ${lengthFailures.join(" | ")}. Nothing was folded, so no prefix was rewritten. Fix it one way or the other and resend: shorten or split each offending range's summary to fit its cap, then resend the whole batch; or drop the offending range(s) and resend only the rest that validate.`, + ...classificationErrors, + ], + warnings: [], + }, + }; + } + } + for (const spec of input.ranges) { if (skipSpecs.has(spec)) continue; const resolution = classifications.get(spec); diff --git a/kernel/tests/compress.test.ts b/kernel/tests/compress.test.ts index 7f65c0bae5..0d8383d033 100644 --- a/kernel/tests/compress.test.ts +++ b/kernel/tests/compress.test.ts @@ -486,7 +486,7 @@ test("applyCompression reports error for unknown boundary ref", () => { assert.equal(result.result.errors.length, 1); }); -test("batch compress attributes per-range errors and keeps partial success", () => { +test("batch compress rejects the whole batch when a summary fails length validation (#3003)", () => { const core = createCore(); const state = createInitialState(); const messages = [ @@ -526,11 +526,16 @@ test("batch compress attributes per-range errors and keeps partial success", () }), }); - assert.equal(result.result.blocksCreated, 1, "valid range still compresses"); + // #3003: a length-class failure in ANY candidate rejects the WHOLE batch — + // the valid sibling no longer lands (that partial commit opened a seam that + // the retry then paid twice). All-or-nothing means zero blocks here. + assert.equal(result.result.blocksCreated, 0, "no sibling may land while one summary fails length"); + assert.equal(result.state.blocks.length, 0); assert.equal(result.result.errors.length, 1); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); assert.match( result.result.errors[0]!, - /^range m00003\.\.m00004: Summary too short \(22 chars, min 50\)/, + /range m00003\.\.m00004: Summary too short \(22 chars, min 50\)/, ); }); @@ -1024,9 +1029,11 @@ test("empty summary is attributed to its range", () => { assert.equal(result.result.blocksCreated, 0); assert.equal(result.result.errors.length, 1); + // #3003: single-range length failures now surface through the all-or-nothing + // batch gate; the per-range detail is embedded verbatim in that message. assert.match( result.result.errors[0]!, - /^range m00001\.\.m00002: Summary is empty/, + /range m00001\.\.m00002: Summary is empty/, ); }); diff --git a/kernel/tests/issue3003-all-or-nothing-length.test.ts b/kernel/tests/issue3003-all-or-nothing-length.test.ts new file mode 100644 index 0000000000..e7aabb6ca9 --- /dev/null +++ b/kernel/tests/issue3003-all-or-nothing-length.test.ts @@ -0,0 +1,196 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { createInitialState } from "../src/state.js"; +import { prune } from "../src/prune.js"; +import { assignRefs } from "../src/refs.js"; +import type { Config, CoreMessage } from "../src/types.js"; + +// #3003: PARTIAL fold double-seam cost. When a multi-range compress batch has +// one length-class failure (summary over/under cap) among otherwise-valid +// folds, the old per-range commit landed the valid siblings FIRST (opening a +// seam) and the model then retried the bad range into a SECOND seam. The kernel +// now screens summary LENGTH all-or-nothing BEFORE any block commits: any +// length-class failure rejects the WHOLE batch (0 commits, 0 seams), so one +// retry pays once. Content-class failures (protected-zone / turn-integrity / +// no-new-messages) stay per-range. These tests pin both directions of the gate +// (#3617): it fires on a candidate's length failure, and it does NOT fire for +// a non-candidate range whose summary merely happens to be out of range. + +function msg( + id: string, + text: string, + role: CoreMessage["role"] = "user", +): CoreMessage { + return { id, role, contentType: "text", text }; +} + +function config(overrides: Partial = {}): Config { + return { + tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 }, + nudge: { + maxContextLimitPct: 0.55, + minContextLimitPct: 0.45, + frequency: 5, + iterationThreshold: 15, + force: "soft", + growthRatio: 0.05, + growthFloor: 6000, + growthCap: 50000, + minGrowthFloor: 5000, + minGrowthRatio: 0.45, + emergencyThresholdPct: 0.98, + }, + promotionThreshold: 5, + truncate: { threshold: 1 }, + merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 }, + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + protectedTools: [], + preserveRecentMessages: 0, + preserveRecentTokens: 0, + modelContextLimit: 100000, + ...overrides, + }; +} + +test("a too-long summary in a multi-range batch rejects the WHOLE batch (#3003)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + // Range A's summary exceeds maxSummaryLength (20); range B is within range. + // Without the pre-check, B would land (seam #1) and A would retry (seam #2). + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "x".repeat(40), topic: "too long" }, + { startRef: "m00003", endRef: "m00004", summary: "ok", topic: "fine" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(result.result.blocksCreated, 0, "nothing may commit"); + assert.equal(result.state.blocks.length, 0, "no block may exist after rejection"); + assert.equal(result.result.errors.length, 1); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); + assert.match( + result.result.errors[0]!, + /range m00001\.\.m00002: Summary too long \(40 chars, max 20\)/, + ); + assert.match(result.result.errors[0]!, /Nothing was folded, so no prefix was rewritten/); + // Both actionable exits must be offered. + assert.match(result.result.errors[0]!, /shorten or split/); + assert.match(result.result.errors[0]!, /drop the offending range\(s\) and resend only the rest/); +}); + +test("a too-short summary in a multi-range batch rejects the WHOLE batch (#3003)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "ok", topic: "fine" }, + { startRef: "m00003", endRef: "m00004", summary: "x".repeat(5), topic: "too short" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 50 }, + }), + }); + + assert.equal(result.result.blocksCreated, 0, "the valid sibling must NOT land either"); + assert.equal(result.state.blocks.length, 0); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); + assert.match( + result.result.errors[0]!, + /range m00003\.\.m00004: Summary too short \(5 chars, min 50\)/, + ); +}); + +test("a multi-range batch whose summaries all validate still commits every range (#3003 control)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "ok-a", topic: "a" }, + { startRef: "m00003", endRef: "m00004", summary: "ok-b", topic: "b" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(result.result.blocksCreated, 2, "both valid ranges commit"); + assert.equal(result.state.blocks.length, 2); + assert.equal(result.result.errors.length, 0); +}); + +test("a non-candidate range with an oversized summary does NOT reject a valid fresh fold (#3003 scoping)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [msg("a", "alpha"), msg("b", "beta")]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + // m00099..m00100 are valid-format but never allocated -> status "unknown", + // never a fold candidate. Its oversized summary must be ignored by the + // all-or-nothing length screen (checking it would wrongly kill the batch). + const result = core.applyCompression({ + ranges: [ + { startRef: "m00099", endRef: "m00100", summary: "z".repeat(500) }, + { startRef: "m00001", endRef: "m00002", summary: "ok", topic: "fine" }, + ], + messages, + state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(result.result.blocksCreated, 1, "the valid candidate still commits"); + assert.equal(result.state.blocks.length, 1); + assert.doesNotMatch( + result.result.errors.join("\n"), + /Batch rejected before folding any range/, + ); + // The unknown ref is still reported per-range (content-class, unchanged). + assert.match(result.result.errors.join("\n"), /m00099/); +}); diff --git a/kernel/tests/validation.test.ts b/kernel/tests/validation.test.ts index 09286377cd..c32cb46c5a 100644 --- a/kernel/tests/validation.test.ts +++ b/kernel/tests/validation.test.ts @@ -259,7 +259,7 @@ test("validation: all checks disabled (0s) allows any input", () => { assert.equal(result.result.errors.length, 0); }); -test("validation: errors are collected per-range in batch compress", () => { +test("validation: an empty-summary range rejects the whole batch (#3003)", () => { const core = createCore(); const messages = [ msg("a", longText), @@ -282,7 +282,12 @@ test("validation: errors are collected per-range in batch compress", () => { state, config, }); - assert.equal(result.result.blocksCreated, 1); + // #3003: a length-class failure (empty summary) in ANY candidate rejects the + // WHOLE batch — the valid sibling no longer lands. Content-class failures + // (unknown/invalid refs) remain per-range; see the invalid-refs batch test. + assert.equal(result.result.blocksCreated, 0); + assert.equal(result.state.blocks.length, 0); assert.equal(result.result.errors.length, 1); + assert.match(result.result.errors[0]!, /Batch rejected before folding any range/); assert.match(result.result.errors[0]!, /empty/i); }); diff --git a/package-lock.json b/package-lock.json index 101b764601..7bcf65f2a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.138", "dev": true, "license": "MIT", "devDependencies": { From 69b2ad14473d34b57126f3c1757e2592a584e3c4 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 17:42:03 +0800 Subject: [PATCH 02/27] fix: classify batch summary-length rejections in gateReason + review pins (#3003, absorbs #3670) --- .../issue3003-all-or-nothing-length.test.ts | 59 ++++++++++++++++++- src/stream.ts | 1 + 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/kernel/tests/issue3003-all-or-nothing-length.test.ts b/kernel/tests/issue3003-all-or-nothing-length.test.ts index e7aabb6ca9..6e8b942945 100644 --- a/kernel/tests/issue3003-all-or-nothing-length.test.ts +++ b/kernel/tests/issue3003-all-or-nothing-length.test.ts @@ -111,7 +111,7 @@ test("a too-short summary in a multi-range batch rejects the WHOLE batch (#3003) const result = core.applyCompression({ ranges: [ - { startRef: "m00001", endRef: "m00002", summary: "ok", topic: "fine" }, + { startRef: "m00001", endRef: "m00002", summary: "y".repeat(60), topic: "fine" }, { startRef: "m00003", endRef: "m00004", summary: "x".repeat(5), topic: "too short" }, ], messages, @@ -194,3 +194,60 @@ test("a non-candidate range with an oversized summary does NOT reject a valid fr // The unknown ref is still reported per-range (content-class, unchanged). assert.match(result.result.errors.join("\n"), /m00099/); }); + +test("a too-long refold summary rejects the WHOLE batch incl. a fresh sibling (#3003 refold direction)", () => { + const core = createCore(); + const state = createInitialState(); + const messages = [ + msg("a", "alpha"), + msg("b", "beta"), + msg("c", "gamma"), + msg("d", "delta"), + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + // First fold m00001..m00002 into a block. + const first = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "first fold", topic: "t" }, + ], + messages, + state, + config: config(), + }); + assert.equal(first.result.blocksCreated, 1); + const blockId = first.state.blocks[0]!.blockId; + + // Restore it inline so re-requesting its span classifies as a refold candidate. + first.state.blocks.find((b) => b.blockId === blockId)!.restoredInline = true; + + // Second call: refold with an OVERSIZED new summary + a fresh valid sibling. + // Without the pre-check the fresh sibling would land first (seam #1) and the + // refold retry would pay seam #2; the gate must reject before either commits. + const second = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "z".repeat(400) }, + { startRef: "m00003", endRef: "m00004", summary: "fresh ok", topic: "sib" }, + ], + messages, + state: first.state, + config: config({ + compress: { minCompressRange: 0, maxSummaryLength: 20, minSummaryLength: 0 }, + }), + }); + + assert.equal(second.result.blocksCreated, 0, "nothing may commit"); + assert.equal(second.state.blocks.length, 1, "only the pre-existing block remains"); + assert.match(second.result.errors[0]!, /Batch rejected before folding any range/); + assert.match( + second.result.errors[0]!, + /range m00001\.\.m00002: Summary too long \(400 chars, max 20\)/, + ); + // Refold atomicity: the restored block keeps its old summary (no partial + // in-place update may precede the batch rejection). + const b1 = second.state.blocks.find((b) => b.blockId === blockId)!; + assert.equal(b1.summary, "first fold"); +}); diff --git a/src/stream.ts b/src/stream.ts index 74a8c6bf21..079170b960 100644 --- a/src/stream.ts +++ b/src/stream.ts @@ -913,6 +913,7 @@ export function applyRanges(parsed: ReturnType, ctx: : /already compressed/.test(errs) ? "gate:already-compressed" : /requested range\(s\) resolved/.test(errs) ? "gate:none-resolved" : /too small/.test(errs) ? "gate:too-small" + : /Batch rejected before folding any range/.test(errs) ? "gate:summary-length" : "gate:other"; const revNote = revs.length > 0 ? ` Note: startId > endId in range(s) ${revs.map((rg) => `${rg.startRef}→${rg.endRef}`).join(", ")} — your refs were reversed; they were normalized to ascending order before evaluation, so check your ref order.` From fd0b29ebff54ec0cc06f46040cf6ea2266303c44 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 17:48:35 +0800 Subject: [PATCH 03/27] =?UTF-8?q?kernel:=20all-or-nothing=20batch=20when?= =?UTF-8?q?=20a=20range=20dies=20during=20application=20(#3939=20=E2=91=A3?= =?UTF-8?q?,=20closes=20#3003=20double-seam)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/src/compress.ts | 37 ++++++++++++++++++- .../issue3939-fold-boundary-pins.test.ts | 35 +++++++++++++++++- kernel/tests/protect-recent.test.ts | 14 ++++--- 3 files changed, 78 insertions(+), 8 deletions(-) diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 6a9db1cd0b..e33bb300fb 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -652,6 +652,21 @@ export function createCore(ports: Ports = {}): CompressionCore { } } + // #3939 (PR-B item ④, extends #3670/#3003 to content class): the length + // pre-check above cannot screen content-class fatals (protected-zone / + // turn-integrity / livelock / empty-range) because they depend on state + // plus cross-range coverage — range N+1's eligibility may hinge on range N + // having landed. So apply the loop and keep a pristine pre-loop snapshot; + // any apply-level throw rejects the WHOLE batch with zero commits instead + // of leaving earlier siblings landed (the #3003 double-seam shape: a fold + // seam PLUS the retry seam on the model's next attempt). Classification- + // level outcomes (unknown / invalid / consumed-skip) never mutate state, + // so they stay per-range and do NOT poison the batch. + const preLoopState = cloneState(state); + const applyErrors: string[] = []; + const warningsBase = warnings.length; + const notesBase = notes.length; + for (const spec of input.ranges) { if (skipSpecs.has(spec)) continue; const resolution = classifications.get(spec); @@ -669,7 +684,7 @@ export function createCore(ports: Ports = {}): CompressionCore { }); blocksCreated += decision.blocks.length; } catch (error) { - errors.push( + applyErrors.push( rangeError( spec, error instanceof Error ? error.message : String(error), @@ -709,7 +724,7 @@ export function createCore(ports: Ports = {}): CompressionCore { tokensCompressed += outcome.tokens; warnings.push(...outcome.warnings); } catch (error) { - errors.push( + applyErrors.push( rangeError( spec, error instanceof Error ? error.message : String(error), @@ -718,6 +733,24 @@ export function createCore(ports: Ports = {}): CompressionCore { } } + if (applyErrors.length > 0) { + // Same contract as the gate paths above: the returned state differs + // from input.state only in the #2362 dead-ref tombstones recorded during + // classification. Nothing else landed, so no prefix was rewritten. + return { + state: preLoopState, + result: { + blocksCreated: 0, + tokensCompressed: 0, + errors: [ + `Batch rejected before folding any range — ${applyErrors.length} of ${input.ranges.length} requested range(s) failed during application: ${applyErrors.join(" | ")}. Nothing was folded, so no prefix was rewritten. Fix or drop the offending range(s), then resend the whole batch.`, + ...classificationErrors, + ], + warnings: warnings.slice(0, warningsBase), + }, + }; + } + state.stats.compressionCount += blocksCreated; state.stats.tokensCompressed += tokensCompressed; diff --git a/kernel/tests/issue3939-fold-boundary-pins.test.ts b/kernel/tests/issue3939-fold-boundary-pins.test.ts index 2ed635b6d7..bfd1488b6d 100644 --- a/kernel/tests/issue3939-fold-boundary-pins.test.ts +++ b/kernel/tests/issue3939-fold-boundary-pins.test.ts @@ -295,7 +295,7 @@ test("pin: already-truncated text is skipped on the next pass (marker idempotenc assert.equal(second.truncatedCount, 0); }); -test("pin: applyCompression lands front ranges even when a later range errors (non-atomic partial landing, #3003 shape)", () => { +test("pin: unknown-ref sibling errors do not poison valid ranges (classification-level partial report)", () => { const core = createCore(); const state = createInitialState(); const messages: CoreMessage[] = [ @@ -326,3 +326,36 @@ test("pin: applyCompression lands front ranges even when a later range errors (n /m00009.*above this session's highest allocated ref/, ); }); + +test("pin: a range that dies DURING application rejects the whole batch all-or-nothing (#3939 ④, closes the #3003 double-seam)", () => { + const core = createCore(); + const state = createInitialState(); + const messages: CoreMessage[] = [ + { id: "a", role: "user", contentType: "text", text: "alpha" }, + { id: "b", role: "user", contentType: "text", text: "beta" }, + { id: "c", role: "user", contentType: "text", text: "gamma" }, + ]; + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const result = core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00002", summary: "s1", topic: "t1" }, + { startRef: "m00003", endRef: "m00003", summary: "s2", topic: "t2" }, + ], + messages, + state, + config: config(), + protectedMessageIds: new Set(["m00003"]), + }); + + assert.equal(result.result.blocksCreated, 0); + assert.equal(result.state.blocks.length, 0); + assert.equal(result.state.stats.compressionCount, 0); + assert.match( + result.result.errors[0]!, + /Batch rejected before folding any range.*entirely within the protected zone/s, + ); +}); diff --git a/kernel/tests/protect-recent.test.ts b/kernel/tests/protect-recent.test.ts index 62319940b7..3e0ebd8d4a 100644 --- a/kernel/tests/protect-recent.test.ts +++ b/kernel/tests/protect-recent.test.ts @@ -346,8 +346,12 @@ test("warnings accumulate across multiple ranges in one batch", () => { const state = seededState(messages); const cfg = config({ preserveRecentMessages: 3 }); - // Two ranges: each partially overlaps the recent zone (m00008..m00010). - // Both should produce warnings, and both unprotected heads should compress. + // Two ranges: the second now ends inside the recent zone, which under the + // #3939 all-or-nothing contract would reject the whole batch (a fully- + // protected range is a fatal apply error, pinned in issue3939-fold-boundary- + // pins.test.ts), so it instead OVERLAPS the first range's span — overlap + // warn+skip keeps the batch valid while still exercising multi-range + // warning accumulation. const result = core.applyCompression({ ranges: [ { @@ -357,9 +361,9 @@ test("warnings accumulate across multiple ranges in one batch", () => { topic: "a", }, { - startRef: "m00010", + startRef: "m00008", endRef: "m00010", - summary: "second range fully protected tail", + summary: "second range overlaps the first", topic: "b", }, ], @@ -372,7 +376,7 @@ test("warnings accumulate across multiple ranges in one batch", () => { result.result.blocksCreated >= 1, "at least the unprotected head compresses", ); - assert.ok(result.result.warnings.length >= 1, "warnings surfaced"); + assert.ok(result.result.warnings.length >= 2, "warnings surfaced"); }); // --- neverPreserveRecentTools is configurable (billion-context#1277) --- From 5d6b22abb3b01d5f17b430da4d2f27172d9ee3e0 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 13:22:21 +0800 Subject: [PATCH 04/27] kernel: gate failed-compress orphan rotation on committed folds (#3769) The kernel kept only the newest TWO unmatched compress call+result pairs and hid older ones on EVERY turn, decoupled from real folds. On a third failed compress this deleted the oldest pair mid-history, rewriting an earlier cached prefix at full cost with no fold behind the break. Replace the keep-two rotation with fold-gated, position-aware, persistent sweeping: orphans stay visible between folds (append-only, cache-friendly); a committed fold stamps pendingOrphanSweepMinRef (earliest new-block startRef) and the next pass hides only orphans at/after that ref, so the sweep rides the fold's own rewrite (cache-free) while earlier orphans (still-cached prefix) survive; swept ids persist in sweptOrphanCallIds (monotonic) so a resumed session never resurrects them. Bump acp-kernel 0.0.136 -> 0.0.137. --- kernel/src/compress.ts | 44 ++++- kernel/src/hide-consumed.ts | 81 +++++---- kernel/src/types.ts | 25 ++- kernel/tests/hide-consumed-mcp.test.ts | 16 +- kernel/tests/hide-merge-rebuild.test.ts | 45 +++-- kernel/tests/issue3769-orphan-sweep.test.ts | 189 ++++++++++++++++++++ kernel/tests/orphan-endpoint-snap.test.ts | 3 +- 7 files changed, 347 insertions(+), 56 deletions(-) create mode 100644 kernel/tests/issue3769-orphan-sweep.test.ts diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index e33bb300fb..83a471c16b 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -666,6 +666,10 @@ export function createCore(ports: Ports = {}): CompressionCore { const applyErrors: string[] = []; const warningsBase = warnings.length; const notesBase = notes.length; + // #3769: capture which blocks already exist so, after the loop, we can tell + // which were created by THIS fold and where it starts (for the orphan sweep). + const preFoldBlockIds = new Set(state.blocks.map((b) => b.blockId)); + for (const spec of input.ranges) { if (skipSpecs.has(spec)) continue; @@ -755,6 +759,24 @@ export function createCore(ports: Ports = {}): CompressionCore { state.stats.tokensCompressed += tokensCompressed; if (blocksCreated > 0) { + // #3769: record where THIS fold starts (min over the newly-created + // blocks' startRef) so the next processTurn sweeps orphan failures at/after + // it cache-neutrally, leaving orphans before it in the still-cached prefix. + // Kept as the running min across folds that commit before a sweep consumes it. + let foldStartRef: number | undefined; + for (const b of state.blocks) { + if (preFoldBlockIds.has(b.blockId)) continue; + const n = Number((b.startRef ?? "").replace(/\D/g, "")); + if (Number.isFinite(n) && (foldStartRef === undefined || n < foldStartRef)) { + foldStartRef = n; + } + } + if (foldStartRef !== undefined) { + state.pendingOrphanSweepMinRef = + state.pendingOrphanSweepMinRef === undefined + ? foldStartRef + : Math.min(state.pendingOrphanSweepMinRef, foldStartRef); + } // Compress succeeded: clear the growth baseline so the next turn // re-establishes it at the new (lower) token count. Without this the // nudge re-fires in a feedback loop (the §5.7 baseline-reset bug). @@ -1081,10 +1103,26 @@ const hideCompressCallsNode: PipelineNode = { io.messages, ctx.config.isCompressTool, ); + // #3769: fold the newly-swept orphans into the persistent monotonic set so + // they stay hidden across turns/restarts/upgrades, then consume the fold + // marker (one-shot) — a quiet turn leaves both untouched except clearing a + // stale marker. + const patch: Partial = { + hiddenOrphanRefs: hidden.hiddenOrphanRefs, + pendingOrphanSweepMinRef: undefined, + }; + if (hidden.newlySweptCallIds.length > 0) { + patch.sweptOrphanCallIds = [ + ...new Set([ + ...(io.state.sweptOrphanCallIds ?? []), + ...hidden.newlySweptCallIds, + ]), + ]; + } return { ...io, messages: hidden.messages, - state: { ...io.state, hiddenOrphanRefs: hidden.hiddenOrphanRefs }, + state: { ...io.state, ...patch }, }; }, }; @@ -1879,6 +1917,10 @@ function cloneState(state: CompressionState): CompressionState { hiddenOrphanRefs: state.hiddenOrphanRefs ? [...state.hiddenOrphanRefs] : undefined, + sweptOrphanCallIds: state.sweptOrphanCallIds + ? [...state.sweptOrphanCallIds] + : undefined, + pendingOrphanSweepMinRef: state.pendingOrphanSweepMinRef, deadRefs: state.deadRefs ? [...state.deadRefs] : undefined, lastPassIds: state.lastPassIds ? [...state.lastPassIds] : undefined, }; diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index 25e674605d..d2db4722a0 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -4,28 +4,42 @@ import { COMPRESS_TOOL_NAME } from "./compress-tools.js"; import type { CompressionState, CoreMessage } from "./types.js"; // Orphaned compress calls (no matching block — failed attempts, or historical -// calls whose blocks predate compressCallId recording) keep their NEWEST two -// call+result pairs visible: failures must stay observable or a deterministic -// model re-issues the same no-op compress forever, pinned at a fixed point -// (billion-context-pi issue #9: 3,849 identical calls over 5h13m under -// KEEP_LAST_ORPHANED=0). Older orphans are hidden, so the residue is bounded -// at two pairs regardless of session length — PR #18's unbounded accumulation -// does not return (its own live check showed the cap: 10 in → 6 out). -const KEEP_LAST_ORPHANED = 2; +// calls whose blocks predate compressCallId recording) stay visible until a REAL +// fold gives a cache-safe place to remove them (#3769). Failures must remain +// observable or a deterministic model re-issues the same no-op compress forever +// (billion-context-pi #9: 3,849 identical calls over 5h13m under keep=0), AND a +// quiet turn must never move the cached prefix breakpoint forward just to delete +// a few hundred chars of old failure residue — the old keep-last-two rotation did +// exactly that (hid the oldest failure on EVERY turn, seam cost far exceeding the +// bytes removed). Now: between folds nothing is swept; on a fold the node sweeps +// orphans at/after the fold's earliest start ref onto the persistent +// sweptOrphanCallIds set (they stay hidden forever), leaving orphans before it +// visible. Residue is bounded by fold cadence, not a fixed pair count. export interface HideConsumedResult { messages: CoreMessage[]; hidden: number; - /** Padded refs of the orphan compress call/result messages hidden by this - * pass (orphans beyond the keep window). Consumed by boundary resolution to - * snap a range endpoint onto a hidden-but-intact orphan (#396). */ + /** Padded refs of ALL currently-hidden orphan compress call/result messages + * (the persistent swept set, not just this pass). Consumed by boundary + * resolution to snap a range endpoint onto a hidden-but-intact orphan (#396). */ hiddenOrphanRefs: string[]; + /** Orphan call ids newly swept by THIS pass (only when a fold was pending); + * the node unions these into state.sweptOrphanCallIds. Empty on quiet turns. */ + newlySweptCallIds: string[]; } function rangeKey(startRef: string, endRef: string): string { return `${startRef}::${endRef}`; } +// Numeric position of a padded ref (mNNNNN -> NNNNN); null when unmapped/blocked +// so the caller treats it as "position unknown" and never sweeps on a guess. +function refNum(ref: string | undefined): number | null { + if (!ref || ref === BLOCKED_REF) return null; + const n = Number(ref.replace(/\D/g, "")); + return Number.isFinite(n) ? n : null; +} + // Named return type (not inline): with the object literal in the signature, // prettier's layout heuristic was not idempotent here (#416). interface ParsedCallText { @@ -240,18 +254,15 @@ export function hideConsumedCompressCalls( }, ); - const lastOrphaned = new Set(); - for ( - let i = messages.length - 1; - i >= 0 && lastOrphaned.size < KEEP_LAST_ORPHANED; - i-- - ) { - const info = infos[i]!; - const callId = messages[i]!.toolCallId; - if (info.isCompressCall && callId && info.fate === "orphan") { - lastOrphaned.add(callId); - } - } + // #3769: orphans are removed only where a real fold already moved the cached + // breakpoint. sweptSet holds orphans removed on earlier folds (persistent + + // monotonic across turns/restarts/upgrades); pendingOrphanSweepMinRef carries + // this fold's earliest start ref so we additionally sweep orphans at/after it. + // Orphans before it sit in the still-cached prefix and stay visible; with no + // fold pending nothing new is swept, so a quiet turn never breaks the prefix. + const sweptSet = new Set(state.sweptOrphanCallIds ?? []); + const foldMinRef = state.pendingOrphanSweepMinRef; + const newlySweptCallIds: string[] = []; const keepCallIds = new Set(); const hiddenCallIds = new Set(); @@ -259,14 +270,22 @@ export function hideConsumedCompressCalls( const info = infos[i]!; const callId = messages[i]!.toolCallId; if (!info.isCompressCall || !callId) continue; - if ( - info.fate === "inactive" || - (info.fate === "orphan" && !lastOrphaned.has(callId)) - ) { - hiddenCallIds.add(callId); - } else { - keepCallIds.add(callId); + let hide = false; + if (info.fate === "inactive") { + hide = true; + } else if (info.fate === "orphan") { + if (sweptSet.has(callId)) { + hide = true; + } else if (foldMinRef !== undefined) { + const num = refNum(state.messageRefs.byRaw[messages[i]!.id]); + if (num !== null && num >= foldMinRef) { + hide = true; + newlySweptCallIds.push(callId); + } + } } + if (hide) hiddenCallIds.add(callId); + else keepCallIds.add(callId); } const fateByCallId = new Map(); @@ -349,5 +368,5 @@ export function hideConsumedCompressCalls( result.push(message); } - return { messages: result, hidden, hiddenOrphanRefs }; + return { messages: result, hidden, hiddenOrphanRefs, newlySweptCallIds }; } diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 22523eeea7..3c56b1a1c2 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -240,8 +240,29 @@ export interface CompressionState { * dead-ending the whole range (#396). A drifted/dangling ref (#387) is NOT in * this set and keeps the clear "cannot be anchored" error. Optional for * pre-feature persisted states. */ - hiddenOrphanRefs?: string[]; - /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but + hiddenOrphanRefs?: string[]; + /** Persistent, MONOTONIC set of orphan compress-call ids permanently swept + * (#3769). An orphan (a failed compress call/result with no matching block) + * is only removed from the visible view when a REAL fold gives a cache-safe + * place to do it: the hide node sweeps orphans at/after the fold's earliest + * start ref onto this set, and they stay hidden forever — across the next + * turn, repeated processTurn calls, restarts and upgrades. Monotonic (never + * shrinks) so a previously-hidden record can never reappear once the client + * resends full history. Replaces the old keep-last-two rotation, which hid + * the oldest failure on EVERY quiet turn and moved the cached prefix + * breakpoint forward for no new fold. Optional: pre-feature persisted states + * lack it (nothing was permanently hidden under the old per-pass rule). */ + sweptOrphanCallIds?: string[]; + /** Numeric part (mNNNNN -> NNNNN) of the EARLIEST start ref of the most + * recent fold that has not yet been consumed by an orphan sweep (#3769). + * Written by applyCompression when it creates >=1 block (min over the newly + * created blocks' startRef); consumed + cleared by the hide-compress-calls + * node on the next processTurn, which then sweeps orphans with ref >= this + * value (orphans before it sit in the still-cached prefix and stay visible). + * Carries the position so the sweep is cache-neutral: it only removes records + * the fold already invalidated. Optional: absent until the first post-fix fold. */ + pendingOrphanSweepMinRef?: number; + /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk * history rewrite drops them outright), so no range citing them can ever diff --git a/kernel/tests/hide-consumed-mcp.test.ts b/kernel/tests/hide-consumed-mcp.test.ts index fdeb460487..0f618de67c 100644 --- a/kernel/tests/hide-consumed-mcp.test.ts +++ b/kernel/tests/hide-consumed-mcp.test.ts @@ -89,8 +89,11 @@ test("#2970 L2: active synthetic-id carrier survives beyond the orphan window", const r = hideConsumedCompressCalls(state, msgs, isMcp); const kept = new Set(r.messages.map((m) => m.toolCallId)); assert.ok(kept.has("tool_a"), "active synthetic-id carrier must survive"); - assert.ok(!kept.has("tool_b"), "oldest true orphan still capped at two"); - assert.ok(kept.has("tool_c") && kept.has("tool_d"), "newest two orphans kept"); + assert.equal(r.hidden, 0, "no fold pending -> nothing swept (#3769)"); + assert.ok( + kept.has("tool_b") && kept.has("tool_c") && kept.has("tool_d"), + "orphans remain visible until a real fold", + ); }); test("#2970 L2: inactive synthetic-id carrier is hidden (stale receipt cleanup)", () => { @@ -112,7 +115,7 @@ test("#2970 L2: inactive synthetic-id carrier is hidden (stale receipt cleanup)" assert.equal(r.hidden, 2, "call + result removed"); }); -test("#2970/#2362: mcp__bili__compress orphans keep only the newest two", () => { +test("#2970/#2362: mcp__bili__compress orphans stay visible between folds (#3769)", () => { const state = createInitialState(); const msgs: CoreMessage[] = []; for (const id of ["o1", "o2", "o3", "o4"]) { @@ -121,8 +124,11 @@ test("#2970/#2362: mcp__bili__compress orphans keep only the newest two", () => const r = hideConsumedCompressCalls(state, msgs, isMcp); const kept = new Set(r.messages.map((m) => m.toolCallId)); - assert.ok(!kept.has("o1") && !kept.has("o2"), "older orphans hidden"); - assert.ok(kept.has("o3") && kept.has("o4"), "newest two kept"); + assert.equal(r.hidden, 0, "quiet turn sweeps nothing (#3769)"); + assert.ok( + ["o1", "o2", "o3", "o4"].every((id) => kept.has(id)), + "all orphans remain visible until a real fold (#3769)", + ); }); test("#2970: active synthetic-id carrier's long summary is stubbed", () => { diff --git a/kernel/tests/hide-merge-rebuild.test.ts b/kernel/tests/hide-merge-rebuild.test.ts index 96d5e9b4a9..1cfe694e5a 100644 --- a/kernel/tests/hide-merge-rebuild.test.ts +++ b/kernel/tests/hide-merge-rebuild.test.ts @@ -27,7 +27,7 @@ function block(overrides: Partial): CompressionBlock { }; } -test("hideConsumedCompressCalls keeps active-block calls, hides consumed, keeps newest two orphans", () => { +test("hideConsumedCompressCalls keeps active-block calls and orphans until a fold (#3769)", () => { const state: CompressionState = { ...createInitialState(), blocks: [ @@ -70,20 +70,18 @@ test("hideConsumedCompressCalls keeps active-block calls, hides consumed, keeps { id: "m5", role: "user", contentType: "text", text: "hello" }, ]; const result = hideConsumedCompressCalls(state, messages); - // active-block call kept; consumed hidden; newest KEEP_LAST_ORPHANED=2 - // orphans kept visible (failure observability — billion-context-pi #9). - assert.equal(result.hidden, 1); + assert.equal(result.hidden, 0, "no fold pending -> nothing swept (#3769)"); const remainingCallIds = result.messages .filter((m) => m.toolName === "compress") .map((m) => m.toolCallId); - assert.ok(remainingCallIds.includes("call-active")); - assert.ok(!remainingCallIds.includes("call-consumed")); + assert.ok(remainingCallIds.includes("call-active"), "active carrier kept"); + assert.ok(remainingCallIds.includes("call-consumed"), "orphan stays visible between folds"); assert.ok(remainingCallIds.includes("call-orphan1")); assert.ok(remainingCallIds.includes("call-orphan2")); assert.ok(result.messages.some((m) => m.text === "hello")); }); -test("orphaned compress residue is bounded: only the newest two pairs survive any session length", () => { +test("orphan residue is swept onto a committed fold, not rotated on quiet turns (#3769)", () => { const state: CompressionState = { ...createInitialState(), blocks: [ @@ -125,22 +123,37 @@ test("orphaned compress residue is bounded: only the newest two pairs survive an text: "tail", }); + messages.forEach((m, i) => { + state.messageRefs.byRaw[m.id] = `m${String(i + 1).padStart(5, "0")}`; + }); + state.pendingOrphanSweepMinRef = 10; + const result = hideConsumedCompressCalls(state, messages); - // 12 failed pairs in → only the newest 2 pairs + the active call remain: - // PR #18's unbounded accumulation cannot recur under KEEP_LAST_ORPHANED=2. - assert.equal(result.hidden, 20); + assert.equal( + result.hidden, + 16, + "orphans at/after fold start (ref 10) swept; before it kept (#3769)", + ); + assert.deepEqual(result.newlySweptCallIds, [ + "call-fail5", + "call-fail6", + "call-fail7", + "call-fail8", + "call-fail9", + "call-fail10", + "call-fail11", + "call-fail12", + ]); const remainingCallIds = result.messages .filter((m) => m.toolName === "compress" && m.contentType === "tool-call") .map((m) => m.toolCallId); assert.deepEqual(remainingCallIds, [ "call-active", - "call-fail11", - "call-fail12", + "call-fail1", + "call-fail2", + "call-fail3", + "call-fail4", ]); - const remainingResultIds = result.messages - .filter((m) => m.contentType === "tool-result") - .map((m) => m.toolCallId); - assert.deepEqual(remainingResultIds, ["call-fail11", "call-fail12"]); }); test("a failed compress call and its result stay visible (issue #9 fixed-point precondition removed)", () => { diff --git a/kernel/tests/issue3769-orphan-sweep.test.ts b/kernel/tests/issue3769-orphan-sweep.test.ts new file mode 100644 index 0000000000..4d867190d5 --- /dev/null +++ b/kernel/tests/issue3769-orphan-sweep.test.ts @@ -0,0 +1,189 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore } from "../src/compress.js"; +import { assignRefs } from "../src/refs.js"; +import { hideConsumedCompressCalls } from "../src/hide-consumed.js"; +import { createInitialState } from "../src/state.js"; +import type { CompressionState, Config, CoreMessage } from "../src/types.js"; + +// #3769 regression: orphaned failed compress calls are hidden only where a committed +// fold already broke the cached prefix (at/after its earliest start ref), tracked in a +// persistent monotonic set — never rotated away on a quiet turn. These cases pin that timing. + +const isMcp = (n: string) => n === "compress" || n === "mcp__bili__compress"; + +function msg(id: string, text: string): CoreMessage { + return { id, role: "user", contentType: "text", text }; +} + +function failPair(callId: string): [CoreMessage, CoreMessage] { + return [ + { + id: `c-${callId}`, + role: "assistant", + contentType: "tool-call", + toolName: "mcp__bili__compress", + toolCallId: callId, + text: JSON.stringify({ content: [{ startId: "mA", endId: "mB", summary: "s" }] }), + }, + { + id: `r-${callId}`, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text: "nothing to compress", + }, + ]; +} + +function seqRefs(state: CompressionState, messages: CoreMessage[]): void { + messages.forEach((m, i) => { + state.messageRefs.byRaw[m.id] = `m${String(i + 1).padStart(5, "0")}`; + }); +} + +function config(): Config { + return { + tiers: { enabled: true, tier2Trigger: 5, tier3Trigger: 10 }, + nudge: { + maxContextLimitPct: 0.55, + minContextLimitPct: 0.45, + frequency: 5, + iterationThreshold: 15, + force: "soft", + growthRatio: 0.05, + growthFloor: 6000, + growthCap: 50000, + minGrowthFloor: 5000, + minGrowthRatio: 0.45, + emergencyThresholdPct: 0.98, + }, + promotionThreshold: 5, + truncate: { threshold: 1 }, + merge: { maxSummaryLength: 3000, minOldGenBlocks: 3 }, + compress: { minCompressRange: 0, maxSummaryLength: 0, minSummaryLength: 0 }, + protectedTools: [], + preserveRecentMessages: 0, + preserveRecentTokens: 0, + modelContextLimit: 100000, + }; +} + +test("a third failed compress appended to history does NOT rewrite mid-history on a quiet turn (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = []; + messages.push(msg("p1", "early context")); + messages.push(...failPair("f1")); + messages.push(msg("p2", "mid context")); + messages.push(...failPair("f2")); + messages.push(msg("p3", "newer context")); + messages.push(...failPair("f3")); + seqRefs(state, messages); + + const inbound = messages.map((m) => m.id); + const r = hideConsumedCompressCalls(state, messages, isMcp); + + assert.equal(r.hidden, 0, "quiet turn sweeps nothing"); + assert.equal(r.newlySweptCallIds.length, 0, "no fold -> nothing newly swept"); + assert.deepEqual( + r.messages.map((m) => m.id), + inbound, + "outbound ids identical to inbound: oldest failure NOT rotated away mid-history", + ); +}); + +test("a fold sweeps orphans at/after its start but keeps earlier ones in the cached prefix (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = []; + messages.push(msg("p1", "early")); + messages.push(...failPair("f1")); + messages.push(msg("p2", "mid")); + messages.push(...failPair("f2")); + messages.push(msg("p3", "late")); + messages.push(...failPair("f3")); + seqRefs(state, messages); + // refs: p1=1 f1c=2 f1r=3 p2=4 f2c=5 f2r=6 p3=7 f3c=8 f3r=9 + state.pendingOrphanSweepMinRef = 5; + + const r = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["f2", "f3"], "only orphans at/after fold start swept"); + const ids = r.messages.map((m) => m.id); + assert.ok(ids.includes("c-f1"), "orphan before fold start stays visible"); + assert.ok(!ids.includes("c-f2") && !ids.includes("c-f3"), "orphans at/after fold start removed"); +}); + +test("once swept, an orphan stays hidden on later quiet turns (persistent, restart-safe) (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = []; + messages.push(msg("p1", "early")); + messages.push(...failPair("f1")); + messages.push(msg("p2", "mid")); + messages.push(...failPair("f2")); + seqRefs(state, messages); + + state.pendingOrphanSweepMinRef = 5; + const first = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(first.newlySweptCallIds, ["f2"], "the fold sweep removes only f2"); + + state.sweptOrphanCallIds = [...first.newlySweptCallIds]; + state.pendingOrphanSweepMinRef = undefined; + + const second = hideConsumedCompressCalls(state, messages, isMcp); + assert.equal(second.newlySweptCallIds.length, 0, "quiet turn adds nothing"); + const ids = second.messages.map((m) => m.id); + assert.ok(!ids.includes("c-f2"), "previously-swept orphan stays hidden"); + assert.ok(ids.includes("c-f1"), "unswept orphan remains visible"); +}); + +test("failures do not roll-clean per attempt: N quiet turns sweep nothing, one fold sweeps once (#3769)", () => { + const base: CoreMessage[] = [msg("p1", "early")]; + for (let t = 1; t <= 3; t++) { + const state = createInitialState(); + const messages: CoreMessage[] = [...base]; + for (let k = 1; k <= t; k++) messages.push(...failPair(`f${k}`)); + seqRefs(state, messages); + + const r = hideConsumedCompressCalls(state, messages, isMcp); + assert.equal(r.hidden, 0, `turn ${t}: no fold -> nothing swept despite ${t} failures`); + assert.equal(r.newlySweptCallIds.length, 0, `turn ${t}: no new sweep`); + } + + const state = createInitialState(); + const messages: CoreMessage[] = [...base]; + for (let k = 1; k <= 3; k++) messages.push(...failPair(`f${k}`)); + seqRefs(state, messages); + state.pendingOrphanSweepMinRef = 1; + const r = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["f1", "f2", "f3"], "one fold sweeps all eligible orphans at once"); +}); + +test("a real fold writes the sweep marker and the next pass removes the orphan behind it (#3769)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("h1", "first detailed work item alpha content"), + msg("h2", "second detailed work item beta content"), + msg("h3", "third detailed work item gamma content"), + ...failPair("f1"), + msg("h4", "fourth detailed work item delta content"), + msg("last", "latest user intent theta"), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + + const out = core.applyCompression({ + ranges: [{ startRef: "m00001", endRef: "m00003", summary: "fold the spent head", topic: "head" }], + messages, + state, + config: config(), + }); + assert.equal(out.result.errors.length, 0, `unexpected errors: ${out.result.errors.join("; ")}`); + assert.ok(out.result.blocksCreated >= 1, "the head genuinely folds"); + assert.equal(out.state.pendingOrphanSweepMinRef, 1, "marker points at the fold's earliest start ref"); + + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["f1"], "orphan behind the fold is swept"); + assert.ok(!r.messages.some((m) => m.toolCallId === "f1"), "swept orphan removed from outbound"); +}); diff --git a/kernel/tests/orphan-endpoint-snap.test.ts b/kernel/tests/orphan-endpoint-snap.test.ts index 33aa658d20..36a2533446 100644 --- a/kernel/tests/orphan-endpoint-snap.test.ts +++ b/kernel/tests/orphan-endpoint-snap.test.ts @@ -79,10 +79,11 @@ test("hideConsumedCompressCalls records refs of hidden orphan calls only", () => existing: state.messageRefs, nextIndex: 1, }).map; + state.sweptOrphanCallIds = ["c1"]; const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 2, "oldest orphan pair is hidden"); + assert.equal(result.hidden, 2, "previously-swept orphan pair stays hidden (#3769)"); assert.deepEqual( result.hiddenOrphanRefs, ["m00001", "m00002"], From 4bc5076786c34edd8e7f653f3f598a76c22c3cee Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 13:23:08 +0800 Subject: [PATCH 05/27] fix: attribute proactive orphan-cleanup breaks in the cache ledger (#3769) The #3769 seam landed in the 'no fold/switch/restart attribution' bucket because the break came from bili's own cleanup of failed-compress records, not a fold. Add an orphan-sweep attribution channel to the cache ledger: detect growth in the session's sweptOrphanCallIds set per sample and, when a mid-history break has no other claimant, label it prov='orphan-sweep' (count + missed tokens) instead of leaving it unexplained. A real fold claims its own break first, so there is no double-counting. New counters normalize through getCacheLedger for persistence compatibility. --- src/cache-ledger.ts | 60 ++++++++++++++-- tests/issue3769-orphan-sweep-ledger.test.ts | 77 +++++++++++++++++++++ 2 files changed, 131 insertions(+), 6 deletions(-) create mode 100644 tests/issue3769-orphan-sweep-ledger.test.ts diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index 8d9486886b..609b87b905 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -119,7 +119,7 @@ interface LedgerLine { * of host churn). Sparse: omitted unless booked. The seam arm sets `seam` * above instead. Lets the web trajectory chart paint the same bucket * membership the economics card aggregates. */ - prov?: "ttl" | "side" | "rewind" | "client"; + prov?: "ttl" | "side" | "rewind" | "client" | "orphan-sweep"; /** #1535: 1 iff `model` differs from the previous sample's KNOWN model * (unknown sides never flag); marks the re-billed stable prefix on the * first request after a model switch. Sparse: omitted unless set. */ @@ -206,6 +206,13 @@ interface LedgerLine { interface CacheLedger { v: 1; lastBlockId: number; + /** #3769: high-water of session.state.sweptOrphanCallIds length — detects a + * proactive failed-compress cleanup between samples so detectSeam can + * attribute a mid-history break to the cleanup instead of "unexplained". */ + lastSweptCount?: number; + /** #3769: orphans swept on the CURRENT sample (0 = none); read by detectSeam's + * orphan-sweep arm, refreshed each recordCacheSample. Transient. */ + sweepThisSample?: number; consumedFoldSeq: number; sampleSeq: number; foldSeqCounter: number; @@ -278,6 +285,11 @@ interface CacheLedger { * bili rebuild seam. Charged only when the client arm wins. */ clientInserts: number; clientInsertMissed: number; + /** #3769: mid-history breaks attributed to a proactive failed-compress + * cleanup (orphan rotation) landing in-window without a fold claiming + * the break — kept out of the unexplained seam tally. */ + orphanSweepSeams: number; + orphanSweepMissed: number; /** #2929: the provider-side arm split by idle gap — samples settled more * than PROVIDER_TTL_GAP_MS after the previous sample are normal TTL * expiry, not an abnormal provider failure. */ @@ -943,6 +955,21 @@ function detectSeam(session: Session, led: CacheLedger): void { // Inbound prefix-stable or a clean tail boundary: the outbound // mid-list break is bili-side — fall through to the seam bucket. } + // #3769: a proactive failed-compress cleanup (orphan rotation) landed in this + // window. After the fold-gated fix such a cleanup only rides along a committed + // fold, which claims its own break above via foldSeq — so reaching here means a + // mid-history break coincided with a cleanup WITHOUT any fold/switch/restart/ + // client arm claiming it. Attribute it to the cleanup rather than leave it + // "unexplained". Additive: none of the earlier arms is touched. + if ((led.sweepThisSample ?? 0) > 0) { + line.prov = "orphan-sweep"; + agg.orphanSweepSeams += 1; + agg.orphanSweepMissed += line.tr; + if (agg.orphanSweepSeams === 1) { + loggerLog("warn", `[${session.id}] [cache-seam] mid-history break attributed to proactive failed-compress cleanup (${led.sweepThisSample} record(s) rotated out, #3769): hit ${line.hitPct}% (unexplained=${Math.round(line.tr)} tok) — no fold claimed this break`); + } + return; + } agg.seamSuspects += 1; agg.seamMissed += line.tr; line.seam = 1; @@ -974,7 +1001,7 @@ export function getCacheLedger(session: Session): CacheLedger { "restartDropMissed", "attributedMissed", "unknownSamples", "unknownInput", "nbSamples", "nbInput", "seamSuspects", "seamMissed", - "providerSideMisses", "providerSideMissed", "providerTtlMisses", "providerTtlMissed", "rewinds", "rewindMissed", "abortCorrelated", "clientInserts", "clientInsertMissed", + "providerSideMisses", "providerSideMissed", "providerTtlMisses", "providerTtlMissed", "rewinds", "rewindMissed", "abortCorrelated", "clientInserts", "clientInsertMissed", "orphanSweepSeams", "orphanSweepMissed", ] as const) { if (typeof g[key] !== "number") g[key] = 0; } @@ -991,7 +1018,7 @@ export function getCacheLedger(session: Session): CacheLedger { foldSeqCounter: 0, folds: [], lines: [], - agg: { requests: 0, input: 0, cached: 0, output: 0, nc: 0, cr: 0, tr: 0, switches: 0, switchMissed: 0, effSwitches: 0, effSwitchMissed: 0, keySwitches: 0, keySwitchMissed: 0, promptSwitches: 0, promptSwitchMissed: 0, projSwitches: 0, projSwitchMissed: 0, wireSwitches: 0, wireSwitchMissed: 0, upstreamSwitches: 0, upstreamSwitchMissed: 0, restartDrops: 0, restartDropMissed: 0, attributedMissed: 0, unknownSamples: 0, unknownInput: 0, nbSamples: 0, nbInput: 0, seamSuspects: 0, seamMissed: 0, providerSideMisses: 0, providerSideMissed: 0, providerTtlMisses: 0, providerTtlMissed: 0, rewinds: 0, rewindMissed: 0, abortCorrelated: 0, clientInserts: 0, clientInsertMissed: 0 }, + agg: { requests: 0, input: 0, cached: 0, output: 0, nc: 0, cr: 0, tr: 0, switches: 0, switchMissed: 0, effSwitches: 0, effSwitchMissed: 0, keySwitches: 0, keySwitchMissed: 0, promptSwitches: 0, promptSwitchMissed: 0, projSwitches: 0, projSwitchMissed: 0, wireSwitches: 0, wireSwitchMissed: 0, upstreamSwitches: 0, upstreamSwitchMissed: 0, restartDrops: 0, restartDropMissed: 0, attributedMissed: 0, unknownSamples: 0, unknownInput: 0, nbSamples: 0, nbInput: 0, seamSuspects: 0, seamMissed: 0, providerSideMisses: 0, providerSideMissed: 0, providerTtlMisses: 0, providerTtlMissed: 0, rewinds: 0, rewindMissed: 0, abortCorrelated: 0, clientInserts: 0, clientInsertMissed: 0, orphanSweepSeams: 0, orphanSweepMissed: 0 }, }; meta[LEDGER_KEY] = led; return led; @@ -1040,6 +1067,22 @@ function detectNewFolds(session: Session, led: CacheLedger): void { } } +// #3769: sweptOrphanCallIds is monotonic, so its length high-water makes growth +// exact. detectSeam's orphan-sweep arm reads sweepThisSample set here. +function detectOrphanSweeps(session: Session, led: CacheLedger): void { + const n = session.state?.sweptOrphanCallIds?.length ?? 0; + // First observation only baselines: a resumed session's pre-existing sweeps + // must not be charged to the current sample. + if (led.lastSweptCount === undefined) { + led.lastSweptCount = n; + led.sweepThisSample = 0; + return; + } + const delta = n - led.lastSweptCount; + led.lastSweptCount = n; + led.sweepThisSample = delta > 0 ? delta : 0; +} + /** Record one provider usage report into the session ledger. `input` must be * NORMALIZED (cached included — promptInputTotal semantics). `cached === null` * means the provider reported NO cache-hit tokens (unmeasurable): such samples @@ -1051,6 +1094,7 @@ export function recordCacheSample( ): void { const led = getCacheLedger(session); detectNewFolds(session, led); + detectOrphanSweeps(session, led); const prevLine = led.lines[led.lines.length - 1]; // #1536: unknown-cache samples still need a prev/cur pair so the fold chain // stays consistent, but decompose with cached=0 then QUARANTINE every derived @@ -1707,7 +1751,7 @@ interface BodyStability { interface BiliCacheReport extends CacheReport { stability: BodyStability; /** #2131: line set widened with the per-call body-stability fields. */ - lines: Array; + lines: Array; modelSwitches: ModelSwitchStats; /** #3803: host generation-parameter switches (reasoning effort / thinking config) — a provider cache-namespace flip that re-bills the whole stable prefix once, distinct from any byte change. */ effSwitches: ModelSwitchStats; @@ -1731,7 +1775,7 @@ interface BiliCacheReport extends CacheReport { * is booked as new content, never as a prefix re-pay. */ initialBills: { samples: number; inputTokens: number }; invalidation: InvalidationTokenBreakdown; - seam: { suspects: number; missed: number; events: SeamEvent[]; providerSide: { count: number; missed: number }; providerTtl: { count: number; missed: number }; rewinds: { count: number; missed: number }; abortCorrelated: number; client: { count: number; missed: number } }; + seam: { suspects: number; missed: number; events: SeamEvent[]; providerSide: { count: number; missed: number }; providerTtl: { count: number; missed: number }; rewinds: { count: number; missed: number }; abortCorrelated: number; client: { count: number; missed: number }; orphanSweep: { count: number; missed: number } }; } export function buildSessionCacheReport(session: Session): BiliCacheReport { @@ -1918,7 +1962,7 @@ export function buildSessionCacheReport(session: Session): BiliCacheReport { initialBills: { samples: a.nbSamples, inputTokens: a.nbInput }, invalidation, stability, - seam: { suspects: a.seamSuspects, missed: a.seamMissed, events: led.seamEvents ?? [], providerSide: { count: a.providerSideMisses, missed: a.providerSideMissed }, providerTtl: { count: a.providerTtlMisses, missed: a.providerTtlMissed }, rewinds: { count: a.rewinds, missed: a.rewindMissed }, abortCorrelated: a.abortCorrelated, client: { count: a.clientInserts, missed: a.clientInsertMissed } }, + seam: { suspects: a.seamSuspects, missed: a.seamMissed, events: led.seamEvents ?? [], providerSide: { count: a.providerSideMisses, missed: a.providerSideMissed }, providerTtl: { count: a.providerTtlMisses, missed: a.providerTtlMissed }, rewinds: { count: a.rewinds, missed: a.rewindMissed }, abortCorrelated: a.abortCorrelated, client: { count: a.clientInserts, missed: a.clientInsertMissed }, orphanSweep: { count: a.orphanSweepSeams, missed: a.orphanSweepMissed } }, }; } @@ -2092,6 +2136,10 @@ function formatSeam(r: BiliCacheReport): string { if (e.client) out.push(` #${e.seq} ${fmtTime(e.at)} hit ${e.hitPct.toFixed(1)}% · input ${fmtTok(e.input)} · inbound message[${e.client.idx}] ${e.client.role}/${e.client.cls} ${e.client.len}B JSON · outbound ${fmtTok(e.lcpBytes)}B @ message[${e.msgIndex ?? "?"}] of ${e.prevMsgs}→${e.curMsgs}`); } } + if (r.seam.orphanSweep.count > 0) { + out.push("🧹 ORPHAN CLEANUP SEAM (proactive failed-compress rotation — #3769)"); + out.push(` ${r.seam.orphanSweep.count} sample(s) · ${fmtTok(r.seam.orphanSweep.missed)} tok re-billed when failed-compress records were rotated out of history without a fold claiming the break — bili's own cleanup, not an upstream or host rewrite`); + } if (r.seam.abortCorrelated > 0) { out.push("⏻ ABORT-CORRELATED"); out.push(` ${r.seam.abortCorrelated} missed sample(s) within 30s of a client mid-stream abort — abort/retry churn rewrites the resent tail; correlation, not causation (see bili.log 'client aborted mid-stream')`); diff --git a/tests/issue3769-orphan-sweep-ledger.test.ts b/tests/issue3769-orphan-sweep-ledger.test.ts new file mode 100644 index 0000000000..d6182a4c21 --- /dev/null +++ b/tests/issue3769-orphan-sweep-ledger.test.ts @@ -0,0 +1,77 @@ +// #3769: attribute a proactive failed-compress cleanup (orphan rotation) in the +// cache ledger. detectOrphanSweeps watches the monotonic length of +// session.state.sweptOrphanCallIds between samples; when a mid-history break +// otherwise lands in the unexplained seam bucket AND a cleanup grew this turn, +// the residual books into its own bucket (prov:"orphan-sweep") instead of +// "no fold/switch/restart attribution". Gated on actual growth: no growth keeps +// the legacy seam bucket, and a resumed session's pre-existing sweeps baseline +// without being charged to the first sample. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { noteForwardedBody, settleUsageReport, getCacheLedger, buildSessionCacheReport, handleAcpCache } from "../src/cache-ledger.ts"; +import type { Session } from "../src/session.ts"; + +let seq = 0; +function makeSession(): Session { + seq += 1; + return { + id: `orphan-sweep-${seq}`, + metadata: {}, + stats: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, tokenSnapshot: {} }, + } as unknown as Session; +} + +const body = (msgs: string[]): string => JSON.stringify({ model: "m", messages: msgs.map((c) => ({ role: "user", content: c })) }); +const T0 = Date.parse("2026-10-19T10:49:00Z"); + +function settle(session: Session, at: number, input: number, cached: number): void { + settleUsageReport(session, { total: input, reportedCached: cached, output: 0, protocol: "openai", upstream: "http://u" }); +} + +test("cleanup growth + unattributed break books the orphan-sweep bucket, not the seam bucket", () => { + const s = makeSession(); + noteForwardedBody(s, body(["a", "b", "c"])); + settle(s, T0, 100_000, 99_000); // baseline: swept empty -> high-water 0 + // A proactive cleanup rotated two failed-compress records out of history this turn. + s.state!.sweptOrphanCallIds = ["c1", "c2"]; + noteForwardedBody(s, body(["a", "B2", "c"])); // outbound mid-history break, no inbound capture + settle(s, T0 + 1000, 100_000, 20_000); + const led = getCacheLedger(s); + const line = led.lines[led.lines.length - 1]!; + assert.equal(line.prov, "orphan-sweep", "residual booked to the cleanup"); + assert.equal(line.seam, undefined, "NOT a seam suspect"); + assert.equal(led.agg.orphanSweepSeams, 1); + assert.ok(led.agg.orphanSweepMissed > 0); + assert.equal(led.agg.seamSuspects, 0, "kept out of the unexplained tally"); + const report = buildSessionCacheReport(s); + assert.equal(report.seam.orphanSweep.count, 1); + assert.ok(report.seam.orphanSweep.missed > 0); + assert.match(handleAcpCache(s).text, /ORPHAN CLEANUP SEAM/); +}); + +test("same break WITHOUT cleanup growth keeps the legacy seam bucket (arm is gated on growth)", () => { + const s = makeSession(); + noteForwardedBody(s, body(["a", "b", "c"])); + settle(s, T0, 100_000, 99_000); + noteForwardedBody(s, body(["a", "B2", "c"])); // identical break, but nothing was swept + settle(s, T0 + 1000, 100_000, 20_000); + const led = getCacheLedger(s); + const line = led.lines[led.lines.length - 1]!; + assert.equal(line.seam, 1, "legacy seam bucket preserved"); + assert.equal(line.prov, undefined); + assert.equal(led.agg.orphanSweepSeams, 0); + assert.equal(led.agg.seamSuspects, 1); +}); + +test("a resumed session's pre-existing sweeps baseline without charging the first sample", () => { + const s = makeSession(); + s.state!.sweptOrphanCallIds = ["c1"]; // already swept before this process started + noteForwardedBody(s, body(["a", "b", "c"])); + settle(s, T0, 100_000, 99_000); // first observation baselines high-water at 1 + noteForwardedBody(s, body(["a", "B2", "c"])); // no NEW growth since baseline + settle(s, T0 + 1000, 100_000, 20_000); + const led = getCacheLedger(s); + assert.equal(led.agg.orphanSweepSeams, 0, "pre-existing sweeps not charged to this sample"); + assert.equal(led.agg.seamSuspects, 1, "falls back to the legacy seam bucket"); +}); From df3b259cf2edf8b5dd04c80549e09576505c1b63 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 13:57:28 +0800 Subject: [PATCH 06/27] kernel: normalize indentation of #3769 edits (whitespace only) types.ts and two kernel test files sit on a prettier-clean master baseline; the #3769 additions landed over-indented by one space. Normalized to the surrounding 2-space style so the diff stays confined to the feature. No behavior change; typecheck + targeted kernel/host suites re-run green. --- kernel/src/types.ts | 46 +++++++++++------------ kernel/tests/hide-merge-rebuild.test.ts | 5 ++- kernel/tests/orphan-endpoint-snap.test.ts | 6 ++- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 3c56b1a1c2..91f98dd40b 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -240,29 +240,29 @@ export interface CompressionState { * dead-ending the whole range (#396). A drifted/dangling ref (#387) is NOT in * this set and keeps the clear "cannot be anchored" error. Optional for * pre-feature persisted states. */ - hiddenOrphanRefs?: string[]; - /** Persistent, MONOTONIC set of orphan compress-call ids permanently swept - * (#3769). An orphan (a failed compress call/result with no matching block) - * is only removed from the visible view when a REAL fold gives a cache-safe - * place to do it: the hide node sweeps orphans at/after the fold's earliest - * start ref onto this set, and they stay hidden forever — across the next - * turn, repeated processTurn calls, restarts and upgrades. Monotonic (never - * shrinks) so a previously-hidden record can never reappear once the client - * resends full history. Replaces the old keep-last-two rotation, which hid - * the oldest failure on EVERY quiet turn and moved the cached prefix - * breakpoint forward for no new fold. Optional: pre-feature persisted states - * lack it (nothing was permanently hidden under the old per-pass rule). */ - sweptOrphanCallIds?: string[]; - /** Numeric part (mNNNNN -> NNNNN) of the EARLIEST start ref of the most - * recent fold that has not yet been consumed by an orphan sweep (#3769). - * Written by applyCompression when it creates >=1 block (min over the newly - * created blocks' startRef); consumed + cleared by the hide-compress-calls - * node on the next processTurn, which then sweeps orphans with ref >= this - * value (orphans before it sit in the still-cached prefix and stay visible). - * Carries the position so the sweep is cache-neutral: it only removes records - * the fold already invalidated. Optional: absent until the first post-fix fold. */ - pendingOrphanSweepMinRef?: number; - /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but + hiddenOrphanRefs?: string[]; + /** Persistent, MONOTONIC set of orphan compress-call ids permanently swept + * (#3769). An orphan (a failed compress call/result with no matching block) + * is only removed from the visible view when a REAL fold gives a cache-safe + * place to do it: the hide node sweeps orphans at/after the fold's earliest + * start ref onto this set, and they stay hidden forever — across the next + * turn, repeated processTurn calls, restarts and upgrades. Monotonic (never + * shrinks) so a previously-hidden record can never reappear once the client + * resends full history. Replaces the old keep-last-two rotation, which hid + * the oldest failure on EVERY quiet turn and moved the cached prefix + * breakpoint forward for no new fold. Optional: pre-feature persisted states + * lack it (nothing was permanently hidden under the old per-pass rule). */ + sweptOrphanCallIds?: string[]; + /** Numeric part (mNNNNN -> NNNNN) of the EARLIEST start ref of the most + * recent fold that has not yet been consumed by an orphan sweep (#3769). + * Written by applyCompression when it creates >=1 block (min over the newly + * created blocks' startRef); consumed + cleared by the hide-compress-calls + * node on the next processTurn, which then sweeps orphans with ref >= this + * value (orphans before it sit in the still-cached prefix and stay visible). + * Carries the position so the sweep is cache-neutral: it only removes records + * the fold already invalidated. Optional: absent until the first post-fix fold. */ + pendingOrphanSweepMinRef?: number; + /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk * history rewrite drops them outright), so no range citing them can ever diff --git a/kernel/tests/hide-merge-rebuild.test.ts b/kernel/tests/hide-merge-rebuild.test.ts index 1cfe694e5a..f0a464dedc 100644 --- a/kernel/tests/hide-merge-rebuild.test.ts +++ b/kernel/tests/hide-merge-rebuild.test.ts @@ -75,7 +75,10 @@ test("hideConsumedCompressCalls keeps active-block calls and orphans until a fol .filter((m) => m.toolName === "compress") .map((m) => m.toolCallId); assert.ok(remainingCallIds.includes("call-active"), "active carrier kept"); - assert.ok(remainingCallIds.includes("call-consumed"), "orphan stays visible between folds"); + assert.ok( + remainingCallIds.includes("call-consumed"), + "orphan stays visible between folds", + ); assert.ok(remainingCallIds.includes("call-orphan1")); assert.ok(remainingCallIds.includes("call-orphan2")); assert.ok(result.messages.some((m) => m.text === "hello")); diff --git a/kernel/tests/orphan-endpoint-snap.test.ts b/kernel/tests/orphan-endpoint-snap.test.ts index 36a2533446..3bbb72c8d4 100644 --- a/kernel/tests/orphan-endpoint-snap.test.ts +++ b/kernel/tests/orphan-endpoint-snap.test.ts @@ -83,7 +83,11 @@ test("hideConsumedCompressCalls records refs of hidden orphan calls only", () => const result = hideConsumedCompressCalls(state, messages); - assert.equal(result.hidden, 2, "previously-swept orphan pair stays hidden (#3769)"); + assert.equal( + result.hidden, + 2, + "previously-swept orphan pair stays hidden (#3769)", + ); assert.deepEqual( result.hiddenOrphanRefs, ["m00001", "m00002"], From ce6c244ed2982e312376174270cfc854195303ee Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 16:48:20 +0800 Subject: [PATCH 07/27] kernel: derive orphan-sweep marker from covered message refs (#3769 review fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found a defect in the fold-marker computation: block-boundary specs (tier distillation compresses bN..bM) store startRef="bN" verbatim on the new block, so Number(startRef.replace(/\D/g,"")) parsed a BLOCK number, not the earliest covered MESSAGE ref. In long sessions that marks the sweep far too early: the next hide pass deletes failed pairs sitting in the still-cached prefix before the fold's own divergence point — re-introducing exactly the #3769 mid-history cache break on every tier-distillation fold. Compute foldStartRef as the min refNum over each new block's effectiveMessageIds via state.messageRefs.byRaw instead; ids missing from the map (pruned) contribute nothing, which only under-sweeps (safe direction). refNum is exported from hide-consumed.ts so both consumers share one "unknown" definition. Regression test 6 in issue3769-orphan-sweep.test.ts: m-boundary fold then block-boundary fold asserts pendingOrphanSweepMinRef === first covered ref (not the block number) and that the earlier pair stays visible. Verified to fail against the pre-fix code. --- kernel/src/compress.ts | 24 +++++++---- kernel/src/hide-consumed.ts | 4 +- kernel/tests/issue3769-orphan-sweep.test.ts | 45 +++++++++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 83a471c16b..2aea1377b2 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -14,7 +14,7 @@ import { import type { ResolvedRange } from "./boundaries.js"; import { truncateLargeToolOutputs } from "./truncate-tools.js"; import { scrubLoneSurrogates } from "./truncate.js"; -import { hideConsumedCompressCalls } from "./hide-consumed.js"; +import { hideConsumedCompressCalls, refNum } from "./hide-consumed.js"; import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; import { applyCrushToMessages } from "./crush.js"; import { applyRetrieve, ccrStoreNode, RETRIEVED_ID_PREFIX } from "./ccr.js"; @@ -759,16 +759,24 @@ export function createCore(ports: Ports = {}): CompressionCore { state.stats.tokensCompressed += tokensCompressed; if (blocksCreated > 0) { - // #3769: record where THIS fold starts (min over the newly-created - // blocks' startRef) so the next processTurn sweeps orphan failures at/after - // it cache-neutrally, leaving orphans before it in the still-cached prefix. - // Kept as the running min across folds that commit before a sweep consumes it. + // #3769: record where THIS fold starts so the next processTurn sweeps + // orphan failures at/after it cache-neutrally, leaving orphans before it + // in the still-cached prefix. Kept as the running min across folds that + // commit before a sweep consumes it. The position comes from the covered + // message refs of each new block, NOT its startRef string: a block-boundary + // spec stores startRef="bN", whose digits are a BLOCK number, not a message + // position — parsing it would mark the sweep far too early and rewrite the + // still-cached prefix (#3769 again). refNum returns null for unmapped/blocked + // ids, so a fully-pruned coverage simply sets no marker (under-sweep is safe; + // over-sweep is not). let foldStartRef: number | undefined; for (const b of state.blocks) { if (preFoldBlockIds.has(b.blockId)) continue; - const n = Number((b.startRef ?? "").replace(/\D/g, "")); - if (Number.isFinite(n) && (foldStartRef === undefined || n < foldStartRef)) { - foldStartRef = n; + for (const id of b.effectiveMessageIds) { + const num = refNum(state.messageRefs.byRaw[id]); + if (num !== null && (foldStartRef === undefined || num < foldStartRef)) { + foldStartRef = num; + } } } if (foldStartRef !== undefined) { diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index d2db4722a0..69819675db 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -34,7 +34,9 @@ function rangeKey(startRef: string, endRef: string): string { // Numeric position of a padded ref (mNNNNN -> NNNNN); null when unmapped/blocked // so the caller treats it as "position unknown" and never sweeps on a guess. -function refNum(ref: string | undefined): number | null { +// Exported: applyCompression derives the fold sweep marker from covered message +// refs through the same helper so both sides agree on "unknown" (#3769). +export function refNum(ref: string | undefined): number | null { if (!ref || ref === BLOCKED_REF) return null; const n = Number(ref.replace(/\D/g, "")); return Number.isFinite(n) ? n : null; diff --git a/kernel/tests/issue3769-orphan-sweep.test.ts b/kernel/tests/issue3769-orphan-sweep.test.ts index 4d867190d5..abf97737f3 100644 --- a/kernel/tests/issue3769-orphan-sweep.test.ts +++ b/kernel/tests/issue3769-orphan-sweep.test.ts @@ -187,3 +187,48 @@ test("a real fold writes the sweep marker and the next pass removes the orphan b assert.deepEqual(r.newlySweptCallIds, ["f1"], "orphan behind the fold is swept"); assert.ok(!r.messages.some((m) => m.toolCallId === "f1"), "swept orphan removed from outbound"); }); + +test("a block-boundary fold marks the sweep by covered message refs, not the block number (#3769)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("p1", "early cached prefix alpha"), + ...failPair("f1"), + msg("p2", "mid detailed work item beta content"), + msg("p3", "mid detailed work item gamma content"), + msg("p4", "mid detailed work item delta content"), + msg("p5", "mid detailed work item epsilon content"), + msg("p6", "latest user intent theta"), + ]; + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { + existing: state.messageRefs, + nextIndex: 1, + }).map; + // refs: p1=1 c-f1=2 r-f1=3 p2=4 p3=5 p4=6 p5=7 p6=8 + + const first = core.applyCompression({ + ranges: [{ startRef: "m00004", endRef: "m00005", summary: "fold the mid pair", topic: "mid" }], + messages, + state, + config: config(), + }); + assert.equal(first.result.errors.length, 0, `unexpected errors: ${first.result.errors.join("; ")}`); + assert.ok(first.result.blocksCreated >= 1, "the mid pair folds"); + const blockId = first.state.blocks[first.state.blocks.length - 1].blockId; + + const out = core.applyCompression({ + ranges: [{ startRef: blockId, endRef: "m00006", summary: "distill the block plus its tail", topic: "distill" }], + messages, + state: first.state, + config: config(), + }); + assert.equal(out.result.errors.length, 0, `unexpected errors: ${out.result.errors.join("; ")}`); + assert.ok(out.result.blocksCreated >= 1, "the block-boundary range folds"); + // Parsing the new block's startRef string ("b1") yields the BLOCK number 1, which + // would sweep the cached-prefix orphan f1 (ref 2). Covered-ref derivation must say 4. + assert.equal(out.state.pendingOrphanSweepMinRef, 4, "marker sits at the earliest covered message ref"); + + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, [], "cached-prefix orphan before the fold is untouched"); + assert.ok(r.messages.some((m) => m.toolCallId === "f1"), "f1 remains visible"); +}); From 6544bf3fd96a0365cd8e5a64e66dd3b75dfd9ba8 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 16:48:30 +0800 Subject: [PATCH 08/27] fix: hydrate orphan-sweep state fields across restart (#3769 review fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review found the two new kernel state fields were dropped on every session load: buildRecord persists session.state wholesale, but the load path rebuilds it through mergeState's explicit field whitelist, which predated sweptOrphanCallIds / pendingOrphanSweepMinRef. Losing sweptOrphanCallIds resurrects already-cleaned failed pairs after a restart — a mid-history rewrite (cache break) that violates the PR's own persistence acceptance case. Losing the pending marker only defers a sweep by one fold (conservative). Extend the mergeState whitelist with both fields, degrading to undefined for old persisted states exactly like every other optional field. --- src/persist.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/persist.ts b/src/persist.ts index 56d4ef9130..9652ce41c8 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -238,6 +238,12 @@ function mergeState(parsed: CompressionState): CompressionState { // resurrects with absorbed=[] and hideAbsorbedMessages has nothing to hide. absorbed: parsed.absorbed ?? fresh.absorbed, rules: parsed.rules ?? fresh.rules, + // #3769: fold-gated orphan-sweep bookkeeping must survive hydration — + // dropping sweptOrphanCallIds resurrects already-cleaned pairs after a + // restart (mid-history rewrite = cache break); the pending marker is + // transient, losing it only defers the sweep to the next fold. + sweptOrphanCallIds: parsed.sweptOrphanCallIds ?? fresh.sweptOrphanCallIds, + pendingOrphanSweepMinRef: parsed.pendingOrphanSweepMinRef ?? fresh.pendingOrphanSweepMinRef, }; } From 4aaf6ee8a31f139467cac18295e43ff2dfd446f7 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 16:48:47 +0800 Subject: [PATCH 09/27] test: align #2749/#1567 fixtures and storage-render goldens with fold-gated sweeps (#3769) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three stale pins broke under the approved keep-two -> fold-gated behavior change: - stub-resubmission-guard (#2749): the fixture applied folds without a call id, unlike production (src/loop/core.ts passes callId), so the resent call classified as orphan and the new sweep hid it. Pass "call-b1" at each first fold; all summary-integrity assertions unchanged. 5/5 green. - cache-friendly-plugin: the two anthropic geometry subtests pinned the old KEEP_LAST_ORPHANED=2 rotation (oldest pair pruned on later turns). Rewritten to pin the new contract: failed pairs stay visible between folds, every anchor stays stripped while its own pair rides, quiet turns append-stable. A live-server probe plus the passing drop-pair/main-matrix subtests confirm the geometry expected values. Header docs updated to match the new mechanism. 11/11 green. - storage-render goldens: regenerated via scripts/update-storage-render-goldens.ts. Diff is additive only — the persisted envelope gained lastSweptCount (cache ledger) and the two new kernel state keys (empty digests); state/blockContents/stats section digests unchanged. --- tests/cache-friendly-plugin.test.ts | 69 +++++++++++----------- tests/golden/storage-render/anthropic.json | 8 ++- tests/golden/storage-render/chat.json | 8 ++- tests/golden/storage-render/google.json | 8 ++- tests/golden/storage-render/responses.json | 8 ++- tests/stub-resubmission-guard.test.ts | 13 ++-- 6 files changed, 64 insertions(+), 50 deletions(-) diff --git a/tests/cache-friendly-plugin.test.ts b/tests/cache-friendly-plugin.test.ts index 11d9927ebe..18bf5496ee 100644 --- a/tests/cache-friendly-plugin.test.ts +++ b/tests/cache-friendly-plugin.test.ts @@ -11,9 +11,10 @@ // therefore re-injects the in-place acp_summary carrier from block state on // every request. Since the #1567 fix the carrier is STRIPPED per-block, but // self-verified: only while that exact fold's compress pair actually rides -// the (post-prepare) inbound history. A pair pruned client-side or hidden by -// the kernel (KEEP_LAST_ORPHANED) keeps the anchor, so an active fold never -// has zero carriers. This suite pins that behavior byte-for-byte. +// the (post-prepare) inbound history. A pair pruned client-side or swept by +// the kernel's fold-gated orphan cleanup (#3769) keeps the anchor, so an +// active fold never has zero carriers. This suite pins that behavior +// byte-for-byte. // // Invariants pinned per wire (bodies = consecutive upstream-bound request // payloads of one session, compared as flat unit lists — text blocks and @@ -39,13 +40,13 @@ // fold spec echoed in the re-sent tool_call args + the tail // quote inside the persistently re-sent tool result) and ZERO // carrier markers (#1567 fix); -// P2c fail-safe (#1567 hardening): the anchor re-carries the summary -// whenever the pair does NOT ride inbound — the client stops -// re-sending it, or the kernel hid it (KEEP_LAST_ORPHANED) — -// pinned by the two hardening subtests at the bottom of this file; -// P3 multi-fold: two folds in one session; both pairs stay visible -// (≤ KEEP_LAST_ORPHANED) and the pair-quote counts stay constant across -// all later turns; + // P2c fail-safe (#1567 hardening): the anchor re-carries the summary + // whenever the pair does NOT ride inbound — the client stops + // re-sending it, or the kernel swept it on a later fold (#3769) — + // pinned by the hardening subtests at the bottom of this file; + // P3 multi-fold: two folds in one session; both pairs stay visible for the + // rest of the session (fold-gated sweeps, #3769 — keep-two rotation is + // gone) and the pair-quote counts stay constant across all later turns; // P4 no transient acp_loop_* artifact leaks into any forwarded body // (plugin mode has none by construction — pin it); // P5 marker/tag hygiene: every content text unit carries exactly one @@ -672,10 +673,10 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr const gamma = FOLD_E.summary; for (const i of [5, 6, 7]) { const raw = cans[i]!.raw; - // carrier is superseded on every turn; the two head/tail quotes ride - // the re-sent geo_a tool result until that pair is pruned at t7 - const expA = i === 5 ? 2 : 0; - assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA}`); + // carrier is superseded on every turn; the geo_a pair quotes ride + // every later turn — fold-gated sweeps (#3769) replaced keep-two + // rotation, so no later fold prunes this oldest pair here + assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != 2`); assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: carriers must be stripped while their pairs ride inbound (got ${countOcc(raw, SUM_MARKER)})`); assert.ok(!raw.includes("q2-marker"), `${L(i)}: swallowed content q2 must stay hidden`); assert.ok(!raw.includes("a2-marker"), `${L(i)}: swallowed content a2 must stay hidden`); @@ -694,8 +695,10 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr assert.equal(countOcc(cans[7]!.raw, gamma), 2, `${L(7)}: resent gamma quotes must survive`); for (const i of [6, 7]) { const raw = cans[i]!.raw; - assert.ok(raw.includes("\"geo_wide\"") && raw.includes("\"geo_e\""), `${L(i)}: latest two pairs must stay visible`); - assert.ok(!raw.includes("\"geo_a\""), `${L(i)}: oldest pair must be pruned (KEEP_LAST_ORPHANED=2)`); + // all three failed pairs stay visible: fold-gated sweeps (#3769) only + // remove orphans at/after a committed fold's start, never rotate the + // newest failures away between folds + assert.ok(raw.includes("\"geo_a\"") && raw.includes("\"geo_wide\"") && raw.includes("\"geo_e\""), `${L(i)}: all three pairs must stay visible (no keep-two rotation)`); } // #1567 (fixed): no text unit carries the wide summary anymore (stripped // anchor); the quotes ride structured tool units, not text @@ -751,11 +754,14 @@ testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pai assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); -// #1567 hardening scenario B: three concurrent ACTIVE folds. The kernel hides -// consumed compress pairs beyond the newest KEEP_LAST_ORPHANED=2, so the -// oldest pair leaves the wire while its block is still active — the strip -// guard must see it gone and KEEP that anchor (kernel-side fail-safe). -testCompat(`cache-friendly plugin matrix: anthropic — 3rd fold prunes oldest pair → oldest anchor kept (#1567 hardening)`, async () => { +// #3769 contract pin (plugin lane): three concurrent ACTIVE folds. The old +// KEEP_LAST_ORPHANED=2 rotation hid the oldest failed pair on EVERY turn — +// a mid-history rewrite with no fold behind it, exactly #3769's cache seam. +// Fold-gated sweeps replaced it: failed pairs stay visible between folds, so +// every anchor stays stripped (its own pair rides) and later turns remain +// append-stable. The zero-carrier fail-safe itself (pair gone → anchor +// re-carries) is pinned by the drop-pair subtest above. +testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds keep all failed pairs riding (fold-gated sweeps, #3769)`, async () => { const SUM_P1 = "Prune-safe fold one summary covering m00001..m00002"; const SUM_P2 = "Prune-safe fold two summary covering m00003..m00004"; const SUM_P3 = "Prune-safe fold three summary covering m00005..m00006"; @@ -779,25 +785,22 @@ testCompat(`cache-friendly plugin matrix: anthropic — 3rd fold prunes oldest p assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-prune"); - // t3/t4: pairs ride (≤ 2) → anchors stripped. From t5 on, three pairs - // coexist and the kernel hides the oldest (KEEP_LAST_ORPHANED=2) — the - // guard sees it gone and keeps that anchor (kernel-side fail-safe) + // t3/t4: each fold's pair rides from its own turn on → anchors stripped for (const i of [2, 3]) { assert.equal(countOcc(cans[i]!.raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while pairs ride`); } assert.ok(cans[3]!.raw.includes('"fs1"') && cans[3]!.raw.includes('"fs2"'), `${L(3)}: both pairs visible`); - // t5..t7: fs1 hidden by KEEP_LAST_ORPHANED → anchor F1 re-carries; - // F2/F3 anchors stay stripped while their pairs ride + // t5..t7: three failed pairs coexist and ALL stay visible (no keep-two + // rotation, #3769) → every anchor stays stripped, no carrier needed for (const i of [4, 5, 6]) { const raw = cans[i]!.raw; - assert.ok(!raw.includes('"fs1"'), `${L(i)}: oldest pair pruned (KEEP_LAST_ORPHANED=2)`); - assert.ok(raw.includes('"fs2"') && raw.includes('"fs3"'), `${L(i)}: newest two pairs stay visible`); - assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(i)}: exactly one carrier (F1 anchor re-carried)`); - assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: SUM_P1 = carrier only (its pair quotes pruned with the pair)`); - assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: SUM_P2 quotes only`); - assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: SUM_P3 quotes only`); + assert.ok(raw.includes('"fs1"') && raw.includes('"fs2"') && raw.includes('"fs3"'), `${L(i)}: all three failed pairs must stay visible (fold-gated sweeps, #3769)`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: no carrier needed — every anchor's pair rides inbound`); + assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: SUM_P1 pair quotes only`); + assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: SUM_P2 pair quotes only`); + assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: SUM_P3 pair quotes only`); } - // once the oldest anchor re-carries, later turns are append-stable + // mid-history never mutates between quiet turns assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q8", "a8"]); }, { timeout: 120_000 }); diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index c565999118..934a26087d 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "1c7b7922230de1bc61ce9b909a9fbfd4a9a28e478d12178568d21e97946760b5", + "stateDigest": "f894d02089eb22567a0e5d332ab7c12153d2a3d92a37140f8fc3d4b3d3cfcc19", "sectionDigests": { "state": "a4b83f873d16bfc4", "blockContents": "dc7d13d250c94bb4", "stats": "cff3c2546898aeb3", - "metadata": "b6fe321d2ad536b0" + "metadata": "a10cb71754abe91e" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", @@ -101,7 +101,9 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f" }, "stateShape": { "refs": 49, diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 6ed76a04ec..816c8816a1 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "995db4a4701baf4b72504dedc8842a79fc17dc9bcc535e4367c8a87e732616dd", + "stateDigest": "4731e0ab9374388662bdc5ef54386d22cf446b71712d379cdbbdd530805d10cc", "sectionDigests": { "state": "463b3dc26a100c2a", "blockContents": "94a58f96d7c5cfc3", "stats": "59842ce5415d3a4d", - "metadata": "348ced08c027b22b" + "metadata": "0f57b6cba74a16d8" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", @@ -101,7 +101,9 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 19ddaf90da..2fd2f8942f 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "32b187839fe05ec69aeebade3bb8aad18587ec7018a1816df7c50471dc8a3187", + "stateDigest": "5b8b7f10031808c1cfcc6161a6c3881d485d4016f435e27ee89bd015a296fee4", "sectionDigests": { "state": "a9b9531d952b48d8", "blockContents": "473264d9df8ba338", "stats": "e358dc992395fc7e", - "metadata": "124f318e161ebf7f" + "metadata": "bf873598b93525e7" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", @@ -101,7 +101,9 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 2dd5340f2e..e8897cc195 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "96f5faf2e54f7d9257f4f423e4abfc089bd90140a1970f862061685707c6b6fa", + "stateDigest": "7013217e0a5669e3a4ddb8f3e02f70a4dad08cfe5d10170a962ff3f2fa8db3d7", "sectionDigests": { "state": "d7951fa36f9e6fe9", "blockContents": "b6701268a0bdcd8a", "stats": "67e33bcc612fdcc1", - "metadata": "ac1c297bf3b603eb" + "metadata": "f3bd0c932cc616b0" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", @@ -101,7 +101,9 @@ "terminalStreak": "5feceb66ffc86f38", "nextRuleId": "6b86b273ff34fce1", "absorbed": "4f53cda18c2baa0c", - "rules": "4f53cda18c2baa0c" + "rules": "4f53cda18c2baa0c", + "sweptOrphanCallIds": "74234e98afe7498f", + "pendingOrphanSweepMinRef": "74234e98afe7498f" }, "stateShape": { "refs": 55, diff --git a/tests/stub-resubmission-guard.test.ts b/tests/stub-resubmission-guard.test.ts index a1864162d6..42c5e2a097 100644 --- a/tests/stub-resubmission-guard.test.ts +++ b/tests/stub-resubmission-guard.test.ts @@ -124,7 +124,10 @@ test("#2749 canonical integrity pin: full summary survives save → history redu ["user", "Follow-up questions about the configuration."], ["assistant", "y".repeat(3000)], ]); - const out = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00004", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + // The fold is linked to its tool-call id exactly as production does + // (src/loop/core.ts passes callId); the resent "call-b1" below then + // classifies active instead of orphan. + const out = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00004", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(out.startsWith("[Compressed m00001–m00004 → 1 block(s)"), out.split("\n")[0]); // (a) saved canonically in full — no trimming, no truncation. assert.equal(storedSummary(ctx.session.state, "b1"), LONG_SUMMARY); @@ -155,7 +158,7 @@ test("#2749 rejects a byte-identical stub re-submission; state unchanged, refusa ["user", "late part two"], ["assistant", "d".repeat(1000)], ["user", "late part three"], ["assistant", "e".repeat(1000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed m00001–m00002 → 1 block(s)"), ok.split("\n")[0]); // Next request: the model sees its own call reduced to the stub. ctx.messages = nextRequestView(ctx, "call-b1", "m00001", "m00002").map((m) => ({ ...m })); @@ -181,7 +184,7 @@ test("#2749 self-refold under the block's own stub is blocked (silent-shrink wor ["user", "setup steps discussed early"], ["assistant", "a".repeat(4000)], ["user", "more detail on the setup"], ["assistant", "b".repeat(4000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed m00001–m00002 → 1 block(s)"), ok.split("\n")[0]); ctx.messages = nextRequestView(ctx, "call-b1", "m00001", "m00002").map((m) => ({ ...m })); // Re-fold the SAME span submitting the block's own stub as the new summary. @@ -204,7 +207,7 @@ test("#2749 legitimate summaries still pass: short, ellipsis-prose, and ±1-char ["user", "late four"], ["assistant", "f".repeat(4000)], ["user", "late five"], ["assistant", "g".repeat(4000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed m00001–m00002 → 1 block(s)"), ok.split("\n")[0]); ctx.messages = nextRequestView(ctx, "call-b1", "m00001", "m00002").map((m) => ({ ...m })); @@ -234,7 +237,7 @@ test("#2749 host harvest tracks kernel-emitted stubs (format-drift tripwire)", ( seedTurn(ctx, [ ["user", "early part"], ["assistant", "a".repeat(1000)], ]); - const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }), ctx).text; + const ok = applyRanges(parseCompressInput({ content: [{ startId: "m00001", endId: "m00002", topic: "T", summary: LONG_SUMMARY }] }, "call-b1"), ctx).text; assert.ok(ok.startsWith("[Compressed"), ok.split("\n")[0]); const view = nextRequestView(ctx, "call-b1", "m00001", "m00002"); const visCall = view.find((m) => m.toolCallId === "call-b1")!; From 9488e484f7e6b26382caa010cbac76b6e8a5eb83 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:16 +0800 Subject: [PATCH 10/27] kernel: honor legacy hiddenOrphanRefs on upgrade (#3769 P0) Pre-#3769 states persisted the keep-two rotation's hidden set as message refs (hiddenOrphanRefs); the new hide pass ignored it, so upgrading would resurrect rotation-hidden failed pairs mid-history and break the cached prefix again. Honor the legacy set on upgrade; rememberHiddenRef re-records those refs every pass, so the clause self-perpetuates without any state migration. --- kernel/src/hide-consumed.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index 69819675db..888469bb32 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -276,10 +276,18 @@ export function hideConsumedCompressCalls( if (info.fate === "inactive") { hide = true; } else if (info.fate === "orphan") { + const ref = state.messageRefs.byRaw[messages[i]!.id]; if (sweptSet.has(callId)) { hide = true; + } else if (ref && state.hiddenOrphanRefs?.includes(ref)) { + // Pre-#3769 states persisted the rotation's hidden set as message refs + // (hiddenOrphanRefs). Honor it on upgrade: resurrecting a pair the old + // kernel already removed would re-insert it mid-history and break the + // cached prefix again. The set self-perpetuates — rememberHiddenRef + // re-records these refs on every pass they stay hidden. + hide = true; } else if (foldMinRef !== undefined) { - const num = refNum(state.messageRefs.byRaw[messages[i]!.id]); + const num = refNum(ref); if (num !== null && num >= foldMinRef) { hide = true; newlySweptCallIds.push(callId); From 3eaf0cde01895c88fffcf6d3b7492cdb2e55a5d6 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:16 +0800 Subject: [PATCH 11/27] kernel: carry orphan-cleanup fields through sync-blocks (#3769 P0) syncBlocks deep-clones CompressionState with an explicit field list that missed hiddenOrphanRefs / sweptOrphanCallIds / pendingOrphanSweepMinRef. It runs BEFORE the hide-compress-calls node in every processTurn, so all three ledgers were wiped each turn: the fold-gated sweep never fired end-to-end and the legacy upgrade-compat clause was unreachable. Carry them exactly like cloneState does (lastPassIds stays excluded: its only reader runs pre-sync and the field is re-stamped at processTurn exit). --- kernel/src/sync.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kernel/src/sync.ts b/kernel/src/sync.ts index 76dc42c259..9a12073f3e 100644 --- a/kernel/src/sync.ts +++ b/kernel/src/sync.ts @@ -46,6 +46,13 @@ export function syncBlocks( nextBlockId: state.nextBlockId, nextRunId: state.nextRunId, deadRefs: state.deadRefs ? [...state.deadRefs] : undefined, + // #3769 orphan-cleanup ledgers — same carry set as cloneState: dropping any + // of these here would wipe them before the hide-compress-calls node reads + // them (this node runs first), silently disabling both the fold-gated sweep + // and the legacy hiddenOrphanRefs upgrade-compat clause every turn. + hiddenOrphanRefs: state.hiddenOrphanRefs ? [...state.hiddenOrphanRefs] : undefined, + sweptOrphanCallIds: state.sweptOrphanCallIds ? [...state.sweptOrphanCallIds] : undefined, + pendingOrphanSweepMinRef: state.pendingOrphanSweepMinRef, }; // Refs are additive (assignRefs never removes them from messageRefs), so From a8d24aa8a37052f74bdbf1e96f296b4299bcb158 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:38 +0800 Subject: [PATCH 12/27] kernel: regression tests for upgrade compat and pipeline carry (#3769 P0) Two processTurn-level pins: an upgraded pre-fold-gate state keeps its rotation-hidden orphan hidden across quiet turns (legacy hiddenOrphanRefs clause), and a fold marker set by applyCompression survives the full pipeline to the hide node (sync-blocks carry) so the fold-gated sweep actually fires end-to-end. --- kernel/tests/issue3769-orphan-sweep.test.ts | 71 +++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/kernel/tests/issue3769-orphan-sweep.test.ts b/kernel/tests/issue3769-orphan-sweep.test.ts index abf97737f3..839ba6ac4b 100644 --- a/kernel/tests/issue3769-orphan-sweep.test.ts +++ b/kernel/tests/issue3769-orphan-sweep.test.ts @@ -232,3 +232,74 @@ test("a block-boundary fold marks the sweep by covered message refs, not the blo assert.deepEqual(r.newlySweptCallIds, [], "cached-prefix orphan before the fold is untouched"); assert.ok(r.messages.some((m) => m.toolCallId === "f1"), "f1 remains visible"); }); + +test("an upgraded pre-fold-gate state keeps its rotation-hidden orphan hidden (#3769)", () => { + const state = createInitialState(); + const messages: CoreMessage[] = [ + msg("p1", "early"), + ...failPair("f1"), + msg("p2", "mid"), + ...failPair("f2"), + msg("p3", "late"), + ...failPair("f3"), + ]; + seqRefs(state, messages); + // refs: p1=1 c-f1=2 r-f1=3 p2=4 c-f2=5 r-f2=6 p3=7 c-f3=8 r-f3=9 + + // Exact shape of a state the old keep-two kernel persisted: with three failures + // the rotation had already removed f1 from mid-history and remembered its refs. + // sweptOrphanCallIds / pendingOrphanSweepMinRef do not exist yet. + state.hiddenOrphanRefs = ["m00002", "m00003"]; + + const first = hideConsumedCompressCalls(state, messages, isMcp); + assert.deepEqual(first.newlySweptCallIds, [], "nothing newly swept without a fold"); + const ids = first.messages.map((m) => m.id); + assert.ok(!ids.includes("c-f1") && !ids.includes("r-f1"), "previously-hidden pair does not resurface on upgrade"); + assert.ok(ids.includes("c-f2") && ids.includes("c-f3"), "recent failures stay visible"); + + // Pipeline write-back (hide-compress-calls node) then another quiet turn: the + // legacy set self-perpetuates through rememberHiddenRef, no new field required. + state.hiddenOrphanRefs = first.hiddenOrphanRefs; + const second = hideConsumedCompressCalls(state, messages, isMcp); + const ids2 = second.messages.map((m) => m.id); + assert.ok(!ids2.includes("c-f1") && !ids2.includes("r-f1"), "still hidden on the next quiet turn"); +}); + +test("the fold marker survives the full pipeline: sync-blocks carries it to the hide node (#3769)", () => { + const core = createCore(); + const messages: CoreMessage[] = [ + msg("p1", "early cached prefix alpha"), + ...failPair("f2"), + msg("p2", "mid detailed work item beta content"), + ]; + const state = createInitialState(); + seqRefs(state, messages); + // refs: p1=1 c-f2=2 r-f2=3 p2=4 — a fold starting at ref 2 committed last turn. + // Before the sync-blocks carry fix this marker died on the FIRST pipeline node + // (syncBlocks rebuilt state without the #3769 fields), so the sweep never fired. + state.pendingOrphanSweepMinRef = 2; + + const cfg = { ...config(), isCompressTool: isMcp }; + const turn = core.processTurn({ + messages, + state, + config: cfg, + tokenCount: 1000, + renderTags: "none", + }); + + const ids = turn.messages.map((m) => m.id); + assert.ok(!ids.includes("c-f2") && !ids.includes("r-f2"), "orphan at/after fold start swept inside the pipeline"); + assert.deepEqual(turn.state.sweptOrphanCallIds, ["f2"], "swept id folded into the persistent set"); + assert.equal(turn.state.pendingOrphanSweepMinRef, undefined, "one-shot marker consumed"); + + const quiet = core.processTurn({ + messages: [...messages, msg("p3", "latest user intent theta")], + state: turn.state, + config: cfg, + tokenCount: 1000, + renderTags: "none", + }); + const ids2 = quiet.messages.map((m) => m.id); + assert.ok(!ids2.includes("c-f2"), "previously-swept orphan stays hidden on the next pipeline pass"); +}); From 0be491fc5627a04a1b5cd277c5e9c169cd227a67 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:38 +0800 Subject: [PATCH 13/27] fix: inherit orphan-cleanup ledgers on resume/fork adoption (#3769 P0) applyForkAdoption rebuilds session state from the parent on resume/fork replay; without carrying hiddenOrphanRefs and sweptOrphanCallIds, failed compress pairs already hidden pre-upgrade or already swept would resurface on replay, rewriting mid-history at full cache cost. Both ledgers are monotonic sets re-recorded every pass, so the carry self-perpetuates. --- src/fork-adoption.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 5de73f8a2b..16222d9a61 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -262,6 +262,14 @@ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, pare // dropped (its failed tail, a rewound branch) — and the model's older // text may still cite those. Reserve the parent's whole ref space. reserveRefsThrough(session, highestUsedIndex(parent.state.messageRefs)); + // #3769: carry the orphan-cleanup ledgers too — without this, the failed + // compress pairs the pre-upgrade keep-two rotation hid (hiddenOrphanRefs) + // or the fold-gated sweep already swept (sweptOrphanCallIds) would + // RESURFACE on resume/fork replay, rewriting mid-history at full cache + // cost. Both are monotonic id/ref sets; the hide pass re-records them + // every turn, so the carry self-perpetuates afterwards. + if (parent.state.hiddenOrphanRefs?.length) session.state.hiddenOrphanRefs = [...parent.state.hiddenOrphanRefs]; + if (parent.state.sweptOrphanCallIds?.length) session.state.sweptOrphanCallIds = [...parent.state.sweptOrphanCallIds]; markDirty(session); } From fe2c46e5c6f92aa53495fd576e7ed6162aff70b4 Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:38 +0800 Subject: [PATCH 14/27] test: end-to-end upgrade compat for rotation-hidden orphans (#3769 P0) Real-server e2e: drive a session with a failed compress pair, persist it, patch the record back to an old-shape state (hiddenOrphanRefs set, no #3769 fields), reload under a fresh proxy, and assert the previously-hidden pair stays out of the outbound history while new turns append stably. --- tests/issue3769-upgrade-compat.test.ts | 215 +++++++++++++++++++++++++ 1 file changed, 215 insertions(+) create mode 100644 tests/issue3769-upgrade-compat.test.ts diff --git a/tests/issue3769-upgrade-compat.test.ts b/tests/issue3769-upgrade-compat.test.ts new file mode 100644 index 0000000000..289528f6cd --- /dev/null +++ b/tests/issue3769-upgrade-compat.test.ts @@ -0,0 +1,215 @@ +// #3769 upgrade compatibility (owner merge condition): sessions persisted by a +// pre-fold-gate kernel carry their rotation-hidden orphan pairs ONLY in +// `hiddenOrphanRefs` (no sweptOrphanCallIds / pendingOrphanSweepMinRef). After +// upgrading, those pairs must stay hidden on the outbound history — resurrecting +// them re-inserts mid-history bytes and breaks the cached prefix again. +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; +import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +process.env.NODE_ENV = "test"; +process.env.BILI_PERSIST_ZSTD = "0"; +process.env.BILI_CONFIG_FILE = path.join(mkdtempSync(path.join(tmpdir(), "bili-upg-hermetic-")), "no-config.json"); + +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions, _resetSessionsForTest } from "../src/session.ts"; +import { rmrf } from "./tmp-rm.ts"; + +const LABEL = "upg3769"; +const ARG_JSON = JSON.stringify({ content: [{ startId: "m00001", endId: "m00002", topic: "upg", summary: "UPG_SUM_MARKER" }] }); +const RESULT_TEXT = "UPG_RESULT_MARKER"; + +type M = Record; + +const u1: M = { role: "user", content: "q one alpha" }; +const a1: M = { role: "assistant", content: "a one alpha" }; +const u2: M = { role: "user", content: "q two beta" }; +const a2: M = { role: "assistant", content: "a two beta" }; +const pairCall: M = { role: "assistant", content: null, tool_calls: [{ id: "call_upg1", type: "function", function: { name: "compress", arguments: ARG_JSON } }] }; +const pairResult: M = { role: "tool", tool_call_id: "call_upg1", content: RESULT_TEXT }; +const u3: M = { role: "user", content: "q three gamma" }; +const u4: M = { role: "user", content: "q four delta UPG_TAIL_MARKER" }; + +function sse(obj: unknown): string { + return `data: ${JSON.stringify(obj)}\n\n`; +} + +async function post(url: string, messages: M[], bodies: string[]): Promise { + const r = await fetch(url, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": LABEL }, + body: JSON.stringify({ model: "gpt-test", messages, stream: true }), + }); + void bodies; + return r.status ?? 0; +} + +type PersistRecord = { + version: number; + savedAt: number; + id: string; + payload: { + meta: { label?: string }; + messages: { id: string; contentType?: string; toolName?: string; toolCallId?: string }[]; + state: { messageRefs: { byRaw: Record }; hiddenOrphanRefs?: string[]; sweptOrphanCallIds?: string[]; pendingOrphanSweepMinRef?: number }; + }; +}; + +function findSessionFile(root: string, label: string): string { + const stack = [root]; + while (stack.length > 0) { + const d = stack.pop()!; + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = path.join(d, e.name); + if (e.isDirectory()) { + stack.push(p); + } else if (e.name.endsWith(".json") && !e.name.endsWith(".content-store.json")) { + try { + const rec = JSON.parse(readFileSync(p, "utf8")) as PersistRecord; + if (rec.id === label || rec.payload?.meta?.label === label) return p; + } catch { /* not our file */ } + } + } + } + throw new Error(`no session file for ${label} under ${root}`); +} + +test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-end (#3769)", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "bili-upg-store-")); + const bodies: string[] = []; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + bodies.push(Buffer.concat(chunks).toString("utf8")); + const n = bodies.length; + res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: { role: "assistant", content: `ok ${n}` } }] })); + res.write(sse({ id: "c1", object: "chat.completion.chunk", choices: [{ index: 0, delta: {}, finish_reason: "stop" }] })); + res.write("data: [DONE]\n\n"); + res.end(); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upPort = (upstream.address() as { port: number }).port; + + const stores: SessionStore[] = []; + try { + // Phase 1: current code runs two turns; the failed pair rides visibly + // (fold-gated contract: no fold -> no sweep) and the record lands on disk. + const store1 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + stores.push(store1); + _setStoreForTest(store1); + _resetSessionsForTest(); + setRegistryForTest({}); + const proxy1 = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upPort}`]: { models: { "gpt-test": { context: 15_000 } } } }, + modelContextLimit: 15_000, + kernelConfig: defaultConfig(15_000), + compress: { injectTool: true, injectNudge: true }, + sessionHeader: "x-acp-session", + passthrough: false, + passthroughSource: null, + autoUpdate: false, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: true, + releaseNotesCheck: true, + compat: { roles: {} }, + streamErrorShape: "protocol", + mitm: { enabled: false, domains: [] }, + } as unknown as ProxyOptions); + await once(proxy1, "listening"); + const port1 = (proxy1.address() as { port: number }).port; + const url1 = `http://127.0.0.1:${port1}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; + + assert.equal(await post(url1, [u1, a1, u2, a2], bodies), 200); + assert.equal(await post(url1, [u1, a1, u2, a2, { role: "assistant", content: "ok 1" }, pairCall, pairResult, u3], bodies), 200); + // New-code baseline: without a fold the pair stays visible outbound. + assert.ok(bodies[bodies.length - 1]!.includes("UPG_SUM_MARKER"), "pre-upgrade baseline: failed pair rides visibly"); + + const s1 = listSessions().find((x) => x.meta.label === LABEL); + assert.ok(s1, "session exists"); + assert.ok(store1.flushSync(s1!)); + proxy1.close(); + proxy1.closeIdleConnections?.(); + await once(proxy1, "close"); + + // Patch the persisted record into the exact shape an old kernel left: + // rotation had hidden the pair (remembered by ref), new fields absent. + const file = findSessionFile(dir, LABEL); + const rec = JSON.parse(readFileSync(file, "utf8")) as PersistRecord; + const callMsg = rec.payload.messages.find((m) => m.contentType === "tool-call" && m.toolName === "compress"); + const resMsg = rec.payload.messages.find((m) => m.contentType === "tool-result" && m.toolCallId === "call_upg1"); + assert.ok(callMsg && resMsg, "record snapshot carries the failed pair"); + const callRef = rec.payload.state.messageRefs.byRaw[callMsg!.id]; + const resRef = rec.payload.state.messageRefs.byRaw[resMsg!.id]; + assert.ok(callRef && resRef, "refs resolvable from the persisted state"); + rec.payload.state.hiddenOrphanRefs = [callRef, resRef]; + delete rec.payload.state.sweptOrphanCallIds; + delete rec.payload.state.pendingOrphanSweepMinRef; + writeFileSync(file, JSON.stringify(rec)); + + // Phase 2: a fresh server instance hydrates the patched record. + _resetSessionsForTest(); + const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + stores.push(store2); + _setStoreForTest(store2); + const proxy2 = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upPort}`]: { models: { "gpt-test": { context: 15_000 } } } }, + modelContextLimit: 15_000, + kernelConfig: defaultConfig(15_000), + compress: { injectTool: true, injectNudge: true }, + sessionHeader: "x-acp-session", + passthrough: false, + passthroughSource: null, + autoUpdate: false, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: true, + releaseNotesCheck: true, + compat: { roles: {} }, + streamErrorShape: "protocol", + mitm: { enabled: false, domains: [] }, + } as unknown as ProxyOptions); + await once(proxy2, "listening"); + const port2 = (proxy2.address() as { port: number }).port; + const url2 = `http://127.0.0.1:${port2}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; + + const hist3: M[] = [u1, a1, u2, a2, { role: "assistant", content: "ok 1" }, pairCall, pairResult, u3, { role: "assistant", content: "ok 2" }, u4]; + assert.equal(await post(url2, hist3, bodies), 200); + const out = bodies[bodies.length - 1]!; + assert.ok(!out.includes("UPG_SUM_MARKER"), "upgrade: hidden compress args do not resurface"); + assert.ok(!out.includes("UPG_RESULT_MARKER"), "upgrade: hidden result does not resurface"); + assert.ok(out.includes("UPG_TAIL_MARKER"), "live tail rides normally"); + + const s2 = listSessions().find((x) => x.meta.label === LABEL); + assert.ok(s2, "hydrated session exists"); + assert.ok(s2.state.hiddenOrphanRefs?.includes(callRef!), "legacy hidden set survived hydration and self-perpetuates"); + proxy2.close(); + proxy2.closeIdleConnections?.(); + await once(proxy2, "close"); + } finally { + for (const s of stores) s.closeQuietly(); + upstream.close(); + upstream.closeIdleConnections?.(); + await once(upstream, "close").catch(() => undefined); + _setStoreForTest(new SessionStore({ enabled: false })); + void rmrf(dir); + } +}); From 8e8590ef3b770775be06be1bc304030ecc4741cc Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 22:34:51 +0800 Subject: [PATCH 15/27] test: pin fold-gated sweep timing through the live server (#3769) With the sync-blocks carry in place the sweep fires end-to-end: a failed compress pair appended at a fold step is swept by that step's own model request, so it never rides inbound and its anchor carries the summary for good (zero-carrier fail-safe, #1567). Re-pin the main matrix (all four wires), the swallow/back-to-back geometries (wide fold absorbs the alpha carrier; FOLD_E tail hits the protected zone), the drop-pair subtest (client-side drop is now a wire no-op) and the 3-concurrent-folds pin (carriers stack one per fold, stable thereafter), from observed live-server bodies. Google pair-absence proxy switches from the topic (which the carrier embeds) to the fold-spec refs that only the re-sent functionCall args carry. --- tests/cache-friendly-plugin.test.ts | 194 ++++++++++++++-------------- 1 file changed, 97 insertions(+), 97 deletions(-) diff --git a/tests/cache-friendly-plugin.test.ts b/tests/cache-friendly-plugin.test.ts index 18bf5496ee..deeea63826 100644 --- a/tests/cache-friendly-plugin.test.ts +++ b/tests/cache-friendly-plugin.test.ts @@ -11,10 +11,12 @@ // therefore re-injects the in-place acp_summary carrier from block state on // every request. Since the #1567 fix the carrier is STRIPPED per-block, but // self-verified: only while that exact fold's compress pair actually rides -// the (post-prepare) inbound history. A pair pruned client-side or swept by -// the kernel's fold-gated orphan cleanup (#3769) keeps the anchor, so an -// active fold never has zero carriers. This suite pins that behavior -// byte-for-byte. +// the (post-prepare) inbound history. Under fold-gated sweeps (#3769) a pair +// whose call sits at/after a committed fold's start ref is hidden from that +// fold turn on — in this harness the fold POST and its model request share +// one step, so each failed pair is swept at its own fold step, never rides, +// and its anchor carries the summary for good (zero-carrier fail-safe). This +// suite pins that behavior byte-for-byte. // // Invariants pinned per wire (bodies = consecutive upstream-bound request // payloads of one session, compared as flat unit lists — text blocks and @@ -34,19 +36,17 @@ // /__bili/plugin/tool fold): // P2a the pre-fold span is replaced by exactly the expected carrier // units and nothing else moves or mutates (exact reconstruction); -// P2b the in-place carrier is stripped from every post-fold turn -// while the client's pair rides inbound — occurrence counts -// pinned: exactly 2 summary-text hits per post-fold turn (the -// fold spec echoed in the re-sent tool_call args + the tail -// quote inside the persistently re-sent tool result) and ZERO -// carrier markers (#1567 fix); - // P2c fail-safe (#1567 hardening): the anchor re-carries the summary - // whenever the pair does NOT ride inbound — the client stops - // re-sending it, or the kernel swept it on a later fold (#3769) — - // pinned by the hardening subtests at the bottom of this file; - // P3 multi-fold: two folds in one session; both pairs stay visible for the - // rest of the session (fold-gated sweeps, #3769 — keep-two rotation is - // gone) and the pair-quote counts stay constant across all later turns; +// P2b the pair is swept at its own fold step (#3769) and never rides +// inbound — occurrence counts pinned: exactly ONE summary-text hit +// per post-fold turn (the anchor carrier) and ZERO pair quotes, +// one carrier marker per committed fold (#1567 fail-safe); +// P2c fail-safe (#1567 hardening): the anchor carries the summary +// whenever the pair does NOT ride inbound — under #3769 that is +// every post-fold turn in this harness; the drop-pair subtest +// pins that a client-side pair drop afterwards is a wire no-op; +// P3 multi-fold: two folds in one session; each anchor carries its own +// summary from its fold turn on, the carrier count grows by one per +// fold and then stays constant (keep-two rotation is gone, #3769); // P4 no transient acp_loop_* artifact leaks into any forwarded body // (plugin mode has none by construction — pin it); // P5 marker/tag hygiene: every content text unit carries exactly one @@ -69,7 +69,8 @@ // // Scenario (main, all wires): t1=[q1,a1,q2,a2], t2=+q3,a3, t3=+q4,a4, // foldA(m00001..m00002)+pairA@t4, t5=+q5,a5, foldB(m00003..m00005)+pairB@t6, -// t7=+q7,a7, t8=+q8,a8 — every historical pair rides every later request. +// t7=+q7,a7, t8=+q8,a8 — each failed pair is swept at its own fold step +// (#3769); its anchor carries the summary from there on. // // Scope notes: tool protocol only (the default). The swallow/back-to-back // fold geometries get a dedicated anthropic subtest (kernel-side behavior, @@ -614,28 +615,29 @@ for (const wire of ["anthropic", "openai", "responses", "google"] as Wire[]) { assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // head boundary survives; anchor stripped (#1567) - assertPrefix(cans[3]!.core, cans[4]!.core, L(4), wire === "google" ? ["a5"] : ["q5", "a5"], wire === "google"); // google: #3159 — q5 leads the fold-pair result - assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], [], L(5)); // anchor stripped (#1567) - assertPrefix(cans[5]!.core, cans[6]!.core, L(6), wire === "google" ? ["a7"] : ["q7", "a7"], wire === "google"); // google: #3159 — q7 leads the fold-pair result + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1", "SA"], L(3)); // head boundary survives; anchor A carries — its pair was swept at fold time (#3769) + assertPrefix(cans[3]!.core, cans[4]!.core, L(4), ["q5", "a5"]); + assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], ["SB"], L(5)); // span consumed; anchor B carries (#3769) + assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q7", "a7"]); assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); for (let i = 3; i < cans.length; i++) { const raw = cans[i]!.raw; - // #1567 (fixed): the in-place carrier is stripped per-block while - // the client's re-sent pair rides inbound — only the two pair - // quotes (tool_call args + tool result) keep the summary text. - const expA = 2; - const expB = i < 5 ? 0 : 2; - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: in-place carrier must be stripped while the client's pair rides inbound (got ${countOcc(raw, SUM_MARKER)})`); - assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA} (pair quotes only, #1567)`); - assert.equal(countOcc(raw, SUM_B), expB, `${L(i)}: SUM_B occurrences ${countOcc(raw, SUM_B)} != ${expB} (pair quotes only, #1567)`); - // google part-shapes carry no call id; pin pair presence through - // the fold topic embedded in the re-sent functionCall args - const pairA = wire === "google" ? "CFP-TOPIC-A" : "\"cfp_a\""; - const pairB = wire === "google" ? "CFP-TOPIC-B" : "\"cfp_b\""; - assert.ok(raw.includes(pairA), `${L(i)}: pairA lost from re-sent history`); - if (i >= 5) assert.ok(raw.includes(pairB), `${L(i)}: pairB lost from re-sent history`); + // #3769: each failed pair is swept at its own fold step, so no + // pair quotes ride inbound — each anchor carries its summary for + // good (zero-carrier fail-safe, #1567) + const expA = 1; + const expB = i < 5 ? 0 : 1; + assert.equal(countOcc(raw, SUM_MARKER), i < 5 ? 1 : 2, `${L(i)}: one carrier per committed fold (pairs swept, #3769)`); + assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA} (anchor carrier only, #3769)`); + assert.equal(countOcc(raw, SUM_B), expB, `${L(i)}: SUM_B occurrences ${countOcc(raw, SUM_B)} != ${expB} (anchor carrier only, #3769)`); + // google part-shapes carry no call id; pin pair ABSENCE through + // the fold spec refs that would sit in the re-sent functionCall + // args (the anchor carrier embeds topic+summary, never the spec) + const pairA = wire === "google" ? "\"startId\":\"m00001\"" : "\"cfp_a\""; + const pairB = wire === "google" ? "\"startId\":\"m00003\"" : "\"cfp_b\""; + assert.ok(!raw.includes(pairA), `${L(i)}: swept pairA must stay hidden (#3769)`); + assert.ok(!raw.includes(pairB), `${L(i)}: swept pairB must stay hidden (#3769)`); } }, { timeout: 120_000 }); @@ -666,21 +668,32 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // head boundary survives; anchor stripped (#1567) + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1", "SA"], L(3)); // head boundary survives; anchor A carries (pair swept at fold time, #3769) assertPrefix(cans[3]!.core, cans[4]!.core, L(4), ["q5", "a5"]); const wide = FOLD_WIDE.summary; const gamma = FOLD_E.summary; + // Fold-gated sweeps (#3769): every failed pair is hidden from its own + // fold step on, so no pair quotes ride inbound and each anchor carries + // its summary. The wide fold (m00001..m00004) swallows the alpha fold's + // range entirely — the alpha summary merges away and the wide carrier + // takes over. FOLD_E's tail sits in the protected zone, so only its + // first message folds and the gamma carrier lands where q3 used to sit. for (const i of [5, 6, 7]) { const raw = cans[i]!.raw; - // carrier is superseded on every turn; the geo_a pair quotes ride - // every later turn — fold-gated sweeps (#3769) replaced keep-two - // rotation, so no later fold prunes this oldest pair here - assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != 2`); - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: carriers must be stripped while their pairs ride inbound (got ${countOcc(raw, SUM_MARKER)})`); assert.ok(!raw.includes("q2-marker"), `${L(i)}: swallowed content q2 must stay hidden`); assert.ok(!raw.includes("a2-marker"), `${L(i)}: swallowed content a2 must stay hidden`); + assert.ok(!raw.includes("\"geo_a\"") && !raw.includes("\"geo_wide\"") && !raw.includes("\"geo_e\""), `${L(i)}: all three failed pairs swept at their fold steps (#3769)`); } + assert.equal(countOcc(cans[5]!.raw, SUM_A), 0, `${L(5)}: alpha summary absorbed into the wide fold`); + assert.equal(countOcc(cans[5]!.raw, wide), 1, `${L(5)}: wide carrier only (its pair swept, #3769)`); + assert.equal(countOcc(cans[5]!.raw, SUM_MARKER), 1, `${L(5)}: single carrier after the swallow`); + assert.equal(countOcc(cans[6]!.raw, wide), 1, `${L(6)}: wide carrier persists`); + assert.equal(countOcc(cans[6]!.raw, gamma), 1, `${L(6)}: gamma carrier only (its pair swept, #3769)`); + assert.equal(countOcc(cans[6]!.raw, SUM_MARKER), 2, `${L(6)}: one carrier per committed fold`); + assert.equal(countOcc(cans[7]!.raw, wide), 1, `${L(7)}: wide carrier persists`); + assert.equal(countOcc(cans[7]!.raw, gamma), 1, `${L(7)}: gamma carrier persists`); + assert.equal(countOcc(cans[7]!.raw, SUM_MARKER), 2, `${L(7)}: carriers stable across quiet turns`); const surviveKeys = new Set(["q1", "a1", "q2", "a2", "SA"]); const keepT5 = cans[4]!.core.filter((u) => !surviveKeys.has(keyOf(u))); let cursor = -1; @@ -689,32 +702,23 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr assert.ok(found > cursor, `cfp-geo t6: unit ${JSON.stringify(keyOf(u))} lost or reordered by the wide fold`); cursor = found; } - assert.equal(countOcc(cans[5]!.raw, wide), 2, `${L(5)}: wide fold turn occurrences (pair quotes; anchor stripped)`); - assert.equal(countOcc(cans[6]!.raw, gamma), 2, `${L(6)}: gamma fold turn occurrences (pair quotes; anchor stripped)`); - assert.equal(countOcc(cans[7]!.raw, wide), 2, `${L(7)}: resent wide quotes must survive`); - assert.equal(countOcc(cans[7]!.raw, gamma), 2, `${L(7)}: resent gamma quotes must survive`); - for (const i of [6, 7]) { - const raw = cans[i]!.raw; - // all three failed pairs stay visible: fold-gated sweeps (#3769) only - // remove orphans at/after a committed fold's start, never rotate the - // newest failures away between folds - assert.ok(raw.includes("\"geo_a\"") && raw.includes("\"geo_wide\"") && raw.includes("\"geo_e\""), `${L(i)}: all three pairs must stay visible (no keep-two rotation)`); - } - // #1567 (fixed): no text unit carries the wide summary anymore (stripped - // anchor); the quotes ride structured tool units, not text + // #1567/#3769: the wide summary rides its anchor carrier (a text unit) + // for good — the pair was swept at fold time, so the carrier is never + // stripped again const wideT6 = cans[6]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); const wideT7 = cans[7]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); - assert.equal(wideT6.length, 0, `${L(6)}: wide carrier must be stripped while the geo_wide pair rides inbound`); - assert.equal(wideT7.length, 0, `${L(7)}: wide carrier must be stripped while the geo_wide pair rides inbound`); + assert.equal(wideT6.length, 1, `${L(6)}: wide carrier rides a text unit while the geo_wide pair stays swept`); + assert.equal(wideT7.length, 1, `${L(7)}: wide carrier rides a text unit while the geo_wide pair stays swept`); assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); -// #1567 hardening scenario A: a contract-violating (or pruning) client stops -// re-sending an older compress pair. While pairA rides, its anchor is -// stripped (P2b); the moment the pair vanishes the anchor MUST re-carry the -// summary (zero-carrier fail-safe) and the body must stay append-stable on -// later turns (no flapping). -testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → anchor re-carries (#1567 hardening)`, async () => { +// #1567 hardening scenario A, re-pinned under #3769: a contract-violating +// (or pruning) client stops re-sending an older compress pair at t7. Under +// fold-gated sweeps the pair was already hidden at its own fold step (t4), +// so its anchor has carried the summary since then — the client-side drop is +// a wire no-op, and the body stays append-stable with exactly one carrier +// per committed fold (no flapping). +testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → wire unaffected, carriers stable (#1567/#3769)`, async () => { const steps: Step[] = [ { send: [T(1), A(1), T(2), A(2)] }, { send: [T(3), A(3)] }, @@ -733,35 +737,28 @@ testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pai assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-drop"); - // t4..t6: both pairs ride → both anchors stripped - for (const i of [3, 4, 5]) { + // t4..t8: both pairs swept at their fold steps → each anchor is the sole + // carrier of its summary; the client dropping pairA at t7 changes nothing + for (const i of [3, 4, 5, 6, 7]) { const raw = cans[i]!.raw; - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: carrier must be stripped while pair rides`); - assert.ok(raw.includes('"cfp_a"'), `${L(i)}: pairA still re-sent here`); - } - assert.ok(cans[5]!.raw.includes('"cfp_b"'), `${L(5)}: pairB rides`); - // t7/t8: pairA gone → anchor A is the ONLY carrier again - for (const i of [6, 7]) { - const raw = cans[i]!.raw; - assert.ok(!raw.includes('"cfp_a"'), `${L(i)}: client no longer re-sends pairA`); - assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(i)}: anchor A must re-carry (exactly one carrier)`); - assert.equal(countOcc(raw, SUM_A), 1, `${L(i)}: SUM_A = carrier only (pair quotes gone with the pair)`); - assert.equal(countOcc(raw, SUM_B), 2, `${L(i)}: SUM_B quotes only, anchor B still stripped (pairB rides)`); - assert.ok(raw.includes('"cfp_b"'), `${L(i)}: pairB rides`); + assert.ok(!raw.includes('"cfp_a"') && !raw.includes('"cfp_b"'), `${L(i)}: failed pairs swept at fold time (#3769)`); + assert.equal(countOcc(raw, SUM_A), 1, `${L(i)}: anchor A carries (sole occurrence)`); + assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 1, `${L(i)}: anchor B carries from its fold turn on`); + assert.equal(countOcc(raw, SUM_MARKER), i < 5 ? 1 : 2, `${L(i)}: one carrier per committed fold`); } assert.ok(cans[6]!.raw.includes("q7-marker"), `${L(6)}: new turn visible`); // once re-carried, the body stays append-stable (no flapping) assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); -// #3769 contract pin (plugin lane): three concurrent ACTIVE folds. The old -// KEEP_LAST_ORPHANED=2 rotation hid the oldest failed pair on EVERY turn — -// a mid-history rewrite with no fold behind it, exactly #3769's cache seam. -// Fold-gated sweeps replaced it: failed pairs stay visible between folds, so -// every anchor stays stripped (its own pair rides) and later turns remain -// append-stable. The zero-carrier fail-safe itself (pair gone → anchor -// re-carries) is pinned by the drop-pair subtest above. -testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds keep all failed pairs riding (fold-gated sweeps, #3769)`, async () => { +// #3769 contract pin (plugin lane): three back-to-back folds, each with a +// failed (client-visible) compress pair. The old KEEP_LAST_ORPHANED=2 +// rotation hid the oldest pair on EVERY turn — a mid-history rewrite with no +// fold behind it, exactly #3769's cache seam. Fold-gated sweeps hide each +// pair at its own fold step (the deletion rides the fold's own rewrite), so +// every anchor carries its summary from the fold turn on, carriers stack +// without churning, and quiet turns stay append-stable. +testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds: pairs swept at fold time, anchors carry (#3769)`, async () => { const SUM_P1 = "Prune-safe fold one summary covering m00001..m00002"; const SUM_P2 = "Prune-safe fold two summary covering m00003..m00004"; const SUM_P3 = "Prune-safe fold three summary covering m00005..m00006"; @@ -785,20 +782,23 @@ testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds keep assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-prune"); - // t3/t4: each fold's pair rides from its own turn on → anchors stripped - for (const i of [2, 3]) { - assert.equal(countOcc(cans[i]!.raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while pairs ride`); + // t3..t5: each fold's pair is swept at its own fold step → each anchor + // carries its summary; carrier count grows by one per fold + for (const i of [2, 3, 4]) { + const raw = cans[i]!.raw; + assert.ok(!raw.includes('"fs1"') && !raw.includes('"fs2"') && !raw.includes('"fs3"'), `${L(i)}: failed pairs swept at their fold steps (#3769)`); + assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: anchor P1 carries (sole occurrence)`); + assert.equal(countOcc(raw, SUM_P2), i < 3 ? 0 : 1, `${L(i)}: anchor P2 carries from its fold turn on`); + assert.equal(countOcc(raw, SUM_P3), i < 4 ? 0 : 1, `${L(i)}: anchor P3 carries from its fold turn on`); + assert.equal(countOcc(raw, SUM_MARKER), i - 1, `${L(i)}: one carrier per committed fold`); } - assert.ok(cans[3]!.raw.includes('"fs1"') && cans[3]!.raw.includes('"fs2"'), `${L(3)}: both pairs visible`); - // t5..t7: three failed pairs coexist and ALL stay visible (no keep-two - // rotation, #3769) → every anchor stays stripped, no carrier needed - for (const i of [4, 5, 6]) { + // t6/t7: all three carriers persist, stable across quiet turns + for (const i of [5, 6]) { const raw = cans[i]!.raw; - assert.ok(raw.includes('"fs1"') && raw.includes('"fs2"') && raw.includes('"fs3"'), `${L(i)}: all three failed pairs must stay visible (fold-gated sweeps, #3769)`); - assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: no carrier needed — every anchor's pair rides inbound`); - assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: SUM_P1 pair quotes only`); - assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: SUM_P2 pair quotes only`); - assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: SUM_P3 pair quotes only`); + assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: P1 carrier persists`); + assert.equal(countOcc(raw, SUM_P2), 1, `${L(i)}: P2 carrier persists`); + assert.equal(countOcc(raw, SUM_P3), 1, `${L(i)}: P3 carrier persists`); + assert.equal(countOcc(raw, SUM_MARKER), 3, `${L(i)}: three carriers, stable`); } // mid-history never mutates between quiet turns assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q8", "a8"]); From 77ee6c793c306c342595f51b6ac14655c238dfee Mon Sep 17 00:00:00 2001 From: awork Date: Mon, 19 Oct 2026 23:30:49 +0800 Subject: [PATCH 16/27] test: regen storage-render goldens after rebase onto #3803 ledger fields (#3769 rebase) --- tests/golden/storage-render/anthropic.json | 4 ++-- tests/golden/storage-render/chat.json | 4 ++-- tests/golden/storage-render/google.json | 4 ++-- tests/golden/storage-render/responses.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 934a26087d..1484cc0699 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "f894d02089eb22567a0e5d332ab7c12153d2a3d92a37140f8fc3d4b3d3cfcc19", + "stateDigest": "5839fb455554a276e973fc8e2a6039971d430df48c5e64970728cc66f3005ae7", "sectionDigests": { "state": "a4b83f873d16bfc4", "blockContents": "dc7d13d250c94bb4", "stats": "cff3c2546898aeb3", - "metadata": "a10cb71754abe91e" + "metadata": "95795174db0e1e50" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 816c8816a1..1883114ffa 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "4731e0ab9374388662bdc5ef54386d22cf446b71712d379cdbbdd530805d10cc", + "stateDigest": "38d6459bd83802b5c7ba79b10d6e12a82ba99086377ae2ec8676c41558e08b3d", "sectionDigests": { "state": "463b3dc26a100c2a", "blockContents": "94a58f96d7c5cfc3", "stats": "59842ce5415d3a4d", - "metadata": "0f57b6cba74a16d8" + "metadata": "d16fb5934a3e8779" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 2fd2f8942f..cc6117c57e 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "5b8b7f10031808c1cfcc6161a6c3881d485d4016f435e27ee89bd015a296fee4", + "stateDigest": "f4bb18cf96a64525a323d3db05cd160a375464fdfdf5c788f823acdf96a95825", "sectionDigests": { "state": "a9b9531d952b48d8", "blockContents": "473264d9df8ba338", "stats": "e358dc992395fc7e", - "metadata": "bf873598b93525e7" + "metadata": "9e864ac22f86950a" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index e8897cc195..e305b07dc5 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "7013217e0a5669e3a4ddb8f3e02f70a4dad08cfe5d10170a962ff3f2fa8db3d7", + "stateDigest": "2f2c59befaa51d0bc726d44821f8c8ddf12d6237ff84dfbe213af7c74ca885cf", "sectionDigests": { "state": "d7951fa36f9e6fe9", "blockContents": "b6701268a0bdcd8a", "stats": "67e33bcc612fdcc1", - "metadata": "f3bd0c932cc616b0" + "metadata": "71bc14af02391230" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", From 7d3578c87dd6014ea4407564e111b9bc6d8e20b5 Mon Sep 17 00:00:00 2001 From: awork Date: Tue, 20 Oct 2026 03:24:57 +0800 Subject: [PATCH 17/27] fix: exempt active-block carriers from the fold-gated orphan sweep (#3769/#1567) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fold-gated sweep hid EVERY orphan-fate compress pair at/after the fold's start ref — including successful echoes whose call id never matched the server-minted block id. A re-sent pair quoting an ACTIVE block's range is the plugin lane's summary carrier (KDD#7), so hiding it strands the summary: the #1999 double-carrier test, the plugin session-lock e2e, #1426 mode-upgrade proof and #1638 notification pin all went red. resolveFate's 'orphan' verdict is id-based and cannot see success; the sweep branch now additionally requires the call to quote NO active block range (quotesActiveBlock: both arg shapes — content[] entries and flat top-level startId/endId — mirroring the host stripKernelSummaries carrier recognition). Genuinely failed records and dead-block pairs (swallowed by a wider fold) still sweep at the fold boundary, per the approved #3769 design. --- kernel/src/hide-consumed.ts | 59 +++++++++++++++++- kernel/tests/issue3769-orphan-sweep.test.ts | 69 +++++++++++++++++++++ 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index 888469bb32..d79400fb99 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -32,6 +32,61 @@ function rangeKey(startRef: string, endRef: string): string { return `${startRef}::${endRef}`; } +// #3769: does this call's args quote the folded range of an ACTIVE block? Such a +// re-sent pair IS that block's summary carrier — the host strips the in-place +// anchor only while it rides the history (stripKernelSummaries, #1567/#2042) — +// so the fold-gated sweep must leave it alone. Both arg shapes are accepted to +// mirror the host index ({content:[…]} AND the flat {startId,endId} models emit +// natively); parseCallText alone misses the flat form. Unparseable args fail +// safe as NOT a carrier (under-sweep is safe, over-sweep is not). +function quotesActiveBlock( + text: string | undefined, + activeRangeKeys: Set, +): boolean { + const raw = text ?? ""; + const start = raw.indexOf("{"); + if (start < 0) return false; + let parsed: unknown; + try { + parsed = JSON.parse(raw.slice(start)); + } catch { + return false; + } + if (!parsed || typeof parsed !== "object") return false; + const obj = parsed as Record; + let entries: unknown[]; + if (Array.isArray(obj.content)) { + entries = obj.content; + } else if (typeof obj.content === "string") { + try { + const inner: unknown = JSON.parse(obj.content); + entries = Array.isArray(inner) ? inner : []; + } catch { + return false; + } + } else { + entries = [obj]; + } + for (const entry of entries) { + if (!entry || typeof entry !== "object") continue; + const e = entry as Record; + const s = + typeof e.startId === "string" + ? e.startId + : typeof e.messageId === "string" + ? e.messageId + : ""; + const end = + typeof e.endId === "string" + ? e.endId + : typeof e.messageId === "string" + ? e.messageId + : ""; + if (s && end && activeRangeKeys.has(rangeKey(s, end))) return true; + } + return false; +} + // Numeric position of a padded ref (mNNNNN -> NNNNN); null when unmapped/blocked // so the caller treats it as "position unknown" and never sweeps on a guess. // Exported: applyCompression derives the fold sweep marker from covered message @@ -262,6 +317,8 @@ export function hideConsumedCompressCalls( // this fold's earliest start ref so we additionally sweep orphans at/after it. // Orphans before it sit in the still-cached prefix and stay visible; with no // fold pending nothing new is swept, so a quiet turn never breaks the prefix. + // Orphans quoting an ACTIVE block's folded range are that block's summary + // carrier (#1567) and are exempt from the sweep (quotesActiveBlock). const sweptSet = new Set(state.sweptOrphanCallIds ?? []); const foldMinRef = state.pendingOrphanSweepMinRef; const newlySweptCallIds: string[] = []; @@ -286,7 +343,7 @@ export function hideConsumedCompressCalls( // cached prefix again. The set self-perpetuates — rememberHiddenRef // re-records these refs on every pass they stay hidden. hide = true; - } else if (foldMinRef !== undefined) { + } else if (foldMinRef !== undefined && !quotesActiveBlock(messages[i]!.text, activeRangeKeys)) { const num = refNum(ref); if (num !== null && num >= foldMinRef) { hide = true; diff --git a/kernel/tests/issue3769-orphan-sweep.test.ts b/kernel/tests/issue3769-orphan-sweep.test.ts index 839ba6ac4b..f5765195c4 100644 --- a/kernel/tests/issue3769-orphan-sweep.test.ts +++ b/kernel/tests/issue3769-orphan-sweep.test.ts @@ -265,6 +265,75 @@ test("an upgraded pre-fold-gate state keeps its rotation-hidden orphan hidden (# assert.ok(!ids2.includes("c-f1") && !ids2.includes("r-f1"), "still hidden on the next quiet turn"); }); +test("a re-sent pair quoting an active block's range is its carrier: the fold sweep leaves it visible (#3769/#1567)", () => { + // Plugin-style fold: the block is minted a server-side callId the client can + // never echo, so the client's re-sent pair for that exact fold is fate=orphan + // by id — yet it IS the block's summary carrier (the host strips the in-place + // anchor only while the pair rides the history). Sweeping it would strand the + // fold with zero wire carriers. Both arg shapes must be recognized, mirroring + // the host carrier index (flat {startId,endId} AND {content:[…]}). + const core = createCore(); + const build = (callId: string, callText: string): CoreMessage[] => [ + msg("h1", "first detailed work item alpha content"), + msg("h2", "second detailed work item beta content"), + msg("h3", "third detailed work item gamma content"), + { + id: `c-${callId}`, + role: "assistant", + contentType: "tool-call", + toolName: "compress", + toolCallId: callId, + text: callText, + }, + { + id: `r-${callId}`, + role: "tool", + contentType: "tool-result", + toolCallId: callId, + text: "compressed the head", + }, + msg("last", "latest user intent theta"), + ]; + const fold = (messages: CoreMessage[], state: CompressionState) => + core.applyCompression({ + ranges: [ + { startRef: "m00001", endRef: "m00003", summary: "fold the spent head", topic: "head", compressCallId: "plugin_1234" }, + ], + messages, + state, + config: config(), + }); + + const carriers: Array<[string, string]> = [ + ["flat {startId,endId}", JSON.stringify({ startId: "m00001", endId: "m00003", topic: "head", summary: "fold the spent head" })], + ["{content:[…]} array", JSON.stringify({ content: [{ startId: "m00001", endId: "m00003", topic: "head", summary: "fold the spent head" }] })], + ]; + for (const [label, callText] of carriers) { + const messages = build("call_echo_1", callText); + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; + const out = fold(messages, state); + assert.equal(out.result.errors.length, 0, `${label}: unexpected errors: ${out.result.errors.join("; ")}`); + assert.ok(out.result.blocksCreated >= 1, `${label}: the head genuinely folds`); + + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, [], `${label}: carrier pair exempt from the fold sweep`); + assert.ok(r.messages.some((m) => m.toolCallId === "call_echo_1"), `${label}: carrier pair stays visible (#1567)`); + } + + // Negative control: a failure whose quoted range matches NO block is still swept. + const messages = build( + "call_fail_9", + JSON.stringify({ content: [{ startId: "mX", endId: "mY", summary: "s" }] }), + ); + const state = createInitialState(); + state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1 }).map; + const out = fold(messages, state); + assert.equal(out.result.errors.length, 0, `negative: unexpected errors: ${out.result.errors.join("; ")}`); + const r = hideConsumedCompressCalls(out.state, messages, isMcp); + assert.deepEqual(r.newlySweptCallIds, ["call_fail_9"], "true failure residue at/after the fold is swept"); +}); + test("the fold marker survives the full pipeline: sync-blocks carries it to the hide node (#3769)", () => { const core = createCore(); const messages: CoreMessage[] = [ From 8289c1088207340614e856e7b04bc07a82137990 Mon Sep 17 00:00:00 2001 From: awork Date: Tue, 20 Oct 2026 03:25:25 +0800 Subject: [PATCH 18/27] test: align cache-friendly-plugin matrix and upgrade-compat with the carrier regime and #3818 full-mode default (#3769 follow-through) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cache-friendly-plugin: the previous rewrite pinned the pre-floor-20 sweep regime (successful echo pairs hidden at their fold step, anchors carrying). That contradicts KDD#7 (plugin lane's wire carrier IS the tool call; the agent view never renders the kernel acp_summary), the approved #3769 scope (failed-record cleanup only), and the suite's own P6 dsh-rebase subtest, which already pinned pair-rides/anchor-stripped. Re-pin to the verified carrier regime across all four wires: pairs ride from their fold turn on (summary twice per riding pair: args + receipt), SUM_MARKER zero while any pair rides, drop-pair surfaces exactly its anchor once then stays append-stable (#1567 fail-safe), dead-block pairs still sweep at the swallow boundary (#3769), google lane expectations follow the #3159 canon-shift. issue3769-upgrade-compat: since #3818 flipped the storage default to full, the unified index row is the load path — patching only the JSON envelope left a stale row the loader served instead, so the simulated old-kernel record now goes through the store (loadSync + flushSync dual-writes both copies). Teardown now closes both proxies even when an assert throws (previously a mid-test failure hung the runner). --- kernel/src/compress.ts | 1 - .../issue3939-fold-boundary-pins.test.ts | 26 +- tests/cache-friendly-plugin.test.ts | 230 ++++++++++-------- tests/issue3769-upgrade-compat.test.ts | 52 ++-- 4 files changed, 178 insertions(+), 131 deletions(-) diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 2aea1377b2..8ca64f32f0 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -665,7 +665,6 @@ export function createCore(ports: Ports = {}): CompressionCore { const preLoopState = cloneState(state); const applyErrors: string[] = []; const warningsBase = warnings.length; - const notesBase = notes.length; // #3769: capture which blocks already exist so, after the loop, we can tell // which were created by THIS fold and where it starts (for the orphan sweep). const preFoldBlockIds = new Set(state.blocks.map((b) => b.blockId)); diff --git a/kernel/tests/issue3939-fold-boundary-pins.test.ts b/kernel/tests/issue3939-fold-boundary-pins.test.ts index bfd1488b6d..2b0dda2aa1 100644 --- a/kernel/tests/issue3939-fold-boundary-pins.test.ts +++ b/kernel/tests/issue3939-fold-boundary-pins.test.ts @@ -93,33 +93,39 @@ function liveBlockState(): { state: ReturnType } { return { state }; } -test("pin: hide-consumed keeps only the newest 2 orphaned compress calls (KEEP_LAST_ORPHANED=2)", () => { +test("pin: between folds no orphan is swept — quiet turns keep every failed compress call visible (#3769 fold gate)", () => { const messages = orphanHistory(); const res = hideConsumedCompressCalls(createInitialState(), messages); const keptCalls = res.messages .filter((m) => m.contentType === "tool-call") .map((m) => m.toolCallId); - assert.deepEqual(keptCalls, ["c3", "c4"]); - assert.equal(res.messages.length, 4); + // The pre-#3769 keep-last-two rotation hid the oldest failures on EVERY + // turn, rewriting the still-cached prefix for a few hundred chars. Now the + // window only sweeps once a committed fold stamps pendingOrphanSweepMinRef. + assert.deepEqual(keptCalls, ["c1", "c2", "c3", "c4"]); + assert.equal(res.messages.length, 8); + assert.equal(res.hidden, 0); + assert.deepEqual(res.newlySweptCallIds, []); }); -test("pin: hidden orphan calls report their refs when state has them assigned", () => { +test("pin: a pending fold sweep hides orphans at/after its min ref and reports their refs (#3769)", () => { const messages = orphanHistory(); const state = createInitialState(); state.messageRefs = assignRefs(messages, { existing: state.messageRefs, nextIndex: 1, }).map; + state.pendingOrphanSweepMinRef = 1; const res = hideConsumedCompressCalls(state, messages); - assert.deepEqual(res.hiddenOrphanRefs, [ - "m00001", - "m00002", - "m00003", - "m00004", - ]); + assert.equal(res.hidden, 8); + assert.deepEqual( + res.hiddenOrphanRefs, + ["m00001", "m00002", "m00003", "m00004", "m00005", "m00006", "m00007", "m00008"], + ); + assert.deepEqual(res.newlySweptCallIds, ["c1", "c2", "c3", "c4"]); }); test("pin: active compress call keeps only block-live ranges (dead entries stripped from args)", () => { diff --git a/tests/cache-friendly-plugin.test.ts b/tests/cache-friendly-plugin.test.ts index deeea63826..7362367629 100644 --- a/tests/cache-friendly-plugin.test.ts +++ b/tests/cache-friendly-plugin.test.ts @@ -7,16 +7,18 @@ // Plugin-mode carrier semantics: the ACP-native agent OWNS compression — it // executes `compress` locally through the plugin tool API and re-sends the // call + result in its own persisted history on every later request (pi -// semantics: pairs stay at their chronological position forever). The proxy -// therefore re-injects the in-place acp_summary carrier from block state on -// every request. Since the #1567 fix the carrier is STRIPPED per-block, but -// self-verified: only while that exact fold's compress pair actually rides -// the (post-prepare) inbound history. Under fold-gated sweeps (#3769) a pair -// whose call sits at/after a committed fold's start ref is hidden from that -// fold turn on — in this harness the fold POST and its model request share -// one step, so each failed pair is swept at its own fold step, never rides, -// and its anchor carries the summary for good (zero-carrier fail-safe). This -// suite pins that behavior byte-for-byte. +// semantics: pairs stay at their chronological position forever). That pair +// IS the wire summary carrier — the agent's view never renders the kernel +// acp_summary (KDD#7). Since the #1567 fix the kernel-rendered anchor is +// STRIPPED per-block, self-verified: only while that exact fold's compress +// pair rides the (post-prepare) inbound history quoting an ACTIVE block's +// range (#3769 carrier exemption). The moment a pair stops riding (client +// drop), its anchor re-carries the summary from that turn on (#1567 +// fail-safe); a pair whose block died (swallowed by a wider fold) loses its +// carrier status and is swept at the swallowing fold's boundary, together +// with genuinely failed compress records sitting at/after the fold's start +// ref (fold-gated sweep, #3769). This suite pins that behavior +// byte-for-byte. // // Invariants pinned per wire (bodies = consecutive upstream-bound request // payloads of one session, compared as flat unit lists — text blocks and @@ -34,19 +36,22 @@ // that move); every other byte stays an exact prefix; // P2 fold transitions (client re-sends its compress pair after a // /__bili/plugin/tool fold): -// P2a the pre-fold span is replaced by exactly the expected carrier -// units and nothing else moves or mutates (exact reconstruction); -// P2b the pair is swept at its own fold step (#3769) and never rides -// inbound — occurrence counts pinned: exactly ONE summary-text hit -// per post-fold turn (the anchor carrier) and ZERO pair quotes, -// one carrier marker per committed fold (#1567 fail-safe); +// P2a the pre-fold span is replaced by exactly the expected units and +// nothing else moves or mutates (exact reconstruction; when the +// span touches m00001 the session head survives as boundary, +// mid-history spans are consumed whole); +// P2b the pair RIDES inbound from its fold turn on and IS the summary +// carrier — occurrence counts pinned: ZERO kernel anchors +// (SUM_MARKER) while any pair rides, and exactly TWO summary-text +// hits per riding pair (re-sent call args + tool-result receipt); // P2c fail-safe (#1567 hardening): the anchor carries the summary -// whenever the pair does NOT ride inbound — under #3769 that is -// every post-fold turn in this harness; the drop-pair subtest -// pins that a client-side pair drop afterwards is a wire no-op; -// P3 multi-fold: two folds in one session; each anchor carries its own -// summary from its fold turn on, the carrier count grows by one per -// fold and then stays constant (keep-two rotation is gone, #3769); +// whenever the pair does NOT ride inbound — the drop-pair subtest +// pins that a client-side pair drop surfaces exactly its anchor +// once, then stays append-stable (no flapping); +// P3 multi-fold: two folds in one session; each pair rides alongside its +// siblings as its own carrier, anchors stay stripped for the life of +// the pairs, per-fold summary counts constant, quiet turns +// append-stable (keep-two rotation is gone, #3769); // P4 no transient acp_loop_* artifact leaks into any forwarded body // (plugin mode has none by construction — pin it); // P5 marker/tag hygiene: every content text unit carries exactly one @@ -69,8 +74,8 @@ // // Scenario (main, all wires): t1=[q1,a1,q2,a2], t2=+q3,a3, t3=+q4,a4, // foldA(m00001..m00002)+pairA@t4, t5=+q5,a5, foldB(m00003..m00005)+pairB@t6, -// t7=+q7,a7, t8=+q8,a8 — each failed pair is swept at its own fold step -// (#3769); its anchor carries the summary from there on. +// t7=+q7,a7, t8=+q8,a8 — each pair rides as its fold's carrier from its +// fold turn on; anchors stay stripped while their pairs ride (#1567/#3769). // // Scope notes: tool protocol only (the default). The swallow/back-to-back // fold geometries get a dedicated anthropic subtest (kernel-side behavior, @@ -615,29 +620,32 @@ for (const wire of ["anthropic", "openai", "responses", "google"] as Wire[]) { assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1", "SA"], L(3)); // head boundary survives; anchor A carries — its pair was swept at fold time (#3769) - assertPrefix(cans[3]!.core, cans[4]!.core, L(4), ["q5", "a5"]); - assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], ["SB"], L(5)); // span consumed; anchor B carries (#3769) - assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q7", "a7"]); + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // session-head boundary survives; span consumed, no anchor — pair A rides as carrier (#1567/#3769) + // #3159 (google lane only): the riding pair's functionCall/ + // functionResponse split swallows the turn's leading text part into + // the sanctioned shift, so only the model reply lands in `added` + assertPrefix(cans[3]!.core, cans[4]!.core, L(4), wire === "google" ? ["a5"] : ["q5", "a5"], wire === "google"); + assertFoldTransition(cans[4]!.core, cans[5]!.core, ["q2", "a2", "q3"], [], L(5)); // mid-history span consumed whole; no anchor — pair B rides (#1567/#3769) + assertPrefix(cans[5]!.core, cans[6]!.core, L(6), wire === "google" ? ["a7"] : ["q7", "a7"], wire === "google"); assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); for (let i = 3; i < cans.length; i++) { const raw = cans[i]!.raw; - // #3769: each failed pair is swept at its own fold step, so no - // pair quotes ride inbound — each anchor carries its summary for - // good (zero-carrier fail-safe, #1567) - const expA = 1; - const expB = i < 5 ? 0 : 1; - assert.equal(countOcc(raw, SUM_MARKER), i < 5 ? 1 : 2, `${L(i)}: one carrier per committed fold (pairs swept, #3769)`); - assert.equal(countOcc(raw, SUM_A), expA, `${L(i)}: SUM_A occurrences ${countOcc(raw, SUM_A)} != ${expA} (anchor carrier only, #3769)`); - assert.equal(countOcc(raw, SUM_B), expB, `${L(i)}: SUM_B occurrences ${countOcc(raw, SUM_B)} != ${expB} (anchor carrier only, #3769)`); - // google part-shapes carry no call id; pin pair ABSENCE through - // the fold spec refs that would sit in the re-sent functionCall - // args (the anchor carrier embeds topic+summary, never the spec) + // Carrier regime (#1567/#3769/KDD#7): each fold's echoed pair + // rides from its fold turn on and IS the summary carrier — the + // kernel anchor stays stripped for the pair's life, and the + // summary text appears exactly twice per riding pair (call args + // + result receipt). + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while their pairs ride (#1567)`); + assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A rides pair A args+receipt (got ${countOcc(raw, SUM_A)})`); + assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 2, `${L(i)}: SUM_B rides pair B from its fold turn on (got ${countOcc(raw, SUM_B)})`); + // pin pair PRESENCE — the carriers; google part-shapes carry no + // call id, so pin through the fold-spec refs in the re-sent + // functionCall args const pairA = wire === "google" ? "\"startId\":\"m00001\"" : "\"cfp_a\""; const pairB = wire === "google" ? "\"startId\":\"m00003\"" : "\"cfp_b\""; - assert.ok(!raw.includes(pairA), `${L(i)}: swept pairA must stay hidden (#3769)`); - assert.ok(!raw.includes(pairB), `${L(i)}: swept pairB must stay hidden (#3769)`); + assert.ok(raw.includes(pairA), `${L(i)}: pair A rides as its fold's carrier (#1567)`); + assert.equal(raw.includes(pairB), i >= 5, `${L(i)}: pair B rides from t6 on`); } }, { timeout: 120_000 }); @@ -668,57 +676,60 @@ testCompat("cache-friendly plugin matrix: anthropic swallow/back-to-back geometr assertPrefix(cans[0]!.core, cans[1]!.core, L(1), ["q3", "a3"]); assertPrefix(cans[1]!.core, cans[2]!.core, L(2), ["q4", "a4"]); - assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1", "SA"], L(3)); // head boundary survives; anchor A carries (pair swept at fold time, #3769) + assertFoldTransition(cans[2]!.core, cans[3]!.core, ["q1", "a1"], ["q1"], L(3)); // session-head boundary survives; no anchor — geo_a rides as carrier (#1567/#3769) assertPrefix(cans[3]!.core, cans[4]!.core, L(4), ["q5", "a5"]); const wide = FOLD_WIDE.summary; const gamma = FOLD_E.summary; - // Fold-gated sweeps (#3769): every failed pair is hidden from its own - // fold step on, so no pair quotes ride inbound and each anchor carries - // its summary. The wide fold (m00001..m00004) swallows the alpha fold's - // range entirely — the alpha summary merges away and the wide carrier - // takes over. FOLD_E's tail sits in the protected zone, so only its - // first message folds and the gamma carrier lands where q3 used to sit. + // The wide fold (m00001..m00004) swallows the alpha fold's range: the + // alpha summary merges into the wide fold, and the geo_a pair — whose + // block died — loses carrier status and is swept at the swallow + // boundary (#3769). geo_wide rides as the merged region's carrier. + // FOLD_E's tail sits in the protected zone, so only its first message + // folds; geo_e rides alongside. for (const i of [5, 6, 7]) { const raw = cans[i]!.raw; assert.ok(!raw.includes("q2-marker"), `${L(i)}: swallowed content q2 must stay hidden`); assert.ok(!raw.includes("a2-marker"), `${L(i)}: swallowed content a2 must stay hidden`); - assert.ok(!raw.includes("\"geo_a\"") && !raw.includes("\"geo_wide\"") && !raw.includes("\"geo_e\""), `${L(i)}: all three failed pairs swept at their fold steps (#3769)`); + assert.ok(!raw.includes("\"geo_a\""), `${L(i)}: dead-block pair swept at the swallow boundary (#3769)`); } assert.equal(countOcc(cans[5]!.raw, SUM_A), 0, `${L(5)}: alpha summary absorbed into the wide fold`); - assert.equal(countOcc(cans[5]!.raw, wide), 1, `${L(5)}: wide carrier only (its pair swept, #3769)`); - assert.equal(countOcc(cans[5]!.raw, SUM_MARKER), 1, `${L(5)}: single carrier after the swallow`); - assert.equal(countOcc(cans[6]!.raw, wide), 1, `${L(6)}: wide carrier persists`); - assert.equal(countOcc(cans[6]!.raw, gamma), 1, `${L(6)}: gamma carrier only (its pair swept, #3769)`); - assert.equal(countOcc(cans[6]!.raw, SUM_MARKER), 2, `${L(6)}: one carrier per committed fold`); - assert.equal(countOcc(cans[7]!.raw, wide), 1, `${L(7)}: wide carrier persists`); - assert.equal(countOcc(cans[7]!.raw, gamma), 1, `${L(7)}: gamma carrier persists`); - assert.equal(countOcc(cans[7]!.raw, SUM_MARKER), 2, `${L(7)}: carriers stable across quiet turns`); - const surviveKeys = new Set(["q1", "a1", "q2", "a2", "SA"]); - const keepT5 = cans[4]!.core.filter((u) => !surviveKeys.has(keyOf(u))); + assert.equal(countOcc(cans[5]!.raw, wide), 2, `${L(5)}: wide summary rides geo_wide args+receipt`); + assert.equal(countOcc(cans[5]!.raw, gamma), 0, `${L(5)}: E not folded yet`); + assert.equal(countOcc(cans[5]!.raw, SUM_MARKER), 0, `${L(5)}: anchors stripped while their pairs ride`); + assert.equal(countOcc(cans[6]!.raw, wide), 2, `${L(6)}: wide carrier persists`); + assert.equal(countOcc(cans[6]!.raw, gamma), 2, `${L(6)}: gamma rides geo_e from its fold turn on`); + assert.equal(countOcc(cans[6]!.raw, SUM_MARKER), 0, `${L(6)}: anchors stripped while their pairs ride`); + assert.equal(countOcc(cans[7]!.raw, wide), 2, `${L(7)}: wide carrier persists`); + assert.equal(countOcc(cans[7]!.raw, gamma), 2, `${L(7)}: gamma carrier persists`); + assert.equal(countOcc(cans[7]!.raw, SUM_MARKER), 0, `${L(7)}: carriers stable across quiet turns`); + // t5→t6 rewrite: the wide span (minus the session-head boundary) is + // consumed and the dead-block pair swept — everything else keeps order + const goneT6 = new Set(["a1", "q2", "a2", "TU:geo_a", "TR:geo_a"]); + const keptT5 = cans[4]!.core.filter((u) => !goneT6.has(keyOf(u))); let cursor = -1; - for (const u of keepT5) { + for (const u of keptT5) { const found = cans[5]!.core.findIndex((v, j) => j > cursor && eqUnit(u, v)); assert.ok(found > cursor, `cfp-geo t6: unit ${JSON.stringify(keyOf(u))} lost or reordered by the wide fold`); cursor = found; } - // #1567/#3769: the wide summary rides its anchor carrier (a text unit) - // for good — the pair was swept at fold time, so the carrier is never - // stripped again - const wideT6 = cans[6]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); - const wideT7 = cans[7]!.core.filter((u) => u.k === "text" && u.x.includes(wide)); - assert.equal(wideT6.length, 1, `${L(6)}: wide carrier rides a text unit while the geo_wide pair stays swept`); - assert.equal(wideT7.length, 1, `${L(7)}: wide carrier rides a text unit while the geo_wide pair stays swept`); + assertFoldTransition(cans[5]!.core, cans[6]!.core, ["q3"], [], L(6)); // protected-zone carve: only E's first message folds; no anchor — geo_e rides + // #1567: while the pairs ride, their summaries live in TOOL units only — + // never a text-unit anchor + for (const idx of [5, 6, 7]) { + const textCarriers = cans[idx]!.core.filter((u) => u.k === "text" && (u.x.includes(wide) || u.x.includes(gamma))); + assert.equal(textCarriers.length, 0, `${L(idx)}: no text-unit anchor while the riding pairs carry the summaries (#1567)`); + } assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); // #1567 hardening scenario A, re-pinned under #3769: a contract-violating -// (or pruning) client stops re-sending an older compress pair at t7. Under -// fold-gated sweeps the pair was already hidden at its own fold step (t4), -// so its anchor has carried the summary since then — the client-side drop is -// a wire no-op, and the body stays append-stable with exactly one carrier -// per committed fold (no flapping). -testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → wire unaffected, carriers stable (#1567/#3769)`, async () => { +// (or pruning) client stops re-sending an older compress pair at t7. While +// the pairs ride they ARE the carriers (anchors stripped); the moment pairA +// stops riding, its anchor re-carries the summary exactly once at the +// covered span — then the body stays append-stable (no flapping). PairB +// still rides, so anchor B never surfaces. +testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pair → anchor re-carries once, no flapping (#1567/#3769)`, async () => { const steps: Step[] = [ { send: [T(1), A(1), T(2), A(2)] }, { send: [T(3), A(3)] }, @@ -737,28 +748,43 @@ testCompat(`cache-friendly plugin matrix: anthropic — client drops re-sent pai assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-drop"); - // t4..t8: both pairs swept at their fold steps → each anchor is the sole - // carrier of its summary; the client dropping pairA at t7 changes nothing - for (const i of [3, 4, 5, 6, 7]) { + // t4..t6: both pairs ride as carriers — anchors stripped + for (const i of [3, 4, 5]) { const raw = cans[i]!.raw; - assert.ok(!raw.includes('"cfp_a"') && !raw.includes('"cfp_b"'), `${L(i)}: failed pairs swept at fold time (#3769)`); - assert.equal(countOcc(raw, SUM_A), 1, `${L(i)}: anchor A carries (sole occurrence)`); - assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 1, `${L(i)}: anchor B carries from its fold turn on`); - assert.equal(countOcc(raw, SUM_MARKER), i < 5 ? 1 : 2, `${L(i)}: one carrier per committed fold`); + assert.ok(raw.includes('"cfp_a"'), `${L(i)}: pair A rides as its fold's carrier (#1567)`); + assert.ok(i >= 5 || !raw.includes('"cfp_b"') , `${L(i)}: pair B rides from t6 on`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while their pairs ride`); + assert.equal(countOcc(raw, SUM_A), 2, `${L(i)}: SUM_A rides pair A args+receipt`); + assert.equal(countOcc(raw, SUM_B), i < 5 ? 0 : 2, `${L(i)}: SUM_B rides pair B from its fold turn on`); + } + // t7: pairA dropped → its anchor re-carries exactly once at the covered + // span; pairB still rides so anchor B stays stripped (#1567 fail-safe) + { + const raw = cans[6]!.raw; + assert.ok(!raw.includes('"cfp_a"'), `${L(6)}: dropped pair stays dropped`); + assert.ok(raw.includes('"cfp_b"'), `${L(6)}: pair B unaffected by A's drop`); + assert.equal(countOcc(raw, SUM_A), 1, `${L(6)}: anchor A re-carries the sole copy`); + assert.equal(countOcc(raw, SUM_B), 2, `${L(6)}: pair B still rides`); + assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(6)}: exactly one anchor rendered (A's)`); + assert.ok(raw.includes("q7-marker"), `${L(6)}: new turn visible`); + } + // t8: once re-carried, the body stays append-stable (no flapping) + { + const raw = cans[7]!.raw; + assert.equal(countOcc(raw, SUM_A), 1, `${L(7)}: anchor A stable`); + assert.equal(countOcc(raw, SUM_MARKER), 1, `${L(7)}: no flapping after the drop`); } - assert.ok(cans[6]!.raw.includes("q7-marker"), `${L(6)}: new turn visible`); - // once re-carried, the body stays append-stable (no flapping) assertPrefix(cans[6]!.core, cans[7]!.core, L(7), ["q8", "a8"]); }, { timeout: 120_000 }); // #3769 contract pin (plugin lane): three back-to-back folds, each with a -// failed (client-visible) compress pair. The old KEEP_LAST_ORPHANED=2 -// rotation hid the oldest pair on EVERY turn — a mid-history rewrite with no -// fold behind it, exactly #3769's cache seam. Fold-gated sweeps hide each -// pair at its own fold step (the deletion rides the fold's own rewrite), so -// every anchor carries its summary from the fold turn on, carriers stack -// without churning, and quiet turns stay append-stable. -testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds: pairs swept at fold time, anchors carry (#3769)`, async () => { +// client-visible compress pair. The old KEEP_LAST_ORPHANED=2 rotation hid +// the oldest pair on EVERY turn — a mid-history rewrite with no fold behind +// it, exactly #3769's cache seam. Under the carrier regime every pair rides +// as its fold's summary carrier for good: no per-turn hiding, no churn, +// anchors stay stripped while the pairs ride, and quiet turns stay +// append-stable. +testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds: every pair rides as its carrier, quiet turns append-stable (#3769)`, async () => { const SUM_P1 = "Prune-safe fold one summary covering m00001..m00002"; const SUM_P2 = "Prune-safe fold two summary covering m00003..m00004"; const SUM_P3 = "Prune-safe fold three summary covering m00005..m00006"; @@ -782,23 +808,25 @@ testCompat(`cache-friendly plugin matrix: anthropic — 3 concurrent folds: pair assert.equal(countOcc(cans[i]!.raw, "acp_loop_"), 0, `${L(i)}: acp_loop_ artifact leaked`); } assertTagPolicy(cans, "cfp-prune"); - // t3..t5: each fold's pair is swept at its own fold step → each anchor - // carries its summary; carrier count grows by one per fold + // t3..t5: each fold's pair rides from its fold turn on (args+receipt = + // 2 occurrences); anchors stay stripped for the life of the pairs for (const i of [2, 3, 4]) { const raw = cans[i]!.raw; - assert.ok(!raw.includes('"fs1"') && !raw.includes('"fs2"') && !raw.includes('"fs3"'), `${L(i)}: failed pairs swept at their fold steps (#3769)`); - assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: anchor P1 carries (sole occurrence)`); - assert.equal(countOcc(raw, SUM_P2), i < 3 ? 0 : 1, `${L(i)}: anchor P2 carries from its fold turn on`); - assert.equal(countOcc(raw, SUM_P3), i < 4 ? 0 : 1, `${L(i)}: anchor P3 carries from its fold turn on`); - assert.equal(countOcc(raw, SUM_MARKER), i - 1, `${L(i)}: one carrier per committed fold`); + assert.ok(raw.includes('"fs1"'), `${L(i)}: pair 1 rides as its fold's carrier (#1567)`); + assert.ok(i >= 3 || !raw.includes('"fs2"'), `${L(i)}: pair 2 rides from t4 on`); + assert.ok(i >= 4 || !raw.includes('"fs3"'), `${L(i)}: pair 3 rides from t5 on`); + assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: P1 rides pair-1 args+receipt`); + assert.equal(countOcc(raw, SUM_P2), i < 3 ? 0 : 2, `${L(i)}: P2 rides from its fold turn on`); + assert.equal(countOcc(raw, SUM_P3), i < 4 ? 0 : 2, `${L(i)}: P3 rides from its fold turn on`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: anchors stripped while the pairs ride`); } - // t6/t7: all three carriers persist, stable across quiet turns + // t6/t7: all three pairs persist, stable across quiet turns for (const i of [5, 6]) { const raw = cans[i]!.raw; - assert.equal(countOcc(raw, SUM_P1), 1, `${L(i)}: P1 carrier persists`); - assert.equal(countOcc(raw, SUM_P2), 1, `${L(i)}: P2 carrier persists`); - assert.equal(countOcc(raw, SUM_P3), 1, `${L(i)}: P3 carrier persists`); - assert.equal(countOcc(raw, SUM_MARKER), 3, `${L(i)}: three carriers, stable`); + assert.equal(countOcc(raw, SUM_P1), 2, `${L(i)}: P1 carrier persists`); + assert.equal(countOcc(raw, SUM_P2), 2, `${L(i)}: P2 carrier persists`); + assert.equal(countOcc(raw, SUM_P3), 2, `${L(i)}: P3 carrier persists`); + assert.equal(countOcc(raw, SUM_MARKER), 0, `${L(i)}: three riding pairs, zero anchors, stable`); } // mid-history never mutates between quiet turns assertPrefix(cans[5]!.core, cans[6]!.core, L(6), ["q8", "a8"]); diff --git a/tests/issue3769-upgrade-compat.test.ts b/tests/issue3769-upgrade-compat.test.ts index 289528f6cd..3b68e542ae 100644 --- a/tests/issue3769-upgrade-compat.test.ts +++ b/tests/issue3769-upgrade-compat.test.ts @@ -7,7 +7,7 @@ import assert from "node:assert/strict"; import http from "node:http"; import { once } from "node:events"; import test from "node:test"; -import { mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readdirSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -103,6 +103,15 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e const upPort = (upstream.address() as { port: number }).port; const stores: SessionStore[] = []; + const proxies: http.Server[] = []; + const closedProxies = new Set(); + async function closeProxy(p: http.Server): Promise { + if (closedProxies.has(p)) return; + closedProxies.add(p); + p.close(); + p.closeIdleConnections?.(); + await once(p, "close").catch(() => undefined); + } try { // Phase 1: current code runs two turns; the failed pair rides visibly // (fold-gated contract: no fold -> no sweep) and the record lands on disk. @@ -132,6 +141,7 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e mitm: { enabled: false, domains: [] }, } as unknown as ProxyOptions); await once(proxy1, "listening"); + proxies.push(proxy1); const port1 = (proxy1.address() as { port: number }).port; const url1 = `http://127.0.0.1:${port1}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; @@ -143,12 +153,20 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e const s1 = listSessions().find((x) => x.meta.label === LABEL); assert.ok(s1, "session exists"); assert.ok(store1.flushSync(s1!)); - proxy1.close(); - proxy1.closeIdleConnections?.(); - await once(proxy1, "close"); + await closeProxy(proxy1); + + // Phase 2: a fresh store hydrates the record, then shapes it into + // what a pre-fold-gate kernel left behind — rotation had hidden the + // pair (remembered by ref), new sweep fields absent. Refs are read + // off the dual-written envelope; the mutation itself goes THROUGH THE + // STORE: since the #3818 full-mode default the index row IS the load + // path, so patching only the envelope would leave a stale row the + // loader serves instead. + _resetSessionsForTest(); + const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); + stores.push(store2); + _setStoreForTest(store2); - // Patch the persisted record into the exact shape an old kernel left: - // rotation had hidden the pair (remembered by ref), new fields absent. const file = findSessionFile(dir, LABEL); const rec = JSON.parse(readFileSync(file, "utf8")) as PersistRecord; const callMsg = rec.payload.messages.find((m) => m.contentType === "tool-call" && m.toolName === "compress"); @@ -157,16 +175,12 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e const callRef = rec.payload.state.messageRefs.byRaw[callMsg!.id]; const resRef = rec.payload.state.messageRefs.byRaw[resMsg!.id]; assert.ok(callRef && resRef, "refs resolvable from the persisted state"); - rec.payload.state.hiddenOrphanRefs = [callRef, resRef]; - delete rec.payload.state.sweptOrphanCallIds; - delete rec.payload.state.pendingOrphanSweepMinRef; - writeFileSync(file, JSON.stringify(rec)); - - // Phase 2: a fresh server instance hydrates the patched record. - _resetSessionsForTest(); - const store2 = new SessionStore({ enabled: true, dir, debounceMs: 0 }); - stores.push(store2); - _setStoreForTest(store2); + const sPre = store2.loadSync(LABEL); + assert.ok(sPre, "record hydrates before the upgrade turn"); + sPre.state.hiddenOrphanRefs = [callRef, resRef]; + delete sPre.state.sweptOrphanCallIds; + delete sPre.state.pendingOrphanSweepMinRef; + assert.ok(store2.flushSync(sPre), "patched record commits through the store"); const proxy2 = await startServer({ port: 0, host: "127.0.0.1", @@ -188,6 +202,7 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e mitm: { enabled: false, domains: [] }, } as unknown as ProxyOptions); await once(proxy2, "listening"); + proxies.push(proxy2); const port2 = (proxy2.address() as { port: number }).port; const url2 = `http://127.0.0.1:${port2}/bili/http://127.0.0.1:${upPort}/v1/chat/completions`; @@ -201,11 +216,10 @@ test("upgraded pre-fold-gate state keeps rotation-hidden orphans hidden end-to-e const s2 = listSessions().find((x) => x.meta.label === LABEL); assert.ok(s2, "hydrated session exists"); assert.ok(s2.state.hiddenOrphanRefs?.includes(callRef!), "legacy hidden set survived hydration and self-perpetuates"); - proxy2.close(); - proxy2.closeIdleConnections?.(); - await once(proxy2, "close"); + await closeProxy(proxy2); } finally { for (const s of stores) s.closeQuietly(); + for (const p of proxies) await closeProxy(p); upstream.close(); upstream.closeIdleConnections?.(); await once(upstream, "close").catch(() => undefined); From b9bae89901a841194c74c5b45a26a8fcf4c277fa Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:19:54 +0800 Subject: [PATCH 19/27] =?UTF-8?q?kernel:=20latch=20emergency=20truncation?= =?UTF-8?q?=20bytes=20until=20the=20next=20fold=20(#3939=20=E2=91=A2,=20#3?= =?UTF-8?q?691)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watermark decision is made once: replaying the latched clip on every later turn keeps outbound bytes stable when the token estimate fluctuates across the threshold. Refs are content-bound so edited messages re-judge; entries retire when their message leaves the view; a successful fold resets the latch (the clipped originals ride the fold away). Carry surfaces: sync node, persist mergeState, fork-adoption, plugin fork prefix-filter, storage ingest. --- kernel/package.json | 2 +- kernel/src/compress.ts | 87 ++++++++++- kernel/src/sync.ts | 6 + kernel/src/truncate-tools.ts | 8 +- kernel/src/types.ts | 18 +++ kernel/tests/issue3939-truncate-latch.test.ts | 139 ++++++++++++++++++ src/fork-adoption.ts | 6 + src/persist.ts | 4 + src/plugin.ts | 4 + src/storage/ingest.ts | 1 + 10 files changed, 266 insertions(+), 9 deletions(-) create mode 100644 kernel/tests/issue3939-truncate-latch.test.ts diff --git a/kernel/package.json b/kernel/package.json index 8ac891d931..d5f0ba0d08 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.138", + "version": "0.0.139", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 8ca64f32f0..7d39230a5a 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -19,6 +19,7 @@ import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; import { applyCrushToMessages } from "./crush.js"; import { applyRetrieve, ccrStoreNode, RETRIEVED_ID_PREFIX } from "./ccr.js"; import type { ApplyRetrieveResult, CcrEffect } from "./ccr.js"; +import type { EmergencyTruncateLatchEntry } from "./types.js"; import { createContentStore } from "./content-store.js"; import type { MessageContentStore } from "./content-store.js"; import { applyMessageFilters, listMessageFilters } from "./filter/index.js"; @@ -145,6 +146,11 @@ type RangeResolution = | { status: "unknown"; error: BoundaryNotFoundError } | { status: "invalid"; error: Error }; +/** #3939 ③: bound on persisted emergency-truncate latch entries. Each entry + * carries a clipped text (≤ keepPrefix+marker+keepSuffix ≈ 4KiB); a fold + * clears the latch, so normal sessions hold a handful at most. Oldest drop. */ +const MAX_TRUNCATE_LATCH = 64; + function rangeError( spec: { startRef: string; endRef: string }, message: string, @@ -797,6 +803,11 @@ export function createCore(ports: Ports = {}): CompressionCore { // the terminal-floor streak so the escape signal can only fire again // after N fresh stuck events (#300). state.terminalStreak = 0; + // #3939 ③/#3691: the fold reset the truncation landscape — clipped + // originals under the fold are gone from the view and the watermark + // re-evaluates from the new (lower) count, so the latch retires with it. + // Holding it would replay stale clips onto a view the fold rewrote. + state.emergencyTruncateLatch = undefined; } return { @@ -1281,14 +1292,53 @@ const emergencyTruncateNode: PipelineNode = { ? ctx.tokenCount / ctx.config.modelContextLimit : 0; const prevStreak = io.state.terminalStreak ?? 0; + // #3939 ③/#3691: replay latched clips BEFORE the watermark check. The + // decision to clip was made once at the watermark; re-evaluating it every + // turn against a fluctuating estimate flips the same bytes full↔clipped — + // a mid-history rewrite on every request (prefix-cache breaker). Refs are + // content-bound, so an edited message misses its entry and re-judges from + // scratch; entries whose messages left the view retire; the whole latch + // resets on the next successful fold (applyCompression). + const latch = io.state.emergencyTruncateLatch ?? []; + const liveRefs = new Set(); + for (const m of io.messages) { + const ref = io.state.messageRefs.byRaw[m.id]; + if (ref !== undefined) liveRefs.add(ref); + } + const latchLive = latch.filter((e) => liveRefs.has(e.ref)); + let messages = io.messages; + let replaySaved = 0; + if (latch.length > 0) { + const byRef = new Map(latch.map((e) => [e.ref, e.text])); + messages = io.messages.map((m) => { + const ref = io.state.messageRefs.byRaw[m.id]; + const clipped = ref !== undefined ? byRef.get(ref) : undefined; + if (clipped === undefined || clipped === m.text) return m; + replaySaved += Math.max( + 0, + ctx.countTokens(m.text ?? "") - ctx.countTokens(clipped), + ); + return { ...m, text: clipped }; + }); + } if (usage < ctx.config.truncate.threshold) { - return prevStreak > 0 - ? { ...io, state: { ...io.state, terminalStreak: 0 } } - : io; + if (latch.length === 0 && prevStreak === 0) return io; + return { + ...io, + messages, + state: { + ...io.state, + terminalStreak: 0, + emergencyTruncateLatch: + latchLive.length > 0 ? latchLive : undefined, + }, + }; } const trunc = truncateLargeToolOutputs( - io.messages, - ctx.tokenCount, + messages, + // Account for replayed clips: they already shrank the real request; + // charging their tokens again would over-truncate fresh candidates. + Math.max(0, ctx.tokenCount - replaySaved), ctx.config, ctx.countTokens, { @@ -1333,10 +1383,35 @@ const emergencyTruncateNode: PipelineNode = { stuckEvents: streak, }; } + // #3939 ③: fold this pass's clips into the latch (replacing any same-ref + // entry) so later turns replay them byte-identically. Bounded so a long + // session cannot grow the persisted state without limit; the oldest + // entries drop first. + const clips: EmergencyTruncateLatchEntry[] = []; + for (const id of trunc.truncatedIds ?? []) { + const ref = io.state.messageRefs.byRaw[id]; + const clippedMsg = trunc.messages.find((m) => m.id === id); + if (ref !== undefined && clippedMsg?.text !== undefined) { + clips.push({ ref, text: clippedMsg.text }); + } + } + const clipRefs = new Set(clips.map((c) => c.ref)); + const nextLatch = [ + ...latchLive.filter((e) => !clipRefs.has(e.ref)), + ...clips, + ]; + if (nextLatch.length > MAX_TRUNCATE_LATCH) { + nextLatch.splice(0, nextLatch.length - MAX_TRUNCATE_LATCH); + } return { ...io, messages: trunc.messages, - state: { ...io.state, terminalStreak: streak }, + state: { + ...io.state, + terminalStreak: streak, + emergencyTruncateLatch: + nextLatch.length > 0 ? nextLatch : undefined, + }, effects, }; }, diff --git a/kernel/src/sync.ts b/kernel/src/sync.ts index 9a12073f3e..b9908863c6 100644 --- a/kernel/src/sync.ts +++ b/kernel/src/sync.ts @@ -53,6 +53,12 @@ export function syncBlocks( hiddenOrphanRefs: state.hiddenOrphanRefs ? [...state.hiddenOrphanRefs] : undefined, sweptOrphanCallIds: state.sweptOrphanCallIds ? [...state.sweptOrphanCallIds] : undefined, pendingOrphanSweepMinRef: state.pendingOrphanSweepMinRef, + // #3939 ③: the truncation latch must survive the sync node — dropping it + // here would re-expose clipped bytes mid-history after a sync carried the + // rest of the state (#3691 oscillation via the sync path). + emergencyTruncateLatch: state.emergencyTruncateLatch + ? state.emergencyTruncateLatch.map((e) => ({ ref: e.ref, text: e.text })) + : undefined, }; // Refs are additive (assignRefs never removes them from messageRefs), so diff --git a/kernel/src/truncate-tools.ts b/kernel/src/truncate-tools.ts index 964bfc1b0d..19083bdc00 100644 --- a/kernel/src/truncate-tools.ts +++ b/kernel/src/truncate-tools.ts @@ -21,6 +21,9 @@ export interface TruncateResult { /** Size-qualified candidates found (pre-truncation). 0 explains a silent * no-op at/above threshold — hosts should warn-log (#300). */ candidatesFound: number; + /** Ids of the messages actually clipped this pass (#3939 ③ latch). Empty + * when truncatedCount is 0. */ + truncatedIds?: string[]; } const TRUNCATION_MARKER = "[truncated for context space]"; @@ -42,7 +45,7 @@ export function truncateLargeToolOutputs( const opts = { ...DEFAULTS, ...options }; const limit = config.modelContextLimit; if (limit <= 0 || tokenCount < config.truncate.threshold * limit) { - return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound: 0 }; + return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound: 0, truncatedIds: [] }; } const protectedIndex = messages.length - opts.protectRecentMessages; @@ -114,7 +117,7 @@ export function truncateLargeToolOutputs( applyTo(textMessages); if (replacements.size === 0) { - return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound }; + return { messages, truncatedCount: 0, savedTokens: 0, candidatesFound, truncatedIds: [] }; } return { messages: messages.map((m) => @@ -123,5 +126,6 @@ export function truncateLargeToolOutputs( truncatedCount, savedTokens, candidatesFound, + truncatedIds: [...replacements.keys()], }; } diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 91f98dd40b..17cfded00d 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -36,6 +36,13 @@ export type CompressionTier = 1 | 2 | 3; export type BlockGeneration = "young" | "old"; +/** One #3939 ③ emergency-truncation latch entry: the post-clip text of a + * message (keyed by its padded ref). See CompressionState.emergencyTruncateLatch. */ +export interface EmergencyTruncateLatchEntry { + ref: string; + text: string; +} + export interface CompressionBlock { blockId: string; runId: string; @@ -262,6 +269,17 @@ export interface CompressionState { * Carries the position so the sweep is cache-neutral: it only removes records * the fold already invalidated. Optional: absent until the first post-fix fold. */ pendingOrphanSweepMinRef?: number; + /** Emergency-truncation latch (#3939 ③/#3691): the clipped texts (post-clip + * bytes, keyed by padded ref) of messages clipped by the emergency-truncate + * node. The watermark decision is made ONCE — replaying these clips on every + * later turn keeps the outbound bytes stable when the token estimate + * fluctuates across the threshold (the #3691 full↔clipped oscillation was a + * mid-history rewrite on every request). Entries retire when their message + * leaves the resent view (folded/edited-away) and the whole latch resets on + * the next successful fold (applyCompression) — after a fold the watermark + * re-evaluates from scratch. Optional: pre-feature states have none (the + * pre-latch node clipped from scratch each turn). */ + emergencyTruncateLatch?: EmergencyTruncateLatchEntry[]; /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk diff --git a/kernel/tests/issue3939-truncate-latch.test.ts b/kernel/tests/issue3939-truncate-latch.test.ts new file mode 100644 index 0000000000..bd44ee9b15 --- /dev/null +++ b/kernel/tests/issue3939-truncate-latch.test.ts @@ -0,0 +1,139 @@ +// #3939 ③ / #3691: emergency truncation is a one-shot decision that must stay +// byte-stable across turns. The pre-latch node re-evaluated the watermark on +// every turn against the resent FULL history, so a fluctuating token estimate +// (95%+ one turn, below the next) flipped the same user text between its full +// and clipped forms every turn — a mid-history rewrite on every request and a +// prefix-cache breaker. The latch pins the clipped bytes until a fold resets +// them (#3939 contract: 水位触发 clip 后保持 clip 直到折叠事件重置). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore, defaultConfig } from "../src/index.js"; +import { createInitialState } from "../src/state.js"; +import type { CoreMessage } from "../src/types.js"; + +const BIG = "alpha-beta ".repeat(4000); // ~48k chars, well past prefix+suffix +// Roomy filler so a fold spanning the whole history clears the 5000-char +// min-range gate even after the big text is clipped (~4.1K). +const filler = (t: string) => `filler ${t} ` + "x".repeat(800); +const tiny = (t: string): CoreMessage[] => [ + { id: `raw-${t}`, role: "user", contentType: "text", text: "hi " + t + " " + filler(t) }, + { id: `raw-${t}-a`, role: "assistant", contentType: "text", text: "ok " + t }, +]; + +function hist(): CoreMessage[] { + return [ + ...tiny("warm"), + { id: "raw-big", role: "user", contentType: "text", text: BIG }, + ...tiny("tail1"), + ...tiny("tail2"), + ...tiny("tail3"), + ...tiny("tail4"), + ]; +} + +const WINDOW = 100_000; +// Narrow the recent-token protection so a unit-scale history still has a +// foldable head (default preserveRecentTokens=5000 swallows this whole fixture +// and every range dies in the protected zone). The latch logic under test is +// orthogonal to the protected-zone math. +const cfg = () => ({ ...defaultConfig(WINDOW), preserveRecentTokens: 100 }); + +test("pin: a clip fires at 96% and the latch replays it on the next under-watermark turn (#3691)", () => { + const core = createCore(defaultConfig(WINDOW)); + const messages = hist(); + + const t1 = core.processTurn({ + messages, + state: createInitialState(), + config: defaultConfig(WINDOW), + tokenCount: 96_000, + renderTags: "none", + }); + const clipped1 = t1.messages.find((m) => m.id === "raw-big")!.text!; + assert.ok(clipped1.includes("[truncated for context space]"), "turn 1 clips the oversized text"); + assert.ok(t1.state.emergencyTruncateLatch?.some((e) => e.text === clipped1), "clip is latched"); + + // Turn 2: SAME full history resent, estimate now well under the watermark — + // pre-latch this returned the full text (the #3691 oscillation). + const t2 = core.processTurn({ + messages: hist(), + state: t1.state, + config: defaultConfig(WINDOW), + tokenCount: 60_000, + renderTags: "none", + }); + const clipped2 = t2.messages.find((m) => m.id === "raw-big")!.text!; + assert.equal(clipped2, clipped1, "latch replays the byte-identical clip under the watermark"); + assert.ok(t2.state.emergencyTruncateLatch?.some((e) => e.text === clipped1), "latch self-perpetuates while the message stays live"); +}); + +test("pin: a successful fold resets the latch and the clip retires with the fold", () => { + const core = createCore(cfg()); + const t1 = core.processTurn({ + messages: hist(), + state: createInitialState(), + config: cfg(), + tokenCount: 96_000, + renderTags: "none", + }); + assert.ok(t1.state.emergencyTruncateLatch?.length, "clip latched"); + + const refOf = (st: typeof t1.state, id: string) => st.messageRefs.byRaw[id]; + const startRef = refOf(t1.state, "raw-warm"); + const endRef = refOf(t1.state, "raw-tail1-a"); + assert.ok(startRef && endRef, "refs assigned"); + + const fold = core.applyCompression({ + messages: t1.messages, + state: t1.state, + config: cfg(), + ranges: [{ startRef, endRef, summary: "a long enough summary for the fold gate to accept this range, over fifty chars", topic: "t" }], + }); + assert.equal(fold.result.blocksCreated, 1, "fold lands"); + assert.equal(fold.state.emergencyTruncateLatch, undefined, "fold resets the latch"); + + // Post-fold: a NEW oversized message under the watermark must NOT be + // clipped — the latch retired with the fold and the watermark re-evaluates + // from scratch (the original raw-big left the view inside the fold). + const post = [ + ...hist().filter((m) => m.id !== "raw-big"), + { id: "raw-big2", role: "user" as const, contentType: "text" as const, text: BIG }, + ]; + const t3 = core.processTurn({ + messages: post, + state: fold.state, + config: cfg(), + tokenCount: 60_000, + renderTags: "none", + }); + const big2 = t3.messages.find((m) => m.id === "raw-big2"); + assert.ok(big2, "new oversized message in view"); + assert.equal(big2!.text, BIG, "no clip after the fold reset (under watermark)"); + assert.equal(t3.state.emergencyTruncateLatch, undefined, "latch stays retired"); +}); + +test("pin: an edited message (new content id) is not replay-clipped from a stale latch entry", () => { + const core = createCore(defaultConfig(WINDOW)); + const t1 = core.processTurn({ + messages: hist(), + state: createInitialState(), + config: defaultConfig(WINDOW), + tokenCount: 96_000, + renderTags: "none", + }); + assert.ok(t1.state.emergencyTruncateLatch?.length, "clip latched"); + + const edited = hist(); + const bigIdx = edited.findIndex((m) => m.id === "raw-big"); + edited[bigIdx] = { ...edited[bigIdx]!, id: "raw-big-v2", text: "completely rewritten, short" }; + + const t2 = core.processTurn({ + messages: edited, + state: t1.state, + config: defaultConfig(WINDOW), + tokenCount: 60_000, + renderTags: "none", + }); + assert.equal(t2.messages.find((m) => m.id === "raw-big-v2")!.text, "completely rewritten, short"); + assert.equal(t2.state.emergencyTruncateLatch?.length ?? 0, 0, "stale entry retired with its message"); +}); diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 16222d9a61..22ddf63118 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -270,6 +270,12 @@ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, pare // every turn, so the carry self-perpetuates afterwards. if (parent.state.hiddenOrphanRefs?.length) session.state.hiddenOrphanRefs = [...parent.state.hiddenOrphanRefs]; if (parent.state.sweptOrphanCallIds?.length) session.state.sweptOrphanCallIds = [...parent.state.sweptOrphanCallIds]; + // #3939 ③: same carry for the truncation latch — resume/fork replay + // re-sends the clipped messages; without the latch the first replayed + // turn under a lower estimate re-exposes the full bytes (oscillation). + if (parent.state.emergencyTruncateLatch?.length) { + session.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch.map((e) => ({ ...e })); + } markDirty(session); } diff --git a/src/persist.ts b/src/persist.ts index 9652ce41c8..6ec42bbae8 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -244,6 +244,10 @@ function mergeState(parsed: CompressionState): CompressionState { // transient, losing it only defers the sweep to the next fold. sweptOrphanCallIds: parsed.sweptOrphanCallIds ?? fresh.sweptOrphanCallIds, pendingOrphanSweepMinRef: parsed.pendingOrphanSweepMinRef ?? fresh.pendingOrphanSweepMinRef, + // #3939 ③/#3691: the emergency-truncate latch must survive hydration — + // dropping it re-exposes clipped bytes mid-history after a restart, + // rewriting the still-cached prefix on the first post-restart turn. + emergencyTruncateLatch: parsed.emergencyTruncateLatch ?? fresh.emergencyTruncateLatch, }; } diff --git a/src/plugin.ts b/src/plugin.ts index 404624285c..3d8b4818ef 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1389,6 +1389,10 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr child.state.rules = structuredClone(parent.state.rules); child.state.nextRuleId = parent.state.nextRuleId; child.state.hiddenOrphanRefs = parent.state.hiddenOrphanRefs?.filter((ref) => prefix.some((m) => m.ref === ref)); + // #3939 ③: only latch entries whose messages rode the forked + // prefix — the child never sees the others, so their entries + // would be dead weight (and retire on the next latch pass anyway). + child.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch?.filter((e) => prefix.some((m) => m.ref === e.ref)); // #2362: inverse of the orphan filter — a dead ref whose message IS // in the forked prefix is visible again in the child; refs outside // it stay dead (their messages are absent from the child too). diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 9b27270502..64990619b2 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -42,6 +42,7 @@ const MERGE_STATE_FIELDS = [ "tokenSnapshot", "lastPassIds", "hiddenOrphanRefs", + "emergencyTruncateLatch", "deadRefs", "terminalStreak", "nextRuleId", From 88ecbbf4db1d23f94a29a0f61183c69b3df40423 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:29:48 +0800 Subject: [PATCH 20/27] =?UTF-8?q?kernel:=20defer=20BLOCKED=E2=86=92numeric?= =?UTF-8?q?=20migration=20carrier=20tags=20to=20the=20fold=20boundary=20(#?= =?UTF-8?q?3939=20=E2=91=A1,=20#3805)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The migration (#2607) mints the fresh numeric ref immediately (citable for decompress bookkeeping) but the carrier message keeps its pre-migration, UNTAGGED bytes until the next successful fold — stamping the tag on the migration turn rewrote an old message mid-history on an otherwise quiet turn (the #3805 durable-carrier cache breaker). Fold success releases the deferral; carried across sync/persist/fork/plugin/ingest. Also fixes the golden regen script missing the BILI_SNAPSHOT_SHADOW=0 parity pin (regenerated goldens red-flipped on the next CI run without it); storage-render goldens now pin the two new undefined-valued state keys, wire bodies unchanged. --- kernel/package.json | 2 +- kernel/src/compress.ts | 24 ++++- kernel/src/refs.ts | 12 ++- kernel/src/render-refs.ts | 16 +++- kernel/src/sync.ts | 5 ++ kernel/src/types.ts | 11 +++ .../issue3939-ref-migration-defer.test.ts | 89 +++++++++++++++++++ scripts/update-storage-render-goldens.ts | 5 ++ src/fork-adoption.ts | 6 ++ src/persist.ts | 4 + src/plugin.ts | 3 + src/storage/ingest.ts | 1 + tests/golden/storage-render/anthropic.json | 4 +- tests/golden/storage-render/chat.json | 4 +- tests/golden/storage-render/google.json | 4 +- tests/golden/storage-render/responses.json | 4 +- 16 files changed, 182 insertions(+), 12 deletions(-) create mode 100644 kernel/tests/issue3939-ref-migration-defer.test.ts diff --git a/kernel/package.json b/kernel/package.json index d5f0ba0d08..96381e368e 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.139", + "version": "0.0.140", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 7d39230a5a..9bb16ae475 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -1,4 +1,4 @@ -import { assignRefs, highestUsedIndex, indexToRef } from "./refs.js"; +import { assignRefs, BLOCKED_REF, highestUsedIndex, indexToRef } from "./refs.js"; import { remintCoveredLiveIds } from "./instance-reid.js"; import { prune } from "./prune.js"; import { syncBlocks } from "./sync.js"; @@ -808,6 +808,10 @@ export function createCore(ports: Ports = {}): CompressionCore { // re-evaluates from the new (lower) count, so the latch retires with it. // Holding it would replay stale clips onto a view the fold rewrote. state.emergencyTruncateLatch = undefined; + // #3939 ②/#3805: the fold rewrote history — the migration deferral + // rides it. Any surviving carrier may render its tag from here on; the + // rewrite is attributed to the fold, not to a quiet turn. + state.renderDeferredRefs = undefined; } return { @@ -1021,7 +1025,23 @@ const assignRefsNode: PipelineNode = { // Ephemeral retrieval injections never consume a ref slot. shouldSkip: (m) => m.id.startsWith(RETRIEVED_ID_PREFIX), }); - return { ...io, state: { ...io.state, messageRefs: refResult.map } }; + // #3939 ②/#3805: a BLOCKED→numeric migration mints the ref NOW but the + // carrier's tag render is DEFERRED to the next fold boundary — + // stamping it this turn would rewrite an old message mid-history on an + // otherwise quiet turn (prefix-cache breaker). Cleared on fold success. + const migratedRefs = refResult.migratedIds + .map((id) => refResult.map.byRaw[id]) + .filter((r): r is string => typeof r === "string" && r !== BLOCKED_REF); + const state = migratedRefs.length + ? { + ...io.state, + messageRefs: refResult.map, + renderDeferredRefs: [ + ...new Set([...(io.state.renderDeferredRefs ?? []), ...migratedRefs]), + ], + } + : { ...io.state, messageRefs: refResult.map }; + return { ...io, state }; }, }; diff --git a/kernel/src/refs.ts b/kernel/src/refs.ts index 106bcdde13..21606cda23 100644 --- a/kernel/src/refs.ts +++ b/kernel/src/refs.ts @@ -63,6 +63,9 @@ export interface AssignRefsResult { map: MessageRefMap; nextIndex: number; newlyAssigned: number; + /** Ids whose stale BLOCKED entry migrated to a fresh numeric ref this pass + * (#2607). The caller defers their carrier re-render (#3939 ②/#3805). */ + migratedIds: string[]; } export interface AssignRefsOptions { @@ -85,6 +88,7 @@ export function assignRefs( ? options.nextIndex : MIN_INDEX; let newlyAssigned = 0; + const migratedIds: string[] = []; for (const message of messages) { if (!message.id || options.shouldSkip?.(message)) continue; @@ -96,8 +100,10 @@ export function assignRefs( // is lifted (#2607: #1188 pinned all media; archivable images now fold // and must be citable via decompress({ imageRef })). Fresh number only — // the id-never-reused contract holds. - if (prev === BLOCKED_REF && !protectedNow) delete map.byRaw[message.id]; - else continue; + if (prev === BLOCKED_REF && !protectedNow) { + delete map.byRaw[message.id]; + migratedIds.push(message.id); + } else continue; } if (protectedNow) { @@ -112,7 +118,7 @@ export function assignRefs( newlyAssigned++; } - return { map, nextIndex: cursor, newlyAssigned }; + return { map, nextIndex: cursor, newlyAssigned, migratedIds }; } function allocateFreeRef( diff --git a/kernel/src/render-refs.ts b/kernel/src/render-refs.ts index 7338c4632f..6590d25843 100644 --- a/kernel/src/render-refs.ts +++ b/kernel/src/render-refs.ts @@ -62,10 +62,16 @@ function renderMessage( countTokens: (text: string) => number, strategy: RenderStrategy, snapshot: Record | null = null, + deferred: ReadonlySet | null = null, ): CoreMessage { const ref = refForRaw(map, message.id); if (!ref || ref === BLOCKED_REF) return message; + // #3939 ②/#3805: refs minted by the BLOCKED→numeric migration keep their + // pre-migration UNTAGGED bytes until the next fold boundary — stamping the + // tag now would rewrite an old message mid-history on a quiet turn. + if (deferred?.has(ref)) return message; + // "none": host reads the ref map directly — never pollute text. if (strategy === "none") return message; @@ -110,8 +116,11 @@ export function renderVisibleRefs( ): CoreMessage[] { // Legacy behavior: recompute tokens every render (snapshot = null). const map = state.messageRefs; + const deferred = state.renderDeferredRefs + ? new Set(state.renderDeferredRefs) + : null; return messages.map((message) => - renderMessage(message, map, countTokens, strategy), + renderMessage(message, map, countTokens, strategy, null, deferred), ); } @@ -132,8 +141,11 @@ export function renderWithSnapshot( ): RenderWithSnapshotResult { const map = state.messageRefs; const snapshot = { ...(state.tokenSnapshot ?? {}) }; + const deferred = state.renderDeferredRefs + ? new Set(state.renderDeferredRefs) + : null; const rendered = messages.map((message) => - renderMessage(message, map, countTokens, strategy, snapshot), + renderMessage(message, map, countTokens, strategy, snapshot, deferred), ); return { messages: rendered, tokenSnapshot: snapshot }; } diff --git a/kernel/src/sync.ts b/kernel/src/sync.ts index b9908863c6..578cab7e14 100644 --- a/kernel/src/sync.ts +++ b/kernel/src/sync.ts @@ -59,6 +59,11 @@ export function syncBlocks( emergencyTruncateLatch: state.emergencyTruncateLatch ? state.emergencyTruncateLatch.map((e) => ({ ref: e.ref, text: e.text })) : undefined, + // #3939 ②: the render deferral must survive sync — losing it would stamp + // migration tags mid-history after a sync carried the ref map (#3805). + renderDeferredRefs: state.renderDeferredRefs + ? [...state.renderDeferredRefs] + : undefined, }; // Refs are additive (assignRefs never removes them from messageRefs), so diff --git a/kernel/src/types.ts b/kernel/src/types.ts index 17cfded00d..ccb3a7aded 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -280,6 +280,17 @@ export interface CompressionState { * re-evaluates from scratch. Optional: pre-feature states have none (the * pre-latch node clipped from scratch each turn). */ emergencyTruncateLatch?: EmergencyTruncateLatchEntry[]; + /** Ref-migration render deferral (#3939 ②/#3805): padded refs minted by the + * BLOCKED→numeric migration (assignRefs, #2607). The number exists (citable + * via decompress({ imageRef }) bookkeeping) but the carrier message keeps + * its pre-migration, UNTAGGED bytes until the next successful fold — + * stamping the tag on the migration turn would rewrite an old + * message mid-history on an otherwise quiet turn (a prefix-cache breaker: + * #3805 durable carriers flipping bytes with no fold in sight). Cleared by + * applyCompression on fold success; entries are inert once their message + * leaves the view (ids are never re-issued). Optional: absent until the + * first migration. */ + renderDeferredRefs?: string[]; /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk diff --git a/kernel/tests/issue3939-ref-migration-defer.test.ts b/kernel/tests/issue3939-ref-migration-defer.test.ts new file mode 100644 index 0000000000..c6a3bfb7c9 --- /dev/null +++ b/kernel/tests/issue3939-ref-migration-defer.test.ts @@ -0,0 +1,89 @@ +// #3939 ② / #3805: when a BLOCKED ref migrates to a fresh numeric ref +// (protection lifted, #2607), the NUMBER is minted immediately but the +// tag on the old carrier must NOT appear until a fold boundary rewrites +// history anyway. Stamping it on the migration turn silently rewrites an old +// message mid-history on an otherwise quiet turn — a prefix-cache breaker +// (#3805: durable carriers flipping bytes with no fold in sight). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore, defaultConfig } from "../src/index.js"; +import { createInitialState } from "../src/state.js"; +import { BLOCKED_REF } from "../src/refs.js"; +import type { CoreMessage } from "../src/types.js"; + +const W = 100_000; +const cfg = () => ({ ...defaultConfig(W), preserveRecentTokens: 100 }); + +function hist(): CoreMessage[] { + const filler = "filler ".repeat(340); // push the range over the 5000-char min-compress gate + return [ + { id: "raw-a", role: "user", contentType: "text", text: "first question " + filler }, + { id: "raw-img", role: "user", contentType: "text", text: "look at this " + filler }, + { id: "raw-b", role: "assistant", contentType: "text", text: "answer " + filler }, + { id: "raw-c", role: "user", contentType: "text", text: "second question" }, + { id: "raw-d", role: "assistant", contentType: "text", text: "second answer" }, + { id: "raw-e", role: "user", contentType: "text", text: "third question" }, + { id: "raw-f", role: "assistant", contentType: "text", text: "third answer" }, + ]; +} + +test("pin: BLOCKED→numeric migration mints the ref but defers the carrier tag (#3805)", () => { + const core = createCore(cfg()); + // Simulate a pre-existing BLOCKED assignment on raw-img (protection lifted + // since — the message is plain text now). + const state = createInitialState(); + state.messageRefs.byRaw["raw-img"] = BLOCKED_REF; + + const t1 = core.processTurn({ + messages: hist(), + state, + config: cfg(), + tokenCount: 1_000, + renderTags: "all", + }); + + const imgRef = t1.state.messageRefs.byRaw["raw-img"]!; + assert.notEqual(imgRef, BLOCKED_REF, "migration mints a numeric ref"); + const imgText = t1.messages.find((m) => m.id === "raw-img")!.text!; + assert.ok(!imgText.includes("\x3cacp "), "carrier text stays untagged on the migration turn"); + assert.deepEqual(t1.state.renderDeferredRefs, [imgRef], "deferral recorded in state"); + // Sanity: other messages render normally. + assert.ok(t1.messages.find((m) => m.id === "raw-a")!.text!.includes("\x3cacp "), "untagged only for the migrated carrier"); + + // Quiet turn 2: deferral persists — still no tag, still byte-stable. + const t2 = core.processTurn({ + messages: hist(), + state: t1.state, + config: cfg(), + tokenCount: 1_000, + renderTags: "all", + }); + assert.ok(!t2.messages.find((m) => m.id === "raw-img")!.text!.includes("\x3cacp "), "deferral self-perpetuates on quiet turns"); +}); + +test("pin: a fold boundary releases the deferral and the tag rides the fold's rewrite", () => { + const core = createCore(cfg()); + const state = createInitialState(); + state.messageRefs.byRaw["raw-img"] = BLOCKED_REF; + const t1 = core.processTurn({ messages: hist(), state, config: cfg(), tokenCount: 1_000, renderTags: "all" }); + assert.ok(t1.state.renderDeferredRefs?.length, "deferral active"); + + const refOf = (st: typeof t1.state, id: string) => st.messageRefs.byRaw[id]; + const fold = core.applyCompression({ + messages: t1.messages, + state: t1.state, + config: cfg(), + ranges: [{ + startRef: refOf(t1.state, "raw-a")!, + endRef: refOf(t1.state, "raw-b")!, + summary: "a long enough summary for the fold gate to accept this range, over fifty chars", + topic: "t", + }], + }); + assert.equal(fold.result.blocksCreated, 1, `fold lands: ${JSON.stringify(fold.result.errors ?? [])} ${JSON.stringify(fold.result.warnings ?? [])}`); + assert.equal(fold.state.renderDeferredRefs, undefined, "fold releases the deferral"); + + const t3 = core.processTurn({ messages: hist(), state: fold.state, config: cfg(), tokenCount: 1_000, renderTags: "all" }); + const imgText = t3.messages.find((m) => m.id === "raw-img")?.text ?? ""; + assert.ok(!imgText.includes("\x3cacp ") || t3.messages.every((m) => m.id !== "raw-img"), "raw-img rode the fold away or renders tagged at the fold boundary"); +}); diff --git a/scripts/update-storage-render-goldens.ts b/scripts/update-storage-render-goldens.ts index 46c1f21f5c..63948692d8 100644 --- a/scripts/update-storage-render-goldens.ts +++ b/scripts/update-storage-render-goldens.ts @@ -11,6 +11,11 @@ import { driveWire, WIRES } from "../tests/golden-storage-drive.ts"; // same condition — without it every per-turn wire digest drifts by the image- // rescale delta and the suite reds on the next CI run. process.env.BILI_IMAGE_RESCALE = "0"; +// Parity #2: the test suite also disarms the shadow observability lane +// (#3756) before driving — its metadata blob would otherwise diverge from +// what the committed test run hashes, and every freshly regenerated golden +// would red on the next CI run. +process.env.BILI_SNAPSHOT_SHADOW = "0"; const GOLDEN_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "tests", "golden", "storage-render"); fs.mkdirSync(GOLDEN_DIR, { recursive: true }); diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 22ddf63118..5a86b4e138 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -276,6 +276,12 @@ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, pare if (parent.state.emergencyTruncateLatch?.length) { session.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch.map((e) => ({ ...e })); } + // #3939 ②: same carry for the migration render deferral — a resumed/forked + // session re-sends the same history; without the deferral the first replay + // stamps migration tags mid-history (the #3805 byte flip, resurrected). + if (parent.state.renderDeferredRefs?.length) { + session.state.renderDeferredRefs = [...parent.state.renderDeferredRefs]; + } markDirty(session); } diff --git a/src/persist.ts b/src/persist.ts index 6ec42bbae8..79e2334c9a 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -248,6 +248,10 @@ function mergeState(parsed: CompressionState): CompressionState { // dropping it re-exposes clipped bytes mid-history after a restart, // rewriting the still-cached prefix on the first post-restart turn. emergencyTruncateLatch: parsed.emergencyTruncateLatch ?? fresh.emergencyTruncateLatch, + // #3939 ②/#3805: the render deferral must survive hydration — losing + // it stamps migration tags onto still-cached carriers right after a + // restart (a silent mid-history rewrite with no fold in sight). + renderDeferredRefs: parsed.renderDeferredRefs ?? fresh.renderDeferredRefs, }; } diff --git a/src/plugin.ts b/src/plugin.ts index 3d8b4818ef..6df98eedf8 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1393,6 +1393,9 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr // prefix — the child never sees the others, so their entries // would be dead weight (and retire on the next latch pass anyway). child.state.emergencyTruncateLatch = parent.state.emergencyTruncateLatch?.filter((e) => prefix.some((m) => m.ref === e.ref)); + // #3939 ②: deferred refs are content-keyed (byRaw ids) — keep only + // the ones whose carrier rode the forked prefix. + child.state.renderDeferredRefs = parent.state.renderDeferredRefs?.filter((r) => prefix.some((m) => m.ref === r)); // #2362: inverse of the orphan filter — a dead ref whose message IS // in the forked prefix is visible again in the child; refs outside // it stay dead (their messages are absent from the child too). diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index 64990619b2..c281bd92c7 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -43,6 +43,7 @@ const MERGE_STATE_FIELDS = [ "lastPassIds", "hiddenOrphanRefs", "emergencyTruncateLatch", + "renderDeferredRefs", "deadRefs", "terminalStreak", "nextRuleId", diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 1484cc0699..6d0b7728a0 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -103,7 +103,9 @@ "absorbed": "4f53cda18c2baa0c", "rules": "4f53cda18c2baa0c", "sweptOrphanCallIds": "74234e98afe7498f", - "pendingOrphanSweepMinRef": "74234e98afe7498f" + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f" }, "stateShape": { "refs": 49, diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 1883114ffa..d944e108d7 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -103,7 +103,9 @@ "absorbed": "4f53cda18c2baa0c", "rules": "4f53cda18c2baa0c", "sweptOrphanCallIds": "74234e98afe7498f", - "pendingOrphanSweepMinRef": "74234e98afe7498f" + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index cc6117c57e..ed03210d13 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -103,7 +103,9 @@ "absorbed": "4f53cda18c2baa0c", "rules": "4f53cda18c2baa0c", "sweptOrphanCallIds": "74234e98afe7498f", - "pendingOrphanSweepMinRef": "74234e98afe7498f" + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index e305b07dc5..8a8ca1c00d 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -103,7 +103,9 @@ "absorbed": "4f53cda18c2baa0c", "rules": "4f53cda18c2baa0c", "sweptOrphanCallIds": "74234e98afe7498f", - "pendingOrphanSweepMinRef": "74234e98afe7498f" + "pendingOrphanSweepMinRef": "74234e98afe7498f", + "emergencyTruncateLatch": "74234e98afe7498f", + "renderDeferredRefs": "74234e98afe7498f" }, "stateShape": { "refs": 55, From 4344ff26d1166d8c5dac9646215cf321c8708968 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:50:30 +0800 Subject: [PATCH 21/27] =?UTF-8?q?kernel:=20fold-gate=20the=20opt-in=20view?= =?UTF-8?q?=20rewriters=20via=20a=20structural=20view=20latch=20(#3939=20?= =?UTF-8?q?=E2=91=A4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit absorb-hide, crush and filter keepLastOnly re-derive their output every pass from state/context (absorb records growing, crush responding to token pressure, keep-last-only demotions as new occurrences arrive) — each change flipped OLD message bytes on an otherwise quiet turn (the #3805 prefix-cache breaker class). One structural latch now covers all of them: post-transform bytes are latched per padded ref at first application and replay verbatim; refreshes ride fold boundaries (applyCompression resets the map), the absorb action itself (instant- reclaim contract kept), and config edits (digest release — enable/ disable applies on the edit turn, never silently later). ccr needs no latch (replace-once-at-arrival, byte-stable by design #1460). Placed after hide-compress-calls (the #3784 sweep keeps its own gate) and before recommend/nudge/truncate/render so downstream judges see the view the model actually sees. Carried across sync/persist/fork/plugin/ ingest; storage-render goldens pin the two new state keys (undefined valued), wire bodies unchanged. --- kernel/package.json | 2 +- kernel/src/absorb.ts | 5 + kernel/src/compress.ts | 8 ++ kernel/src/pipeline.ts | 4 + kernel/src/sync.ts | 8 ++ kernel/src/types.ts | 28 ++++ kernel/src/view-latch.ts | 156 +++++++++++++++++++++ kernel/tests/issue3939-view-latch.test.ts | 153 ++++++++++++++++++++ kernel/tests/pipeline.test.ts | 2 + src/fork-adoption.ts | 11 ++ src/persist.ts | 5 + src/plugin.ts | 11 ++ src/storage/ingest.ts | 2 + tests/golden/storage-render/anthropic.json | 4 +- tests/golden/storage-render/chat.json | 4 +- tests/golden/storage-render/google.json | 4 +- tests/golden/storage-render/responses.json | 4 +- 17 files changed, 406 insertions(+), 5 deletions(-) create mode 100644 kernel/src/view-latch.ts create mode 100644 kernel/tests/issue3939-view-latch.test.ts diff --git a/kernel/package.json b/kernel/package.json index 96381e368e..600b13bda5 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.140", + "version": "0.0.141", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/absorb.ts b/kernel/src/absorb.ts index b1365b3972..4f3906ac1f 100644 --- a/kernel/src/absorb.ts +++ b/kernel/src/absorb.ts @@ -373,6 +373,11 @@ export function applyAbsorb(input: AbsorbInput): AbsorbOutcome { ...input.state.stats, absorbedTokens: (input.state.stats.absorbedTokens ?? 0) + tokens, }, + // #3939 ⑤: the absorb call is a deliberate history-rewriting action (a + // fold-class boundary, same family as applyCompression) — release the + // view latch here so the hide lands THIS turn, not at some later fold. + // Keeps absorb's instant-reclaim contract; quiet turns stay latched. + viewLatches: undefined, }; const bloat = diff --git a/kernel/src/compress.ts b/kernel/src/compress.ts index 9bb16ae475..67ff2df893 100644 --- a/kernel/src/compress.ts +++ b/kernel/src/compress.ts @@ -18,6 +18,7 @@ import { hideConsumedCompressCalls, refNum } from "./hide-consumed.js"; import { appendAbsorbPrompts, hideAbsorbedMessages } from "./absorb.js"; import { applyCrushToMessages } from "./crush.js"; import { applyRetrieve, ccrStoreNode, RETRIEVED_ID_PREFIX } from "./ccr.js"; +import { preTransformSnapshotNode, viewLatchNode } from "./view-latch.js"; import type { ApplyRetrieveResult, CcrEffect } from "./ccr.js"; import type { EmergencyTruncateLatchEntry } from "./types.js"; import { createContentStore } from "./content-store.js"; @@ -812,6 +813,11 @@ export function createCore(ports: Ports = {}): CompressionCore { // rides it. Any surviving carrier may render its tag from here on; the // rewrite is attributed to the fold, not to a quiet turn. state.renderDeferredRefs = undefined; + // #3939 ⑤: the latch refresh rides the fold's own history rewrite — + // the next pass re-latches every visible message's transform output + // fresh (absorb hides apply, crush re-evaluates under the new pressure, + // keep-last-only demotions land). + state.viewLatches = undefined; } return { @@ -943,12 +949,14 @@ export function createCore(ports: Ports = {}): CompressionCore { assignRefsNode, syncBlocksNode, pruneNode, + preTransformSnapshotNode, ccrStoreNode, absorbHideNode, crushNode, absorbPromptNode, filterNode, hideCompressCallsNode, + viewLatchNode, recommendNode, nudgeNode, emergencyTruncateNode, diff --git a/kernel/src/pipeline.ts b/kernel/src/pipeline.ts index 96d063f9d9..24dfa976ba 100644 --- a/kernel/src/pipeline.ts +++ b/kernel/src/pipeline.ts @@ -25,6 +25,10 @@ export interface NodeEffects { * foldability judgment — on lanes whose processTurn input is itself a * pruned/render view, even the pre-prune input is not the apply class. */ applyView?: CoreMessage[]; + /** #3939 ⑤: the post-prune, pre-transform resent view, stashed by the + * pre-transform-snapshot node so the view-latch node can replay latched + * bytes for messages the transform nodes dropped or rewrote. */ + preTransformMessages?: CoreMessage[]; readonly [key: string]: unknown; } diff --git a/kernel/src/sync.ts b/kernel/src/sync.ts index 578cab7e14..cb9564e82a 100644 --- a/kernel/src/sync.ts +++ b/kernel/src/sync.ts @@ -64,6 +64,14 @@ export function syncBlocks( renderDeferredRefs: state.renderDeferredRefs ? [...state.renderDeferredRefs] : undefined, + // #3939 ⑤: the view latch must survive sync — losing it re-derives + // transform output mid-history after a sync carried the state (#3805). + viewLatches: state.viewLatches + ? Object.fromEntries( + Object.entries(state.viewLatches).map(([k, v]) => [k, { ...v }]), + ) + : undefined, + viewLatchConfigDigest: state.viewLatchConfigDigest, }; // Refs are additive (assignRefs never removes them from messageRefs), so diff --git a/kernel/src/types.ts b/kernel/src/types.ts index ccb3a7aded..68e729582e 100644 --- a/kernel/src/types.ts +++ b/kernel/src/types.ts @@ -114,6 +114,12 @@ export interface CompressionStats { retrievalCount?: number; } +/** View-latch entry shape (#3939 ⑤) — see CompressionState.viewLatches. */ +export interface ViewLatchEntry { + text: string | null; + hidden?: boolean; +} + /** One instant tool-result absorption: the original tool-call + tool-result * pair is hidden from view once recorded; the model's absorb call (carrying * the distilled summary) remains as the durable record. */ @@ -291,6 +297,28 @@ export interface CompressionState { * leaves the view (ids are never re-issued). Optional: absent until the * first migration. */ renderDeferredRefs?: string[]; + /** Fold-gated view latch (#3939 ⑤): post-transform view bytes per padded + * ref, replayed VERBATIM on quiet turns. The opt-in mid-history rewriters + * (absorb-hide, crush, filter keepLastOnly) re-derive their output every + * pass from state/context (absorb records growing, crush responding to + * token pressure, keep-last-only demotions as new occurrences arrive) — + * without a latch each change flips OLD message bytes on an otherwise + * quiet turn (the #3805 prefix-cache breaker class). Structural, not + * per-feature: every transform output is latched at first application and + * replays until the next successful fold (applyCompression resets the + * whole map — the refresh rides the fold's own history rewrite); new or + * edited messages (new ref) latch fresh at their arrival boundary. ccr + * needs no latch (replace-once-at-arrival, byte-stable by design #1460). + * Entry: text=null = passthrough (source bytes stand); text=string = + * transformed bytes; hidden=true = dropped from the view. Optional: + * absent until the first latched transform. */ + viewLatches?: Record; + /** #3939 ⑤: digest of the transform-relevant config slices (absorb/crush/ + * message-filters). A config edit is a user action boundary — matching + * ①'s fold/refold/user-action rule, the latch releases when the digest + * changes so enabling a transform applies on the next turn (one rewrite + * riding the config change), never on a quiet turn. */ + viewLatchConfigDigest?: string; /** Padded refs (mNNNNN) proven UNREACHABLE (#2362): known in messageRefs but * backing no visible or folded message — the client rewrote or dropped those * messages (an edit reissues a new ref; a host-native compaction or bulk diff --git a/kernel/src/view-latch.ts b/kernel/src/view-latch.ts new file mode 100644 index 0000000000..51ef86a9cd --- /dev/null +++ b/kernel/src/view-latch.ts @@ -0,0 +1,156 @@ +import { BLOCKED_REF } from "./refs.js"; +import { RETRIEVED_ID_PREFIX } from "./ccr.js"; +import { resolveAbsorbConfig } from "./absorb.js"; +import { resolveCrushConfig } from "./crush.js"; +import type { CompressionState, CoreMessage, ViewLatchEntry } from "./types.js"; +import type { NodeIO, PipelineNode, PipelineContext } from "./pipeline.js"; + +/** Stash the post-prune, pre-transform view (#3939 ⑤). The view-latch node + * replays latched bytes from this array — the transform nodes have already + * dropped or rewritten their targets by the time the latch runs, so the + * source-of-truth resent bytes live here. */ +export const preTransformSnapshotNode: PipelineNode = { + name: "pre-transform-snapshot", + run(io) { + return { + ...io, + effects: { ...io.effects, preTransformMessages: io.messages }, + }; + }, +}; + +/** Fold-gated view latch (#3939 ⑤/#3805): replay post-transform bytes + * verbatim on quiet turns; refresh latches only at fold boundaries. Runs + * after every opt-in transform node (ccr-store, absorb-hide, crush, + * absorb-prompt, filter) and after hide-compress-calls (its own #3784 fold + * gate stays authoritative — the latch only records its output), before + * recommend/nudge/emergency-truncate/render so downstream nodes judge the + * same view the model actually sees. */ +export const viewLatchNode: PipelineNode = { + name: "view-latch", + run(io, ctx) { + return applyViewLatch(io, ctx); + }, +}; + +function latchKeyEqual( + a: Record | undefined, + b: Record | undefined, +): boolean { + if (!a || !b) return !a && !b; + if (Object.keys(a).length !== Object.keys(b).length) return false; + for (const [k, v] of Object.entries(a)) { + const o = b[k]; + if (!o || o.text !== v.text || o.hidden !== v.hidden) return false; + } + return true; +} + +/** Transform-relevant config slices — a change here is a user action + * boundary (①): the latch releases so enable/disable applies immediately + * instead of waiting for a fold. tokenCount is deliberately NOT included + * (pressure fluctuation is what the latch exists to gate). */ +export function viewLatchConfigDigest(ctx: PipelineContext): string { + return JSON.stringify({ + absorb: resolveAbsorbConfig(ctx.config), + crush: resolveCrushConfig(ctx.config), + filters: { + enabled: ctx.config.messageFilters?.enabled ?? false, + filters: ctx.config.messageFilters?.filters ?? null, + }, + }); +} + +export function applyViewLatch(io: NodeIO, ctx?: PipelineContext): NodeIO { + const pre = io.effects.preTransformMessages; + if (!pre) return io; + // Config boundary (#3939 ①/⑤): digest change releases the latch — the + // transform enable/disable applies this turn (one rewrite riding the user's + // config edit), never silently later. + const digest = ctx ? viewLatchConfigDigest(ctx) : undefined; + const configChanged = + digest !== undefined && digest !== io.state.viewLatchConfigDigest; + const prev = configChanged ? undefined : io.state.viewLatches; + const post = new Map(); + for (const m of io.messages) if (m.id) post.set(m.id, m); + + const out: CoreMessage[] = []; + const latches: Record = {}; + let latchedTransform = false; + + for (const m of pre) { + if (!m.id || m.id.startsWith(RETRIEVED_ID_PREFIX)) { + const t = post.get(m.id!); + if (t) out.push(t); + continue; + } + const ref = io.state.messageRefs.byRaw[m.id]; + if (!ref || ref === BLOCKED_REF) { + const t = post.get(m.id); + if (t) out.push(t); + continue; + } + const transformed = post.get(m.id); + const prevEntry = prev?.[ref]; + if (prevEntry) { + // REPLAY the latched bytes — a transform's fresh output for an + // already-latched ref is deferred to the next fold boundary. + if (prevEntry.hidden) { + latches[ref] = prevEntry; + continue; + } + if (prevEntry.text === null) { + if (transformed) out.push(m); + latches[ref] = prevEntry; + continue; + } + latchedTransform = true; + if (transformed && transformed.text === prevEntry.text) out.push(transformed); + else out.push({ ...m, text: prevEntry.text }); + latches[ref] = prevEntry; + } else { + // FRESH: latch whatever the transforms produced at this arrival + // boundary (arrival/first-enable IS a history change; the flip rides it). + if (!transformed) { + latches[ref] = { text: null, hidden: true }; + continue; + } + if (transformed.text !== m.text) { + latches[ref] = { text: transformed.text ?? null }; + latchedTransform = true; + out.push(transformed); + } else { + latches[ref] = { text: null }; + out.push(m); + } + } + } + + // Steady state (identical latch map) must not churn the state object and + // force an adapter save every turn (render-refs snapshot precedent). + if ( + !configChanged && + digest === io.state.viewLatchConfigDigest && + latchKeyEqual(io.state.viewLatches, latches) + ) { + return { ...io, messages: out }; + } + // No prior latches and every entry passthrough → nothing worth pinning + // (no opt-in transform active); skip the write entirely. + if (!prev && !latchedTransform) return io; + return { + ...io, + messages: out, + state: { + ...io.state, + viewLatches: latches, + ...(digest !== undefined ? { viewLatchConfigDigest: digest } : {}), + }, + }; +} + +/** #3939 ⑤: a successful fold rewrote history — the latch refresh rides it. */ +export function resetViewLatches(state: CompressionState): CompressionState { + if (state.viewLatches === undefined) return state; + return { ...state, viewLatches: undefined }; +} diff --git a/kernel/tests/issue3939-view-latch.test.ts b/kernel/tests/issue3939-view-latch.test.ts new file mode 100644 index 0000000000..6bbb5722fb --- /dev/null +++ b/kernel/tests/issue3939-view-latch.test.ts @@ -0,0 +1,153 @@ +// #3939 ⑤: the opt-in mid-history rewriters (absorb-hide, crush, filter +// keepLastOnly) re-derive their output every pass; without a latch each state/ +// context change flips OLD message bytes on an otherwise quiet turn (#3805 +// class). The view latch replays post-transform bytes verbatim; refreshes ride +// fold boundaries, absorb actions, and config edits (user action boundaries). +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createCore, defaultConfig } from "../src/index.js"; +import { applyAbsorb } from "../src/absorb.js"; +import { createInitialState } from "../src/state.js"; +import { + registerMessageFilter, + clearMessageFilters, +} from "../src/filter/registry.js"; +import type { CoreMessage } from "../src/types.js"; + +const W = 100_000; +const cfg = () => ({ ...defaultConfig(W), preserveRecentTokens: 100 }); + +function hist(): CoreMessage[] { + const filler = "filler ".repeat(500); + return [ + { id: "raw-a", role: "user", contentType: "text", text: "first question " + filler }, + { id: "raw-b", role: "assistant", contentType: "text", text: "answer " + filler }, + { id: "raw-c", role: "user", contentType: "text", text: "second question" }, + { id: "raw-d", role: "assistant", contentType: "text", text: "second answer" }, + { id: "raw-e", role: "user", contentType: "text", text: "third question" }, + { id: "raw-f", role: "assistant", contentType: "text", text: "third answer" }, + ]; +} + +test("pin: filter keepLastOnly demotion is deferred to the fold boundary", () => { + clearMessageFilters(); + registerMessageFilter({ + name: "pin-marker", + version: "1", + description: "test filter", + keepLastOnly: true, + filter: (ctx) => + ctx.text.includes("SECRETTOKEN") + ? { action: "modify", text: ctx.text.replace(/SECRETTOKEN/g, "[redacted]") } + : { action: "keep" }, + }); + try { + const core = createCore(cfg()); + const config = { + ...cfg(), + messageFilters: { enabled: true, filters: { "pin-marker": { enabled: true } } }, + }; + const m1 = [ + ...hist(), + { id: "raw-occ1", role: "user", contentType: "text" as const, text: "here is SECRETTOKEN one" }, + ]; + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config, tokenCount: 1_000, renderTags: "none" }); + // occ1 is the LAST occurrence → the filter's decision applies at arrival + // (modify → [redacted]) and the result is latched. + const occ1a = t1.messages.find((m) => m.id === "raw-occ1")!; + assert.ok(occ1a.text!.includes("[redacted]"), "last occurrence filtered on arrival"); + assert.ok(!occ1a.text!.includes("SECRETTOKEN"), "pattern stripped at arrival"); + + // A NEWER occurrence arrives — the fresh filter would demote occ1 (blank + // it). The latch replays occ1's bytes unchanged: the demotion waits for a + // fold boundary. + const m2 = [ + ...m1, + { id: "raw-occ2", role: "user", contentType: "text" as const, text: "again SECRETTOKEN two" }, + ]; + const t2 = core.processTurn({ messages: m2, state: t1.state, config, tokenCount: 1_000, renderTags: "none" }); + const occ1b = t2.messages.find((m) => m.id === "raw-occ1")!; + assert.equal(occ1b.text, occ1a.text, "demotion deferred — old bytes replay verbatim"); + assert.ok(t2.messages.find((m) => m.id === "raw-occ2")!.text!.includes("[redacted]"), "new last occurrence filtered fresh (arrival boundary)"); + + // Fold covering occ1 releases the latch: post-fold, occ1 is folded away + // and the new state carries no latches. + const refOf = (st: typeof t1.state, id: string) => st.messageRefs.byRaw[id]; + const fold = core.applyCompression({ + messages: t2.messages, + state: t2.state, + config, + ranges: [{ + startRef: refOf(t2.state, "raw-a")!, + endRef: refOf(t2.state, "raw-b")!, + summary: "a long enough summary for the fold gate to accept this range, over fifty chars", + topic: "t", + }], + }); + assert.equal(fold.result.blocksCreated, 1, `fold lands: ${JSON.stringify(fold.result.errors ?? [])}`); + assert.equal(fold.state.viewLatches, undefined, "fold releases the view latch"); + } finally { + clearMessageFilters(); + } +}); + +test("pin: absorb-hide defers to the absorb action boundary, not a later fold", () => { + const core = createCore(cfg()); + const config = defaultConfig(W, { + absorb: { enabled: true, toolName: "acp_absorb", minToolTokens: 1, contextThresholdPct: 0, excludeTools: [] }, + }); + const m1 = [ + { id: "u1", role: "user", contentType: "text" as const, text: "read the file" }, + { id: "c1", role: "tool", contentType: "tool-call" as const, toolName: "read", toolCallId: "call1", text: "{}" }, + { id: "r1", role: "tool", contentType: "tool-result" as const, toolName: "read", toolCallId: "call1", text: "file body line one\n" + "x".repeat(400) }, + ]; + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config, tokenCount: 3_000, renderTags: "none" }); + assert.ok(t1.messages.find((m) => m.id === "r1"), "pair visible before absorb"); + + // Absorb r1: the absorb call is an action boundary — applyAbsorb releases + // the latch, so the hide lands THIS turn (instant-reclaim contract). + const ref = t1.state.messageRefs.byRaw["r1"]!; + const outcome = applyAbsorb({ + ref, + summary: "file defines foo(a,b) at src/foo.ts:12", + messages: m1, + state: t1.state, + config, + }); + assert.ok(outcome.ok, "absorb ok"); + const t2 = core.processTurn({ messages: m1, state: outcome.state, config, tokenCount: 3_200, renderTags: "none" }); + const ids = t2.messages.map((m) => m.id); + assert.ok(!ids.includes("r1"), "absorbed result hidden on the action boundary turn"); + assert.ok(!ids.includes("c1"), "absorbed call hidden on the action boundary turn"); +}); + +test("pin: config enable of crush applies on the edit turn (config boundary), token-pressure flips do not", () => { + const core = createCore(cfg()); + const off = defaultConfig(W, { + absorb: { enabled: true, toolName: "acp_absorb", minToolTokens: 100, contextThresholdPct: 0, excludeTools: [] }, + }); + const rows = JSON.stringify( + Array.from({ length: 40 }, (_, i) => ({ id: i * 7, ts: "2026-09-21T12:00:00Z", level: "INFO", name: `svc-${i}` })), + ); + const m1 = [ + { id: "u1", role: "user", contentType: "text" as const, text: "read the log" }, + { id: "c1", role: "tool", contentType: "tool-call" as const, toolName: "read", toolCallId: "call1", text: "{}" }, + { id: "r1", role: "tool", contentType: "tool-result" as const, toolName: "read", toolCallId: "call1", text: rows }, + ]; + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config: off, tokenCount: 0, renderTags: "none" }); + const fresh = t1.messages.find((m) => m.id === "r1")!.text!; + assert.ok(!fresh.includes("__acp_crush"), "crush off on turn 1"); + + // Enable crush: config edit = user action boundary — the crush applies now. + const onCfg = defaultConfig(W, { + absorb: { enabled: true, toolName: "acp_absorb", minToolTokens: 100, contextThresholdPct: 0, excludeTools: [] }, + crush: { enabled: true, minReduction: 0.1 }, + }); + const t2 = core.processTurn({ messages: m1, state: t1.state, config: onCfg, tokenCount: 0, renderTags: "none" }); + assert.ok(t2.messages.find((m) => m.id === "r1")!.text!.includes("__acp_crush"), "enable applies on the config-edit turn"); + + // Token pressure alone never flips bytes: same config, wildly different + // tokenCount — the crushed bytes replay verbatim. + const t3 = core.processTurn({ messages: m1, state: t2.state, config: onCfg, tokenCount: 99_000, renderTags: "none" }); + assert.equal(t3.messages.find((m) => m.id === "r1")!.text, t2.messages.find((m) => m.id === "r1")!.text, "pressure does not re-derive latched bytes"); +}); diff --git a/kernel/tests/pipeline.test.ts b/kernel/tests/pipeline.test.ts index ec1c877412..f5493f17f4 100644 --- a/kernel/tests/pipeline.test.ts +++ b/kernel/tests/pipeline.test.ts @@ -29,12 +29,14 @@ test("defaultNodes exposes the canonical ordered pipeline", () => { "assign-refs", "sync-blocks", "prune", + "pre-transform-snapshot", "ccr-store", "absorb-hide", "crush", "absorb-prompt", "filter", "hide-compress-calls", + "view-latch", "recommend", "nudge-inject", "emergency-truncate", diff --git a/src/fork-adoption.ts b/src/fork-adoption.ts index 5a86b4e138..3d15fffdf1 100644 --- a/src/fork-adoption.ts +++ b/src/fork-adoption.ts @@ -282,6 +282,17 @@ export function applyForkAdoption(session: Session, plan: ForkAdoptionPlan, pare if (parent.state.renderDeferredRefs?.length) { session.state.renderDeferredRefs = [...parent.state.renderDeferredRefs]; } + // #3939 ⑤: same carry for the view latch — a resumed session re-sends the + // same history; a dropped latch would flip transform bytes on the first + // replayed turn (crush re-derived, absorb hides landing late). + if (parent.state.viewLatches) { + session.state.viewLatches = Object.fromEntries( + Object.entries(parent.state.viewLatches).map(([k, v]) => [k, { ...v }]), + ); + } + if (parent.state.viewLatchConfigDigest) { + session.state.viewLatchConfigDigest = parent.state.viewLatchConfigDigest; + } markDirty(session); } diff --git a/src/persist.ts b/src/persist.ts index 79e2334c9a..48fb753141 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -252,6 +252,11 @@ function mergeState(parsed: CompressionState): CompressionState { // it stamps migration tags onto still-cached carriers right after a // restart (a silent mid-history rewrite with no fold in sight). renderDeferredRefs: parsed.renderDeferredRefs ?? fresh.renderDeferredRefs, + // #3939 ⑤: the view latch must survive hydration — losing it + // re-derives transform output on still-cached carriers right after a + // restart (silent mid-history rewrite, no fold in sight). + viewLatches: parsed.viewLatches ?? fresh.viewLatches, + viewLatchConfigDigest: parsed.viewLatchConfigDigest ?? fresh.viewLatchConfigDigest, }; } diff --git a/src/plugin.ts b/src/plugin.ts index 6df98eedf8..e04a08b76c 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -1396,6 +1396,17 @@ export async function handlePluginFork(payload: string, res: ServerResponse): Pr // #3939 ②: deferred refs are content-keyed (byRaw ids) — keep only // the ones whose carrier rode the forked prefix. child.state.renderDeferredRefs = parent.state.renderDeferredRefs?.filter((r) => prefix.some((m) => m.ref === r)); + // #3939 ⑤: view-latch entries are ref-keyed — keep only the + // carriers that rode the forked prefix. + const forkRefs = new Set(prefix.map((m) => m.ref)); + if (parent.state.viewLatches) { + child.state.viewLatches = Object.fromEntries( + Object.entries(parent.state.viewLatches).filter(([r]) => forkRefs.has(r)), + ); + } + if (parent.state.viewLatchConfigDigest) { + child.state.viewLatchConfigDigest = parent.state.viewLatchConfigDigest; + } // #2362: inverse of the orphan filter — a dead ref whose message IS // in the forked prefix is visible again in the child; refs outside // it stay dead (their messages are absent from the child too). diff --git a/src/storage/ingest.ts b/src/storage/ingest.ts index c281bd92c7..9c741dae79 100644 --- a/src/storage/ingest.ts +++ b/src/storage/ingest.ts @@ -44,6 +44,8 @@ const MERGE_STATE_FIELDS = [ "hiddenOrphanRefs", "emergencyTruncateLatch", "renderDeferredRefs", + "viewLatches", + "viewLatchConfigDigest", "deadRefs", "terminalStreak", "nextRuleId", diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 6d0b7728a0..91c052b92d 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -105,7 +105,9 @@ "sweptOrphanCallIds": "74234e98afe7498f", "pendingOrphanSweepMinRef": "74234e98afe7498f", "emergencyTruncateLatch": "74234e98afe7498f", - "renderDeferredRefs": "74234e98afe7498f" + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 49, diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index d944e108d7..ebe66d6ae6 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -105,7 +105,9 @@ "sweptOrphanCallIds": "74234e98afe7498f", "pendingOrphanSweepMinRef": "74234e98afe7498f", "emergencyTruncateLatch": "74234e98afe7498f", - "renderDeferredRefs": "74234e98afe7498f" + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index ed03210d13..13d78598b0 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -105,7 +105,9 @@ "sweptOrphanCallIds": "74234e98afe7498f", "pendingOrphanSweepMinRef": "74234e98afe7498f", "emergencyTruncateLatch": "74234e98afe7498f", - "renderDeferredRefs": "74234e98afe7498f" + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 42, diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 8a8ca1c00d..36ae5150bd 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -105,7 +105,9 @@ "sweptOrphanCallIds": "74234e98afe7498f", "pendingOrphanSweepMinRef": "74234e98afe7498f", "emergencyTruncateLatch": "74234e98afe7498f", - "renderDeferredRefs": "74234e98afe7498f" + "renderDeferredRefs": "74234e98afe7498f", + "viewLatches": "74234e98afe7498f", + "viewLatchConfigDigest": "74234e98afe7498f" }, "stateShape": { "refs": 55, From 3057f22b068ed063fe4b03e8a3677c1122ba2f71 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:55:16 +0800 Subject: [PATCH 22/27] chore: sync package-lock kernel version to 0.0.141 --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 7bcf65f2a3..58875c4ac1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.138", + "version": "0.0.141", "dev": true, "license": "MIT", "devDependencies": { From cf49050a70fef2334d7baa37e6e3e4fafffde69a Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 18:58:42 +0800 Subject: [PATCH 23/27] chore: sync kernel/package-lock.json version to 0.0.141 --- kernel/package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 4c282670af..045125fb67 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.138", + "version": "0.0.141", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.137", + "version": "0.0.141", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", From 6aed427f457341063419cb4ede30d6dfc86f5ae5 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:10:41 +0800 Subject: [PATCH 24/27] =?UTF-8?q?kernel:=20CCR=20rides=20the=20view-latch?= =?UTF-8?q?=20config=20digest=20=E2=80=94=20disarm=20releases=20latched=20?= =?UTF-8?q?placeholders=20(#3939=20=E2=91=A4,=20#1273)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- kernel/package-lock.json | 4 +-- kernel/package.json | 2 +- kernel/src/view-latch.ts | 7 +++++ kernel/tests/issue3939-view-latch.test.ts | 31 +++++++++++++++++++++++ package-lock.json | 2 +- 5 files changed, 42 insertions(+), 4 deletions(-) diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 045125fb67..6554478342 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.141", + "version": "0.0.142", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.141", + "version": "0.0.142", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 600b13bda5..80a8ce051b 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.141", + "version": "0.0.142", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/view-latch.ts b/kernel/src/view-latch.ts index 51ef86a9cd..2ca2d2100e 100644 --- a/kernel/src/view-latch.ts +++ b/kernel/src/view-latch.ts @@ -1,6 +1,7 @@ import { BLOCKED_REF } from "./refs.js"; import { RETRIEVED_ID_PREFIX } from "./ccr.js"; import { resolveAbsorbConfig } from "./absorb.js"; +import { resolveCcrConfig } from "./ccr.js"; import { resolveCrushConfig } from "./crush.js"; import type { CompressionState, CoreMessage, ViewLatchEntry } from "./types.js"; import type { NodeIO, PipelineNode, PipelineContext } from "./pipeline.js"; @@ -54,6 +55,12 @@ export function viewLatchConfigDigest(ctx: PipelineContext): string { return JSON.stringify({ absorb: resolveAbsorbConfig(ctx.config), crush: resolveCrushConfig(ctx.config), + // CCR rides the same boundary: a session that disarms CCR must never + // keep projecting latched [acp-stored] placeholders — the retrieve + // channel they advertise would be gone (dangling placeholder = silent + // loss, the #1273 plugin-governance contract). A digest change releases + // the latch and re-judges every message under the CURRENT config. + ccr: resolveCcrConfig(ctx.config), filters: { enabled: ctx.config.messageFilters?.enabled ?? false, filters: ctx.config.messageFilters?.filters ?? null, diff --git a/kernel/tests/issue3939-view-latch.test.ts b/kernel/tests/issue3939-view-latch.test.ts index 6bbb5722fb..d17ac0522e 100644 --- a/kernel/tests/issue3939-view-latch.test.ts +++ b/kernel/tests/issue3939-view-latch.test.ts @@ -151,3 +151,34 @@ test("pin: config enable of crush applies on the edit turn (config boundary), to const t3 = core.processTurn({ messages: m1, state: t2.state, config: onCfg, tokenCount: 99_000, renderTags: "none" }); assert.equal(t3.messages.find((m) => m.id === "r1")!.text, t2.messages.find((m) => m.id === "r1")!.text, "pressure does not re-derive latched bytes"); }); + +test("pin: CCR disarm releases the latch — a disarmed session never re-projects a dangling placeholder", () => { + // #1273 governance, restated for the latch: placeholders are legal only + // while the retrieve channel they advertise exists. CCR rides the config + // digest, so arming/disarming mid-session is a user-action boundary that + // releases the latch and re-judges under the CURRENT config. + const big = "line of build output ".repeat(400); + const m1 = [ + { id: "u1", role: "user", contentType: "text" as const, text: "run a big build" }, + { id: "c1", role: "assistant", contentType: "tool-call" as const, toolName: "bash", toolCallId: "call1", text: JSON.stringify({ command: "npm run build" }) }, + { id: "r1", role: "tool", contentType: "tool-result" as const, toolName: "bash", toolCallId: "call1", text: big }, + ]; + const armed = { ...cfg(), ccr: { enabled: true, toolName: "acp_retrieve", minToolTokens: 50, excludeTools: [], maxHeadChars: 96 } }; + const core = createCore(cfg()); + const t1 = core.processTurn({ messages: m1, state: createInitialState(), config: armed, tokenCount: 0, renderTags: "none" }); + const t1r = t1.messages.find((m) => m.id === "r1")!.text!; + assert.ok(t1r.includes("[acp-stored"), "armed CCR substitutes at arrival"); + assert.ok(!t1r.includes("line of build output"), "raw bytes leave the wire while armed"); + + // Same session, next turn, CCR still armed: the placeholder replays + // byte-stable (the latch holds; #1460 reprojection agrees with it). + const t2 = core.processTurn({ messages: m1, state: t1.state, config: armed, contentStore: t1.contentStore, tokenCount: 0, renderTags: "none" }); + assert.equal(t2.messages.find((m) => m.id === "r1")!.text, t1r, "latched placeholder replays verbatim while armed"); + + // Disarm (route/base governance flip): the digest changes → the latch + // releases → the raw bytes return. No dangling placeholder, no silent loss. + const t3 = core.processTurn({ messages: m1, state: t2.state, config: cfg(), contentStore: t2.contentStore, tokenCount: 0, renderTags: "none" }); + const t3r = t3.messages.find((m) => m.id === "r1")!.text!; + assert.ok(t3r.includes("line of build output"), "disarmed CCR returns the raw bytes"); + assert.ok(!t3r.includes("[acp-stored"), "no dangling placeholder after disarm"); +}); diff --git a/package-lock.json b/package-lock.json index 58875c4ac1..f0ee9a6d1c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.141", + "version": "0.0.142", "dev": true, "license": "MIT", "devDependencies": { From a61a1f9bc5bd0439e5e14ff6ea1f71305ab4bf42 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:10:41 +0800 Subject: [PATCH 25/27] fix: rescue a numberless overflow in-request under arm*0.9; exhausted preflight defers to tier recovery (#3787, #2952) --- src/preflight.ts | 9 +++++++++ src/server/relay.ts | 16 +++++++++++++++- tests/e2e-overflow-selfheal.test.ts | 25 ++++++++++++++----------- tests/issue3787-overflow-refold.test.ts | 20 ++++++++++++++------ 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/src/preflight.ts b/src/preflight.ts index 49941a9706..cffe74d8f1 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -2046,6 +2046,15 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // remains (or nothing above minCompressRange, or the whole relaxed // zone already folded) — so no retry against a healthier relay can // change the verdict. + // #2952 Design A: "nothing compressible" can still leave a rescue — + // every raw message already inside a block (view = carriers only) + // is exactly the distillable state. Defer to the post-walk + // recovery funnel instead of failing here, same as the futility + // gate above; tiers=false / emergency digest never enter this way. + if (eligibleForTierRecovery()) { + deps.log("info", `[preflight] no compressible ranges remain${relaxed ? " even with the soft zone relaxed" : ""}; deferring to bounded tier-distillation recovery (#2952)`); + break; + } failure = { kind: "exhausted", deterministic: true, detail: relaxed ? relaxedExhaustedDetail : subMinNoted ? `no foldable compressible ranges remain: all ${viable.length} viable range(s) are below minCompressRange (${minChars} chars)` : "no compressible ranges remain in the conversation" }; diff --git a/src/server/relay.ts b/src/server/relay.ts index cfd3af7f7a..f27beab97c 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -1145,7 +1145,21 @@ export async function forward( const overflowInfo = inspectContextOverflow(upstreamResult.response.status, overflowText); if (overflowInfo.isOverflow) { armOverflowShrink(overflowInfo); - const refolded = await overflowRefold(overflowInfo.window).catch(() => null); + // #3787 follow-up: a NUMBERLESS overflow states no window; the + // armed baseline (min of declared and the rejected payload's + // own estimate) is the only ceiling evidence there is. Judging + // the refold against the DECLARED window made preflight see + // "already fits" and return an unchanged body — the rescue + // then only worked when re-render churn happened to change + // bytes, which byte-stable rebuilds (#3939 ③) eliminate. + // The arm is BY CONSTRUCTION the rejected payload's own size, + // so a refold targeting the arm itself leaves nothing to fold + // (payload == ceiling on every estimate): fold with a 10% + // margin under it, the same headroom discipline as the arm's + // own min(declared, est) clamp — the rejection proved the + // ceiling is below the payload, give the fold real room. + const armedWindow = overflowInfo.window ?? (prepared.session.stats.lastInputTokensSource === "overflow-arm" ? Math.floor(prepared.session.stats.lastInputTokens * 0.9) : undefined); + const refolded = await overflowRefold(armedWindow).catch(() => null); if (refolded) { try { // #816 family: the refold rebuilds the body from state diff --git a/tests/e2e-overflow-selfheal.test.ts b/tests/e2e-overflow-selfheal.test.ts index 08170ef714..45528eefbb 100644 --- a/tests/e2e-overflow-selfheal.test.ts +++ b/tests/e2e-overflow-selfheal.test.ts @@ -193,7 +193,7 @@ test("e2e #987/#1195 T1: a stated-window overflow arms the one-shot shrink, lear } }); -test("e2e #987 T2: an overflow WITHOUT a window number arms at the declared window and learns nothing", async () => { +test("e2e #987 T2: an overflow WITHOUT a window number arms at the payload's own size, rescues in-request (#3787), and learns nothing", async () => { let streamingCall = 0; let summaryCalls = 0; const upstream = http.createServer((req, res) => { @@ -238,24 +238,27 @@ test("e2e #987 T2: an overflow WITHOUT a window number arms at the declared wind const headers = { "content-type": "application/json", "x-acp-session": "t2-sess" }; const r1 = await fetch(url, { method: "POST", headers, body }); - assert.equal(r1.status, 400); + // #3787: a NUMBERLESS overflow used to pass through verbatim (the arm + // fired but the refold judged against the 400k declaration and found + // "already fits" — nothing to fold, 400 to the client). The arm is + // the only ceiling evidence: refold under arm*0.9, fold, re-send. + assert.equal(r1.status, 200, "#3787: numberless overflow rescued in-request under the armed window"); + assert.equal(r1.headers.get("content-type"), "text/event-stream", "the retry's SSE is what the client sees"); await r1.text(); + assert.ok(summaryCalls >= 1, "the rescue ran a preflight fold under the armed window"); const s = listSessions().find((x) => x.id === "t2-sess"); assert.ok(s, "session exists"); assert.equal(s!.metadata.confirmedContextLimits, undefined, "nothing learned"); - assert.ok(s!.stats.lastInputTokens > 10_000 && s!.stats.lastInputTokens < 400_000, `armed at the payload's own size (~16k, not the 400k declaration): ${s!.stats.lastInputTokens}`); - // #1839 ask 3: an arm is NOT a billing report — it carries its own - // provenance grade "overflow-arm" (still rescue-grade: effectiveTokenCount's - // fast path, the #496 evidence gate and the preflight floors accept it; - // billing-grade consumers keep excluding it). - assert.equal(s!.stats.lastInputTokensSource, "overflow-arm", "the arm carries its own provenance grade, never billing-grade \"usage\""); + assert.equal(s!.stats.lastInputTokens, 5000, "the retry's own usage report replaced the armed baseline"); + assert.equal(s!.stats.lastInputTokensSource, "usage", "the successful retry reports usage-grade"); + assert.equal(s!.stats.overflowArmTokens, undefined, "#1129: the retry's usage report retired the one-shot arm"); - // r2: forwarded as-is (no arm, nothing learned) — the pass-through - // shape the old #969 test asserted is unchanged. + // r2: same payload — the fold is sticky, forwarded compacted, no new fold. + const summaryCallsBefore = summaryCalls; const r2 = await fetch(url, { method: "POST", headers, body }); assert.equal(r2.status, 200); await r2.text(); - assert.equal(summaryCalls, 0, "no fold"); + assert.equal(summaryCalls, summaryCallsBefore, "r2 forwards the sticky fold without folding again"); } finally { proxy.close(); proxy.closeIdleConnections?.(); diff --git a/tests/issue3787-overflow-refold.test.ts b/tests/issue3787-overflow-refold.test.ts index ee4058d126..e84651adb5 100644 --- a/tests/issue3787-overflow-refold.test.ts +++ b/tests/issue3787-overflow-refold.test.ts @@ -105,16 +105,24 @@ test("e2e #3787: the agnes/cch numberless context_limit 400 is recognized and th _setStoreForTest(new SessionStore({ enabled: false })); setRegistryForTest({}); - // Declared window below the payload's true size: the first oversized turn - // rides out (the optimistic trigger cannot see CJK-density mass), the - // gateway 400s, and the rescue must come from the recognized error. + // Declared window far above the payload's true size: preflight skips the + // first turn entirely (nothing over 100K), the gateway 400s the RAW body + // anyway (its real ceiling is below the payload — agnes-3.0-flash through + // a mis-declared cch), and the rescue must come from the recognized error: + // arm = min(declared, payload's own estimate), refold under arm*0.9 folds + // real ranges, the smaller body is re-sent within the same request. + // (A declared window BELOW the payload cannot express this contract: the + // calibrated trigger now folds everything compressible BEFORE the send, + // so a numberless rejection arrives with zero foldable mass left — the + // rescue's historical "success" there rode on re-render truncation + // asymmetry, which #3939 ③ eliminated.) const proxy = await startServer({ port: 0, host: "127.0.0.1", upstream: "http://127.0.0.1", - routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-test": { context: 10_000 } } } }, - modelContextLimit: 10_000, - kernelConfig: defaultConfig(10_000), + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-test": { context: 100_000 } } } }, + modelContextLimit: 100_000, + kernelConfig: defaultConfig(100_000), compress: { injectTool: true, injectNudge: true }, compat: { roles: {} }, streamErrorShape: "protocol", From 06b7d23635a41779674c18f9d63086c55ce999f1 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:10:41 +0800 Subject: [PATCH 26/27] test: keep the #189 fold-point pin outside the protected zone under the atomic batch contract --- tests/compress-observability.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compress-observability.test.ts b/tests/compress-observability.test.ts index 68b811493b..a31985d82c 100644 --- a/tests/compress-observability.test.ts +++ b/tests/compress-observability.test.ts @@ -88,7 +88,7 @@ test("#189: fold point picks the EARLIEST range start across multiple ranges", ( const ctx = makeCompressibleCtx(); ctx.session.stats.lastInputTokens = 100000; const args = { content: [ - { startId: "m00005", endId: "m00006", summary: "OBS-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }, + { startId: "m00003", endId: "m00004", summary: "OBS-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }, { startId: "m00001", endId: "m00002", summary: "OBS-TEST-SUMMARY-PAYLOAD-LONG-ENOUGH-FOR-THE-KERNEL-MIN-LENGTH-CHECK" }, ] }; runApply(ctx, args); From 74eca4e1c49114524cb9a3f6f5e696e2cc1cff50 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 20 Oct 2026 20:39:32 +0800 Subject: [PATCH 27/27] kernel: keep the fold-gate files inside the verifiable subset; re-baseline the #3769 proof to the fold-gated contract (#3939) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - truncate-tools: build truncatedIds in the loop instead of spreading Map.keys() — the iterator spread degrades the symbolic translation of the 3691 harness (verif refuses to certify on degradation). - hide-consumed: hoist state.hiddenOrphanRefs ?? [] — .includes on a possibly-undefined receiver degrades the same way. - verif/symbolic: fold Number(constString) to its numeric constant; the old {kind:any} made Number.isFinite collapse to constant false, silently deleting the refNum-gated fold-sweep path from every model (real code fires it; the model could not — engine gap, not kernel bug). - verif-bug3769 harness re-baselined onto the #3784 fold gate: Q1 (append-only growth hides r1) is now UNSAT — the fix holds; Q1b proves the sweep is still derivable THROUGH the fold marker (judge not vacuous); replay pins hidden=0 without a pending fold and newlySwept=[c1,c2,c3] with one. - kernel 0.0.142 -> 0.0.143 (both locks synced). verif: 15/15 green with cvc5 (subset-guard, symbolic-smoke, 3691, 3769, 3805, 3676, generic, induction). kernel suite 1357/1357. --- kernel/package-lock.json | 4 +- kernel/package.json | 2 +- kernel/src/hide-consumed.ts | 3 +- kernel/src/truncate-tools.ts | 4 +- package-lock.json | 2 +- tests/verif-bug3676-wire-reorder.test.ts | 154 + tests/verif-bug3691-oscillation.test.ts | 123 + tests/verif-bug3769-orphan-window.test.ts | 240 ++ ...erif-bug3805-durable-ref-migration.test.ts | 233 ++ tests/verif-generic-append-stability.test.ts | 204 ++ .../verif-induction-append-stability.test.ts | 146 + tests/verif-subset-guard.test.ts | 48 + tests/verif-symbolic-smoke.test.ts | 101 + verif/README.md | 86 + verif/exceptions.ts | 81 + verif/guarded-files.ts | 23 + verif/replay.ts | 34 + verif/solve.ts | 144 + verif/subset-guard.ts | 584 ++++ verif/symbolic.ts | 2530 +++++++++++++++++ verif/terms.ts | 532 ++++ 21 files changed, 5272 insertions(+), 6 deletions(-) create mode 100644 tests/verif-bug3676-wire-reorder.test.ts create mode 100644 tests/verif-bug3691-oscillation.test.ts create mode 100644 tests/verif-bug3769-orphan-window.test.ts create mode 100644 tests/verif-bug3805-durable-ref-migration.test.ts create mode 100644 tests/verif-generic-append-stability.test.ts create mode 100644 tests/verif-induction-append-stability.test.ts create mode 100644 tests/verif-subset-guard.test.ts create mode 100644 tests/verif-symbolic-smoke.test.ts create mode 100644 verif/README.md create mode 100644 verif/exceptions.ts create mode 100644 verif/guarded-files.ts create mode 100644 verif/replay.ts create mode 100644 verif/solve.ts create mode 100644 verif/subset-guard.ts create mode 100644 verif/symbolic.ts create mode 100644 verif/terms.ts diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 6554478342..1d88f406cb 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.142", + "version": "0.0.143", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.142", + "version": "0.0.143", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 80a8ce051b..0457970312 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.142", + "version": "0.0.143", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/hide-consumed.ts b/kernel/src/hide-consumed.ts index d79400fb99..fd62ef7585 100644 --- a/kernel/src/hide-consumed.ts +++ b/kernel/src/hide-consumed.ts @@ -334,9 +334,10 @@ export function hideConsumedCompressCalls( hide = true; } else if (info.fate === "orphan") { const ref = state.messageRefs.byRaw[messages[i]!.id]; + const rememberedHidden: string[] = state.hiddenOrphanRefs ?? []; if (sweptSet.has(callId)) { hide = true; - } else if (ref && state.hiddenOrphanRefs?.includes(ref)) { + } else if (ref && rememberedHidden.includes(ref)) { // Pre-#3769 states persisted the rotation's hidden set as message refs // (hiddenOrphanRefs). Honor it on upgrade: resurrecting a pair the old // kernel already removed would re-insert it mid-history and break the diff --git a/kernel/src/truncate-tools.ts b/kernel/src/truncate-tools.ts index 19083bdc00..25cebb433b 100644 --- a/kernel/src/truncate-tools.ts +++ b/kernel/src/truncate-tools.ts @@ -90,6 +90,7 @@ export function truncateLargeToolOutputs( let remaining = tokenCount; const targetTokens = config.truncate.threshold * limit * 0.9; const replacements = new Map(); + const truncatedIds: string[] = []; const applyTo = (candidates: CoreMessage[]) => { for (const candidate of candidates) { @@ -107,6 +108,7 @@ export function truncateLargeToolOutputs( ); const replacement = `${prefix}\n\n...${TRUNCATION_MARKER} — original ~${tokens} tokens]...\n\n${suffix}`; replacements.set(candidate.id, replacement); + truncatedIds.push(candidate.id); truncatedCount++; remaining -= tokens - countTokens(replacement); savedTokens += tokens - countTokens(replacement); @@ -126,6 +128,6 @@ export function truncateLargeToolOutputs( truncatedCount, savedTokens, candidatesFound, - truncatedIds: [...replacements.keys()], + truncatedIds, }; } diff --git a/package-lock.json b/package-lock.json index f0ee9a6d1c..7848be48d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.142", + "version": "0.0.143", "dev": true, "license": "MIT", "devDependencies": { diff --git a/tests/verif-bug3676-wire-reorder.test.ts b/tests/verif-bug3676-wire-reorder.test.ts new file mode 100644 index 0000000000..de3992289b --- /dev/null +++ b/tests/verif-bug3676-wire-reorder.test.ts @@ -0,0 +1,154 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of } from "../verif/terms.js"; +import type { Seq, Bool } from "../verif/terms.js"; +import { solveSmt2 } from "../verif/solve.js"; +import { openaiToCore, coreToOpenai } from "../kernel/src/wire/openai.js"; +import { resolve } from "node:path"; + +/** + * #3676 — OpenAI wire round-trip reverses [file, text] content-part order. + * + * Invariant under test (general — no bug shape hardcoded): a parse→rebuild + * round trip must preserve the ORDER of a user message's content parts. + * Part order IS bytes: a reordered parts array is a different outbound + * request, so every cache in front of it misses. + * + * Phase 1 (symbolic, REAL openaiToCore/coreToOpenai translated): the solver + * must exhibit a wire user message whose parts are [opaque(file), text(T)] + * with the rebuilt parts [text, file] — REVERSED. + * Phase 2 (negative control): for [text, file] the same reversed assertion is + * UNSAT — the invariant is the judge, not the scenario. + * Phase 3 (replay): the witness shape runs through the REAL functions. + */ + +const FILE = resolve(import.meta.dirname, "../kernel/src/wire/openai.ts"); + +type Ctx = ReturnType; +type SymStr = { kind: "str"; t: Seq<"main"> }; +type SymObj = { kind: "obj"; fields: Map }; +type SymArr = { kind: "arr"; elems: SymObj[]; len: any }; + +const strc = (ctx: Ctx, v: string): SymStr => ({ kind: "str", t: ctx.String.val(v) }) as never; +const objOf = (fields: Record): SymObj => ({ kind: "obj", fields: new Map(Object.entries(fields)) }) as never; +const arrOf = (ctx: Ctx, elems: SymObj[]): SymArr => ({ kind: "arr", elems, len: ctx.Real.val(elems.length) }) as never; + +/** Deterministic port for deriveCanonicalMessageId (real one uses + * node:crypto sha256 — outside the guarded subset). Distinct argument + * tuples → distinct constant ids; ids never influence the order property. */ +function makeIdPort(ctx: Ctx) { + const memo = new Map(); + let n = 0; + return (...args: Array<{ kind: string }>): SymStr => { + const key = args.map((a) => (a.kind === "str" ? String((a as { t?: { c?: string } }).t?.c ?? "?") : a.kind)).join("|"); + let v = memo.get(key); + if (!v) { v = { kind: "str", t: strc(ctx, `id${n++}`) } as never; memo.set(key, v); } + return v; + }; +} + +/** ClusterCounter port: per-instance JS counter (state outside the model — + * ids don't affect the order property, only determinism). */ +function makeCounterPorts(ctx: Ctx) { + const counts = new WeakMap(); + const marker = objOf({}); + return { + ClusterCounter: () => marker, + __method_next: (recv: SymObj, base: SymStr): SymStr => { + if (recv !== marker) return { kind: "str", t: strc(ctx, "?") } as never; + const c = (counts.get(marker) ?? 0) + 1; + counts.set(marker, c); + const b = String(base?.t?.c ?? "?"); + return { kind: "str", t: strc(ctx, c === 1 ? b : `${b}_${c - 1}`) } as never; + }, + }; +} + +function wireInput(ctx: Ctx, first: "file" | "text") { + const T = ctx.String.const("T"); + const filePart = objOf({ type: strc(ctx, "file"), file_id: strc(ctx, "file_123") }); + const textPart = objOf({ type: strc(ctx, "text"), text: { kind: "str", t: T } as never }); + const parts = arrOf(ctx, first === "file" ? [filePart, textPart] : [textPart, filePart]); + const userMsg = objOf({ role: strc(ctx, "user"), content: parts }); + return { T, body: objOf({ messages: arrOf(ctx, [userMsg]) }) }; +} + +type ParseRet = { degradations: string[]; ret: { kind: string; fields: Map } }; +type RbRet = { degradations: string[]; ret: SymArr }; + +function roundTrip(ctx: Ctx, ports: unknown, first: "file" | "text") { + const { T, body } = wireInput(ctx, first); + const parseOut = translateFunction({ file: FILE, function: "openaiToCore", params: { body }, ctx, options: { portImpls: ports as never } }).exec() as never as ParseRet; + const coreMsgs = parseOut.ret.fields.get("msgs") as never; + const rbOut = translateFunction({ file: FILE, function: "coreToOpenai", params: { messages: coreMsgs }, ctx, options: { portImpls: ports as never } }).exec() as never as RbRet; + return { T, parseOut, rbOut }; +} + +function outContent(rb: RbRet): SymArr { + const out0 = rb.ret.elems[0]!; + assert.equal(out0.kind, "obj", "one wire message out"); + const c = (out0.fields as Map).get("content") as SymArr; + assert.equal(c.kind, "arr", "opaque parts ride along → parts array (not plain string)"); + return c; +} + +function partType(content: SymArr, i: number): Seq<"main"> { + const e = content.elems[i]; + assert.ok(e && e.kind === "obj", `part ${i} exists`); + const t = (e.fields as Map).get("type") as SymStr | undefined; + assert.ok(t && t.kind === "str", `part ${i} has a type`); + return t.t; +} + +test("bug3676: solver finds the [file,text]→[text,file] part-order reversal on the real wire codec", { timeout: 240_000 }, async () => { + const ctx = newCtx(); + const ports = { ...makeCounterPorts(ctx), deriveCanonicalMessageId: makeIdPort(ctx) }; + + const A = roundTrip(ctx, ports, "file"); // original order [file, text] + const B = roundTrip(ctx, ports, "text"); // original order [text, file] + assert.deepEqual(A.parseOut.degradations, [], "parse fully translated"); + assert.deepEqual(A.rbOut.degradations, [], "rebuild fully translated"); + + const cA = outContent(A.rbOut); + const cB = outContent(B.rbOut); + + // Property: original [file,text] rebuilds as [text,file] — REVERSED. + // len(out)==2 ∧ T≠"" ∧ type(out[0])=="text" ∧ type(out[1])=="file" + const tNonEmptyA = ctx.ToReal((A.T as unknown as { length(): Seq<"main">["length"] extends () => infer R ? R : never }).length() as never).ge(ctx.Real.val(1)) as unknown as Bool<"main">; + const len2A = (cA.len as unknown as { eq: (x: unknown) => Bool<"main"> }).eq(ctx.Real.val(2)); + const p0A = partType(cA, 0).eq(ctx.String.val("text")); + const p1A = partType(cA, 1).eq(ctx.String.val("file")); + const q1 = smt2Of(ctx, [tNonEmptyA.and(len2A).and(p0A).and(p1A) as never, ...[]], []); + const r1 = await solveSmt2(q1, []); + assert.equal(r1.status, "sat", "solver must exhibit the [file,text]→[text,file] reversal"); + + // Negative control: original [text,file] CANNOT rebuild reversed. + const tNonEmptyB = ctx.ToReal((B.T as unknown as { length(): never }).length() as never).ge(ctx.Real.val(1)) as unknown as Bool<"main">; + const len2B = (cB.len as unknown as { eq: (x: unknown) => Bool<"main"> }).eq(ctx.Real.val(2)); + const p0B = partType(cB, 0).eq(ctx.String.val("file")); + const q2 = smt2Of(ctx, [tNonEmptyB.and(len2B).and(p0B) as never], []); + const r2 = await solveSmt2(q2, []); + assert.equal(r2.status, "unsat", "no reversal exists for [text,file] — order preserved (the invariant is the judge)"); + + // ---- replay on the REAL codec (tsx runs the real imports incl. crypto) ---- + const wireFileFirst = [ + { + role: "user" as const, + content: [ + { type: "file" as const, file_id: "file_123" }, + { type: "text" as const, text: "hello world" }, + ], + }, + ]; + const core = openaiToCore({ messages: wireFileFirst } as never); + const rebuilt = coreToOpenai(core.msgs as never); + const c = (rebuilt[0] as { content: Array<{ type: string }> }).content; + assert.ok(Array.isArray(c), "rebuild emits a parts array when opaque parts ride along"); + assert.equal(c.length, 2, "both parts survive (never dropped, #1205)"); + assert.deepEqual( + c.map((p) => p.type), + ["text", "file"], + "REPLAY: [file,text] round-trips as [text,file] — part order REVERSED (#3676)", + ); +}); diff --git a/tests/verif-bug3691-oscillation.test.ts b/tests/verif-bug3691-oscillation.test.ts new file mode 100644 index 0000000000..7a17c3fa9f --- /dev/null +++ b/tests/verif-bug3691-oscillation.test.ts @@ -0,0 +1,123 @@ +// #3691 (kernel/src/truncate-tools.ts): the emergency clip has no persistent +// record — the decision to clip depends only on the per-turn token meter, so +// the same message renders clipped this turn and full next turn. Outbound +// bytes for an unchanged message flip-flop: a provider prefix-cache seam on +// every flip. This harness proves the mechanism WITHOUT pointing the solver +// at the bug: the symbolic phase only asks "can the clip fire at all under a +// gate-level meter?", then the REAL two-turn replay checks the general +// stability invariant (same message in ⇒ same bytes out across turns when the +// client resent identical history). The mitigation check pins that raising +// the threshold to 1.0 restores stability. +// +// No-cheat rules (#3848): entry functions are the real kernel exports; the +// property is general (byte-stability), never asserted at bug lines; the +// counterexample witness must replay on the real code. +import test from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.ts"; +import { newCtx, smt2Of } from "../verif/terms.ts"; +import { solveSmt2, parseNum } from "../verif/solve.ts"; +import { truncateLargeToolOutputs } from "../kernel/src/truncate-tools.ts"; +import { defaultCountTokens } from "../kernel/src/tokenize.ts"; + +const LIMIT = 2000; +const GATE = LIMIT * 0.95; // emergencyTruncateNode gate: usage >= threshold + +const realConfig = { modelContextLimit: LIMIT, truncate: { threshold: 0.95 } } as never; +const mkMsgs = (T: string): { id: string; role: "tool" | "user"; contentType: "tool-result" | "text"; toolCallId?: string; text: string }[] => [ + { id: "m0", role: "tool", contentType: "tool-result" as const, toolCallId: "tc0", text: T }, + { id: "m1", role: "user", contentType: "text" as const, text: "short 1" }, + { id: "m2", role: "user", contentType: "text" as const, text: "short 2" }, + { id: "m3", role: "user", contentType: "text" as const, text: "short 3" }, +]; + +test("bug3691: solver finds a clip-triggering witness on the real truncation code", { timeout: 240_000 }, async (t) => { + const ctx = newCtx(); + const T = ctx.String.const("T"); + const tok1 = ctx.Real.const("tok1"); + const msgs = { kind: "arr" as const, len: ctx.Real.val(4), elems: [ + { kind: "obj" as const, fields: new Map([ + ["id", { kind: "str" as const, t: ctx.String.val("m0") }], + ["role", { kind: "str" as const, t: ctx.String.val("tool") }], + ["contentType", { kind: "str" as const, t: ctx.String.val("tool-result") }], + ["toolCallId", { kind: "str" as const, t: ctx.String.val("tc0") }], + ["text", { kind: "str" as const, t: T }], + ]) }, + ...[1, 2, 3].map((i) => ({ kind: "obj" as const, fields: new Map([ + ["id", { kind: "str" as const, t: ctx.String.val(`m${i}`) }], + ["role", { kind: "str" as const, t: ctx.String.val("user") }], + ["contentType", { kind: "str" as const, t: ctx.String.val("text") }], + ["text", { kind: "str" as const, t: ctx.String.val(`short ${i}`) }], + ]) })), + ] }; + const config = { kind: "obj" as const, fields: new Map([ + ["modelContextLimit", { kind: "num" as const, t: ctx.Real.val(LIMIT) }], + ["truncate", { kind: "obj" as const, fields: new Map([ + ["threshold", { kind: "num" as const, t: ctx.Real.val(0.95) }], + ]) }], + ]) }; + const meter = (sv: { kind: string; t?: unknown }) => + sv.kind === "str" ? { kind: "num" as const, t: ctx.ToReal((sv.t as import("../verif/terms.js").Seq).length()).div(ctx.Real.val(4)).add(ctx.Real.val(1)) } : { kind: "any" as const }; + + const { exec } = translateFunction({ + file: new URL("../kernel/src/truncate-tools.ts", import.meta.url).pathname, + function: "truncateLargeToolOutputs", + params: { messages: msgs, tokenCount: { kind: "num", t: tok1 }, config, countTokens: { kind: "any" }, options: { kind: "obj", fields: new Map([["includeTextMessages", { kind: "bool", t: ctx.Bool.val(true) }]]) } }, + ctx, + options: { portImpls: { countTokens: meter } }, + }); + const out1 = exec(); + assert.deepEqual(out1.degradations, [], "translation must be degradation-free"); + const tc = (out1.ret as { kind: string; fields: Map }).fields.get("truncatedCount") as { kind: string; t: import("../verif/terms.js").Arith }; + assert.equal(tc.kind, "num", "truncatedCount must stay symbolic"); + + // Ask the solver: is there ANY input where the meter sits at the gate and + // the clip fires? (Pure existence — no bug-specific shaping beyond the + // scenario constraint "usage at the emergency threshold".) + const q1 = smt2Of(out1.ctx, [ + tok1.ge(ctx.Real.val(GATE)), + ctx.ToReal(T.length()).ge(ctx.Real.val(1)), + tc.t.ge(ctx.Real.val(1)), + ...out1.assumptions, + ]); + let res; + try { res = await solveSmt2(q1, ["(str.len T)", "tok1"]); } + catch (e) { t.skip(`solver unavailable: ${(e as Error).message.slice(0, 200)}`); return; } + assert.equal(res.status, "sat", `expected sat, got ${res.status}: ${res.raw.slice(0, 400)}`); + const lenT = parseNum(res.values[0]!); + const tokW = parseNum(res.values[1]!); + assert.ok(lenT >= 1, "witness must be a non-empty string"); + + // Negative control: below the gate the clip can NEVER fire (translation + // must not be able to produce phantom clips). + const q2 = smt2Of(out1.ctx, [ + tok1.lt(ctx.Real.val(GATE)), + tc.t.ge(ctx.Real.val(1)), + ...out1.assumptions, + ]); + const res2 = await solveSmt2(q2, []); + assert.equal(res2.status, "unsat", "clip must be unreachable below the gate"); + + // ---- Phase 2: replay the witness on the REAL code, two turns ---- + const Tstr = "x".repeat(lenT); + const r1 = truncateLargeToolOutputs(mkMsgs(Tstr) as never, Math.max(1, Math.round(tokW)), realConfig, defaultCountTokens, { includeTextMessages: true }); + assert.ok(r1.truncatedCount >= 1, `real code must clip with the witness (len=${lenT}, tok=${tokW})`); + + // Turn 2: the client resents the identical history; the provider meters + // turn-1's OUTBOUND (clipped) text. + const tok2 = defaultCountTokens(r1.messages[0]!.text ?? "") + 3; + const r2 = truncateLargeToolOutputs(mkMsgs(Tstr) as never, tok2, realConfig, defaultCountTokens, { includeTextMessages: true }); + const out1b = r1.messages[0]!.text!; + const out2 = r2.messages[0]!.text!; + assert.notEqual(out1b, out2, "same resent history must not produce different outbound bytes across turns — #3691 flip"); +}); + +test("bug3691 mitigation: threshold=1.0 makes the render stable across the same two turns", { timeout: 60_000 }, () => { + const Tstr = "x".repeat(4001); + const stable = { modelContextLimit: LIMIT, truncate: { threshold: 1.0 } } as never; + const r1 = truncateLargeToolOutputs(mkMsgs(Tstr) as never, 1900, stable, defaultCountTokens, { includeTextMessages: true }); + const tok2 = defaultCountTokens(r1.messages[0]!.text ?? "") + 3; + const r2 = truncateLargeToolOutputs(mkMsgs(Tstr) as never, tok2, stable, defaultCountTokens, { includeTextMessages: true }); + assert.equal(r1.messages[0]!.text, Tstr, "no clip with threshold=1"); + assert.equal(r2.messages[0]!.text, Tstr, "stable across turns"); +}); diff --git a/tests/verif-bug3769-orphan-window.test.ts b/tests/verif-bug3769-orphan-window.test.ts new file mode 100644 index 0000000000..4bd2888b85 --- /dev/null +++ b/tests/verif-bug3769-orphan-window.test.ts @@ -0,0 +1,240 @@ +/** + * Bug #3769 — hide-consumed orphan window breaks append-only stability. + * + * ORIGINAL bug (pre-#3784): kernel/src/hide-consumed.ts kept only the NEWEST + * KEEP_LAST_ORPHANED=2 orphan compress-call pairs visible; a THIRD failed + * pair silently dropped the OLDEST mid-history (issue #3769: 184,423 tokens + * re-billed, hit rate 97.7% → 19.2%). + * + * FIXED contract (#3784, absorbed into #3939 ①): orphan sweeping is + * FOLD-GATED — nothing hides between folds; the sweep rides the fold's own + * sanctioned rewrite (state.pendingOrphanSweepMinRef). This harness was + * re-baselined when the fix landed: the append-only flip must now be + * UNSAT (Q1), the mechanism must remain derivable THROUGH the fold marker + * only (Q1b sat) — proving the judge still sees the gated mechanism, not a + * vacuous truth — and non-compress appends stay stable (Q2 unsat). + * + * No-cheat contract (#3848 / user directive m00852): + * - we execute the REAL hideConsumedCompressCalls (translateFunction); + * - the invariant is a GENERAL hyperproperty: "a message that was visible + * must remain visible when the input history only GROWS (append-only), + * state unchanged" — nothing references the keep-window internals; + * - the solver searches over the appended pair's symbolic identity + * (toolName / toolCallId): the flip must be derivable, not asserted; + * - the counterexample replays on the real code. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { resolve } from "node:path"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of, strc } from "../verif/terms.js"; +import { solveSmt2, parseNum } from "../verif/solve.js"; +import { hideConsumedCompressCalls } from "../kernel/src/hide-consumed.js"; +import type { CoreMessage, CompressionState } from "../kernel/src/types.js"; + +const file = resolve(import.meta.dirname, "../kernel/src/hide-consumed.ts"); + +const asMsg = (m: CoreMessage): CoreMessage => m; + +// A: two orphan failed-compression pairs + surrounding user traffic. +// B: one MORE orphan pair, appended (nothing else changes). +const histA = (): CoreMessage[] => [ + asMsg({ id: "u0", role: "user", contentType: "text", text: "q1" }), + asMsg({ id: "t1", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c1", text: "{}" }), + asMsg({ id: "r1", role: "tool", contentType: "tool-result", toolCallId: "c1", text: "fail" }), + asMsg({ id: "t2", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c2", text: "{}" }), + asMsg({ id: "r2", role: "tool", contentType: "tool-result", toolCallId: "c2", text: "fail" }), +]; + +test("bug3769: solver proves append-only growth hides a previously-visible pair", { timeout: 240_000 }, async (t) => { + const solver = await (async () => { + try { return await solveSmt2("(set-logic ALL)(check-sat)", []); } catch { return null; } + })(); + if (!solver) return t.skip("no SMT solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + + // Symbolic identity of the appended pair. + const tn3 = ctx.String.const("tn3"); + const tc3 = ctx.String.const("tc3"); + + const mkSym = (id: string, role: string, ct: string, toolName: import("../verif/terms.js").Seq | string, toolCallId: import("../verif/terms.js").Seq | string, text: string) => { + const fields = new Map(); + fields.set("id", { kind: "str", t: ctx.String.val(id) }); + fields.set("role", { kind: "str", t: ctx.String.val(role) }); + fields.set("contentType", { kind: "str", t: ctx.String.val(ct) }); + fields.set("text", { kind: "str", t: ctx.String.val(text) }); + if (typeof toolName === "string") fields.set("toolName", { kind: "str", t: ctx.String.val(toolName) }); + else fields.set("toolName", { kind: "str", t: toolName }); + if (typeof toolCallId === "string") fields.set("toolCallId", { kind: "str", t: ctx.String.val(toolCallId) }); + else fields.set("toolCallId", { kind: "str", t: toolCallId }); + return { kind: "obj" as const, fields }; + }; + const msgs = (grown: boolean): import("../verif/symbolic.js").SymVal => { + const list: ReturnType[] = [ + mkSym("u0", "user", "text", "", "", "q1"), + mkSym("t1", "assistant", "tool-call", "compress", "c1", "{}"), + mkSym("r1", "tool", "tool-result", "", "c1", "fail"), + mkSym("t2", "assistant", "tool-call", "compress", "c2", "{}"), + mkSym("r2", "tool", "tool-result", "", "c2", "fail"), + ]; + if (grown) { + list.push(mkSym("t3", "assistant", "tool-call", tn3, tc3, "{}")); + list.push(mkSym("r3", "tool", "tool-result", "", tc3, "fail")); + } + return { kind: "arr", elems: list, len: ctx.Real.val(list.length) }; + }; + const stateObj = (foldGate: boolean): import("../verif/symbolic.js").SymVal => { + const fields = new Map(); + fields.set("blocks", { kind: "arr", elems: [], len: ctx.Real.val(0) }); + // real state shape: messageRefs = { byRaw: Record } + const byRawInner = new Map(); + if (foldGate) { + // A fold just committed: refs are minted and the sweep is pending from + // the fold's own min ref (m00001) — the sanctioned rewrite window. + byRawInner.set("t1", { kind: "str", t: ctx.String.val("m00001") }); + byRawInner.set("r1", { kind: "str", t: ctx.String.val("m00002") }); + byRawInner.set("t2", { kind: "str", t: ctx.String.val("m00003") }); + byRawInner.set("r2", { kind: "str", t: ctx.String.val("m00004") }); + byRawInner.set("t3", { kind: "str", t: ctx.String.val("m00005") }); + byRawInner.set("r3", { kind: "str", t: ctx.String.val("m00006") }); + fields.set("pendingOrphanSweepMinRef", { kind: "num", t: ctx.Real.val(1) }); + } + fields.set("messageRefs", { kind: "obj", fields: new Map([["byRaw", { kind: "obj", fields: byRawInner }]]) }); + return { kind: "obj", fields }; + }; + + const run = (grown: boolean, foldGate = false) => { + const { exec } = translateFunction({ + file, function: "hideConsumedCompressCalls", + params: { state: stateObj(foldGate), messages: msgs(grown), isCompressTool: { kind: "undef" } }, + ctx, + options: { + portImpls: { + // injectable classifier port (default param in source): symbolic + // membership test — the harness does NOT hardcode the outcome + isCompressTool: (sv: any) => ({ kind: "bool", t: sv.t.eq(strc(ctx, "compress")) }) as any, + }, + }, + }); + const r = exec(); + assert.deepEqual(r.degradations, [], `no degradations expected (got: ${r.degradations.join("; ")})`); + assert.ok(r.ret && r.ret.kind === "obj", "result object"); + return r as import("../verif/symbolic.js").ExecOut; + }; + const outA = run(false); + const outB = run(true); + const outGate = run(true, true); + + const arrIds = (o: import("../verif/symbolic.js").ExecOut) => { + const ret = o.ret as Extract; + const out = ret.fields.get("messages"); + assert.ok(out && out.kind === "arr", "messages array in result"); + return out as Extract; + }; + const hasR1 = (a: Extract) => { + let acc: import("../verif/terms.js").Bool | null = null; + for (let k = 0; k < a.elems.length; k++) { + const e = a.elems[k]!; + const idv = e.kind === "obj" ? e.fields.get("id") : undefined; + const hit = (idv && idv.kind === "str" ? idv.t : ctx.String.val("")).eq(ctx.String.val("r1")); + const inLen = a.len.gt(ctx.Real.val(k)); + const term = inLen.and(hit); + acc = acc ? acc.or(term) : term; + } + return acc!; + }; + const visibleBefore = hasR1(arrIds(outA)); + const visibleAfter = hasR1(arrIds(outB)); + + // Q1 (fix holds): with the third pair a genuine compress pair (fresh id), + // append-only growth alone can NO LONGER hide the previously-visible r1 — + // the sweep is fold-gated (#3784). No counterexample exists. + const q1 = smt2Of(outB.ctx, [ + tn3.eq(ctx.String.val("compress")), + tc3.eq(ctx.String.val("c1")).not(), + tc3.eq(ctx.String.val("c2")).not(), + visibleBefore, + visibleAfter.not(), + ...outA.assumptions, ...outB.assumptions, + ]); + const r1 = await solveSmt2(q1, ["tn3", "tc3"]); + assert.equal(r1.status, "unsat", "append-only growth must not hide anything after the #3784 fold gate"); + + // Q1b (mechanism still reachable, only through the fold): the same grown + // history under a pending fold sweep (pendingOrphanSweepMinRef minted refs) + // DOES hide r1 — the rewrite is sanctioned to ride the fold boundary. + // SAT here proves the judge is not vacuous: the gated path is derivable. + const gateHidden = hasR1(arrIds(outGate)); + const q1b = smt2Of(outGate.ctx, [ + tn3.eq(ctx.String.val("compress")), + tc3.eq(ctx.String.val("c1")).not(), + tc3.eq(ctx.String.val("c2")).not(), + visibleBefore, + gateHidden.not(), + ...outA.assumptions, ...outGate.assumptions, + ]); + const r1b = await solveSmt2(q1b, ["tn3", "tc3"]); + assert.equal(r1b.status, "sat", "the fold-gated sweep must remain derivable through the fold marker"); + + // Q2 (negative control): when the appended pair is NOT a compress call, + // the same invariant holds — no counterexample exists. + const q2 = smt2Of(outB.ctx, [ + tn3.eq(ctx.String.val("compress")).not(), + visibleBefore, + visibleAfter.not(), + ...outA.assumptions, ...outB.assumptions, + ]); + const r2 = await solveSmt2(q2, []); + assert.equal(r2.status, "unsat", "non-compress appends must not hide anything"); + + // Replay on the REAL code: same shape, concrete distinct ids. + const state = { blocks: [], messageRefs: { byRaw: {} } } as unknown as CompressionState; + const a = hideConsumedCompressCalls(state, histA()); + assert.equal(a.messages.length, 5, "all five messages visible before the third pair"); + assert.ok(a.messages.some((m) => m.id === "r1"), "r1 visible before"); + const b = hideConsumedCompressCalls(state, [ + ...histA(), + asMsg({ id: "t3", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c3", text: "{}" }), + asMsg({ id: "r3", role: "tool", contentType: "tool-result", toolCallId: "c3", text: "fail" }), + ]); + assert.equal(b.hidden, 0, "#3784: appending an orphan pair no longer sweeps anything between folds"); + assert.ok(b.messages.some((m) => m.id === "r1"), "r1 STAYS visible after the append — the #3769 flip is gone"); + assert.equal(b.newlySweptCallIds.length, 0, "nothing newly swept without a pending fold"); + + // Replay the sanctioned side: a pending fold sweep (refs minted, min ref 1) + // hides the orphan pairs in one pass — the rewrite rides the fold. + const gateState = { + blocks: [], + messageRefs: { byRaw: { t1: "m00001", r1: "m00002", t2: "m00003", r2: "m00004", t3: "m00005", r3: "m00006" } }, + pendingOrphanSweepMinRef: 1, + } as unknown as CompressionState; + const c = hideConsumedCompressCalls(gateState, [ + ...histA(), + asMsg({ id: "t3", role: "assistant", contentType: "tool-call", toolName: "compress", toolCallId: "c3", text: "{}" }), + asMsg({ id: "r3", role: "tool", contentType: "tool-result", toolCallId: "c3", text: "fail" }), + ]); + assert.ok(!c.messages.some((m) => m.id === "r1"), "r1 hidden AT the fold boundary — the gated sweep fires"); + assert.deepEqual(c.newlySweptCallIds, ["c1", "c2", "c3"], "all three orphan pairs ride the fold sweep"); + void solverBool; +}); + +// helper: print a z3 Bool in smt2 form via its solver-companion — +// the translateFunction result is in ctx "main", so its .toString() IS smt2. +const solverBool = (b: import("../verif/terms.js").Bool): string => b.toString(); + +test("bug3769 counter-test: with a larger window the same append is stable (shows the invariant is the judge, not the shape)", { timeout: 240_000 }, async (t) => { + // Only TWO orphan pairs exist and the appended traffic is user text (not a + // compress call) — visibility must be stable under the fold gate too. + const state = { blocks: [], messageRefs: { byRaw: {} } } as unknown as CompressionState; + const before = hideConsumedCompressCalls(state, histA()); + const after = hideConsumedCompressCalls(state, [ + ...histA(), + asMsg({ id: "u9", role: "user", contentType: "text", text: "q2" }), + ]); + const idsAfter = after.messages.map((m) => m.id); + assert.deepEqual(idsAfter.slice(0, before.messages.length), before.messages.map((m) => m.id), + "non-compress appends never change visibility of the existing prefix"); + assert.ok(after.messages.some((m) => m.id === "r1")); + void parseNum; // keep import used in both branches +}); diff --git a/tests/verif-bug3805-durable-ref-migration.test.ts b/tests/verif-bug3805-durable-ref-migration.test.ts new file mode 100644 index 0000000000..bcb8d8ed17 --- /dev/null +++ b/tests/verif-bug3805-durable-ref-migration.test.ts @@ -0,0 +1,233 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of } from "../verif/terms.js"; +import type { Bool, Term } from "../verif/terms.js"; +import { solveSmt2 } from "../verif/solve.js"; +import { assignRefs, emptyRefMap, BLOCKED_REF } from "../kernel/src/refs.js"; +import { renderVisibleRefs } from "../kernel/src/render-refs.js"; +import type { CoreMessage } from "../kernel/src/types.js"; + +/** + * #3805: durable carrier ref migration re-tags an already-sent message. + * + * Mechanism (all in guarded kernel code): + * - refs.ts assignRefs: a message whose previous ref is BLOCKED (it was + * protected last turn) and whose protection has lifted gets the BLOCKED + * entry DELETED and a fresh numeric ref allocated. + * - render-refs.ts renderMessage: BLOCKED renders verbatim; a numeric ref + * renders as an "m00001\n" tag PREFIX. + * - The durable-protection window is "latest carrier only" + * (collectLatestProtected + durable-message-guards.ts classifier + * `text.includes("")`): when a NEWER carrier for the + * same identity arrives, the OLDER one's protection lifts — mid-history. + * + * Invariant under test (general, #3848-style I1 tail-only growth): + * appending NEWER messages must not change the rendered bytes of an + * already-sent message. + * + * No-cheat composition: assignRefs + renderVisibleRefs are the REAL + * translated kernel functions. The only injected seams are + * - isProtected (the durable classifier + latest-wins window — its real + * impl lives outside the guarded kernel subset; modeled exactly as + * "text contains the skill marker" ∧ "is the newest carrier"), + * - countTokens (meter, len/4+1), + * - the dynamic ownTag regex is over-approximated by the engine's + * anchored-prefix rule (documented in verif/README). + * The replay phase runs the REAL functions end-to-end. + */ + +type Ctx = ReturnType; +type SymStr = { kind: "str"; t: Term }; +type SymBool = { kind: "bool"; t: Bool<"main"> }; +type SymObj = { kind: "obj"; fields: Map }; +type SymArr = { kind: "arr"; elems: any[]; len: any }; +type SymVal = SymStr | SymBool | SymObj | SymArr | { kind: string; t?: unknown }; + +const strv = (ctx: Ctx, v: string): SymStr => ({ kind: "str", t: ctx.String.val(v) }) as never; +const numv = (ctx: Ctx, v: number): { kind: "num"; t: unknown } => ({ kind: "num", t: ctx.Real.val(v) }) as never; +const boolv = (ctx: Ctx, v: boolean): SymBool => ({ kind: "bool", t: ctx.Bool.val(v) }) as never; +const objOf = (fields: Record): SymObj => ({ kind: "obj", fields: new Map(Object.entries(fields)) }) as never; +const arrOf = (ctx: Ctx, elems: SymVal[]): SymArr => ({ kind: "arr", elems, len: ctx.Real.val(elems.length) }) as never; + +const MARKER = ""; +const REPO = new URL("..", import.meta.url).pathname; +const REFS = `${REPO}kernel/src/refs.ts`; +const RREFS = `${REPO}kernel/src/render-refs.ts`; + +function coreMsg(ctx: Ctx, id: string, text: SymStr): SymObj { + return objOf({ + id: strv(ctx, id), + role: strv(ctx, "user"), + contentType: strv(ctx, "text"), + text, + }); +} + +function meterPort(ctx: Ctx) { + return (text: SymVal): { kind: "num"; t: unknown } => { + if (text && (text as SymStr).kind === "str") { + const len = ctx.ToReal((text as SymStr).t.length() as never); + return { kind: "num", t: (len as unknown as { div(x: unknown): { add(y: unknown): unknown } }).div(ctx.Real.val(4)).add(ctx.Real.val(1)) } as never; + } + return { kind: "num", t: ctx.Real.val(1) } as never; + }; +} + +interface TurnArtifacts { + mapByRaw: SymObj; + mapByRef: SymObj; + nextIndex: unknown; + rendered0: SymStr; + assumptions: any[]; + degradations: string[]; +} + +/** One full turn: assignRefs then renderVisibleRefs — both REAL translated + * kernel functions. `protected1`/`protected2` are the symbolic protection + * bits for m1/m2 (the durable classifier, injected as the isProtected port). */ +function turn( + ctx: Ctx, + msgs: SymVal, + existing: { byRaw: SymObj; byRef: SymObj } | null, + nextIndex: unknown, + protected1: SymBool, + protected2: SymBool, +): TurnArtifacts { + const emptyRec = objOf({}); + const options = objOf({ + existing: objOf({ byRaw: existing ? existing.byRaw : emptyRec, byRef: existing ? existing.byRef : emptyRec }), + nextIndex: numv(ctx, typeof nextIndex === "number" ? nextIndex : 1), + }); + const portsA = { + __method_isProtected: (_recv: SymVal, m: SymVal): SymBool => { + const id = (m as SymObj).fields.get("id") as SymStr | undefined; + const idc = id && typeof id.t.c === "string" ? id.t.c : ""; + if (idc === "u1") return protected1; + return protected2; + }, + countTokens: meterPort(ctx), + }; + const outA = translateFunction({ + file: REFS, function: "assignRefs", + params: { messages: msgs, options } as never, + ctx, options: { portImpls: portsA as never }, + }).exec(); + const retA = outA.ret as SymObj; + const mapA = retA.fields.get("map") as SymObj; + + const state = objOf({ messageRefs: mapA }); + const outR = translateFunction({ + file: RREFS, function: "renderVisibleRefs", + params: { messages: msgs, state } as never, + ctx, options: { portImpls: { countTokens: meterPort(ctx) } as never }, + }).exec(); + const retR = outR.ret as SymArr; + const rendered0 = retR.elems[0] as SymObj; + return { + mapByRaw: mapA.fields.get("byRaw") as SymObj, + mapByRef: mapA.fields.get("byRef") as SymObj, + nextIndex: retA.fields.get("nextIndex"), + rendered0: rendered0.fields.get("text") as SymStr, + assumptions: outA.assumptions ?? [], + degradations: [...(outA.degradations ?? []), ...(outR.degradations ?? [])], + }; +} + +function runSymbolic(ctx: Ctx) { + const T1 = ctx.String.const("T1"); + const T2 = ctx.String.const("T2"); + const marker = ctx.String.val(MARKER); + const d1: SymBool = { kind: "bool", t: T1.contains(marker) } as never; + const d2: SymBool = { kind: "bool", t: T2.contains(marker) } as never; + + const m1 = coreMsg(ctx, "u1", { kind: "str", t: T1 } as never); + const m2 = coreMsg(ctx, "u2", { kind: "str", t: T2 } as never); + + // Turn A: only m1. Protection = "is the newest durable carrier" = d1. + const A = turn(ctx, arrOf(ctx, [m1 as SymVal]), null, 1, d1, boolv(ctx, false)); + // Turn B: m2 appended. Latest-wins: m1 protected iff d1 ∧ ¬d2; m2 iff d2. + const notd2: SymBool = { kind: "bool", t: d2.t.not() } as never; + const keep1: SymBool = { kind: "bool", t: d1.t.and(notd2.t) } as never; + const B = turn(ctx, arrOf(ctx, [m1 as SymVal, m2 as SymVal]), { byRaw: A.mapByRaw, byRef: A.mapByRef }, A.nextIndex, keep1, d2); + return { T1, T2, d1, d2, A, B }; +} + +test("bug3805: solver finds the mid-history re-tag when protection lifts (append-only bytes must stay stable)", { timeout: 240_000 }, async (t) => { + const haveSolver = await (async () => { try { return solveSmt2("(set-logic ALL)(check-sat)", []).status === "sat"; } catch { return false; } })(); + if (!haveSolver) return t.skip("no solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + const { T1, T2, d1, d2, A, B } = runSymbolic(ctx); + assert.deepEqual(A.degradations, [], `A degradations: ${JSON.stringify(A.degradations)}`); + assert.deepEqual(B.degradations, [], `B degradations: ${JSON.stringify(B.degradations)}`); + assert.equal(A.rendered0.kind, "str"); + assert.equal(B.rendered0.kind, "str"); + + // Property: appending m2 changed m1's rendered bytes (mid-history seam). + const differ = (A.rendered0 as SymStr).t.neq((B.rendered0 as SymStr).t); + const q1 = smt2Of(ctx, [...(A.assumptions as never[]), ...(B.assumptions as never[]), d1.t, d2.t, differ] as never[], ["str.len T1", "str.len T2"]); + const r1 = await solveSmt2(q1, []); + assert.equal(r1.status, "sat", "solver must find the re-tag witness (d1 ∧ d2 → flip)"); + + // Negative control: m2 is NOT a durable carrier → m1 keeps its protection + // (or never had one) → bytes stay stable. + const q2 = smt2Of(ctx, [...(A.assumptions as never[]), ...(B.assumptions as never[]), d1.t, d2.t.not(), differ] as never[], []); + const r2 = await solveSmt2(q2, []); + assert.notEqual(r2.status, "sat", "no byte change may occur when the appended message is not a durable carrier"); + + // ---- replay on the REAL code (witness semantics: both carry the marker) ---- + const realCount = (text: string) => Math.ceil(text.length / 4) + 1; + const t1r = MARKER + "\nskill catalog v1 (some body)"; + const t2r = MARKER + "\nskill catalog v2 (some body)"; + const mk = (id: string, text: string): CoreMessage => + ({ id, role: "user", contentType: "text", text }) as unknown as CoreMessage; + + // Turn A: m1 is the newest (only) durable carrier → protected → BLOCKED. + const rA = assignRefs([mk("u1", t1r)], { + existing: emptyRefMap(), nextIndex: 1, + isProtected: () => true, + }); + assert.equal(rA.map.byRaw.u1, BLOCKED_REF, "protected carrier renders verbatim (BLOCKED)"); + const renderA = renderVisibleRefs([mk("u1", t1r)], { messageRefs: rA.map } as never, realCount, "all"); + assert.equal(renderA[0]!.text, t1r, "turn A: no tag (BLOCKED → verbatim)"); + + // Turn B: a NEWER carrier for the same identity arrives → m1's protection + // lifts → BLOCKED deleted → fresh numeric ref → tag prefix mid-history. + const rB = assignRefs([mk("u1", t1r), mk("u2", t2r)], { + existing: rA.map, nextIndex: rA.nextIndex, + isProtected: (m) => m.id === "u2", + }); + assert.notEqual(rB.map.byRaw.u1, BLOCKED_REF, "older carrier lost protection (latest-wins window)"); + const renderB = renderVisibleRefs([mk("u1", t1r), mk("u2", t2r)], { messageRefs: rB.map } as never, realCount, "all"); + const t1After: string = renderB[0]!.text ?? ""; + assert.notEqual(t1After, t1r, "#3805 seam: already-sent message re-tagged mid-history"); + assert.ok(t1After.startsWith(" { + const haveSolver = await (async () => { try { return solveSmt2("(set-logic ALL)(check-sat)", []).status === "sat"; } catch { return false; } })(); + if (!haveSolver) return t.skip("no solver available"); + + const ctx = newCtx(); + const { T1, T2, d1, A, B } = runSymbolic(ctx); + void T2; + // m2 without the marker: m1's protection never lifts. + const same = (A.rendered0 as SymStr).t.eq((B.rendered0 as SymStr).t); + const q = smt2Of(ctx, [...(A.assumptions as never[]), ...(B.assumptions as never[]), d1.t, same] as never[], []); + const r = await solveSmt2(q, []); + assert.equal(r.status, "sat", "stable model exists (d1 ∧ ¬d2 → identical bytes)"); + + // replay: plain user append + const realCount = (text: string) => Math.ceil(text.length / 4) + 1; + const t1r = MARKER + "\nskill catalog v1"; + const t2r = "just a plain user message"; + const mk = (id: string, text: string): CoreMessage => + ({ id, role: "user", contentType: "text", text }) as unknown as CoreMessage; + const rA = assignRefs([mk("u1", t1r)], { existing: emptyRefMap(), nextIndex: 1, isProtected: () => true }); + const rB = assignRefs([mk("u1", t1r), mk("u2", t2r)], { existing: rA.map, nextIndex: rA.nextIndex, isProtected: (m) => m.id === "u1" }); + assert.equal(rB.map.byRaw.u1, BLOCKED_REF, "protection persists for a non-durable append"); + const a = renderVisibleRefs([mk("u1", t1r)], { messageRefs: rA.map } as never, realCount, "all")[0]!.text; + const b = renderVisibleRefs([mk("u1", t1r), mk("u2", t2r)], { messageRefs: rB.map } as never, realCount, "all")[0]!.text; + assert.equal(a, b, "bytes stable"); +}); diff --git a/tests/verif-generic-append-stability.test.ts b/tests/verif-generic-append-stability.test.ts new file mode 100644 index 0000000000..22c36a01b2 --- /dev/null +++ b/tests/verif-generic-append-stability.test.ts @@ -0,0 +1,204 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of } from "../verif/terms.js"; +import type { Seq } from "../verif/terms.js"; +import { solveSmt2 } from "../verif/solve.js"; +import { assignRefs } from "../kernel/src/refs.js"; +import { renderVisibleRefs } from "../kernel/src/render-refs.js"; +import { defaultCountTokens } from "../kernel/src/tokenize.js"; +import type { CoreMessage } from "../kernel/src/types.js"; + +/** + * Generic arbitrary-input discovery (issue #3848 acceptance, directive: find + * bugs under ARBITRARY inputs — not scenario-shaped harnesses). + * + * Input model: EVERYTHING symbolic. Message ids, roles, contentTypes and + * texts are free string constants; protection flags are free booleans per + * turn. Nothing encodes a known bug shape. + * + * Invariant under test (I1, append-only stability): when the host resends + * the same history with one more message appended (turn A → turn B), the + * rendered bytes of every message that was already visible in turn A must + * be IDENTICAL in turn B. + * + * Q1: violation reachable? (expect SAT — at minimum the known #3805 + * durable-ref migration re-emerges from the general model) + * Q2: violation reachable WITHOUT any protection dropping? + * (p_i → q_i for all i). Expect UNSAT: no NEW violation shapes within + * bounds (≤3 messages, text length ≤ 200, unroll bounds). SAT here = + * a candidate NEW bug → must replay on real code. + */ + +const REPO = new URL("..", import.meta.url).pathname; +const REFS = `${REPO}kernel/src/refs.ts`; +const RENDER_REFS = `${REPO}kernel/src/render-refs.ts`; + +type SymVal = Record; + +interface TurnOut { + rendered: SymVal[]; + assumptions: unknown[]; + ctx: ReturnType; + degradations: string[]; +} + +test("generic append-only stability: arbitrary inputs over assignRefs + renderVisibleRefs", { timeout: 300_000 }, async (t) => { + const haveSolver = (() => { try { return solveSmt2("(set-logic ALL)(check-sat)", []).status === "sat"; } catch { return false; } })(); + if (!haveSolver) t.skip("no SMT solver available (set VERIF_PYTHON or VERIF_SOLVER_CMD)"); + + const ctx = newCtx(); + const strv = (v: string) => ({ kind: "str", t: ctx.String.val(v) }) as never; + const numv = (n: number) => ({ kind: "num", t: ctx.Real.val(n) }) as never; + const boolv = (b: boolean) => ({ kind: "bool", t: ctx.Bool.val(b) }) as never; + const objOf = (f: Record) => ({ kind: "obj", fields: new Map(Object.entries(f)) }) as never; + const arrOf = (elems: unknown[]) => ({ kind: "arr", elems, len: ctx.Real.val(elems.length) }) as never; + + // Arbitrary messages: every field a free symbolic constant. + const mkMsg = (tag: string) => objOf({ + id: { kind: "str", t: ctx.String.const(`${tag}_id`) }, + role: { kind: "str", t: ctx.String.const(`${tag}_role`) }, + contentType: { kind: "str", t: ctx.String.const(`${tag}_ct`) }, + text: { kind: "str", t: ctx.String.const(`${tag}_text`) }, + }); + const m1 = mkMsg("m1"), m2 = mkMsg("m2"), m3 = mkMsg("m3"); + // Texts non-empty and reasonably short (solver bound, not a shape). + const textLenBound = (name: string): unknown => { + const x = ctx.String.const(name) as Seq<"main">; + const lenR = ctx.ToReal(x.length() as never) as unknown as { ge(a: unknown): { and(b: unknown): unknown }; le(a: unknown): unknown }; + return lenR.ge(ctx.Real.val(1)).and(lenR.le(ctx.Real.val(200))); + }; + const p1 = ctx.Bool.const("p1") as { implies(x: unknown): { and(b: unknown): unknown } }, p2 = ctx.Bool.const("p2") as { implies(x: unknown): unknown }; + const q1 = ctx.Bool.const("q1"), q2 = ctx.Bool.const("q2"), q3 = ctx.Bool.const("q3"); + void q3; + + const meterPort = (a: unknown) => { + const t = (a as { t: { length(): unknown } }).t; + const r = ctx.ToReal(t.length() as never) as unknown as { div(x: unknown): { add(y: unknown): unknown } }; + return { kind: "num", t: r.div(ctx.Real.val(4)).add(ctx.Real.val(1)) } as never; + }; + const protFor = (table: [SymVal, unknown][]) => (_recv: unknown, m: unknown) => { + // The port receives the MESSAGE object; match by SymVal identity — obj + // values carry no .t, an earlier .t-based compare never matched and + // silently pinned every protection to false (Q1 went UNSAT). + for (const [want, val] of table) if (want === (m as SymVal)) return { kind: "bool", t: val }; + return { kind: "bool", t: ctx.Bool.val(false) }; + }; + + const emptyMap = objOf({ byRaw: objOf({}), byRef: objOf({}) }); + + const runAssign = (msgs: unknown[], existing: unknown, nextIndex: unknown, prot: unknown) => + translateFunction({ + file: REFS, function: "assignRefs", + params: { messages: arrOf(msgs), options: objOf({ existing, nextIndex }) } as never, + ctx, + options: { portImpls: { __method_isProtected: prot as never, countTokens: meterPort } } as never, + }).exec(); + + const runRender = (msgs: unknown[], state: unknown) => + translateFunction({ + file: RENDER_REFS, function: "renderVisibleRefs", + params: { messages: arrOf(msgs), state, countTokens: numv(0), strategy: strv("text-only") } as never, + ctx, + options: { portImpls: { countTokens: meterPort } } as never, + }).exec(); + + // ---- turn A: [m1, m2] ---- + const outA = runAssign([m1, m2], emptyMap, numv(1), protFor([[m1, p1], [m2, p2]])); + assert.deepEqual(outA.degradations, [], `turn-A assignRefs degradations: ${JSON.stringify(outA.degradations)}`); + const mapA = (outA.ret as { fields: Map }).fields.get("map"); + const nextA = (outA.ret as { fields: Map }).fields.get("nextIndex"); + const stateA = objOf({ messageRefs: mapA }); + const rendA = runRender([m1, m2], stateA as never); + assert.deepEqual(rendA.degradations, [], `turn-A render degradations: ${JSON.stringify(rendA.degradations)}`); + const rendAArr = (rendA.ret as { elems: SymVal[] }).elems; + + // ---- turn B: same history + m3 appended ---- + const outB = runAssign([m1, m2, m3], mapA, nextA, protFor([[m1, q1], [m2, q2], [m3, q3]])); + assert.deepEqual(outB.degradations, [], `turn-B assignRefs degradations: ${JSON.stringify(outB.degradations)}`); + const mapB = (outB.ret as { fields: Map }).fields.get("map"); + const stateB = objOf({ messageRefs: mapB }); + const rendB = runRender([m1, m2, m3], stateB as never); + assert.deepEqual(rendB.degradations, [], `turn-B render degradations: ${JSON.stringify(rendB.degradations)}`); + const rendBArr = (rendB.ret as { elems: SymVal[] }).elems; + + // ---- I1 property: rendered[i] turn A == rendered[i] turn B, i ∈ {0,1} ---- + const textOf = (m: SymVal): unknown => (m.fields as Map).get("text"); + const valEqStr = (a: unknown, b: unknown): unknown => (a as { t: { eq(x: unknown): unknown } }).t.eq((b as { t: unknown }).t); + const stable0 = valEqStr(textOf(rendAArr[0]!), textOf(rendBArr[0]!)) as { not(): { or(b: unknown): unknown } }; + const stable1 = valEqStr(textOf(rendAArr[1]!), textOf(rendBArr[1]!)) as { not(): unknown }; + const violation = stable0.not().or(stable1.not()) as unknown; + + const baseAsserts = [ + textLenBound("m1_text"), textLenBound("m2_text"), textLenBound("m3_text"), + ...outA.assumptions, ...rendA.assumptions, ...outB.assumptions, ...rendB.assumptions, + ] as never[]; + + // Q1: can ANY arbitrary input violate append-only stability? + const names = ["p1", "p2", "q1", "q2", "q3"]; + const q1res = solveSmt2(smt2Of(ctx, [...baseAsserts, violation] as never[], names as never), []); + assert.equal(q1res.status, "sat", "Q1 expected SAT — append-only violation must be reachable under arbitrary inputs"); + const w: Record = {}; + (q1res.values ?? []).forEach((v, i) => { w[names[i]!] = v; }); + + // Q2: violation without ANY protection drop (p_i → q_i)? SAT here found a + // NEW shape the scenario harnesses never covered: m3 reusing m1's raw id + // while being unprotected — the per-id BLOCKED binding is shared but + // protection is per-message, so the colliding message steals the binding + // (refs.ts:99 delete + fresh alloc re-tags m1's already-sent bytes). + const noDrop = p1.implies(q1).and(p2.implies(q2)); + const q2res = solveSmt2(smt2Of(ctx, [...baseAsserts, noDrop, violation] as never[], [] as never), []); + assert.equal(q2res.status, "sat", "Q2 expected SAT — the id-collision re-tag must be reachable without protection drops"); + // Q3: with pairwise-distinct ids and no protection drop, NO violation may + // remain within bounds (bounded absence — the known shapes are gone). + const idt = (n: string) => ctx.String.const(n); + const distinct = idt("m1_id").neq(idt("m2_id")).and(idt("m1_id").neq(idt("m3_id"))).and(idt("m2_id").neq(idt("m3_id"))); + const q3res = solveSmt2(smt2Of(ctx, [...baseAsserts, noDrop, distinct, violation] as never[], [] as never), []); + assert.equal(q3res.status, "unsat", "Q3 expected UNSAT — nothing beyond protection-drop and id-collision shapes within bounds"); + + // ---- replay on REAL code with a witness-shaped input ---- + // Interpret witness booleans through the real durable classifier semantics + // (latest marker-carrier wins): p=true ⇒ carries "". + const marker = ""; + const textFor = (tag: string, prot: boolean | undefined) => + (prot ? `${marker} catalog ${tag}` : `plain ${tag} body`); + const bool = (v: unknown): boolean => { + if (v === "true") return true; + if (v === "false") return false; + if (typeof v === "string" && (v === "1" || v === "0")) return v === "1"; + return false; + }; + const p1w = bool(w.p1), p2w = bool(w.p2), q1w = bool(w.q1), q2w = bool(w.q2), q3w = bool(w.q3); + const rmsg = (tag: string, prot: boolean): CoreMessage => ({ + id: `${tag}-real`, role: "user", contentType: "text", text: textFor(tag, prot), + } as CoreMessage); + const turnProt = (table: Record) => (m: { id: string }) => table[m.id] === true; + // Real code, turn A then turn B, threading map/nextIndex like the proxy does. + const fresh = { byRaw: {} as Record, byRef: {} as Record }; + const a1 = assignRefs([rmsg("m1", p1w), rmsg("m2", p2w)], { existing: fresh, nextIndex: 1, isProtected: turnProt({ "m1-real": p1w, "m2-real": p2w }) }); + const ra = renderVisibleRefs([rmsg("m1", p1w), rmsg("m2", p2w)], { messageRefs: a1.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + const a2 = assignRefs([rmsg("m1", q1w), rmsg("m2", q2w), rmsg("m3", q3w)], { existing: a1.map, nextIndex: a1.nextIndex, isProtected: turnProt({ "m1-real": q1w, "m2-real": q2w, "m3-real": q3w }) }); + const rb = renderVisibleRefs([rmsg("m1", q1w), rmsg("m2", q2w), rmsg("m3", q3w)], { messageRefs: a2.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + // The witness must ACTUALLY violate on real code (replay lock) when its + // protection pattern is realizable; protection-drop lifts (#3805 shape) + // are the expected realization. + const realFlip = ra[0]!.text !== rb[0]!.text || ra[1]!.text !== rb[1]!.text; + const lifted = (p1w && !q1w) || (p2w && !q2w); + assert.ok(realFlip === true || lifted === false, + `witness p=${p1w},${p2w} q=${q1w},${q2w},${q3w} replayFlip=${realFlip} lifted=${lifted}`); + // Q2 shape replayed for real: duplicate raw id, protection NOT dropped. + const mkDup = (id: string, text: string, prot: boolean): CoreMessage => + ({ id, role: "user", contentType: "text", text: prot ? `${marker} catalog ${id}` : text } as CoreMessage); + const d1 = mkDup("C", "marker body catalog", true); + const d2 = mkDup("B", "plain tool payload", false); + const d3 = mkDup("C", "A", false); + const dupProt = (m: { id: string; text?: string }) => m.text !== undefined && m.text.includes(marker) && m.id === "C"; + const dupA = assignRefs([d1, d2], { existing: { byRaw: {}, byRef: {} }, nextIndex: 1, isProtected: dupProt as never }); + const dupRA = renderVisibleRefs([d1, d2], { messageRefs: dupA.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + const dupB = assignRefs([d1, d2, d3], { existing: dupA.map, nextIndex: dupA.nextIndex, isProtected: dupProt as never }); + const dupRB = renderVisibleRefs([d1, d2, d3], { messageRefs: dupB.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + assert.ok(dupRA[0]!.text !== dupRB[0]!.text, + "id-collision witness must flip real code: protected m1 verbatim in A, re-tagged in B"); + assert.equal(dupA.map.byRaw.C, "BLOCKED"); + assert.notEqual(dupB.map.byRaw.C, "BLOCKED"); +}); diff --git a/tests/verif-induction-append-stability.test.ts b/tests/verif-induction-append-stability.test.ts new file mode 100644 index 0000000000..8ffebecc42 --- /dev/null +++ b/tests/verif-induction-append-stability.test.ts @@ -0,0 +1,146 @@ +/** + * k-induction judge for append-only send-byte stability (#3848, #3945 follow-up). + * + * Claim (for ARBITRARY message count n): + * If turn k's outbound bytes for messages [1..n] were derived from map M_n, + * and the host only APPENDS message n+1 (prefix byte-identical, same ids), + * then turn k+1's rendered bytes for [1..n] are unchanged — PROVIDED the + * appended message does not REUSE a raw id already present in M_n + * (the #3945 id-collision shape is the one real counterexample). + * + * Why this is induction, not another bounded scan: + * - The prefix is ABSTRACTED to its effect: a symbolic map M with one + * observed binding (id_p -> r_p). No 1..n loop, no per-message unroll — + * formula size is independent of n. + * - Step: run the REAL translated assignRefs([m], M) and prove the observed + * binding is untouched when m.id != id_p. UNSAT = step holds for every M, + * reachable or not (weaker premise than reachability — the abstraction + * characterization is the part we deliberately skip; it cannot weaken the + * conclusion, only ever strengthen it). + * - Render side: renderVisibleRefs renders each message independently + * (messages.map(renderMessage) over the legacy path) and rendered bytes + * are a pure function of (message fields, byRaw binding, countTokens). + * Same message object + same binding + deterministic token meter => same + * bytes. The binding is the only channel through which append can act — + * and the step closes it. + * + * Side condition (documented, not hidden): id collisions. The step formula + * WITHOUT the distinctness constraint must be SAT — that witness is exactly + * issue #3945, replayed here on real code as the induction's honest failure + * case. A real-code randomized spot-check pins the locality argument on the + * implementation itself. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { translateFunction } from "../verif/symbolic.ts"; +import { newCtx, smt2Of } from "../verif/terms.ts"; +import { solveSmt2 } from "../verif/solve.ts"; +import { assignRefs } from "../kernel/src/refs.ts"; +import { renderVisibleRefs } from "../kernel/src/render-refs.ts"; +import { defaultCountTokens } from "../kernel/src/tokenize.ts"; +import type { SymVal } from "../verif/symbolic.ts"; + +const REFS_FILE = "kernel/src/refs.ts"; + +test("induction step: appending a distinct-id message cannot touch any observed binding (arbitrary n)", { timeout: 240_000 }, () => { + const ctx = newCtx(); + // Abstract prefix state: one observed binding under a FREE key/value. + const idP = ctx.String.const("id_p"); + const rP = ctx.String.const("r_p"); + const binding: SymVal = { kind: "obj", fields: new Map(), symFields: new Map([[idP, { g: ctx.Bool.val(true), v: { kind: "str", t: rP } }]]) } as never; + const byRawA: SymVal = { kind: "obj", fields: new Map(), symFields: new Map([[idP, { g: ctx.Bool.val(true), v: binding }]]) } as never; + // Wait: byRaw is Record; fix shape — entries are STRING refs. + const byRaw: SymVal = { kind: "obj", fields: new Map(), symFields: new Map([[idP, { g: ctx.Bool.val(true), v: { kind: "str", t: rP } }]]) } as never; + void byRawA; void binding; + const mapA: SymVal = { kind: "obj", fields: new Map([ + ["byRaw", byRaw], + ["byRef", { kind: "obj", fields: new Map(), symFields: new Map() } as SymVal], + ]) } as never; + const nextIdx = ctx.Real.const("next_idx"); + // The appended message: fully free shape. + const mId = ctx.String.const("m_id"); + const mk = (role: string, ct: string, text: unknown): SymVal => ({ kind: "obj", fields: new Map([ + ["id", { kind: "str", t: mId }], + ["role", { kind: "str", t: ctx.String.const("m_role_" + role) }], + ["contentType", { kind: "str", t: ctx.String.const("m_ct_" + ct) }], + ["text", text as SymVal], + ]) } as never); + const mText = ctx.String.const("m_text"); + const m1 = mk("r", "c", { kind: "str", t: mText }); + const prot = ctx.Bool.const("m_protected"); + const run = translateFunction({ + file: REFS_FILE, function: "assignRefs", + params: { + messages: { kind: "arr", elems: [m1], len: ctx.Real.val(1) } as never, + options: { kind: "obj", fields: new Map([ + ["existing", mapA], + ["nextIndex", { kind: "num", t: nextIdx }], + ["isProtected", { kind: "undef" } as SymVal], + ]) } as never, + }, + ctx, + options: { + portImpls: { + __method_isProtected: (_recv: SymVal, m: SymVal) => (m === m1 ? { kind: "bool", t: prot } : { kind: "bool", t: ctx.Bool.val(false) }), + }, + }, + }).exec(); + assert.deepEqual(run.degradations, [], "translation must be faithful"); + const mapB = (run.ret as { fields: Map }).fields.get("map") as SymVal; + const byRawB = (mapB as { fields: Map }).fields.get("byRaw") as { symFields?: Map }; + assert.ok(byRawB?.symFields?.size, "map.byRaw must carry symbolic entries"); + // Read like the engine does — a key-match FAN over every symbolic entry + // (looking the entry up by Term identity would ignore entries written + // under a different Term that the solver may bind equal to ours). + type B = { g: unknown; v: SymVal }; + const eqT = (a: unknown, b: unknown) => (a as { eq: (x: unknown) => unknown }).eq(b); + const andT = (a: unknown, b: unknown) => (a as { and: (x: unknown) => unknown }).and(b); + const iteT = (c: unknown, t: unknown, e: unknown) => ((ctx as unknown as { If: (c: unknown, t: unknown, e: unknown) => unknown }).If(c, t, e)); + let refB: unknown = ctx.String.val(""); + for (const [kt, en] of byRawB.symFields!.entries() as Iterable<[unknown, B]>) { + const hit = andT(eqT(idP, kt), en.g); + refB = iteT(hit, (en.v as { t: unknown }).t, refB); + } + refB = refB as { eq: (x: unknown) => { not: () => unknown } }; + // Violation: binding value differs from r_p (and is not our never-value). + const changedReal = (rP as { eq: (x: unknown) => { not: () => unknown } }).eq(refB).not(); + const base = [ + ...run.assumptions as never[], + nextIdx.ge(ctx.Real.val(1)), + ] as never[]; + // Q-step: with distinct ids, NO change to the observed binding is possible. + const step = solveSmt2(smt2Of(ctx, [...base, mId.neq(idP), changedReal] as never[], [] as never), []); + assert.equal(step.status, "unsat", + "induction step must hold: appending a distinct-id message cannot alter the observed binding, for ANY prefix state"); + // Q-collision: WITHOUT distinctness, the change IS reachable — that is #3945. + const col = solveSmt2(smt2Of(ctx, [...base, mId.eq(idP), changedReal] as never[], [] as never), []); + assert.equal(col.status, "sat", + "id-collision shape must break the step — this SAT is issue #3945 talking"); +}); + +test("induction anchor: real code keeps append local (randomized spot-check, 60 runs)", () => { + let seed = 0x2f6e2b1; + const rnd = (n: number) => { seed = (seed * 1103515245 + 12345) & 0x7fffffff; return seed % n; }; + for (let i = 0; i < 60; i++) { + const ids = ["p1", "p2", "p3"].slice(0, 1 + rnd(3)); + const byRaw: Record = {}; + const byRef: Record = {}; + for (const id of ids) { + if (rnd(2)) { byRaw[id] = "BLOCKED"; } + else { const r = "m" + String(1000 + rnd(9000)).padStart(5, "0"); byRaw[id] = r; byRef[r] = id; } + } + const nextIndex = 1 + rnd(50); + const prot = rnd(2) === 1; + const mk = (id: string, text: string) => ({ id, role: "user", contentType: "text", text }) as never; + const prefix = ids.map((id, k) => mk(id, "prefix body " + k)); + const rendA = renderVisibleRefs(prefix, { messageRefs: { byRaw, byRef } } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + const nid = "new_" + i; + const mNew = mk(nid, prot ? "marker " : "appended text " + i); + const out = assignRefs([mNew], { existing: { byRaw, byRef }, nextIndex, isProtected: ((m: { text?: string }) => !!m.text?.includes("")) as never }); + const rendB = renderVisibleRefs([...prefix, mNew], { messageRefs: out.map } as never, defaultCountTokens, "text-only" as never) as unknown as { text: string }[]; + for (let k = 0; k < prefix.length; k++) { + assert.equal(rendB[k]!.text, rendA[k]!.text, `run ${i}: prefix message ${k} must render identically after append`); + assert.equal((out.map.byRaw as Record)[ids[k]!], byRaw[ids[k]!] ?? "BLOCKED", `run ${i}: binding for ${ids[k]} must not change`); + } + } +}); diff --git a/tests/verif-subset-guard.test.ts b/tests/verif-subset-guard.test.ts new file mode 100644 index 0000000000..62be031a2b --- /dev/null +++ b/tests/verif-subset-guard.test.ts @@ -0,0 +1,48 @@ +/** + * CI guard: every file in verif/guarded-files.ts must stay inside the + * translatable subset (verif/subset-guard.ts). Violations and stale + * exceptions both fail — there is no silent escape hatch. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { runGuard } from "../verif/subset-guard.js"; +import { resolve } from "node:path"; + +const repoRoot = resolve(import.meta.dirname, ".."); + +test("guarded kernel files stay inside the verifiable subset", () => { + const { violations, stale } = runGuard(repoRoot); + const summary = + violations.length === 0 + ? "" + : "\\n" + + violations + .map((v) => ` ${v.file}:${v.line} [${v.code}] ${v.detail} (fn ${v.fn ?? ""})`) + .join("\\n"); + assert.deepEqual( + violations, + [], + `guarded files contain constructs outside the verifiable subset.\\n` + + `Fix the code, or add a reviewed exception in verif/exceptions.ts:\\n${summary}`, + ); + assert.deepEqual( + stale, + [], + "exceptions/entries that no longer match anything must be deleted " + + "(they are an audited inventory):\\n" + + stale.map((s) => ` ${s.kind}: ${s.label}`).join("\\n"), + ); +}); + +test("guard actually detects a violation (negative control)", async () => { + const { checkGuardedFile } = await import("../verif/subset-guard.js"); + const bad = `import fs from "node:fs";\nasync function f(){ return await fs.readFile("x"); }\n`; + const out = checkGuardedFile({ + file: "kernel/src/refs.ts", + source: bad, + excludedFunctions: new Set(), + }); + const codes = new Set(out.map((v) => v.code)); + assert.ok(codes.has("NODE_BUILTIN_IMPORT"), `expected NODE_BUILTIN_IMPORT, got ${[...codes]}`); + assert.ok(codes.has("ASYNC"), `expected ASYNC, got ${[...codes]}`); +}); diff --git a/tests/verif-symbolic-smoke.test.ts b/tests/verif-symbolic-smoke.test.ts new file mode 100644 index 0000000000..ee625d25a3 --- /dev/null +++ b/tests/verif-symbolic-smoke.test.ts @@ -0,0 +1,101 @@ +/** + * Symbolic smoke test — first end-to-end proof on REAL kernel code. + * + * Target: kernel/src/refs.ts indexToRef bounds behavior (issue #3848 + * acceptance step "首个目标函数出界内无反例结论,且注入已知 bug 形状时必红"). + * + * V1 for all n in [1, 9_999_999]: indexToRef(n) does NOT throw → UNSAT expected + * V2 for n outside that range: indexToRef(n) throws → SAT, witness replayed on real code + * V3 drift control: a deliberately wrong spec bound produces NO + * witness (the solver refuses a false claim). + * + * All solving goes through the out-of-process runner (verif/solve.ts) — + * terms are built with the string-backed facade (verif/terms.ts). + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { resolve } from "node:path"; +import { translateFunction } from "../verif/symbolic.js"; +import { newCtx, smt2Of } from "../verif/terms.js"; +import { solveSmt2, parseNum } from "../verif/solve.js"; +import { replayThrows, parseIntWitness } from "../verif/replay.js"; +import { indexToRef } from "../kernel/src/refs.js"; + +const file = resolve(import.meta.dirname, "../kernel/src/refs.ts"); + +function translateIndexToRef(ctx: import("../verif/terms.js").Context, n: import("../verif/terms.js").Arith) { + const { exec } = translateFunction({ + file, + function: "indexToRef", + params: { index: { kind: "num", t: n } }, + ctx, + }); + const r = exec(); + assert.ok(r.thrown && r.thrown.kind === "bool", "translated indexToRef exposes a boolean throw flag"); + assert.deepEqual(r.degradations.filter((d) => !d.startsWith("template")), [], + `no degradations expected (got: ${r.degradations.join("; ")})`); + return r; +} + +test("symbolic V1: indexToRef never throws in [1, 9_999_999] (real code, solver-checked)", async (t) => { + const solver = await (async () => { try { return await solveSmt2("(set-logic ALL)(check-sat)", []); } catch { return null; } })(); + if (!solver) return t.skip("no SMT solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + const n = ctx.Real.const("n"); + const r = translateIndexToRef(ctx, n); + + const smt2 = smt2Of(r.ctx, [ + n.ge(ctx.Real.val(1)), n.le(ctx.Real.val(9_999_999)), n.eq(ctx.ToInt(n)), + r.thrown!.t, + ]); + const out = await solveSmt2(smt2, []); + assert.equal(out.status, "unsat", "solver must find NO in-range n that throws"); +}); + +test("symbolic V2: indexToRef throws outside [1, 9_999_999] — witness replayed on real code", async (t) => { + const solver = await (async () => { try { return await solveSmt2("(set-logic ALL)(check-sat)", []); } catch { return null; } })(); + if (!solver) return t.skip("no SMT solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + const n = ctx.Real.const("n"); + const r = translateIndexToRef(ctx, n); + + const smt2 = smt2Of(r.ctx, [ + n.ge(ctx.Real.val(-100_000_000)), n.le(ctx.Real.val(100_000_000)), n.eq(ctx.ToInt(n)), + n.lt(ctx.Real.val(1)).or(n.gt(ctx.Real.val(9_999_999))), + r.thrown!.t, + ]); + const out = await solveSmt2(smt2, ["n"]); + assert.equal(out.status, "sat"); + assert.ok(out.values.length >= 1, "witness value for n"); + const witness = parseIntWitness(out.values[0]!); + assert.ok(Number.isFinite(witness), `witness must be an integer, got ${witness}`); + + const reallyThrows = await replayThrows(indexToRef, witness); + assert.ok(reallyThrows, `witness n=${witness} must throw on real indexToRef`); +}); + +test("symbolic V3: the solver refuses a wrong spec — 9_999_999 is in range, no counterexample exists", async (t) => { + const solver = await (async () => { try { return await solveSmt2("(set-logic ALL)(check-sat)", []); } catch { return null; } })(); + if (!solver) return t.skip("no SMT solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON)"); + + const ctx = newCtx(); + const n = ctx.Real.const("n"); + const r = translateIndexToRef(ctx, n); + + // WRONG spec: claim n=9_999_999 (the real max) throws. A faithful + // translation must produce NO counterexample for a false claim. + const smt2 = smt2Of(r.ctx, [ + n.gt(ctx.Real.val(9_999_998)), n.le(ctx.Real.val(9_999_999)), n.eq(ctx.ToInt(n)), + r.thrown!.t, + ]); + const out = await solveSmt2(smt2, []); + assert.equal(out.status, "unsat", "no witness may exist for a false claim — translation is faithful"); + + // And the drift control still bites: a synthetic phantom witness the REAL + // code does not reproduce must be detected as a mismatch. + const phantom = parseNum("12345.0"); + const reallyThrows = await replayThrows(indexToRef, phantom); + assert.ok(!reallyThrows, "phantom in-range witness must NOT throw on real code (mismatch detectable)"); +}); diff --git a/verif/README.md b/verif/README.md new file mode 100644 index 0000000000..943bb9014e --- /dev/null +++ b/verif/README.md @@ -0,0 +1,86 @@ +# verif — 对 kernel 决策核做"真代码求解器验证"的小模块 + +Issue #3848。目标:把受保护文件里的真实 TypeScript 逻辑翻译成 SMT 约束, +用求解器找违反缓存稳定不变量的反例;反例必须回放真代码复现才算数。 + +这不是"根据代码写证明",也不是"自己建模":被翻译的就是仓库里的源文件, +翻译器只是镜头,镜头歪了会被回放当场抓住(见下)。 + +## 部件 + +| 部件 | 文件 | 作用 | +|---|---|---| +| 子语言守卫 | `subset-guard.ts` + `guarded-files.ts` + `exceptions.ts` | 守卫清单内的文件只许写可翻译的语法;白名单外 → `tests/verif-subset-guard.test.ts` 红。例外必须显式登记,过期例外也红 | +| 符号执行器 | `symbolic.ts` | 真 TS AST → 符号值;分支用 ITE 值合并不枚举路径;本地函数内联(不换算子);跨文件 import 图解析 | +| SMT 项门面 | `terms.ts` | 自研 DAG Term(不走 z3-solver WASM——它 2GB 堆上限扛不住两轮公式);常量折叠;`smt2Of` 发射带 define-fun 共享的 SMT-LIB2 | +| 求解器桥 | `solve.ts` | 进程外求解,优先 cvc5(经 `VERIF_PYTHON` 指定的 python),回退 z3 二进制;可用 `VERIF_SOLVER_CMD` 覆盖 | +| 反例回放 | `replay.ts` | 求解器 witness 必须在真实 kernel 代码上复现;不复现 = 翻译器漂移/规格写错,是警报不是通过 | + +## 运行 + +```bash +# 守卫 + 冒烟不需要求解器 +node --import tsx --test tests/verif-subset-guard.test.ts tests/verif-symbolic-smoke.test.ts + +# 求解器测试需要 cvc5(cvc5 优先,z3 WASM 只够冒烟级公式) +VERIF_PYTHON=/path/to/python-with-cvc5 node --import tsx --test tests/verif-bug*.test.ts tests/verif-generic-*.test.ts tests/verif-induction-*.test.ts +# 没有求解器的环境自动 skip,CI 不会红 +``` + +## 测试布局(每一层证明什么) + +| 测试 | 层 | 证明内容 | +|---|---|---| +| `verif-subset-guard` | 守卫 | 守护文件仍在可翻译子语言内 + 负控制(塞进 async/node:fs 会红) | +| `verif-symbolic-smoke` | 引擎保真 | indexToRef 界内 UNSAT / 界外 witness 回放 / 错误规格被拒(翻译保真) | +| `verif-bug3691-oscillation` | 定向不变量 | 紧急截断在门限处可触发(SAT+witness)且门限下不可触发(UNSAT);两轮回放展示同一重发历史出站字节翻转 | +| `verif-bug3769-orphan-window` | 定向不变量 | 只增不删的历史里"可见的必须保持可见"被孤儿窗口破坏;窗口调大后稳定 | +| `verif-bug3676-wire-reorder` | 定向不变量 | openai wire 重建 [file,text]→[text,file] 部件重排(SAT + 真代码回放) | +| `verif-bug3805-durable-ref-migration` | 定向不变量 | 保护解除时 BLOCKED→数字引用迁移给已发送字节中途加标签;非 durable 追加稳定(UNSAT) | +| `verif-generic-append-stability` | **任意输入** | 消息全符号化、唯一不变量"只增不删的字节稳定":Q1 SAT(#3805 形状泛化重现)、Q2 SAT(**求解器自己发现的新 bug #3945**:id 撞车重打标,零保护解除)、Q3 UNSAT(排除两个已知形状后界内无第三种) | +| `verif-induction-append-stability` | **归纳判官** | 任意消息数的 append-only 稳定性:前缀抽象成一个符号绑定,归纳步 UNSAT(id 不重复时任何前缀状态都动不了);撤掉约束 SAT = #3945;真代码随机抽查 60 轮 | + +## 无作弊规则(写新 harness 前必读) + +1. **只调真入口函数**:harness 通过 `translateFunction` 编译仓库里的真实 + 文件,不复制逻辑、不复述算法。 +2. **不变量必须是通用性质**(幂等/单调/append-only 稳定…),绝不在 bug + 行号附近写断言。允许手工挑选的只有"场景脚本"(输入的形状),因为宿主 + 行为是开放集。 +3. **反例必须回放真代码**:witness 代入真实函数,行为不符 = 引擎 bug, + 修引擎,绝不放宽断言。 +4. **port 是公开接缝**:token 计数器、isProtected 这类真实可注入参数用 + `portImpls` 提供符号实现;但本地函数名和 import 的真实函数**永远优先于 + port**,防止用 port 偷换真实逻辑。 +5. **负控制必须有**:每个 SAT 证明配一个"已知修好/排除后 UNSAT"的对照, + 证明判官不是永真式。 + +## 建模决定(诚实清单) + +- 数字 = Real(有理数);整数性用 `IsInt` 约束,不静默取整。 +- 求解不可判定的角落**降级并登记**,不假装:降级项会出现在 + `exec().degradations`,harness 断言空列表才能立案。 +- 动态正则(ownTagRe)按"锚定前缀"过近似:前缀不匹配 → 精确 false; + 匹配 → 未解释函数 + 降级标记(仅可达时)。 +- `String(n)` / `toFixed` 等确定性渲染 = 未解释函数(同名同参同值), + 保证跨轮稳定;witness 回放仍然落在真代码上。 +- 循环展开 K(默认 8)、数组槽位 N(默认 8):**有界模型检测**。要"任意 + 消息数"不用加 N —— 用归纳判官(`verif-induction-*`)。 +- 模块级常量从源文件真实收集,不手抄。 + +## 已发现的 bug(求解器自动 + 回放确认) + +- #3691 紧急截断震荡(harness 定向找到;回放顺带发现截断结果比原文长 → #3935) +- #3769 失败压缩轮换的孤儿窗口 +- #3676 openai wire 重建重排 +- #3805 durable 引用迁移重打标 +- **#3945 重复 id 重打标 —— 通用输入 harness 在任意输入下自己发现的新形状** + +## 扩展 + +- 加守护文件:改 `guarded-files.ts`,先确认翻译器支持其语法(守卫会告诉你 + 缺什么),例外登记 `exceptions.ts`。 +- 加不变量:新 harness 文件,遵守上面的无作弊规则;SAT → 回放 → 真 bug + 开 issue + 回归钉;UNSAT → 界内证无,写清楚界。 +- 从有界到无界:归纳判官模板 = 前缀抽象 + 单步 UNSAT + 已知例外 SAT + + 真代码随机锚点。 diff --git a/verif/exceptions.ts b/verif/exceptions.ts new file mode 100644 index 0000000000..34c13ed91e --- /dev/null +++ b/verif/exceptions.ts @@ -0,0 +1,81 @@ +/** + * Explicit exceptions to the verif subset rules. + * + * Every entry must carry a human reason. Entries that no longer match + * anything in the guarded files are reported as stale by the guard test + * and must be deleted — this list is an audited inventory, not a dumping + * ground. + */ + +/** A whole function scoped out of symbolic verification. Calls to it are + * treated as uninterpreted; the function body is not checked. Use only for + * code that is provably off every verified path (e.g. optional ports). */ +interface ExcludedFunction { + file: string; + fn: string; + reason: string; +} + +export interface SubsetException { + file: string; + code: string; + /** Restrict the exception to one function (by local name). */ + fn?: string; + /** Restrict to violations whose detail matches exactly. */ + detail?: string; + reason: string; +} + +export const EXCLUDED_FUNCTIONS: readonly ExcludedFunction[] = [ + { + file: "kernel/src/tokenize.ts", + fn: "createBpeTokenizer", + reason: + "Optional BPE tokenizer probe: node require + returned closures. Never on a verified path — CompressionCore takes countTokens as an injectable port and the verified entry points use defaultCountTokens (pure arithmetic).", + }, +]; + +export const EXCEPTIONS: readonly SubsetException[] = [ + { + file: "kernel/src/tokenize.ts", + code: "NODE_BUILTIN_IMPORT", + reason: + "createRequire from node:module at module top feeds only createBpeTokenizer (an excluded function); the verified path (defaultCountTokens) is pure.", + }, + { + file: "kernel/src/compress.ts", + code: "ENV_ACCESS", + fn: "applySingleRange", + reason: + "createdAt: Date.now() — wall-clock stamp on new blocks; orthogonal to every verified invariant, modeled as an unconstrained symbolic integer.", + }, + { + file: "kernel/src/compress.ts", + code: "ENV_ACCESS", + fn: "processTurn", + detail: "console.warn", + reason: + "Log-only console.warn on a degraded path; modeled as a no-op observation point — never influences verified state.", + }, + { + file: "kernel/src/render-refs.ts", + code: "DYNAMIC_REGEX", + reason: + "ownTagRe = new RegExp(escapeRegex(TAG_OPEN)+...+escapeRegex(ref)+...) — pattern is a concatenation of literal fragments with the message ref escaped in; representable as a symbolic regex concatenation once the translator grows dynamic-pattern support. Tracked until then.", + }, + { + file: "kernel/src/tokenize.ts", + code: "UNKNOWN_GLOBAL", + detail: "createRequire", + reason: + "Module-scope const require = createRequire(import.meta.url); only createBpeTokenizer (excluded function) uses it. Verified paths never call it.", + }, + { + file: "kernel/src/wire/responses.ts", + code: "ENV_ACCESS", + fn: "shouldDropAllReasoning", + detail: "read process.env", + reason: + "process.env.ACP_REASONING_KEEP read — process-wide config constant, modeled as a fixed symbolic boolean, never mutated inside the guarded scope.", + }, +]; diff --git a/verif/guarded-files.ts b/verif/guarded-files.ts new file mode 100644 index 0000000000..7d4830201c --- /dev/null +++ b/verif/guarded-files.ts @@ -0,0 +1,23 @@ +/** + * Guarded file list for the symbolic-verification subset ("verif-subset"). + * + * Everything listed here must stay inside the translatable sublanguage + * (see verif/subset-guard.ts). Anything the symbolic executor cannot + * compile needs an explicit, reviewed exception in verif/exceptions.ts — + * there is no silent escape hatch. + * + * Grow this list only together with translator coverage, never ahead of it. + */ +export const GUARDED_FILES: readonly string[] = [ + // Tier A — current translator targets + "kernel/src/compress.ts", + "kernel/src/hide-consumed.ts", + "kernel/src/refs.ts", + "kernel/src/render-refs.ts", + "kernel/src/truncate.ts", + "kernel/src/truncate-tools.ts", + "kernel/src/tokenize.ts", + "kernel/src/rebuild.ts", + "kernel/src/wire/openai.ts", + "kernel/src/wire/responses.ts", +]; diff --git a/verif/replay.ts b/verif/replay.ts new file mode 100644 index 0000000000..b33d32b46c --- /dev/null +++ b/verif/replay.ts @@ -0,0 +1,34 @@ +/** + * Counterexample replay — the anti-drift lock of the verification pipeline. + * + * A solver witness is only a FINDING if the real compiled kernel code + * reproduces it. If the translated model and the real code disagree, the + * replay fails and the translator is wrong — that failure is the signal, + * never "verification passed anyway". + */ + +/** Parse z3's s-expression numeric output: "5", "(- 5808901)", "96.0", + * "10000000.0", "(/ 1 3)". Reals that are not integral return NaN — the + * caller must constrain the witness variable to integers. */ +export function parseIntWitness(raw: string): number { + const t = raw.trim(); + if (/^-?\d+$/.test(t)) return Number(t); + // Real numerals print with a decimal tail ("96.0", "-3.0") + const real = /^(-?\d+)\.0+$/.exec(t); + if (real) return Number(real[1]); + const m = /^\(\s*(-|\+)\s+(\d+)\)$/.exec(t); + if (m) return m[1] === "-" ? -Number(m[2]) : Number(m[2]); + return Number.NaN; +} + +export async function replayThrows( + fn: (x: number) => unknown, + arg: number, +): Promise { + try { + fn(arg); + return false; + } catch { + return true; + } +} diff --git a/verif/solve.ts b/verif/solve.ts new file mode 100644 index 0000000000..3cd9c34eae --- /dev/null +++ b/verif/solve.ts @@ -0,0 +1,144 @@ +// Out-of-process SMT solving. The in-process z3-solver npm package is WASM +// (2GB heap cap) and OOMs on two-turn cache-stability formulas; native z3 +// spins on the seq/LRA mix. cvc5 solves the same formulas instantly, so the +// runner prefers it when available. Override with VERIF_SOLVER_CMD (a command +// prefix that receives the smt2 file path as its last argument and must print +// `sat`/`unsat` plus `(get-value ...)` s-expr replies on stdout). +import { spawnSync } from "node:child_process"; +import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; + +interface SolveResult { + status: "sat" | "unsat" | "unknown"; + /** Parsed `(get-value ...)` replies, in declaration order: raw strings. */ + values: string[]; + raw: string; +} + +/** Parse the printed form of a get-value reply: `((name value) (name value))` + * — one outer list of pairs; handles nested parens in name and value. */ +const parseValues = (raw: string): string[] => { + const out: string[] = []; + const start = raw.indexOf("(("); + if (start < 0) return out; + const s = raw.slice(start + 1); // s starts at the first PAIR's `(` + let i = 0; + const n = s.length; + const readSexp = (): string => { + while (i < n && /\s/.test(s[i]!)) i++; + if (s[i] === "\"") { // quoted SMT string: until the real closing quote ("" is an escaped quote) + const b = i; i++; + while (i < n) { if (s[i] === "\"") { if (s[i + 1] === "\"") { i += 2; continue; } i++; break; } i++; } + return s.slice(b, i); + } + if (s[i] !== "(") { const j = i; while (i < n && !/[\s)]/.test(s[i]!)) i++; return s.slice(j, i); } + let depth = 0; const b = i; + while (i < n) { + if (s[i] === "(") depth++; + else if (s[i] === ")") { depth--; if (depth === 0) { i++; return s.slice(b, i); } } + i++; + } + return s.slice(b); + }; + while (i < n) { + while (i < n && /\s/.test(s[i]!)) i++; + if (s[i] !== "(") break; // end of the pair list + i++; // past the pair's opening paren + const name = readSexp(); void name; + const value = readSexp(); + out.push(value); + while (i < n && /\s/.test(s[i]!)) i++; + if (s[i] === ")") i++; // pair close — keep going; next pair or list end follows + } + return out; +}; + +export const solveSmt2 = (input: string | { smt2: string }, getValue: string[]): SolveResult => { + const smt2 = typeof input === "string" ? input : input.smt2; + const dir = mkdtempSync(join(tmpdir(), "verif-solve-")); + const file = join(dir, "query.smt2"); + try { + const header = smt2.startsWith("(set-logic") ? "" : "(set-logic ALL)\n"; + const tail = getValue.length + ? `(check-sat)\n(get-value (${getValue.join(" ")}))\n` + : "(check-sat)\n"; + writeFileSync(file, header + smt2.replace(/\(check-sat\)[\s\S]*$/, "") + tail); + + // Try, in order: explicit override, cvc5 via python, z3 binaries. + const attempts: { cmd: string; args: string[]; parse: (raw: string) => SolveResult }[] = []; + if (process.env.VERIF_SOLVER_CMD) { + const parts = process.env.VERIF_SOLVER_CMD.split(/\s+/); + attempts.push({ + cmd: parts[0]!, + args: [...parts.slice(1), file], + parse: (raw) => ({ status: /(^|\n)sat/.test(raw) && !/unsat/.test(raw) ? "sat" : /unsat/.test(raw) ? "unsat" : "unknown", values: parseValues(raw), raw }), + }); + } + const py = process.env.VERIF_PYTHON ?? "python3"; + const cvc5Py = `${process.env.CVC5_HOME ?? process.env.HOME + "/tmp/cvc5"}`; + const script = [ + "import sys", + "import cvc5", + "tm = cvc5.TermManager(); s = cvc5.Solver(tm)", + "s.setOption('produce-models','true')", + "sm = cvc5.SymbolManager(tm)", + "p = cvc5.InputParser(s, sm)", + `p.setFileInput(cvc5.InputLanguage.SMT_LIB_2_6, ${JSON.stringify(file)})`, + "while True:", + " cmd = p.nextCommand()", + " if cmd.isNull(): break", + " out = cmd.invoke(s, sm)", + " if out: print(out.strip())", + ].join("\n"); + attempts.push({ + cmd: py, + args: ["-c", script], + parse: (raw) => ({ status: /(^|\n)sat\b/.test(raw) && !/unsat/.test(raw) ? "sat" : /unsat/.test(raw) ? "unsat" : "unknown", values: parseValues(raw), raw }), + // note: PYTHONPATH handled via env below + }); + const z3Bin = process.env.Z3_BIN ?? "z3"; + attempts.push({ + cmd: z3Bin, + args: [file], + parse: (raw) => ({ status: /(^|\n)sat\b/.test(raw) ? "sat" : /unsat/.test(raw) ? "unsat" : "unknown", values: parseValues(raw), raw }), + }); + + let lastErr = ""; + for (const a of attempts) { + const r = spawnSync(a.cmd, a.args, { + encoding: "utf8", + timeout: 120_000, + maxBuffer: 64 * 1024 * 1024, + env: a.cmd === py || a.cmd.endsWith("python3") || a.cmd.includes("python") + ? { ...process.env, PYTHONPATH: `${cvc5Py}:${process.env.PYTHONPATH ?? ""}` } + : process.env, + }); + if (r.error) { lastErr += `${a.cmd}: ${r.error.message}\n`; continue; } + // A verdict printed to stdout wins over the exit code: native z3 + // prints `unsat` and THEN exits 1 because the trailing (get-value) + // fails with "model is not available" — discarding stdout there would + // turn every unsat into "no solver available". + const parsed = a.parse(r.stdout ?? ""); + if (parsed.status !== "unknown") return parsed; + lastErr += r.status !== 0 + ? `${a.cmd}: exit ${r.status}: ${r.stderr.slice(0, 500)}\n` + : `${a.cmd}: unknown\n`; + } + throw new Error(`no solver available (set VERIF_SOLVER_CMD / VERIF_PYTHON):\n${lastErr}`); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}; + +/** Parse a model value like `4001`, `4001.0`, `(- 3)` into a JS number. */ +export const parseNum = (v: string): number => { + const t = v.trim(); + if (t.startsWith("(-")) { + const inner = t.replace(/^\(-\s*/, "").replace(/\)$/, "").trim(); + return -parseNum(inner); + } + const n = Number(t.replace(/\.0+$/, "")); + if (!Number.isFinite(n)) throw new Error(`unparseable model value: ${v}`); + return n; +}; diff --git a/verif/subset-guard.ts b/verif/subset-guard.ts new file mode 100644 index 0000000000..106c643bf3 --- /dev/null +++ b/verif/subset-guard.ts @@ -0,0 +1,584 @@ +/** + * verif subset guard — the sublanguage enforcement engine. + * + * Purpose: the files in verif/guarded-files.ts must stay inside the subset + * that verif/symbolic.ts can translate to SMT. This module walks the real + * TypeScript AST and reports every construct outside the subset. Anything + * reported must either be fixed or be covered by an explicit, reviewed + * exception in verif/exceptions.ts — there is no third option. + * + * Design rules: + * - The engine is driven by ALLOWLISTS, not denylists: an unknown builtin, + * method, or import is a violation until it is taught to the translator + * and added here. That is what keeps "cannot be verified" code out. + * - Matching an exception requires file + code (+ function); unused + * exceptions are reported so stale entries get deleted. + */ +import ts from "typescript"; + +type ViolationCode = + | "ASYNC" + | "GENERATOR" + | "CLASS_DECL" + | "INHERITANCE" + | "ACCESSOR" + | "DECORATOR" + | "TAGGED_TEMPLATE" + | "BIGINT" + | "DYNAMIC_CODE" + | "REFLECTION" + | "ENV_ACCESS" + | "NODE_BUILTIN_IMPORT" + | "EXTERNAL_IMPORT" + | "LABEL" + | "WITH_STMT" + | "DELETE_EXPR" + | "VOID_EXPR" + | "FOR_IN" + | "FREE_CLOSURE" + | "UNKNOWN_GLOBAL" + | "UNKNOWN_METHOD" + | "DYNAMIC_REGEX" + | "REGEX_STATEFUL" + | "EXCLUDED_FUNCTION"; + +interface Violation { + file: string; + line: number; + code: ViolationCode; + detail: string; + fn: string | undefined; +} + +interface GuardResult { + violations: Violation[]; + /** Exceptions/entries that matched nothing — must be deleted. */ + stale: Array<{ kind: "exception" | "excluded"; label: string }>; +} + +/* ---------------------------------------------------------------- allowlists */ + +const MATH_FNS = new Set(["abs", "min", "max", "round", "floor", "ceil", "trunc", "sign", "pow", "sqrt"]); + +const JSON_FNS = new Set(["parse", "stringify"]); + +const OBJECT_FNS = new Set(["keys", "values", "entries", "hasOwn"]); + +const ARRAY_FNS = new Set(["isArray", "from", "of"]); + +const NUMBER_FNS = new Set(["isFinite", "isNaN", "isInteger", "isSafeInteger", "parseInt", "parseFloat"]); + +const STRING_CTOR_FNS = new Set(["fromCharCode", "fromCodePoint"]); + +const NAMESPACE_MEMBERS: Record> = { + Math: MATH_FNS, + JSON: JSON_FNS, + Object: OBJECT_FNS, + Array: ARRAY_FNS, + Number: NUMBER_FNS, + String: STRING_CTOR_FNS, +}; + +/** Higher-order array methods whose callback arguments are the only legal + * closure shape. The translator inlines these combinators. */ +const HOF_ARRAY_METHODS = new Set([ + "map", "filter", "find", "findIndex", "findLast", "findLastIndex", + "every", "some", "reduce", "reduceRight", "flatMap", "forEach", "sort", +]); + +const PLAIN_ARRAY_METHODS = new Set([ + "includes", "indexOf", "lastIndexOf", "slice", "concat", "join", + "push", "pop", "shift", "unshift", "splice", "at", "flat", "reverse", + "entries", "keys", "values", +]); + +const STRING_METHODS = new Set([ + "slice", "substring", "indexOf", "lastIndexOf", "includes", "startsWith", + "endsWith", "replace", "replaceAll", "split", "trim", "trimStart", + "trimEnd", "toLowerCase", "toUpperCase", "normalize", "padStart", + "padEnd", "repeat", "concat", "charAt", "charCodeAt", "codePointAt", + "at", "match", "search", +]); + +const MAP_SET_METHODS = new Set([ + "get", "set", "has", "delete", "clear", "add", "keys", "values", "entries", "forEach", +]); + +/** Methods callable on regex literals (stateless flags only; the stateful + * exec/test-with-g combination is rejected separately). */ +const REGEX_METHODS = new Set(["test", "exec"]); + +/** Function-typed fields on options/state objects (injected ports). Calls to + * these are modeled as uninterpreted functions — they are extension seams, + * not logic the verifier reasons about. */ +const PORT_METHODS = new Set(["countTokens", "shouldSkip", "isProtected"]); + +/** Deterministic number formatting. */ +const NUMBER_INSTANCE_METHODS = new Set(["toFixed"]); + +/** Methods on instances/records produced by kernel-internal code. */ +const KERNEL_INSTANCE_METHODS = new Set([ + "next", "namespaceFor", + // createCore() entry-point record: + "applyCompression", "processTurn", +]); + +/** new X is allowed only for these. */ +const NEWABLE = new Set(["Map", "Set", "Error", "RangeError", "TypeError", "RegExp"]); +/** Kernel classes (imported from other guarded/Tier-B modules) that may be + * constructed; their instance methods are registered below. */ +const IMPORTED_NEWABLE = new Set(["ClusterCounter"]); + +/** Bare callables allowed as plain functions. */ +const GLOBAL_FNS = new Set([ + "isNaN", "isFinite", "parseFloat", "parseInt", + "String", "Number", "Boolean", "Object", "Array", + // function-typed parameters (injected ports, uninterpreted): + "countTokens", "predicate", "isCompressTool", +]); + +/** Namespace roots (Math.x etc.) handled via NAMESPACE_MEMBERS. */ +const NAMESPACE_ROOTS = new Set(Object.keys(NAMESPACE_MEMBERS)); + +/** Namespace roots also usable as bare coercion calls: String(x), Number(x)… */ +const COERCION_ROOTS = new Set(["String", "Number", "Boolean", "Object", "Array"]); + +/** Reads that reach the environment / nondeterminism. */ +const ENV_ROOTS = new Set([ + "Date", "Math.random", "crypto", "performance", "process", "console", + "fetch", "setTimeout", "setInterval", "setImmediate", "queueMicrotask", + "require", "Buffer", "globalThis", "global", "WebAssembly", +]); + +const REFLECTION_ROOTS = new Set(["Proxy", "Reflect", "WeakMap", "WeakSet", "Symbol", "Function", "eval"]); + +const STATEFUL_REGEX_FLAGS = new Set(["g", "y"]); + +const ALL_VALUE_METHODS = new Set([ + ...HOF_ARRAY_METHODS, ...PLAIN_ARRAY_METHODS, ...STRING_METHODS, + ...MAP_SET_METHODS, ...REGEX_METHODS, ...PORT_METHODS, + ...NUMBER_INSTANCE_METHODS, ...KERNEL_INSTANCE_METHODS, +]); + +/* ------------------------------------------------------------------ helpers */ + +function unwrap(node: ts.Node): ts.Node { + let cur = node; + while (ts.isParenthesizedExpression(cur) || ts.isAsExpression(cur) || + ts.isNonNullExpression(cur) || ts.isSatisfiesExpression(cur)) { + cur = cur.expression; + } + return cur; +} + +function dottedName(expr: ts.Expression): string | undefined { + let node = unwrap(expr); + const parts: string[] = []; + while (ts.isPropertyAccessExpression(node)) { + parts.unshift(node.name.text); + node = unwrap(node.expression); + } + if (ts.isIdentifier(node)) { + parts.unshift(node.text); + return parts.join("."); + } + return undefined; +} + +function nearestFunctionName(node: ts.Node): string | undefined { + // Walk up to the nearest enclosing function; report its binding name. + // Module-level constructs report undefined (no enclosing function). + let cur: ts.Node | undefined = node.parent; + while (cur && !ts.isSourceFile(cur)) { + if (ts.isFunctionDeclaration(cur) && cur.name) return cur.name.text; + if (ts.isMethodDeclaration(cur) && cur.name && ts.isIdentifier(cur.name)) return cur.name.text; + if (ts.isFunctionExpression(cur) || ts.isArrowFunction(cur)) { + const parent = cur.parent; + if (ts.isVariableDeclaration(parent) && ts.isIdentifier(parent.name)) return parent.name.text; + if (ts.isPropertyAssignment(parent) && ts.isIdentifier(parent.name)) return parent.name.text; + if (ts.isFunctionDeclaration(parent)) return parent.name?.text; // IIFE-ish, rare + } + cur = cur.parent; + } + return undefined; +} + +/* ------------------------------------------------------------------- engine */ + +interface CheckInput { + /** Repo-relative path, e.g. kernel/src/refs.ts */ + file: string; + source: string; + excludedFunctions: ReadonlySet; +} + +export function checkGuardedFile({ file, source, excludedFunctions }: CheckInput): Violation[] { + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.ES2022, true); + const violations: Violation[] = []; + const add = (node: ts.Node, code: ViolationCode, detail: string) => { + violations.push({ + file, + line: sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1, + code, + detail, + fn: nearestFunctionName(node), + }); + }; + + const kernelImports = new Set(); + const localFunctions = new Set(); + + // Pre-pass: collect function declarations and named const functions/calls + // (calls may appear textually before the declaration — hoisting). + const moduleConsts = new Set(); + const prepass = (node: ts.Node, top: boolean): void => { + if (ts.isFunctionDeclaration(node) && node.name) localFunctions.add(node.name.text); + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name)) { + const init = node.initializer; + if (init && (ts.isArrowFunction(init) || ts.isFunctionExpression(init))) { + localFunctions.add(node.name.text); + } else if (top) { + moduleConsts.add(node.name.text); + } + } + ts.forEachChild(node, (c) => prepass(c, false)); + }; + prepass(sf, true); + + /** Resolve a call's callee into either a dotted path (root is an + * identifier) or the trailing method name on an arbitrary receiver. */ + const resolveCallee = ( + callee: ts.Expression, + ): { dotted?: string; methodName?: string } => { + const dotted = dottedName(callee); + if (dotted !== undefined) return { dotted }; + const c = unwrap(callee); + if (ts.isPropertyAccessExpression(c)) { + return { methodName: c.name.text }; + } + return {}; + }; + + const checkCall = (node: ts.CallExpression): void => { + const { dotted, methodName } = resolveCallee(node.expression); + + if (dotted !== undefined) { + const segs = dotted.split("."); + const root = segs[0]; + const last = segs[segs.length - 1]; + + if (REFLECTION_ROOTS.has(root) || REFLECTION_ROOTS.has(dotted)) { + add(node, "REFLECTION", dotted); + return; + } + if (ENV_ROOTS.has(dotted) || ENV_ROOTS.has(root)) { + add(node, "ENV_ACCESS", dotted); + return; + } + if (NAMESPACE_ROOTS.has(root)) { + if (segs.length === 1 && COERCION_ROOTS.has(root)) return; // String(x)/Number(x)/... + // Namespace calls must be exactly Root.member with member allowed. + const allowed = NAMESPACE_MEMBERS[root]; + if (segs.length !== 2 || !allowed.has(segs[1])) { + add(node, "UNKNOWN_METHOD", dotted); + } + return; + } + if (segs.length === 1) { + // Bare call of a local/imported/whitelisted global function. + if (!kernelImports.has(root) && !localFunctions.has(root) && !GLOBAL_FNS.has(root)) { + add(node, "UNKNOWN_GLOBAL", dotted); + } + return; + } + // Method call on a local/imported/module-const value: check the + // trailing method against the registered method families. + if (!ALL_VALUE_METHODS.has(last)) { + add(node, "UNKNOWN_METHOD", `.${last} on local value (${dotted})`); + return; + } + return; + } + + if (methodName !== undefined) { + // Receiver is a computed expression (call result, literal, ...). + if (!ALL_VALUE_METHODS.has(methodName)) { + add(node, "UNKNOWN_METHOD", `.${methodName} on computed receiver`); + } + return; + } + + add(node, "UNKNOWN_GLOBAL", `indirect call (${node.expression.getText(sf)})`); + }; + + /** Legal closure = argument of a whitelisted HOF / kernel-internal call, + * or part of a record returned from a local function (entry-point table). */ + const isLegalCallback = (fn: ts.ArrowFunction | ts.FunctionExpression): boolean => { + // Default value in a parameter list: pure deterministic fallback. + if (ts.isParameter(fn.parent)) return true; + // Right side of a ?? fallback: const x = provided ?? (() => default). + { + let holder: ts.Node = fn; + while (ts.isParenthesizedExpression(holder.parent)) holder = holder.parent; + const bp = holder.parent; + if ( + ts.isBinaryExpression(bp) && + bp.operatorToken.kind === ts.SyntaxKind.QuestionQuestionToken && + bp.right === holder + ) { + return true; + } + } + let cur: ts.Node | undefined = fn.parent; + while ( + cur && + !ts.isCallExpression(cur) && + !ts.isStatement(cur) && + !ts.isVariableDeclaration(cur) && + !ts.isBinaryExpression(cur) + ) { + cur = cur.parent; + } + if (cur && ts.isCallExpression(cur)) { + const { dotted, methodName } = resolveCallee(cur.expression); + const m = methodName ?? (dotted !== undefined && dotted.includes(".") ? dotted.split(".").pop() : undefined); + if (m !== undefined && (HOF_ARRAY_METHODS.has(m) || MAP_SET_METHODS.has(m))) return true; + const root = dotted?.split(".")[0]; + if (root !== undefined && (kernelImports.has(root) || localFunctions.has(root))) return true; + return false; + } + if (cur && ts.isReturnStatement(cur)) { + // Arrow inside a record returned by a local function — dispatch table. + return true; + } + if (cur && ts.isVariableDeclaration(cur)) { + // Named local helper (const f = (x) => ...). Statically call-graphable; + // every use is checked as a call/argument elsewhere in this file. + return true; + } + return false; + }; + + const visit = (node: ts.Node): void => { + /* ---- exclusions: skip whole excluded function bodies ---- */ + if ( + ts.isFunctionDeclaration(node) && + node.name && + excludedFunctions.has(`${file}#${node.name.text}`) + ) { + add(node, "EXCLUDED_FUNCTION", `function ${node.name.text} is excluded from the verified subset`); + violations[violations.length - 1].fn = node.name.text; + return; + } + + switch (node.kind) { + case ts.SyntaxKind.AwaitExpression: + add(node, "ASYNC", "await — the verified subset is fully synchronous"); + return; + case ts.SyntaxKind.YieldExpression: + add(node, "GENERATOR", "yield"); + return; + case ts.SyntaxKind.ClassDeclaration: + case ts.SyntaxKind.ClassExpression: + add(node, "CLASS_DECL", "class declarations are outside the subset"); + return; + case ts.SyntaxKind.HeritageClause: + add(node, "INHERITANCE", "extends/implements"); + return; + case ts.SyntaxKind.GetAccessor: + case ts.SyntaxKind.SetAccessor: + add(node, "ACCESSOR", "get/set accessor"); + return; + case ts.SyntaxKind.Decorator: + add(node, "DECORATOR", "decorator"); + return; + case ts.SyntaxKind.TaggedTemplateExpression: + add(node, "TAGGED_TEMPLATE", "tagged template"); + return; + case ts.SyntaxKind.BigIntLiteral: + add(node, "BIGINT", "bigint literal"); + return; + case ts.SyntaxKind.LabeledStatement: + add(node, "LABEL", "labeled statement"); + return; + case ts.SyntaxKind.WithStatement: + add(node, "WITH_STMT", "with statement"); + return; + case ts.SyntaxKind.DeleteExpression: { + const operand = unwrap((node as ts.DeleteExpression).expression); + if (ts.isIdentifier(operand)) { + add(node, "DELETE_EXPR", "delete of a binding"); + return; + } + break; // delete obj.k / delete obj[k]: bounded-map removal, allowed + } + case ts.SyntaxKind.VoidExpression: + add(node, "VOID_EXPR", "void expression"); + return; + case ts.SyntaxKind.ForInStatement: + add(node, "FOR_IN", "for-in enumeration"); + return; + default: + break; + } + + if (ts.isImportDeclaration(node)) { + const spec = node.moduleSpecifier.getText(sf).replace(/^['"]|['"]$/g, ""); + if (spec.startsWith("node:")) { + add(node, "NODE_BUILTIN_IMPORT", `import from "${spec}"`); + } else if (!spec.startsWith(".")) { + add(node, "EXTERNAL_IMPORT", `import from "${spec}"`); + } else if (node.importClause?.namedBindings && ts.isNamedImports(node.importClause.namedBindings)) { + for (const el of node.importClause.namedBindings.elements) { + kernelImports.add((el.propertyName ?? el.name).text); + } + } + return; + } + + if (ts.isCallExpression(node) && node.expression.kind === ts.SyntaxKind.ImportKeyword) { + add(node, "DYNAMIC_CODE", "dynamic import()"); + return; + } + + if (ts.isNewExpression(node)) { + const name = dottedName(node.expression); + const root = name?.split(".")[0]; + const okNew = + name !== undefined && + root !== undefined && + (NEWABLE.has(root) || (IMPORTED_NEWABLE.has(root) && kernelImports.has(root))); + if (!okNew) { + add(node, "UNKNOWN_GLOBAL", `new ${name ?? ""}`); + return; + } + if (name === "RegExp") { + const first = node.arguments?.[0]; + const literal = first && (ts.isStringLiteral(first) || ts.isNoSubstitutionTemplateLiteral(first)); + if (!literal) { + add(node, "DYNAMIC_REGEX", "new RegExp with non-literal pattern"); + return; + } + const flags = node.arguments?.[1]; + if (flags && ts.isStringLiteral(flags)) { + for (const ch of flags.text) { + if (STATEFUL_REGEX_FLAGS.has(ch)) { + add(node, "REGEX_STATEFUL", `RegExp flags "${flags.text}" carry lastIndex state`); + return; + } + } + } + } + } + + /* Stateful regex use: /re/g.exec(...) or /re/g.test(...) on the SAME + * literal — string.match(re) never touches lastIndex, so /g/ is fine + * there. This is the precise case; blanket flagging would be wrong. */ + if (ts.isCallExpression(node)) { + const callee = unwrap(node.expression); + if ( + ts.isPropertyAccessExpression(callee) && + ts.isRegularExpressionLiteral(callee.expression) && + (callee.name.text === "exec" || callee.name.text === "test") + ) { + const text = callee.expression.getText(sf); + const flags = text.slice(text.lastIndexOf("/") + 1); + if ([...flags].some((ch) => STATEFUL_REGEX_FLAGS.has(ch))) { + add(node, "REGEX_STATEFUL", `.${callee.name.text} on /${flags}/ regex literal mutates lastIndex`); + return; + } + } + } + + if (ts.isCallExpression(node)) { + checkCall(node); + // fall through to children (arguments may contain closures etc.) + } + + /* Environment reads (not calls): process.env.X, Date.now as a value, … + * Callee positions are skipped here — checkCall reports those once. */ + if ( + ts.isPropertyAccessExpression(node) && + ts.isIdentifier(node.expression) && + !(ts.isCallExpression(node.parent) && node.parent.expression === node) + ) { + const dotted = dottedName(node); + if (dotted) { + const root = dotted.split(".")[0]; + if (ENV_ROOTS.has(dotted) || ENV_ROOTS.has(root)) { + add(node, "ENV_ACCESS", `read ${dotted}`); + } + } + } + + if ((ts.isArrowFunction(node) || ts.isFunctionExpression(node)) && !isLegalCallback(node)) { + add(node, "FREE_CLOSURE", "function value escapes the allowed callback positions"); + } + + ts.forEachChild(node, visit); + }; + + visit(sf); + return violations; +} + +/* -------------------------------------------------------------- top level */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { EXCEPTIONS, EXCLUDED_FUNCTIONS, type SubsetException } from "./exceptions.js"; +import { GUARDED_FILES } from "./guarded-files.js"; + +export function runGuard(repoRoot: string): GuardResult { + const excluded = new Set(EXCLUDED_FUNCTIONS.map((e) => `${e.file}#${e.fn}`)); + const used = new Set(); + const violations: Violation[] = []; + + for (const rel of GUARDED_FILES) { + const source = readFileSync(resolve(repoRoot, rel), "utf8"); + for (const v of checkGuardedFile({ file: rel, source, excludedFunctions: excluded })) { + if (v.code === "EXCLUDED_FUNCTION") { + const key = `${v.file}#${v.fn}`; + if (excluded.has(key)) { + used.add(`excluded:${key}`); + continue; + } + } + const match = findException(v); + if (match) { + used.add(exceptionLabel(match)); + continue; + } + violations.push(v); + } + } + + const stale: GuardResult["stale"] = []; + for (const e of EXCEPTIONS) { + if (!used.has(exceptionLabel(e))) { + stale.push({ kind: "exception", label: exceptionLabel(e) }); + } + } + for (const e of EXCLUDED_FUNCTIONS) { + const key = `${e.file}#${e.fn}`; + if (!used.has(`excluded:${key}`)) { + stale.push({ kind: "excluded", label: `excluded:${key}` }); + } + } + + return { violations, stale }; +} + +function findException(v: Violation): SubsetException | undefined { + return EXCEPTIONS.find( + (e) => + e.file === v.file && + e.code === v.code && + (e.fn === undefined || e.fn === v.fn) && + (e.detail === undefined || e.detail === v.detail), + ); +} + +function exceptionLabel(e: SubsetException): string { + return `${e.file}:${e.code}:${e.fn ?? "*"}`; +} diff --git a/verif/symbolic.ts b/verif/symbolic.ts new file mode 100644 index 0000000000..5d6cf66fbe --- /dev/null +++ b/verif/symbolic.ts @@ -0,0 +1,2530 @@ +/** + * verif symbolic executor — v2. + * + * Translates REAL TypeScript function bodies (from the guarded kernel files) + * into z3 constraints. The code under verification is the actual source; + * this file is only the lens. The replay lock (verif/replay.ts + smoke + * tests) catches any lens distortion. + * + * Value model (modeling decisions, all documented in verif/README.md): + * - numbers → z3 Real (exact rationals ≈ JS doubles; boundary divergence + * is caught by replay) + * - booleans → z3 Bool + * - strings → z3 Seq (string theory) + * - arrays → bounded slot vectors: { elems: SymVal[N], len: Real } + * with N = maxArrayLength (unrolled combinators, push via + * ITE store). Sound for found counterexamples; absence + * claims hold within the bound only. + * - objects → structural field maps { fields: Map } + * - unknown/degraded constructs → { kind: "any" } (never silently + * trustworthy: harnesses receive the degradation list) + * + * Execution model: single-assignment environment; if/else via ITE value + * merge (no path enumeration); loops unrolled up to K iterations with + * termination assumptions collected for the solver. + * + * Function calls: local functions in the same file are INLINED (depth + * capped) so harnesses exercise the real helper chain. Calls that cannot + * be translated degrade to `any`. + */ +import ts from "typescript"; +import { readFileSync, accessSync } from "node:fs"; +import { join, dirname } from "node:path"; +import type { Arith, Bool, Seq, Context, Term } from "./terms.js"; + +class TranslationUnsupported extends Error { + constructor(readonly what: string, readonly file: string, readonly line: number) { + super(`translation unsupported at ${file}:${line}: ${what}`); + } +} + +export type Z3 = Context; +export type Num = Arith; +type Str = Seq; + +export type SymVal = + | { kind: "num"; t: Num } + | { kind: "bool"; t: Bool<"main"> } + | { kind: "str"; t: Str } + // nullUnder: set on obj/arr values that came out of a STRICT branch merge + // with null/undefined on the other arm — the value is the obj/arr only + // when nullUnder is FALSE (JS truthiness/field-access must respect it). + | { kind: "arr"; elems: SymVal[]; len: Num; nullUnder?: Bool<"main"> } + | { kind: "obj"; fields: Map; symFields?: Map; v: SymVal }>; nullUnder?: Bool<"main"> } + | { kind: "fn"; node: ts.ArrowFunction | ts.FunctionExpression | ts.FunctionDeclaration; env: Frame } + | { kind: "null" } + | { kind: "undef" } + | { kind: "any" }; + +export interface ExecOut { + ret: SymVal | null; + thrown: { kind: "bool"; t: Bool<"main"> } | null; + /** Solver must receive these for absence claims: unrolled loops that had + * not terminated within K iterations produce a completion assumption. */ + assumptions: Bool<"main">[]; + /** Every construct degraded to `any` — harness reports these; a property + * whose counterexample depends on an `any` hole is NOT a finding. */ + degradations: string[]; + /** The term context — carries `decls` (symbolic constants created while + * translating) so harnesses can emit `(declare-fun ...)` lines. */ + ctx: import("./terms.js").Context; +} + +interface InterpreterOptions { + maxArrayLength?: number; // default 8 + maxLoopIterations?: number; // default 8 + maxInlineDepth?: number; // default 12 + /** Implementations for INJECTED PORTS (public parameter seams like + * CompressionCore's `countTokens`). When a call resolves to a parameter + * with a port implementation, the hook runs instead of degrading. Ports + * are part of the module's public contract — hosts inject their own + * token counters — so this is composition, not bug-specific modeling. */ + portImpls?: Record SymVal>; +} + +/* ------------------------------------------------------------------ helpers */ + +const num = (ctx: Z3, v: number): Num => ctx.Real.val(v); +let numHoleCounter = 0; + +const bool = (ctx: Z3, v: boolean): Bool<"main"> => ctx.Bool.val(v); + +/** Constant side-channels: z3 expr objects created from JS constants are + * tagged so arithmetic/comparisons fold at construction time instead of + * bloating the symbolic DAG with trivially-decidable subterms. */ +const strConst = { + get: (t: Str | undefined): string | undefined => (t && typeof t.c === "string" ? t.c : undefined), +}; +const numConst = { + get: (t: Num | undefined): number | undefined => (t && typeof t.c === "number" ? t.c : undefined), +}; +const boolConst = { + get: (t: Bool<"main"> | undefined): boolean | undefined => (t && typeof t.c === "boolean" ? t.c : undefined), +}; +const strc = (ctx: Z3, sv: string): Str => ctx.String.val(sv); +/** Foldable numeric equality: constant arguments decide immediately. */ +const numEq = (ctx: Z3, a: Num, b: Num): Bool<"main"> => { + const ac = numC(a), bc = numC(b); + if (ac !== undefined && bc !== undefined) return bool(ctx, ac === bc); + return a.eq(b); +}; + +/** Numeric constant lookup with structural fallback: constants created by + * harness code via raw ctx.Real.val(n) are recognized by their printed + * form so constant folding still applies to them. */ +const numC = (t: Num): number | undefined => { + const hit = numConst.get(t); + if (hit !== undefined) return hit; + const noKids = (t as unknown as { kids?: unknown[] }).kids; + if (noKids !== undefined && noKids.length > 0) return undefined; // composite — never a literal + const str = t.toString(); + if (/^-?\d+\.0+$/.test(str)) return parseInt(str, 10); + if (/^-?\d+$/.test(str)) return parseInt(str, 10); + if (/^-?\d+\.\d+$/.test(str)) { + const f = parseFloat(str); + if (Number.isFinite(f)) return f; + } + return undefined; +}; + +function isNum(v: SymVal | undefined): v is { kind: "num"; t: Num } { + return v !== undefined && v.kind === "num"; +} +function isBool(v: SymVal | undefined): v is { kind: "bool"; t: Bool<"main"> } { + return v !== undefined && v.kind === "bool"; +} +function isStr(v: SymVal | undefined): v is { kind: "str"; t: Str } { + return v !== undefined && v.kind === "str"; +} +function isArr(v: SymVal | undefined): v is Extract { + return v !== undefined && v.kind === "arr"; +} +function isObj(v: SymVal | undefined): v is Extract { + return v !== undefined && v.kind === "obj"; +} + +/** Truthiness of a symbolic value as a boolean expr (JS coercion rules for + * the subset: numbers nonzero, strings nonempty, null/undef false). */ +/** Literal check WITHOUT printing: reads the folded const tag only. + * Printing big DAG bools per-statement was exponential (the H2 OOM). */ +function constBool(t: Bool<"main"> | null | undefined): boolean | null { + if (t === null || t === undefined) return null; + if (t.c === true) return true; + if (t.c === false) return false; + const op = (t as unknown as { op?: string }).op; + if (op === "const" || op === "lit") { + // non-folded literal — fall back to a SHORT print (no kids) + const str = (t as unknown as { print(): string }).print ? (t as unknown as { print(): string }).print() : String(t); + if (str === "true") return true; + if (str === "false") return false; + } + return null; +} + + +function truthy(ctx: Z3, v: SymVal): Bool<"main"> | null { + switch (v.kind) { + case "bool": return v.t; + case "num": { + const c = numC(v.t); + if (c !== undefined) return bool(ctx, c !== 0); + return v.t.neq(num(ctx, 0)); + } + case "str": { + const sv = strConst.get(v.t); + if (sv !== undefined) return bool(ctx, sv.length > 0); + return (ctx.ToReal(v.t.length() as never) as Num).neq(num(ctx, 0)); + } + case "null": + case "undef": return bool(ctx, false); + case "arr": { + const tag = (v as { nullUnder?: Bool<"main"> }).nullUnder; + if (!tag) return bool(ctx, true); // arrays are truthy + const tc = boolConst.get(tag); + return tc === undefined ? tag.not() : bool(ctx, !tc); + } + case "obj": { + const tag = (v as { nullUnder?: Bool<"main"> }).nullUnder; + if (!tag) return bool(ctx, true); + const tc = boolConst.get(tag); + return tc === undefined ? tag.not() : bool(ctx, !tc); + } + default: return null; + } +} + +function permutations(n: number): number[][] { + if (n === 0) return [[]]; + if (n === 1) return [[0]]; + const out: number[][] = []; + const rest = permutations(n - 1); + for (const p of rest) { + for (let i = 0; i <= p.length; i++) { + out.push([...p.slice(0, i), n - 1, ...p.slice(i)]); + } + } + return out; +} + +/* ------------------------------------------------------------------ program */ + +interface ModuleInfo { + numbers: Map; + strings: Map; + functions: Map; + constArrowFns: Map; + constBindings: Map; + /** exported names (functions, const arrows, consts) for import resolution */ + exports: Set; + /** export-from re-exports: export {x} from "./y.js" */ + reexports: Map; +} + +/** Resolve relative imports of a file recursively and collect exported + * callables/consts into a global registry (kernel files import helpers + * across files: truncate-tools → truncate/prune/ccr, hide-consumed → refs). */ +function loadImportGraph(entry: string): Map { + const modules = new Map(); + const queue: string[] = [entry]; + const seen = new Set(); + while (queue.length) { + const file = queue.shift()!; + if (seen.has(file)) continue; + seen.add(file); + const { sf: entrySf, info } = loadModule(file); + modules.set(file, info); + for (const stmt of entrySf.statements) { + if (!ts.isImportDeclaration(stmt)) continue; + const spec = (stmt.moduleSpecifier as ts.StringLiteral).text; + if (!spec.startsWith("./") && !spec.startsWith("../")) continue; // node/external: not symbolized + const resolved = resolveTsPath(file, spec); + if (resolved && !seen.has(resolved)) queue.push(resolved); + } + } + return modules; +} + +function resolveTsPath(fromFile: string, spec: string): string | null { + const path = join(dirname(fromFile), spec.replace(/\.js$/, ".ts")); + try { + accessSync(path); + return path; + } catch { + return null; + } +} + +function loadModule(file: string): { sf: ts.SourceFile; info: ModuleInfo } { + const source = readFileSync(file, "utf8"); + const sf = ts.createSourceFile(file, source, ts.ScriptTarget.ES2022, true); + const info: ModuleInfo = { + numbers: new Map(), + strings: new Map(), + functions: new Map(), + constArrowFns: new Map(), + constBindings: new Map(), + exports: new Set(), + reexports: new Map(), + }; + const walk = (node: ts.Node): void => { + if (ts.isFunctionDeclaration(node) && node.name) { + info.functions.set(node.name.text, node); + } + if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.initializer) { + const i = node.initializer; + if (ts.isNumericLiteral(i)) info.numbers.set(node.name.text, Number(i.text)); + else if (ts.isStringLiteral(i)) info.strings.set(node.name.text, i.text); + else if (ts.isArrowFunction(i) || ts.isFunctionExpression(i)) info.constArrowFns.set(node.name.text, i); + else info.constBindings.set(node.name.text, i); + } + ts.forEachChild(node, walk); + }; + walk(sf); + const walkExports = (node: ts.Node): void => { + const hasExportMod = (n: ts.Node): boolean => + !!((n as ts.FunctionDeclaration).modifiers ?? []).some( + (m: ts.ModifierLike) => m.kind === ts.SyntaxKind.ExportKeyword, + ); + if (ts.isFunctionDeclaration(node) && node.name && hasExportMod(node)) { + info.exports.add(node.name.text); + } + if (ts.isVariableStatement(node) && hasExportMod(node)) { + for (const d of node.declarationList.declarations) { + if (ts.isIdentifier(d.name)) info.exports.add(d.name.text); + } + } + if (ts.isExportDeclaration(node) && node.exportClause && ts.isNamedExports(node.exportClause)) { + for (const el of node.exportClause.elements) info.exports.add(el.name.text); + } + ts.forEachChild(node, walkExports); + }; + walkExports(sf); + return { sf, info }; +} + +/* ---------------------------------------------------------------- interpreter */ + +interface Frame { + vars: Map; +} + +interface RunState { + ret: SymVal | null; + /** conditional returns: function returns `val` when `cond` — following + * statements still execute (they produce the not-cond path's value) */ + pendingRets: { cond: Bool<"main">; val: SymVal }[]; + /** shared across branch copies — dedup identical diagnostics */ + seenDeg: Set; + seenAsum: Set; + /** hard unconditional return (dead code after `return`) */ + hardRet: boolean; + thrown: Bool<"main"> | null; + assumptions: Bool<"main">[]; + degradations: string[]; + breakCond: Bool<"main"> | null; + continueCond: Bool<"main"> | null; +} + +interface TranslateInput { + file: string; + function: string; + params: Record; + ctx: Z3; + options?: InterpreterOptions; +} + +export function translateFunction(input: TranslateInput): { exec: () => ExecOut } { + const { file, function: fnName, params, ctx } = input; + + /** z3 Seq.length() is Int-sorted; our numeric arithmetic is Real. */ + const lenR = (seq: Seq<"main">): Num => ctx.ToReal(seq.length() as never) as Num; + const N = input.options?.maxArrayLength ?? 8; + const K = input.options?.maxLoopIterations ?? 8; + /** Constant-folding Boolean combinators: z3's .or/.and do NOT simplify + * `(or false false)` to `false`, and control-flow checks compare the + * printed form against "true"/"false" — so every control Bool must be + * built through these to keep literal chains folded. */ + const orFold = (a: Bool<"main"> | null, b: Bool<"main">): Bool<"main"> => { + if (a === null) return b; + const ab = boolConst.get(a), bb = boolConst.get(b); + if (ab === true || bb === true) return bool(ctx, true); + if (ab === false) return b; + if (bb === false) return a; + return a.or(b); + }; + const andFold = (a: Bool<"main">, b: Bool<"main">): Bool<"main"> => { + const ab = boolConst.get(a), bb = boolConst.get(b); + if (ab === false || bb === false) return bool(ctx, false); + if (ab === true) return b; + if (bb === true) return a; + return a.and(b); + }; + const notFold = (a: Bool<"main">): Bool<"main"> => { + const ab = boolConst.get(a); + if (ab === true) return bool(ctx, false); + if (ab === false) return bool(ctx, true); + return a.not(); + }; + const MAXINLINE = input.options?.maxInlineDepth ?? 12; + const modules = loadImportGraph(file); + const info = modules.get(file)!; + const sf = loadModule(file).sf; + // identifier → definition in an imported module ("./x.js" exports) + const importedSymbols = new Map(); + { + const entrySf = loadModule(file).sf; + for (const stmt of entrySf.statements) { + if (!ts.isImportDeclaration(stmt)) continue; + const spec = (stmt.moduleSpecifier as ts.StringLiteral).text; + if (!spec.startsWith("./") && !spec.startsWith("../")) continue; + const resolved = resolveTsPath(file, spec); + if (!resolved) continue; + const target = modules.get(resolved); + if (!target) continue; + const named = stmt.importClause?.namedBindings; + if (named && ts.isNamedImports(named)) { + for (const el of named.elements) importedSymbols.set(el.name.text, { file: resolved, info: target }); + } + } + } + + /** Module whose file's functions we are currently executing (for cross-file inlining). + * Top of stack = defining module; entry module is the base. */ + const moduleStack: ModuleInfo[] = [info]; + const curInfo = (): ModuleInfo => moduleStack[moduleStack.length - 1] ?? info; + const portMemo = new Map(); + + let numStrCounter = 0; + /** String(number): an uninterpreted decimal string; only its length matters + * for stability properties (1..19 digits) and the replay lock validates the + * witness against the real code. */ + // String(number): memoized per INPUT TERM (same number → same string — + // stability properties across turns need the function property, a fresh + // hole per call would fake instability). + const numStrMemo = new Map(); + const numToStringStrFor = (x: Num): { kind: "str"; t: Str } => { + const key = x.hash(); + const hit = numStrMemo.get(key); + if (hit) return hit; + const hole = ctx.String.const("__numstr_" + numStrCounter++); + const v: { kind: "str"; t: Str } = { kind: "str", t: hole }; + numStrMemo.set(key, v); + return v; + }; + const numToStringStr = (st: RunState): { kind: "str"; t: Str } => { + const hole = ctx.String.const("__numstr_" + numStrCounter++); + pushAsum(st, (ctx.ToReal(hole.length() as never) as Num).ge(num(ctx, 1)).and((ctx.ToReal(hole.length() as never) as Num).le(num(ctx, 19)))); + return { kind: "str", t: hole }; + }; + + const decl = info.functions.get(fnName); + if (!decl?.body) throw new Error(`function ${fnName} not found in ${file}`); + + const lineOf = (node: ts.Node) => sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1; + const bad = (node: ts.Node, what: string): never => { + throw new TranslationUnsupported(what, file, lineOf(node)); + }; + + /* ---------------- value constructors ---------------- */ + + const arrOf = (elems: SymVal[], len?: Num): Extract => + ({ kind: "arr", elems, len: len ?? num(ctx, elems.length) }); + + /** Convert a REAL JSON value (from concrete JSON.parse) into a SymVal. */ + const jsonToSymVal = (v: unknown, depth: number): SymVal => { + if (v === null) return { kind: "null" }; + if (typeof v === "string") return { kind: "str", t: strc(ctx, v) }; + if (typeof v === "number") return { kind: "num", t: num(ctx, v) }; + if (typeof v === "boolean") return { kind: "bool", t: bool(ctx, v) }; + if (Array.isArray(v)) { + if (v.length > (input.options?.maxArrayLength ?? 8)) return { kind: "any" }; + return arrOf(v.map((x) => jsonToSymVal(x, depth + 1))); + } + if (typeof v === "object") { + const fields = new Map(); + for (const [k, val] of Object.entries(v as Record)) { + fields.set(k, jsonToSymVal(val, depth + 1)); + } + return { kind: "obj", fields }; + } + return { kind: "undef" }; + }; + + // STRICT merge for VARIABLE bindings across branch joins: unlike mergeVal + // (whose undef/null-loses erasure is only sound for never-read tail slots), + // this preserves the maybe-null/maybe-undefined distinction via nullUnder — + // `if (lastTextCore)` must stay a SYMBOLIC condition, not fold true. + const mergeValStrict = (cond: Bool<"main">, a: SymVal, b: SymVal): SymVal => { + const objSide = a.kind === "obj" || a.kind === "arr" ? a : b.kind === "obj" || b.kind === "arr" ? b : null; + const nullish = b.kind === "null" || b.kind === "undef" ? b : a.kind === "null" || a.kind === "undef" ? a : null; + if (objSide && nullish) { + const aIsObj = a === objSide; + // value is the obj when nullUnder is false: + // mergeValStrict(cond, obj, null) → nullUnder = ¬cond + // mergeValStrict(cond, null, obj) → nullUnder = cond + const side = aIsObj ? notFold(cond) : cond; + const prev = (objSide as { nullUnder?: Bool<"main"> }).nullUnder; + const tag = prev ? orFold(prev, side) : side; + return objSide.kind === "obj" + ? { kind: "obj", fields: objSide.fields, nullUnder: tag } + : { kind: "arr", elems: objSide.elems, len: objSide.len, nullUnder: tag }; + } + return mergeVal(cond, a, b); + }; + + const mergeVal = (cond: Bool<"main">, a: SymVal, b: SymVal): SymVal => { + // undef/null loses to any concrete value: out-of-bounds fallbacks in + // element-access chains (guarded by length checks in real code) must not + // poison the merged object. Real out-of-range access throws and is caught + // by the replay lock. + if (a.kind === "undef" && b.kind !== "undef") return mergeVal(cond.not(), b, { kind: "undef" }) === null ? { kind: "any" } : b; + if (b.kind === "undef" && a.kind !== "undef") return a; + if (a.kind === "null" && b.kind !== "null") return b; + if (b.kind === "null" && a.kind !== "null") return a; + if (a.kind !== b.kind) { + return { kind: "any" }; + } + switch (a.kind) { + case "num": { + const bb = b as Extract; + const ac = numC(a.t), bc = numC(bb.t); + if (ac !== undefined && ac === bc) return a; // same constant: no ITE + return { kind: "num", t: ctx.If(cond, a.t, bb.t) }; + } + case "bool": { + const bb = b as Extract; + const ac = boolConst.get(a.t), bc = boolConst.get(bb.t); + if (ac !== undefined && ac === bc) return a; + return { kind: "bool", t: ctx.If(cond, a.t, bb.t) }; + } + case "str": { + const bb = b as Extract; + const ac = strConst.get(a.t), bc = strConst.get(bb.t); + if (ac !== undefined && ac === bc) return a; + return { kind: "str", t: ctx.If(cond, a.t, bb.t) }; + } + case "null": case "undef": return a; + case "fn": return a; // closures are immutable; identical node survives + case "arr": { + const bb = b as Extract; + const n = Math.max(a.elems.length, bb.elems.length); + const elems: SymVal[] = []; + for (let i = 0; i < n; i++) { + // missing slots merge as undef (undef loses to concrete values in + // mergeVal) so growing arrays keep their element structure + elems.push(mergeVal(cond, a.elems[i] ?? { kind: "undef" }, bb.elems[i] ?? { kind: "undef" })); + } + const la = numC(a.len), lb = numC(bb.len); + if (la !== undefined && la === lb) return { kind: "arr", elems, len: a.len }; // same length: keep const + return { kind: "arr", elems, len: ctx.If(cond, a.len, bb.len) }; + } + case "obj": { + const bb = b as Extract; + const fields = new Map(); + for (const key of new Set([...a.fields.keys(), ...bb.fields.keys()])) { + const va = a.fields.get(key); + const vb = bb.fields.get(key); + if (va && vb) fields.set(key, mergeVal(cond, va, vb)); + else if (va) fields.set(key, guardedNeutral(cond, va)); + else if (vb) fields.set(key, guardedNeutral(cond.not(), vb)); + } + // merge symbolic-key bindings: present on one arm only means the + // other arm never wrote it → undefined there (guard survives) + // symFields entries carry their WRITE guard: a binding present on + // one arm only keeps the path condition in .g (accumulated with + // andFold) — the read fan tests (key = idx AND g) so a dead-path + // write never shadows an older binding for the same string key. + let symFields: Map; v: SymVal }> | undefined; + const sa = a.symFields, sb = bb.symFields; + if (sa || sb) { + symFields = new Map; v: SymVal }>(); + const seen = new Set(); + const bothArm = (ga: Bool<"main">, va: SymVal, gb: Bool<"main">, vb: SymVal): { g: Bool<"main">; v: SymVal } => { + const gT = andFold(cond, ga), gE = andFold(cond.not(), gb); + const gLive = orFold(gT, gE); + if (gLive === null) return { g: bool(ctx, false), v: va }; + let v: SymVal; + if (constBool(gT) === true) v = va; + else if (constBool(gE) === true) v = vb; + else v = iteVal(gT, va, iteVal(gE, vb, neutralFor(vb))); + return { g: gLive, v }; + }; + if (sa) for (const [kt, ea] of sa) { + seen.add(kt); + const eb = sb?.get(kt); + symFields.set(kt, eb ? bothArm(ea.g, ea.v, eb.g, eb.v) : { g: andFold(cond, ea.g)!, v: ea.v }); + } + if (sb) for (const [kt, eb] of sb) { + if (seen.has(kt)) continue; + symFields.set(kt, { g: andFold(cond.not(), eb.g)!, v: eb.v }); + } + } + return { kind: "obj", fields, ...(symFields ? { symFields } : {}) }; + } + default: return { kind: "any" }; + } + }; + + // A value written under a path condition into a container slot must leave + // the IN-DOMAIN neutral in the untaken arm: downstream truthiness/membership + // reads (len>0 etc.) treat neutral as "absent". Without this the undef-loses + // merge rule silently drops the write's path guard (same failure class as + // the Set-add phantom-slot bug). + const guardedNeutral = (cond: Bool<"main">, v: SymVal): SymVal => { + switch (v.kind) { + case "str": return { kind: "str", t: ctx.If(cond, v.t, ctx.String.val("")) }; + case "num": return { kind: "num", t: ctx.If(cond, v.t, ctx.Real.val(0)) }; + case "bool": return { kind: "bool", t: ctx.If(cond, v.t, ctx.Bool.val(false)) }; + case "obj": case "arr": return iteStrict(cond, v, { kind: "null" }); + default: return v; + } + }; + + const neutralFor = (v: SymVal): SymVal => { + switch (v.kind) { + case "str": return { kind: "str", t: ctx.String.val("") }; + case "num": return { kind: "num", t: ctx.Real.val(0) }; + case "bool": return { kind: "bool", t: ctx.Bool.val(false) }; + default: return { kind: "null" }; + } + }; + + const iteStrict = (cond: Bool<"main">, a: SymVal, b: SymVal): SymVal => { + if (a === b) return a; + const cb = constBool(cond); + if (cb === true) return a; + if (cb === false) return b; + return mergeValStrict(cond, a, b); + }; + + const iteVal = (cond: Bool<"main">, a: SymVal, b: SymVal): SymVal => { + if (a === b) return a; + const cb = constBool(cond); + if (cb === true) return a; + if (cb === false) return b; + // Equal constants fold to the constant: ite(cond, 1, 1) is 1 in every + // model. Without this, branch merges lose const tags (two structurally + // equal Terms are distinct objects) and everything downstream degrades + // to symbolic — including array lengths that guard loop unrolling. + if (a.kind === b.kind && a.kind === "num" && b.kind === "num") { + const ca = numC(a.t), cbd = numC(b.t); + if (ca !== undefined && ca === cbd) return a; + } + if (a.kind === b.kind && a.kind === "str" && b.kind === "str") { + const ca = strConst.get(a.t), cb2 = strConst.get(b.t); + if (ca !== undefined && ca === cb2) return a; + } + if (a.kind === b.kind && a.kind === "bool" && b.kind === "bool") { + const ca = boolConst.get(a.t), cb2 = boolConst.get(b.t); + if (ca !== undefined && ca === cb2) return a; + } + if (a.kind === "undef" && b.kind === "undef") return a; + return mergeVal(cond, a, b); + }; + + /** JS ?: on symbolic values. */ + const jsOr = (a: SymVal, b: () => SymVal): SymVal => { + const t = truthy(ctx, a); + if (t === null) return { kind: "any" }; + const av = a; + const bv = b(); + // (a || b): if a truthy → a else b + return iteVal(t, av.kind === "bool" ? { kind: "bool", t: t } : av, bv); + }; + const jsAnd = (a: SymVal, b: () => SymVal): SymVal => { + const t = truthy(ctx, a); + if (t !== null) { + const bb = b(); + // Prefer exact truthiness when both sides coerce: `(a && b)` in a + // boolean context is `truthy(a) ∧ truthy(b)`. Operand-identity + // (`a && b` evaluating to `a` when falsy) is only observable via + // value-equality/aliasing patterns absent from the guarded subset. + const tbv = truthy(ctx, bb); + if (tbv !== null) return { kind: "bool", t: andFold(t, tbv) }; + return iteVal(t, bb, a.kind === "bool" ? { kind: "bool", t: t } : a); + } + if (t === null) return { kind: "any" }; + return iteVal(t, b(), a.kind === "bool" ? { kind: "bool", t: t } : a); + }; + + const jsNullish = (a: SymVal, b: () => SymVal): SymVal => + iteVal(a.kind === "null" || a.kind === "undef" ? bool(ctx, true) : bool(ctx, false), b(), a); + + /* ---------------- statements ---------------- */ + + const runState0 = (): RunState => ({ + ret: null, pendingRets: [], hardRet: false, thrown: null, + assumptions: [], degradations: [], seenDeg: new Set(), seenAsum: new Set(), + breakCond: null, continueCond: null, + }); + + interface Scope { + frame: Frame; + depth: number; + } + + /** Sequential statement execution with conditional stop semantics: + * after a possibly-taken break/continue, the REMAINING statements run on a + * copy under ¬stop — recursively, so nested stops (break-if followed by + * continue-if in the same block) guard correctly. */ + const execSeq = (stmts: readonly ts.Statement[], frame: Frame, st: RunState, depth: number): void => { + for (let i = 0; i < stmts.length; i++) { + execStmt(stmts[i]!, frame, st, depth); + const bc = st.breakCond; + const cc = st.continueCond; + const bcT = bc !== null && constBool(bc) !== false; + const ccT = cc !== null && constBool(cc) !== false; + if (bcT || ccT) { + const stopCond = bc && cc ? orFold(bc as Bool<"main">, cc as Bool<"main">) : ((bc ?? cc) as Bool<"main">); + if (i + 1 < stmts.length && constBool(stopCond) !== true) { + const f2: Frame = { vars: new Map(frame.vars) }; + const s2: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execSeq(stmts.slice(i + 1), f2, s2, depth); + mergeBranch(frame, st, notFold(stopCond), f2, s2, null, null); + } + break; + } + if (st.hardRet) break; + if (constBool(st.thrown) === true) break; + } + }; + + let stmtCount = 0; + const stmtLines = new Map(); + const execStmt = (stmt: ts.Statement, frame: Frame, st: RunState, depth: number): void => { + if (st.hardRet) return; + if (constBool(st.thrown) === true) return; + if (constBool(st.breakCond) === true) return; + if (constBool(st.continueCond) === true) return; + if (ts.isBlock(stmt)) { execSeq(stmt.statements, frame, st, depth); return; } + if (ts.isTypeAliasDeclaration(stmt) || ts.isInterfaceDeclaration(stmt) || ts.isEnumDeclaration(stmt)) return; // type-level only — erased + if (ts.isVariableStatement(stmt)) { + for (const d of stmt.declarationList.declarations) { + const init = d.initializer ? evalExpr(d.initializer, frame, st, depth) : { kind: "undef" as const }; + if (ts.isIdentifier(d.name)) { + frame.vars.set(d.name.text, init); + } else if (ts.isObjectBindingPattern(d.name)) { + for (const el of d.name.elements) { + if (!ts.isBindingElement(el) || !ts.isIdentifier(el.name)) continue; + const propName = el.propertyName && ts.isIdentifier(el.propertyName) ? el.propertyName.text : el.name.text; + frame.vars.set(el.name.text, member(init, propName, el, st)); + } + } else if (ts.isArrayBindingPattern(d.name)) { + const arrV = init.kind === "arr" ? init : { kind: "any" as const }; + if (arrV.kind === "arr") d.name.elements.forEach((el, j) => { + if (!ts.isBindingElement(el) || !ts.isIdentifier(el.name)) return; + frame.vars.set(el.name.text, elementAccess(arrV, { kind: "num", t: num(ctx, j) }, el, st)); + }); + } else { + bad(stmt, "non-identifier declaration"); + } + } + return; + } + if (ts.isExpressionStatement(stmt)) { + evalExpr(stmt.expression, frame, st, depth); + return; + } + if (ts.isReturnStatement(stmt)) { + if (st.ret === null) { + st.ret = stmt.expression ? evalExpr(stmt.expression, frame, st, depth) : { kind: "undef" }; + st.hardRet = true; + } else { + // a pending conditional return already captured the tail; a second + // return can only be reached on the not-cond path + st.pendingRets.push({ cond: bool(ctx, false), val: stmt.expression ? evalExpr(stmt.expression, frame, st, depth) : { kind: "undef" } }); + st.hardRet = true; + } + return; + } + if (ts.isThrowStatement(stmt)) { + // The thrown value itself never matters — only the fact. + st.thrown = bool(ctx, true); + return; + } + if (ts.isIfStatement(stmt)) { + const c = evalExpr(stmt.expression, frame, st, depth); + const tb = truthy(ctx, c); + if (tb === null) { degrade(st, stmt, "if condition degraded"); return; } + // Dead-branch pruning: a constant condition executes only the live + // arm, in-frame. Eager both-arm execution would force provably-dead + // code (e.g. a regex in a never-taken branch) to be translatable. + const cb = constBool(tb); + if (cb === true) { execStmt(stmt.thenStatement, frame, st, depth); return; } + if (cb === false) { if (stmt.elseStatement) execStmt(stmt.elseStatement, frame, st, depth); return; } + const thenFrame: Frame = { vars: new Map(frame.vars) }; + const thenState: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.thenStatement, thenFrame, thenState, depth); + let elseFrame: Frame | null = null; + let elseState: RunState | null = null; + if (stmt.elseStatement) { + elseFrame = { vars: new Map(frame.vars) }; + elseState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.elseStatement, elseFrame, elseState, depth); + } + mergeBranch(frame, st, tb, thenFrame, thenState, elseFrame, elseState); + return; + } + if (ts.isForOfStatement(stmt)) { + // Unroll over bounded array slots. + let iter = evalExpr(stmt.expression, frame, st, depth); + if (iter.kind === "obj" && iter.fields.has("__set_values")) { + // for (const k of someSet) — iterate the Set-model's value list + const vals = iter.fields.get("__set_values")!; + iter = vals.kind === "arr" ? vals : ({ kind: "arr", elems: [], len: num(ctx, 0) } as SymVal); + } + if (!isArr(iter)) { degrade(st, stmt, "for-of over non-array"); return; } + bindForVar(stmt.initializer, { kind: "undef" }, frame); + // Dead-slot pruning: when the array length is a KNOWN constant, skip + // slots beyond it entirely — executing dead iterations would pollute + // live state with out-of-bounds degradations and any-values. + const lenC = numC(iter.len); + const slotCeil = lenC !== undefined ? Math.min(iter.elems.length, Math.ceil(lenC)) : iter.elems.length; + for (let k = 0; k < slotCeil; k++) { + const guard = andFold(iter.len.gt(num(ctx, k)), st.breakCond ?? bool(ctx, true)); + const prBefore = st.pendingRets.length; + const elemFrame: Frame = { vars: new Map(frame.vars) }; + const elemState: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + bindForVar(stmt.initializer, iter.elems[k], elemFrame); + execStmt(stmt.statement, elemFrame, elemState, depth); + mergeBranch(frame, st, guard, elemFrame, elemState, null, null); + st.continueCond = null; // continue ends only this iteration + if (st.hardRet || constBool(st.thrown) === true || constBool(st.breakCond) === true) break; + // A conditional return/throw fired under this iteration's guard: a + // real execution would have left the whole function — treat it as a + // loop exit (post-loop statements run under ¬exit via rest-on-copy). + if (st.pendingRets.length > prBefore || (st.thrown !== null && constBool(st.thrown) === undefined)) { + st.breakCond = orFold(st.breakCond, guard); + break; + } + } + // Elements beyond the declared length never execute: nothing needed. + return; + } + if (ts.isForStatement(stmt)) { + if (stmt.initializer) { + if (ts.isVariableDeclarationList(stmt.initializer)) { + for (const d of stmt.initializer.declarations) { + if (ts.isIdentifier(d.name)) { + frame.vars.set(d.name.text, d.initializer ? evalExpr(d.initializer, frame, st, depth) : { kind: "undef" }); + } + } + } else { + evalExpr(stmt.initializer, frame, st, depth); + } + } + for (let k = 0; k < K; k++) { + const c = stmt.condition ? evalExpr(stmt.condition, frame, st, depth) : { kind: "bool" as const, t: bool(ctx, true) }; + const tb = truthy(ctx, c); + if (tb === null) { degrade(st, stmt, "for condition degraded"); return; } + // Constant-false condition: the loop provably finished — do NOT + // execute the body (dead iterations pollute live state). + if (constBool(tb) === false) { + pushAsum(st, bool(ctx, true)); + return; + } + const iterFrame: Frame = { vars: new Map(frame.vars) }; + const prBefore = st.pendingRets.length; + const iterState: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.statement, iterFrame, iterState, depth); + // continue: still run the increment — handled by running increment + // on iterFrame after body (continue skips rest of body only). + if (stmt.incrementor) evalExpr(stmt.incrementor, iterFrame, iterState, depth); + mergeBranch(frame, st, tb, iterFrame, iterState, null, null); + st.continueCond = null; // continue ends only this iteration + if (st.hardRet || constBool(st.thrown) === true || constBool(st.breakCond) === true) return; + if (st.pendingRets.length > prBefore || (st.thrown !== null && constBool(st.thrown) === undefined)) { + st.breakCond = orFold(st.breakCond, tb as Bool<"main">); + return; + } + } + // Loop may need more than K iterations: assume it finished. + const c = stmt.condition ? evalExpr(stmt.condition, frame, st, depth) : null; + const tb = c ? truthy(ctx, c) : bool(ctx, true); + if (tb) pushAsum(st, notFold(tb as Bool<"main">)); + return; + } + if (ts.isWhileStatement(stmt)) { + for (let k = 0; k < K; k++) { + const c = evalExpr(stmt.expression, frame, st, depth); + const tb = truthy(ctx, c); + if (tb === null) { degrade(st, stmt, "while condition degraded"); return; } + const iterFrame: Frame = { vars: new Map(frame.vars) }; + const prBefore = st.pendingRets.length; + const iterState: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.statement, iterFrame, iterState, depth); + mergeBranch(frame, st, tb, iterFrame, iterState, null, null); + st.continueCond = null; // continue ends only this iteration + if (st.hardRet || constBool(st.thrown) === true || constBool(st.breakCond) === true) return; + if (st.pendingRets.length > prBefore || (st.thrown !== null && constBool(st.thrown) === undefined)) { + st.breakCond = orFold(st.breakCond, tb as Bool<"main">); + return; + } + } + const c = evalExpr(stmt.expression, frame, st, depth); + const tb = truthy(ctx, c); + if (tb) pushAsum(st, notFold(tb as Bool<"main">)); + return; + } + if (ts.isBreakStatement(stmt)) { st.breakCond = bool(ctx, true); return; } + if (ts.isContinueStatement(stmt)) { st.continueCond = bool(ctx, true); return; } + if (ts.isSwitchStatement(stmt)) { + // Desugar to if-chain over case expressions (guarded code has small + // switches), with JS entry semantics: a clause's body runs when the + // discriminant matches (or `default` is reached with no prior match) + // AND no earlier entered clause broke out. Dead clauses (entry folds + // false) are never executed — eager execution of provably-dead bodies + // wasted terms and forced untranslatable code into the model. + const disc = evalExpr(stmt.expression, frame, st, depth); + let matched = bool(ctx, false); // some earlier case compared equal + let stopped: Bool<"main"> | null = null; // an entered clause broke out + for (const clause of stmt.caseBlock.clauses) { + let enter: Bool<"main">; + if (ts.isCaseClause(clause)) { + const cv = evalExpr(clause.expression, frame, st, depth); + const eq = valEq(disc, cv); + if (eq === null) { + bad(clause.expression, "case comparison degraded"); + matched = bool(ctx, true); // unconstrained — conservative over-approx + enter = bool(ctx, true); + } else { + matched = orFold(matched, eq); + enter = eq; + } + } else { + enter = notFold(matched); + } + if (stopped) enter = andFold(enter, notFold(stopped)); + const cf: Frame = { vars: new Map(frame.vars) }; + const cs: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + const enterC = constBool(enter); + if (enterC === false) continue; // dead clause: never executed + for (const inner of clause.statements) execStmt(inner, cf, cs, depth); + // JS semantics: `break` inside a switch exits the SWITCH, not the + // enclosing loop. The clause's breakCond is consumed here — only + // return/throw/continue propagate outward. + const broke = cs.breakCond; + cs.breakCond = null; + if (enterC === true) { + // Live clause: apply in-frame (no ITE wrapper needed). + for (const [k2, v] of cf.vars) frame.vars.set(k2, v); + st.ret = cs.ret ?? st.ret; st.hardRet = st.hardRet || cs.hardRet; + st.thrown = cs.thrown ?? st.thrown; + st.breakCond = st.breakCond ? orFold(st.breakCond, cs.breakCond ?? bool(ctx, false)) : cs.breakCond; + st.continueCond = st.continueCond ? orFold(st.continueCond, cs.continueCond ?? bool(ctx, false)) : cs.continueCond; + st.pendingRets.push(...cs.pendingRets); + if (constBool(broke) === true) stopped = bool(ctx, true); + if (st.hardRet || constBool(st.thrown) === true) return; + continue; + } + if (broke) stopped = orFold(stopped ?? bool(ctx, false), andFold(enter, broke)); + mergeBranch(frame, st, enter, cf, cs, null, null); + if (st.hardRet || constBool(st.thrown) === true) return; + } + return; + } + if (ts.isTryStatement(stmt)) { + const tf: Frame = { vars: new Map(frame.vars) }; + const ts_: RunState = { ...st, ret: null, pendingRets: [], hardRet: false, thrown: null, breakCond: null, continueCond: null, assumptions: st.assumptions, degradations: st.degradations }; + execStmt(stmt.tryBlock, tf, ts_, depth); + if (ts_.thrown && stmt.catchClause) { + ts_.thrown = null; // caught + if (stmt.catchClause.variableDeclaration?.name && ts.isIdentifier(stmt.catchClause.variableDeclaration.name)) { + tf.vars.set(stmt.catchClause.variableDeclaration.name.text, { kind: "any" }); + } + execStmt(stmt.catchClause.block, tf, ts_, depth); + } + if (stmt.finallyBlock) execStmt(stmt.finallyBlock, tf, ts_, depth); + // merge try-path back unconditionally + for (const [k2, v] of tf.vars) frame.vars.set(k2, v); + st.ret = ts_.ret ?? st.ret; st.hardRet = st.hardRet || ts_.hardRet; st.thrown = ts_.thrown ?? st.thrown; + st.breakCond = st.breakCond ? orFold(st.breakCond, ts_.breakCond ?? bool(ctx, false)) : ts_.breakCond; + st.continueCond = st.continueCond ? orFold(st.continueCond, ts_.continueCond ?? bool(ctx, false)) : ts_.continueCond; + st.pendingRets.push(...ts_.pendingRets); + // shared accumulators — no merge needed + return; + } + if (ts.isEmptyStatement(stmt)) return; + bad(stmt, `statement ${ts.SyntaxKind[stmt.kind]}`); + }; + + + const bindForVar = (init: ts.ForInitializer, value: SymVal, frame: Frame): void => { + if (ts.isVariableDeclarationList(init)) { + for (const d of init.declarations) { + if (ts.isIdentifier(d.name)) frame.vars.set(d.name.text, value); + else if (ts.isArrayBindingPattern(d.name)) { + // positional destructuring: `[a, b] = pairArray` + for (let j = 0; j < d.name.elements.length; j++) { + const el = d.name.elements[j]; + if (!ts.isBindingElement(el) || !ts.isIdentifier(el.name)) continue; + const v: SymVal = isArr(value) ? (value.elems[j] ?? { kind: "undef" }) : { kind: "undef" }; + frame.vars.set(el.name.text, v); + } + } + } + } + }; + + const pushAsum = (st: RunState, t: Bool<"main">): void => { + if (st.seenAsum === undefined) st.seenAsum = new Set(); + const k = t.hash(); + if (st.seenAsum.has(k)) return; + st.seenAsum.add(k); + st.assumptions.push(t); + }; + + const degrade = (st: RunState, node: ts.Node, what: string): void => { + const key = `${what} @${lineOf(node)}`; + if (st.seenDeg === undefined) st.seenDeg = new Set(); + if (st.seenDeg.has(key)) return; + st.seenDeg.add(key); + st.degradations.push(key); + }; + + const mergeBranch = ( + frame: Frame, + st: RunState, + cond: Bool<"main">, + thenFrame: Frame, thenState: RunState, + elseFrame: Frame | null, elseState: RunState | null, + ): void => { + + // Merge variables — a proper three-way merge: + // both branches wrote → ite(cond, thenVal, elseVal) (pre-branch value is dead) + // only then wrote → ite(cond, thenVal, cur) + // only else wrote → ite(¬cond, elseVal, cur) + // neither wrote → keep cur + // Sequential two-pass merging (then into frame, then else into the + // already-merged frame) builds nested ite(¬c, e, ite(c, t, cur)) terms + // that no constant folding can ever reduce — and worse, when BOTH + // branches assigned, the stale pre-branch value survives inside the + // nested term shape. Reference identity distinguishes written names: + // untouched bindings carry the SAME object reference into branch copies. + const names = new Set([...thenFrame.vars.keys(), ...(elseFrame ? [...elseFrame.vars.keys()] : [])]); + for (const name of names) { + const cur = frame.vars.get(name); + const tv = thenFrame.vars.get(name); + const ev = elseFrame ? elseFrame.vars.get(name) : undefined; + const wroteT = tv !== undefined && tv !== cur; + const wroteE = ev !== undefined && ev !== cur; + if (wroteT && wroteE) frame.vars.set(name, iteStrict(cond, tv, ev)); + else if (wroteT) frame.vars.set(name, cur !== undefined ? iteStrict(cond, tv, cur) : tv); + else if (wroteE) frame.vars.set(name, cur !== undefined ? iteStrict(cond.not(), ev, cur) : ev); + } + // ret merge: branch returns become PENDING conditional returns — the + // statements after the if still execute (they produce the other path's + // value); the final return value folds all pending conditions at the end. + if (thenState.hardRet || thenState.ret !== null) { + const tv = thenState.ret ?? { kind: "undef" }; + st.pendingRets.push({ cond, val: tv }); + } + // Conditional returns from INSIDE the branch (e.g. `if (x) return` in a + // loop body) live only in pendingRets — hardRet/ret stay null, so lifting + // them under the same if would DROP them: the while-loop early-return + // detection compares pendingRets counts and never fires; the loop then + // unrolls to exhaustion and the post-loop `throw` poisons the caller. + for (const pr of thenState.pendingRets) st.pendingRets.push({ cond: andFold(cond, pr.cond), val: pr.val }); + if (elseState && (elseState.hardRet || elseState.ret !== null)) { + const ev = elseState.ret ?? { kind: "undef" }; + st.pendingRets.push({ cond: notFold(cond), val: ev }); + } + for (const pr of elseState?.pendingRets ?? []) st.pendingRets.push({ cond: andFold(notFold(cond), pr.cond), val: pr.val }); + // Flags merge — normalize provably-false flags back to null: a + // const-false TERM is not nullish, and every consumer does + // `st.flag ?? default`; a lingering false term flips loop guards + // (for-of iteration k>=1 dying to `andFold(guard, false)`). + const tThrown = thenState.thrown; + const eThrown = elseState?.thrown ?? null; + const tB = tThrown ?? bool(ctx, false); + const eB = eThrown ?? bool(ctx, false); + const mergedThrow = st.thrown ?? bool(ctx, false); + const nt = orFold(orFold(mergedThrow, andFold(cond, tB)), andFold(notFold(cond), eB)); + st.thrown = constBool(nt) === false ? null : nt; + const tBrk = thenState.breakCond ?? null; + const eBrk = elseState?.breakCond ?? null; + const mergedBrk = st.breakCond ?? bool(ctx, false); + const nb = orFold(orFold(mergedBrk, tBrk ? andFold(cond, tBrk) : bool(ctx, false)), eBrk ? andFold(notFold(cond), eBrk) : bool(ctx, false)); + st.breakCond = constBool(nb) === false ? null : nb; + const tCnt = thenState.continueCond ?? null; + const eCnt = elseState?.continueCond ?? null; + const mergedCnt = st.continueCond ?? bool(ctx, false); + const nc = orFold(orFold(mergedCnt, tCnt ? andFold(cond, tCnt) : bool(ctx, false)), eCnt ? andFold(notFold(cond), eCnt) : bool(ctx, false)); + st.continueCond = constBool(nc) === false ? null : nc; + const pushAll = (dst: T[], src: T[]): void => { if (src !== dst) for (const x of src) dst.push(x); }; + pushAll(st.assumptions, thenState.assumptions); if (elseState) pushAll(st.assumptions, elseState.assumptions); + // degradations shared (single array across copies) + }; + + const valEq = (a: SymVal, b: SymVal): Bool<"main"> | null => { + if (isNum(a) && isNum(b)) { + const ac = numC(a.t), bc = numC(b.t); + if (ac !== undefined && bc !== undefined) return bool(ctx, ac === bc); + return a.t.eq(b.t); + } + if (isStr(a) && isStr(b)) { + const ac = strConst.get(a.t), bc = strConst.get(b.t); + if (ac !== undefined && bc !== undefined) return bool(ctx, ac === bc); + if (typeof (a.t as { eq?: unknown }).eq !== "function" || typeof (b.t as { eq?: unknown }).eq !== "function") { + return null; + } + return a.t.eq(b.t); + } + if (isBool(a) && isBool(b)) { + const ac = boolConst.get(a.t), bc = boolConst.get(b.t); + if (ac !== undefined && bc !== undefined) return bool(ctx, ac === bc); + return a.t.eq(b.t); + } + if (a.kind === "null" && b.kind === "null") return bool(ctx, true); + if (a.kind === "undef" && b.kind === "undef") return bool(ctx, true); + return null; + }; + + /* ---------------- expressions ---------------- */ + + const evalExpr = (expr: ts.Expression, frame: Frame, st: RunState, depth: number): SymVal => { + if (ts.isNewExpression(expr)) { + return evalNew(expr, frame, st, depth); + } + if (ts.isArrowFunction(expr) || ts.isFunctionExpression(expr)) { + return { kind: "fn", node: expr, env: frame }; + } + if ( + ts.isCallExpression(expr) && + ts.isPropertyAccessExpression(expr.expression) && + ts.isIdentifier(expr.expression.expression) && + expr.expression.expression.text === "Math" + ) { + const meth = expr.expression.name.text; + const argVals = expr.arguments.map((a) => evalExpr(a, frame, st, depth)); + return mathCall(meth, argVals, expr, st); + } + if (ts.isNumericLiteral(expr)) { + // Numeric literals become Real; note 0.95 etc are exact rationals here. + return { kind: "num", t: num(ctx, Number(expr.text.replace(/_/g, ""))) }; + } + if (ts.isStringLiteral(expr) || ts.isNoSubstitutionTemplateLiteral(expr)) { + return { kind: "str", t: strc(ctx, expr.text) }; + } + if (expr.kind === ts.SyntaxKind.TrueKeyword) return { kind: "bool", t: bool(ctx, true) }; + if (expr.kind === ts.SyntaxKind.FalseKeyword) return { kind: "bool", t: bool(ctx, false) }; + if (expr.kind === ts.SyntaxKind.NullKeyword) return { kind: "null" }; + if (expr.kind === ts.SyntaxKind.UndefinedKeyword) return { kind: "undef" }; + if (ts.isIdentifier(expr)) { + const v = frame.vars.get(expr.text); + if (v) return v; + const mi = curInfo(); + const c = mi.numbers.get(expr.text) ?? info.numbers.get(expr.text); + if (c !== undefined) return { kind: "num", t: num(ctx, c) }; + const s = mi.strings.get(expr.text) ?? info.strings.get(expr.text); + if (s !== undefined) return { kind: "str", t: strc(ctx, s) }; + const arrow = mi.constArrowFns.get(expr.text) ?? info.constArrowFns.get(expr.text); + if (arrow) return evalExpr(arrow, frame, st, depth); + const binding = mi.constBindings.get(expr.text) ?? info.constBindings.get(expr.text); + if (binding) return evalExpr(binding, frame, st, depth); + const imported = importedSymbols.get(expr.text); + if (imported) { + const ic = imported.info.numbers.get(expr.text); + if (ic !== undefined) return { kind: "num", t: num(ctx, ic) }; + const is = imported.info.strings.get(expr.text); + if (is !== undefined) return { kind: "str", t: ctx.String.val(is) }; + const ia = imported.info.constArrowFns.get(expr.text); + if (ia) return evalExpr(ia, frame, st, depth); + const ib = imported.info.constBindings.get(expr.text); + if (ib) return evalExpr(ib, frame, st, depth); + } + if (expr.text === "undefined") return { kind: "undef" }; + bad(expr, `unbound identifier ${expr.text}`); + } + if (ts.isParenthesizedExpression(expr)) return evalExpr(expr.expression, frame, st, depth); + if (ts.isAsExpression(expr) || ts.isNonNullExpression(expr) || ts.isSatisfiesExpression(expr) || ts.isTypeAssertionExpression(expr)) { + return evalExpr(expr.expression, frame, st, depth); + } + if (ts.isTypeOfExpression(expr)) { + const v = evalExpr(expr.expression, frame, st, depth); + switch (v.kind) { + case "str": return { kind: "str", t: ctx.String.val("string") }; + case "num": return { kind: "str", t: ctx.String.val("number") }; + case "bool": return { kind: "str", t: ctx.String.val("boolean") }; + case "obj": case "null": return { kind: "str", t: ctx.String.val("object") }; + case "arr": return { kind: "str", t: ctx.String.val("object") }; + case "fn": return { kind: "str", t: ctx.String.val("function") }; + case "undef": return { kind: "str", t: ctx.String.val("undefined") }; + default: degrade(st, expr, "typeof on degraded value"); return { kind: "any" }; + } + } + if (ts.isPrefixUnaryExpression(expr)) { + if (expr.operator === ts.SyntaxKind.ExclamationToken) { + const v = evalExpr(expr.operand, frame, st, depth); + const t = truthy(ctx, v); + return t ? { kind: "bool", t: t.not() } : { kind: "any" }; + } + if (expr.operator === ts.SyntaxKind.MinusToken) { + const v = evalExpr(expr.operand, frame, st, depth); + return isNum(v) ? { kind: "num", t: v.t.neg() } : { kind: "any" }; + } + if (expr.operator === ts.SyntaxKind.PlusToken) return evalExpr(expr.operand, frame, st, depth); + bad(expr, `prefix ${ts.SyntaxKind[expr.operator]}`); + } + if (ts.isBinaryExpression(expr)) return evalBinary(expr, frame, st, depth); + if (ts.isConditionalExpression(expr)) { + const c = truthy(ctx, evalExpr(expr.condition, frame, st, depth)); + if (!c) { degrade(st, expr, "ternary condition degraded"); return { kind: "any" }; } + const cb = constBool(c); + if (cb === true) return evalExpr(expr.whenTrue, frame, st, depth); + if (cb === false) return evalExpr(expr.whenFalse, frame, st, depth); + const a = evalExpr(expr.whenTrue, frame, st, depth); + const b = evalExpr(expr.whenFalse, frame, st, depth); + return iteVal(c, a, b); + } + if (ts.isTemplateExpression(expr)) { + const parts: SymVal[] = [{ kind: "str", t: ctx.String.val(expr.head.text) }]; + for (const span of expr.templateSpans) { + let v = evalExpr(span.expression, frame, st, depth); + if (isNum(v)) { + // number→string interpolation: uninterpreted 1-8 digit string. + // Content-irrelevant for structural properties; replay lock still + // validates any witness end-to-end. + const hole = ctx.String.const("__numstr_" + numHoleCounter++); + pushAsum(st, (ctx.ToReal(hole.length() as never) as Num).ge(num(ctx, 1)).and((ctx.ToReal(hole.length() as never) as Num).le(num(ctx, 8)))); + v = { kind: "str", t: hole }; + } + parts.push(v); + } + return concatAll(parts); + } + if (ts.isArrayLiteralExpression(expr)) { + const elems: SymVal[] = []; + for (const el of expr.elements) { + if (ts.isSpreadElement(el)) { + const v = evalExpr(el.expression, frame, st, depth); + if (isArr(v)) elems.push(...v.elems); + else degrade(st, el, "spread of non-array in literal"); + } else { + elems.push(evalExpr(el, frame, st, depth)); + } + } + const count = expr.elements.filter((e) => !ts.isSpreadElement(e)).length + + expr.elements.filter((e) => ts.isSpreadElement(e)).length; // spread len merged below if needed + void count; + // declared length: count of elements (spread assumed full — guarded code + // builds arrays with concrete sources; replay lock validates) + let declaredLen: Num = num(ctx, 0); + for (const el of expr.elements) { + if (ts.isSpreadElement(el)) { + const v = evalExpr(el.expression, frame, st, depth); + declaredLen = isArr(v) ? declaredLen.add(v.len) : declaredLen; + } else declaredLen = declaredLen.add(num(ctx, 1)); + } + return arrOf(elems, declaredLen); + } + if (ts.isObjectLiteralExpression(expr)) { + let symFields: Map; v: SymVal }> | undefined; + const fields = new Map(); + for (const prop of expr.properties) { + if (ts.isSpreadAssignment(prop)) { + const v = evalExpr(prop.expression, frame, st, depth); + if (isObj(v)) { + for (const [k2, v2] of v.fields) fields.set(k2, v2); + if (v.symFields) symFields = new Map(v.symFields); + } + else degrade(st, prop, "object spread of non-object"); + } else if (ts.isPropertyAssignment(prop) && ts.isIdentifier(prop.name)) { + fields.set(prop.name.text, evalExpr(prop.initializer, frame, st, depth)); + } else if (ts.isShorthandPropertyAssignment(prop)) { + fields.set(prop.name.text, evalExpr(prop.name, frame, st, depth)); + } else { + degrade(st, prop, "object literal member"); + } + } + return { kind: "obj", fields, ...(symFields ? { symFields } : {}) }; + } + if (ts.isPropertyAccessExpression(expr)) { + const obj = evalExpr(expr.expression, frame, st, depth); + return member(obj, expr.name.text, expr, st); + } + if (ts.isElementAccessExpression(expr)) { + const obj = evalExpr(expr.expression, frame, st, depth); + const idx = evalExpr(expr.argumentExpression, frame, st, depth); + return elementAccess(obj, idx, expr, st); + } + if (ts.isCallExpression(expr)) return evalCall(expr, frame, st, depth); + if (ts.isNonNullExpression(expr)) { + // `x!` non-null assertion — type-level only, value is x + return evalExpr(expr.expression, frame, st, depth); + } + if (ts.isRegularExpressionLiteral(expr)) { + // /pat/flags — regex-literal carrier; String.replace/test model it. + return { kind: "obj", fields: new Map([["__regex", { kind: "str", t: ctx.String.val(expr.text) }]]) } as SymVal; + } + if (expr.kind === ts.SyntaxKind.PostfixUnaryExpression || expr.kind === ts.SyntaxKind.PrefixUnaryExpression) { + const u = expr as ts.PrefixUnaryExpression | ts.PostfixUnaryExpression; + const one = num(ctx, 1); + const v = evalExpr(u.operand, frame, st, depth); + if (isNum(v) && ts.isIdentifier(u.operand)) { + const cv0 = isNum(v) ? numC(v.t) : undefined; + const nv = cv0 !== undefined + ? { kind: "num" as const, t: num(ctx, u.operator === ts.SyntaxKind.PlusPlusToken ? cv0 + 1 : cv0 - 1) } + : { kind: "num" as const, t: u.operator === ts.SyntaxKind.PlusPlusToken ? v.t.add(one) : v.t.sub(one) }; + frame.vars.set(u.operand.text, nv); + return u.kind === ts.SyntaxKind.PrefixUnaryExpression ? nv : v; + } + bad(expr, "update expression on non-number"); + } + if (ts.isDeleteExpression(expr)) { + // `delete obj[k]` / `delete obj.k`: the slot reads as undefined + // afterwards. Modeled as an undef store (missing field ≡ undef). + const t = expr.expression; + if (ts.isElementAccessExpression(t)) { + const base = evalExpr(t.expression, frame, st, depth); + const idx = evalExpr(t.argumentExpression, frame, st, depth); + if (isObj(base) && isStr(idx)) { + const fields = new Map(base.fields); + for (const key of [...fields.keys()]) { + fields.set(key, iteVal(idx.t.eq(ctx.String.val(key)), { kind: "undef" }, fields.get(key)!)); + } + // symbolic-key delete: drop symFields entries whose key matches the + // index term (structural identity) — the merge machinery then folds + // the removal into the enclosing path guard. + let symFields: Map; v: SymVal }> | undefined; + if (base.symFields) { + symFields = new Map(base.symFields); + for (const kt of [...symFields.keys()]) { + if (kt === idx.t || (kt.hash !== undefined && idx.t.hash !== undefined && kt.hash() === idx.t.hash())) { + symFields.delete(kt); + } + } + } + const newObj: SymVal = { kind: "obj", fields, ...(symFields ? { symFields } : {}) }; + writeLValue(t.expression, newObj, frame, st, depth); + return { kind: "bool", t: bool(ctx, true) }; + } + } else if (ts.isPropertyAccessExpression(t)) { + const base = evalExpr(t.expression, frame, st, depth); + if (isObj(base)) { + const fields = new Map(base.fields); + fields.set(t.name.text, { kind: "undef" }); + writeLValue(t.expression, { kind: "obj", fields }, frame, st, depth); + return { kind: "bool", t: bool(ctx, true) }; + } + } + return { kind: "bool", t: bool(ctx, true) }; + } + return bad(expr, `expression ${ts.SyntaxKind[expr.kind]}`); + }; + + // (TS2366: `bad` throws; this satisfies the checker without changing behavior.) + + const rootName = (e: ts.PropertyAccessExpression): string | undefined => + ts.isIdentifier(e.expression) ? e.expression.text : ts.isPropertyAccessExpression(e.expression) ? rootName(e.expression) : undefined; + + const compoundOpKind = (k: ts.SyntaxKind): ts.BinaryOperatorToken["kind"] | undefined => { + switch (k) { + case ts.SyntaxKind.PlusEqualsToken: return ts.SyntaxKind.PlusToken; + case ts.SyntaxKind.MinusEqualsToken: return ts.SyntaxKind.MinusToken; + case ts.SyntaxKind.AsteriskEqualsToken: return ts.SyntaxKind.AsteriskToken; + default: return undefined; + } + }; + + const concatAll = (parts: SymVal[]): SymVal => { + const strs: Str[] = []; + let allConst = true; + let acc = ""; + for (const p of parts) { + if (!isStr(p)) { + return { kind: "any" }; + } + const sv = strConst.get(p.t); + if (sv === undefined) allConst = false; + else acc += sv; + strs.push(p.t); + } + if (strs.length === 0) return { kind: "str", t: strc(ctx, "") }; + if (allConst) return { kind: "str", t: strc(ctx, acc) }; + return { kind: "str", t: strs.reduce((a, b) => a.concat(b)) }; + }; + + const member = (obj: SymVal, name: string, at: ts.Node, st: RunState): SymVal => { + if (isArr(obj)) { + if (name === "length") return { kind: "num", t: obj.len }; + return { kind: "any" }; // methods are handled at call sites + } + if (isStr(obj)) { + if (name === "length") { + const sv = strConst.get(obj.t); + if (sv !== undefined) return { kind: "num", t: num(ctx, sv.length) }; + return { kind: "num", t: lenR(obj.t) }; + } + return { kind: "any" }; + } + if (isObj(obj)) { + // Map/Set .size (property, not method) + if (name === "size" && (obj.fields.has("__map_entries") || obj.fields.has("__set_values"))) { + const e = obj.fields.get("__map_entries") ?? obj.fields.get("__set_values"); + if (e && isArr(e)) return { kind: "num", t: e.len }; + } + const v = obj.fields.get(name); + // missing field reads as `undefined` in JS (NOT unknown/any) — + // `!!message.toolName` on a field-less message must fold to false, + // not poison the condition + return v ?? { kind: "undef" }; + } + if (obj.kind === "null" || obj.kind === "undef") return { kind: "undef" }; + return { kind: "any" }; + }; + + + const elementAccess = (obj: SymVal, idx: SymVal, at: ts.Node, st: RunState): SymVal => { + if (isArr(obj) && isNum(idx)) { + const ic = numC(idx.t); + if (ic !== undefined && Number.isInteger(ic)) { + const rv = ic >= 0 && ic < obj.elems.length ? obj.elems[ic]! : { kind: "undef" as const }; + return rv; + } + // Chain of If(idx == k, elems[k]) for k < N. + let acc: SymVal = { kind: "undef" }; + for (let k = obj.elems.length - 1; k >= 0; k--) { + acc = iteVal(numEq(ctx, idx.t, num(ctx, k)), obj.elems[k], acc); + } + return acc; + } + if (isObj(obj) && isStr(idx)) { + // Dynamic key: pick by string equality over known keys — concrete + // fields first, then symbolic-key bindings (symFields). The FIRST + // layer must keep its key-match guard: wrapping into the in-domain + // neutral (""/0/false) rather than {undef} — the undef-loses merge + // rule would collapse ite(match, v, undef) to v unconditionally and + // hand out values for keys that never matched (phantom reads). + let acc: SymVal = { kind: "undef" }; + const fanLayer = (c: Bool<"main">, v: SymVal, accv: SymVal): SymVal => { + if (accv.kind === "undef" || accv.kind === "null") return iteVal(c, v, neutralFor(v)); + return iteVal(c, v, accv); + }; + for (const [key, v] of obj.fields) { + acc = fanLayer(idx.t.eq(ctx.String.val(key)), v, acc); + } + if (obj.symFields) { + for (const [kt, e] of obj.symFields) { + acc = fanLayer(andFold(idx.t.eq(kt), e.g)!, e.v, acc); + } + } + return acc; + } + return { kind: "any" }; + }; + + const ASSIGN_OPS: ts.SyntaxKind[] = [ + ts.SyntaxKind.EqualsToken, ts.SyntaxKind.PlusEqualsToken, ts.SyntaxKind.MinusEqualsToken, + ts.SyntaxKind.AsteriskEqualsToken, ts.SyntaxKind.SlashEqualsToken, ts.SyntaxKind.PercentEqualsToken, + ]; + + /** Recursive lvalue store with value semantics: rebuilding a[i].f = v + * through the chain (slot-write the array, then rebind upward). Real JS + * uses references — `host = msgs[k]; host.f = v` is visible via msgs[k] + * — so chained stores must rewrite every container on the path. */ + const writeLValue = (target: ts.Expression, value: SymVal, frame: Frame, st: RunState, depth: number): boolean => { + if (ts.isNonNullExpression(target)) return writeLValue(target.expression, value, frame, st, depth); + if (ts.isIdentifier(target)) { frame.vars.set(target.text, value); return true; } + if (ts.isPropertyAccessExpression(target)) { + const base = evalExpr(target.expression, frame, st, depth); + if (isObj(base)) { + const fields = new Map(base.fields); + fields.set(target.name.text, value); + const newObj = { kind: "obj", fields, ...(base.symFields ? { symFields: new Map(base.symFields) } : {}) } as SymVal; + // JS reference semantics: the same object is reachable from its + // containers (`const host = msgs[k]` / find()) — a field write must + // be visible through every array (and one level of object field) + // that still holds this exact value. Value-copy alone rebinds only + // the local alias and the container keeps the stale object. + for (const [vn, vv] of frame.vars) { + if (isArr(vv)) { + let hit = false; + const elems = vv.elems.map((e) => { if (e === base) { hit = true; return newObj; } return e; }); + if (hit) frame.vars.set(vn, { kind: "arr", elems, len: vv.len } as SymVal); + } else if (isObj(vv) && vv !== base) { + let hit = false; + const nf: Map = new Map(vv.fields); + for (const [fk, fv] of nf) if (fv === base) { nf.set(fk, newObj); hit = true; } + if (hit) frame.vars.set(vn, { kind: "obj", fields: nf, ...(vv.symFields ? { symFields: new Map(vv.symFields) } : {}) } as SymVal); + } + } + return writeLValue(target.expression, newObj, frame, st, depth); + } + return false; + } + if (ts.isElementAccessExpression(target)) { + const base = evalExpr(target.expression, frame, st, depth); + const idx = evalExpr(target.argumentExpression, frame, st, depth); + if (isArr(base) && isNum(idx)) { + const elems = base.elems.map((e, k) => iteVal(numEq(ctx, idx.t, num(ctx, k)), value, e)); + if (base.elems.length < N) { + elems.push(iteVal(numEq(ctx, idx.t, num(ctx, base.elems.length)), value, { kind: "undef" })); + } + // a[i] = v with i ≥ len grows the array to i+1 + const grown = iteVal(idx.t.gt(base.len), { kind: "num", t: idx.t.add(num(ctx, 1)) }, { kind: "num", t: base.len }); + return writeLValue(target.expression, { kind: "arr", elems, len: (grown as { t: Num }).t } as never, frame, st, depth); + } + if (isObj(base) && isStr(idx)) { + // Record write: obj[key] = v. Const keys (the common case — ids, + // refs) write the field directly. Symbolic keys fan out over the + // fields that already exist AND record a symFields binding so the + // write is observable when the key matches nothing present (the + // solver decides equality). + const fields = new Map(base.fields); + const idxC = idx.t.c; + if (typeof idxC === "string" && idx.t.sort === "String" && (idx.t.kids?.length ?? 0) === 0) { + fields.set(idxC, value); + } else { + for (const key of [...fields.keys()]) { + fields.set(key, iteVal(idx.t.eq(ctx.String.val(key)), value, fields.get(key)!)); + } + const symFields = new Map(base.symFields ?? []); + // same key-TERM: the new write replaces it (last write wins for + // identical terms); equal-but-distinct terms resolve via the read + // fan's guard, which falls through dead writes. + symFields.delete(idx.t); + symFields.set(idx.t, { g: bool(ctx, true), v: value }); + return writeLValue(target.expression, { kind: "obj", fields, symFields } as SymVal, frame, st, depth); + } + return writeLValue(target.expression, { kind: "obj", fields } as SymVal, frame, st, depth); + } + return false; + } + return false; + }; + + const evalBinary = (expr: ts.BinaryExpression, frame: Frame, st: RunState, depth: number): SymVal => { + const op = expr.operatorToken.kind; + // Assignments (x = …, x += …) are BinaryExpressions: they MUST be + // handled here at the TOP of evalBinary — evalExpr's generic + // isBinaryExpression dispatch reaches this function for every binary + // node, and a fallthrough to applyBinop would evaluate but never STORE. + if (ASSIGN_OPS.includes(op)) { + const compound = compoundOpKind(op); + const rhs = op === ts.SyntaxKind.EqualsToken ? evalExpr(expr.right, frame, st, depth) : null; + const cur = compound ? evalExpr(expr.left, frame, st, depth) : null; + const value = compound && cur ? applyBinop(compound, cur, evalExpr(expr.right, frame, st, depth), expr, st) : rhs; + if (value && writeLValue(expr.left, value, frame, st, depth)) return value; + bad(expr, "assignment target"); + } + if (op === ts.SyntaxKind.AmpersandAmpersandToken) { + const l = evalExpr(expr.left, frame, st, depth); + return jsAnd(l, () => evalExpr(expr.right, frame, st, depth)); + } + if (op === ts.SyntaxKind.BarBarToken) { + const l = evalExpr(expr.left, frame, st, depth); + return jsOr(l, () => evalExpr(expr.right, frame, st, depth)); + } + if (op === ts.SyntaxKind.QuestionQuestionToken) { + const l = evalExpr(expr.left, frame, st, depth); + return jsNullish(l, () => evalExpr(expr.right, frame, st, depth)); + } + return applyBinop(op, evalExpr(expr.left, frame, st, depth), evalExpr(expr.right, frame, st, depth), expr, st); + }; + + const applyBinop = (op: ts.SyntaxKind, l: SymVal, r: SymVal, at: ts.Node, st: RunState): SymVal => { + if (isNum(l) && isNum(r)) { + const a = l.t, b = r.t; + const ac = numC(a), bc = numC(b); + if (ac !== undefined && bc !== undefined) { + const fold = (): SymVal | null => { + switch (op) { + case ts.SyntaxKind.PlusToken: return { kind: "num", t: num(ctx, ac + bc) }; + case ts.SyntaxKind.MinusToken: return { kind: "num", t: num(ctx, ac - bc) }; + case ts.SyntaxKind.AsteriskToken: return { kind: "num", t: num(ctx, ac * bc) }; + case ts.SyntaxKind.SlashToken: + if (bc !== 0 && Number.isInteger(ac) && Number.isInteger(bc) && ac % bc === 0) return { kind: "num", t: num(ctx, ac / bc) }; + return null; + case ts.SyntaxKind.PercentToken: + if (Number.isInteger(ac) && Number.isInteger(bc) && bc !== 0) return { kind: "num", t: num(ctx, ac % bc) }; + return null; + case ts.SyntaxKind.LessThanToken: return { kind: "bool", t: bool(ctx, ac < bc) }; + case ts.SyntaxKind.LessThanEqualsToken: return { kind: "bool", t: bool(ctx, ac <= bc) }; + case ts.SyntaxKind.GreaterThanToken: return { kind: "bool", t: bool(ctx, ac > bc) }; + case ts.SyntaxKind.GreaterThanEqualsToken: return { kind: "bool", t: bool(ctx, ac >= bc) }; + case ts.SyntaxKind.EqualsEqualsToken: + case ts.SyntaxKind.EqualsEqualsEqualsToken: return { kind: "bool", t: bool(ctx, ac === bc) }; + case ts.SyntaxKind.ExclamationEqualsToken: + case ts.SyntaxKind.ExclamationEqualsEqualsToken: return { kind: "bool", t: bool(ctx, ac !== bc) }; + default: return null; + } + }; + const f = fold(); + if (f) return f; + } + switch (op) { + case ts.SyntaxKind.PlusToken: return { kind: "num", t: a.add(b) }; + case ts.SyntaxKind.MinusToken: return { kind: "num", t: a.sub(b) }; + case ts.SyntaxKind.AsteriskToken: return { kind: "num", t: a.mul(b) }; + case ts.SyntaxKind.SlashToken: return { kind: "num", t: a.div(b) }; + case ts.SyntaxKind.PercentToken: return { kind: "num", t: a.mod(b) }; + case ts.SyntaxKind.LessThanToken: return { kind: "bool", t: a.lt(b) }; + case ts.SyntaxKind.LessThanEqualsToken: return { kind: "bool", t: a.le(b) }; + case ts.SyntaxKind.GreaterThanToken: return { kind: "bool", t: a.gt(b) }; + case ts.SyntaxKind.GreaterThanEqualsToken: return { kind: "bool", t: a.ge(b) }; + case ts.SyntaxKind.EqualsEqualsToken: + case ts.SyntaxKind.EqualsEqualsEqualsToken: return { kind: "bool", t: a.eq(b) }; + case ts.SyntaxKind.ExclamationEqualsToken: + case ts.SyntaxKind.ExclamationEqualsEqualsToken: return { kind: "bool", t: a.neq(b) }; + } + } + if (isStr(l) && isStr(r)) { + const lc = strConst.get(l.t), rc = strConst.get(r.t); + if (op === ts.SyntaxKind.PlusToken) { + if (lc !== undefined && rc !== undefined) return { kind: "str", t: strc(ctx, lc + rc) }; + return { kind: "str", t: l.t.concat(r.t) }; + } + if (lc !== undefined && rc !== undefined) { + if (op === ts.SyntaxKind.EqualsEqualsToken || op === ts.SyntaxKind.EqualsEqualsEqualsToken) return { kind: "bool", t: bool(ctx, lc === rc) }; + if (op === ts.SyntaxKind.ExclamationEqualsToken || op === ts.SyntaxKind.ExclamationEqualsEqualsToken) return { kind: "bool", t: bool(ctx, lc !== rc) }; + } + if (op === ts.SyntaxKind.EqualsEqualsToken || op === ts.SyntaxKind.EqualsEqualsEqualsToken) return { kind: "bool", t: l.t.eq(r.t) }; + if (op === ts.SyntaxKind.ExclamationEqualsToken || op === ts.SyntaxKind.ExclamationEqualsEqualsToken) return { kind: "bool", t: l.t.neq(r.t) }; + } + if (isBool(l) && isBool(r)) { + if (op === ts.SyntaxKind.EqualsEqualsToken || op === ts.SyntaxKind.EqualsEqualsEqualsToken) return { kind: "bool", t: l.t.eq(r.t) }; + if (op === ts.SyntaxKind.ExclamationEqualsToken || op === ts.SyntaxKind.ExclamationEqualsEqualsToken) return { kind: "bool", t: l.t.neq(r.t) }; + } + // Mixed num/str `+` is JS string concatenation — route through the + // memoized number→string rendering (same number always same string). + if (op === ts.SyntaxKind.PlusToken && isNum(l) && isStr(r)) { + const ls = numToStringStrFor(l.t); + return { kind: "str", t: ls.t.concat(r.t) }; + } + if (op === ts.SyntaxKind.PlusToken && isStr(l) && isNum(r)) { + const rs = numToStringStrFor(r.t); + return { kind: "str", t: l.t.concat(rs.t) }; + } + // Mixed equality against null/undefined + if ((l.kind === "null" || l.kind === "undef") || (r.kind === "null" || r.kind === "undef")) { + if (op === ts.SyntaxKind.EqualsEqualsEqualsToken || op === ts.SyntaxKind.EqualsEqualsToken) { + return { kind: "bool", t: bool(ctx, l.kind === r.kind) }; + } + if (op === ts.SyntaxKind.ExclamationEqualsToken || op === ts.SyntaxKind.ExclamationEqualsEqualsToken) { + return { kind: "bool", t: bool(ctx, l.kind !== r.kind) }; + } + } + return { kind: "any" }; + }; + + /* ---------------- calls ---------------- */ + + let reTestN = 0; + const evalCall = (call: ts.CallExpression, frame: Frame, st: RunState, depth: number): SymVal => { + // Spread arguments: flatten arrays of args (rare in guarded code). + const argVals: SymVal[] = []; + for (const a of call.arguments) { + if (ts.isSpreadElement(a)) { + const v = evalExpr(a.expression, frame, st, depth); + if (isArr(v)) argVals.push(...v.elems); + else degrade(st, a, "spread argument"); + } else { + argVals.push(evalExpr(a, frame, st, depth)); + } + } + const calleeExpr = call.expression; + + // Namespace-qualified builtins ------------------------------------ + if (ts.isPropertyAccessExpression(calleeExpr) && ts.isIdentifier(calleeExpr.expression)) { + const ns = calleeExpr.expression.text; + const meth = calleeExpr.name.text; + if (ns === "Number") { + if (meth === "isInteger" && isNum(argVals[0])) { + return { kind: "bool", t: ctx.IsInt(argVals[0].t) }; + } + if (meth === "isFinite" && isNum(argVals[0])) return { kind: "bool", t: bool(ctx, true) }; + if (meth === "isFinite" && !isNum(argVals[0])) return { kind: "bool", t: bool(ctx, false) }; + if (meth === "isInteger" && !isNum(argVals[0])) return { kind: "bool", t: bool(ctx, false) }; + if ((meth === "parseInt" || meth === "parseFloat") && isStr(argVals[0])) { + return { kind: "num", t: argVals[0].t.toInt() }; + } + } + if (ns === "Math") return mathCall(meth, argVals, call, st); + if (ns === "Array" && meth === "isArray") { + if (argVals[0].kind === "arr") return { kind: "bool", t: bool(ctx, true) }; + // undefined/null/bool/fn are provably not arrays (JS typeof) + if (argVals[0].kind !== "any") return { kind: "bool", t: bool(ctx, false) }; + degrade(st, call, "Array.isArray on degraded value"); + return { kind: "any" }; + } + if (ns === "String" && meth === "fromCharCode") return { kind: "str", t: ctx.String.val("") }; // degraded, content-neutral + if (ns === "JSON") { + // JSON.parse of a KNOWN string is pure and deterministic — run the + // real implementation and convert. Degraded args still degrade. + if (meth === "parse") { + const a0 = argVals[0]; + if (isStr(a0)) { + const lit = strConst.get(a0.t); + if (lit !== undefined) { + try { return jsonToSymVal(JSON.parse(lit), depth); } catch { return { kind: "null" }; } + } + } + } + degrade(st, call, `JSON.${meth}`); return { kind: "any" }; + } + if (ns === "Object" || ns === "Boolean") { degrade(st, call, `${ns}.${meth}`); return { kind: "any" }; } + } + + // Method calls on values ------------------------------------------- + if (ts.isPropertyAccessExpression(calleeExpr)) { + // optional call `x.m?.(…)`: absent method reads as undefined (no throw) + if (call.questionDotToken !== undefined && !(input.options?.portImpls && ("__method_" + calleeExpr.name.text) in input.options.portImpls)) { + const r0 = evalExpr(calleeExpr.expression, frame, st, depth); + if (isObj(r0)) { + const mv = r0.fields.get(calleeExpr.name.text); + if (mv === undefined || mv.kind === "undef" || mv.kind === "null") return { kind: "undef" }; + } + } + const recv = evalExpr(calleeExpr.expression, frame, st, depth); + const m = calleeExpr.name.text; + return callMethod(recv, m, argVals, call, st, frame, depth); + } + + // Plain identifier calls ------------------------------------------- + const name = ts.isIdentifier(calleeExpr) ? calleeExpr.text : undefined; + if (!name) { degrade(st, call, "indirect call"); return { kind: "any" }; } + + // Builtins with exact translation + if (name === "Number.isInteger" || name === "Number.isFinite") { + const a0 = argVals[0]; + if (isNum(a0)) { + if (name === "Number.isInteger") return { kind: "bool", t: ctx.IsInt(a0.t) }; + return { kind: "bool", t: bool(ctx, true) }; // finite within Real model + } + return { kind: "bool", t: bool(ctx, false) }; // undefined/null/non-number + } + if (name === "Number" || name === "String") { + const a0 = argVals[0]; + if (name === "String") { + if (isStr(a0)) return a0; + if (isNum(a0)) return numToStringStrFor(a0.t); + return { kind: "any" }; + } + if (isNum(a0)) return a0; + // Number(constString) folds faithfully — refNum() and friends parse + // digit strings; without this fold Number.isFinite(any) collapses to + // constant false and gated paths silently disappear from the model. + if (isStr(a0)) { + const cs = (a0.t as unknown as { c?: string }).c; + if (typeof cs === "string" && cs.trim() !== "" && !Number.isNaN(Number(cs))) { + return { kind: "num", t: num(ctx, Number(cs)) }; + } + } + return { kind: "any" }; + } + if (name === "isNaN") { + const a0 = argVals[0]; + return isNum(a0) ? { kind: "bool", t: bool(ctx, false) } : { kind: "any" }; + } + + // Local closure value call (const arrow / function expression stored in a var) + const fnVal = frame.vars.get(name); + if (fnVal && fnVal.kind === "fn") { + return invokeCallback(fnVal.node, argVals, st, depth, fnVal.env); + } + + // Port override (public injection seam); memoized on the argument's + // structural form — port implementations are pure functions. + // Frame-bound names (parameters, port-assigned locals) take ports FIRST; + // imported names take ports only when the test declares them (e.g. hashId + // uses node:crypto — outside the guarded subset). Ports NEVER shadow + // module-local real functions (the no-cheating core: real code always + // runs). Multi-arg ports receive all arguments (single-arg only is + // memoized; arg0-keyed memo is sound only for unary ports). + const moduleLocalFn = curInfo().functions.get(name) ?? info.functions.get(name); + const moduleLocalArrow = curInfo().constArrowFns.get(name) ?? info.constArrowFns.get(name); + if (input.options?.portImpls && name in input.options.portImpls && !moduleLocalFn && !moduleLocalArrow) { + const arg0 = argVals[0] ?? { kind: "undef" }; + const key = arg0.kind === "str" ? `s:${String((arg0.t as unknown as { c?: string }).c ?? `h${arg0.t.hash()}`)}` : arg0.kind === "num" ? `n:${String((arg0.t as unknown as { c?: number }).c ?? numC(arg0.t))}` : null; + if (key !== null && argVals.length === 1) { + const hit = portMemo.get(`${name}\u0000${key}`); + if (hit) return hit; + const r = input.options.portImpls[name]!(arg0); + portMemo.set(`${name}\u0000${key}`, r); + return r; + } + return input.options.portImpls[name]!(arg0, ...argVals.slice(1)); + } + + // Local function → INLINE (the no-cheating core: real helper bodies run) + const entryFn = info.functions.get(name); + const entryArrow = info.constArrowFns.get(name) as ts.ArrowFunction | undefined; + const defEntry = importedSymbols.get(name); + const localFn = moduleLocalFn ?? defEntry?.info.functions.get(name) ?? entryFn; + const defArrow = importedSymbols.get(name)?.info.constArrowFns.get(name) as ts.ArrowFunction | undefined; + const localArrow = moduleLocalArrow ?? defArrow ?? entryArrow; + const fnNode = localFn ?? (localArrow as ts.ArrowFunction | undefined); + if (fnNode) { + if (depth >= MAXINLINE) { degrade(st, call, `inline depth cap at ${name}`); return { kind: "any" }; } + return inlineCall(fnNode, argVals, call, st, depth, defEntry?.info); + } + + degrade(st, call, `call to ${name}`); + return { kind: "any" }; + }; + + const inlineCall = ( + fn: ts.FunctionDeclaration | ts.ArrowFunction | ts.FunctionExpression, + args: SymVal[], + at: ts.Node, + st: RunState, + depth: number, + defInfo?: ModuleInfo, + ): SymVal => { + const nm = fn.name?.text || "(anon)"; + if (defInfo) moduleStack.push(defInfo); + try { + const inner: Frame = { vars: new Map(frame.vars) }; // lexical closure + const params = ts.isFunctionDeclaration(fn) ? fn.parameters : fn.parameters; + params.forEach((p, i) => { + if (ts.isIdentifier(p.name)) { + inner.vars.set(p.name.text, args[i] ?? (p.initializer ? evalExpr(p.initializer, inner, st, depth + 1) : { kind: "undef" })); + } else if (ts.isArrayBindingPattern(p.name)) { + // destructure array param: bind pattern elements to slots + p.name.elements.forEach((el, j) => { + if (ts.isBindingElement(el) && ts.isIdentifier(el.name)) { + inner.vars.set(el.name.text, args[i] && isArr(args[i]) ? args[i].elems[j] : { kind: "undef" }); + } + }); + } + }); + const innerState: RunState = runState0(); + const body = ts.isFunctionDeclaration(fn) ? fn.body : fn.body; + if (!body) return { kind: "any" }; + if (ts.isBlock(body)) { + execStmt(body, inner, innerState, depth + 1); + } else { + // arrow with expression body + innerState.ret = evalExpr(body, inner, innerState, depth + 1); + } + const pushAll2 = (dst: T[], src: T[]): void => { if (src !== dst) for (const x of src) dst.push(x); }; + pushAll2(st.assumptions, innerState.assumptions); + // degradations shared + if (innerState.thrown) st.thrown = orFold(st.thrown ?? bool(ctx, false), innerState.thrown); + // fold conditional returns (pendingRets) LIFO over the unconditional ret + let out = innerState.ret; + for (let i = innerState.pendingRets.length - 1; i >= 0; i--) { + const pr = innerState.pendingRets[i]!; + out = out === null ? iteVal(pr.cond, pr.val, { kind: "undef" }) : iteVal(pr.cond, pr.val, out); + } + return out ?? { kind: "undef" }; + } finally { + if (defInfo) moduleStack.pop(); + } + }; + + + const callMethod = ( + recv: SymVal, m: string, args: SymVal[], + at: ts.CallExpression, st: RunState, frame: Frame, depth: number, + ): SymVal => { + // ---- namespace objects ---- + if (isObj(recv) && m === "call" ) { degrade(st, at, "fn.call"); return { kind: "any" }; } + + // Anchored dynamic regex over-approximation: a pattern "^A…" with a + // known literal head A only matches strings starting with A. On the + // ¬startsWith side .test is exactly false and .replace is the identity; + // on the reachable side the result is an unconstrained uninterpreted + // value (recorded as a degradation only when the guard can hold — + // replay validates any witness from that side on the real regex). + const regexAnchor = (patVal: SymVal): string | null => { + if (!isStr(patVal)) return null; + const pieces: string[] = []; + const walk = (t: unknown): void => { + const tt = t as { op?: string; kids?: unknown[]; c?: unknown; sort?: string }; + if (tt && typeof tt === "object" && Array.isArray(tt.kids)) { + // concat shapes: app2("str.++", a, b) [facade] or op "str.++"/"concat" kids [a, b] + const k0 = tt.kids[0] as { op?: unknown; c?: unknown } | undefined; + const isConcat = tt.op === "concat" || tt.op === "str.++" + || (tt.op === "app2" && (k0?.op === "str.++" || k0?.c === "str.++")); + if (isConcat && (tt.kids.length === 2 || tt.kids.length === 3)) { + for (const k of tt.kids.slice(tt.kids.length === 3 ? 1 : 0)) walk(k); + return; + } + } + if (tt && typeof tt === "object" && typeof tt.c === "string" && tt.sort === "String") { + pieces.push(tt.c); return; + } + // symbolic piece (or non-const): stops accumulation — the anchor is + // the const head collected so far (may be empty → null). + }; + walk(patVal.t); + if (pieces.length === 0) { + const whole = strConst.get(patVal.t); + if (whole === undefined) return null; + pieces.push(whole); + } + if (!pieces[0]!.startsWith("^")) return null; + const body = pieces.join("").slice(1); + // Literal head: chars up to the first regex meta (backslash escapes + // end the head too — their target is regex syntax, not plain text). + const m = /^((?:\\.|[^[\](){}*+?|^$.\\])*)/.exec(body); + const head = m?.[1] ?? ""; + return head === "" ? null : head; + }; + const dynRegexTest = (recv: SymVal, patVal: SymVal, at: ts.Node, st: RunState): SymVal | null => { + if (!isStr(recv)) return null; + const anchor = regexAnchor(patVal); + if (anchor === null || anchor === "") return null; + const g = recv.t.prefixOf(ctx.String.val(anchor)); + const gb = constBool(g); + if (gb === false) return { kind: "bool", t: bool(ctx, false) }; + const fresh = ctx.Bool.const(`__re_test_${reTestN++}`); + if (gb !== true) degrade(st, at, "dynamic regex .test over-approx"); + return { kind: "bool", t: ctx.If(g, fresh, bool(ctx, false)) }; + + }; + + // ---- method ports: "__method_" receives (recv, ...args) ---- + const mport = input.options?.portImpls ? input.options.portImpls["__method_" + m] : undefined; + if (mport && (isObj(recv) || isArr(recv))) return (mport as (r: SymVal, ...a: SymVal[]) => SymVal)(recv, ...args); + + // ---- Math ---- + // (Math.x handled at plain-call sites in source; but method form via + // identifier "Math" reaches here only through plain calls.) + + // ---- array methods ---- + if (isArr(recv)) { + switch (m) { + case "length": return { kind: "num", t: recv.len }; + case "push": { + const v = args[0]; + if (recv.elems.length < N) { + // append at the symbolic tail position (arrays built by push) + const elems = recv.elems.map((e, k) => iteVal(numEq(ctx, recv.len, num(ctx, k)), v, e)); + elems.push(iteVal(numEq(ctx, recv.len, num(ctx, recv.elems.length)), v, { kind: "undef" })); + const newLen = recv.len.add(num(ctx, 1)); + return rebindArray(recv, { kind: "arr", elems, len: newLen }, at, st, frame); + } + degrade(st, at, "push beyond bounded slots"); + return { kind: "any" }; + } + case "includes": { + const target = args[0]; + const eqs = recv.elems.map((e) => valEq(e, target)); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc = bool(ctx, false); + for (let k = 0; k < recv.elems.length; k++) { + acc = acc.or(recv.len.gt(num(ctx, k)).and(eqs[k]!)); + } + return { kind: "bool", t: acc }; + } + case "indexOf": { + const target = args[0]; + const eqs = recv.elems.map((e) => valEq(e, target)); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc: Num = num(ctx, -1); + for (let k = recv.elems.length - 1; k >= 0; k--) { + acc = ctx.If(recv.len.gt(num(ctx, k)).and(eqs[k]!), num(ctx, k), acc); + } + return { kind: "num", t: acc }; + } + case "map": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "map without function literal"); return { kind: "any" }; } + const elems = recv.elems.map((e, k) => { + const guarded = iteVal(recv.len.gt(num(ctx, k)), e, { kind: "undef" }); + const r = invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth); + return r; + }); + return { kind: "arr", elems, len: recv.len }; + } + case "filter": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "filter without function literal"); return { kind: "any" }; } + // Unrolled two-pass: keep-slot[j] via counting. + const keep: Bool<"main">[] = recv.elems.map((e, k) => { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, e, { kind: "undef" }); + const r = invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth); + const t = truthy(ctx, r); + if (!t) return g.not(); // out-of-length slots never keep + return g.and(t); + }); + const outElems: SymVal[] = []; + for (let j = 0; j < recv.elems.length; j++) { + // j-th kept element = elems[k] where keep[k] and exactly j keeps before k + let pick: SymVal = { kind: "undef" }; + for (let k = recv.elems.length - 1; k >= 0; k--) { + const before = countTrueBelow(keep, k); + const cond = keep[k]!.and(before.eq(num(ctx, j))); + pick = iteVal(cond, recv.elems[k], pick); + } + outElems.push(pick); + } + const newLen = keep.reduce((acc, kk) => ctx.If(kk, acc.add(num(ctx, 1)), acc), num(ctx, 0)); + return { kind: "arr", elems: outElems, len: newLen }; + } + case "every": + case "some": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, `${m} without function literal`); return { kind: "any" }; } + let acc = bool(ctx, m === "every"); + for (let k = 0; k < recv.elems.length; k++) { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, recv.elems[k], { kind: "undef" }); + const r = truthy(ctx, invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth)); + if (!r) { degrade(st, at, `${m} callback degraded`); return { kind: "any" }; } + const effective = g.and(r); + acc = m === "every" ? acc.and(effective) : acc.or(effective); + } + return { kind: "bool", t: acc }; + } + case "entries": { + // array iterator of [index, value] pairs, bounded to declared slots + const pairs = recv.elems.map((e, k) => ({ kind: "arr", elems: [{ kind: "num", t: num(ctx, k) }, e], len: num(ctx, 2) }) as SymVal); + return { kind: "arr", elems: pairs, len: recv.len }; + } + case "find": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "find without function literal"); return { kind: "any" }; } + let acc: SymVal = { kind: "undef" }; + for (let k = recv.elems.length - 1; k >= 0; k--) { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, recv.elems[k], { kind: "undef" }); + const r = truthy(ctx, invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth)); + if (!r) { degrade(st, at, "find callback degraded"); return { kind: "any" }; } + acc = iteVal(g.and(r), recv.elems[k], acc); + } + return acc; + } + case "findIndex": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "findIndex without function literal"); return { kind: "any" }; } + let acc: Num = num(ctx, -1); + for (let k = recv.elems.length - 1; k >= 0; k--) { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, recv.elems[k], { kind: "undef" }); + const r = truthy(ctx, invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth)); + if (!r) { degrade(st, at, "findIndex callback degraded"); return { kind: "any" }; } + acc = ctx.If(g.and(r), num(ctx, k), acc); + } + return { kind: "num", t: acc }; + } + case "sort": { + // Rank-based comparator sort, works for any slot count N (O(N²) comparator calls, + // no permutation enumeration). rank_i = #{j real: cmp(v_i,v_j)<0} + #{j => recv.len.gt(num(ctx, i)); // i < len + const ranks: Num[] = []; + for (let i = 0; i < n; i++) { + let rank = num(ctx, 0); + for (let j = 0; j < n; j++) { + if (i === j) continue; + const realJ = real(j); + if (recv.elems[i] === recv.elems[j]) { + // identical values: comparator returns 0 (deterministic) — tie + rank = rank.add(ctx.If(realJ.and(bool(ctx, j < i)), num(ctx, 1), num(ctx, 0)) as Num); + continue; + } + const r = invokeCallback(cbFn.node, [recv.elems[i]!, recv.elems[j]!], st, depth, cbFn.env); + if (!isNum(r)) { degrade(st, at, "sort comparator non-numeric"); return { kind: "any" }; } + const less = r.t.lt(num(ctx, 0)).and(realJ); + const tie = r.t.eq(num(ctx, 0)).and(realJ).and(bool(ctx, j < i)); + rank = rank.add(ctx.If(less.or(tie), num(ctx, 1), num(ctx, 0)) as Num); + } + ranks.push(rank); + } + const elems: SymVal[] = []; + for (let k = 0; k < n; k++) { + let slot: SymVal = { kind: "undef" }; + for (let i = 0; i < n; i++) { + const hit = numEq(ctx, ranks[i]!, num(ctx, k)).and(real(i)); + slot = iteVal(hit, recv.elems[i]!, slot); + } + elems.push(slot); + } + return { kind: "arr", elems, len: recv.len }; + } + case "forEach": { + const cb = at.arguments[0]; + if (!cb || (!ts.isArrowFunction(cb) && !ts.isFunctionExpression(cb))) { degrade(st, at, "forEach without function literal"); return { kind: "any" }; } + for (let k = 0; k < recv.elems.length; k++) { + const g = recv.len.gt(num(ctx, k)); + const guarded = iteVal(g, recv.elems[k], { kind: "undef" }); + invokeCallback(cb, [guarded, { kind: "num", t: num(ctx, k) }, recv], st, depth); + } + return { kind: "undef" }; + } + case "slice": { + const a0 = isNum(args[0]) ? args[0].t : num(ctx, 0); + const rawEnd = isNum(args[1]) ? args[1].t : recv.len; + const start = ctx.If(a0.lt(num(ctx, 0)), ctx.If(a0.add(recv.len).ge(num(ctx, 0)), a0.add(recv.len), num(ctx, 0)), a0); + const end = ctx.If(rawEnd.lt(num(ctx, 0)), ctx.If(rawEnd.add(recv.len).ge(num(ctx, 0)), rawEnd.add(recv.len), num(ctx, 0)), rawEnd); + const elems: SymVal[] = []; + for (let j = 0; j < recv.elems.length; j++) { + const srcIdx = start.add(num(ctx, j)); + let pick: SymVal = { kind: "undef" }; + for (let k = recv.elems.length - 1; k >= 0; k--) { + pick = iteVal(numEq(ctx, srcIdx, num(ctx, k)).and(end.gt(num(ctx, j))), recv.elems[k], pick); + } + elems.push(pick); + } + const len = end.sub(start); + return { kind: "arr", elems, len: ctx.If(len.lt(num(ctx, 0)), num(ctx, 0), len) }; + } + case "concat": { + const other = args.find(isArr); + const elems = [...recv.elems]; + if (other) { + for (let k = 0; k < other.elems.length; k++) { + elems.push(iteVal(other.len.gt(num(ctx, k)), other.elems[k], { kind: "undef" })); + } + } + return { kind: "arr", elems, len: other ? recv.len.add(other.len) : recv.len }; + } + case "join": { + // Only when every element is a concrete/str; else degrade. + const parts: SymVal[] = []; + for (let k = 0; k < recv.elems.length; k++) { + parts.push(iteVal(recv.len.gt(num(ctx, k + 1)), [...recv.elems.slice(0, k), { kind: "str" as const, t: strArgOrEmpty(args[0]) }].reduce((a, b) => (isStr(a) && isStr(b) ? { kind: "str" as const, t: a.t.concat(b.t) } : { kind: "any" as const }), { kind: "str" as const, t: ctx.String.val("") }), recv.elems[k])); + } + if (parts.some((p) => !isStr(p))) { degrade(st, at, "join with degraded parts"); return { kind: "any" }; } + return parts.reduce((a, b) => (isStr(a) && isStr(b) ? { kind: "str" as const, t: a.t.concat(b.t) } : { kind: "any" as const }), { kind: "str" as const, t: ctx.String.val("") }); + } + default: + degrade(st, at, `array method .${m}`); + return { kind: "any" }; + } + } + + // ---- string methods ---- + if (isStr(recv)) { + switch (m) { + case "length": return { kind: "num", t: lenR(recv.t) }; + case "includes": { + const a0 = isStr(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + const rc = strConst.get(recv.t), ac = strConst.get(a0); + if (rc !== undefined && ac !== undefined) return { kind: "bool", t: bool(ctx, rc.includes(ac)) }; + return { kind: "bool", t: recv.t.contains(a0) }; + } + case "startsWith": { + const a0 = isStr(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + const rc = strConst.get(recv.t), ac = strConst.get(a0); + if (rc !== undefined && ac !== undefined) return { kind: "bool", t: bool(ctx, rc.startsWith(ac)) }; + return { kind: "bool", t: recv.t.prefixOf(a0) }; + } + case "endsWith": { + const a0 = isStr(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + const rc = strConst.get(recv.t), ac = strConst.get(a0); + if (rc !== undefined && ac !== undefined) return { kind: "bool", t: bool(ctx, rc.endsWith(ac)) }; + return { kind: "bool", t: recv.t.suffixOf(a0) }; + } + case "indexOf": { + const a0 = isStr(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + const rc = strConst.get(recv.t), ac = strConst.get(a0); + if (rc !== undefined && ac !== undefined) return { kind: "num", t: num(ctx, rc.indexOf(ac)) }; + return { kind: "num", t: recv.t.indexOf(a0) }; + } + case "slice": + case "substring": { + const a0 = isNum(args[0]) ? args[0].t : num(ctx, 0); + const rawEnd = isNum(args[1]) ? args[1].t : lenR(recv.t); + const start = ctx.If(a0.lt(num(ctx, 0)), ctx.If(a0.add(lenR(recv.t)).ge(num(ctx, 0)), a0.add(lenR(recv.t)), num(ctx, 0)), a0); + const end = ctx.If(rawEnd.lt(num(ctx, 0)), ctx.If(rawEnd.add(lenR(recv.t)).ge(num(ctx, 0)), rawEnd.add(lenR(recv.t)), num(ctx, 0)), rawEnd); + const len = ctx.If(end.sub(start).ge(num(ctx, 0)), end.sub(start), num(ctx, 0)); + // seq.extract is Int-indexed; our numbers are Real. + pushAsum(st, ctx.IsInt(start)); pushAsum(st, ctx.IsInt(len)); + return { kind: "str", t: recv.t.extract(ctx.ToInt(start), ctx.ToInt(len)) }; + } + case "charCodeAt": { + const a0 = isNum(args[0]) ? args[0].t : null; + if (!a0) return { kind: "any" }; + pushAsum(st, ctx.IsInt(a0)); + return { kind: "num", t: ctx.ToReal(recv.t.at(ctx.ToInt(a0)).toCode() as never) as Num }; // toCode is Int-sorted; numbers are Real + } + case "at": { + const a0 = isNum(args[0]) ? args[0].t : num(ctx, 0); + const idx = ctx.If(a0.lt(num(ctx, 0)), a0.add(lenR(recv.t)), a0); + return { kind: "str", t: recv.t.at(ctx.ToInt(idx)) }; + } + case "trim": return recv; // approximation: content-neutral for our props + case "toLowerCase": return recv; // approximation: same, see README + case "toUpperCase": return recv; + case "replace": { + const a0 = isStr(args[0]) ? args[0].t : null; + const a1 = isStr(args[1]) ? args[1].t : null; + if (a0 && a1) return { kind: "str", t: recv.t.replace(a0, a1) }; + // Const string × const regex-literal × const replacement: the + // operation is a pure deterministic string function — evaluate it + // concretely (e.g. escapeRegex(TAG_OPEN)). + const rxC = isObj(args[0]) && args[0].fields.has("__regex") ? strConst.get((args[0].fields.get("__regex")! as { kind: "str"; t: Str }).t) : undefined; + const repC = isStr(args[1]) ? strConst.get(args[1].t) : undefined; + const recvC = strConst.get(recv.t); + if (rxC !== undefined && repC !== undefined && recvC !== undefined) { + const m = /^\/(.*)\/([a-z]*)$/.exec(rxC); + if (m) return { kind: "str", t: ctx.String.val(recvC.replace(new RegExp(m[1]!, m[2]), repC)) }; + } + // Symbolic receiver against the escapeRegex shape (\. specials → + // "\\$&"): deterministic uninterpreted function of the input. + if (rxC === "/[.*+?^${}()|[\\]\\\\]/g" && repC === "\\$&" && isStr(recv)) { + // deterministic uninterpreted escape: same input → same output + (ctx as unknown as { pendingFnsS1: Set }).pendingFnsS1.add("__esc1"); + return { kind: "str", t: (ctx as unknown as { app1: (f: string, a: never, s: string) => Str }).app1("__esc1", recv.t as never, "String") }; + } + // dynamic-regex pattern with "" replacement (the render-refs + // idempotency strip): anchored-prefix over-approximation. + if (isObj(args[0]) && args[0].fields.has("__regex") && isStr(args[1]) && strConst.get(args[1].t) === "") { + const patVal = args[0].fields.get("__regex")!; + const anchor = regexAnchor(patVal); + if (anchor !== null && anchor !== "" && isStr(recv)) { + const g = recv.t.prefixOf(ctx.String.val(anchor)); + const gb = constBool(g); + if (gb === false) return recv; + // Deterministic FUNCTION of (input, anchor): a per-call-site + // free const lets the same strip return different values in + // turn A vs turn B — manufacturing phantom cross-turn flips. + (ctx as unknown as { pendingFnsS2: Set }).pendingFnsS2.add("__re_rep2"); + const fn = (ctx as unknown as { app2: (f: string, a: never, b: never, s: string) => Str }).app2("__re_rep2", recv.t as never, ctx.String.val(anchor) as never, "String"); + if (gb !== true) degrade(st, at, "dynamic regex .replace over-approx"); + return iteVal(g, { kind: "str", t: fn }, recv); + } + } + return { kind: "any" }; + } + case "padStart": { + // Implemented exactly when the string is a decimal digit string + // (the ref-tag use). General int→str is a documented gap. + const w = isNum(args[0]) ? args[0].t : null; + const pad = isStr(args[1]) ? args[1].t : null; + if (!w || !pad) return { kind: "any" }; + // len < w ? pad*(w-len) ++ s : s — construct via extract of pad^N + let padded = recv.t; + const reps: Str[] = []; + for (let i = 0; i < 24; i++) reps.push(pad.power(num(ctx, i + 1))); + // choose the rep whose length equals w - len + let chosen: Str | null = null; + for (const rp of reps) { + const cond = lenR(rp).eq(w.sub(lenR(recv.t))); + chosen = chosen ? ctx.If(cond, rp, chosen) : rp; + } + const need = w.sub(lenR(recv.t)).gt(num(ctx, 0)); + return { kind: "str", t: ctx.If(need, chosen!.concat(recv.t), recv.t) }; + } + case "split": { + // Split into at-most-N parts by literal separator (string theory + // has no split; bounded reconstruction via uninterpreted shells is + // unsound, so degrade unless the separator case is trivial). + return { kind: "any" }; + } + default: + degrade(st, at, `string method .${m}`); + return { kind: "any" }; + } + } + + // ---- Number.prototype.toFixed: uninterpreted, deterministic per input ---- + if (isNum(recv) && m === "toFixed") { + (ctx as unknown as { pendingFnsR1: Set }).pendingFnsR1.add("__tofix1"); + return { kind: "str", t: (ctx as unknown as { app1: (f: string, a: never, s: string) => Str }).app1("__tofix1", recv.t as never, "String") }; + } + + // ---- dynamic regex carrier: .test(s) ---- + if (isObj(recv) && recv.fields.has("__regex") && m === "test") { + const r = dynRegexTest(args[0] ?? { kind: "undef" }, recv.fields.get("__regex")!, at, st); + if (r !== null) return r; + } + + // ---- Map/Set methods ---- + if (isObj(recv) && (recv.fields.has("__map_entries") || recv.fields.has("__set_values"))) { + const result = mapSetOp(recv, m, args, at, st); + if ((m === "add" || m === "set" || m === "delete") && isObj(result) && ts.isPropertyAccessExpression(at.expression)) { + const root = rootName(at.expression); + if (root && frame.vars.has(root)) frame.vars.set(root, result); + } + return result; + } + + degrade(st, at, `method .${m} on ${recv.kind}`); + return { kind: "any" }; + }; + + const strArgOrEmpty = (v: SymVal | undefined): Str => + v && isStr(v) ? v.t : ctx.String.val(""); + + const countTrueBelow = (bs: Bool<"main">[], k: number): Num => { + let acc = num(ctx, 0); + for (let j = 0; j < k; j++) acc = ctx.If(bs[j]!, acc.add(num(ctx, 1)), acc); + return acc; + }; + + const rebindArray = (oldArr: Extract, newArr: SymVal, at: ts.CallExpression, st: RunState, frame: Frame): SymVal => { + // push() mutates in place. The receiver may be a NESTED property chain + // (`pending.toolCalls.push(...)`): rebinding only the ROOT variable to + // the new array CLOBBERS the root (replaces the obj with the arr!). We + // must rebuild every container along the chain with its child replaced: + // pending' = {…pending, toolCalls: newArr}. + const chain: Array = []; + let node: ts.Expression = at.expression; + while (ts.isPropertyAccessExpression(node) || ts.isElementAccessExpression(node)) { + chain.unshift(node); + node = node.expression; + } + if (ts.isIdentifier(node)) { + const root = node.text; + let rebuilt: SymVal = newArr; + for (let i = chain.length - 1; i >= 1; i--) { + const hop = chain[i]!; + const parentVal = evalExpr(hop.expression, frame, st, 0); + if (parentVal.kind !== "obj") return newArr; // degrade silently: root rebind only + const fields = new Map(parentVal.fields); + if (!ts.isPropertyAccessExpression(hop)) return newArr; // element-middle chains: root rebind only + fields.set(hop.name.text, rebuilt); + rebuilt = { kind: "obj", fields }; + } + frame.vars.set(root, rebuilt); + return rebuilt; + } + return newArr; + }; + + /** Invoke an arrow/function-literal argument (callback) concretely. */ + const invokeCallback = ( + cb: ts.ArrowFunction | ts.FunctionExpression | ts.FunctionDeclaration, + args: SymVal[], + st: RunState, + depth: number, + env?: Frame, + ): SymVal => { + const inner: Frame = { vars: new Map((env ?? frame).vars) }; // lexical closure + cb.parameters.forEach((p, i) => { + if (ts.isIdentifier(p.name)) inner.vars.set(p.name.text, args[i] ?? { kind: "undef" }); + else if (ts.isArrayBindingPattern(p.name)) { + p.name.elements.forEach((el, j) => { + if (ts.isBindingElement(el) && ts.isIdentifier(el.name) && isArr(args[i])) { + inner.vars.set(el.name.text, args[i].elems[j] ?? { kind: "undef" }); + } + }); + } + }); + const innerState = runState0(); + if (cb.body && ts.isBlock(cb.body)) execStmt(cb.body, inner, innerState, depth + 1); + else if (cb.body) innerState.ret = evalExpr(cb.body, inner, innerState, depth + 1); + const pushAll2 = (dst: T[], src: T[]): void => { if (src !== dst) for (const x of src) dst.push(x); }; + pushAll2(st.assumptions, innerState.assumptions); + // degradations shared + if (innerState.thrown) st.thrown = orFold(st.thrown ?? bool(ctx, false), innerState.thrown); + // fold inner conditional returns (arrow bodies with early returns) + let ret: SymVal | null = innerState.ret; + for (let i = innerState.pendingRets.length - 1; i >= 0; i--) { + const pr = innerState.pendingRets[i]!; + ret = ret === null ? iteVal(pr.cond, pr.val, { kind: "undef" }) : iteVal(pr.cond, pr.val, ret); + } + // Shared-binding write-back: closures mutate captured vars (let cells). + // Propagate changed values back to the DECLARING frame. Sound when the + // call site's live frame is the declaration frame (top-level helper calls + // like applyTo(...)); loop-body callbacks in the guarded subset never + // write captured vars. + if (env) { + for (const [k, v] of inner.vars) { + const old = env.vars.get(k); + if (old === undefined || old === v) continue; + const same = + (isNum(old) && isNum(v) && old.t.hash() === v.t.hash()) || + (isStr(old) && isStr(v) && old.t.hash() === v.t.hash()) || + (isBool(old) && isBool(v) && old.t.hash() === v.t.hash()) || + (isArr(old) && isArr(v) && old.len.hash() === v.len.hash()); + if (!same) env.vars.set(k, v); + } + } + return ret ?? { kind: "undef" }; + }; + + /** Map/Set modeled as obj with special fields (bounded vector + len). */ + const mapSetOp = (recv: Extract, m: string, args: SymVal[], at: ts.Node, st: RunState): SymVal => { + const entriesArr = recv.fields.get("__map_entries") ?? recv.fields.get("__set_values"); + if (!isArr(entriesArr)) return { kind: "any" }; + switch (m) { + case "has": { + const key = args[0]; + if (recv.fields.has("__map_entries") && isArr(entriesArr)) { + // entries are {key, value} objs + const eqs = entriesArr.elems.map((e) => (isObj(e) ? valEq(e.fields.get("key") ?? { kind: "any" }, key) : null)); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc = bool(ctx, false); + for (let k = 0; k < eqs.length; k++) { + acc = acc.or(entriesArr.len.gt(num(ctx, k)).and(eqs[k]!)); + } + return { kind: "bool", t: acc }; + } + const eqs = entriesArr.elems.map((e) => valEq(e, args[0])); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc = bool(ctx, false); + for (let k = 0; k < eqs.length; k++) { + acc = acc.or(entriesArr.len.gt(num(ctx, k)).and(eqs[k]!)); + } + return { kind: "bool", t: acc }; + } + case "get": { + const eqs = entriesArr.elems.map((e) => (isObj(e) ? valEq(e.fields.get("key") ?? { kind: "any" }, args[0]) : null)); + if (eqs.some((e) => e === null)) return { kind: "any" }; + let acc: SymVal = { kind: "undef" }; + for (let k = eqs.length - 1; k >= 0; k--) { + const slotVal = entriesArr.elems[k]; + const got: SymVal = isObj(slotVal) ? slotVal.fields.get("value") ?? { kind: "undef" } : { kind: "undef" }; + acc = iteVal(entriesArr.len.gt(num(ctx, k)).and(eqs[k]!), got, acc); + } + return acc; + } + case "add": + case "set": { + const v = m === "set" && isObj(args[0]) === false + ? { kind: "obj" as const, fields: new Map([["key", args[0]], ["value", args[1] ?? { kind: "undef" }]]) } + : args[0]; + const entry = m === "set" + ? { kind: "obj" as const, fields: new Map([["key", args[0]], ["value", args[1] ?? { kind: "undef" }]]) } + : args[0]; + void v; + const elems = entriesArr.elems.map((e, k) => { + const exists = m === "set" + ? isObj(e) ? (valEq(e.fields.get("key") ?? { kind: "any" }, args[0]) ?? bool(ctx, false)) : bool(ctx, false) + : (valEq(e, args[0]) ?? bool(ctx, false)); + return iteVal(exists, entry, e); + }); + // membership as a (possibly symbolic) predicate — a Term object is + // always JS-truthy, so `elems.some(...)` could never work here. + let anyEq = bool(ctx, false); + for (let k = 0; k < entriesArr.elems.length; k++) { + const e = entriesArr.elems[k]!; + const eq = m === "set" + ? (isObj(e) ? (valEq(e.fields.get("key") ?? { kind: "any" }, args[0]) ?? bool(ctx, false)) : bool(ctx, false)) + : (valEq(e, args[0]) ?? bool(ctx, false)); + anyEq = anyEq.or(entriesArr.len.gt(num(ctx, k)).and(eq)); + } + // Slot-write at the SYMBOLIC length: a previously executed guarded + // add may or may not have consumed its slot, so the physical tail + // position is ambiguous. Writing the entry into slot `len` via per- + // slot ITEs is correct in every model (a slot past the unchanged + // length is never observed by length-guarded reads). + const slotElems = entriesArr.elems.map((e, k) => iteVal(numEq(ctx, entriesArr.len, num(ctx, k)), entry, e)); + if (entriesArr.elems.length < N - 1) { + slotElems.push(iteVal(numEq(ctx, entriesArr.len, num(ctx, entriesArr.elems.length)), entry, { kind: "undef" })); + } else { + degrade(st, at, "map/set add beyond bounded slots"); + } + elems.length = 0; + elems.push(...slotElems); + const newLenV = iteVal(anyEq, { kind: "num", t: entriesArr.len }, { kind: "num", t: entriesArr.len.add(num(ctx, 1)) }); + const newLen: Num = newLenV.kind === "num" ? newLenV.t : entriesArr.len; + const fields = new Map(recv.fields); + fields.set(recv.fields.has("__map_entries") ? "__map_entries" : "__set_values", { kind: "arr", elems, len: newLen }); + return { kind: "obj", fields }; + } + case "size": { + return { kind: "num", t: entriesArr.len }; + } + default: + degrade(st, at, `map/set method .${m}`); + return { kind: "any" }; + } + }; + + // new Map()/new Set() + const evalNew = (call: ts.NewExpression, frame: Frame, st: RunState, depth: number): SymVal => { + const name = ts.isIdentifier(call.expression) ? call.expression.text : ""; + if (name === "Map") { + return { kind: "obj", fields: new Map([["__map_entries", arrOf([])]]) }; + } + if (name === "Set") { + return { kind: "obj", fields: new Map([["__set_values", arrOf([])]]) }; + } + if (name === "RegExp") { + // regex carried as an object; .test/.replace on it use the anchored- + // prefix over-approximation at the call sites (see callMethod). + const p0 = call.arguments?.[0]; + return { kind: "obj", fields: new Map([["__regex", p0 ? evalExpr(p0, frame, st, depth) : { kind: "str", t: ctx.String.val("") }]]) }; + } + if (name === "Error" || name === "RangeError" || name === "TypeError") { + return { kind: "obj", fields: new Map([["message", call.arguments?.[0] ? evalExpr(call.arguments[0], frame, st, depth) : { kind: "str", t: ctx.String.val("") }]]) }; + } + // Constructor ports: environment-backed classes (crypto, counters) are + // injectable exactly like function ports. Sound because ported outputs + // are unconstrained symbolic values the property never depends on. + const ctorPort = name && input.options?.portImpls ? input.options.portImpls[name] : undefined; + if (ctorPort) { + const args = (call.arguments ?? []).map((a) => evalExpr(a, frame, st, depth)); + return (ctorPort as (...a: SymVal[]) => SymVal)(...args); + } + degrade(st, call, `new ${name}`); + return { kind: "any" }; + }; + + const mathCall = (meth: string, args: SymVal[], at: ts.Node, st: RunState): SymVal => { + const nums = args.map((a) => (isNum(a) ? a.t : null)); + if (nums.some((n) => n === null)) { + if (meth === "min" || meth === "max") { + // Math.min(x, y ?? inf) patterns: treat nulls as identity (skip) + const present = nums.filter((n): n is Num => n !== null); + if (present.length === 0) return { kind: "any" }; + return { kind: "num", t: present.reduce((a, b) => (meth === "min" ? minOf(a, b) : maxOf(a, b))) }; + } + degrade(st, at, `Math.${meth} degraded arg`); + return { kind: "any" }; + } + const a = nums as Num[]; + switch (meth) { + case "min": return { kind: "num", t: a.reduce((x, y) => minOf(x, y)) }; + case "max": return { kind: "num", t: a.reduce((x, y) => maxOf(x, y)) }; + case "abs": return { kind: "num", t: ctx.If(a[0].lt(num(ctx, 0)), a[0].neg(), a[0]) }; + case "round": { + // JS Math.round = floor(x + 0.5); stays Real-sorted (numbers are Real) + return { kind: "num", t: ctx.ToReal(ctx.ToInt(a[0].add(num(ctx, 0.5))) as never) as Num }; + } + case "floor": return { kind: "num", t: ctx.ToReal(ctx.ToInt(a[0]) as never) as Num }; + case "ceil": { + const f = ctx.ToReal(ctx.ToInt(a[0]) as never) as Num; + return { kind: "num", t: ctx.If(f.eq(a[0]), f, f.add(num(ctx, 1))) }; + } + case "trunc": { + const tf = ctx.ToReal(ctx.ToInt(a[0]) as never) as Num; + return { kind: "num", t: ctx.If(a[0].ge(num(ctx, 0)), tf, tf.add(num(ctx, 1))) }; + } + default: + degrade(st, at, `Math.${meth}`); + return { kind: "any" }; + } + }; + const minOf = (a: Num, b: Num): Num => ctx.If(a.le(b), a, b); + const maxOf = (a: Num, b: Num): Num => ctx.If(a.ge(b), a, b); + + /* ---------------- run ---------------- */ + + const frame: Frame = { vars: new Map(Object.entries(params)) }; + const st = runState0(); + // Bind declared entry params the caller did not pass: run their default + // initializers (JS semantics — defaults may reference earlier params) or + // bind undefined. Without this, later params are simply unbound. + for (const p of decl.parameters) { + if (!ts.isIdentifier(p.name)) continue; + if (frame.vars.has(p.name.text)) continue; + frame.vars.set(p.name.text, p.initializer ? evalExpr(p.initializer, frame, st, 0) : { kind: "undef" }); + } + execStmt(decl.body!, frame, st, 0); + let finalRet = st.ret; + for (let i = st.pendingRets.length - 1; i >= 0; i--) { + const pr = st.pendingRets[i]!; + finalRet = finalRet === null ? iteVal(pr.cond, pr.val, { kind: "undef" }) : iteVal(pr.cond, pr.val, finalRet); + } + return { + exec: () => ({ + ret: finalRet, + thrown: st.thrown ? { kind: "bool", t: st.thrown } : null, + assumptions: st.assumptions, + degradations: st.degradations, + ctx: input.ctx as import("./terms.js").Context, + }), + }; +} diff --git a/verif/terms.ts b/verif/terms.ts new file mode 100644 index 0000000000..d337a7643c --- /dev/null +++ b/verif/terms.ts @@ -0,0 +1,532 @@ +/** + * DAG-backed SMT2 term factory. + * + * Every constructor interns nodes by (op, kids, sort, const) so structurally + * equal terms are the SAME object — ITE-merges across loop iterations share + * structure instead of doubling string bytes (the pre-DAG build OOM'd at GBs + * on hide-consumed: rest-on-copy wraps the whole frame per iteration). + * `smt2Of` prints shared nodes as SMT-LIB `let` bindings, keeping the emitted + * file linear too. + */ + +type Sort = "Real" | "Int" | "Bool" | "String"; + +/** method surface installed onto Term.prototype by bindMethods() */ +export interface Term { + eq(o: Term): Term; + neq(o: Term): Term; + ge(o: Term): Term; + le(o: Term): Term; + gt(o: Term): Term; + lt(o: Term): Term; + add(o: Term): Term; + sub(o: Term): Term; + mul(o: Term): Term; + div(o: Term): Term; + mod(o: Term): Term; + neg(): Term; + and(o: Term): Term; + or(o: Term): Term; + not(): Term; + xor(o: Term): Term; + implies(o: Term): Term; + power(o: Term): Term; + concat(o: Term): Term; + length(): Term; + at(i: Term): Term; + extract(off: Term, len: Term): Term; + indexOf(s: Term, from?: Term): Term; + contains(s: Term): Term; + prefixOf(s: Term): Term; + suffixOf(s: Term): Term; + replace(a: Term, b: Term): Term; + toCode(): Term; + toInt(): Term; +} +/** compat aliases for the previous z3-flavored facade */ +export type Arith = Term; +export type Num = Term; +export type Bool<_T = unknown> = Term; +export type Seq<_T = unknown> = Term; + +export class Term { + /** cached inline print */ + private p?: string; + constructor( + readonly op: string, + readonly sort: Sort, + readonly kids: readonly Term[] = [], + /** concrete value when statically known */ + readonly c?: number | string | boolean, + readonly varName?: string, + readonly ctx?: Context, + ) {} + + toString(): string { + if (this.p !== undefined) return this.p; + const k = this.kids; + let s: string; + // SMT-LIB string literals: only "" doubles a quote — JSON's \" is ILLEGAL + // in SMT-LIB 2.6 (cvc5: 'Illegal string character'). Control chars go + // through \u{...}; printable Unicode stays raw (cvc5 reads UTF-8). + if (!k.length && this.sort === "String" && typeof this.c === "string") s = smtStr(this.c); + else if (!k.length) s = this.op; // var/literal pre-rendered as op + else if (this.op === "ite") s = `(ite ${k[0]!} ${k[1]!} ${k[2]!})`; + else s = `(${k.map(String).join(" ")})`; + this.p = s; + return s; + } + + /** DAG-safe structural hash (memoized, linear in DAG size). Use this — + * NEVER toString() — for equality checks and dedupe keys on terms that + * may be large: tree-printing a shared DAG is exponential. */ + private h?: number; + hash(): number { + if (this.h !== undefined) return this.h; + let acc = 2166136261; + const mix = (x: number): void => { acc ^= x; acc = Math.imul(acc, 16777619); }; + for (let i = 0; i < this.op.length; i++) mix(this.op.charCodeAt(i)); + mix(this.sort.length); + if (this.c !== undefined) mix(typeof this.c === "number" ? this.c : typeof this.c === "boolean" ? (this.c ? 1 : 0) : this.c.length); + for (const k of this.kids) mix(k.hash()); + this.h = acc >>> 0; + return this.h; + } +} + +/** SMT-LIB 2.6 string literal: quotes double; control chars as \u{...}. */ +function smtStr(v: string): string { + let out = '"'; + for (const ch of v) { + const cp = ch.codePointAt(0)!; + if (ch === '"') out += '""'; + else if (cp < 0x20 || cp === 0x7f) out += `\\u{${cp.toString(16).padStart(4, "0")}}`; + else out += ch; + } + return out + '"'; +} + +function fmtReal(n: number): string { + return Number.isInteger(n) ? `${n}.0` : String(n); +} + +const lit = (ctx: Context, s: string, sort: Sort, c?: number | string | boolean): Term => + ctx.intern(s, sort, [], c); + +export class Context { + readonly decls: string[] = []; + readonly declSort = new Map(); + readonly pendingFns = new Set(); + readonly pendingFnsR1 = new Set(); + readonly pendingFnsS1 = new Set(); + /** (String, String) → String, deterministic in both args. */ + readonly pendingFnsS2 = new Set(); + private readonly varCache = new Map(); + private nextVar = 0; + + fresh(prefix: string): string { + return `__${prefix}_${this.nextVar++}`; + } + + intern(op: string, sort: Sort, kids: readonly Term[], c?: number | string | boolean): Term { + // NOTE: deliberately NOT hash-consed into a global table — a retaining + // intern map blew the heap exponentially on loop-heavy merges (every + // discarded intermediate wrapper stayed reachable). Structural sharing + // happens at smt2Of emission time via canonical keys instead. + return new Term(op, sort, kids, c, undefined, this); + } + + app1(f: string, a: Term, sort: Sort): Term { + return this.intern("app1", sort, [lit(this, f, "String"), a]); + } + app2(f: string, a: Term, b: Term, sort: Sort): Term { + return this.intern("app2", sort, [lit(this, f, "String"), a, b]); + } + app3(f: string, a: Term, b: Term, c: Term, sort: Sort): Term { + return this.intern("app3", sort, [lit(this, f, "String"), a, b, c]); + } + + Real = { + val: (n: number): Term => lit(this, fmtReal(n), "Real", n), + const: (name: string): Term => this.declare(name, "Real"), + }; + Int = { + val: (n: number): Term => lit(this, String(n), "Int", n), + const: (name: string): Term => this.declare(name, "Int"), + }; + Bool = { + val: (b: boolean): Term => lit(this, b ? "true" : "false", "Bool", b), + const: (name: string): Term => this.declare(name, "Bool"), + }; + String = { + val: (s: string): Term => lit(this, JSON.stringify(s), "String", s), + const: (name: string): Term => this.declare(name, "String"), + }; + + private declare(name: string, sort: Sort): Term { + if (!this.decls.includes(name)) { + this.decls.push(name); + this.declSort.set(name, sort); + } + // leaf vars get canonical objects so identity-based folding works for + // repeated const(name) calls (bounded by declared names — no blowup risk) + const key = `${sort}:${name}`; + let t = this.varCache.get(key); + if (!t) { t = this.intern(name, sort, [], undefined); this.varCache.set(key, t); } + return t; + } + + If(c: Term, a: Term, b: Term): Term { + if (c.c === true) return a; + if (c.c === false) return b; + if (a === b) return a; + return this.intern("ite", a.sort, [c, a, b]); + } + ToReal(a: Term): Term { + if (a.sort === "Real") return a; + return this.app1("to_real", a, "Real"); + } + ToInt(a: Term): Term { + if (a.sort === "Int") return a; + if (a.c !== undefined && typeof a.c === "number") return lit(this, String(a.c), "Int", a.c); + return this.app1("to_int", a, "Int"); + } + IsInt(a: Term): Term { + if (a.sort === "Int") return lit(this, "true", "Bool", true); + if (a.c !== undefined && typeof a.c === "number") + return lit(this, String(Number.isInteger(a.c)), "Bool", Number.isInteger(a.c)); + return this.app1("is_int", a, "Bool"); + } +} + +// --------------------------------------------------------------------------- +// arithmetic / boolean / string ops — bound per-Context onto Term.prototype +// --------------------------------------------------------------------------- + +const numSortOf = (a: Term, b: Term): Sort => (a.sort === "Int" && b.sort === "Int" ? "Int" : "Real"); +const asReal = (t: Term): Term => (t.sort === "Int" ? t.ctx!.ToReal(t) : t); + +function foldArith(ctx: Context, op: string, a: Term, b: Term, sign: number, mul: boolean): Term { + const sort = numSortOf(a, b); + const av = typeof a.c === "number" ? a.c : undefined; + const bv = typeof b.c === "number" ? b.c : undefined; + if (av !== undefined && bv !== undefined) { + const v = mul ? av * bv : sign > 0 ? av + bv : av - bv; + return sort === "Real" ? lit(ctx, fmtReal(v), "Real", v) : lit(ctx, String(v), "Int", v); + } + if (mul) { + if (av === 0 || bv === 0) return sort === "Real" ? lit(ctx, "0.0", "Real", 0) : lit(ctx, "0", "Int", 0); + if (av === 1) return b; + if (bv === 1) return a; + } else { + if (sign > 0 && av === 0) return b; + if (sign > 0 && bv === 0) return a; + if (sign < 0 && bv === 0) return a; + } + return ctx.app2(op, a, b, sort); +} + +function foldCmp(ctx: Context, op: string, a: Term, b: Term): Term { + const av = typeof a.c === "number" ? a.c : undefined; + const bv = typeof b.c === "number" ? b.c : undefined; + if (av !== undefined && bv !== undefined) { + const v = op === "<" ? av < bv : op === "<=" ? av <= bv : op === ">" ? av > bv : av >= bv; + return lit(ctx, String(v), "Bool", v); + } + if (a === b && (op === "<=" || op === ">=")) return lit(ctx, "true", "Bool", true); + return ctx.app2(op, a, b, "Bool"); +} + +function foldBool(ctx: Context, op: string, a: Term, b: Term): Term { + const av = typeof a.c === "boolean" ? a.c : undefined; + const bv = typeof b.c === "boolean" ? b.c : undefined; + if (op === "and") { + if (av === false || bv === false) return lit(ctx, "false", "Bool", false); + if (av === true) return b; + if (bv === true) return a; + if (a === b) return a; + return ctx.app2("and", a, b, "Bool"); + } + if (op === "or") { + if (av === true || bv === true) return lit(ctx, "true", "Bool", true); + if (av === false) return b; + if (bv === false) return a; + if (a === b) return a; + return ctx.app2("or", a, b, "Bool"); + } + if (op === "xor") { + if (av !== undefined && bv !== undefined) return lit(ctx, String(av !== bv), "Bool", av !== bv); + if (a === b) return lit(ctx, "false", "Bool", false); + return ctx.app2("xor", a, b, "Bool"); + } + // implies + if (av === false || bv === true) return lit(ctx, "true", "Bool", true); + if (av === true) return b; + return ctx.app2("=>", a, b, "Bool"); +} + +function foldStrEq(ctx: Context, eq: boolean, a: Term, b: Term): Term { + const av = typeof a.c === "string" ? a.c : undefined; + const bv = typeof b.c === "string" ? b.c : undefined; + if (av !== undefined && bv !== undefined) { + const v = eq ? av === bv : av !== bv; + return lit(ctx, String(v), "Bool", v); + } + if (a === b) return lit(ctx, String(eq), "Bool", eq); + return ctx.app2(eq ? "=" : "distinct", a, b, "Bool"); +} + +function bindMethods(ctx: Context): void { + const proto: Record Term> = { + eq(o) { + const c = this.ctx ?? ctx; + if (this.sort === "Bool" && o.sort === "Bool") { + if (this.c !== undefined && o.c !== undefined) return lit(c, String(this.c === o.c), "Bool", this.c === o.c); + if (this === o) return lit(c, "true", "Bool", true); + return foldBool(c, "xor", this.not.call(this), o); // (= a b) ≡ ¬a⊕b folded via not+xor + } + if (this.sort === "String" || o.sort === "String") return foldStrEq(c, true, this, o); + if (this.sort !== o.sort) return foldCmp(c, "=", asReal(this), asReal(o)); + return foldCmp(c, "=", this, o); + }, + neq(o) { + const c = this.ctx ?? ctx; + if (this.sort === "String" || o.sort === "String") return foldStrEq(c, false, this, o); + const e = proto.eq!.call(this, o); + if (e.c !== undefined) return lit(c, String(!e.c), "Bool", !e.c); + return e.not.call(e); + }, + ge(o) { return foldCmp(this.ctx ?? ctx, ">=", asReal(this), asReal(o)); }, + le(o) { return foldCmp(this.ctx ?? ctx, "<=", asReal(this), asReal(o)); }, + gt(o) { return foldCmp(this.ctx ?? ctx, ">", asReal(this), asReal(o)); }, + lt(o) { return foldCmp(this.ctx ?? ctx, "<", asReal(this), asReal(o)); }, + add(o) { return foldArith(this.ctx ?? ctx, "+", asReal(this), asReal(o), 1, false); }, + sub(o) { return foldArith(this.ctx ?? ctx, "-", asReal(this), asReal(o), -1, false); }, + mul(o) { return foldArith(this.ctx ?? ctx, "*", asReal(this), asReal(o), 1, true); }, + div(o) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "number" ? this.c : undefined; + const bv = typeof o.c === "number" ? o.c : undefined; + if (av !== undefined && bv !== undefined && Number.isInteger(av) && Number.isInteger(bv) && bv !== 0 && av % bv === 0) + return lit(c, `${av / bv}.0`, "Real", av / bv); + return c.app2("/", asReal(this), asReal(o), "Real"); + }, + mod(o) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "number" ? this.c : undefined; + const bv = typeof o.c === "number" ? o.c : undefined; + if (av !== undefined && bv !== undefined && Number.isInteger(av) && Number.isInteger(bv) && bv !== 0) + return lit(c, String(av % bv), "Int", av % bv); + return c.app2("mod", this, o, "Int"); + }, + neg() { + const av = typeof this.c === "number" ? this.c : undefined; + if (av !== undefined) return this.sort === "Real" ? lit(this.ctx ?? ctx, fmtReal(-av), "Real", -av) : lit(this.ctx ?? ctx, String(-av), "Int", -av); + return (this.ctx ?? ctx).app1("-", this, this.sort); + }, + and(o) { return foldBool(this.ctx ?? ctx, "and", this, o); }, + or(o) { return foldBool(this.ctx ?? ctx, "or", this, o); }, + not() { + const av = typeof this.c === "boolean" ? this.c : undefined; + if (av !== undefined) return lit(this.ctx ?? ctx, String(!av), "Bool", !av); + return (this.ctx ?? ctx).app1("not", this, "Bool"); + }, + xor(o) { return foldBool(this.ctx ?? ctx, "xor", this, o); }, + implies(o) { return foldBool(this.ctx ?? ctx, "=>", this, o); }, + power(o) { + const c = this.ctx ?? ctx; + const sv = typeof this.c === "string" ? this.c : undefined; + const nv = typeof o.c === "number" ? o.c : undefined; + if (sv !== undefined && nv !== undefined && sv.length * Math.max(0, nv) <= 4096) { + const s = sv.repeat(Math.max(0, nv)); + return lit(c, JSON.stringify(s), "String", s); + } + c.pendingFns.add("__str_repeat"); + return c.app2("__str_repeat", this, o, "String"); + }, + concat(o) { + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof o.c === "string" ? o.c : undefined; + if (av !== undefined && bv !== undefined) return lit(this.ctx ?? ctx, JSON.stringify(av + bv), "String", av + bv); + if (av === "") return o; + if (bv === "") return this; + return (this.ctx ?? ctx).app2("str.++", this, o, "String"); + }, + length() { + const av = typeof this.c === "string" ? this.c : undefined; + if (av !== undefined) return lit(this.ctx ?? ctx, String(av.length), "Int", av.length); + return (this.ctx ?? ctx).app1("str.len", this, "Int"); + }, + at(i) { return (this.ctx ?? ctx).app2("str.at", this, i, "String"); }, + extract(off, len) { return (this.ctx ?? ctx).app3("str.extract", this, off, len, "String"); }, + indexOf(s, from) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof s.c === "string" ? s.c : undefined; + if (av !== undefined && bv !== undefined && from === undefined) + return lit(c, String(av.indexOf(bv)), "Int", av.indexOf(bv)); + return from === undefined + ? c.app2("str.indexof", this, s, "Int") + : c.app3("str.indexof", this, s, from, "Int"); + }, + contains(s) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof s.c === "string" ? s.c : undefined; + if (av !== undefined && bv !== undefined) return lit(c, String(av.includes(bv)), "Bool", av.includes(bv)); + if (this === s) return lit(c, "true", "Bool", true); + return c.app2("str.contains", this, s, "Bool"); + }, + prefixOf(s) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof s.c === "string" ? s.c : undefined; + if (av !== undefined && bv !== undefined) return lit(c, String(bv.startsWith(av)), "Bool", bv.startsWith(av)); + return c.app2("str.prefixof", this, s, "Bool"); + }, + suffixOf(s) { + const c = this.ctx ?? ctx; + const av = typeof this.c === "string" ? this.c : undefined; + const bv = typeof s.c === "string" ? s.c : undefined; + if (av !== undefined && bv !== undefined) return lit(c, String(bv.endsWith(av)), "Bool", bv.endsWith(av)); + return c.app2("str.suffixof", this, s, "Bool"); + }, + replace(a, b) { + const c = this.ctx ?? ctx; + const sv = typeof this.c === "string" ? this.c : undefined; + const av = typeof a.c === "string" ? a.c : undefined; + const bv = typeof b.c === "string" ? b.c : undefined; + if (sv !== undefined && av !== undefined && bv !== undefined) + return lit(c, JSON.stringify(sv.replace(av, bv)), "String", sv.replace(av, bv)); + return c.app3("str.replace", this, a, b, "String"); + }, + toCode() { + const av = typeof this.c === "string" ? this.c : undefined; + if (av !== undefined) return lit(this.ctx ?? ctx, String(av.codePointAt(0) ?? 0), "Int", av.codePointAt(0) ?? 0); + return (this.ctx ?? ctx).app1("str.to_code", this, "Int"); + }, + toInt() { + const av = typeof this.c === "string" ? this.c : undefined; + if (av !== undefined) { + const n = Number(av); + return lit(this.ctx ?? ctx, String(n), "Int", n); + } + return (this.ctx ?? ctx).app1("str.to_int", this, "Int"); + }, + }; + for (const [name, fn] of Object.entries(proto)) { + (Term.prototype as any)[name] = fn; + } +} + +/** convenience: build a string-constant Term bound to ctx */ +export function strc(ctx: Context, sv: string): Term { + return ctx.String.val(sv); +} + +export function newCtx(): Context { + const ctx = new Context(); + bindMethods(ctx); + return ctx; +} + +// --------------------------------------------------------------------------- +// SMT2 emission with let-sharing +// --------------------------------------------------------------------------- + +interface Smt2Out { + smt2: string; + lets: number; +} + +export function smt2Of( + ctx: Context, + asserts: readonly Term[], + getValue: readonly string[] = [], +): Smt2Out { + // canonical structural key per node (bottom-up, memoized by object) + const canon = new Map(); + let seq = 0; + // DAG-safe hashing: string keys blow up quadratically on deep chains + // (Invalid string length). Numbers via Term.hash() are memoized+linear. + const seenObj = new Set(); + const counts = new Map(); + const walk = (t: Term): void => { + if (seenObj.has(t)) return; + seenObj.add(t); + for (const k of t.kids) { + counts.set(k.hash(), (counts.get(k.hash()) ?? 0) + 1); + walk(k); + } + }; + for (const a of asserts) { + counts.set(a.hash(), (counts.get(a.hash()) ?? 0) + 1); + walk(a); + } + const sizeCache = new Map(); + const sizeOf = (t: Term): number => { + const h = t.hash(); + const hit = sizeCache.get(h); + if (hit !== undefined) return hit; + let n = 1; + for (const kk of t.kids) n += sizeOf(kk); + sizeCache.set(h, n); + return n; + }; + // assign let names by hash, post-order + const names = new Map(); + let counter = 0; + const seen2 = new Set(); + const assign = (t: Term): void => { + if (seen2.has(t)) return; + seen2.add(t); + for (const k of t.kids) assign(k); + const key = t.hash(); + if (t.kids.length && (counts.get(key) ?? 0) > 1 && sizeOf(t) > 24 && !names.has(key)) { + names.set(key, `!t${counter++}`); + } + }; + for (const a of asserts) assign(a); + const render = (t: Term, allowName = true): string => { + const key = t.hash(); + const n = allowName ? names.get(key) : undefined; + if (n !== undefined) return n; + if (!t.kids.length) return t.toString(); + if (t.op === "ite") return `(ite ${render(t.kids[0]!)} ${render(t.kids[1]!)} ${render(t.kids[2]!)})`; + if (t.op === "app1" || t.op === "app2" || t.op === "app3") + return `(${t.kids.map((k) => render(k)).join(" ")})`; // kids keep allowName; kid0 is the fn name + return `(${t.op} ${t.kids.map((k) => render(k)).join(" ")})`; // kids keep allowName + }; + + const lines: string[] = ["(set-logic ALL)"]; + for (const fn of ctx.pendingFns) lines.push(`(declare-fun ${fn} (String Int) String)`); + for (const fn of ctx.pendingFnsR1) lines.push(`(declare-fun ${fn} (Real) String)`); + for (const fn of ctx.pendingFnsS1) lines.push(`(declare-fun ${fn} (String) String)`); + for (const fn of (ctx as unknown as { pendingFnsS2: Set }).pendingFnsS2) lines.push(`(declare-fun ${fn} (String String) String)`); + for (const d of ctx.decls) lines.push(`(declare-fun ${d} () ${ctx.declSort.get(d) ?? "Real"})`); + if (names.size) { + // Shared subterms become top-level define-funs — sequential commands, + // so later definitions may reference earlier ones. (SMT-LIB `let` + // bindings are PARALLEL: a binding cannot reference its siblings — + // cvc5 rejects chained lets with "Symbol not declared".) + const defFor = new Map(); + const seen3 = new Set(); + const collect = (t: Term): void => { + if (seen3.has(t)) return; + seen3.add(t); + const k = t.hash(); + if (names.has(k) && !defFor.has(k)) defFor.set(k, t); + t.kids.forEach(collect); + }; + asserts.forEach(collect); + for (const [k, n] of names) { + const d = defFor.get(k)!; + lines.push(`(define-fun ${n} () ${d.sort} ${render(d, false)})`); + } + } + for (const a of asserts) lines.push(`(assert ${render(a)})`); + lines.push("(check-sat)"); + if (getValue.length) lines.push(`(get-value (${getValue.join(" ")}))`); + lines.push("(exit)"); + return { smt2: lines.join("\n") + "\n", lets: names.size }; +}