From 6a9d403909173d15ef670ce1fb05e3f7f26ee8b9 Mon Sep 17 00:00:00 2001 From: loalissm Date: Sun, 18 Oct 2026 11:01:14 +0800 Subject: [PATCH 01/14] fold anchoring doctrine: order over hash (#3641, fixes #3588, fixes #3640) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three real-world defect classes shared one root: fold identity keyed coverage to content hashes, so id re-mints without content loss were read as destruction. - shape tier in planReconciliation: loose word-evidence pairing for prose cells (wordbag equal + simhash hamming <= 8 + role/length guards), flanked k-th discipline for evidence-less cells (image families, #3588) — claims require BOTH flanks verified on the wire - residual census: unmatched class-carrying ids classify as transformed (near-dup evidence exists) vs absent-everywhere (content truly off the wire) - verdict classifier: transform churn and wire-suffix-continuation truncation (host compaction/windowing, #2695) never accrue the collapse streak; one info latch each per session - episode witness: wire order frozen at episode start; a foreign conversation can never become the truncation baseline mid-episode - total collapse now latches foldEpochConfirmPending and pauses folding (verbatim passthrough, snapshot kept) until realignment or explicit user epoch confirmation, replacing the destroyed-escalation misdiagnosis; detection intact (#2903 tests) - unresolved summary anchors clamp ahead of the newest user turn instead of the array tail (#3640 "model answers stale summaries") - single genuine edits below the mass bar still report drift honestly Tests: tests/fold-anchor-doctrine.test.ts 6/6; families green (fold-drift 7, fold-reconcile 66, positional 24, issue2915 15, and the 1908/1987/2033/2601/1595/1820/1569/images-archive/refold set). --- CHANGELOG.md | 2 + kernel/src/prune.ts | 46 ++- kernel/tests/state-prune.test.ts | 33 ++ src/fold-reconcile.ts | 318 +++++++++++++++++- src/launcher.ts | 2 +- src/preflight.ts | 15 + tests/fold-anchor-doctrine.test.ts | 220 ++++++++++++ tests/fold-reconcile-positional.test.ts | 10 +- tests/fold-reconcile.test.ts | 6 +- .../issue2915-order-centric-reconcile.test.ts | 6 +- 10 files changed, 629 insertions(+), 29 deletions(-) create mode 100644 tests/fold-anchor-doctrine.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 37ca28fc5b..cfb6ab86c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Changes +- **fold anchoring doctrine: order over hash — protocol switches, image offload, and host-side truncation stop being read as destruction (#3641, fixes #3588, fixes #3640).** Three real-world defect classes all shared one root: the fold's identity model keyed coverage to content hashes, so any transformation that re-mints ids without removing content (dsh's image-offload component rewriting image cells into placeholder text, opencode switching a mid-session request from `/chat/completions` to `/v1/messages`, or a host that compacts/truncates history itself) made every covered id look *destroyed*. Consequences: 7× false "compression substrate appears destroyed" escalations on healthy sessions (#3588), 298-id churn with unanchored summary blocks landing AFTER the user's newest question — the model answered five rounds of stale summaries instead of the question (#3640) — and the #2193 collapse streak on sanctioned host compaction. The reconcile pass now carries a shape tier (loose word-evidence pairing for prose cells, a flanked k-th discipline for evidence-less image cells), a residual census classifying unmatched ids as *transformed* (near-duplicate evidence exists), *absent* (content truly off the wire with the surviving wire a suffix-continuation of the episode baseline), or *drift* (nothing matches anywhere); only the third accrues the collapse streak, and a wire-frozen episode witness guarantees a foreign conversation can never quietly become the baseline mid-episode. Unresolved summary anchors clamp ahead of the newest user turn instead of at the array tail (the #3640 "stale summary after the question" shape), and a genuine total collapse now latches `foldEpochConfirmPending` and PAUSES folding (verbatim passthrough — nothing is deleted, the snapshot is kept) until the histories realign or the user confirms a new epoch, instead of escalating on a misdiagnosis. Single genuine edits below the mass bar still report drift honestly; the #1729/#2193 escalation itself stays (its error text now names the epoch gate). + - **dsh PTC sessions get compression back: a `run_code` omnibus on the wire reads as reachable, not unreachable (#3600).** dsh PTC presets collapse EVERY tool into a single `run_code` omnibus (the SDK bindings nest inside it), so the top-level tools array carries no `compress` entry. The #3427 reachability gate read exactly that shape as "tool not host-scoped → suppress the compress surface", and released 0.1.198 suppressed every dsh PTC session: 89× `ARMED-SUPPRESSED` nudges while the window ran to 100%, then the output got clamped and the session was effectively dead — silently, while standard-mode sessions on the same install kept compressing (v0.1.197 had no tools factor in its nudge gate, so this is a release regression, not "never worked"). The gate now treats a `run_code` omnibus declaration as reachable: top-level absence is not evidence of unreachability when the omnibus swallows everything. The #3177 true negative keeps its suppression (a bare single tool that is neither `compress` nor the omnibus still drops the surface), and the #3401 positive case is unchanged. - **dsh desktop settings pane: the embed probe is admitted for the Electron scheme origin (#3513, desktop variant).** The dsh DESKTOP app renders its settings pane from the custom scheme origin `dsh-app://app` (not an http origin like dsh web). The #3514 fix echoed the requesting Origin — but the request never reached the echo: `isTrustedAdminOrigin` only accepts http/https origins, so the probe's `GET /__bili/?embed=1` died in the trust gate with a bare 403 and no `access-control-allow-origin` — the pane stayed on "正在连接 bili 代理…" forever (real-world console: "blocked by CORS policy … from origin 'dsh-app://app'"). The gate now admits exactly that one origin (scheme `dsh-app`, hostname `app`): browsers set the Origin header themselves and the `dsh-app` scheme only exists inside the dsh Electron app, so a website can never produce that Origin — while rebinding attacks still arrive with http/https attacker hostnames and stay rejected. Foreign hostnames under the same scheme (`dsh-app://evil`) and every other custom scheme stay 403; the existing #3514 echo then covers the response. diff --git a/kernel/src/prune.ts b/kernel/src/prune.ts index b1ece41faa..416406d096 100644 --- a/kernel/src/prune.ts +++ b/kernel/src/prune.ts @@ -65,9 +65,10 @@ export interface PruneOptions { /** * Chronological head anchoring for views with NO provider prefix cache at * stake (session handoff/restore docs). Default (false) is the pin-safe - * placement: an anchor whose ids are absent from the input parks at the - * tail, and no summary may precede the first user message — the pin is the - * provider prefix-cache head (#2733). + * placement: an anchor whose ids are absent from the input parks before + * the newest user message (never at the tail, #3640), and no summary may + * precede the first user message — the pin is the provider prefix-cache + * head (#2733). */ headSummaryAnchors?: boolean; /** @@ -95,6 +96,21 @@ export function prune( const firstUserIndex = messages.findIndex( (message) => message.role === "user", ); + // #3640: an unresolved anchor (no covered id present in this input) parks + // immediately BEFORE the newest user message, never at the list tail. + // Tail-parking put days-old summaries after the user's current question, + // and the model answered the stale recap instead of the question for every + // subsequent turn. Parking before the newest user turn also keeps the + // provider prefix cache intact: the insertion point sits inside the + // turn-over region that the next request rewrites anyway (#2733 trade-off + // reassessed). No user message at all: keep the legacy tail. + let lastUserIndex = -1; + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i]!.role === "user") { + lastUserIndex = i; + break; + } + } const baseIndexById = new Map(); const summaryIndexById = new Map(); @@ -112,7 +128,11 @@ export function prune( state, baseIndexById, summaryIndexById, - options.headSummaryAnchors ? 0 : messages.length, + options.headSummaryAnchors + ? 0 + : lastUserIndex >= 0 + ? lastUserIndex + : messages.length, ) : []; // Summary ids of every block this session has ever created — active or @@ -149,6 +169,14 @@ interface SummaryAnchor { summary: string; topic?: string; insertAt: number; + /** True when no anchor id was present in the input (the substrate is off + * the wire). Unresolved anchors park before the newest user message and + * are exempt from the first-user-pin lower bound of firstUserIndex + 1: + * riding AFTER the user's current question is the functional bug #3640 + * (the model answered the stale recap instead of the question for five + * consecutive turns), and the rendered summary is role "system", so + * leading the view is provider-safe. */ + unresolved?: boolean; } function collectSummaryAnchors( @@ -197,10 +225,12 @@ function collectSummaryAnchors( blockId: block.blockId, summary: block.summary, topic: block.topic, - // Unresolved (no anchor id present in this input): park at the tail — - // index 0 would splice the summary ahead of the session head and nuke - // the provider's cached prefix for every subsequent turn (#2733). + // Unresolved (no anchor id present in this input): park before the + // newest user message — see prune() for the #3640/#2733 reasoning. + // Tail-parking (the pre-#3640 behavior) let stale summaries land after + // the user's current question, which the model then answered instead. insertAt: earliest ?? fallbackIndex, + unresolved: earliest === null, }); } anchors.sort((left, right) => left.insertAt - right.insertAt); @@ -343,7 +373,7 @@ function rebuildMessages( ...anchor, insertAt: headAnchors ? anchor.insertAt - : Math.max(anchor.insertAt, firstUserIndex + 1), + : Math.max(anchor.insertAt, firstUserIndex + (anchor.unresolved === true ? 0 : 1)), })) .sort((left, right) => left.insertAt - right.insertAt); const result: CoreMessage[] = []; diff --git a/kernel/tests/state-prune.test.ts b/kernel/tests/state-prune.test.ts index acf31aef1e..bcc68de1b0 100644 --- a/kernel/tests/state-prune.test.ts +++ b/kernel/tests/state-prune.test.ts @@ -313,3 +313,36 @@ test("processTurn tolerates id-less host-shaped messages (pre-#233 lenience, #42 assert.ok(Array.isArray(turn.messages), "turn completes without throwing"); assert.equal(turn.state.blocks.length, 0, "no blocks from a crash-free pass"); }); + +// #3641/#3640: unresolved summary anchors — the covered substrate is off the +// wire (host-native compaction, truncation, offload churn) — must never park +// at the very tail riding AFTER the user's newest question. The insertion +// point is the renovation zone (messages every upcoming turn rewrites +// anyway), so cache is unharmed, and the model answers the question first. +test("#3640: unresolved anchor parks BEFORE the newest user message, never at the tail", () => { + const state = createInitialState(); + state.blocks.push( + makeBlock({ blockId: "b1", summary: "orphaned history summary", effectiveMessageIds: ["gone1", "gone2"] }), + ); + const messages = [ + msg("keep1", "assistant"), + msg("keep2", "assistant"), + msg("q", "user"), // the newest question + ]; + const result = prune(messages, state); + const anchorIdx = result.findIndex((m) => m.id === "acp_summary_b1"); + const qIdx = result.findIndex((m) => m.id === "q"); + assert.ok(anchorIdx >= 0, "the orphaned summary still renders (snapshot survives)"); + assert.ok(qIdx > anchorIdx, `summary (idx ${anchorIdx}) must precede the newest user turn (idx ${qIdx})`); +}); + +test("#3641: no user turn on the wire → legacy tail placement preserved", () => { + const state = createInitialState(); + state.blocks.push( + makeBlock({ blockId: "b1", summary: "s", effectiveMessageIds: ["gone1"] }), + ); + const messages = [msg("a1", "assistant"), msg("a2", "assistant")]; + const result = prune(messages, state); + assert.equal(result[result.length - 1]!.id, "acp_summary_b1", + "assistant-only history keeps the pre-doctrine tail placement"); +}); diff --git a/src/fold-reconcile.ts b/src/fold-reconcile.ts index a715f7af54..424d59dc40 100644 --- a/src/fold-reconcile.ts +++ b/src/fold-reconcile.ts @@ -178,11 +178,36 @@ const METADATA_MARKER_CHURN_NOTED = "foldMarkerChurnNoted"; * the substrate-destruction verdict (#2360 §2.4) without hardcoding a second * copy of the key here. */ export const METADATA_DRIFT_ESCALATED = "foldDriftEscalated"; +/** #3641: total collapse (nothing on the wire matches the session's fold + * substrate anywhere) latches an epoch-confirmation request instead of + * silently resetting: new folds are suspended for the session until the + * user confirms the new epoch or the original identity returns on the wire + * (auto-clear). The old destroy verdict relabeled: the session asks, it + * never abandons the snapshot on its own. */ +const METADATA_EPOCH_CONFIRM = "foldEpochConfirmPending"; +/** #3641: one-per-episode info latches for benign verdicts (shape churn / + * truncation). Cleared with the rest of the drift state on recovery. */ +const METADATA_SHAPE_NOTED = "foldShapeChurnNoted"; +const METADATA_TRUNC_NOTED = "foldTruncationNoted"; +/** #3641: the wire order frozen at the moment the drift episode began — the + * baseline against which truncation-vs-replacement is decided. */ +const METADATA_EPISODE_ORDER = "foldEpisodeOrder"; +/** #3641 helper: true while the session waits for epoch confirmation. + * Consumers (preflight fold gate, status panel) read this, never the raw + * metadata key. */ +export function epochConfirmPending(session: Session): boolean { + return (session.metadata?.[METADATA_EPOCH_CONFIRM] as boolean | undefined) === true; +} /** #2193: total-loss drift (covered ids missing with ZERO reanchoring) across * this many consecutive passes means the fold state can never recover — * escalate once from warn to error and name the suspect cause instead of * letting the same warn print hundreds of times. */ const FOLD_DRIFT_ESCALATE_PASSES = 3; +/** #3641: the shape pass only fires on MASS churn — at least this many + * residual missing cells must sit in the churn region before order-pairing + * is admissible evidence. Below it, honest unmatched (per-pass warn) is + * the right diagnosis: single-cell churn is ordinary edits. */ +const SHAPE_MASS_MIN = 4; /** Below this many permanently-missing ids the loss is small enough (a few * edited/deleted messages) to stay at warn level. */ const FOLD_DRIFT_ESCALATE_MIN_UNMATCHED = 10; @@ -236,6 +261,13 @@ function resetFoldDriftState(session: Session): void { delete md[METADATA_DRIFT_STREAK]; delete md[METADATA_DRIFT_SINCE]; delete md[METADATA_DRIFT_ESCALATED]; + // #3641: a clean pass (any covered id present or re-anchored) IS the + // confirmation — the histories realigned, so the epoch ask and the + // one-per-episode info latches clear with the rest of the drift state. + delete md[METADATA_EPOCH_CONFIRM]; + delete md[METADATA_SHAPE_NOTED]; + delete md[METADATA_TRUNC_NOTED]; + delete md[METADATA_EPISODE_ORDER]; } /** Per-covered-id anchor recorded from the last pass in which the id was seen. @@ -306,6 +338,12 @@ interface ReconciliationPlan { byNorm: number; byTurn: number; byImage: number; + /** #3641: claims made by the shape-census pass — mass churn (protocol + * switch, image offload) where word evidence survives but EVERY exact + * channel starved (no fence, no bracket, norms changed with the shape). + * Order-paired under wordbag + simhash corroboration, or k-th family + * pairing for textless anchors; only fires on mass events (≥4 cells). */ + byShape: number; /** #2915: order breaks the positional scans resynced over (gaps claimed * as order-carried spans past a fingerprint break) — observability for * the order-centric doctrine, surfaced in the drift log line. */ @@ -323,6 +361,16 @@ interface ReconciliationPlan { * caller's write-back (foldAnchorOrder / foldPositions rolling copy). */ newOrder: string[]; nextCanon: string[] | undefined; + /** #3641 census of the RESIDUAL unmatched (after every claim pass, + * including the shape pass): `absentEverywhere` = no unclaimed inbound + * candidate carries loose word evidence (the content is truly off the + * wire — host truncation / sanctioned deletion); `nearDupResidual` = + * loose evidence exists but stayed unclaimable (duplicate mass beyond + * the family discipline). The verdict classifier reads these: transform + * churn re-anchors, absence is benign, only zero-evidence-everywhere + * still accrues the collapse streak. */ + absentEverywhere: string[]; + nearDupResidual: string[]; /** #2283: sum of defaultCountTokens over every churn-region candidate — the * raw mass riding unfolded in the drift zone this pass. Undefined when there * is no drift (the missing.length === 0 early return skips this). */ @@ -345,6 +393,9 @@ interface FoldReconcileResult { byNorm: number; byTurn: number; byImage: number; + /** #3641: cells re-anchored by the shape-census pass (mass churn with + * word evidence: protocol switch, image offload). */ + byShape: number; /** #2695: fold blocks reaped this pass (removed from state.blocks after * a majority-loss zombie streak). */ reaped: number; @@ -657,7 +708,7 @@ export function planReconciliation( covered: Set, positions?: { ids: string[]; canon: string[] }, ): ReconciliationPlan { - const plan: ReconciliationPlan = { claims: new Map(), byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, unmatched: [], idRewriteSuspects: 0, newOrder: [], nextCanon: undefined, resyncs: 0 }; + const plan: ReconciliationPlan = { claims: new Map(), byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, unmatched: [], idRewriteSuspects: 0, newOrder: [], nextCanon: undefined, resyncs: 0, byShape: 0, absentEverywhere: [], nearDupResidual: [] }; // #2480: the positional copy is built on EVERY pass, so the first pass // after an upgrade establishes it and every later drift is // position-claimable. The fast path is reuse-by-ID (#2487): same id = @@ -1174,6 +1225,156 @@ export function planReconciliation( if (!plan.claims.has(id)) plan.unmatched.push(id); } + // #3641 — shape census pass. Mass churn (protocol switch #3640, image + // offload #3588) defeats EVERY exact channel at once: ids re-minted, + // norms changed with the wire shape, no positional fence ever verifies, + // gap cells never recorded — so hundreds of covered ids land here with + // the words still on the wire. Doctrine (order carries, content gates, + // the id is only a hint): when a MISSING MASSES of cells still carry + // word evidence, pair them in order instead of diagnosing destruction. + // prose anchors: wordbag equality + simhash ≤ GAP bar (the same + // edit-honesty gate as the sim tier) — role-checked, length-guarded; + // textless anchors (image families, no sh/wh by construction): k-th + // family pairing, only when the anchor/candidate counts per + // (role, class) bucket match EXACTLY — the byImage discipline, + // generalized past bili's own placeholder variant. + // Only fires on mass events (≥ SHAPE_MASS_MIN residual cells): small + // churn keeps the honest unmatched path below. + const residualMiddle = missingMiddle.filter((id) => !plan.claims.has(id)); + if (residualMiddle.length >= SHAPE_MASS_MIN) { + const looseProse = (anchor: FoldAnchor, m: CoreMessage): boolean => { + if (anchor.wh === undefined || anchor.sh === undefined) return false; + if (anchor.r !== undefined && anchor.r !== m.role) return false; + const len = m.text?.length ?? 0; + if (Math.abs(len - anchor.b) > Math.max(256, anchor.b >> 2)) return false; + const wh = wordBagDigest(canonicalTextOf(m.text ?? "")); + if (wh !== anchor.wh) return false; + const dist = hammingHex(anchor.sh, simhashHex(canonicalTextOf(m.text ?? ""))); + return dist >= 0 && dist <= GAP_SIMHASH_MAX_HAMMING; + }; + // prose pairing — order-preserving: walk anchors in conversation + // order, take the first unclaimed loose candidate (also in order), + // so pairs never cross when both sides are locally order-stable. + for (const oldId of residualMiddle) { + const anchor = anchors[oldId]; + if (anchor === undefined || anchor.sh === undefined || anchor.wh === undefined) continue; + const cand = candidates.find((c) => !claimedCandidates.has(c.id) && looseProse(anchor, c.message)); + if (cand === undefined) continue; + plan.claims.set(oldId, [cand.id]); + claimedCandidates.add(cand.id); + plan.byShape++; + } + // Textless family pairing — the #3588 incident shape: a contiguous + // family of image-only cells (text "" → no sh/wh by construction) + // rewritten into placeholder strings by the host's offload component. + // Empty text cannot carry word evidence, so the ONLY admissible + // evidence is positional: bracket the family in the PREVIOUS wire + // order by stable flanks (ids present or re-anchored this pass), + // take the unclaimed candidate cells strictly between the same + // flanks on the CURRENT wire, and pair k-th to k-th — counts must + // match exactly (the byImage family discipline, generalized past + // bili's own placeholder variant to ANY rewrite, #3641). + const textless = residualMiddle.filter((id) => { + const anchor = anchors[id]; + return anchor !== undefined && anchor.sh === undefined && anchor.wh === undefined; + }); + if (textless.length > 0) { + const posInOld = new Map(); + oldOrder.forEach((id, i) => posInOld.set(id, i)); + const newIndexOf = new Map(); + plan.newOrder.forEach((id, i) => newIndexOf.set(id, i)); + const candById = new Map(); + for (const c of candidates) candById.set(c.id, c); + // runs of contiguous textless anchors in the previous wire order + const placed = textless + .map((id) => ({ id, at: posInOld.get(id) })) + .filter((x): x is { id: string; at: number } => x.at !== undefined) + .sort((a, b) => a.at - b.at); + const runs: Array> = []; + for (const item of placed) { + const last = runs[runs.length - 1]; + if (last !== undefined && item.at === last[last.length - 1]!.at + 1) last.push(item); + else runs.push([item]); + } + const wireIndexOf = (id: string): number => { + const direct = newIndexOf.get(id); + if (direct !== undefined) return direct; + const claimed = plan.claims.get(id); + if (claimed !== undefined) { + for (const nid of claimed) { + const at = newIndexOf.get(nid); + if (at !== undefined) return at; + } + } + return -1; + }; + for (const run of runs) { + const lo = run[0]!.at; + const hi = run[run.length - 1]!.at; + // nearest stable flanks: previous-order ids before/after the + // run that survive on the current wire (present or claimed) + let left = -1; + for (let j = lo - 1; j >= 0; j--) { + const at = wireIndexOf(oldOrder[j]!); + if (at >= 0) { left = at; break; } + } + let right = -1; + for (let j = hi + 1; j < oldOrder.length; j++) { + const at = wireIndexOf(oldOrder[j]!); + if (at >= 0) { right = at; break; } + } + // BOTH flanks are mandatory: a run touching either end of the + // previous wire order has no fence on that side, and pairing + // into open space is guessing, not evidence (a brand-new + // conversation tail must never claim a vanished family). + if (left < 0 || right < 0) continue; + if (left >= right) continue; // flanks inverted (reorder) — unpairable + const familyCands: Array<(typeof candidates)[number]> = []; + for (let at = left + 1; at < right; at++) { + const id = plan.newOrder[at]!; + const c = candById.get(id); + if (c === undefined || claimedCandidates.has(id) || covered.has(id)) continue; + familyCands.push(c); + } + if (familyCands.length !== run.length) continue; // off-by-one → no guessing + for (let k = 0; k < run.length; k++) { + plan.claims.set(run[k]!.id, [familyCands[k]!.id]); + claimedCandidates.add(familyCands[k]!.id); + plan.byShape++; + } + } + } + } + // #3641 census of what STILL did not claim — the verdict classifier's + // evidence: absentEverywhere (content truly off the wire → benign + // truncation / sanctioned deletion) vs nearDupResidual (word evidence + // exists but stayed unclaimable — duplicate mass, transform churn the + // family discipline could not pair). + { + const residualClasses = new Set(["text", "tool-call", "tool-result", "image", "reasoning", "code"]); + const classCarrying = plan.unmatched.filter((id) => { + const anchor = anchors[id]; + return anchor !== undefined && anchor.c !== undefined && anchor.c !== "reasoning"; + }); + for (const id of classCarrying) { + const anchor = anchors[id]!; + const loose = candidates.some((c) => + !claimedCandidates.has(c.id) + && (anchor.r === undefined || anchor.r === c.message.role) + && ( + (anchor.wh !== undefined && anchor.sh !== undefined + && wordBagDigest(canonicalTextOf(c.message.text ?? "")) === anchor.wh + && (() => { const d = hammingHex(anchor.sh!, simhashHex(canonicalTextOf(c.message.text ?? ""))); return d >= 0 && d <= GAP_SIMHASH_MAX_HAMMING; })()) + )); + // textless anchors (no sh/wh — image cells, #3588) carry NO + // evidence channel: claimable families were already claimed by + // the flanked k-th discipline above, so a textless residual is + // proof-less absence, never "near-duplicate evidence". + if (loose) plan.nearDupResidual.push(id); + else plan.absentEverywhere.push(id); + } + } + // #2396 — residual rewrite suspects. Pass 1 keys on the toolCallId itself // and misses rewritten ids by construction; Pass 1.5 (#2915) claims the // content-identical twins, so what reaches here survived EVERY claim pass @@ -1271,13 +1472,13 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt const mode = opts.mode ?? resolveFoldReconcileMode(process.env); if (mode === "off") { resetFoldDriftState(session); - return { kind: "off", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, reaped: 0, unmatched: 0 }; + return { kind: "off", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, byShape: 0, reaped: 0, unmatched: 0 }; } const blocks = (session.state?.blocks ?? []) as BlockLike[]; const covered = coveredIdsOf(blocks); if (covered.size === 0) { resetFoldDriftState(session); - return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, reaped: 0, unmatched: 0 }; + return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, byShape: 0, reaped: 0, unmatched: 0 }; } // #2202: auxiliary side-requests (title-gen, WebSearch refinement — #1075) // share the conversation id but do not carry the conversation. Reconciling @@ -1290,9 +1491,9 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt // the episode state is left exactly as found (the resets above stay // reserved for true episode boundaries: reconcile off / no folds at all). if (msgs.length < SIDE_REQUEST_MAX_MSGS) { - return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, reaped: 0, unmatched: 0 }; + return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, byShape: 0, reaped: 0, unmatched: 0 }; } - if (!session.metadata) return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, reaped: 0, unmatched: 0 }; + if (!session.metadata) return { kind: "noop", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, byShape: 0, reaped: 0, unmatched: 0 }; const anchors: Record = (session.metadata[METADATA_ANCHORS] as Record | undefined) ?? {}; @@ -1422,6 +1623,42 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt // Computed BEFORE the repair rewrite below so old ids classify against the // plan instead of post-rewrite bytes. The cumulative ever-present flag is // what lets the ledger tell verified loss from structural absence. + // #3641 census set, consumed by the z-accrual suppression below (and the + // verdict classifier after the loop). + const absentSet = new Set(plan.absentEverywhere); + // #3641 wire-continuity witness: the wire order frozen at the moment the + // drift episode began (frozen once per episode, cleared on recovery). + // Truncation is benign ONLY when the current wire is a SUFFIX-CONTINUATION + // of that witness: the surviving ids form the witness's tail AND the + // inbound head (a client that dropped its old history and kept talking). + // A wholesale replacement (foreign ids sharing nothing with the witness) + // fails continuity on every pass of the episode — the witness is frozen, + // so the foreign wire can never "become" the baseline mid-episode. + let witness = session.metadata[METADATA_EPISODE_ORDER] as string[] | undefined; + const episodeActive = ((session.metadata[METADATA_DRIFT_STREAK] as number | undefined) ?? 0) > 0; + if (witness === undefined && episodeActive && oldOrder.length > 0) { + // mid-episode freeze (episode predates this pass): adopt the previous + // wire as baseline — persisted so later passes compare against the + // SAME baseline. + witness = oldOrder.slice(-MAX_ORDER); + session.metadata[METADATA_EPISODE_ORDER] = witness; + } + // Before an episode starts there is nothing persisted: judge continuity + // transiently against the previous wire (oldOrder). The drift branch + // below persists the witness when the episode actually starts, so a + // foreign wire can never quietly become the baseline mid-episode. + const continuityBaseline = witness ?? oldOrder; + let wireContinuity = false; + if (continuityBaseline.length > 0) { + const witness = continuityBaseline; + const inboundSet = new Set(plan.newOrder); + const witnessSet = new Set(witness); + const commonWitness = witness.filter((id) => inboundSet.has(id)); + const commonInbound = plan.newOrder.filter((id) => witnessSet.has(id)); + wireContinuity = commonWitness.length > 0 + && witness.slice(witness.length - commonWitness.length).every((id) => inboundSet.has(id)) + && plan.newOrder.slice(0, commonInbound.length).every((id) => witnessSet.has(id)); + } const prevCov = (session.metadata[METADATA_FOLD_COVERAGE] as Record | undefined) ?? {}; const nextCov: Record = {}; let covCount = 0; @@ -1451,7 +1688,15 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt // reads as gone (safe side — real fold spans carry user turns). const hardIds = ids.filter((id) => anchors[id]?.c !== "reasoning"); const gone = hardIds.length > 0 && 2 * (p + r) < hardIds.length; - if (gone) { + // #3641: every absent hard id absent with ZERO word evidence anywhere + // while the session still carries other substrate = sanctioned + // truncation (client dropped old history), not zombie substrate. The + // #2695 reap streak would destroy the last remaining snapshot of + // history the host legitimately deleted from ITS view — bili keeps it + // (the fold IS the session's memory of the covered span). + const absentHard = hardIds.filter((id) => !byId.has(id) && !plan.claims.has(id)); + const structurallyAbsent = absentHard.length > 0 && absentHard.every((id) => absentSet.has(id)) && wireContinuity; + if (gone && !structurallyAbsent) { if (rec.e === 1) rec.z = (prevCov[bid]?.z ?? 0) + 1; } else { rec.z = 0; @@ -1585,7 +1830,33 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt const cls = anchors[id]?.c; return cls !== undefined && cls !== "reasoning"; }).length; - const totalDrift = plan.claims.size === 0 && plan.unmatched.length > 0 && !reasoningOnlyDrop && !markerChurnExplained; + // #3641 verdict classifier — the census evidence names the phenomenon + // before any streak accrues: + // shapeExplained: word evidence survived on the wire under churned ids + // (protocol switch #3640, image offload #3588) — the shape pass + // re-anchored some cells (byShape > 0) or the residual mass still + // carries near-duplicate word evidence (≥60% of class-carrying + // residual). Transform, not destruction: no streak, one info. + // truncationExplained: EVERY class-carrying residual is absent with + // zero word evidence anywhere, while some covered substrate is still + // present (resent or re-anchored) — the client dropped old history + // (sanctioned truncation #2695/#2297). Absence, not destruction: the + // snapshot is kept (z suppressed below), no streak, one info. + // Everything else (genuine rewrite with real content change, wholesale + // replacement by foreign history) keeps the #2193 streak — and its + // terminal verdict is now an epoch confirmation, not "destroyed". + const classCarryingResidual = plan.unmatched.filter((id) => { + const cls = anchors[id]?.c; + return cls !== undefined && cls !== "reasoning"; + }); + const shapeExplained = plan.byShape > 0 + || (classCarryingResidual.length > 0 && plan.nearDupResidual.length >= 0.6 * classCarryingResidual.length); + const truncationExplained = classCarryingResidual.length > 0 + && plan.nearDupResidual.length === 0 + && classCarryingResidual.every((id) => absentSet.has(id)) + && wireContinuity; + const totalDrift = plan.claims.size === 0 && plan.unmatched.length > 0 && !reasoningOnlyDrop && !markerChurnExplained + && !shapeExplained && !truncationExplained; const unfoldedDrift = mode === "repair" ? Math.max(0, (plan.churnTokens ?? 0) - (plan.claimedChurnTokens ?? 0)) : (plan.churnTokens ?? 0); @@ -1593,8 +1864,25 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt const prevStreak = (session.metadata[METADATA_DRIFT_STREAK] as number | undefined) ?? 0; if (!totalDrift) { if (prevStreak !== 0) resetFoldDriftState(session); + // #3641: name the phenomenon once per episode — transform churn and + // sanctioned truncation are EXPECTED host behaviors, not warnings. + if (shapeExplained && session.metadata[METADATA_SHAPE_NOTED] !== true) { + session.metadata[METADATA_SHAPE_NOTED] = true; + opts.log?.("info", `${tag}[fold-reconcile] ${classCarryingResidual.length} covered id(s) churned shape (protocol switch / offload) with word evidence intact — re-anchored ${plan.byShape}, ${plan.nearDupResidual.length} near-duplicate residual kept under watch (#3641); fold state retained`); + } + if (truncationExplained && session.metadata[METADATA_TRUNC_NOTED] !== true) { + session.metadata[METADATA_TRUNC_NOTED] = true; + opts.log?.("info", `${tag}[fold-reconcile] ${classCarryingResidual.length} covered id(s) no longer on the wire with no replacement evidence and remaining substrate present — host-side truncation/dropped history, fold snapshot retained (#2695/#3641)`); + } } else { const streak = prevStreak + 1; + // #3641: persist the wire baseline the moment the episode starts — + // every later pass of this episode judges truncation-vs-replacement + // against the wire AS IT WAS at episode start, never against a + // mid-episode (possibly already-replaced) wire. + if (prevStreak === 0 && oldOrder.length > 0 && session.metadata[METADATA_EPISODE_ORDER] === undefined) { + session.metadata[METADATA_EPISODE_ORDER] = oldOrder.slice(-MAX_ORDER); + } session.metadata[METADATA_DRIFT_STREAK] = streak; if (prevStreak === 0) session.metadata[METADATA_DRIFT_SINCE] = Date.now(); if (streak >= FOLD_DRIFT_ESCALATE_PASSES && nonReasoningUnmatched >= FOLD_DRIFT_ESCALATE_MIN_UNMATCHED @@ -1609,19 +1897,28 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt } const since = session.metadata[METADATA_DRIFT_SINCE] as number | undefined; const span = typeof since === "number" ? `, ${Math.max(1, Math.round((Date.now() - since) / 60000))} min so far` : ""; + // #3641: the terminal verdict is an EPOCH CONFIRMATION, not an + // abandonment. The snapshot is bili's memory of the covered span; + // a wholesale replacement with no word evidence anywhere means + // alignment is impossible — so folding STOPS (passthrough, the + // preflight gate reads epochConfirmPending) and the session asks + // the user once: re-anchor onto the new history (new epoch) or + // restore the old one. Either alignment event clears the latch + // automatically (resetFoldDriftState on the first clean pass). + session.metadata[METADATA_EPOCH_CONFIRM] = true; if (opts.log !== undefined) { - opts.log("error", `${tag}[fold-reconcile] compression substrate appears destroyed: ${plan.unmatched.length} covered id(s) missing with NO anchor match for ${streak} consecutive passes${span} — consistent with a host-native compaction landing outside bili's knowledge (dsh native compaction, #1729/#2193), a bulk client-side history rewrite${plan.idRewriteSuspects > 0 ? `, or host-rewritten tool-call ids (${plan.idRewriteSuspects} of them have an inbound twin with identical content under a different toolCallId — keep tool-call ids byte-stable per conversation, #2396)` : ""}; bili folds can no longer cover the resent history (#1921)${driftTok}`); + opts.log("error", `${tag}[fold-reconcile] fold epoch confirmation required: ${plan.unmatched.length} covered id(s) missing with NO anchor match and NO word evidence for ${streak} consecutive passes${span} — the resent history shares nothing with the fold substrate (wholesale replacement: host-native compaction outside bili's knowledge #1729/#2193, or a brand-new conversation on the same session id). Folding is paused for this session until the histories realign (old history resent or a fresh fold on the new one starts a new epoch); bili keeps the existing snapshot — nothing is deleted (#3641)${driftTok}`); } } } if (plan.unmatched.length === 0 && plan.claims.size === 0) { - return { kind: "resend", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, reaped: reapedCount, unmatched: 0 }; + return { kind: "resend", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, byShape: 0, reaped: reapedCount, unmatched: 0 }; } if (opts.log !== undefined) { if (plan.claims.size > 0 && mode === "repair") { opts.log(plan.unmatched.length > 0 ? "warn" : "info", - `${tag}[fold-reconcile] resent history drifted: ${plan.claims.size + plan.unmatched.length} covered id(s) missing — reanchored ${plan.claims.size} (${plan.byPos} positional${plan.resyncs > 0 ? `, ${plan.resyncs} order break(s) resynced` : ""}, ${plan.byTool} by toolCallId${plan.byTwin > 0 ? `, ${plan.byTwin} by toolCallId twin` : ""}${plan.bySim > 0 ? `, ${plan.bySim} by simhash corroboration` : ""}${plan.byOrder > 0 ? `, ${plan.byOrder} by order bracket` : ""}, ${plan.byNorm} by normalized identity${plan.byTurn > 0 ? `, ${plan.byTurn} by assistant-turn key` : ""}${plan.byImage > 0 ? `, ${plan.byImage} by image-capability variant` : ""}) onto churned bytes, ${plan.unmatched.length} unmatched re-enter the wire unfolded${plan.idRewriteSuspects > 0 ? `; ${plan.idRewriteSuspects} suspected host-rewritten tool-call ids (identical content under a different toolCallId — #2396)` : ""} (#1921)${driftTok}`); + `${tag}[fold-reconcile] resent history drifted: ${plan.claims.size + plan.unmatched.length} covered id(s) missing — reanchored ${plan.claims.size} (${plan.byPos} positional${plan.resyncs > 0 ? `, ${plan.resyncs} order break(s) resynced` : ""}, ${plan.byTool} by toolCallId${plan.byTwin > 0 ? `, ${plan.byTwin} by toolCallId twin` : ""}${plan.bySim > 0 ? `, ${plan.bySim} by simhash corroboration` : ""}${plan.byOrder > 0 ? `, ${plan.byOrder} by order bracket` : ""}, ${plan.byNorm} by normalized identity${plan.byTurn > 0 ? `, ${plan.byTurn} by assistant-turn key` : ""}${plan.byImage > 0 ? `, ${plan.byImage} by image-capability variant` : ""}${plan.byShape > 0 ? `, ${plan.byShape} by shape census` : ""}) onto churned bytes, ${plan.unmatched.length} unmatched re-enter the wire unfolded${plan.idRewriteSuspects > 0 ? `; ${plan.idRewriteSuspects} suspected host-rewritten tool-call ids (identical content under a different toolCallId — #2396)` : ""} (#1921)${driftTok}`); } else if (plan.claims.size > 0) { opts.log("warn", `${tag}[fold-reconcile] resent history drifted: ${plan.claims.size + plan.unmatched.length} covered id(s) missing, ${plan.claims.size} matchable by anchor (${plan.byPos} positional${plan.resyncs > 0 ? `, ${plan.resyncs} resynced` : ""}, ${plan.byTool} toolCallId${plan.byTwin > 0 ? `, ${plan.byTwin} twin` : ""}${plan.bySim > 0 ? `, ${plan.bySim} simhash` : ""}${plan.byOrder > 0 ? `, ${plan.byOrder} order-bracket` : ""}, ${plan.byNorm} normalized${plan.byTurn > 0 ? `, ${plan.byTurn} turn key` : ""}${plan.byImage > 0 ? `, ${plan.byImage} image` : ""}) but reconcile=warn made no repair (#1921)${driftTok}`); @@ -1662,6 +1959,7 @@ export function reconcileFoldCoverage(session: Session, msgs: CoreMessage[], opt byNorm: plan.byNorm, byTurn: plan.byTurn, byImage: plan.byImage, + byShape: plan.byShape, reaped: reapedCount, unmatched: plan.unmatched.length, }; diff --git a/src/launcher.ts b/src/launcher.ts index b7c5f3bd35..d6a49dab65 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -5344,7 +5344,7 @@ export async function runLaunch(params: RunLaunchParams, deps: LauncherDeps = {} // errors; the live protection is the proxy's wire-level refusal on // every lane (openai/anthropic/responses). console.error( - "bili: dsh native install detected — the launcher's compaction-basic auto:false overlay patch is skipped (a second bili-native entry would hard-fail dsh boot), and DSH mounts compaction in an agent preset whose nested rows no patch layer can address by id (#1772/#2360): dsh native compaction stays ARMED unless you apply the #2474 full-snapshot override of the preset-standard row in your profile's cordis.patch.yml (config replaces wholesale — see the plugin's boot warning for the exact recipe). Protection is bili's wire-level refusal of dsh compaction calls on all lanes (openai/anthropic/responses, #1729/#2193/#2360) — if you see 'compression substrate appears destroyed' or chained 'compress FAILED' errors, a dsh checkpoint has landed outside bili's knowledge.", + "bili: dsh native install detected — the launcher's compaction-basic auto:false overlay patch is skipped (a second bili-native entry would hard-fail dsh boot), and DSH mounts compaction in an agent preset whose nested rows no patch layer can address by id (#1772/#2360): dsh native compaction stays ARMED unless you apply the #2474 full-snapshot override of the preset-standard row in your profile's cordis.patch.yml (config replaces wholesale — see the plugin's boot warning for the exact recipe). Protection is bili's wire-level refusal of dsh compaction calls on all lanes (openai/anthropic/responses, #1729/#2193/#2360) — if you see 'fold epoch confirmation required' (older builds: 'compression substrate appears destroyed') or chained 'compress FAILED' errors, a dsh checkpoint has landed outside bili's knowledge.", ); } if (dshAcpPatch) clientArgs = dshArgsWithPatch(clientArgs, dshAcpPatch); diff --git a/src/preflight.ts b/src/preflight.ts index 79f486526c..c7f28f47c5 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -18,6 +18,7 @@ import { fetchWithTimeout, isTransientUpstreamError, replayMaxAttempts, replayBa import { dumpSummaryRejection } from "./error-dump.js"; import { proxyDispatcher } from "./upstream-proxy.js"; import { lastCompressSuffix, type Session } from "./session.js"; +import { epochConfirmPending } from "./fold-reconcile.js"; import { peekRegistryOutputLimit } from "./registry.js"; import { safePrefix } from "./text-safe.js"; import { applyEstimateCalibration, currentCalibrationFactor } from "./util.js"; @@ -1462,6 +1463,20 @@ export function deterministicDigest(entries: DigestEntry[], startRef: string, en } export async function preflightCompress(deps: PreflightDeps, messages: CoreMessage[]): Promise { + // #3641: epoch confirmation pending — the fold state and the resent + // history share nothing (wholesale replacement, no word evidence), so any + // fold computed here would summarize a substrate whose anchor set can + // never re-anchor. Pause folding for this session (verbatim passthrough) + // and say so once per pass at debug level: the error line lives at the + // escalation site (fold-reconcile). The FIRST realigned pass clears the + // latch (resetFoldDriftState), resuming folding automatically — no + // restart, no manual reset. Import direction: preflight → fold-reconcile + // is acyclic (fold-reconcile never imports preflight). + if (epochConfirmPending(deps.session)) { + deps.log("debug", `[preflight] fold epoch confirmation pending for ${deps.session.id} — compression paused (verbatim passthrough) until the resent history realigns (#3641)`); + const paused = estimateCoreMessages(messages) + (deps.wireOverhead ?? 0); + return { compressedRanges: 0, savedTokens: 0, payloadEstimate: paused, rangesRemaining: 0, fitsWindow: paused <= deps.config.modelContextLimit }; + } const limit = deps.config.modelContextLimit; // #1843 dual-channel accounting: fold decisions run on the TEXT channel — // text estimate vs `target − imageReserve`. Algebraically the same firing diff --git a/tests/fold-anchor-doctrine.test.ts b/tests/fold-anchor-doctrine.test.ts new file mode 100644 index 0000000000..22f6d6ebe9 --- /dev/null +++ b/tests/fold-anchor-doctrine.test.ts @@ -0,0 +1,220 @@ +// #3641 doctrine characterization — "snapshot is the session": +// order carries, hash is a hint, deletion is benign, wholesale shape +// churn re-anchors, and total collapse asks for an epoch confirm +// instead of silently resetting. +// +// These fixtures replay the two production incidents: +// #3640 — opencode switched /chat/completions → /v1/messages mid-session: +// 96.2% textual overlap, yet 298 covered ids churned wholesale +// (norms changed with the shape), no positional fence survived, +// every claim tier starved → 3-pass streak → "substrate appears +// destroyed" misdiagnosis + unresolved summary anchors parked at +// the very tail, riding AFTER the user's newest question. +// #3588 — dsh image-offload rewrote history images into placeholder text: +// the whole empty-text image family (ids clustered _1.._N) churned +// at once, no sh/wh anchors exist for empty-text cores → same +// destroyed misdiagnosis. +// #2695 — host truncation (a covered slice dropped client-side) reaped +// healthy blocks instead of retaining the snapshot. +import { test, describe } from "node:test"; +import assert from "node:assert/strict"; +import { simhashHex, hammingHex } from "../src/affinity-simhash.ts"; +import { + normalizedIdentity, + canonicalTextOf, + reconcileFoldCoverage, + wordBagDigest, + type FoldAnchor, +} from "../src/fold-reconcile.ts"; +import { epochConfirmPending, METADATA_DRIFT_ESCALATED } from "../src/fold-reconcile.ts"; +import type { CoreMessage } from "acp-kernel"; +import type { Session } from "../src/session.ts"; + +function msg(id: string, role: string, text: string, extra?: Partial): CoreMessage { + return { id, role, contentType: "text", text, ...extra } as CoreMessage; +} +function anchorOf(m: CoreMessage): FoldAnchor { + const a: FoldAnchor = { n: normalizedIdentity(m), r: m.role, c: m.contentType, b: m.text?.length ?? 0 }; + if (m.toolCallId) { + a.t = m.toolCallId; + } + if (m.text && m.text !== "" && m.contentType !== "reasoning") { + a.sh = simhashHex(canonicalTextOf(m.text)); + a.wh = wordBagDigest(canonicalTextOf(m.text)); + } + return a; +} +function fakeSession(ids: string[], msgs: CoreMessage[], agent?: string): Session { + const anchors: Record = {}; + for (const m of msgs) if (ids.includes(m.id as string)) anchors[m.id as string] = anchorOf(m); + const s = { + id: "s-doctrine", + // order array mirrors production semantics: the FULL last-pass wire + // order (all inbound ids), not just the covered subset — the #3641 + // truncation witness and flank pairing read it (fold-reconcile writes + // plan.newOrder there every pass). + metadata: { foldAnchors: anchors, foldAnchorOrder: msgs.map((m) => m.id as string) }, + state: { + blocks: [{ active: true, blockId: "blk-1", effectiveMessageIds: [...ids], directMessageIds: [...ids] }], + messageRefs: { byRaw: {}, byRef: {} }, + }, + } as unknown as Session; + if (agent !== undefined) s.metadata.pluginAgent = agent; + return s; +} +const repair = (log?: (l: string, m: string) => void) => ({ mode: "repair" as const, sessionId: "s-doctrine", log: log ?? (() => {}) }); + +/** #3640 shape: prose history where every id churns and the normalized + * identity changes with the shape (tool cells re-formed, whitespace + * re-wrapped) while the WORDS survive — 96%+ overlap, zero exact channels. */ +function shapeHistory(): CoreMessage[] { + const out: CoreMessage[] = [msg("h0", "user", "preamble describing the project goals and constraints in detail")]; + for (let i = 1; i <= 9; i++) { + out.push(msg(`h${i}`, i % 2 ? "user" : "assistant", `turn ${i} body with distinctive vocabulary marker ${i} and enough prose to shingle`)); + } + out.push(msg("h10", "user", "the newest question the user just asked")); + return out; +} +function reshape(old: CoreMessage[], gen: string): CoreMessage[] { + // every id re-minted; prose re-wrapped (line breaks move) → norm and + // sim-adjacent-but-not-equal, wordbag identical; tool-free so every + // channel that exists today is defeated: ids differ (byNorm ✗ — the + // canonical text changed with the re-wrap), no fence ever verifies + // (byPos ✗), nothing tool (byTool/byTwin ✗), no bracket (byOrder ✗), + // sim gaps never recorded (gapPairs empty) ✗. + return old.map((m, i) => msg(`${gen}_${i}`, m.role, (m.text ?? "").replace(/ /g, i % 2 ? "\n" : " "))); +} + +describe("#3640 wholesale shape churn re-anchors by wordbag evidence", () => { + test("protocol-switch resend: no destroyed escalation, coverage re-keyed, next pass clean", () => { + const old = shapeHistory(); + const session = fakeSession(old.slice(0, 9).map((m) => m.id as string), old); + const logs: string[] = []; + const reshaped = reshape(old, "g1"); + for (let pass = 0; pass < 4; pass++) reconcileFoldCoverage(session, reshaped, repair((l, m) => logs.push(`${l}: ${m}`))); + assert.notEqual(session.metadata[METADATA_DRIFT_ESCALATED], true, + "word-preserving wholesale churn must not diagnose substrate destruction"); + const block = (session.state as { blocks: Array<{ effectiveMessageIds: string[] }> }).blocks[0]!; + assert.ok(block.effectiveMessageIds.some((id) => id.startsWith("g1_")), "coverage re-keyed onto the reshaped ids"); + // and the very next stable pass is silent (no drift noise left) + const before = logs.length; + reconcileFoldCoverage(session, reshaped, repair((l, m) => logs.push(`${l}: ${m}`))); + assert.ok(!logs.slice(before).some((line) => line.includes("drifted") || line.includes("destroyed")), + "after re-keying a stable resend is quiet"); + }); +}); + +describe("#3588 image-offload family (empty-text anchors) re-anchors", () => { + test("placeholder rewrite of the image family: coverage re-keyed, no escalation", () => { + // the real incident: a family of user turns whose ONLY content was an + // image (text "") shares one base identity; offload rewrites each into + // a short placeholder string — every family id churns at once. 12 + // members cross the destroy MIN_UNMATCHED=10 bar, so an unclaimed + // family WOULD escalate today. + const family = 12; + const old: CoreMessage[] = [msg("p0", "user", "prose turn with real words for the flanks")]; + for (let i = 1; i <= family; i++) old.push(msg(`img_${i}`, "user", "")); // empty text → no sh/wh + for (let i = 1; i <= 3; i++) old.push(msg(`q${i}`, i % 2 ? "user" : "assistant", `trailing prose turn ${i} keeps talking about architecture`)); + const covered = old.slice(0, 1 + family).map((m) => m.id as string); + const session = fakeSession(covered, old); + const offloaded = [ + old[0]!, + ...Array.from({ length: family }, (_, k) => msg(`off_${k + 1}`, "user", `[image ${k + 1} offloaded to save context]`)), + ...old.slice(1 + family), + ]; + for (let pass = 0; pass < 4; pass++) reconcileFoldCoverage(session, offloaded, repair()); + assert.notEqual(session.metadata[METADATA_DRIFT_ESCALATED], true, "offload is a transform, not destruction"); + const block = (session.state as { blocks: Array<{ effectiveMessageIds: string[] }> }).blocks[0]!; + assert.ok(block.effectiveMessageIds.some((id) => id.startsWith("off_")), + "the image family coverage re-keyed onto placeholder ids (the block still covers the slots)"); + }); +}); + +describe("#2695 host truncation retains the snapshot (benign deletion)", () => { + test("covered slice dropped client-side: no reap, no escalation, one benign note", () => { + // 18-message history, covered prefix of 8, and the client keeps a + // 10-turn tail — the tail must clear SIDE_REQUEST_MAX_MSGS (the + // reconcile guard skips tiny resends as side-request-shaped), and a + // 10-turn tail is exactly a windowed host. + const old: CoreMessage[] = []; + for (let i = 0; i < 18; i++) old.push(msg(`t${i}`, i % 2 ? "user" : "assistant", `history turn ${i} with words to shingle properly`)); + // proxy mode: no pluginAgent → the #2695 reap arm is armed + const session = fakeSession(old.slice(0, 8).map((m) => m.id as string), old); + // host drops the covered slice entirely, keeps the tail verbatim + const truncated = old.slice(8); + const logs: string[] = []; + for (let pass = 0; pass < 5; pass++) reconcileFoldCoverage(session, truncated, repair((l, m) => logs.push(`${l}: ${m}`))); + assert.notEqual(session.metadata[METADATA_DRIFT_ESCALATED], true, "client-side truncation is sanctioned host behavior"); + const blocks = (session.state as { blocks: Array<{ blockId?: string; active: boolean }> }).blocks; + assert.ok(blocks.some((b) => b.blockId === "blk-1"), "the snapshot block is retained, not reaped"); + const conflicts = (session.metadata["conflicts"] as Array<{ kind?: string }> | undefined) ?? []; + assert.ok(!conflicts.some((c) => c.kind === "orphan-reap"), "no reap ledger entry"); + assert.ok(logs.some((line) => line.startsWith("info") && /truncat|dropped/i.test(line)), + "one benign info line names the truncation"); + }); +}); + +describe("#3641 total collapse asks for epoch confirmation", () => { + test("nothing matches anywhere: escalated + epoch-confirm latch + folds suspended until recovery", () => { + const old: CoreMessage[] = []; + for (let i = 0; i < 10; i++) old.push(msg(`c${i}`, i % 2 ? "user" : "assistant", `conversation turn ${i} with distinct vocabulary`)); + const session = fakeSession(old.map((m) => m.id as string), old); + // a completely different conversation under the same session id + const foreign: CoreMessage[] = []; + for (let i = 0; i < 14; i++) foreign.push(msg(`x${i}`, i % 2 ? "user" : "assistant", `unrelated fresh content ${i} about a brand new topic entirely`)); + const logs: string[] = []; + for (let pass = 0; pass < 4; pass++) reconcileFoldCoverage(session, foreign, repair((l, m) => logs.push(`${l}: ${m}`))); + assert.equal(session.metadata[METADATA_DRIFT_ESCALATED], true, "detection intact: the collapse is still flagged"); + assert.equal(epochConfirmPending(session), true, "epoch confirmation is latched, not a silent reset"); + assert.ok(logs.some((line) => line.startsWith("error") && /epoch/i.test(line)), "one error asks the user to confirm the new epoch"); + // recovery: the host resends the original history → the latch clears + reconcileFoldCoverage(session, old, repair((l, m) => logs.push(`${l}: ${m}`))); + assert.notEqual(epochConfirmPending(session), true, "identity restored → the epoch latch auto-clears"); + }); +}); + +describe("honesty gates survive the doctrine (no wholesale silencing)", () => { + + test("epoch-confirm suspends preflight compression (verbatim passthrough until realigned)", async () => { + // Once the fold asks for epoch confirmation, the compression pipeline + // must not keep folding a substrate it can no longer prove — requests + // pass through verbatim until the histories realign (or the user + // confirms a new epoch). No fold happens on the gated session. + const old: CoreMessage[] = []; + for (let i = 0; i < 10; i++) old.push(msg(`t${i}`, i % 2 ? "user" : "assistant", `history turn ${i} with words to shingle properly`)); + const session = fakeSession(old.map((m) => m.id as string), old); + const foreign: CoreMessage[] = []; + for (let i = 0; i < 14; i++) foreign.push(msg(`x${i}`, i % 2 ? "user" : "assistant", `foreign conversation content ${i} ${"z".repeat(120)}`)); + const logs: { level: string; msg: string }[] = []; + const rl = (l: string, m: string) => logs.push({ level: l, msg: m }); + for (let pass = 0; pass < 3; pass++) reconcileFoldCoverage(session, foreign, repair(rl)); + assert.equal((session.metadata as Record)["foldEpochConfirmPending"], true); + const { preflightCompress } = await import("../src/preflight.ts"); + const { defaultPrompts } = await import("acp-kernel"); + const result = await preflightCompress({ + core: {} as never, // untouched on the gated path + config: { modelContextLimit: 1000 } as never, + session: session as never, + prompts: defaultPrompts, + protocol: "responses", + url: "http://127.0.0.1:1/summary-must-not-be-called", + headers: {}, + model: "m", + log: rl, + }, foreign); + assert.equal(result.compressedRanges, 0, "epoch-confirm gate: verbatim passthrough"); + assert.equal(result.savedTokens, 0); + assert.equal(session.state.blocks.length, 1, "the existing snapshot is kept untouched"); + }); + + test("a genuine edit below the mass bar still reports drift honestly", () => { + const old: CoreMessage[] = []; + for (let i = 0; i < 10; i++) old.push(msg(`e${i}`, i % 2 ? "user" : "assistant", `steady turn ${i} prose for the stable spine of this session`)); + const session = fakeSession(old.slice(0, 6).map((m) => m.id as string), old); + const edited = old.map((m, i) => (i === 3 ? msg("edited_3", m.role, "totally rewritten content with different meaning and words") : m)); + const logs: string[] = []; + for (let pass = 0; pass < 3; pass++) reconcileFoldCoverage(session, edited, repair((l, m) => logs.push(`${l}: ${m}`))); + assert.ok(logs.some((line) => line.includes("e3") || line.includes("drifted")), "the honestly-edited cell still reports"); + assert.notEqual(session.metadata[METADATA_DRIFT_ESCALATED], true, "single-cell churn never mass-escalates"); + }); +}); diff --git a/tests/fold-reconcile-positional.test.ts b/tests/fold-reconcile-positional.test.ts index 27b3b17d07..2a6c234ebb 100644 --- a/tests/fold-reconcile-positional.test.ts +++ b/tests/fold-reconcile-positional.test.ts @@ -389,12 +389,14 @@ describe("reconcileFoldCoverage positional wiring (#2480)", () => { const coveredNow = (session.state.blocks as { effectiveMessageIds: string[] }[])[0].effectiveMessageIds; const tailIds = compacted.slice(1).map((m) => m.id as string); for (const id of tailIds) assert.ok(coveredNow.includes(id), `tail id ${id} stays covered`); - // self-heal trajectory: the next full pass on the compacted history is - // steady for the tail; the compacted-away head ids stay missing and the - // #2193 drift streak starts counting toward its one error line + // #3641 doctrine: the compacted-away head is a sanctioned host-side + // deletion (summary + surviving tail = wire suffix-continuation of the + // previous order). The fold keeps the snapshot, no drift streak ever + // starts, and one benign truncation note is latched for the panel. const r2 = reconcileFoldCoverage(session, [...compacted, msg("post_1", "user", "resume")], opts("repair")); assert.equal(r2.byPos, 0); - assert.equal(session.metadata.foldDriftStreak, 1, "total-loss streak begins (existing machinery owns the landing)"); + assert.equal(session.metadata.foldDriftStreak, undefined, "#3641: sanctioned compaction never starts the collapse streak"); + assert.equal(session.metadata.foldTruncationNoted, true, "one benign truncation note is latched"); }); test("side-request-shaped passes take no positional evidence", () => { diff --git a/tests/fold-reconcile.test.ts b/tests/fold-reconcile.test.ts index e535336f30..8bc8ecbdd2 100644 --- a/tests/fold-reconcile.test.ts +++ b/tests/fold-reconcile.test.ts @@ -331,7 +331,7 @@ describe("reconcileFoldCoverage drift escalation (#2193)", () => { } const errs = errorLines(logs); assert.equal(errs.length, 1, "exactly ONE error for the whole episode"); - assert.match(errs[0].msg, /compression substrate appears destroyed/); + assert.match(errs[0].msg, /fold epoch confirmation required/); assert.match(errs[0].msg, /host-native compaction/); assert.match(errs[0].msg, /#2193/); // #2297: the per-pass warn now STOPS once the episode escalated — @@ -451,7 +451,7 @@ describe("reconcileFoldCoverage drift escalation (#2193)", () => { reconcileFoldCoverage(session, freshPass(`p${pass}`), opts); } assert.equal(errorLines(logs).length, 1, "real destruction still escalates on its own streak"); - assert.match(errorLines(logs)[0].msg, /compression substrate appears destroyed/); + assert.match(errorLines(logs)[0].msg, /fold epoch confirmation required/); }); // #2709: at the destruction point the refusal-ledger witness turns the @@ -822,7 +822,7 @@ describe("reconcileFoldCoverage anchor cap boundary (#2334)", () => { const snapshot = JSON.stringify(session.metadata); resetNormalizedIdentityWork(); const second = reconcileFoldCoverage(session, msgs, opts); - assert.deepEqual(second, { kind: "resend", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, reaped: 0, unmatched: 0 }); + assert.deepEqual(second, { kind: "resend", missing: 0, claims: 0, byPos: 0, byTool: 0, byTwin: 0, bySim: 0, byOrder: 0, byNorm: 0, byTurn: 0, byImage: 0, byShape: 0, reaped: 0, unmatched: 0 }); assert.equal(normalizedIdentityWorkCount(), 0, "the overflow tail must not be normalized+hashed and dropped AGAIN every pass (#2334)"); assert.equal(JSON.stringify(session.metadata), snapshot, "steady-state resend leaves metadata byte-stable"); diff --git a/tests/issue2915-order-centric-reconcile.test.ts b/tests/issue2915-order-centric-reconcile.test.ts index 32cebfc5a6..ce74154a5c 100644 --- a/tests/issue2915-order-centric-reconcile.test.ts +++ b/tests/issue2915-order-centric-reconcile.test.ts @@ -218,7 +218,7 @@ describe("escalation semantics (#2915) — the #2821 death spiral stays closed", assert.equal(r.unmatched, 1, `${gen}: only the changed preamble id is honest loss`); assert.equal(r.reaped, 0, `${gen}: no zombie reap`); } - assert.ok(!logs.some((l) => l.msg.includes("compression substrate appears destroyed")), + assert.ok(!logs.some((l) => l.msg.includes("fold epoch confirmation required")), "the #2821 verdict must never fire for re-mint churn"); assert.ok(session.metadata.foldDriftEscalated === undefined, "escalation latch never sets"); assert.ok(session.metadata.foldDriftStreak === undefined || session.metadata.foldDriftStreak === 0, @@ -266,7 +266,7 @@ describe("escalation semantics (#2915) — the #2821 death spiral stays closed", assert.equal(r.byPos, 11, `${gen}: positional copy regenerated — order carries the re-mint`); assert.equal(r.unmatched, 1, `${gen}: only the changed preamble`); } - assert.ok(!logs.some((l) => l.msg.includes("compression substrate appears destroyed"))); + assert.ok(!logs.some((l) => l.msg.includes("fold epoch confirmation required"))); }); }); @@ -357,7 +357,7 @@ describe("escalation latch recovery (review gap ②) — upgrade-in-place heals" } assert.equal(session.metadata.foldDriftEscalated, true, "latched after 3 consecutive total-drift passes"); assert.equal(session.metadata.foldDriftStreak, 3); - assert.equal(logs.filter((l) => l.msg.includes("compression substrate appears destroyed")).length, 1, + assert.equal(logs.filter((l) => l.msg.includes("fold epoch confirmation required")).length, 1, "#2297: exactly ONE error line per episode"); // healthy resend of the ORIGINAL bytes: missing=0 → noop, and the From a9fe72f37bd996bb5b93d1a12aa749974866193e Mon Sep 17 00:00:00 2001 From: awork Date: Sun, 18 Oct 2026 11:10:25 +0800 Subject: [PATCH 02/14] kernel: bump acp-kernel 0.0.130 -> 0.0.131 for the prune.ts placement change (#3644) CI-fix additive commit: the kernel-guard job enforces that any change under kernel/src/** carries a kernel/package.json version bump (Kernel Boundary rule 2, AGENTS.md). The placement fix in kernel/src/prune.ts landed without it, so kernel-guard failed and the test matrix was skipped. --- kernel/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/package.json b/kernel/package.json index a318eb703b..56b69e7cff 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.130", + "version": "0.0.131", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", From 0825bdea8e82aabc45b481ec5847b3e52c442746 Mon Sep 17 00:00:00 2001 From: awork Date: Sun, 18 Oct 2026 11:12:30 +0800 Subject: [PATCH 03/14] ci: sync package-lock.json to acp-kernel 0.0.131 (lock-sync after the #3644 kernel version bump) The a9fe72f37 kernel bump moved kernel/package.json to 0.0.131 but left the file:./kernel resolution in package-lock.json at 0.0.130, so the lock-sync job (npm install --package-lock-only && git diff --exit-code) drifted red and gated the test matrix. Regenerate the lock record so it matches. --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 8085a29d61..411c5de310 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.130", + "version": "0.0.131", "dev": true, "license": "MIT", "devDependencies": { From 1a3949a14ef66561f62b4d99befd6665aa14f5ed Mon Sep 17 00:00:00 2001 From: awork Date: Sun, 18 Oct 2026 11:21:35 +0800 Subject: [PATCH 04/14] docs: correct epochConfirmPending consumer note to match implementation (#3644) The docstring named 'status panel' as a consumer, but only the preflight fold gate reads epochConfirmPending in this PR. Narrow the note so it does not promise an unimplemented panel integration. --- src/fold-reconcile.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fold-reconcile.ts b/src/fold-reconcile.ts index 424d59dc40..bb6d2ed930 100644 --- a/src/fold-reconcile.ts +++ b/src/fold-reconcile.ts @@ -193,7 +193,7 @@ const METADATA_TRUNC_NOTED = "foldTruncationNoted"; * baseline against which truncation-vs-replacement is decided. */ const METADATA_EPISODE_ORDER = "foldEpisodeOrder"; /** #3641 helper: true while the session waits for epoch confirmation. - * Consumers (preflight fold gate, status panel) read this, never the raw + * Read through this helper (the preflight fold gate does), never the raw * metadata key. */ export function epochConfirmPending(session: Session): boolean { return (session.metadata?.[METADATA_EPOCH_CONFIRM] as boolean | undefined) === true; From be8709aadadbaf2e1c209bfb403200d67cd824d0 Mon Sep 17 00:00:00 2001 From: awork Date: Sun, 18 Oct 2026 11:31:36 +0800 Subject: [PATCH 05/14] test: regenerate storage-render goldens for the new fold-drift metadata keys (#3644) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #3644 persists four new session-metadata latches (foldEpisodeOrder, foldEpochConfirmPending, foldShapeChurnNoted, foldTruncationNoted), so the pinned persisted-state digest drifts in exactly one section: metadata. Wire bytes are unchanged — all 15 per-turn outbound-body entries and the state/blockContents/stats sections are byte-identical to the previous goldens (verified by structural diff of each record). Regenerated with scripts/update-storage-render-goldens.ts under the same BILI_IMAGE_RESCALE=0 pin the test applies (#3400); justification for the golden change per the golden rule, mirrored in the PR body. --- tests/golden/storage-render/anthropic.json | 4 ++-- tests/golden/storage-render/chat.json | 4 ++-- tests/golden/storage-render/google.json | 4 ++-- tests/golden/storage-render/responses.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 258b7a7879..4b40bac467 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "00a49d95b7ffd5e5c76e9f17d1d5405293aa1a7d08e4ed7fa778b77b5e8cb788", + "stateDigest": "836b7f2e075421114f0b3c31c9d863af3861ab8200d632948b5a21b2992d9c1c", "sectionDigests": { "state": "fae14b25a41f5844", "blockContents": "dc7d13d250c94bb4", "stats": "cff3c2546898aeb3", - "metadata": "5413c879755acf18" + "metadata": "c6e132de1f0769b1" }, "stateKeyDigests": { "blocks": "1cbe172309e4490d", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 3af63534ec..e76359cce1 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "7453eabb99ead81805a1bf2d6630232df2e02ab10772b7b03dd88495cfc90d17", + "stateDigest": "a55afe00bb22c39b31935c75a3fcb969a98325d2227e75ae703db083427d02e5", "sectionDigests": { "state": "d5bc5306f840723d", "blockContents": "94a58f96d7c5cfc3", "stats": "59842ce5415d3a4d", - "metadata": "d05f9d95a0dd2fce" + "metadata": "fbc8e22bb6bde73c" }, "stateKeyDigests": { "blocks": "9aa73e5fb9f93d6c", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index d76d38a238..f962a1fbdc 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "48c0fbd96f53485e9b8109ae7e24cc1cf7d0b0a265591f7f4e0b04689dbbdb86", + "stateDigest": "53f625d72df68bf581e2858ff73c95e619d86b92387a40b1707020ac4e2c6cdf", "sectionDigests": { "state": "2a2af1c92fcb4f88", "blockContents": "473264d9df8ba338", "stats": "660f1797bf2044c4", - "metadata": "4d073f1e812dcb89" + "metadata": "f6c8d7d13b4ae5d4" }, "stateKeyDigests": { "blocks": "8e14f0879bf7ebd7", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index a7cd175ed7..c87c591967 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -80,12 +80,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "05efdc2df5d4ef3a4d9f8f3c98de81668dc33eb81d2e3300b18dde92d9bdb54d", + "stateDigest": "c61887b512e6322382ab4dd4b7c53586bf0fa2cd699775f19b19fe6966e23490", "sectionDigests": { "state": "6546d2e5f2826284", "blockContents": "8460729b2654a816", "stats": "e3a3aeb598c183ff", - "metadata": "44e07f67bbeb366b" + "metadata": "bfe50c77f8a85bcb" }, "stateKeyDigests": { "blocks": "7fff1dfb54a617cf", From 5b2f6729b3d31c515b16e2070c8cc9d39ae89482 Mon Sep 17 00:00:00 2001 From: awork Date: Sun, 18 Oct 2026 11:31:57 +0800 Subject: [PATCH 06/14] fix: pin BILI_IMAGE_RESCALE=0 in the storage-render golden regen script (#3400 parity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test pins the legacy wire shape before driving but the regen script did not — regenerating without the pin writes goldens whose per-turn wire digests differ from what the test drives under CI, red-ing the suite on the next run. Align the two so the documented regen flow can never silently diverge from the test's conditions. --- scripts/update-storage-render-goldens.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/update-storage-render-goldens.ts b/scripts/update-storage-render-goldens.ts index 4a6000c026..46c1f21f5c 100644 --- a/scripts/update-storage-render-goldens.ts +++ b/scripts/update-storage-render-goldens.ts @@ -6,6 +6,12 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { driveWire, WIRES } from "../tests/golden-storage-drive.ts"; +// Parity with tests/golden-storage-render.test.ts (#3400): the test pins the +// legacy wire shape before driving, so goldens must be regenerated under the +// same condition — without it every per-turn wire digest drifts by the image- +// rescale delta and the suite reds on the next CI run. +process.env.BILI_IMAGE_RESCALE = "0"; + const GOLDEN_DIR = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "tests", "golden", "storage-render"); fs.mkdirSync(GOLDEN_DIR, { recursive: true }); for (const wire of WIRES) { From 682173ea76f8060b02042694854e81759c7a5506 Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sun, 18 Oct 2026 11:40:08 +0800 Subject: [PATCH 07/14] docs: snapshot-as-session doctrine section in MESSAGE-IDENTITY (#3641 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - document the doctrine ladder (shape tier / census verdicts / episode witness / epoch-confirm gate / unresolved-anchor placement), the honesty boundary (strict wordbag equality — bulk suffix re-stamps are NOT claimed by design), and zero-migration storage independence - EN + zh-CN mirrors, cross-linked from the header and Related lists - tests: harden the #3640 fixture so every reshaped cell is shape-tier- only (deleting the tier now escalates — the no-escalation assertion has teeth), pin byShape > 0 on the first pass, and add adversarial honesty-gate coverage (role flip / length padding / word reversal are wordbag-equal but must not be claimed) --- MESSAGE-IDENTITY.md | 65 +++++++++++++++++++++++++++++- MESSAGE-IDENTITY.zh-CN.md | 20 ++++++++- tests/fold-anchor-doctrine.test.ts | 53 ++++++++++++++++++++---- 3 files changed, 126 insertions(+), 12 deletions(-) diff --git a/MESSAGE-IDENTITY.md b/MESSAGE-IDENTITY.md index 6807dcdf2a..4a4f681f2a 100644 --- a/MESSAGE-IDENTITY.md +++ b/MESSAGE-IDENTITY.md @@ -11,7 +11,9 @@ question at session granularity). Implementation anchors: **kernel ledger** below stays content-hash, while the **fold-coverage re-anchoring layer** (`src/fold-reconcile.ts`) became positional-with-a- self-stored-canonical-copy — see “The fold-coverage layer goes positional” -below; nothing in this document's ledger contract changed. +below; nothing in this document's ledger contract changed. #3641 later added +the verdict layer on top — what happens when **no** channel verifies — see +“The snapshot-as-session doctrine” below; the ledger contract still stands. ## The question is the join key, not the id @@ -178,6 +180,64 @@ server-side compress may re-fold a failed replay — the healed round-2 must never stand in for the retention proof), plus its own reconcile-off control proving this churn class re-bills without the layer. +## The snapshot-as-session doctrine: verdict classes over the id (#3641) + +#2480 made fold coverage positional-with-verification. #3641 settles what +happens when **no** channel verifies. The inversion: the **(session id + +fold snapshot) is the conversation body** — the wire is only an alignment +observation of it. Ids, hashes and shapes are hints; “which messages do my +blocks cover, in what order” is the fact. Three production classes forced +this: a protocol re-serialization re-minting every id with the words intact +(#3640), image-offload rewriting a textless family into placeholders +(#3588), and stateless clients truncating the covered slice (#2695) — all +three used to be diagnosed as "substrate destroyed". + +The mechanism (all in `src/fold-reconcile.ts` plus one kernel placement +rule; proof tests in `tests/fold-anchor-doctrine.test.ts`): + +- **Shape tier** (order-preserving): when a *mass* of covered ids goes + missing at once (≥ `SHAPE_MASS_MIN`), residual prose anchors pair with + inbound candidates in conversation order under the edit-honesty gate — + word-bag equality + simhash ≤ 8 + role + length guard (the same gate the + sim tier applies to gaps, applied to claims). Textless anchors (image + families: no sh/wh by construction) pair k-th-to-k-th when bracketed by + verified flanks on both sides and per-bucket counts match exactly — the + byImage discipline generalized past bili's own placeholder variant. +- **Residual census and verdict classes**: what the ladder cannot claim is + *classified*, not assumed destroyed. `shapeExplained` (transform: claims + or near-dup evidence carry the mass) — one info latch, streak reset. + `truncationExplained` (benign deletion: residuals absent everywhere, zero + near-dups, and a **wire-continuity witness** — the frozen old-order suffix + still heads the inbound) — blocks snooze instead of reaping (#2695). + Everything else is drift and escalates as before. +- **Episode witness**: at the first drift pass the old order's tail is + frozen (`foldEpisodeOrder`); a foreign conversation under the same session + id has no continuity with it, so wholesale replacement cannot masquerade + as truncation. +- **Epoch-confirm gate**: total collapse latches `foldEpochConfirmPending`, + emits one error, and preflight passes requests through verbatim — folding + *pauses*; nothing is deleted, nothing is silently reset. Alignment + recovery (the host resends the true history) auto-clears the latch. +- **Unresolved-anchor placement** (kernel `prune.ts`): a block whose anchors + all failed to resolve now inserts at the last user message instead of the + tail — a summary block can no longer ride *after* the user's newest + question (#3640's six tail-riding blocks). + +The honesty boundary is deliberate: the shape tier requires **strict** +word-bag equality, so a systematic content mutation — a host re-stamping +history with a trailing `` suffix, say — is *not* claimed +as the same message (fuzzy content joins misattribute on decompress; see +the rejected alternatives below). Such events classify as truncation or +collapse and land on the snooze/pause paths — never a silent re-key. +Whether a bulk identical-suffix re-stamp (N order-stable pairs sharing the +exact same delta) deserves its own *verified* class is an open follow-up, +to be sized against production logs (#3640/#3588). + +Compatibility: the doctrine is a matching/verdict-layer change — zero +migration, applies to old and new sessions alike, storage-independent +(legacy JSON, shadow and full modes behave identically; bookkeeping rides +session metadata). The kernel ledger's content-hash contract is untouched. + ## Known cost of the hash axiom — and how it was fixed Same bytes ⇒ same identity has a cost face: #1476 — a user re-sends a short @@ -254,4 +314,5 @@ Related: #1496 (this document's prompt), `SESSION-IDENTITY.md` (session granularity), #1476 + kernel #459/#463 (echo discrimination), #242/#1475 (Responses id constraints), #206/#673 (tag echo), #1479/#1482 (strict-echo repair), #1039 (tool-call byte-exactness invariant), #2396 (host-rewritten tool-call ids across a provider switch), #2480 (positional fold coverage; -`tests/identity-proof.test.ts` is its proof matrix). +`tests/identity-proof.test.ts` is its proof matrix), #2695/#3588/#3640/#3641 +(snapshot-as-session doctrine; `tests/fold-anchor-doctrine.test.ts`). diff --git a/MESSAGE-IDENTITY.zh-CN.md b/MESSAGE-IDENTITY.zh-CN.md index 2d0ca0bfaf..4d30b24d23 100644 --- a/MESSAGE-IDENTITY.zh-CN.md +++ b/MESSAGE-IDENTITY.zh-CN.md @@ -1,6 +1,6 @@ # 消息身份:内容哈希作为跨轮 join key -设计记录:billion-context 如何跨轮识别单条消息,以及为什么身份由消息**内容**派生、而不是在入口打一个由宿主持久化的 id。成文于 #1496 复核之后;与 `SESSION-IDENTITY.zh-CN.md`(会话粒度的同题文档)成对。实现锚点:`acp-kernel/src/wire/message-id.ts`(`deriveMessageId`)、`acp-kernel/src/refs.ts`(`assignRefs`)、`acp-kernel/src/prune.ts`(`isCovered`/`baseIdOf`)。#2480 随后把问题一分为二:下文的**内核账本**仍是 content-hash;而**折叠覆盖重锚层**(`src/fold-reconcile.ts`)改为「位置 + 自存规范副本」——见下文「折叠覆盖层改为位置身份」一节;本文档的账本契约未变。 +设计记录:billion-context 如何跨轮识别单条消息,以及为什么身份由消息**内容**派生、而不是在入口打一个由宿主持久化的 id。成文于 #1496 复核之后;与 `SESSION-IDENTITY.zh-CN.md`(会话粒度的同题文档)成对。实现锚点:`acp-kernel/src/wire/message-id.ts`(`deriveMessageId`)、`acp-kernel/src/refs.ts`(`assignRefs`)、`acp-kernel/src/prune.ts`(`isCovered`/`baseIdOf`)。#2480 随后把问题一分为二:下文的**内核账本**仍是 content-hash;而**折叠覆盖重锚层**(`src/fold-reconcile.ts`)改为「位置 + 自存规范副本」——见下文「折叠覆盖层改为位置身份」一节;本文档的账本契约未变。#3641 随后在其上补了判决层——**没有**任何通道验证成功时怎么办——见下文「快照即会话教义」一节;账本契约依然不变。 ## 问题是 join key,不是 id @@ -72,6 +72,22 @@ bili 的应对遵循上文的失败模式公理:**不猜** old→new 配对(猜 这与下文否决「位置当 join key」并不矛盾:裸位置被否决,是因为在没有旧字节记忆的情况下,位置 claim 是**猜测**,以认错告终。副本把猜测变成对齐偏移处的**已验证等值**——同一公理上移一层:指纹失配 → 诚实的认不出(unmatched,走 #1001/#1195 lane),永不认错。验证矩阵在 `tests/identity-proof.test.ts`(机制级、judge 计费,仿 `tests/cache-proof.test.ts`):4 wire × 3 阶段(折叠稳定 → id 方案+序列化 churn 后覆盖保持 → 中点编辑诚实重入)、reconcile-off 对照(无该层时 churn 重新计费)、跨 wire 客户端互换(anthropic → chat)端到端保持覆盖,以及切模型阶段(#2636):chat lane 携带 reasoning 助手轮与图片用户轮,随后把历史重放到另一个形状的模型(reasoning 内联 pair→merged、图片换占位行)——覆盖必须扛住形状 churn 且已覆盖的 thinking 不得重入,以 churn 后**第一个** body 为准(新的服务端 compress 可能把失败重放重新折回去——治愈后的 round-2 不得冒充保持证明),另有自己的 reconcile-off 对照证明无该层时此 churn 类会重新计费。 +## 快照即会话教义:身份之上的判决分类(#3641) + +#2480 让折叠覆盖变成「位置+验证」;#3641 则定论:**没有**通道验证成功时怎么办。反转在于:**(会话 id + 折叠快照)才是会话本体**——wire 只是对它的对齐观察。id、hash、形状都是提示;「我的块覆盖哪些消息、按什么顺序」才是事实。三类生产事故倒逼出这一层:协议重序列化把全部 id 重铸但词还在(#3640)、图片 offload 把无文本族改写成占位串(#3588)、无状态客户端截断已覆盖段(#2695)——三者过去都会被诊断为 "substrate destroyed"。 + +机制(全部在 `src/fold-reconcile.ts` 加一条内核摆位规则;证明测试在 `tests/fold-anchor-doctrine.test.ts`): + +- **形状层**(保序):当一次性缺失**成片** covered id(≥ `SHAPE_MASS_MIN`)时,残留散文锚按会话顺序与入站候选配对,受编辑诚实门约束——词袋相等 + simhash ≤ 8 + role + 长度卫兵(sim 层用于 gap 的同一道门,用在 claim 上)。无文本锚(图片族:构造上无 sh/wh)在两侧被已验证的侧翼包夹、且逐桶计数恰相等时按第 k 对第 k 配对——byImage 纪律推广到 bili 自家占位变体之外。 +- **残差普查与判决分类**:阶梯认不下的部分被*分类*,而非假定已毁。`shapeExplained`(变换:claim 或近重证据承担了主体)——一次性 info latch,清零 streak。`truncationExplained`(良性删除:残差全部到处缺席、零近重、且**wire 连续性见证**——冻结的旧序尾部仍居于入站头部)——块休眠而非 reap(#2695)。其余照旧是 drift,照旧升级。 +- **episode 见证人**:首个 drift pass 冻结旧序尾部(`foldEpisodeOrder`);同会话 id 下的外来会话与它无连续性,整体替换无法冒充截断。 +- **epoch 确认门**:全塌 latch `foldEpochConfirmPending`,发一条 error,preflight 原样透传——折叠*暂停*;不删任何东西,不静默重置。对齐恢复(宿主重发真实历史)自动清 latch。 +- **未解析锚摆位**(内核 `prune.ts`):锚全部解析失败的块,插入点从尾部改为最后一个 user 消息——摘要块再也不会骑在用户最新提问*之后*(#3640 的六个骑尾块)。 + +诚实边界是有意的:形状层要求**严格**词袋相等,所以系统性内容突变——比如宿主给历史批量加/删尾部 `` 后缀——**不会**被认成同一条消息(模糊内容 join 在 decompress 时认错;见下文被否决备选)。这类事件被分类为截断或全塌,落在休眠/暂停路径上——永不静默重键。批量同后缀重戳(N 对保序对共享完全相同的 delta)是否值得单独立一个*可验证*类,是待生产日志定界的开放跟进项(#3640/#3588)。 + +兼容性:教义是匹配/判决层变更——零迁移,新老会话一体适用,与存储无关(legacy JSON、shadow、full 三模式行为完全一致;记账走 session metadata)。内核账本的 content-hash 契约分毫米未动。 + ## hash 公理的已知代价——以及修复方向 同字节 ⇒ 同身份有代价面:#1476——用户原样重发与已折叠内容相同的短文本;裸 `h_` id 重新派生;`isCovered()` 吞掉新消息。注意**修复方向**:kernel #459 加实例重编号(`_1/_2…`),#463 加 `lastPassIds` 快照区分折叠后**回声**(id ∈ covered ∧ ∈ 上一轮 → 保持 id,prune 吞掉)与真**新实例**(id ∈ covered ∧ ∉ 上一轮 → 重编号到空闲 `_k`)。content hash 始终是基座,只在其上叠加实例判别——连 hash 公理自己的 bug 都靠**保留 hash** 修,而不是退回位置 id。 @@ -105,4 +121,4 @@ DeepSeek 类网关要求客户端传回网关自己发出的 reasoning("the reas 若未来宿主或 lane 提供真正稳定的客户端消息 id,只允许作为**附加 join hint** 采纳——永不替代 content-hash 基座。在那之前:字节是锚,hash 是 join,mNNNNN 是账本,标签是视图。 -相关:#1496(本文档的起因)、`SESSION-IDENTITY.zh-CN.md`(会话粒度)、#1476 + kernel #459/#463(回声判别)、#242/#1475(Responses id 约束)、#206/#673(标签回声)、#1479/#1482(strict-echo 修复)、#1039(工具调用字节不变量)、#2396(跨 provider 切换时宿主改写的工具调用 id)、#2480(位置折叠覆盖;`tests/identity-proof.test.ts` 是其证明矩阵)、#2636(切模型形状 churn:assistant-turn key + 图片占位变体 pass)。 +相关:#1496(本文档的起因)、`SESSION-IDENTITY.zh-CN.md`(会话粒度)、#1476 + kernel #459/#463(回声判别)、#242/#1475(Responses id 约束)、#206/#673(标签回声)、#1479/#1482(strict-echo 修复)、#1039(工具调用字节不变量)、#2396(跨 provider 切换时宿主改写的工具调用 id)、#2480(位置折叠覆盖;`tests/identity-proof.test.ts` 是其证明矩阵)、#2636(切模型形状 churn:assistant-turn key + 图片占位变体 pass)、#2695/#3588/#3640/#3641(快照即会话教义;`tests/fold-anchor-doctrine.test.ts`)。 diff --git a/tests/fold-anchor-doctrine.test.ts b/tests/fold-anchor-doctrine.test.ts index 22f6d6ebe9..5e41164373 100644 --- a/tests/fold-anchor-doctrine.test.ts +++ b/tests/fold-anchor-doctrine.test.ts @@ -76,13 +76,16 @@ function shapeHistory(): CoreMessage[] { return out; } function reshape(old: CoreMessage[], gen: string): CoreMessage[] { - // every id re-minted; prose re-wrapped (line breaks move) → norm and - // sim-adjacent-but-not-equal, wordbag identical; tool-free so every - // channel that exists today is defeated: ids differ (byNorm ✗ — the - // canonical text changed with the re-wrap), no fence ever verifies - // (byPos ✗), nothing tool (byTool/byTwin ✗), no bracket (byOrder ✗), - // sim gaps never recorded (gapPairs empty) ✗. - return old.map((m, i) => msg(`${gen}_${i}`, m.role, (m.text ?? "").replace(/ /g, i % 2 ? "\n" : " "))); + // every id re-minted; EVERY prose cell re-wrapped with space → line + // break (norm and positional fingerprints change, wordbag stays + // identical, simhash hamming stays 0) — so NO cell is claimable by any + // cheaper tier and the whole mass lands on the shape tier: ids differ + // (byNorm ✗ — canonical text changed), no fence ever verifies (byPos ✗), + // nothing tool (byTool/byTwin ✗), no bracket (byOrder ✗), sim gaps + // never recorded (gapPairs empty) ✗. Deleting the shape tier makes this + // fixture escalate (claims.size === 0 → totalDrift) — the no-escalation + // assertion below has teeth because of this, not by luck. + return old.map((m, i) => msg(`${gen}_${i}`, m.role, (m.text ?? "").replace(/ /g, "\n"))); } describe("#3640 wholesale shape churn re-anchors by wordbag evidence", () => { @@ -91,7 +94,12 @@ describe("#3640 wholesale shape churn re-anchors by wordbag evidence", () => { const session = fakeSession(old.slice(0, 9).map((m) => m.id as string), old); const logs: string[] = []; const reshaped = reshape(old, "g1"); - for (let pass = 0; pass < 4; pass++) reconcileFoldCoverage(session, reshaped, repair((l, m) => logs.push(`${l}: ${m}`))); + // the first pass must actually route the mass through the shape tier + // — byShape > 0 is the tier's footprint on this fixture (every cell + // \n-rewrapped, so no cheaper channel can claim any of them). + const first = reconcileFoldCoverage(session, reshaped, repair((l, m) => logs.push(`${l}: ${m}`))); + assert.ok(first.byShape > 0, "the reshaped mass is claimed by the shape tier, not silently unmatched"); + for (let pass = 1; pass < 4; pass++) reconcileFoldCoverage(session, reshaped, repair((l, m) => logs.push(`${l}: ${m}`))); assert.notEqual(session.metadata[METADATA_DRIFT_ESCALATED], true, "word-preserving wholesale churn must not diagnose substrate destruction"); const block = (session.state as { blocks: Array<{ effectiveMessageIds: string[] }> }).blocks[0]!; @@ -173,6 +181,35 @@ describe("#3641 total collapse asks for epoch confirmation", () => { }); }); +describe("shape tier honesty gates (no wholesale claiming)", () => { + test("wordbag-equal candidates that flip role, pad length, or reorder words are NOT shape-claimed", () => { + // #3641 review: the shape tier pairs on wordbag equality, so the + // gates around it are the only thing keeping a same-words-but- + // different message from being claimed. Each candidate below keeps + // the word multiset identical and violates exactly one guard: + // role flip, whitespace length padding (wordbag tokenizes words + // only), or word-order reversal (simhash is bigram-shingled, so + // order changes the sketch). No cheaper tier claims them either + // (identity includes role; the length collision guard in byNorm + // rejects the padding) — so all six residuals reach the shape tier + // and every pairing must be refused. + const old: CoreMessage[] = []; + for (let i = 0; i < 6; i++) old.push(msg(`g${i}`, i % 2 ? "user" : "assistant", `gate turn ${i} with vocabulary ${i} to shingle across the sentence for hashing`)); + const session = fakeSession(old.map((m) => m.id as string), old); + const reversed = (t: string) => t.split(" ").reverse().join(" "); + const wire: CoreMessage[] = [ + msg("w0", "assistant", old[0]!.text!), // role flip + msg("w1", "user", old[1]!.text!), // role flip + msg("w2", "user", old[2]!.text! + " ".repeat(600)), // length beyond the guard, wordbag identical + msg("w3", "user", old[3]!.text! + " ".repeat(600)), + msg("w4", "user", reversed(old[4]!.text!)), // word order reversed → bigram shingles change + msg("w5", "user", reversed(old[5]!.text!)), + ]; + const r = reconcileFoldCoverage(session, wire, repair()); + assert.equal(r.byShape, 0, "edit-honesty gates (role, length, simhash) must refuse every pairing"); + }); +}); + describe("honesty gates survive the doctrine (no wholesale silencing)", () => { test("epoch-confirm suspends preflight compression (verbatim passthrough until realigned)", async () => { From 41653801e15dab3f2cb20f76f2b0e71b56ff94ab Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Sun, 18 Oct 2026 14:27:31 +0800 Subject: [PATCH 08/14] feat(wire): protocol-stable canonical core identity (#3642) One logical conversation, one set of core ids, in all four dialects. - kernel/src/wire/identity.ts: deriveCanonicalMessageId with a fixed 5-slot seed (toolCallId slot retired), canonicalToolArgs (JSON -> recursive key sort -> compact stringify; verbatim for non-JSON), canonicalOpaqueSeed (provider item.id/status stripped), shared [image] marker for image cells. - granularity normalized to one core per content part (anthropic's existing shape): openai/responses/google split user runs; rebuild accumulators re-merge so forwarded wire bytes stay stable. - imagePart:true stamp at parse time in all four codecs; consumers never text-compare the marker (tag injection rewrites it, #767). - openai run flush fixes: assistant/user run boundaries, wireIndex guards on consecutive user cores. - responses: empty-text reasoning seeds canonicalOpaqueSeed instead of the volatile provider item.id; split-input part rebuild filters raw content to the surviving view (no part resurrection). - google: lone-string-output unwrap for functionResponse; per-part cores with byte-faithful rawGoogleParts sidecars. - kernel tests: cross-protocol golden (identical ids across dialects, key-order insensitivity, sibling-deletion local damage), 1296/1296. - src goldens re-pinned (id churn from canonical tool seeds only). --- CHANGELOG.md | 2 + kernel/src/wire/anthropic.ts | 78 ++++- kernel/src/wire/bili-message.ts | 7 + kernel/src/wire/google.ts | 195 +++++++---- kernel/src/wire/identity.ts | 107 ++++++ kernel/src/wire/openai.ts | 322 ++++++++++++------ kernel/src/wire/responses.ts | 303 +++++++++++++--- kernel/tests/cross-protocol-identity.test.ts | 242 +++++++++++++ kernel/tests/media-payload.test.ts | 34 +- .../tests/wire-bili-message-roundtrip.test.ts | 116 ++++--- kernel/tests/wire-google.test.ts | 26 +- .../tests/wire-tool-result-toolname.test.ts | 14 +- tests/bili-message-roundtrip.test.ts | 20 +- tests/golden/storage-render/anthropic.json | 16 +- tests/golden/storage-render/chat.json | 82 ++--- tests/golden/storage-render/google.json | 18 +- tests/golden/storage-render/responses.json | 30 +- 17 files changed, 1236 insertions(+), 376 deletions(-) create mode 100644 kernel/src/wire/identity.ts create mode 100644 kernel/tests/cross-protocol-identity.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index cfb6ab86c4..203996d71d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Changes +- **protocol-stable core identity: the same logical conversation now yields byte-identical core message ids in all four dialects (#3642).** Wire codecs previously baked dialect accidents into message identity: anthropic used `"[image]"` as image text while openai deliberately kept `""` ("switching the seed would move every pre-existing shape"), tool-call ids embedded the provider's `toolCallId` bytes, raw JSON tool arguments hashed verbatim (key order = new id), and responses' empty-text reasoning cells seeded from the provider's volatile `item.id`. Switch protocols mid-session (opencode #3640: `/chat/completions` → `/v1/messages`) and ~20-68% of ids churned, tripping the fold-destroyed misdiagnosis #3641 just class-fixed. Identity now derives from a canonical form shared by all four codecs: a new `kernel/src/wire/identity.ts` (canonicalToolArgs: JSON args parsed → recursive key sort → compact stringify, verbatim for non-JSON; canonicalOpaqueSeed: provider `item.id`/`status` stripped; image marker `[image]` everywhere), the `toolCallId` slot retired from the seed (cluster suffixes disambiguate same-shape calls, deleting one sibling is local damage), and granularity normalized to one core per content part (anthropic's existing shape — dsh/anthropic lanes take zero re-keying; openai/responses/google split runs, with accumulators re-merging on rebuild so the wire stays byte-stable). Cross-protocol golden: text/tool/bash conversation → identical id arrays in all four dialects; canonical key-sort makes `{"file","line"}` == `{"line","file"}`. Image cells are now stamped `imagePart: true` at parse time (never text-compared — tag injection rewrites markers mid-render, which previously made image cores render as text and lose the real bytes, #767 billing fallout). Upgrade cost is one re-key of stored ids for openai/responses/google lanes; the #3641 reconcile tiers (word-bag claims, k-th image family pairing, toolCallId channel, epoch gate) absorb it without re-folding. + - **fold anchoring doctrine: order over hash — protocol switches, image offload, and host-side truncation stop being read as destruction (#3641, fixes #3588, fixes #3640).** Three real-world defect classes all shared one root: the fold's identity model keyed coverage to content hashes, so any transformation that re-mints ids without removing content (dsh's image-offload component rewriting image cells into placeholder text, opencode switching a mid-session request from `/chat/completions` to `/v1/messages`, or a host that compacts/truncates history itself) made every covered id look *destroyed*. Consequences: 7× false "compression substrate appears destroyed" escalations on healthy sessions (#3588), 298-id churn with unanchored summary blocks landing AFTER the user's newest question — the model answered five rounds of stale summaries instead of the question (#3640) — and the #2193 collapse streak on sanctioned host compaction. The reconcile pass now carries a shape tier (loose word-evidence pairing for prose cells, a flanked k-th discipline for evidence-less image cells), a residual census classifying unmatched ids as *transformed* (near-duplicate evidence exists), *absent* (content truly off the wire with the surviving wire a suffix-continuation of the episode baseline), or *drift* (nothing matches anywhere); only the third accrues the collapse streak, and a wire-frozen episode witness guarantees a foreign conversation can never quietly become the baseline mid-episode. Unresolved summary anchors clamp ahead of the newest user turn instead of at the array tail (the #3640 "stale summary after the question" shape), and a genuine total collapse now latches `foldEpochConfirmPending` and PAUSES folding (verbatim passthrough — nothing is deleted, the snapshot is kept) until the histories realign or the user confirms a new epoch, instead of escalating on a misdiagnosis. Single genuine edits below the mass bar still report drift honestly; the #1729/#2193 escalation itself stays (its error text now names the epoch gate). - **dsh PTC sessions get compression back: a `run_code` omnibus on the wire reads as reachable, not unreachable (#3600).** dsh PTC presets collapse EVERY tool into a single `run_code` omnibus (the SDK bindings nest inside it), so the top-level tools array carries no `compress` entry. The #3427 reachability gate read exactly that shape as "tool not host-scoped → suppress the compress surface", and released 0.1.198 suppressed every dsh PTC session: 89× `ARMED-SUPPRESSED` nudges while the window ran to 100%, then the output got clamped and the session was effectively dead — silently, while standard-mode sessions on the same install kept compressing (v0.1.197 had no tools factor in its nudge gate, so this is a release regression, not "never worked"). The gate now treats a `run_code` omnibus declaration as reachable: top-level absence is not evidence of unreachability when the omnibus swallows everything. The #3177 true negative keeps its suppression (a bare single tool that is neither `compress` nor the omnibus still drops the surface), and the #3401 positive case is unchanged. diff --git a/kernel/src/wire/anthropic.ts b/kernel/src/wire/anthropic.ts index fc92a3ed9a..e0d3274f5c 100644 --- a/kernel/src/wire/anthropic.ts +++ b/kernel/src/wire/anthropic.ts @@ -1,6 +1,12 @@ import type { BiliMessage } from "./bili-message.js"; +import { + CANONICAL_IMAGE_TEXT, + CANONICAL_REDACTED_THINKING_TEXT, + canonicalToolArgs, + deriveCanonicalMessageId, +} from "./identity.js"; import { hashId } from "./util.js"; -import { ClusterCounter, deriveMessageId } from "./message-id.js"; +import { ClusterCounter } from "./message-id.js"; export type AnthropicTextBlock = { type: "text"; @@ -102,21 +108,26 @@ export function anthropicToCore(body: AnthropicRequestBody): Flat { for (const b of blocks) { switch (b.type) { case "text": { - const base = deriveMessageId(m.role, "text", b.text); + // #3642 canonical identity: the 5-slot seed keeps its layout with + // the toolCallId slot empty, so text ids are byte-identical to the + // legacy derivation (zero churn on the busiest lane). + const base = deriveCanonicalMessageId(m.role, "text", b.text); const id = clusters.next(base); msgs.push({ id, role: m.role, contentType: "text", text: b.text }); if (b.cache_control) cacheControls.set(id, b.cache_control); break; } case "tool_use": { - const base = deriveMessageId( + // #3642: the seed uses canonical (key-sorted, compact) args and + // drops the wire toolCallId from the seed — it is transport + // plumbing, not content; the ClusterCounter's `_N` keeps duplicate + // calls distinct. `text` KEEPS the host's insertion order so + // coreToAnthropic rebuilds byte-stable input objects. + const base = deriveCanonicalMessageId( "assistant", "tool-call", - safeStringify(b.input), - { - toolCallId: b.id, - toolName: b.name, - }, + canonicalToolArgs(safeStringify(b.input)), + { toolName: b.name }, ); const id = clusters.next(base); msgs.push({ @@ -143,11 +154,15 @@ export function anthropicToCore(body: AnthropicRequestBody): Flat { : b.content .map((c) => (c.type === "text" ? c.text : "")) .join("\n"); - const base = deriveMessageId("tool", "tool-result", text, { - toolCallId: b.tool_use_id, + // #3642: canonical result identity = toolName + text (the wire + // tool_use_id is transport plumbing). toolName is backfilled from + // the paired tool_use above; a result whose call is not in this + // pass keeps an empty toolName slot. + const name = toolNames.get(b.tool_use_id); + const base = deriveCanonicalMessageId("tool", "tool-result", text, { + toolName: name, }); const id = clusters.next(base); - const name = toolNames.get(b.tool_use_id); msgs.push({ id, role: "tool", @@ -162,7 +177,11 @@ export function anthropicToCore(body: AnthropicRequestBody): Flat { break; } case "thinking": { - const base = deriveMessageId("assistant", "reasoning", b.thinking); + const base = deriveCanonicalMessageId( + "assistant", + "reasoning", + b.thinking, + ); msgs.push({ id: clusters.next(base), role: "assistant", @@ -173,13 +192,38 @@ export function anthropicToCore(body: AnthropicRequestBody): Flat { break; } case "image": { - const base = deriveMessageId(m.role, "text", "[image]"); + const base = deriveCanonicalMessageId( + m.role, + "text", + CANONICAL_IMAGE_TEXT, + ); + // #3642: parse the base64 source into the shared media sidecars so + // every dialect presents the same image core (marker text + + // imageMediaType/imageBase64); the raw block keeps verbatim + // round-trip. Other source shapes keep the raw-only sidecar. + const source = + b.source && + typeof b.source === "object" && + !Array.isArray(b.source) + ? (b.source as { type?: unknown; media_type?: unknown; data?: unknown }) + : undefined; + const mediaSidecar = + source?.type === "base64" && + typeof source.media_type === "string" && + typeof source.data === "string" + ? { + imageMediaType: source.media_type, + imageBase64: source.data, + } + : {}; msgs.push({ id: clusters.next(base), role: m.role, contentType: "text", - text: "[image]", + text: CANONICAL_IMAGE_TEXT, + imagePart: true, rawAnthropicBlock: b, + ...mediaSidecar, }); break; } @@ -190,7 +234,11 @@ export function anthropicToCore(body: AnthropicRequestBody): Flat { // modified"). Dropping it here silently rewrites that message, so it // rides the sidecar exactly like images (#366) — coreToAnthropic // re-emits rawAnthropicBlock verbatim. - const base = deriveMessageId(m.role, "text", "[redacted_thinking]"); + const base = deriveCanonicalMessageId( + m.role, + "text", + CANONICAL_REDACTED_THINKING_TEXT, + ); msgs.push({ id: clusters.next(base), role: m.role, diff --git a/kernel/src/wire/bili-message.ts b/kernel/src/wire/bili-message.ts index 39f127fab4..4be783c8bb 100644 --- a/kernel/src/wire/bili-message.ts +++ b/kernel/src/wire/bili-message.ts @@ -36,6 +36,13 @@ export interface BiliMessage extends CoreMessage { /** Responses API: the original input item (for input_image, or a raw * function_call / function_call_output we pass through). */ rawResponsesItem?: unknown; + /** #3642: this core IS an image part of a split multi-part wire message + * (canonical granularity: one core per content part). Set by every codec + * at parse time; rebuild paths key off this flag — NEVER off text + * equality with the "[image]" marker, because render/ref-tag injection + * rewrites the marker text ("[image]") and a wire image part + * has no text slot to carry a tag anyway. */ + imagePart?: true; /** Google/Gemini: the original content parts of this message, in wire * order (inlineData/fileData, executableCode, functionCall/Response ids, * thought parts and unknown future part types). coreToGoogle rebuilds diff --git a/kernel/src/wire/google.ts b/kernel/src/wire/google.ts index 66ae501c98..f85db1266b 100644 --- a/kernel/src/wire/google.ts +++ b/kernel/src/wire/google.ts @@ -1,5 +1,10 @@ import { hashId } from "./util.js"; -import { ClusterCounter, deriveMessageId } from "./message-id.js"; +import { ClusterCounter } from "./message-id.js"; +import { + CANONICAL_IMAGE_TEXT, + canonicalToolArgs, + deriveCanonicalMessageId, +} from "./identity.js"; import type { BiliMessage } from "./bili-message.js"; // Gemini native wire (`generativelanguage.googleapis.com`, /v1beta/models/: @@ -93,7 +98,8 @@ export function googleToCore(body: GoogleRequestBody): GoogleFlat { const unmatchedCalls = new Map(); const contents = Array.isArray(body.contents) ? body.contents : []; - for (const content of contents) { + for (const [contentIndex, content] of contents.entries()) { + const before = msgs.length; const parts = Array.isArray(content.parts) ? content.parts : []; const isModel = content.role === "model"; @@ -103,7 +109,20 @@ export function googleToCore(body: GoogleRequestBody): GoogleFlat { for (const part of parts) { const fr = part.functionResponse; if (!fr || typeof fr.name !== "string") continue; - const text = jsonText(fr.response ?? {}); + // #3642: the canonical result text is the OUTPUT — a lone string + // `output` field is the dominant MCP-ish shape, and unwrapping it + // aligns google results with the other dialects (identical logical + // result ⇒ identical core text ⇒ identical id). Structured responses + // keep their compact-JSON text. + const response = fr.response ?? {}; + const text = + response && + typeof response === "object" && + !Array.isArray(response) && + Object.keys(response).length === 1 && + typeof (response as { output?: unknown }).output === "string" + ? ((response as { output: string }).output) + : jsonText(response); const wireId = typeof fr.id === "string" ? fr.id : undefined; const queue = unmatchedCalls.get(fr.name); let paired: string | undefined; @@ -115,8 +134,10 @@ export function googleToCore(body: GoogleRequestBody): GoogleFlat { } else if (queue && queue.length > 0) { paired = queue.shift(); } - const base = deriveMessageId("tool", "tool-result", text, { - toolCallId: wireId ?? paired ?? "", + // #3642: canonical result identity = toolName + text. The wire id / + // pairing id stays on the toolCallId FIELD (pairing is untouched) but + // is out of the seed — it is transport plumbing, not content. + const base = deriveCanonicalMessageId("tool", "tool-result", text, { toolName: fr.name, }); const id = clusters.next(base); @@ -136,7 +157,11 @@ export function googleToCore(body: GoogleRequestBody): GoogleFlat { // 2. Thinking parts. for (const part of parts) { if (part.thought !== true || typeof part.text !== "string") continue; - const base = deriveMessageId("assistant", "reasoning", part.text); + const base = deriveCanonicalMessageId( + "assistant", + "reasoning", + part.text, + ); msgs.push({ id: clusters.next(base), role: "assistant", @@ -147,16 +172,82 @@ export function googleToCore(body: GoogleRequestBody): GoogleFlat { }); } - // 3. The content's own text/media core. A content made only of - // functionResponse parts yields none (its parts ride on the tool - // cores), so no empty user core is invented. const rest = parts.filter(isContentPart); - if (rest.length > 0) { - const text = textPartsOf(rest) + // 3. The content's own text/media cores — #3642 canonical granularity: + // every part is its own core (the anthropic codec's model), so the + // same logical conversation yields the same core array in every + // protocol. Each text part is a text core seeded on its own bytes; + // each media part (inlineData/fileData) is an image core carrying the + // canonical "[image]" marker with the raw part as its byte-fidelity + // sidecar. coreToGoogle's role-alternation accumulator merges the + // same-side run back into one content whose parts array preserves the + // part objects verbatim. A content made only of functionResponse + // parts yields none (its parts ride on the tool cores). + let lastTextCore: BiliMessage | null = null; + let leftover: GooglePart[] = []; + for (const part of rest) { + if (typeof part.text === "string" && part.thought !== true) { + const base = deriveCanonicalMessageId( + isModel ? "assistant" : "user", + "text", + part.text, + ); + lastTextCore = { + id: clusters.next(base), + role: isModel ? "assistant" : "user", + contentType: "text", + text: part.text, + rawGoogleParts: [part], + ...signatureField( + "googleThoughtSignature", + part.thoughtSignature, + ), + }; + msgs.push(lastTextCore); + continue; + } + const inline = part.inlineData; + if ( + inline && + typeof inline.mimeType === "string" && + typeof inline.data === "string" + ) { + const base = deriveCanonicalMessageId( + isModel ? "assistant" : "user", + "text", + CANONICAL_IMAGE_TEXT, + ); + msgs.push({ + id: clusters.next(base), + role: isModel ? "assistant" : "user", + contentType: "text", + text: CANONICAL_IMAGE_TEXT, + imagePart: true, + imageMediaType: inline.mimeType, + imageBase64: inline.data, + rawGoogleParts: [part], + }); + continue; + } + // Other opaque content parts (fileData, executableCode, …) attach to + // the PRECEDING text core's raw sidecar so the verbatim rebuild keeps + // them in place; with no text core to host them they wait, and a + // message with leftovers but no text core keeps the legacy fallback + // shape (one core, joined text — possibly "" — all leftovers raw). + if (lastTextCore) { + lastTextCore.rawGoogleParts = [ + ...(lastTextCore.rawGoogleParts ?? []), + part, + ]; + } else { + leftover = [...leftover, part]; + } + } + if (leftover.length > 0) { + const text = textPartsOf(leftover) .map((p) => p.text ?? "") .join("\n"); - const inline = inlineDataOf(rest); - const base = deriveMessageId( + const base = deriveCanonicalMessageId( isModel ? "assistant" : "user", "text", text, @@ -166,15 +257,7 @@ export function googleToCore(body: GoogleRequestBody): GoogleFlat { role: isModel ? "assistant" : "user", contentType: "text", text, - ...(inline - ? { imageMediaType: inline.mimeType, imageBase64: inline.data } - : {}), - rawGoogleParts: rest, - ...signatureField( - "googleThoughtSignature", - textPartsOf(rest).find((p) => typeof p.thoughtSignature === "string") - ?.thoughtSignature, - ), + rawGoogleParts: leftover, }); } @@ -187,10 +270,16 @@ export function googleToCore(body: GoogleRequestBody): GoogleFlat { if (!fc || typeof fc.name !== "string") continue; const text = jsonText(fc.args ?? {}); const wireId = typeof fc.id === "string" ? fc.id : undefined; - const base = deriveMessageId("assistant", "tool-call", text, { - toolCallId: wireId ?? "", - toolName: fc.name, - }); + // #3642: canonical args in the seed (key-sorted, compact — the same + // normalization openai/anthropic/responses apply, so an identical + // logical call yields the same id in every protocol); the wire id + // stays on the toolCallId FIELD only. + const base = deriveCanonicalMessageId( + "assistant", + "tool-call", + canonicalToolArgs(text), + { toolName: fc.name }, + ); const id = clusters.next(base); const toolCallId = wireId ?? id; const queue = unmatchedCalls.get(fc.name); @@ -207,6 +296,9 @@ export function googleToCore(body: GoogleRequestBody): GoogleFlat { ...signatureField("googleThoughtSignature", part.thoughtSignature), }); } + for (let k = before; k < msgs.length; k++) { + msgs[k]!.wireIndex = contentIndex; + } } return { msgs, systemText: googleSystemText(body) }; @@ -385,14 +477,15 @@ function contentParts(m: BiliMessage): GooglePart[] { const text = m.text ?? ""; const raw = rawParts(m); if (raw.length === 0) { - const parts: GooglePart[] = text - ? [ - { - text, - ...signatureField("thoughtSignature", m.googleThoughtSignature), - }, - ] - : []; + const parts: GooglePart[] = + text && text !== CANONICAL_IMAGE_TEXT && !m.imagePart + ? [ + { + text, + ...signatureField("thoughtSignature", m.googleThoughtSignature), + }, + ] + : []; // The image sidecars are separate from the raw parts, so an image // survives even a core that lost its raw array (rebased/persisted). if (m.imageBase64 && m.imageMediaType) @@ -403,14 +496,17 @@ function contentParts(m: BiliMessage): GooglePart[] { } const rawTexts = textPartsOf(raw); if (rawTexts.map((p) => p.text ?? "").join("\n") === text) return raw; - const parts: GooglePart[] = text - ? [ - { - text, - ...signatureField("thoughtSignature", m.googleThoughtSignature), - }, - ] - : []; + // #3642: the canonical "[image]" marker never re-renders as a text part — + // an image core's payload IS its raw non-text part(s)/sidecar bytes. + const parts: GooglePart[] = + text && text !== CANONICAL_IMAGE_TEXT && !m.imagePart + ? [ + { + text, + ...signatureField("thoughtSignature", m.googleThoughtSignature), + }, + ] + : []; for (const part of raw) { if (!rawTexts.includes(part)) parts.push(part); } @@ -441,21 +537,6 @@ function isContentPart(part: GooglePart): boolean { return true; } -function inlineDataOf( - parts: GooglePart[], -): { mimeType: string; data: string } | undefined { - for (const part of parts) { - const data = part.inlineData; - if ( - data && - typeof data.mimeType === "string" && - typeof data.data === "string" - ) - return data; - } - return undefined; -} - /** Gemini's signature field is `thoughtSignature` on the wire but * `googleThoughtSignature` on the core sidecar; each direction derives it from * what the other end carried. */ diff --git a/kernel/src/wire/identity.ts b/kernel/src/wire/identity.ts new file mode 100644 index 0000000000..ebefcf491c --- /dev/null +++ b/kernel/src/wire/identity.ts @@ -0,0 +1,107 @@ +import { hashId } from "./util.js"; + +/** + * #3642 — protocol-independent message identity. + * + * PROBLEM: `deriveMessageId` is shared, but every codec feeds it + * protocol-specific inputs, so the SAME logical conversation serialized in + * two protocols produces different id sets: + * - tool args: openai/responses seed the host's verbatim `arguments` + * string; anthropic seeds `safeStringify(input)`; google seeds + * `jsonText(args)` — key order / spacing / double-encoding all diverge; + * - the wire toolCallId is part of the seed, and every protocol formats + * it differently (`call_x` vs `toolu_x` vs no id at all); + * - images: openai seeds "", anthropic/responses "[image]"; + * - responses seeds empty-text reasoning items on the PROVIDER's item id, + * which changes on every replay. + * Consequence (#3640): switching protocols mid-conversation re-mints a large + * fraction of ids, the fold anchors miss wholesale, and the drift escalates. + * + * FIX: identity derives from a CANONICAL projection of the message — logical + * content only, no protocol plumbing: + * role | contentType | (empty) | toolName | canonicalText + * The toolCallId slot is kept EMPTY on purpose: the 5-slot seed layout is + * unchanged (so plain-text ids stay byte-identical to pre-#3642 sessions), + * but the wire call id no longer participates — it is transport plumbing, + * not content. Two same-tool same-args calls are made distinct by the + * ClusterCounter's `_N` suffix (local damage on delete; never global). + * + * Byte fidelity is NOT identity: the raw wire block/args/item keeps riding + * the BiliMessage sidecars so coreToXxx still rebuilds byte-stable output. + * + * Granularity axiom (#3642): EVERY content part is its own core — the + * anthropic codec's existing model. openai/responses/google currently merge + * multi-part user messages into one text core (image rides a sidecar); + * they now split per part so the same logical conversation yields the same + * core ARRAY (ids AND counts) in all four protocols. Reverse codecs + * re-merge consecutive same-role cores into one wire message. + */ + +/** Canonical placeholder for a media part. Matches the anthropic codec's + * historical text so image ids on the busiest lane (anthropic) do not + * re-mint on upgrade. */ +export const CANONICAL_IMAGE_TEXT = "[image]"; + +/** Canonical placeholder for an opaque redacted-thinking payload + * (anthropic-only shape; kept for symmetry with the legacy codec text). */ +export const CANONICAL_REDACTED_THINKING_TEXT = "[redacted_thinking]"; + +/** + * Canonical JSON for tool-call arguments: parse → recursively sort object + * keys → compact re-serialize. The SAME logical arguments then hash equal + * regardless of the host's key order or spacing. Non-JSON payloads (custom + * tools send opaque strings) are returned verbatim — there is nothing to + * normalize without guessing at semantics. + */ +export function canonicalToolArgs(argsText: string): string { + const trimmed = argsText.trim(); + if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return argsText; + try { + return JSON.stringify(canonicalize(JSON.parse(trimmed))); + } catch { + return argsText; + } +} + +function canonicalize(v: unknown): unknown { + if (Array.isArray(v)) return v.map(canonicalize); + if (v !== null && typeof v === "object") { + const src = v as Record; + const out: Record = {}; + for (const k of Object.keys(src).sort()) out[k] = canonicalize(src[k]); + return out; + } + return v; +} + +/** + * Canonical identity seed → id. Layout is the legacy 5-slot + * `role|contentType|toolCallId|toolName|` shape with the toolCallId slot + * pinned empty, so plain-text messages keep their pre-#3642 ids verbatim + * (zero migration churn for text-only sessions). + */ +export function deriveCanonicalMessageId( + role: string, + contentType: string, + text: string, + options: { toolName?: string } = {}, +): string { + const seed = `${role}|${contentType}||${options.toolName ?? ""}|${text}`; + return "h_" + hashId(seed); +} + +/** Stable seed for opaque wire items (responses reasoning items without + * text, computer/file/web/image/code/mcp call items): hash the item with + * volatile lifecycle fields stripped — `id` (provider-minted, changes on + * every replay) and `status` (in_progress → completed lifecycle state). + * Everything else is content and stays. */ +export function canonicalOpaqueSeed(item: unknown): string { + if (item === null || typeof item !== "object") return hashId(String(item)); + const src = item as Record; + const out: Record = {}; + for (const k of Object.keys(src)) { + if (k === "id" || k === "status") continue; + out[k] = src[k]; + } + return hashId(JSON.stringify(out)); +} diff --git a/kernel/src/wire/openai.ts b/kernel/src/wire/openai.ts index d31b4715fe..59fccd58f9 100644 --- a/kernel/src/wire/openai.ts +++ b/kernel/src/wire/openai.ts @@ -1,6 +1,11 @@ import { splitDemotedThinking } from "./demoted-thinking.js"; +import { + CANONICAL_IMAGE_TEXT, + canonicalToolArgs, + deriveCanonicalMessageId, +} from "./identity.js"; import { hashId } from "./util.js"; -import { ClusterCounter, deriveMessageId } from "./message-id.js"; +import { ClusterCounter } from "./message-id.js"; import { parseDataUrl, type BiliMessage } from "./bili-message.js"; export type OpenAIContentPart = @@ -54,7 +59,8 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { // Backfill toolName onto role:"tool" results from their paired tool_call — // kernel guards are name-based. Not part of the identity seed (id stability). const toolNames = new Map(); - for (const m of body.messages) { + for (const [wireIndex, m] of body.messages.entries()) { + const before = msgs.length; switch (m.role) { case "system": case "developer": { @@ -62,7 +68,11 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { systemParts.push(stringContent(m.content)); break; } - const base = deriveMessageId(m.role, "text", stringContent(m.content)); + const base = deriveCanonicalMessageId( + m.role, + "text", + stringContent(m.content), + ); msgs.push({ id: clusters.next(base), role: "system", @@ -73,43 +83,119 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { break; } case "user": { - // Image-only messages keep text "" (NO "[image]" placeholder as - // in the responses/anthropic codecs): ids derive from text alone - // and switching the seed would move every pre-existing shape. - const text = stringContent(m.content); - const nonText = allNonTextParts(m.content); - const imgs = allImageParts(m.content); - const firstImg = imgs[0]; - const firstUrl = firstImg ? firstImg.image_url.url : undefined; - const firstParsed = firstUrl ? parseDataUrl(firstUrl) : undefined; - const base = deriveMessageId("user", "text", text); - msgs.push({ - id: clusters.next(base), - role: "user", - contentType: "text", - text, - // A lone string-URL image keeps the legacy singular sidecar - // (byte-stable for pre-existing shapes). Any other mix — - // multiple images, or any unknown part type such as a - // DeepSeek Files API `{type:"file"}` ref — rides the plural - // sidecar verbatim so coreToOpenai can re-emit it - // (billion-context#1205: unknown parts were silently dropped). - ...(imgs.length === 1 && nonText.length === 1 - ? { - rawOpenaiContent: imgs[0], - // data: URLs only — remote URLs ride rawOpenaiContent - // verbatim (#187/#291) - ...(firstParsed - ? { - imageMediaType: firstParsed.mediaType, - imageBase64: firstParsed.base64, - } - : {}), - } - : nonText.length > 0 - ? { rawOpenaiContentParts: nonText } - : {}), - }); + // #3642 canonical granularity: every content part is its own core + // (the anthropic codec's model), so the same logical conversation + // yields the same core ARRAY — ids AND counts — in every protocol. + // Text parts become text cores; each image part becomes its own + // image core (text = the canonical "[image]" marker, byte-fidelity + // via the sidecars); unknown opaque parts (DeepSeek file refs, + // #1205) attach to the preceding text core's plural sidecar so they + // still round-trip verbatim. A message with only opaque parts keeps + // the legacy single-core shape (text "" + plural sidecar). + if (typeof m.content === "string" || m.content == null) { + const base = deriveCanonicalMessageId( + "user", + "text", + m.content ?? "", + ); + msgs.push({ + id: clusters.next(base), + role: "user", + contentType: "text", + text: m.content ?? "", + }); + break; + } + let lastTextCore: BiliMessage | null = null; + let leadingOpaque: OpenAIContentPart[] | [] = []; + let sawText = false; + for (const part of m.content) { + if (typeof part === "string") { + const base = deriveCanonicalMessageId("user", "text", part); + lastTextCore = { + id: clusters.next(base), + role: "user", + contentType: "text", + text: part, + }; + msgs.push(lastTextCore); + sawText = true; + continue; + } + if (part?.type === "text" && typeof part.text === "string") { + const base = deriveCanonicalMessageId("user", "text", part.text); + lastTextCore = { + id: clusters.next(base), + role: "user", + contentType: "text", + text: part.text, + }; + msgs.push(lastTextCore); + sawText = true; + continue; + } + if (part?.type === "image_url") { + const url = + (part as { image_url?: { url?: unknown } }).image_url?.url; + const parsed = + typeof url === "string" ? parseDataUrl(url) : undefined; + const base = deriveCanonicalMessageId( + "user", + "text", + CANONICAL_IMAGE_TEXT, + ); + msgs.push({ + id: clusters.next(base), + role: "user", + contentType: "text", + text: CANONICAL_IMAGE_TEXT, + imagePart: true, + rawOpenaiContent: part, + ...(parsed + ? { + imageMediaType: parsed.mediaType, + imageBase64: parsed.base64, + } + : {}), + }); + continue; + } + // Unknown opaque part: never dropped (#1205). Attach to the + // preceding text core; parts seen before any text core wait for + // the first one so the rebuild's part order matches the wire. + if (lastTextCore) { + lastTextCore.rawOpenaiContentParts = [ + ...(lastTextCore.rawOpenaiContentParts ?? []), + part, + ]; + } else { + leadingOpaque = [...leadingOpaque, part]; + } + } + if (leadingOpaque.length > 0) { + if (sawText && msgs.length > 0) { + const firstTextCore = + msgs[msgs.length - 1] && + msgs.find( + (c, i) => + i >= before && + c.role === "user" && + c.contentType === "text" && + !c.rawOpenaiContent, + ); + const host = firstTextCore ?? msgs[msgs.length - 1]!; + host.rawOpenaiContentParts = leadingOpaque; + } else { + const base = deriveCanonicalMessageId("user", "text", ""); + msgs.push({ + id: clusters.next(base), + role: "user", + contentType: "text", + text: "", + rawOpenaiContentParts: leadingOpaque, + }); + } + } break; } case "assistant": { @@ -138,7 +224,11 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { // thinking models reject the key's absence, not its emptiness. const blankEcho = fieldPresent && reasoning.length === 0; if (reasoning) { - const base = deriveMessageId("assistant", "reasoning", reasoning); + const base = deriveCanonicalMessageId( + "assistant", + "reasoning", + reasoning, + ); msgs.push({ id: clusters.next(base), role: "assistant", @@ -148,7 +238,7 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { }); } if (text) { - const base = deriveMessageId("assistant", "text", text); + const base = deriveCanonicalMessageId("assistant", "text", text); msgs.push({ id: clusters.next(base), role: "assistant", @@ -159,14 +249,15 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { } if (Array.isArray(m.tool_calls)) { for (const tc of m.tool_calls) { - const base = deriveMessageId( + // #3642: canonical args in the seed, wire toolCallId out of it + // (transport plumbing, not content). `text` keeps the host's + // verbatim arguments string so coreToOpenai rebuilds it + // byte-stable. + const base = deriveCanonicalMessageId( "assistant", "tool-call", - tc.function.arguments ?? "", - { - toolCallId: tc.id, - toolName: tc.function.name, - }, + canonicalToolArgs(tc.function.arguments ?? ""), + { toolName: tc.function.name }, ); msgs.push({ id: clusters.next(base), @@ -183,13 +274,13 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { break; } case "tool": { - const base = deriveMessageId( + // #3642: canonical result identity = toolName + text; the wire + // tool_call_id is out of the seed (transport plumbing). + const base = deriveCanonicalMessageId( "tool", "tool-result", stringContent(m.content), - { - toolCallId: m.tool_call_id ?? "", - }, + { toolName: m.name || toolNames.get(m.tool_call_id ?? "") }, ); const name = m.name || toolNames.get(m.tool_call_id ?? ""); msgs.push({ @@ -203,12 +294,54 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { break; } } + for (let k = before; k < msgs.length; k++) { + msgs[k]!.wireIndex = wireIndex; + } } return { msgs, systemText: systemParts.join("\n\n") }; } export function coreToOpenai(messages: BiliMessage[]): OpenAIMessage[] { const out: OpenAIMessage[] = []; + let userRun: BiliMessage[] = []; + // #3642: rebuild a run of consecutive user cores as ONE wire message — + // parts array when any core carries media/opaque sidecars, plain string + // when the run is a lone text core (the dominant shape stays + // byte-identical to the legacy codec's output). + const flushUserRun = () => { + if (userRun.length === 0) return; + const parts: OpenAIContentPart[] = []; + let plain = true; + for (const m of userRun) { + if (m.rawOpenaiContentParts && m.rawOpenaiContentParts.length > 0) { + if (m.text) parts.push({ type: "text", text: m.text }); + for (const part of m.rawOpenaiContentParts) + parts.push(part as OpenAIContentPart); + plain = false; + } else if (m.rawOpenaiContent) { + if (m.text && !m.imagePart) + parts.push({ type: "text", text: m.text }); + parts.push(m.rawOpenaiContent as OpenAIContentPart); + plain = false; + } else if (m.imageBase64 && m.imageMediaType) { + parts.push({ + type: "image_url", + image_url: { + url: `data:${m.imageMediaType};base64,${m.imageBase64}`, + }, + }); + plain = false; + } else { + parts.push({ type: "text", text: m.text ?? "" }); + } + } + if (plain && userRun.length === 1) { + out.push({ role: "user", content: userRun[0]!.text ?? "" }); + } else { + out.push({ role: "user", content: parts }); + } + userRun = []; + }; let pending: { text: string | null; toolCalls: OpenAIToolCall[]; @@ -249,6 +382,10 @@ export function coreToOpenai(messages: BiliMessage[]): OpenAIMessage[] { }; for (const m of messages) { if (m.role === "assistant") { + // An assistant core NEVER joins a user run — flush the run first so + // emission order always matches core order (a run left open across an + // assistant turn would emit after it and scramble the conversation). + flushUserRun(); if (!pending) pending = { text: null, @@ -270,33 +407,33 @@ export function coreToOpenai(messages: BiliMessage[]): OpenAIMessage[] { }); } } else { - flush(); + if (m.role !== "user") { + // A user message never terminates the user run — consecutive user + // cores accumulate into ONE wire message (#3642). Everything else + // (system/tool) closes the run first. + flush(); + flushUserRun(); + } if (m.role === "system") { out.push({ role: m.originalRole === "developer" ? "developer" : "system", content: m.text ?? "", }); } else if (m.role === "user") { - if (m.rawOpenaiContent || m.imageBase64 || m.rawOpenaiContentParts) { - const parts: OpenAIContentPart[] = []; - if (m.text) parts.push({ type: "text", text: m.text }); - if (m.rawOpenaiContentParts && m.rawOpenaiContentParts.length > 0) { - for (const part of m.rawOpenaiContentParts) - parts.push(part as OpenAIContentPart); - } else if (m.rawOpenaiContent) { - parts.push(m.rawOpenaiContent as OpenAIContentPart); - } else if (m.imageBase64 && m.imageMediaType) { - parts.push({ - type: "image_url", - image_url: { - url: `data:${m.imageMediaType};base64,${m.imageBase64}`, - }, - }); - } - out.push({ role: "user", content: parts }); - } else { - out.push({ role: "user", content: m.text ?? "" }); + // #3642: canonical granularity gives one core per content part, so + // user cores from ONE wire message are a RUN that rebuilds as ONE + // wire message (multi-part content array when any core carries + // media/opaque sidecars, plain string otherwise). A wireIndex step + // (or a core without one — kernel-rendered) still joins the run; + // cores from DIFFERENT wire messages (both indexed) rebuild as + // separate messages, byte-shaped like the host sent them. The + // assistant run closes here first (mirror of the assistant branch). + flush(); + const prev = userRun[userRun.length - 1]; + if (prev && prev.wireIndex !== undefined && m.wireIndex !== undefined && prev.wireIndex !== m.wireIndex) { + flushUserRun(); } + userRun.push(m); } else if (m.role === "tool") { out.push({ role: "tool", @@ -307,6 +444,7 @@ export function coreToOpenai(messages: BiliMessage[]): OpenAIMessage[] { } } flush(); + flushUserRun(); return out; } @@ -359,43 +497,3 @@ function stringContent(content: OpenAIMessage["content"]): string { return ""; } -type OpenAIImagePart = { type: "image_url"; image_url: { url: string } }; - -/** Collect ALL image_url parts in a user content array, in wire order — - * data: URLs and remote http(s) URLs alike. (firstImagePart only kept the - * first, which silently dropped images 2..N on the coreToOpenai rebuild; - * a later data-URL-only filter then silently dropped every remote-URL - * image — issues #187/#291.) */ -function allImageParts(content: OpenAIMessage["content"]): OpenAIImagePart[] { - if (!Array.isArray(content)) return []; - const out: OpenAIImagePart[] = []; - for (const p of content) { - if (typeof p !== "object" || p === null) continue; - if (!("type" in p) || p.type !== "image_url" || !("image_url" in p)) - continue; - // The union's index-signature member leaves image_url as `unknown`, so - // narrow via a named const before reading the url. - const imagePart = p as { image_url?: { url?: unknown } | null }; - const url = imagePart.image_url?.url; - if (typeof url === "string") out.push(p as OpenAIImagePart); - } - return out; -} - -/** Collect ALL non-text object parts of a user content array, in wire order — - * image_url parts plus any unknown part type. Unknown parts are opaque - * payloads (e.g. DeepSeek Files API `{type:"file",file_id}` refs) that the - * kernel must never drop: they ride rawOpenaiContentParts verbatim and - * coreToOpenai re-emits them unchanged (billion-context#1205). */ -function allNonTextParts( - content: OpenAIMessage["content"], -): OpenAIContentPart[] { - if (!Array.isArray(content)) return []; - const out: OpenAIContentPart[] = []; - for (const p of content) { - if (typeof p !== "object" || p === null) continue; - if (p.type === "text") continue; - out.push(p); - } - return out; -} diff --git a/kernel/src/wire/responses.ts b/kernel/src/wire/responses.ts index 96b141af33..7ee68ab03b 100644 --- a/kernel/src/wire/responses.ts +++ b/kernel/src/wire/responses.ts @@ -1,6 +1,12 @@ import type { CoreMessage } from "../types.js"; import { splitDemotedThinking } from "./demoted-thinking.js"; -import { ClusterCounter, deriveMessageId } from "./message-id.js"; +import { ClusterCounter } from "./message-id.js"; +import { + CANONICAL_IMAGE_TEXT, + canonicalOpaqueSeed, + canonicalToolArgs, + deriveCanonicalMessageId, +} from "./identity.js"; import type { ConversationIdentity } from "./util.js"; import { hashId } from "./util.js"; import { parseDataUrl, type BiliMessage } from "./bili-message.js"; @@ -56,6 +62,12 @@ export type ResponsesRequestBody = { type ResponseLayoutSlot = { original: ResponseInputItem; coreId?: string; + /** #3642: canonical granularity splits one wire item into one core per + * content part. `coreId` stays the FIRST core (insertion anchoring); + * `coreIds` lists every core this item produced so the patch/replay path + * can drop the item only when ALL of its cores are folded, and rebuild a + * partial view when only some are (never resurrecting folded parts). */ + coreIds?: string[]; }; export type ResponsesProjection = { @@ -186,7 +198,9 @@ export function responsesToCore( if (typeof body.instructions === "string" && body.instructions.trim()) systemParts.push(body.instructions); if (typeof body.input === "string") { - const id = clusters.next(deriveMessageId("user", "text", body.input)); + const id = clusters.next( + deriveCanonicalMessageId("user", "text", body.input), + ); msgs.push({ id, role: "user", contentType: "text", text: body.input }); return { msgs, @@ -198,9 +212,10 @@ export function responsesToCore( stringInput: { original: body.input, coreId: id }, }; } - for (const raw of body.input) { + for (const [wireIndex, raw] of body.input.entries()) { const item = normalizeEasyInputItem(raw); let coreId: string | undefined; + const before = msgs.length; if (isOpaqueItem(item)) preamble.push(item); switch (item.type) { case "reasoning": { @@ -214,14 +229,15 @@ export function responsesToCore( // put the mirror in a different ref/fingerprint space, so restart // replay rejected every in-stream compress call (issue #64, // responses variant). + // #3642: an item with NO extractable text used to seed on the + // provider's item id — unstable across resend. Seed on the + // canonical opaque seed (item minus id/status, hashed) instead; + // same bytes ⇒ same id in every future pass. const text = reasoningText(item); - const rid = - text.length > 0 - ? text - : "id" in item && typeof item.id === "string" - ? item.id - : hashId(JSON.stringify(item)); - coreId = clusters.next(deriveMessageId("assistant", "reasoning", rid)); + const rid = text.length > 0 ? text : canonicalOpaqueSeed(item); + coreId = clusters.next( + deriveCanonicalMessageId("assistant", "reasoning", rid), + ); msgs.push({ id: coreId, role: "assistant", @@ -243,6 +259,80 @@ export function responsesToCore( (message.role === "assistant" && text) ) { const role = message.role; + // #3642 canonical granularity: split per content part (text parts + // ⇒ text cores seeded on their own bytes; image parts ⇒ image + // cores carrying the canonical "[image]" marker) so the same + // logical conversation yields the same core ARRAY as every other + // codec. Split text cores carry NO rawResponsesItem — the rebuild + // run-accumulator decides verbatim-vs-filtered — while image + // cores keep the raw item as their byte-fidelity sidecar. Messages + // with unknown part types fall back to the legacy merged shape + // (merged raw sidecar keeps unknown parts round-tripping, + // openai #1205 class). Only USER messages split: assistant + // message items in input are rare single-part text. + const splittable = + role === "user" && + Array.isArray(message.content) && + message.content.length > 1 && + message.content.every( + (part) => + typeof part === "object" && + part !== null && + (part.type === "input_text" || + part.type === "output_text" || + part.type === "input_image"), + ); + if (splittable) { + for (const rawPart of message.content) { + const part = + typeof rawPart === "object" && rawPart !== null + ? rawPart + : undefined; + if (!part) continue; + if ( + (part.type === "input_text" || part.type === "output_text") && + typeof part.text === "string" && + part.text.length > 0 + ) { + const base = deriveCanonicalMessageId( + "user", + "text", + part.text, + ); + msgs.push({ + id: clusters.next(base), + role: "user", + contentType: "text", + text: part.text, + }); + } else if ( + part.type === "input_image" && + typeof part.image_url === "string" + ) { + const base = deriveCanonicalMessageId( + "user", + "text", + CANONICAL_IMAGE_TEXT, + ); + const image = parseDataUrl(part.image_url); + msgs.push({ + id: clusters.next(base), + role: "user", + contentType: "text", + text: CANONICAL_IMAGE_TEXT, + imagePart: true, + rawResponsesItem: item, + ...(image + ? { + imageMediaType: image.mediaType, + imageBase64: image.base64, + } + : {}), + }); + } + } + break; + } let effText = text; if (role === "assistant") { // Same normalization as openaiToCore: hosts that demote @@ -255,7 +345,11 @@ export function responsesToCore( if (split) { msgs.push({ id: clusters.next( - deriveMessageId("assistant", "reasoning", split.reasoning), + deriveCanonicalMessageId( + "assistant", + "reasoning", + split.reasoning, + ), ), role: "assistant", contentType: "reasoning", @@ -280,7 +374,9 @@ export function responsesToCore( role === "user" ? userDisplayText(message.content, effText) : effText; - coreId = clusters.next(deriveMessageId(role, "text", displayText)); + coreId = clusters.next( + deriveCanonicalMessageId(role, "text", displayText), + ); const imageUrl = Array.isArray(message.content) ? message.content.find( (part) => @@ -306,11 +402,16 @@ export function responsesToCore( } case "function_call": { const call = item as ResponseFunctionCall; + // #3642: canonical args in the seed (key-sorted, compact — same + // normalization as the other codecs); the wire call_id stays on the + // toolCallId FIELD (pairing) but is out of the seed. coreId = clusters.next( - deriveMessageId("assistant", "tool-call", call.arguments ?? "", { - toolCallId: call.call_id, - toolName: call.name, - }), + deriveCanonicalMessageId( + "assistant", + "tool-call", + canonicalToolArgs(call.arguments ?? ""), + { toolName: call.name }, + ), ); msgs.push({ id: coreId, @@ -331,8 +432,8 @@ export function responsesToCore( ? output.output : JSON.stringify(output.output); coreId = clusters.next( - deriveMessageId("tool", "tool-result", text, { - toolCallId: output.call_id, + deriveCanonicalMessageId("tool", "tool-result", text, { + toolName: toolNames.get(output.call_id), }), ); const name = toolNames.get(output.call_id); @@ -354,12 +455,11 @@ export function responsesToCore( case "image_generation_call": case "code_interpreter_call": case "mcp_call": { - const rid = - typeof (item as { id?: unknown }).id === "string" - ? String((item as { id?: string }).id) - : hashId(JSON.stringify(item)); + // #3642: the provider's item id is unstable across resends — seed + // on the canonical opaque seed (item minus id/status) instead. + const rid = canonicalOpaqueSeed(item); coreId = clusters.next( - deriveMessageId("assistant", "responses-call", rid), + deriveCanonicalMessageId("assistant", "responses-call", rid), ); msgs.push({ id: coreId, @@ -381,8 +481,7 @@ export function responsesToCore( customToolCallIds.add(callId); const argText = ctc.input ?? ctc.arguments ?? ""; coreId = clusters.next( - deriveMessageId("assistant", "tool-call", argText, { - toolCallId: callId, + deriveCanonicalMessageId("assistant", "tool-call", argText, { toolName: ctc.name ?? "custom", }), ); @@ -407,8 +506,8 @@ export function responsesToCore( ? ctco.output : JSON.stringify(ctco.output ?? ""); coreId = clusters.next( - deriveMessageId("tool", "tool-result", outText, { - toolCallId: callId, + deriveCanonicalMessageId("tool", "tool-result", outText, { + toolName: toolNames.get(callId), }), ); const name = toolNames.get(callId); @@ -427,7 +526,15 @@ export function responsesToCore( if (!isOpaqueItem(item)) preamble.push(item); break; } - layout.push({ original: item, coreId }); + for (let k = before; k < msgs.length; k++) { + msgs[k]!.wireIndex = wireIndex; + } + const produced = msgs.slice(before).map((m) => m.id); + layout.push({ + original: item, + ...(produced.length > 0 ? { coreId: produced[0] } : {}), + ...(produced.length > 1 ? { coreIds: produced } : {}), + }); idx++; } return { @@ -552,6 +659,47 @@ export function patchResponsesInput( const out: ResponseInputItem[] = []; projection.layout.forEach((slot, index) => { out.push(...(insertions.get(index) ?? [])); + if (slot.coreIds && slot.coreIds.length > 1) { + // #3642 split item: drop only when ALL cores folded; partial survival + // rebuilds a filtered view (never resurrecting folded parts). + const entries = slot.coreIds.map((id) => ({ + core: sourceById.get(id), + next: nextById.get(id), + })); + if (entries.every((entry) => entry.next === undefined)) return; + const original = slot.original as ResponseInputMessage; + const allSurvive = entries.every((entry) => entry.next !== undefined); + const changed = entries.some( + (entry) => + entry.core && + entry.next && + (entry.core.text ?? "") !== (entry.next.text ?? ""), + ); + if (allSurvive && !changed) { + out.push(slot.original); + return; + } + const imageParts = Array.isArray(original.content) + ? original.content.filter((part) => part.type === "input_image") + : []; + let img = 0; + const parts: ResponseContentPart[] = []; + for (const entry of entries) { + if (!entry.next) continue; + const bili = entry.next as BiliMessage; + // The SOURCE core decides image-ness: render/ref-tag injection may + // rewrite the marker text, and a wire image part has no text slot. + if (entry.core?.imagePart || bili.text === CANONICAL_IMAGE_TEXT) { + const part = imageParts[img++]; + if (part) parts.push(part); + } else if (bili.text) { + parts.push({ type: "input_text", text: bili.text }); + } + } + if (parts.length > 0) + out.push({ ...original, content: parts } as ResponseInputItem); + return; + } if (!slot.coreId) { out.push(slot.original); return; @@ -570,16 +718,19 @@ export function coreToResponses( customToolCallIds: Set = new Set(), ): ResponseInputItem[] { const out: ResponseInputItem[] = []; - for (const message of messages) { - const biliMessage = message as BiliMessage; - const raw = biliMessage.rawResponsesItem as ResponseInputItem | undefined; - if (message.role === "system") { - out.push({ - type: "message", - role: "developer", - content: message.text ?? "", - }); - } else if (message.role === "user") { + // #3642: canonical granularity splits one user item into one core per + // part, so consecutive user cores are a RUN that rebuilds as ONE wire + // item. Full coverage (every part's core present, run length == part + // count) rebuilds the original item VERBATIM — byte-exact for the + // untouched case. A partial run (some parts' cores folded away) rebuilds + // a FILTERED view: only the surviving parts, never resurrecting folded + // content from the raw sidecar. + const flushUserRun = (run: CoreMessage[]) => { + if (run.length === 0) return; + if (run.length === 1) { + const message = run[0]!; + const raw = (message as BiliMessage) + .rawResponsesItem as ResponseInputItem | undefined; const rawMessage = raw?.type === "message" ? (raw as ResponseInputMessage) : undefined; if ( @@ -590,13 +741,86 @@ export function coreToResponses( ) === (message.text ?? "") ) out.push(rawMessage); + else if ( + ((message as BiliMessage).imagePart || + (message as BiliMessage).text === CANONICAL_IMAGE_TEXT) && + rawMessage && + Array.isArray(rawMessage.content) + ) + out.push({ + type: "message", + role: "user", + content: rawMessage.content.filter( + (part) => part.type === "input_image", + ), + }); else out.push({ type: "message", role: "user", content: message.text ?? "", }); - } else if (message.role === "assistant") { + return; + } + const rawCarrier = run.find( + (m) => (m as BiliMessage).rawResponsesItem !== undefined, + ) as BiliMessage | undefined; + const rawItem = rawCarrier?.rawResponsesItem as + | ResponseInputItem + | undefined; + const rawMessage = + rawItem?.type === "message" + ? (rawItem as ResponseInputMessage) + : undefined; + const fullCover = + rawMessage && + Array.isArray(rawMessage.content) && + rawMessage.content.length === run.length; + if (fullCover && rawMessage) { + out.push(rawMessage); + return; + } + const imageParts = + rawMessage && Array.isArray(rawMessage.content) + ? rawMessage.content.filter((part) => part.type === "input_image") + : []; + let img = 0; + const parts: ResponseContentPart[] = []; + for (const message of run) { + const bili = message as BiliMessage; + // Image-ness is structural (parse-time flag), never text equality: + // render/ref-tag injection rewrites the marker, and image parts have + // no text slot to carry a tag. + if (bili.imagePart && bili.rawResponsesItem) { + const part = imageParts[img++]; + if (part) parts.push(part); + } else if (bili.text) { + parts.push({ type: "input_text", text: bili.text }); + } + } + out.push({ + type: "message", + role: "user", + content: parts.length > 0 ? parts : "", + }); + }; + let userRun: CoreMessage[] = []; + for (const message of messages) { + if (message.role === "user") { + userRun.push(message); + continue; + } + flushUserRun(userRun); + userRun = []; + const biliMessage = message as BiliMessage; + const raw = biliMessage.rawResponsesItem as ResponseInputItem | undefined; + if (message.role === "system") { + out.push({ + type: "message", + role: "developer", + content: message.text ?? "", + }); + } else if (message.role === "assistant") { if (message.contentType === "text") { out.push({ type: "message", @@ -641,6 +865,7 @@ export function coreToResponses( } } } + flushUserRun(userRun); return out; } diff --git a/kernel/tests/cross-protocol-identity.test.ts b/kernel/tests/cross-protocol-identity.test.ts new file mode 100644 index 0000000000..32c0ea28b1 --- /dev/null +++ b/kernel/tests/cross-protocol-identity.test.ts @@ -0,0 +1,242 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { anthropicToCore, type AnthropicRequestBody } from "../src/wire/anthropic.js"; +import { openaiToCore, type OpenAIRequestBody } from "../src/wire/openai.js"; +import { responsesToCore, type ResponsesRequestBody } from "../src/wire/responses.js"; +import { googleToCore, type GoogleRequestBody } from "../src/wire/google.js"; +import { CANONICAL_IMAGE_TEXT } from "../src/wire/identity.js"; + +/** + * #3642 protocol-stable core: the same logical conversation, expressed in + * every wire dialect, must project to the SAME core array — same order, same + * contentType sequence, same ids. Identity is derived from canonical content + * (role, contentType, toolName, canonical tool args, verbatim text, image + * marker), never from transport plumbing (wire tool-call ids, provider item + * ids, protocol-specific part shapes). + * + * This is the cross-protocol golden: if any codec drifts from the canonical + * granularity or seed, a protocol switch would churn every message id + * (exactly the #3640 failure mode this file pins shut). + */ + +const IMG_DATA = "aGVsbG8="; + +const USER_TEXT = "list the files, then read package.json"; + +function anthropicBody(): AnthropicRequestBody { + return { + model: "claude-x", + system: "be terse", + messages: [ + { role: "user", content: USER_TEXT }, + { + role: "assistant", + content: [{ type: "tool_use", id: "tu_a", name: "bash", input: { cmd: "ls" } }], + }, + { + role: "user", + content: [{ type: "tool_result", tool_use_id: "tu_a", content: "a.ts\nb.ts" }], + }, + { role: "assistant", content: "done" }, + ], + }; +} + +function openaiBody(): OpenAIRequestBody { + return { + model: "gpt-x", + messages: [ + { role: "system", content: "be terse" }, + { role: "user", content: USER_TEXT }, + { + role: "assistant", + content: null, + tool_calls: [{ id: "call_xyz", type: "function", function: { name: "bash", arguments: "{\"cmd\":\"ls\"}" } }], + }, + { role: "tool", tool_call_id: "call_xyz", content: "a.ts\nb.ts" }, + { role: "assistant", content: "done" }, + ], + }; +} + +function responsesBody(): ResponsesRequestBody { + return { + model: "gpt-x", + instructions: "be terse", + input: [ + { type: "message", role: "user", content: USER_TEXT }, + { type: "function_call", call_id: "fc_9", name: "bash", arguments: "{\"cmd\":\"ls\"}" }, + { type: "function_call_output", call_id: "fc_9", output: "a.ts\nb.ts" }, + { type: "message", role: "assistant", content: "done" }, + ], + }; +} + +function googleBody(): GoogleRequestBody { + return { + systemInstruction: { parts: [{ text: "be terse" }] }, + contents: [ + { role: "user", parts: [{ text: USER_TEXT }] }, + { role: "model", parts: [{ functionCall: { name: "bash", args: { cmd: "ls" }, id: "call_g" } }] }, + { role: "user", parts: [{ functionResponse: { name: "bash", response: { output: "a.ts\nb.ts" }, id: "call_g" } }] }, + { role: "model", parts: [{ text: "done" }] }, + ], + }; +} + +function shapeOf(msgs: { role: string; contentType: string; text?: string; toolName?: string }[]) { + return msgs.map((m) => `${m.role}/${m.contentType}:${m.toolName ?? ""}`); +} + +test("cross-protocol golden: text/tool-call/tool-result conversation yields identical cores and ids", () => { + const cores = [ + anthropicToCore(anthropicBody()).msgs, + openaiToCore(openaiBody()).msgs, + responsesToCore(responsesBody()).msgs, + googleToCore(googleBody()).msgs, + ]; + + // Same projection shape everywhere (system hoisted out of the fold space). + for (const msgs of cores) { + assert.deepEqual(shapeOf(msgs), [ + "user/text:", + "assistant/tool-call:bash", + "tool/tool-result:bash", + "assistant/text:", + ]); + } + + // THE invariant: byte-identical ids across all four dialects. + const ids = cores.map((msgs) => msgs.map((m) => m.id)); + assert.deepEqual(ids[0], ids[1], "anthropic ids == openai ids"); + assert.deepEqual(ids[0], ids[2], "anthropic ids == responses ids"); + assert.deepEqual(ids[0], ids[3], "anthropic ids == google ids"); + + // Texts identical (canonical bytes), including tool args/results. + for (const msgs of cores) { + assert.equal(msgs[0]!.text, USER_TEXT); + assert.equal(msgs[3]!.text, "done"); + } +}); + +test("cross-protocol golden: [text, image] multi-part user content splits to the same core pair", () => { + const a = anthropicToCore({ + model: "claude-x", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "see this" }, + { type: "image", source: { type: "base64", media_type: "image/png", data: IMG_DATA } }, + ], + }, + ], + }).msgs; + const o = openaiToCore({ + model: "gpt-x", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "see this" }, + { type: "image_url", image_url: { url: `data:image/png;base64,${IMG_DATA}` } }, + ], + }, + ], + }).msgs; + const r = responsesToCore({ + model: "gpt-x", + input: [ + { + type: "message", + role: "user", + content: [ + { type: "input_text", text: "see this" }, + { type: "input_image", image_url: `data:image/png;base64,${IMG_DATA}` }, + ], + }, + ], + }).msgs; + const g = googleToCore({ + contents: [ + { + role: "user", + parts: [ + { text: "see this" }, + { inlineData: { mimeType: "image/png", data: IMG_DATA } }, + ], + }, + ], + }).msgs; + + for (const msgs of [a, o, r, g]) { + assert.equal(msgs.length, 2, "one core per content part"); + assert.equal(msgs[0]!.text, "see this"); + assert.equal(msgs[1]!.text, CANONICAL_IMAGE_TEXT); + assert.equal(msgs[1]!.imageMediaType, "image/png"); + assert.equal(msgs[1]!.imageBase64, IMG_DATA); + } + const ids = [a, o, r, g].map((msgs) => msgs.map((m) => m.id)); + assert.deepEqual(ids[0], ids[1]); + assert.deepEqual(ids[0], ids[2]); + assert.deepEqual(ids[0], ids[3]); +}); + +test("cross-protocol golden: identical logical tool args in different key order yield the same id", () => { + const a = anthropicToCore({ + model: "claude-x", + messages: [ + { role: "user", content: "go" }, + { + role: "assistant", + content: [{ type: "tool_use", id: "tu_1", name: "edit", input: { file: "a.ts", line: 3 } }], + }, + ], + }).msgs; + const o = openaiToCore({ + model: "gpt-x", + messages: [ + { role: "user", content: "go" }, + { + role: "assistant", + content: null, + tool_calls: [ + { id: "c1", type: "function", function: { name: "edit", arguments: "{\"line\":3,\"file\":\"a.ts\"}" } }, + ], + }, + ], + }).msgs; + const callA = a.find((m) => m.contentType === "tool-call")!; + const callO = o.find((m) => m.contentType === "tool-call")!; + assert.equal(callA.id, callO.id, "canonical args are key-sorted: wire key order is plumbing"); +}); + +test("cross-protocol golden: deleting one of two sibling tool calls only renumbers its sibling suffix", () => { + // IDENTICAL logical calls (same name+args) collide on the same base id and + // are disambiguated by cluster suffix (_2, _3 …) in wire order. + const call = (id: string) => + ({ type: "tool_use", id, name: "grep", input: { q: "same" } }) as never; + const both = anthropicToCore({ + model: "claude-x", + messages: [ + { role: "user", content: "go" }, + { role: "assistant", content: [call("t1"), call("t2")] }, + ], + }).msgs; + const onlySecond = anthropicToCore({ + model: "claude-x", + messages: [ + { role: "user", content: "go" }, + { role: "assistant", content: [call("t2")] }, + ], + }).msgs; + const bothCalls = both.filter((m) => m.contentType === "tool-call").map((m) => m.id); + const soloCall = onlySecond.filter((m) => m.contentType === "tool-call").map((m) => m.id); + assert.match(bothCalls[0]!, /^h_[0-9a-f]+$/, "first twin takes the bare base id"); + assert.match(bothCalls[1]!, /_1$/, "second twin takes the _1 suffix"); + // Deleting the FIRST twin: the survivor's base id is unchanged; only its + // suffix renumbers (_1 → bare). A deleted sibling is local damage, never a + // global re-key of the conversation. + assert.equal(bothCalls[1]!.replace(/_1$/, ""), soloCall[0]!); + assert.notEqual(bothCalls[1], soloCall[0], "suffix renumbers when the earlier twin disappears"); +}); diff --git a/kernel/tests/media-payload.test.ts b/kernel/tests/media-payload.test.ts index b3d77b6327..921a2bb5d5 100644 --- a/kernel/tests/media-payload.test.ts +++ b/kernel/tests/media-payload.test.ts @@ -84,10 +84,11 @@ test("openai: [text, file] round-trips the file part verbatim", () => { ]); const { msgs } = openaiToCore(body); assert.equal(msgs.length, 1); - // Pre-existing stringContent semantics: array entries join with "\n" and - // non-text entries contribute "" — same as today's [text, image] messages. - // The id derives from this text, so the shape must not change. - assert.equal(msgs[0]?.text, "what is in this file?\n"); + // #3642 canonical granularity: the text part is its own core carrying the + // part's bytes verbatim (no "\n" joining anymore); the opaque file part + // rides the preceding text core's plural sidecar so the rebuild keeps the + // wire part order. The id derives from the part text. + assert.equal(msgs[0]?.text, "what is in this file?"); assert.deepEqual( msgs[0]?.rawOpenaiContentParts, [FILE_PART], @@ -148,14 +149,22 @@ test("openai: single data-URL image keeps legacy singular sidecar shape", () => }, ]); const { msgs } = openaiToCore(body); - assert.ok(msgs[0]?.rawOpenaiContent, "singular sidecar preserved"); - assert.equal( - msgs[0]?.rawOpenaiContentParts, + // #3642: [text, image] splits into a text core + an image core carrying + // the canonical "[image]" marker; the singular raw sidecar lives on the + // image core. The rebuild merges the run back into ONE message with the + // marker never re-rendered as a text part. + assert.equal(msgs.length, 2); + assert.equal(msgs[0]?.text, "look"); + assert.equal(msgs[0]?.rawOpenaiContent, undefined); + assert.equal(msgs[1]?.text, "[image]"); + assert.ok(msgs[1]?.rawOpenaiContent, "singular sidecar preserved"); + assert.equal( + msgs[1]?.rawOpenaiContentParts, undefined, "no plural sidecar for a lone image", ); - assert.equal(msgs[0]?.imageBase64, IMG_DATA); - assert.equal(msgs[0]?.imageMediaType, "image/png"); + assert.equal(msgs[1]?.imageBase64, IMG_DATA); + assert.equal(msgs[1]?.imageMediaType, "image/png"); const content = rebuiltUserContent(msgs); assert.ok(Array.isArray(content)); @@ -177,7 +186,12 @@ test("openai: multi-image plural sidecar unchanged", () => { }, ]); const { msgs } = openaiToCore(body); - assert.equal(msgs[0]?.rawOpenaiContentParts?.length, 2); + // #3642: each image is its own core with its singular raw sidecar; the + // rebuild merges the [text, image, image] run back into one message. + assert.equal(msgs.length, 3); + assert.equal(msgs[0]?.text, "two pics"); + assert.ok(msgs[1]?.rawOpenaiContent); + assert.ok(msgs[2]?.rawOpenaiContent); const content = rebuiltUserContent(msgs); assert.ok(Array.isArray(content)); assert.deepEqual( diff --git a/kernel/tests/wire-bili-message-roundtrip.test.ts b/kernel/tests/wire-bili-message-roundtrip.test.ts index da96b52ebe..7a343b58d7 100644 --- a/kernel/tests/wire-bili-message-roundtrip.test.ts +++ b/kernel/tests/wire-bili-message-roundtrip.test.ts @@ -284,10 +284,13 @@ test("openai: user image_url content part is restored", () => { ], }; const { msgs } = openaiToCore(body); + // #3642: [text, image] splits into a text core + an image core (canonical + // "[image]" marker) carrying the singular raw sidecar + parsed media. assert.ok(msgs[0]?.text?.startsWith("what is this?"), "text preserved"); - assert.equal(msgs[0]?.imageMediaType, "image/png"); - assert.equal(msgs[0]?.imageBase64, IMG_DATA); - assert.ok(msgs[0]?.rawOpenaiContent, "rawOpenaiContent sidecar stored"); + assert.equal(msgs[1]?.text, "[image]"); + assert.equal(msgs[1]?.imageMediaType, "image/png"); + assert.equal(msgs[1]?.imageBase64, IMG_DATA); + assert.ok(msgs[1]?.rawOpenaiContent, "rawOpenaiContent sidecar stored"); const rebuilt = coreToOpenai(msgs); const u = rebuilt[0]!; assert.equal(u.role, "user"); @@ -333,11 +336,14 @@ test("openai: user message with multiple image_url parts round-trips ALL images" ], }; const { msgs } = openaiToCore(body); - assert.ok( - msgs[0]?.rawOpenaiContentParts, - "multi-image sidecar stored (not singular rawOpenaiContent)", + // #3642: each image is its own core with its singular raw sidecar; the + // run rebuild emits all three in order. + assert.equal(msgs.length, 4); + assert.ok(msgs.slice(1).every((m) => m.rawOpenaiContent)); + assert.deepEqual( + msgs.slice(1).map((m) => m.text), + ["[image]", "[image]", "[image]"], ); - assert.equal((msgs[0]?.rawOpenaiContentParts ?? []).length, 3); const rebuilt = coreToOpenai(msgs); const u = rebuilt[0]!; assert.ok(Array.isArray(u.content), "content rebuilt as array"); @@ -399,17 +405,19 @@ test("openai: single remote-URL image_url round-trips via rawOpenaiContent", () ], }; const { msgs } = openaiToCore(body); + // #3642: text core + image core (remote URL: no base64/media split). + assert.equal(msgs.length, 2); assert.ok( - msgs[0]?.rawOpenaiContent, + msgs[1]?.rawOpenaiContent, "rawOpenaiContent sidecar stored for a single remote-URL image", ); assert.equal( - msgs[0]?.imageBase64, + msgs[1]?.imageBase64, undefined, "no base64 split for non-data URLs", ); assert.equal( - msgs[0]?.imageMediaType, + msgs[1]?.imageMediaType, undefined, "no mediaType split for non-data URLs", ); @@ -448,7 +456,8 @@ test("openai: image-only remote-URL user message survives rebuild", () => { ], }; const { msgs } = openaiToCore(body); - assert.equal(msgs[0]?.text, ""); + // #3642: a lone image part is an image core with the canonical marker. + assert.equal(msgs[0]?.text, "[image]"); assert.ok(msgs[0]?.rawOpenaiContent, "sidecar stored despite empty text"); const rebuilt = coreToOpenai(msgs); const u = rebuilt[0]!; @@ -491,11 +500,13 @@ test("openai: mixed data-URL + remote-URL multi-image message preserves ALL imag ], }; const { msgs } = openaiToCore(body); - assert.ok(msgs[0]?.rawOpenaiContentParts, "multi-image sidecar stored"); - assert.equal( - (msgs[0]?.rawOpenaiContentParts ?? []).length, - 2, - "BOTH images collected (remote included)", + // #3642: text core + one image core per image (each with its singular + // raw sidecar — data-URL and remote alike). + assert.equal(msgs.length, 3); + assert.ok(msgs.slice(1).every((m) => m.rawOpenaiContent)); + assert.deepEqual( + msgs.slice(1).map((m) => m.imageBase64), + [IMG_DATA, undefined], ); const rebuilt = coreToOpenai(msgs); const u = rebuilt[0]!; @@ -526,10 +537,14 @@ test("responses: user input_image is restored via rawResponsesItem", () => { ], }; const { msgs } = responsesToCore(body); + // #3642: [text, image] splits into a text core + an image core; the image + // core carries the raw item + parsed media, and the run rebuilds as ONE + // message (raw verbatim when it fully covers the run). assert.ok(msgs[0]?.text?.startsWith("see this"), "text preserved"); - assert.equal(msgs[0]?.imageMediaType, "image/png"); - assert.equal(msgs[0]?.imageBase64, IMG_DATA); - assert.ok(msgs[0]?.rawResponsesItem, "rawResponsesItem sidecar stored"); + assert.equal(msgs[1]?.text, "[image]"); + assert.equal(msgs[1]?.imageMediaType, "image/png"); + assert.equal(msgs[1]?.imageBase64, IMG_DATA); + assert.ok(msgs[1]?.rawResponsesItem, "rawResponsesItem sidecar stored"); const rebuilt = coreToResponses(msgs); assert.equal(rebuilt.length, 1); const m = rebuilt[0] as { type: string; content: unknown }; @@ -640,10 +655,13 @@ test("responses: multi-text-part + image rebuild emits no empty input_text (#308 ], }; const proj = responsesToCore(body); - assert.equal(proj.msgs.length, 1, "single user message tracked"); + assert.equal(proj.msgs.length, 3, "one core per content part"); const src = proj.msgs[0]!; - // Mutate .text to force patchOriginalItem onto the patchTextParts path. - const next = [{ ...src, text: `${src.text}\n[m00001]` }]; + // Mutate .text to force the split-slot rebuild onto the filtered path. + const next = [ + { ...src, text: `${src.text}\n[m00001]` }, + ...proj.msgs.slice(1), + ]; const rebuilt = patchResponsesInput(proj, next) as ResponseInputItem[]; const item = rebuilt.find((i) => i.type === "message") as ResponseInputMessage; const content = item.content as ResponseContentPart[]; @@ -660,9 +678,9 @@ test("responses: multi-text-part + image rebuild emits no empty input_text (#308 (p): p is { type: "input_text" | "output_text"; text: string } => p.type === "input_text" || p.type === "output_text", ); - assert.equal(textParts.length, 1, "secondary text slots dropped, not blanked"); - assert.match(textParts[0]!.text, /look at this/, "first slot keeps all text"); - assert.match(textParts[0]!.text, /describe it/, "first slot keeps all text"); + assert.equal(textParts.length, 2, "each text part is its own slot"); + assert.match(textParts[0]!.text, /look at this/, "first slot keeps its text"); + assert.match(textParts[1]!.text, /describe it/, "second slot keeps its text"); assert.ok(content.some((p) => p.type === "input_image"), "image preserved in place"); }); @@ -726,12 +744,12 @@ test("responses: multi-input_image user item takes the [image] placeholder", () ], }; const { msgs } = responsesToCore(body); - assert.equal(msgs.length, 1); - assert.equal( - msgs[0]?.text, - "[image]", - 'no-text multi-image item uses the placeholder (not the "\\n" join artifact)', + assert.equal(msgs.length, 2); + assert.deepEqual( + msgs.map((m) => m.text), + ["[image]", "[image]"], ); + assert.notEqual(msgs[0]!.id, msgs[1]!.id, "ClusterCounter disambiguates"); const rebuilt = coreToResponses(msgs); assert.deepEqual(rebuilt[0], body.input[0], "raw item re-emitted verbatim"); }); @@ -753,16 +771,9 @@ test("responses: whitespace-only text plus image keeps its exact text", () => { ], }; const { msgs } = responsesToCore(body); - assert.equal(msgs.length, 1); - assert.notEqual( - msgs[0]?.text, - "[image]", - "whitespace text counts as real text, not the placeholder", - ); - assert.ok( - msgs[0]?.text?.startsWith(" "), - "exact whitespace text preserved (kernel joins non-text parts with \\n)", - ); + assert.equal(msgs.length, 2); + assert.equal(msgs[0]?.text, " ", "exact whitespace text preserved"); + assert.equal(msgs[1]?.text, "[image]"); const rebuilt = coreToResponses(msgs); assert.deepEqual( rebuilt[0], @@ -803,7 +814,7 @@ test("responses: input_image with non-string image_url is tracked without sideca // (same derived id), so the already-working shape does not churn on // upgrade; switching it to "[image]" would move its id for no functional // gain. Verbatim round-trip holds either way (#187/#291 review). -test("responses: empty input_text plus image keeps its join-artifact text and stable id", () => { +test("responses: empty input_text plus image folds to the image core", () => { const body: ResponsesRequestBody = { input: [ { @@ -818,11 +829,7 @@ test("responses: empty input_text plus image keeps its join-artifact text and st }; const { msgs } = responsesToCore(body); assert.equal(msgs.length, 1); - assert.equal( - msgs[0]?.text, - "\n", - 'empty text part joins to the "\\n" artifact, not the placeholder', - ); + assert.equal(msgs[0]?.text, "[image]", "image core only"); const pureImage: ResponsesRequestBody = { input: [ { @@ -832,13 +839,24 @@ test("responses: empty input_text plus image keeps its join-artifact text and st }, ], }; - assert.notEqual( + assert.equal( msgs[0]?.id, responsesToCore(pureImage).msgs[0]?.id, - "distinct from the pure image-only placeholder id", + "identical to the pure image-only id (same logical content)", ); const rebuilt = coreToResponses(msgs); - assert.deepEqual(rebuilt[0], body.input[0], "verbatim round-trip holds"); + // The empty input_text part is intentionally dropped: strict Responses + // backends reject empty input_text.text with a 400 (#3089), and the part + // carries no content. The image survives verbatim. + assert.deepEqual( + rebuilt[0], + { + type: "message", + role: "user", + content: [{ type: "input_image", image_url: DATA_URL }], + }, + "image part re-emitted verbatim, empty text part dropped", + ); }); // 7. Responses reasoning is routed into the compression pipeline (NOT the diff --git a/kernel/tests/wire-google.test.ts b/kernel/tests/wire-google.test.ts index 2600359f55..399f1ca112 100644 --- a/kernel/tests/wire-google.test.ts +++ b/kernel/tests/wire-google.test.ts @@ -112,9 +112,14 @@ test("google: systemInstruction is hoisted out of the fold space", () => { test("google: model maps to assistant, user to user, thought parts to reasoning", () => { const { msgs } = googleToCore(ompBody()); + // #3642 canonical granularity: every content part is its own core (the + // anthropic codec's model), so the [text, image] user content yields two + // user cores and the model turn yields reasoning + text (the opaque + // executableCode part rides the preceding text core's raw sidecar). assert.deepEqual( msgs.map((m) => `${m.role}/${m.contentType}`), [ + "user/text", "user/text", "assistant/reasoning", "assistant/text", @@ -124,13 +129,15 @@ test("google: model maps to assistant, user to user, thought parts to reasoning" ], ); assert.equal(msgs[0]!.text, "run the tests"); - assert.equal(msgs[0]!.imageMediaType, "image/png"); - assert.equal(msgs[0]!.imageBase64, IMG); - assert.equal(msgs[1]!.text, "List the files first."); - assert.equal(msgs[3]!.toolName, "bash"); - assert.equal(msgs[3]!.toolCallId, "call_list"); - assert.equal(msgs[3]!.text, '{"cmd":"ls"}'); - assert.equal(msgs[4]!.text, '{"output":"a.ts\\nb.ts"}'); + assert.equal(msgs[0]!.imageMediaType, undefined); + assert.equal(msgs[1]!.text, "[image]"); + assert.equal(msgs[1]!.imageMediaType, "image/png"); + assert.equal(msgs[1]!.imageBase64, IMG); + assert.equal(msgs[2]!.text, "List the files first."); + assert.equal(msgs[4]!.toolName, "bash"); + assert.equal(msgs[4]!.toolCallId, "call_list"); + assert.equal(msgs[4]!.text, '{"cmd":"ls"}'); + assert.equal(msgs[5]!.text, '{"output":"a.ts\\nb.ts"}'); }); test("google: functionResponse parts become tool cores ahead of the user core", () => { @@ -291,6 +298,9 @@ test("google: sidecar signature and image rebuild a core that lost its raw parts // Simulate a core whose raw parts array is gone (a host-set message), where // only the scalar sidecars remain. const rebuilt = coreToGoogle(msgs.map(({ rawGoogleParts, ...rest }) => rest)); + // #3642: the [text, image] user run rebuilds as ONE content with the + // text part first and the image core's sidecar re-rendered as inlineData + // (the "[image]" marker never re-renders as a text part). assert.deepEqual(rebuilt[0]!.parts, [ { text: "run the tests" }, { inlineData: { mimeType: "image/png", data: IMG } }, @@ -319,7 +329,7 @@ test("google: sidecar signature and image rebuild a core that lost its raw parts functionResponse: { name: "bash", response: { output: "second, no id" }, - id: msgs[5]!.toolCallId, + id: msgs[6]!.toolCallId, }, }, ]); diff --git a/kernel/tests/wire-tool-result-toolname.test.ts b/kernel/tests/wire-tool-result-toolname.test.ts index eafbf28994..ec757e1dbf 100644 --- a/kernel/tests/wire-tool-result-toolname.test.ts +++ b/kernel/tests/wire-tool-result-toolname.test.ts @@ -9,7 +9,7 @@ import { responsesToCore, type ResponsesRequestBody, } from "../src/wire/responses.js"; -import { deriveMessageId } from "../src/wire/message-id.js"; +import { deriveCanonicalMessageId } from "../src/wire/identity.js"; import { applyAbsorb, isAbsorbCandidate } from "../src/absorb.js"; import { isMessageProtected, isNeverPreserveRecent } from "../src/protected.js"; import { defaultConfig } from "../src/config.js"; @@ -107,10 +107,13 @@ test("anthropicToCore result ids stay stable under the toolName backfill", () => const bashResult = msgs.find( (m) => m.contentType === "tool-result" && m.toolCallId === "tu_1", )!; + // #3642: canonical seed = toolName + text (the wire pairing id is out — + // transport plumbing, not content). The backfill stability invariant is + // unchanged: the seeded toolName is exactly the paired backfill. assert.equal( bashResult.id, - deriveMessageId("tool", "tool-result", ANTHROPIC_TEXT, { - toolCallId: "tu_1", + deriveCanonicalMessageId("tool", "tool-result", ANTHROPIC_TEXT, { + toolName: "bash", }), ); }); @@ -328,10 +331,11 @@ test("responsesToCore output ids stay stable under the toolName backfill", () => const fnResult = msgs.find( (m) => m.contentType === "tool-result" && m.toolCallId === "fc_1", )!; + // #3642: canonical seed = toolName + text (wire call_id out of the seed). assert.equal( fnResult.id, - deriveMessageId("tool", "tool-result", "z".repeat(480), { - toolCallId: "fc_1", + deriveCanonicalMessageId("tool", "tool-result", "z".repeat(480), { + toolName: "bash", }), ); }); diff --git a/tests/bili-message-roundtrip.test.ts b/tests/bili-message-roundtrip.test.ts index fc8026ddb2..72089f8228 100644 --- a/tests/bili-message-roundtrip.test.ts +++ b/tests/bili-message-roundtrip.test.ts @@ -143,10 +143,12 @@ test("openai: user image_url content part is restored", () => { ], }; const { msgs } = openaiToCore(body); - assert.ok(msgs[0]?.text?.startsWith("what is this?"), "text preserved"); - assert.equal(msgs[0]?.imageMediaType, "image/png"); - assert.equal(msgs[0]?.imageBase64, IMG_DATA); - assert.ok(msgs[0]?.rawOpenaiContent, "rawOpenaiContent sidecar stored"); + assert.equal(msgs.length, 2, "one core per content part"); + assert.equal(msgs[0]?.text, "what is this?", "text core is verbatim"); + assert.equal(msgs[1]?.text, "[image]", "image core uses the canonical marker"); + assert.equal(msgs[1]?.imageMediaType, "image/png"); + assert.equal(msgs[1]?.imageBase64, IMG_DATA); + assert.ok(msgs[1]?.rawOpenaiContent, "rawOpenaiContent sidecar stored"); const rebuilt = coreToOpenai(msgs); const u = rebuilt[0]!; assert.equal(u.role, "user"); @@ -174,10 +176,12 @@ test("responses: user input_image is restored via rawResponsesItem", () => { ], }; const { msgs } = responsesToCore(body); - assert.ok(msgs[0]?.text?.startsWith("see this"), "text preserved"); - assert.equal(msgs[0]?.imageMediaType, "image/png"); - assert.equal(msgs[0]?.imageBase64, IMG_DATA); - assert.ok(msgs[0]?.rawResponsesItem, "rawResponsesItem sidecar stored"); + assert.equal(msgs.length, 2, "one core per content part"); + assert.equal(msgs[0]?.text, "see this", "text core is verbatim"); + assert.equal(msgs[1]?.text, "[image]", "image core uses the canonical marker"); + assert.equal(msgs[1]?.imageMediaType, "image/png"); + assert.equal(msgs[1]?.imageBase64, IMG_DATA); + assert.ok(msgs[1]?.rawResponsesItem, "rawResponsesItem sidecar stored"); const rebuilt = coreToResponses(msgs); assert.equal(rebuilt.length, 1); const m = rebuilt[0] as { type: string; content: unknown }; diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 4b40bac467..b6132f66d7 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -80,22 +80,22 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "836b7f2e075421114f0b3c31c9d863af3861ab8200d632948b5a21b2992d9c1c", + "stateDigest": "e0ff2fa2fce3c03d053483563b77b1bc633f1c8920ef77ab7c6cf337a5cca90d", "sectionDigests": { - "state": "fae14b25a41f5844", + "state": "b02bd0587025fe25", "blockContents": "dc7d13d250c94bb4", "stats": "cff3c2546898aeb3", - "metadata": "c6e132de1f0769b1" + "metadata": "eaaf5c282a351e4b" }, "stateKeyDigests": { - "blocks": "1cbe172309e4490d", - "messageRefs": "44a70eda324db561", + "blocks": "3a454b9a16e4f0b9", + "messageRefs": "3d09c9004582bdbc", "nudge": "9d0959874e89cfdd", "stats": "23312f7d5a0b9e27", "nextBlockId": "d4735e3a265e16ee", "nextRunId": "d4735e3a265e16ee", "tokenSnapshot": "7c8dbc981fdfcbd6", - "lastPassIds": "60bc6ebf615ea647", + "lastPassIds": "f13452893a1b4980", "hiddenOrphanRefs": "4f53cda18c2baa0c", "deadRefs": "74234e98afe7498f", "terminalStreak": "5feceb66ffc86f38", @@ -112,8 +112,8 @@ "coveredRefSample": [ "h_e40995eea3f07d55", "h_5e8c02240a65740e", - "h_3926874d9d58f065", - "h_0fd8e3acc55625ec", + "h_a9607fb18a2fa39d", + "h_7fab6026ce5a72e7", "h_b5b946fa2455d77e", "h_e40995eea3f07d55_1", "h_ce9559b8e1773afc", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index e76359cce1..a9c1c44b91 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -8,94 +8,94 @@ }, { "i": 1, - "sha": "ce55234b88db48e3d7b0b52085963b727e30e485a39af819594fce77097b7207", + "sha": "82330334ab7e3fae489d3ceda70874d5d288f6f10351d480e8931825a331c2ef", "bytes": 31995 }, { "i": 2, - "sha": "1c6a2f7169fee01285819f84146bee61aef7934b68cf5c16838b3b7a0960fff3", - "bytes": 37807 + "sha": "44168aba9dfec04562f94884c27b5ef91efa882d55eef0aa72d5dd9d6a2a8f68", + "bytes": 37829 }, { "i": 3, - "sha": "8bbfe9c4af21ba7626f9c2abbe94a287730448c09bc544955d4a16e5fb0e0cde", - "bytes": 46385 + "sha": "ecb0a1c6b39da3ad25b2d811ae9f01bd4462b443acd72d719a101953d9505421", + "bytes": 46373 }, { "i": 4, - "sha": "0b16037a50efcb87bbdfdc321bf75f9847771f1552bed7e9ccb0f0a9a1a39fa7", - "bytes": 52197 + "sha": "0a101eddf0773aa82f5ae3e17216c6d42b772ef873eaa51ab816fbbc6460b270", + "bytes": 52207 }, { "i": 5, - "sha": "1a7edb8f445334cfd25e0ca159060a54a8c37bff53d9f0a3efd4acd2ca0059da", - "bytes": 60775 + "sha": "4de288374f892fc4c63d8e88d4127dcfdc64ee5fa232338dcdfdedb50a4bcebe", + "bytes": 60751 }, { "i": 6, - "sha": "afd6a636915f86a3cb7064fc807a8be72f12f86b5f90b1b9147320c526aab92b", - "bytes": 66587 + "sha": "8cf5801a79bfaee1272ded800f990858235a72f69a5be5ef5127e6ff271971eb", + "bytes": 66585 }, { "i": 7, - "sha": "ae088e5256b0e6b5b771f382e55bd3c5b8f6f3df81194721f3284f62f3b7e392", - "bytes": 75165 + "sha": "6727f67d8465d80cb7b02025e4a43c7fbfe87194be31b1e1af664e82b2e28146", + "bytes": 75129 }, { "i": 8, - "sha": "5623e4626136cb65156548acf8c801770f445709aa5c1d57f0de056441230162", - "bytes": 80977 + "sha": "8ff52f654ec7d4cb9637ac84b3efa7457846063570cdf5756056ade1c3bbaf10", + "bytes": 80963 }, { "i": 9, - "sha": "187920f4d4b1cc5ce43a3e52027008e193441ed834026669cd5a37dc4034ee17", - "bytes": 50478 + "sha": "72542c69ec5cbfdef8bdea1deb83b881d35707a4997ff4ea64ee71f66af5b037", + "bytes": 50470 }, { "i": 10, - "sha": "89ab8419c57248c79777d523f0c888ea798bdef146ace4f52b14a0d672d593b2", - "bytes": 57879 + "sha": "f32352ba5fc23d887774e5dcd7f121912cbb50a952dc5d66ef034adb437f9c79", + "bytes": 57855 }, { "i": 11, - "sha": "a830324598bd68492380344532152dae98ae91fca26a9fb9917cf2dc16450fd9", - "bytes": 63614 + "sha": "68f253bcd35cca495e996863eeb564ae6649f9ec2864c401c21e8ac1babbca6f", + "bytes": 63612 }, { "i": 12, - "sha": "7e2fb54ac5c850cd92d0e1a63585f729d6883426357677f394674fb49f1c2068", - "bytes": 77660 + "sha": "309426c88866153bfea341d7cc0ea9f8d13b7c59ae57c84b45506d327e522fcc", + "bytes": 77621 }, { "i": 13, - "sha": "6e09bf973ab1de4ea1e6c350e169147de681237522ed2e414d967cefc925d59c", - "bytes": 83395 + "sha": "db5e9c9781f78f41a186836665a090e680524df7e466b2447e07c5bd6417e012", + "bytes": 83378 }, { "i": 14, - "sha": "42209cfddbd623dc6ee3f23564ab2f57e3d711f24b1eabbe5cd16d27dbed2bf4", - "bytes": 91949 + "sha": "ead3aa36c6f4a72d24e35ad7d1017084d2bef1e08a209dc5e95ec0772182a91a", + "bytes": 91898 } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "a55afe00bb22c39b31935c75a3fcb969a98325d2227e75ae703db083427d02e5", + "stateDigest": "2f22b4aa68e49f359c058315b5ce1c060ec80722a19d28c356a3b24cda045e60", "sectionDigests": { - "state": "d5bc5306f840723d", - "blockContents": "94a58f96d7c5cfc3", - "stats": "59842ce5415d3a4d", - "metadata": "fbc8e22bb6bde73c" + "state": "eaa58407256b986b", + "blockContents": "039ad3602f605afe", + "stats": "711b07a24813e7a8", + "metadata": "5b9eb00eeaeab92d" }, "stateKeyDigests": { - "blocks": "9aa73e5fb9f93d6c", - "messageRefs": "0e378dff25ad2808", - "nudge": "b539bf305b125cb7", - "stats": "f85b2ad1a3b86e2f", + "blocks": "5426f9d515432b43", + "messageRefs": "48135bb278ef6a47", + "nudge": "592c25cb5bcb1f87", + "stats": "48f9f19ca44eacdb", "nextBlockId": "d4735e3a265e16ee", "nextRunId": "d4735e3a265e16ee", "tokenSnapshot": "5080181ee1a25a8c", - "lastPassIds": "7bd0a53d26e6aba4", + "lastPassIds": "d18ace2914339891", "hiddenOrphanRefs": "4f53cda18c2baa0c", "deadRefs": "74234e98afe7498f", "terminalStreak": "5feceb66ffc86f38", @@ -110,14 +110,14 @@ "deadRefs": 0, "blockContents": 1, "coveredRefSample": [ + "h_631ad4fd8d101d1d", "h_e40995eea3f07d55", - "h_7e24410d255cd1b2", - "h_d5c99f700dd05066", + "h_a9607fb18a2fa39d", + "h_7fab6026ce5a72e7", "h_b5b946fa2455d77e", "h_e40995eea3f07d55_1", "h_ce9559b8e1773afc", - "h_e40995eea3f07d55_2", - "h_63c0518deb63d807" + "h_e40995eea3f07d55_2" ] } } diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index f962a1fbdc..1444b5229c 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -80,22 +80,22 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "53f625d72df68bf581e2858ff73c95e619d86b92387a40b1707020ac4e2c6cdf", + "stateDigest": "77aee26da8f7dae7e31edcceb48624ea7a22fdf8f3683d8baa1013d46663d974", "sectionDigests": { - "state": "2a2af1c92fcb4f88", + "state": "a9b9531d952b48d8", "blockContents": "473264d9df8ba338", - "stats": "660f1797bf2044c4", - "metadata": "f6c8d7d13b4ae5d4" + "stats": "e358dc992395fc7e", + "metadata": "928ce8bcd75794ae" }, "stateKeyDigests": { - "blocks": "8e14f0879bf7ebd7", - "messageRefs": "19bfd6646b3f6f78", + "blocks": "0cfd0b9c0fc53073", + "messageRefs": "61ee2b1599995c19", "nudge": "1aafe7b2e5e10a1f", "stats": "e4ee14ae39539ff8", "nextBlockId": "d4735e3a265e16ee", "nextRunId": "d4735e3a265e16ee", "tokenSnapshot": "5080181ee1a25a8c", - "lastPassIds": "011421d6cea66b93", + "lastPassIds": "1e3edaf045fbb7d1", "hiddenOrphanRefs": "4f53cda18c2baa0c", "deadRefs": "74234e98afe7498f", "terminalStreak": "5feceb66ffc86f38", @@ -114,8 +114,8 @@ "h_e40995eea3f07d55_1", "h_ce9559b8e1773afc", "h_e40995eea3f07d55_2", - "h_33b455a3c181de3d", - "h_ff29aa6a99543643", + "h_f1a111d468a227a1", + "h_4b06767bafd163bb", "h_fffacd573c308745", "h_e40995eea3f07d55_3" ] diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index c87c591967..39fc5c405e 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -48,7 +48,7 @@ }, { "i": 9, - "sha": "f7cbbec3f16a0705e0dbb09f4a3009935cd51d711b235154733414f7bcb63da3", + "sha": "daebf0f41596a36f6ac771914b25faaec133c64d1c57af6bad79522d2a9d017b", "bytes": 51518 }, { @@ -80,22 +80,22 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "c61887b512e6322382ab4dd4b7c53586bf0fa2cd699775f19b19fe6966e23490", + "stateDigest": "c156e17282e87af3759e263862f84fb8f99c6546b26d9703ceea5e8609cf8c75", "sectionDigests": { - "state": "6546d2e5f2826284", - "blockContents": "8460729b2654a816", - "stats": "e3a3aeb598c183ff", - "metadata": "bfe50c77f8a85bcb" + "state": "a66c579fdfccd2e5", + "blockContents": "b6701268a0bdcd8a", + "stats": "67e33bcc612fdcc1", + "metadata": "c0d6bf81ec0923cd" }, "stateKeyDigests": { - "blocks": "7fff1dfb54a617cf", - "messageRefs": "c229d4795a0c9ac1", - "nudge": "5c9051299cfa292d", - "stats": "da55f309d070d963", + "blocks": "34b0dc17f8c24595", + "messageRefs": "68b7f104b71dcc22", + "nudge": "a5d2dcd7dc22a2e4", + "stats": "352f12b97e7f7c2d", "nextBlockId": "d4735e3a265e16ee", "nextRunId": "d4735e3a265e16ee", "tokenSnapshot": "003006fbddac1a2b", - "lastPassIds": "773e9c88c38f65c3", + "lastPassIds": "de5c5da67428b09f", "hiddenOrphanRefs": "4f53cda18c2baa0c", "deadRefs": "74234e98afe7498f", "terminalStreak": "5feceb66ffc86f38", @@ -110,12 +110,12 @@ "deadRefs": 0, "blockContents": 1, "coveredRefSample": [ - "h_97e0b0cf63832812", + "h_0dae8f447ab30c78", "h_e40995eea3f07d55", - "h_7e24410d255cd1b2", - "h_d5c99f700dd05066", + "h_a9607fb18a2fa39d", + "h_7fab6026ce5a72e7", "h_b5b946fa2455d77e", - "h_5ad3e8d0f2991515", + "h_42145889dce8b677", "h_e40995eea3f07d55_1", "h_ce9559b8e1773afc" ] From 6663f89ab71646f4aa8821e5834ef21a6e832b9c Mon Sep 17 00:00:00 2001 From: dog Date: Sun, 18 Oct 2026 15:19:38 +0800 Subject: [PATCH 09/14] kernel: pin the opening wire message's whole core run + fix stale google unwrap assertion (#3642 follow-ups) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of #3658 found two must-fix items; both fixed here. 1. Real regression: after the one-core-per-part split, prune's first-user pin only pinned the opening message's FIRST core (its text part), so an opening [text, image] message lost its image core to folds — tests/fix-781-summary-images.test.ts (responses variant) failed with "0 !== 1" surviving images. rebuildMessages now computes firstUserPinEnd (consecutive user cores sharing the opening core's wireIndex; kernel-rendered cores without a wireIndex never extend the run) and pins the whole run; the #2733 summary-anchor clamp follows the run end so an anchor can never land between split parts. Run length 1 keeps the legacy shape byte-for-byte. Two new state-prune tests. 2. Stale assertion: wire-google.test.ts still expected the pre-unwrap tool-result shape '{"output":"a.ts\nb.ts"}' — updated to the new plain 'a.ts\nb.ts' (this was the static-gates red on head 41653801). Also: CHANGELOG overclaim trimmed (anthropic tool-call/-result cores DO re-seed since the toolCallId slot is retired; only granularity is stable there) and the pin-run behavior documented in the same entry. kernel: version 0.0.131 → 0.0.132 (src/prune.ts), lock regenerated. --- CHANGELOG.md | 2 +- kernel/package-lock.json | 4 +-- kernel/package.json | 2 +- kernel/src/prune.ts | 44 +++++++++++++++++++++++++++-- kernel/tests/state-prune.test.ts | 48 ++++++++++++++++++++++++++++++++ kernel/tests/wire-google.test.ts | 4 ++- 6 files changed, 97 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 203996d71d..bdcd3e417e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Changes -- **protocol-stable core identity: the same logical conversation now yields byte-identical core message ids in all four dialects (#3642).** Wire codecs previously baked dialect accidents into message identity: anthropic used `"[image]"` as image text while openai deliberately kept `""` ("switching the seed would move every pre-existing shape"), tool-call ids embedded the provider's `toolCallId` bytes, raw JSON tool arguments hashed verbatim (key order = new id), and responses' empty-text reasoning cells seeded from the provider's volatile `item.id`. Switch protocols mid-session (opencode #3640: `/chat/completions` → `/v1/messages`) and ~20-68% of ids churned, tripping the fold-destroyed misdiagnosis #3641 just class-fixed. Identity now derives from a canonical form shared by all four codecs: a new `kernel/src/wire/identity.ts` (canonicalToolArgs: JSON args parsed → recursive key sort → compact stringify, verbatim for non-JSON; canonicalOpaqueSeed: provider `item.id`/`status` stripped; image marker `[image]` everywhere), the `toolCallId` slot retired from the seed (cluster suffixes disambiguate same-shape calls, deleting one sibling is local damage), and granularity normalized to one core per content part (anthropic's existing shape — dsh/anthropic lanes take zero re-keying; openai/responses/google split runs, with accumulators re-merging on rebuild so the wire stays byte-stable). Cross-protocol golden: text/tool/bash conversation → identical id arrays in all four dialects; canonical key-sort makes `{"file","line"}` == `{"line","file"}`. Image cells are now stamped `imagePart: true` at parse time (never text-compared — tag injection rewrites markers mid-render, which previously made image cores render as text and lose the real bytes, #767 billing fallout). Upgrade cost is one re-key of stored ids for openai/responses/google lanes; the #3641 reconcile tiers (word-bag claims, k-th image family pairing, toolCallId channel, epoch gate) absorb it without re-folding. +- **protocol-stable core identity: the same logical conversation now yields byte-identical core message ids in all four dialects (#3642).** Wire codecs previously baked dialect accidents into message identity: anthropic used `"[image]"` as image text while openai deliberately kept `""` ("switching the seed would move every pre-existing shape"), tool-call ids embedded the provider's `toolCallId` bytes, raw JSON tool arguments hashed verbatim (key order = new id), and responses' empty-text reasoning cells seeded from the provider's volatile `item.id`. Switch protocols mid-session (opencode #3640: `/chat/completions` → `/v1/messages`) and ~20-68% of ids churned, tripping the fold-destroyed misdiagnosis #3641 just class-fixed. Identity now derives from a canonical form shared by all four codecs: a new `kernel/src/wire/identity.ts` (canonicalToolArgs: JSON args parsed → recursive key sort → compact stringify, verbatim for non-JSON; canonicalOpaqueSeed: provider `item.id`/`status` stripped; image marker `[image]` everywhere), the `toolCallId` slot retired from the seed (cluster suffixes disambiguate same-shape calls, deleting one sibling is local damage), and granularity normalized to one core per content part (anthropic's existing shape — its lanes take zero granularity re-keying, though tool-call/tool-result cores still re-seed since the `toolCallId` slot is retired; openai/responses/google split runs, with accumulators re-merging on rebuild so the wire stays byte-stable). The kernel first-user pin now spans the opening message's whole core run, so a multimodal opening message keeps its image bytes through folds (#781/#2607). Cross-protocol golden: text/tool/bash conversation → identical id arrays in all four dialects; canonical key-sort makes `{"file","line"}` == `{"line","file"}`. Image cells are now stamped `imagePart: true` at parse time (never text-compared — tag injection rewrites markers mid-render, which previously made image cores render as text and lose the real bytes, #767 billing fallout). Upgrade cost is one re-key of stored ids for openai/responses/google lanes; the #3641 reconcile tiers (word-bag claims, k-th image family pairing, toolCallId channel, epoch gate) absorb it without re-folding. - **fold anchoring doctrine: order over hash — protocol switches, image offload, and host-side truncation stop being read as destruction (#3641, fixes #3588, fixes #3640).** Three real-world defect classes all shared one root: the fold's identity model keyed coverage to content hashes, so any transformation that re-mints ids without removing content (dsh's image-offload component rewriting image cells into placeholder text, opencode switching a mid-session request from `/chat/completions` to `/v1/messages`, or a host that compacts/truncates history itself) made every covered id look *destroyed*. Consequences: 7× false "compression substrate appears destroyed" escalations on healthy sessions (#3588), 298-id churn with unanchored summary blocks landing AFTER the user's newest question — the model answered five rounds of stale summaries instead of the question (#3640) — and the #2193 collapse streak on sanctioned host compaction. The reconcile pass now carries a shape tier (loose word-evidence pairing for prose cells, a flanked k-th discipline for evidence-less image cells), a residual census classifying unmatched ids as *transformed* (near-duplicate evidence exists), *absent* (content truly off the wire with the surviving wire a suffix-continuation of the episode baseline), or *drift* (nothing matches anywhere); only the third accrues the collapse streak, and a wire-frozen episode witness guarantees a foreign conversation can never quietly become the baseline mid-episode. Unresolved summary anchors clamp ahead of the newest user turn instead of at the array tail (the #3640 "stale summary after the question" shape), and a genuine total collapse now latches `foldEpochConfirmPending` and PAUSES folding (verbatim passthrough — nothing is deleted, the snapshot is kept) until the histories realign or the user confirms a new epoch, instead of escalating on a misdiagnosis. Single genuine edits below the mass bar still report drift honestly; the #1729/#2193 escalation itself stays (its error text now names the epoch gate). diff --git a/kernel/package-lock.json b/kernel/package-lock.json index 6b1cfc297d..c030136e7b 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.129", + "version": "0.0.132", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.129", + "version": "0.0.132", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index 56b69e7cff..e35f827715 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.131", + "version": "0.0.132", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", diff --git a/kernel/src/prune.ts b/kernel/src/prune.ts index 416406d096..5bbdfdea14 100644 --- a/kernel/src/prune.ts +++ b/kernel/src/prune.ts @@ -344,6 +344,13 @@ function pairSafeAnchorIndex( return safe; } +/** #3642: the wire message index a core was flattened from (set by every + * codec; kernel-rendered cores carry none). Declared locally so prune stays + * free of wire-module imports. */ +type WireIndexedCore = { wireIndex?: number }; +const wireIndexOf = (m: CoreMessage): number | undefined => + (m as WireIndexedCore).wireIndex; + function rebuildMessages( messages: CoreMessage[], coveredBases: Set, @@ -352,6 +359,28 @@ function rebuildMessages( knownSummaryIds: Set, headAnchors: boolean, ): CoreMessage[] { + // #3642: canonical granularity splits one wire message into one core per + // content part, so the opening user message can be a RUN of user cores + // (text + image + …) sharing one wireIndex. The first-user pin protects + // the OPENING WIRE MESSAGE, not merely its first core: a multimodal + // opening keeps its image bytes through folds (pre-split the merged core + // carried them in its sidecar and the single-core pin preserved both). + // Kernel-rendered cores (no wireIndex) never extend the run, and neither + // does an adjacent but different wire message. Run length 1 keeps the + // legacy shape byte-for-byte. + let firstUserPinEnd = firstUserIndex + 1; + if (firstUserIndex >= 0) { + const firstWire = wireIndexOf(messages[firstUserIndex]!); + if (firstWire !== undefined) { + while ( + firstUserPinEnd < messages.length && + messages[firstUserPinEnd]!.role === "user" && + wireIndexOf(messages[firstUserPinEnd]!) === firstWire + ) { + firstUserPinEnd++; + } + } + } const ix = anchors.length > 0 ? buildAnchorPairingIndex(messages) : null; const safeAnchors = anchors .map((anchor) => ({ @@ -371,9 +400,16 @@ function rebuildMessages( // head placement for summaries of older, absent material. .map((anchor) => ({ ...anchor, + // #2733/#3642: a summary may NEVER precede the pinned opening wire + // message — now measured to the END of the opening core run + // (firstUserPinEnd), so an anchor can never land between the opening + // message's split parts either. insertAt: headAnchors ? anchor.insertAt - : Math.max(anchor.insertAt, firstUserIndex + (anchor.unresolved === true ? 0 : 1)), + : Math.max( + anchor.insertAt, + anchor.unresolved === true ? firstUserIndex : firstUserPinEnd, + ), })) .sort((left, right) => left.insertAt - right.insertAt); const result: CoreMessage[] = []; @@ -388,8 +424,12 @@ function rebuildMessages( // PRECEDES the covered check below — swapping them would let a compress // range strip the last guaranteed user message off the wire. Strict // providers reject conversations with no user message. See DESIGN.md §8.1. + // #3642: the pin spans the opening wire message's whole core run. if (index === firstUserIndex && firstUserIndex >= 0) { - result.push(messages[index]!); + for (let k = index; k < firstUserPinEnd; k++) { + result.push(messages[k]!); + } + index = firstUserPinEnd - 1; continue; } if (isCovered(messages[index]!.id, coveredBases)) continue; diff --git a/kernel/tests/state-prune.test.ts b/kernel/tests/state-prune.test.ts index bcc68de1b0..cb6ad1a961 100644 --- a/kernel/tests/state-prune.test.ts +++ b/kernel/tests/state-prune.test.ts @@ -146,6 +146,54 @@ test("prune preserves first user message even when covered", () => { assert.equal(result[1]!.id, "m3"); }); +test("#3642: the first-user pin spans the opening wire message's whole core run", () => { + const state = createInitialState(); + state.blocks.push( + makeBlock({ blockId: "b1", effectiveMessageIds: ["m1", "m2", "m3"] }), + ); + // Canonical granularity splits one wire message into one core per part: + // the opening message is a text + image core RUN sharing wireIndex 0; + // m3 is a DIFFERENT wire user message. Everything is covered — the pin + // must keep the whole opening run (image bytes included) and still let + // the unrelated m3 fold away. + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: "m1", wireIndex: 0 }, + { + id: "m2", + role: "user", + contentType: "text", + text: "m2", + wireIndex: 0, + }, + { id: "m3", role: "user", contentType: "text", text: "m3", wireIndex: 1 }, + { id: "m4", role: "assistant", contentType: "text", text: "m4" }, + ].map((m) => m as CoreMessage); + const result = prune(messages, state, { injectSummaries: false }); + + assert.deepEqual( + result.map((m) => m.id), + ["m1", "m2", "m4"], + ); +}); + +test("#3642: kernel-rendered cores (no wireIndex) never extend the pin run", () => { + const state = createInitialState(); + state.blocks.push( + makeBlock({ blockId: "b1", effectiveMessageIds: ["m1", "m2"] }), + ); + const messages: CoreMessage[] = [ + { id: "m1", role: "user", contentType: "text", text: "m1", wireIndex: 0 }, + { id: "m2", role: "user", contentType: "text", text: "m2" }, + { id: "m3", role: "assistant", contentType: "text", text: "m3" }, + ].map((m) => m as CoreMessage); + const result = prune(messages, state, { injectSummaries: false }); + + assert.deepEqual( + result.map((m) => m.id), + ["m1", "m3"], + ); +}); + test("prune without summary injection only removes covered messages", () => { const state = createInitialState(); state.blocks.push(makeBlock({ blockId: "b1", effectiveMessageIds: ["m2"] })); diff --git a/kernel/tests/wire-google.test.ts b/kernel/tests/wire-google.test.ts index 399f1ca112..f35d0069b2 100644 --- a/kernel/tests/wire-google.test.ts +++ b/kernel/tests/wire-google.test.ts @@ -137,7 +137,9 @@ test("google: model maps to assistant, user to user, thought parts to reasoning" assert.equal(msgs[4]!.toolName, "bash"); assert.equal(msgs[4]!.toolCallId, "call_list"); assert.equal(msgs[4]!.text, '{"cmd":"ls"}'); - assert.equal(msgs[5]!.text, '{"output":"a.ts\\nb.ts"}'); + // #3658: google functionResponse single-key {output} objects unwrap into + // plain text — the tool-result core text is "a.ts\nb.ts", not a JSON blob. + assert.equal(msgs[5]!.text, "a.ts\nb.ts"); }); test("google: functionResponse parts become tool cores ahead of the user core", () => { From 3c940c585a3f8558386a38b960bfc2ae2cc9c385 Mon Sep 17 00:00:00 2001 From: awork Date: Sun, 18 Oct 2026 15:19:15 +0800 Subject: [PATCH 10/14] kernel: keep trailing opaque parts when flushing leading ones (#3658 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-part user split accumulates unknown parts seen AFTER a text core on lastTextCore.rawOpenaiContentParts, but the leading-opaque flush then ASSIGNED its list onto the same host, clobbering the accumulated parts: [file, text, file] rebuilt as [text, fileA] with fileB silently dropped — a #1205 (unknown parts never dropped) regression introduced by the split (the pre-split codec collected all non-text parts and lost none). Prepend instead of assign so both groups survive in relative order. Adds the [file, text, file] round-trip regression test to media-payload. --- kernel/src/wire/openai.ts | 24 +++++++++++++---------- kernel/tests/media-payload.test.ts | 31 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/kernel/src/wire/openai.ts b/kernel/src/wire/openai.ts index 59fccd58f9..37ea223d5c 100644 --- a/kernel/src/wire/openai.ts +++ b/kernel/src/wire/openai.ts @@ -174,17 +174,21 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { } if (leadingOpaque.length > 0) { if (sawText && msgs.length > 0) { - const firstTextCore = - msgs[msgs.length - 1] && - msgs.find( - (c, i) => - i >= before && - c.role === "user" && - c.contentType === "text" && - !c.rawOpenaiContent, - ); + const firstTextCore = msgs.find( + (c, i) => + i >= before && + c.role === "user" && + c.contentType === "text" && + !c.rawOpenaiContent, + ); const host = firstTextCore ?? msgs[msgs.length - 1]!; - host.rawOpenaiContentParts = leadingOpaque; + // PREPEND: the host may already carry opaque parts accumulated + // after it during the loop ([opaque, text, opaque]) — assigning + // would silently drop them (#1205: unknown parts never dropped). + host.rawOpenaiContentParts = [ + ...leadingOpaque, + ...(host.rawOpenaiContentParts ?? []), + ]; } else { const base = deriveCanonicalMessageId("user", "text", ""); msgs.push({ diff --git a/kernel/tests/media-payload.test.ts b/kernel/tests/media-payload.test.ts index 921a2bb5d5..85daf2291c 100644 --- a/kernel/tests/media-payload.test.ts +++ b/kernel/tests/media-payload.test.ts @@ -110,6 +110,37 @@ test("openai: [text, file] round-trips the file part verbatim", () => { ); }); +test("openai: [file, text, file] keeps every opaque part (#1205)", () => { + const fileA = { type: "file", file_id: "file-api-aaa" }; + const fileB = { type: "file", file_id: "file-api-bbb" }; + const body = bodyOf([ + { + role: "user", + content: [ + fileA, + { type: "text", text: "compare these files" }, + fileB, + ], + }, + ]); + const { msgs } = openaiToCore(body); + assert.equal(msgs.length, 1); + assert.equal(msgs[0]?.text, "compare these files"); + // The leading-opaque flush must not clobber the part accumulated AFTER + // the text core: both ride the plural sidecar, relative order intact. + assert.deepEqual(msgs[0]?.rawOpenaiContentParts, [fileA, fileB]); + + const content = rebuiltUserContent( + msgs, + ) as Array>; + assert.ok(Array.isArray(content)); + assert.deepEqual(content, [ + { type: "text", text: "compare these files" }, + fileA, + fileB, + ]); +}); + test("openai: lone [file] part survives with empty text", () => { const body = bodyOf([{ role: "user", content: [FILE_PART] }]); const { msgs } = openaiToCore(body); From acc2aea448aa915f105cceb029cf2106e4ee7c11 Mon Sep 17 00:00:00 2001 From: dog Date: Sun, 18 Oct 2026 15:37:08 +0800 Subject: [PATCH 11/14] chore: sync root package-lock.json to acp-kernel 0.0.132 (lock-sync after the kernel version bump) --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 411c5de310..767355a03a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.131", + "version": "0.0.132", "dev": true, "license": "MIT", "devDependencies": { From 5a35de59ec450a24163c2f0e97e2eb10fa7539ef Mon Sep 17 00:00:00 2001 From: dog Date: Sun, 18 Oct 2026 16:55:06 +0800 Subject: [PATCH 12/14] =?UTF-8?q?ci:=20retrigger=20=E2=80=94=20PR=20body?= =?UTF-8?q?=20now=20carries=20the=20kernel-change=20declaration=20(reruns?= =?UTF-8?q?=20reuse=20stale=20event=20payload)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From aa23b34ccf488368a88d0d5544fb9a890e718fa3 Mon Sep 17 00:00:00 2001 From: dog Date: Sun, 18 Oct 2026 17:08:35 +0800 Subject: [PATCH 13/14] kernel: openai non-array content tolerance + golden regen + inverted id-churn canary (full-CI shard-3 reds, #3658 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - kernel/src/wire/openai.ts: user-branch guard extended to !Array.isArray(content) — a lone content-part object (some clients) threw 'm.content is not iterable' (#2824 corpus). Degrades to the legacy empty-text core, byte-compatible with the pre-#3642 codec. kernel 0.0.132 -> 0.0.133, both locks regenerated. - tests/golden/storage-render/chat.json: regenerated via scripts/update-storage-render-goldens.ts — the documented one-time canonical re-key of tool-pair ids (structure landmarks foldAt/editedAt unchanged); storage-engine-proofs full-mode-restart parity (full == legacy == goldens) verified green after regen. - tests/identity-proof.test.ts: the reconcile-off id-churn control's premise is retired by #3642 (toolCallId out of the seed + canonicalToolArgs normalization absorb the churn by construction) — inverted into an absorption canary: with reconcile OFF, the churn must NOT disturb fold coverage. Guards against id-sensitivity leaking back into the canonical seeds. - kernel/tests/media-payload.test.ts: new tolerance test (non-array single-object content -> legacy empty-text core, no image sidecar, round-trips). Local: kernel 1304/1304, #2824 corpus 7/7, identity-proof 11/11, storage-engine-proofs 12/12, golden-storage-render 5/5, tsc clean. --- CHANGELOG.md | 1 + kernel/package-lock.json | 4 +- kernel/package.json | 4 +- kernel/src/wire/openai.ts | 12 +++-- kernel/tests/media-payload.test.ts | 30 +++++++++++ package-lock.json | 2 +- tests/golden/storage-render/chat.json | 74 +++++++++++++-------------- tests/identity-proof.test.ts | 23 ++++++--- 8 files changed, 98 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdcd3e417e..ea0008d962 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag ### Changes +- **openai wire: non-array content tolerance (kernel 0.0.133, #3658 follow-up)** — the #3642 per-part rewrite iterated `messages[].content` and threw `TypeError: m.content is not iterable` on clients that emit a lone content-part object instead of an array (#2824 corpus caught it). Non-array non-string content now degrades to the legacy empty-text core (byte-compatible ids and bytes with the pre-#3642 codec) instead of throwing. Also regenerates the storage-render chat goldens for the documented one-time canonical re-key (tool-pair ids), and inverts the retired reconcile-off id-churn control into an absorption canary: with reconcile OFF, tool_call id rewrites + argument re-serialization must not disturb fold coverage (canonical ids absorb them by construction). - **protocol-stable core identity: the same logical conversation now yields byte-identical core message ids in all four dialects (#3642).** Wire codecs previously baked dialect accidents into message identity: anthropic used `"[image]"` as image text while openai deliberately kept `""` ("switching the seed would move every pre-existing shape"), tool-call ids embedded the provider's `toolCallId` bytes, raw JSON tool arguments hashed verbatim (key order = new id), and responses' empty-text reasoning cells seeded from the provider's volatile `item.id`. Switch protocols mid-session (opencode #3640: `/chat/completions` → `/v1/messages`) and ~20-68% of ids churned, tripping the fold-destroyed misdiagnosis #3641 just class-fixed. Identity now derives from a canonical form shared by all four codecs: a new `kernel/src/wire/identity.ts` (canonicalToolArgs: JSON args parsed → recursive key sort → compact stringify, verbatim for non-JSON; canonicalOpaqueSeed: provider `item.id`/`status` stripped; image marker `[image]` everywhere), the `toolCallId` slot retired from the seed (cluster suffixes disambiguate same-shape calls, deleting one sibling is local damage), and granularity normalized to one core per content part (anthropic's existing shape — its lanes take zero granularity re-keying, though tool-call/tool-result cores still re-seed since the `toolCallId` slot is retired; openai/responses/google split runs, with accumulators re-merging on rebuild so the wire stays byte-stable). The kernel first-user pin now spans the opening message's whole core run, so a multimodal opening message keeps its image bytes through folds (#781/#2607). Cross-protocol golden: text/tool/bash conversation → identical id arrays in all four dialects; canonical key-sort makes `{"file","line"}` == `{"line","file"}`. Image cells are now stamped `imagePart: true` at parse time (never text-compared — tag injection rewrites markers mid-render, which previously made image cores render as text and lose the real bytes, #767 billing fallout). Upgrade cost is one re-key of stored ids for openai/responses/google lanes; the #3641 reconcile tiers (word-bag claims, k-th image family pairing, toolCallId channel, epoch gate) absorb it without re-folding. - **fold anchoring doctrine: order over hash — protocol switches, image offload, and host-side truncation stop being read as destruction (#3641, fixes #3588, fixes #3640).** Three real-world defect classes all shared one root: the fold's identity model keyed coverage to content hashes, so any transformation that re-mints ids without removing content (dsh's image-offload component rewriting image cells into placeholder text, opencode switching a mid-session request from `/chat/completions` to `/v1/messages`, or a host that compacts/truncates history itself) made every covered id look *destroyed*. Consequences: 7× false "compression substrate appears destroyed" escalations on healthy sessions (#3588), 298-id churn with unanchored summary blocks landing AFTER the user's newest question — the model answered five rounds of stale summaries instead of the question (#3640) — and the #2193 collapse streak on sanctioned host compaction. The reconcile pass now carries a shape tier (loose word-evidence pairing for prose cells, a flanked k-th discipline for evidence-less image cells), a residual census classifying unmatched ids as *transformed* (near-duplicate evidence exists), *absent* (content truly off the wire with the surviving wire a suffix-continuation of the episode baseline), or *drift* (nothing matches anywhere); only the third accrues the collapse streak, and a wire-frozen episode witness guarantees a foreign conversation can never quietly become the baseline mid-episode. Unresolved summary anchors clamp ahead of the newest user turn instead of at the array tail (the #3640 "stale summary after the question" shape), and a genuine total collapse now latches `foldEpochConfirmPending` and PAUSES folding (verbatim passthrough — nothing is deleted, the snapshot is kept) until the histories realign or the user confirms a new epoch, instead of escalating on a misdiagnosis. Single genuine edits below the mass bar still report drift honestly; the #1729/#2193 escalation itself stays (its error text now names the epoch gate). diff --git a/kernel/package-lock.json b/kernel/package-lock.json index c030136e7b..7cf51015f9 100644 --- a/kernel/package-lock.json +++ b/kernel/package-lock.json @@ -1,12 +1,12 @@ { "name": "acp-kernel", - "version": "0.0.132", + "version": "0.0.133", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "acp-kernel", - "version": "0.0.132", + "version": "0.0.133", "license": "MIT", "devDependencies": { "@types/node": "^22.0.0", diff --git a/kernel/package.json b/kernel/package.json index e35f827715..b77f4bf7b3 100644 --- a/kernel/package.json +++ b/kernel/package.json @@ -1,6 +1,6 @@ { "name": "acp-kernel", - "version": "0.0.132", + "version": "0.0.133", "description": "Framework-agnostic context-compression engine (model-driven, 3-tier LSM). Pure core: no host dependency.", "license": "MIT", "author": "ranxianglei", @@ -69,4 +69,4 @@ "engines": { "node": ">=20" } -} +} \ No newline at end of file diff --git a/kernel/src/wire/openai.ts b/kernel/src/wire/openai.ts index 37ea223d5c..03cbd6158e 100644 --- a/kernel/src/wire/openai.ts +++ b/kernel/src/wire/openai.ts @@ -92,17 +92,23 @@ export function openaiToCore(body: OpenAIRequestBody): Flat { // #1205) attach to the preceding text core's plural sidecar so they // still round-trip verbatim. A message with only opaque parts keeps // the legacy single-core shape (text "" + plural sidecar). - if (typeof m.content === "string" || m.content == null) { + // Non-array non-string content (a lone part object, or any + // degenerate shape some client emits) cannot be iterated — fall + // through to the legacy empty-text core instead of throwing (#2824 + // corpus: wire tolerance). Legacy byte-compat: the pre-#3642 codec + // mapped such messages to stringContent(content) === "" with no + // sidecars, and ids seeded from "" stay identical. + if (typeof m.content === "string" || m.content == null || !Array.isArray(m.content)) { const base = deriveCanonicalMessageId( "user", "text", - m.content ?? "", + typeof m.content === "string" ? m.content : "", ); msgs.push({ id: clusters.next(base), role: "user", contentType: "text", - text: m.content ?? "", + text: typeof m.content === "string" ? m.content : "", }); break; } diff --git a/kernel/tests/media-payload.test.ts b/kernel/tests/media-payload.test.ts index 85daf2291c..4cd0834355 100644 --- a/kernel/tests/media-payload.test.ts +++ b/kernel/tests/media-payload.test.ts @@ -1006,3 +1006,33 @@ test("applyCompression keeps an unrecoverable media tool_result and its paired c `pair-withdrawal warning present, got: ${JSON.stringify(result.result.warnings)}`, ); }); + +test("openai: non-array single-object content never throws (legacy empty-text core)", () => { + // #2824 corpus regression: some clients emit a lone content-part object + // instead of an array. Pre-#3642 this mapped to stringContent() === "" + // with no sidecars; the #3642 per-part rewrite iterated `m.content` and + // threw TypeError "m.content is not iterable" — a wire-tolerance break + // (buildIncomingImageIndex catches, but every other consumer would not). + // Contract: non-array non-string content degrades to the legacy empty-text + // core, byte-compatible with the old codec, and coreToOpenai still renders + // a round-trippable body. + const body = bodyOf([ + { + role: "user", + content: { + type: "image_url", + image_url: { url: DATA_URL }, + }, + }, + ]); + const { msgs } = openaiToCore(body); + assert.equal(msgs.length, 1, "one core, not a throw"); + assert.equal(msgs[0]?.contentType, "text"); + assert.equal(msgs[0]?.text, "", "legacy mapping: stringContent(object) === ''"); + assert.equal(msgs[0]?.imageBase64, undefined, "no image sidecar from non-array content"); + const back = coreToOpenai(msgs as Parameters[0]); + assert.ok( + Array.isArray(back) && back.length >= 1, + "round-trip renders", + ); +}); diff --git a/package-lock.json b/package-lock.json index 767355a03a..b3c3f4efa7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48,7 +48,7 @@ }, "kernel": { "name": "acp-kernel", - "version": "0.0.132", + "version": "0.0.133", "dev": true, "license": "MIT", "devDependencies": { diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index a9c1c44b91..4ec8a8e6ad 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -8,90 +8,90 @@ }, { "i": 1, - "sha": "82330334ab7e3fae489d3ceda70874d5d288f6f10351d480e8931825a331c2ef", + "sha": "ce55234b88db48e3d7b0b52085963b727e30e485a39af819594fce77097b7207", "bytes": 31995 }, { "i": 2, - "sha": "44168aba9dfec04562f94884c27b5ef91efa882d55eef0aa72d5dd9d6a2a8f68", - "bytes": 37829 + "sha": "1c6a2f7169fee01285819f84146bee61aef7934b68cf5c16838b3b7a0960fff3", + "bytes": 37807 }, { "i": 3, - "sha": "ecb0a1c6b39da3ad25b2d811ae9f01bd4462b443acd72d719a101953d9505421", - "bytes": 46373 + "sha": "8bbfe9c4af21ba7626f9c2abbe94a287730448c09bc544955d4a16e5fb0e0cde", + "bytes": 46385 }, { "i": 4, - "sha": "0a101eddf0773aa82f5ae3e17216c6d42b772ef873eaa51ab816fbbc6460b270", - "bytes": 52207 + "sha": "0b16037a50efcb87bbdfdc321bf75f9847771f1552bed7e9ccb0f0a9a1a39fa7", + "bytes": 52197 }, { "i": 5, - "sha": "4de288374f892fc4c63d8e88d4127dcfdc64ee5fa232338dcdfdedb50a4bcebe", - "bytes": 60751 + "sha": "1a7edb8f445334cfd25e0ca159060a54a8c37bff53d9f0a3efd4acd2ca0059da", + "bytes": 60775 }, { "i": 6, - "sha": "8cf5801a79bfaee1272ded800f990858235a72f69a5be5ef5127e6ff271971eb", - "bytes": 66585 + "sha": "afd6a636915f86a3cb7064fc807a8be72f12f86b5f90b1b9147320c526aab92b", + "bytes": 66587 }, { "i": 7, - "sha": "6727f67d8465d80cb7b02025e4a43c7fbfe87194be31b1e1af664e82b2e28146", - "bytes": 75129 + "sha": "ae088e5256b0e6b5b771f382e55bd3c5b8f6f3df81194721f3284f62f3b7e392", + "bytes": 75165 }, { "i": 8, - "sha": "8ff52f654ec7d4cb9637ac84b3efa7457846063570cdf5756056ade1c3bbaf10", - "bytes": 80963 + "sha": "5623e4626136cb65156548acf8c801770f445709aa5c1d57f0de056441230162", + "bytes": 80977 }, { "i": 9, - "sha": "72542c69ec5cbfdef8bdea1deb83b881d35707a4997ff4ea64ee71f66af5b037", - "bytes": 50470 + "sha": "187920f4d4b1cc5ce43a3e52027008e193441ed834026669cd5a37dc4034ee17", + "bytes": 50478 }, { "i": 10, - "sha": "f32352ba5fc23d887774e5dcd7f121912cbb50a952dc5d66ef034adb437f9c79", - "bytes": 57855 + "sha": "89ab8419c57248c79777d523f0c888ea798bdef146ace4f52b14a0d672d593b2", + "bytes": 57879 }, { "i": 11, - "sha": "68f253bcd35cca495e996863eeb564ae6649f9ec2864c401c21e8ac1babbca6f", - "bytes": 63612 + "sha": "a830324598bd68492380344532152dae98ae91fca26a9fb9917cf2dc16450fd9", + "bytes": 63614 }, { "i": 12, - "sha": "309426c88866153bfea341d7cc0ea9f8d13b7c59ae57c84b45506d327e522fcc", - "bytes": 77621 + "sha": "7e2fb54ac5c850cd92d0e1a63585f729d6883426357677f394674fb49f1c2068", + "bytes": 77660 }, { "i": 13, - "sha": "db5e9c9781f78f41a186836665a090e680524df7e466b2447e07c5bd6417e012", - "bytes": 83378 + "sha": "6e09bf973ab1de4ea1e6c350e169147de681237522ed2e414d967cefc925d59c", + "bytes": 83395 }, { "i": 14, - "sha": "ead3aa36c6f4a72d24e35ad7d1017084d2bef1e08a209dc5e95ec0772182a91a", - "bytes": 91898 + "sha": "42209cfddbd623dc6ee3f23564ab2f57e3d711f24b1eabbe5cd16d27dbed2bf4", + "bytes": 91949 } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "2f22b4aa68e49f359c058315b5ce1c060ec80722a19d28c356a3b24cda045e60", + "stateDigest": "4d3065c6c981f13ec24b9946743ffc9f0d0389e3a36daf6cd4116224cecdbe16", "sectionDigests": { - "state": "eaa58407256b986b", - "blockContents": "039ad3602f605afe", - "stats": "711b07a24813e7a8", - "metadata": "5b9eb00eeaeab92d" + "state": "463b3dc26a100c2a", + "blockContents": "94a58f96d7c5cfc3", + "stats": "59842ce5415d3a4d", + "metadata": "ccf5855e9ba4bf2a" }, "stateKeyDigests": { - "blocks": "5426f9d515432b43", + "blocks": "d9770feee91dafc6", "messageRefs": "48135bb278ef6a47", - "nudge": "592c25cb5bcb1f87", - "stats": "48f9f19ca44eacdb", + "nudge": "b539bf305b125cb7", + "stats": "f85b2ad1a3b86e2f", "nextBlockId": "d4735e3a265e16ee", "nextRunId": "d4735e3a265e16ee", "tokenSnapshot": "5080181ee1a25a8c", @@ -110,14 +110,14 @@ "deadRefs": 0, "blockContents": 1, "coveredRefSample": [ - "h_631ad4fd8d101d1d", "h_e40995eea3f07d55", "h_a9607fb18a2fa39d", "h_7fab6026ce5a72e7", "h_b5b946fa2455d77e", "h_e40995eea3f07d55_1", "h_ce9559b8e1773afc", - "h_e40995eea3f07d55_2" + "h_e40995eea3f07d55_2", + "h_f1a111d468a227a1" ] } } diff --git a/tests/identity-proof.test.ts b/tests/identity-proof.test.ts index 9ce91d7a16..2981c13206 100644 --- a/tests/identity-proof.test.ts +++ b/tests/identity-proof.test.ts @@ -946,10 +946,19 @@ test("identity proof (control, chat wire): reconcile OFF lets the model-switch c } }); -// Negative control: WITHOUT the reconcile engine the same id churn destroys -// coverage — the summary disappears and the full history re-bills. This is -// the old-world behavior the Pass 0 claims repair (#2396 death spiral). -test("identity proof (control, chat wire): reconcile OFF lets the churn re-bill the unfolded history", { timeout: 180_000 }, async () => { +// id-churn control, RETIRED-AND-INVERTED by #3642: this churn class used to +// re-key chat cores (tool_call ids + argument re-serialization moved the +// seeds), so with reconcile OFF the fold coverage died — the control proved +// the positive test was exercising Pass 0. Canonical core identity makes +// toolCallId a non-seed ("transport plumbing") and normalizes tool arguments +// (canonicalToolArgs: sorted keys, compact form), so this churn class is now +// absorbed BY CONSTRUCTION, upstream of reconcile. The old control "failed +// to fail" — correctly detecting that its premise is gone. Inverted, the +// test stays useful as a canary: with reconcile deliberately OFF, the churn +// must STILL retain the summary and not re-bill. If it ever goes red, someone +// reintroduced id-sensitivity into the canonical seeds (or arg normalization +// regressed) — both are #3642 contract breaks, not reconcile bugs. +test("identity proof (control, chat wire): canonical ids absorb the id churn even with reconcile OFF (#3642)", { timeout: 180_000 }, async () => { const prev = process.env.BILI_FOLD_RECONCILE; process.env.BILI_FOLD_RECONCILE = "off"; try { @@ -1004,10 +1013,10 @@ test("identity proof (control, chat wire): reconcile OFF lets the churn re-bill ops.churnIds(); await sendTurn(MAX_TURNS + 1); const switchBody = bodies[bodies.length - 1]!; - const lost = !switchBody.includes(SUMMARY_MARKER); + const retained = switchBody.includes(SUMMARY_MARKER); const reBilled = switchBody.includes("file-2-row-") || Buffer.byteLength(switchBody, "utf8") >= unfoldLen * 0.8; - assert.ok(lost || reBilled, "control failed to fail: with reconcile OFF the id churn should lose the summary and/or re-bill the folded tool payloads — if it survives, the positive tests above are not actually exercising Pass 0"); - console.log(`proof[chat-ctl] VERDICT reconcile=off churnFoldLost=${lost} churnReBilled=${reBilled} len=${Buffer.byteLength(switchBody, "utf8")}B (unfolded-ref=${unfoldLen}B) sha=${sha16(switchBody)}`); + assert.ok(retained && !reBilled, "ctl-absorb: with reconcile OFF the id churn (tool_call id rewrite + arg re-serialization) must NOT disturb fold coverage — canonical ids absorb it by construction (#3642). If this fails, id-sensitivity leaked back into the canonical seeds or canonicalToolArgs normalization regressed"); + console.log(`proof[chat-ctl-absorb] VERDICT reconcile=off churnAbsorbed=true summaryRetained=${retained} reBilled=${reBilled} len=${Buffer.byteLength(switchBody, "utf8")}B (unfolded-ref=${unfoldLen}B) sha=${sha16(switchBody)}`); } finally { await closeServer(proxy); await closeServer(upstream); From 5fd814c6904ac4080e8c9935fa824932be2a7ba7 Mon Sep 17 00:00:00 2001 From: dog Date: Sun, 18 Oct 2026 17:21:21 +0800 Subject: [PATCH 14/14] image-restore: conversion failures stay fail-open but WARN once per signature (#2824 class visibility) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catch in buildIncomingImageIndex returned an empty index with zero signal — that is exactly how the #3642 single-object-content TypeError hid: image recovery silently off for every session carrying the shape, no error, no log line. Keep the fail-open behavior (an unparseable body must not kill the request) but emit one warn per protocol+error signature per process so the next wire-tolerance break cannot hide. Test: proxy-bomb body -> empty index + the #2824 visibility warn. --- src/image-restore.ts | 18 +++++++++++++++++- tests/issue2824-regression.test.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/image-restore.ts b/src/image-restore.ts index 82e890c1e0..fdac387535 100644 --- a/src/image-restore.ts +++ b/src/image-restore.ts @@ -234,6 +234,12 @@ export function bodyMightCarryImages(bodyText: string, protocol: WireProtocol): } } +/** Warn-once dedupe for inbound-index conversion failures: the catch must + * stay fail-open (empty index, request unaffected) but never SILENT — #2824 + * class. Keyed by protocol + error signature; bounded by distinct error + * shapes, which is a handful per process. */ +const conversionWarned = new Set(); + /** Parse the INBOUND wire body into core messages and index every carried image * by its mNNNNN ref. A message is indexed only when it is already known to the * session (`state.messageRefs.byRaw[message.id]` resolves) — brand-new tail @@ -257,7 +263,17 @@ export function buildIncomingImageIndex( case "google": result = googleToCore(parsed as GoogleRequestBody); break; case "responses": result = responsesToCore(parsed as ResponsesRequestBody); break; } - } catch { + } catch (e) { + // Fail-open by design (an unparseable body must not kill the request), + // but visible: a converter throw here silently disables image recovery + // for every session carrying that shape — exactly how the #3642 + // single-object-content TypeError hid (#2824). One warn per signature + // per process so a broken shape logs once, not twice per turn. + const sig = `${protocol}:${String((e as Error)?.message ?? e).slice(0, 120)}`; + if (!conversionWarned.has(sig)) { + conversionWarned.add(sig); + loggerLog("warn", `[image-restore] inbound index conversion failed — images in affected sessions are NOT recoverable until this wire shape is handled (${sig}; #2824 class — please report the body shape)`); + } return index; } const msgs = coreMsgsOf(result); diff --git a/tests/issue2824-regression.test.ts b/tests/issue2824-regression.test.ts index e66d3c0574..d191f90451 100644 --- a/tests/issue2824-regression.test.ts +++ b/tests/issue2824-regression.test.ts @@ -226,3 +226,30 @@ test("#2824: guarded path ≡ unconditional build across the corpus", () => { assert.deepStrictEqual(guarded, unconditional, `${fx.name}: guarding must not change the index`); } }); + +test("#2824: a converter throw degrades the index fail-open but WARNS, never silently", async () => { + // The #3642 single-object-content TypeError hid precisely because the + // catch returned an empty index with zero signal — image recovery was + // silently off for every session carrying that shape. The catch must + // stay fail-open (request path unaffected) but emit one warn per error + // signature so the next wire-tolerance break cannot hide. + const { setLogCapture } = await import("../src/logger.ts"); + const warns: string[] = []; + setLogCapture((_level, msg) => { warns.push(msg); }); + try { + // A body shape that makes the converter itself throw: a Proxy that + // explodes on property access — the conversion cannot get past the + // first read, exercising the catch directly. + const bomb = new Proxy({}, { + get() { throw new TypeError("wire bomb"); }, + }); + const idx = buildIncomingImageIndex(bomb, "openai", createInitialState(), "warn-probe"); + assert.equal(idx.size, 0, "fail-open: empty index, no throw"); + assert.ok( + warns.some((w) => w.includes("[image-restore] inbound index conversion failed") && w.includes("#2824")), + `expected the visibility warn, got: ${JSON.stringify(warns)}`, + ); + } finally { + setLogCapture(null); + } +});