diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json index d0f7772d77..9515dcc38d 100644 --- a/.agents/plugins/marketplace.json +++ b/.agents/plugins/marketplace.json @@ -6,7 +6,7 @@ "plugins": [ { "name": "billion-context", - "version": "0.1.205", + "version": "0.1.207", "source": { "source": "local", "path": "./plugins/codex" diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 0345018cac..12953b3934 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ "name": "billion-context", "source": "./", "description": "Context-compression plugin for Claude Code · month-long single sessions (billions of tokens) · high compression quality even in small context windows (100K is enough) · 5× fewer tokens", - "version": "0.1.205", + "version": "0.1.207", "author": { "name": "ranxianglei" }, diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 4b5ce9de31..4538febdfc 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "billion-context", - "version": "0.1.205", + "version": "0.1.207", "description": "Context-compression plugin for Claude Code · month-long single sessions (billions of tokens) · high compression quality even in small context windows (100K is enough) · 5× fewer tokens", "author": { "name": "ranxianglei" diff --git a/.github/workflows/release-bugfix.yml b/.github/workflows/release-bugfix.yml index 98d14b9ad2..d687addcca 100644 --- a/.github/workflows/release-bugfix.yml +++ b/.github/workflows/release-bugfix.yml @@ -52,7 +52,17 @@ jobs: contents: write id-token: write actions: write + pull-requests: write # annotate the ship PR + auto-open the alignment PR (#4744 note) steps: + - name: Ensure jq (2026-10 runner images are unreliable about it — always install our own; ported back from v0.1.208, #4744) + run: | + type -a jq || true + mkdir -p "$HOME/.local/bin" + curl -fsSL "https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-linux-amd64" -o "$HOME/.local/bin/jq" + chmod +x "$HOME/.local/bin/jq" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + "$HOME/.local/bin/jq" --version + - name: Validate trigger + resolve the publish ref id: resolve run: | diff --git a/plugins/codex/.codex-plugin/plugin.json b/plugins/codex/.codex-plugin/plugin.json index 8ee8250a7b..862a8a231a 100644 --- a/plugins/codex/.codex-plugin/plugin.json +++ b/plugins/codex/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "billion-context", - "version": "0.1.205", + "version": "0.1.207", "description": "Context-compression plugin for Codex — small context windows (100K is enough), 5x fewer tokens, month-long single sessions (billions of tokens), and high compression quality.", "author": { "name": "ranxianglei" diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index a16b2b299f..e3bde3891f 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -1291,10 +1291,16 @@ export function recordCacheSample( if (pending.length > 0 && known && s.input > 0) { const rawMissed = s.input - effCached; if (rawMissed > 8192 && rawMissed > 0.3 * s.input) { - const curB = session.transient?.sentBody; + // #4748: read the CURRENT send through pendingCapture — the relay + // lane keeps pending evidence in the AsyncLocalStorage capture + // (#4430), so transient.sentBody is undefined there and this + // measurement (whose target scenario IS the relay lane) never + // fired. prev reads stay on transient: settled evidence is moved + // OUT of the capture at settle time. + const curB = pendingCapture(session).sentBody ?? session.transient?.sentBody; const prevB = session.transient?.settledBody; if (curB !== undefined && prevB !== undefined && curB.bytes <= SEAM_BODY_CAP && prevB.bytes <= SEAM_BODY_CAP) { - const curIn = session.transient?.inBody; + const curIn = pendingCapture(session).inBody ?? session.transient?.inBody; const prevIn = session.transient?.settledInBody; const clientBreak = curIn !== undefined && prevIn !== undefined && curIn.msgs <= INBOUND_MSG_CAP && prevIn.msgs <= INBOUND_MSG_CAP && (() => { const k = firstDivergentInboundMsg(prevIn, curIn); diff --git a/src/degenerate-retry.ts b/src/degenerate-retry.ts index ba3ea55498..a21b28bd73 100644 --- a/src/degenerate-retry.ts +++ b/src/degenerate-retry.ts @@ -1,4 +1,5 @@ import { fetchWithTimeout } from "./fetch-util.js"; +import { noteUpstreamThrottle429, noteUpstreamThrottleResolved, upstreamThrottleLocalReject } from "./upstream-throttle.js"; import { hasCompactionTrigger } from "./codex-compact.js"; import { proxyDispatcher } from "./upstream-proxy.js"; import type { WireProtocol } from "./util.js"; @@ -82,6 +83,17 @@ export function makeContinuationRefetch(opts: ContinuationRetryOpts): () => Prom const wireRetryBody = opts.wrapBody ? opts.wrapBody(retryBody) : retryBody; try { opts.resign?.(opts.reqHeaders, wireRetryBody); + // #4743: the retry is an egress send like any other — feed the 429 + // governor (observe only, no local gate: the retry is bounded by + // the degenerate-turn ladder, and holding IT locally would strand + // the client's already-open response stream). A 429 verdict arms + // the hold for the NEXT main-lane send; any other resolved status + // clears it — same one-observation-per-request discipline as relay. + const throttleReject = upstreamThrottleLocalReject(opts.upstreamUrl, opts.reqHeaders); + if (throttleReject !== undefined) { + opts.log("info", `[${opts.label}] [plugin] degenerate-terminal retry answered locally by the 429 hold (#4204) — passing the empty turn through`); + return null; + } const r = await fetchWithTimeout( opts.upstreamUrl, { @@ -95,9 +107,12 @@ export function makeContinuationRefetch(opts: ContinuationRetryOpts): () => Prom ); if (!r.response.ok || !r.response.body) { r.clearTimer(); + if (r.response.status === 429) noteUpstreamThrottle429(opts.upstreamUrl, opts.reqHeaders, r.response.headers.get("retry-after")); + else noteUpstreamThrottleResolved(opts.upstreamUrl, opts.reqHeaders, r.response.status); opts.log("warn", `[${opts.label}] [plugin] degenerate-terminal retry rejected (HTTP ${r.response.status}); passing the empty turn through`); return null; } + noteUpstreamThrottleResolved(opts.upstreamUrl, opts.reqHeaders, r.response.status); const body = r.response.body as ReadableStream; // The retry stream is spliced into a response the caller already // owns, so its own idle timer is dropped — a stalled retry then ends diff --git a/src/dsh-channel.ts b/src/dsh-channel.ts index 308c385f54..b04273ab25 100644 --- a/src/dsh-channel.ts +++ b/src/dsh-channel.ts @@ -407,7 +407,10 @@ export function planDshSpawn( "Set BILI_DSH_BIN to the dsh executable, or reword the argument.", ); }); - const comspec = env.COMSPEC?.trim() || "cmd.exe"; + // #4599: sanitized env copies keep original key casing (`ComSpec`) — an + // exact `env.COMSPEC` probe misses it on plain objects. + const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec; + const comspec = comspecRaw?.trim() || "cmd.exe"; const line = `"${[command, ...args].map(quoteWinToken).join(" ")}"`; return { command: comspec, args: ["/d", "/s", "/c", line], windowsVerbatimArguments: true }; } diff --git a/src/fold-decomposition-lookup.ts b/src/fold-decomposition-lookup.ts index 543e6b160b..f7aad8b68e 100644 --- a/src/fold-decomposition-lookup.ts +++ b/src/fold-decomposition-lookup.ts @@ -23,7 +23,7 @@ export interface StoredRow { content: string; } -interface DecompositionSource { +export interface DecompositionSource { /** null = no index available (legacy/shadow/degraded); [] = index has none of them. */ rows(sessionId: string, msgIds: readonly string[]): StoredRow[] | null; } @@ -89,10 +89,13 @@ export function resolveStoredDecompositions( // Latest seq per id wins (edits append rows); neighbors stay in so rc // pairs keep adjacency for turn-key derivation. Unparseable rows are // auxiliary-evidence misses, not errors — count them, don't throw. + // #4747: no in-loop index maintenance — interleaved rows (A1,B1,A2,B2 — + // the append shape for edits, exactly this lookup's target scenario) + // desynced a per-id index map on every splice, deleting the wrong + // element and silently dropping another id's LATEST row. Collect + // per-id latest now; rebuild + sort once at the end. let skipped = 0; - const latestSeq = new Map(); - const ordered: { seq: number; m: CoreMessage }[] = []; - const orderedIdx = new Map(); + const latestById = new Map(); for (const row of rows) { let parsed: unknown; try { @@ -110,16 +113,12 @@ export function resolveStoredDecompositions( skipped++; continue; } - const prev = latestSeq.get(row.msgId); - if (prev !== undefined && prev >= row.seq) continue; - latestSeq.set(row.msgId, row.seq); - const prevIdx = orderedIdx.get(row.msgId); - if (prevIdx !== undefined) ordered.splice(prevIdx, 1); - ordered.push({ seq: row.seq, m }); - orderedIdx.set(row.msgId, ordered.length - 1); + const prev = latestById.get(row.msgId); + if (prev !== undefined && prev.seq >= row.seq) continue; + latestById.set(row.msgId, { seq: row.seq, m }); } if (skipped > 0) loggerLog("debug", `[fold-reconcile] stored decomposition: ${skipped} unparseable row(s) skipped`); - if (ordered.length === 0) return undefined; - ordered.sort((a, b) => a.seq - b.seq); + if (latestById.size === 0) return undefined; + const ordered = [...latestById.values()].sort((a, b) => a.seq - b.seq); return ordered.map((o) => o.m); } diff --git a/src/launcher.ts b/src/launcher.ts index 42f66e34fd..d9d865c2ff 100644 --- a/src/launcher.ts +++ b/src/launcher.ts @@ -5418,7 +5418,12 @@ export function planClientSpawn( "Set BILI_CLIENT_BIN to the native executable (e.g. the real codex.exe) or to a script entry run under node, or reword the argument.", ); }); - const comspec = nonEmpty(env.COMSPEC) ? env.COMSPEC : "cmd.exe"; + // #4599: Windows env vars are case-insensitive, but sanitized copies + // (spread objects, e.g. stripInheritedProxy's inlineEnv) keep the original + // key casing — `ComSpec` — where an exact `env.COMSPEC` probe misses it + // and silently degrades to the bare-name fallback. Probe every spelling. + const comspecRaw = env.COMSPEC ?? env.ComSpec ?? env.comspec; + const comspec = nonEmpty(comspecRaw) ? comspecRaw : "cmd.exe"; return { command: comspec, args: ["/d", "/s", "/c", buildWindowsCommandLine(cmd, args)], diff --git a/src/persist.ts b/src/persist.ts index bdf9abc0d1..5d33c32c68 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -1610,6 +1610,30 @@ export class SessionStore { if (outcome.superseded.size > 0) { session.restampSuperseded = outcome.superseded; } + // The token snapshot is keyed by ref (#3488 render-refs): renumbered + // refs would otherwise inherit the stale count of whichever message + // held the new number pre-alignment — the wire tag then disagrees + // with the legacy-restart leg beyond citation renumbering (storage + // engine-proofs wire-parity, #4599). Two-pass moves: stage all + // (oldRef → newRef → count) first, then delete, then write — a + // single-pass swap would lose entries whose newRef is another + // move's oldRef. + const snap = (state as { tokenSnapshot?: Record }).tokenSnapshot; + if (snap && outcome.superseded.size > 0) { + const byRawNow = state.messageRefs.byRaw; + const moves: Array<[oldRef: string, newRef: string, count: number]> = []; + for (const [oldRef, rawId] of outcome.superseded) { + const count = snap[oldRef]; + const newRef = byRawNow[rawId]; + if (count === undefined || newRef === undefined || newRef === oldRef) continue; + moves.push([oldRef, newRef, count]); + } + if (moves.length > 0) { + for (const [oldRef] of moves) delete snap[oldRef]; + for (const [, newRef, count] of moves) snap[newRef] = count; + this.log("info", `[persist] remapped ${moves.length} token-snapshot entries onto ledger-aligned refs for ${session.id} (#4599)`); + } + } state.refLedgerAligned = true; this.log("info", `[persist] ledger-aligned refs for ${session.id}: kept=${outcome.kept} restamped=${outcome.restamped} rehomed=${outcome.rehomed} blocked=${outcome.blocked} superseded=${outcome.superseded.size} (#4316 PR3/3)`); this.scheduleSave(session); diff --git a/src/plugin-render.ts b/src/plugin-render.ts index d0faafff5d..cc175918c9 100644 --- a/src/plugin-render.ts +++ b/src/plugin-render.ts @@ -105,9 +105,15 @@ export async function handlePluginRender(req: http.IncomingMessage, res: http.Se if (blockSeqs.length === 0) continue; for (const s of blockSeqs) coveredSeqs.add(s); const maxCover = Math.max(...blockSeqs); + // #4746: the anchor's slot is the first view row PAST the block's + // coverage — when the coverage ends at (or beyond) the window's last + // row there IS no later row, and the anchor belongs at the very end + // (mirror of snapshot-render.ts's `let at = d` default + its tail + // flush). Skipping it (the old `continue`) suppressed the covered + // rows AND emitted no acp_summary anchor: the model silently lost + // the fold's context while the log still counted folded=N. const pos = viewRows.findIndex((r) => r.seq > maxCover); - if (pos === -1) continue; // the fold extends past the requested window - anchors.push({ pos, msg: { id: `acp_summary_${block.blockId}`, role: "system", contentType: "text", text: "" } as BiliMessage }); + anchors.push({ pos: pos === -1 ? viewRows.length : pos, msg: { id: `acp_summary_${block.blockId}`, role: "system", contentType: "text", text: "" } as BiliMessage }); } anchors.sort((a, b) => a.pos - b.pos); const merged: BiliMessage[] = []; @@ -119,6 +125,13 @@ export async function handlePluginRender(req: http.IncomingMessage, res: http.Se } if (!coveredSeqs.has(viewRows[i].seq)) merged.push(prefix[i]); } + // #4746: anchors slotted past the last view row (coverage reached the + // window's end) must still be flushed — the loop above never reaches + // their index (mirror of snapshot-render.ts's tail flush). + while (ai < anchors.length) { + merged.push(anchors[ai].msg); + ai++; + } const system = loadSystem(sessionId); // v1 renders the prefix WITHOUT cache-control marks: marks are per-turn diff --git a/src/preflight.ts b/src/preflight.ts index 4c90bf5c26..230bc00f39 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -258,6 +258,17 @@ export interface PreflightDeps { upstreamOrigin?: string; /** #2133: compress.streamSummary resolved true for this request (three-level cascade). The self-learn flag only sees 400 "stream required" rejections, so gateways that time out long non-streaming completions (Cloudflare 524) can never self-heal — this forces SSE from the first attempt instead. */ forceStreamSummary?: boolean; + /** #4288 rework: true ONLY when this invocation re-runs after the upstream + * PROVED an overflow (the #1195/#3787 refold lanes pass the stated or + * armed window as a per-call override). There the usage baseline carries + * the upstream's own stated number (armOverflowShrink ran before us), so + * it is EVIDENCE, not staleness — the own-estimate round exit must stay + * off and folding judges on the floored quantity exactly as before. + * Absent (the normal growth/window path): a mid-session model switch + * leaves a stale baseline measured under the prior wider window, and the + * own-estimate exit stops the summarization ladder on payloads that + * already fit (#4288). */ + upstreamProvedOverflow?: boolean; /** One external-summary deadline shared by every range/chunk in this invocation. */ externalSummary?: ConfiguredSummaryPlan; /** #2155: compress.streamSummary resolved FALSE for this request (explicit operator opt-out anywhere in the cascade). Neither learn path (400 "stream required" nor the 524/504 gateway-timeout first-hit learn) may arm, and an already-armed session flag is ignored — the operator said this upstream must never stream summaries. */ @@ -2049,6 +2060,29 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // (the floor can be stale — see PreflightResult.payloadEstimate). result.payloadEstimate = roundText + imageReserve; if (startTokens < 0) startTokens = currentTokens; + // #4288: the round-boundary exit must judge on the payload's OWN + // estimate — the same floorless quantity the caller's forward gate + // trusts (#1839/#1492) — NOT on decisionTokens, which is floored by + // the session's usage baseline. After a mid-session switch to a + // smaller-window model that baseline was measured under the prior + // (wider) model and sits above the new target, so a floored + // comparison never trips and the loop burns the whole slow + // summarization ladder on a payload that already fits (the client + // then times out mid-ladder and the session appears stuck). + // EXCEPTION (#4288 rework): when the upstream PROVED the overflow + // (#1195/#3787 refold lanes — upstreamProvedOverflow), the baseline + // carries the upstream's stated number and outranks our own estimate: + // a locally-fitting payload was just rejected by the real window, so + // folding must continue on the floored quantity exactly as before + // (the original #4294 exit without this guard killed the #1195 + // rescue: the mock upstream's stated 130k made a 17k payload look + // fitting and preflight returned an unchanged body). + if (deps.upstreamProvedOverflow !== true && result.payloadEstimate < limit) { + if (baselineFloor >= limit) { + deps.log("info", `[preflight] payload fits by its own estimate (~${Math.round(result.payloadEstimate)}/${limit}); stopping instead of folding further despite a higher usage baseline (~${Math.round(baselineFloor)} tok measured under a prior model/route) — forwarding (#4288)`); + } + break; + } if (decisionTokens < textTarget) break; // #847: drop sub-minimum ranges at list level too — every chunk of a // sub-min range fails the apply-side gate, so walking them only burns diff --git a/src/server.ts b/src/server.ts index 2dd74daee8..0f0c4cff1b 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3301,6 +3301,9 @@ export async function preflightCompressIfNeeded( anonymous: boolean, log: (level: string, msg: string) => void, instanceId: string, + /** #4288 rework: true only on the #1195/#3787 refold lanes (the upstream + * PROVED the overflow); see PreflightDeps.upstreamProvedOverflow. */ + upstreamProvedOverflow?: boolean, ): Promise { const session = prepared.session; // #3450: when an upstream-stated overflow proved this provider bills the @@ -3733,6 +3736,7 @@ export async function preflightCompressIfNeeded( upstreamOrigin: currentOrigin, forceStreamSummary, streamSummaryOff, + ...(upstreamProvedOverflow === true ? { upstreamProvedOverflow: true } : {}), }, prepared.originalMessages, ); diff --git a/src/server/handle.ts b/src/server/handle.ts index 1ec676fcd4..e1cc42e53b 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -2524,8 +2524,15 @@ async function handleRequest( // request (the two-walls split let "usage=105%" and // "target 1048576" coexist in the same second). Downward only // — a stated ceiling above the declaration never loosens it. + let upstreamProvedOverflow = false; if (overflowWindow !== undefined && typeof reqConfig.modelContextLimit === "number" && reqConfig.modelContextLimit > 0 && overflowWindow < reqConfig.modelContextLimit) { reqConfig = { ...reqConfig, modelContextLimit: overflowWindow }; + // #4288 rework: this clamp is the signature of the #1195/#3787 + // refold lanes — the upstream PROVED the payload overflows, so + // the usage baseline (armed from the stated number just before + // us) is evidence, not staleness. preflight's own-estimate + // round exit must stay OFF here. + upstreamProvedOverflow = true; } prepared = await runPrepare(); memProbe(opts.debug, log, session.id, "prepared", typeof prepared.body === "string" ? Buffer.byteLength(prepared.body) : prepared.body.byteLength); @@ -2622,6 +2629,7 @@ async function handleRequest( anonAffinity !== null, log, instanceId, + upstreamProvedOverflow, ); if (isPreflightFailFast(outcome)) { // #301: the payload still overflows the window and diff --git a/src/storage/mysql.ts b/src/storage/mysql.ts index f3c46ea373..1763125dbc 100644 --- a/src/storage/mysql.ts +++ b/src/storage/mysql.ts @@ -520,20 +520,35 @@ export class MysqlBackend implements StorageBackend { } // #4316 layer 3 (mirror of store.ts): resolve coverage to canonical // seq bounds from the append-only transcript; unresolvable keeps 0. + // #4741/#4742 (mysql twin): point-query only the covered ids; an + // unresolvable span is UNKNOWN — stored as NULL, never a + // degenerate [0,0] that reads like a real interval claim. + const coveredIds = new Set(); + for (const b of input.blocks) { + try { + const cov = JSON.parse(b.payloadJson) as { effectiveMessageIds?: string[]; directMessageIds?: string[] }; + for (const id of [...(cov.effectiveMessageIds ?? []), ...(cov.directMessageIds ?? [])]) coveredIds.add(id); + } catch { /* malformed payload: bounds stay unknown */ } + } const seqSpanById = new Map(); - if (input.blocks.length > 0) { - const spanRows = (await c.query( - "SELECT msg_id, seq FROM messages WHERE session_id = ?", - [s.sessionId], - )) as unknown as Row[][]; - for (const r of spanRows[0]) { - const msgId = str(r.msg_id); - const seq = Number(r.seq); - const cur = seqSpanById.get(msgId); - if (!cur) seqSpanById.set(msgId, [seq, seq]); - else { - if (seq < cur[0]) cur[0] = seq; - if (seq > cur[1]) cur[1] = seq; + { + const ids = [...coveredIds]; + for (let i = 0; i < ids.length; i += 500) { + const chunk = ids.slice(i, i + 500); + const ph = chunk.map(() => "?").join(","); + const spanRows = (await c.query( + `SELECT msg_id, seq FROM messages WHERE session_id = ? AND msg_id IN (${ph})`, + [s.sessionId, ...chunk], + )) as unknown as Row[][]; + for (const r of spanRows[0]) { + const msgId = str(r.msg_id); + const seq = Number(r.seq); + const cur = seqSpanById.get(msgId); + if (!cur) seqSpanById.set(msgId, [seq, seq]); + else { + if (seq < cur[0]) cur[0] = seq; + if (seq > cur[1]) cur[1] = seq; + } } } } @@ -548,8 +563,9 @@ export class MysqlBackend implements StorageBackend { if (sp[0] < lo) lo = sp[0]; if (sp[1] > hi) hi = sp[1]; } - } catch { /* malformed payload: keep the 0 placeholder */ } - return hi < 0 ? [0, 0] : [lo, hi]; + } catch { /* malformed payload: bounds stay unknown */ } + // Unknown → -1/-1 no-claim sentinel, never degenerate [0,0] (#4741). + return hi < 0 ? [-1, -1] : [lo, hi]; }; for (const b of input.blocks) { const [bStartSeq, bEndSeq] = boundsForCovered(b.payloadJson); diff --git a/src/storage/store.ts b/src/storage/store.ts index a13acf4edb..c1fb121119 100644 --- a/src/storage/store.ts +++ b/src/storage/store.ts @@ -863,18 +863,38 @@ export class UnifiedStore { // bounds from the append-only transcript. Final absolute seqs exist // only here (delta placeholders are renumbered in-transaction), so // the pure planner cannot know them. Coverage whose ids have no - // transcript row (imported pre-fold) keeps the 0 placeholder. - const seqSpanById = new Map(); + // transcript row (imported pre-fold) is UNKNOWN — stored as NULL + // (no claim), never a degenerate [0,0]: kernel's interval + // consumers treat [0,0] as a live claim over seq 0 and would + // mis-cover an unrelated message (#4741). + // #4742: point-query only the covered ids (the full-table scan ran + // on EVERY save of every folded session; span resolution needs at + // most the ids one block's payload names). + const coveredIds = new Set(); if (input.blocks.length > 0) { - const rows = this.db - .prepare("SELECT msg_id, seq FROM messages WHERE session_id = ?") - .all(s.sessionId) as Array<{ msg_id: string; seq: number }>; - for (const r of rows) { - const cur = seqSpanById.get(r.msg_id); - if (!cur) seqSpanById.set(r.msg_id, [r.seq, r.seq]); - else { - if (r.seq < cur[0]) cur[0] = r.seq; - if (r.seq > cur[1]) cur[1] = r.seq; + for (const b of input.blocks) { + try { + const cov = JSON.parse(b.payloadJson) as { effectiveMessageIds?: string[]; directMessageIds?: string[] }; + for (const id of [...(cov.effectiveMessageIds ?? []), ...(cov.directMessageIds ?? [])]) coveredIds.add(id); + } catch { /* malformed payload: bounds stay unknown */ } + } + } + const seqSpanById = new Map(); + { + const ids = [...coveredIds]; + for (let i = 0; i < ids.length; i += 500) { + const chunk = ids.slice(i, i + 500); + const ph = chunk.map(() => "?").join(","); + const rows = this.db + .prepare(`SELECT msg_id, seq FROM messages WHERE session_id = ? AND msg_id IN (${ph})`) + .all(s.sessionId, ...chunk) as Array<{ msg_id: string; seq: number }>; + for (const r of rows) { + const cur = seqSpanById.get(r.msg_id); + if (!cur) seqSpanById.set(r.msg_id, [r.seq, r.seq]); + else { + if (r.seq < cur[0]) cur[0] = r.seq; + if (r.seq > cur[1]) cur[1] = r.seq; + } } } } @@ -889,8 +909,11 @@ export class UnifiedStore { if (sp[0] < lo) lo = sp[0]; if (sp[1] > hi) hi = sp[1]; } - } catch { /* malformed payload: keep the 0 placeholder */ } - return hi < 0 ? [0, 0] : [lo, hi]; + } catch { /* malformed payload: bounds stay unknown */ } + // Unknown → the -1/-1 no-claim sentinel (planner convention, + // ingest.ts:372): never a degenerate [0,0] that reads like a + // real interval claim over seq 0 (#4741). + return hi < 0 ? [-1, -1] : [lo, hi]; }; for (const b of input.blocks) { const [bStartSeq, bEndSeq] = boundsForCovered(b.payloadJson); @@ -935,8 +958,8 @@ export class UnifiedStore { b.blockId, s.sessionId, b.ord, - b.startSeq, - b.endSeq, + bStartSeq, + bEndSeq, b.startRef, b.endRef, b.tier, diff --git a/src/web/blogfeed.ts b/src/web/blogfeed.ts index 6efe1560a7..bdf692f3dd 100644 --- a/src/web/blogfeed.ts +++ b/src/web/blogfeed.ts @@ -78,14 +78,17 @@ export function _setBlogRootForTest(root: string | null): void { function parseFrontmatter(text: string): { fm: Record; body: string } { const fm: Record = {}; let body = text; - if (text.startsWith("---\n")) { - const end = text.indexOf("\n---\n", 4); + // Git-for-Windows autocrlf checks .md posts out with CRLF; normalize so + // the exact-match fences below survive Windows checkouts (#4599). + const src = text.startsWith("---\r\n") ? text.replace(/\r\n/g, "\n") : text; + if (src.startsWith("---\n")) { + const end = src.indexOf("\n---\n", 4); if (end !== -1) { - for (const line of text.slice(4, end).split("\n")) { + for (const line of src.slice(4, end).split("\n")) { const j = line.indexOf(":"); if (j > 0) fm[line.slice(0, j).trim()] = line.slice(j + 1).trim().replace(/^["']|["']$/g, ""); } - body = text.slice(end + 5); + body = src.slice(end + 5); } } return { fm, body }; diff --git a/tests/corrupt-index-child.ts b/tests/corrupt-index-child.ts index 2d59e4a869..76eb4d5335 100644 --- a/tests/corrupt-index-child.ts +++ b/tests/corrupt-index-child.ts @@ -43,7 +43,7 @@ import type { ProxyOptions } from "../src/config.ts"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; import { loadLegacyView, toLegacyLike } from "../src/storage/ingest.ts"; -import { canonicalize, sha256 } from "./golden-canonical.ts"; +import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts"; // --------------------------------------------------------------------------- // Shared with the parent test files (moved verbatim, single definition). @@ -81,6 +81,73 @@ export function viewStateDigest(view: ReturnType): string }))); } +/** #4316 layer 3 PR3/3 (380ef4dac): a unified full-mode cold load performs a + * ONE-TIME ledger alignment — citation numbers are re-homed onto ledger + * sequence (kernel restampRefsToLedger) and state.refLedgerAligned is stamped. + * Cross-mode / cross-read-path "identical state" proofs must therefore compare + * the state modulo that one-time renumbering: this strips the alignment stamp + * and re-homes every ref number to a canonical order (k-th smallest raw id -> + * m0000k), so two views differing ONLY in which numbering scheme produced + * their refs hash identically under viewDigest/viewStateDigest. Object key + * order is normalized too: different hydration paths build identical content + * under different insertion orders, and canonicalize preserves them. Pure + * function — returns a new object, input untouched. */ +export function stripOneTimeAlignment(view: T): T { + const v = view as { state?: unknown; blockContents?: unknown }; + if (!v.state || typeof v.state !== "object") return view; + const byRaw = (v.state as { messageRefs?: { byRaw?: Record } }).messageRefs?.byRaw; + const remap = new Map(); + if (byRaw) { + for (const [raw, ref] of Object.entries(byRaw).sort((a, b) => (a[0] < b[0] ? -1 : 1))) { + remap.set(ref, `m${String(remap.size + 1).padStart(5, "0")}`); + } + } + // The JSON pass below flattens Maps — pull blockContents out while it is + // still a Map, renumber its keys afterwards, rebuild it. Everything else + // in these views is JSON-native. + const bcIsMap = v.blockContents instanceof Map; + let bcEntries: Array<[string, string]> | null = null; + let source: unknown = view; + if (bcIsMap) { + bcEntries = [...(v.blockContents as Map).entries()]; + source = { ...(v as Record), blockContents: undefined }; + } + let json = JSON.stringify(canonicalize(source)); + if (remap.size > 0) { + // Single-pass alternation: sequential per-token replacement would chain + // (a target that is also another source gets re-mapped on the next pass). + const re = new RegExp(`\\b(${[...remap.keys()].join("|")})\\b`, "g"); + json = json.replace(re, (tok) => remap.get(tok)!); + } + const out = JSON.parse(json) as (T & { state?: Record; blockContents?: unknown }) & Record; + if (out.state && "refLedgerAligned" in out.state) delete out.state["refLedgerAligned"]; + // cacheLedger line `bd` digests hash the RAW outbound bodies — bodies whose + // only cross-leg delta is this same citation renumbering. The bodies are + // compared directly (per-turn shaCiteNorm), so masking the derived digests + // hides nothing; keeping them would fail every aligned comparison. + const ledger = (out.metadata as { cacheLedger?: { lines?: Array> } } | undefined)?.cacheLedger; + // Segment bookkeeping, not content, in the aligned view (#4599): a restart + // appends a fresh open tail line and re-anchors the closed-segment byte + // boundaries (be/bl/bs), so no-restart and restart legs legitimately differ + // there. Keep the closed segments' content; drop the unfinalized tail and + // the bookkeeping trio everywhere. + if (Array.isArray(ledger?.lines)) { + while (ledger.lines.length > 0 && ledger.lines[ledger.lines.length - 1]?.bl === undefined) ledger.lines.pop(); + for (const line of ledger.lines) { delete line["bd"]; delete line["be"]; delete line["bl"]; delete line["bs"]; delete line["prov"]; delete line["seam"]; } + } + // agg is the same experience telemetry summed: a restart re-partitions the + // provider-cache misses into seam buckets (seamMissed/seamSuspects vs + // providerSideMissed), so legs legitimately disagree. Per-line content + // fields stay; only the derived aggregate goes. + if (ledger) delete (ledger as { agg?: unknown }).agg; + // snapshotShadow is the shadow-copier's OWN health telemetry — it reports + // the ingest pipeline's experience of building this view (errors, watermark + // drops), which is inherently load-path-dependent, not conversation state. + if (out.metadata && "snapshotShadow" in (out.metadata as Record)) delete (out.metadata as Record)["snapshotShadow"]; + if (bcEntries) out.blockContents = new Map(bcEntries.map(([k, val]) => [remap.get(k) ?? k, val])); + return sortObjectKeys(out) as T; +} + export interface StorageWire { requested: string; effective: string; @@ -277,8 +344,12 @@ async function engproofScenario(argv: string[], run: CheckRun): Promise { // Regression (#2781): the failed open must have CLOSED its handle. await runGuard(run, ENGPROOF_CHECK_NAMES.midFlowGuard, dir, "-moved"); + // #4316 PR3/3: the clean full-mode read carried the one-time ledger + // alignment; the legacy fallback over the corrupt index did not. The + // parent passes its digest already modulo that alignment (single + // definition), so strip it here too before comparing. await run.run(ENGPROOF_CHECK_NAMES.stateDigest, () => - assert.equal(viewStateDigest(toLegacyLike(reloaded)), expectedStateDigest, + assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(reloaded))), expectedStateDigest, "corrupt-index fallback state digest drift vs clean full-mode read")); // Writes must keep working (legacy path) — no throw, file refreshed. diff --git a/tests/golden-canonical.ts b/tests/golden-canonical.ts index 09f491f2a7..6736b33311 100644 --- a/tests/golden-canonical.ts +++ b/tests/golden-canonical.ts @@ -4,6 +4,22 @@ export { canonicalize, sha256 } from "../src/storage/canonical.ts"; import { canonicalize, sha256 } from "../src/storage/canonical.ts"; +/** Deterministic key order for cross-read-path digest comparisons: two views + * built by different hydration paths carry identical content under different + * map insertion orders, and canonicalize preserves insertion order — sorting + * here makes the digests path-independent. Maps/Sets pass through untouched + * (callers that need them iterable keep them intact; canonicalize flattens + * them downstream). */ +export function sortObjectKeys(v: unknown): unknown { + if (Array.isArray(v)) return v.map(sortObjectKeys); + if (v && typeof v === "object" && !(v instanceof Map) && !(v instanceof Set)) { + const out: Record = {}; + for (const k of Object.keys(v as Record).sort()) out[k] = sortObjectKeys((v as Record)[k]); + return out; + } + return v; +} + /** Canonical JSON → sha256, with a final pass of STRING-level environment * normalization (ports, session ids embedded inside text fields). */ function digestOf(value: unknown, extra?: { normalize?: (s: string) => string }): string { diff --git a/tests/golden-storage-drive.ts b/tests/golden-storage-drive.ts index a6b14205f8..4d65412aca 100644 --- a/tests/golden-storage-drive.ts +++ b/tests/golden-storage-drive.ts @@ -16,10 +16,11 @@ import http from "node:http"; import os from "node:os"; import path from "node:path"; import { once } from "node:events"; -import { canonicalize, sha256 } from "./golden-canonical.ts"; +import { canonicalize, sha256, sortObjectKeys } from "./golden-canonical.ts"; import { defaultConfig } from "acp-kernel"; import { startServer } from "../src/server.ts"; import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { stripOneTimeAlignment } from "./corrupt-index-child.ts"; import { flushAllSessions, _resetSessionsForTest } from "../src/session.ts"; import type { ProxyOptions } from "../src/config.ts"; @@ -110,7 +111,7 @@ const GROW_TURNS = 20; // hard cap for the grow loop const SUMMARY_MARKER = "[Compressed conversation section]"; const EDIT_SENTINEL = "GOLDEN-EDIT-SENTINEL"; -interface GoldenTurn { i: number; sha: string; bytes: number } +interface GoldenTurn { i: number; sha: string; bytes: number; shaCiteNorm: string } interface GoldenRecord { wire: Wire; turns: GoldenTurn[]; @@ -118,6 +119,7 @@ interface GoldenRecord { editedAt: number; // body index after the mid-history edit summaryFrom: number; // first body index carrying the fold summary stateDigest: string; // sha256 of the canonicalized persisted session + stateDigestAligned: string; // #4316 PR3/3: same, modulo the one-time ledger alignment sectionDigests: Record; // drift localization (same normalization) stateKeyDigests: Record; // per-top-level-state-key digests stateShape: { // human-readable facts baked into the snapshot @@ -169,6 +171,18 @@ function canonicalBody(body: string): string { .replace(/msg-fix-[0-9a-f]+/g, "msg-fix-"); } +// #4316 PR3/3 (380ef4dac): a full-mode cold load re-homes citation numbers +// once (one-time ledger alignment) — that renumbering is the ONLY permitted +// wire delta vs the legacy restart. Wire-parity proofs therefore pin a second +// digest per turn with every citation number masked to : equality says +// "only the citation numbers may differ", everything else stays byte-exact. +// All m\d{5} tokens are masked (not just tag payloads) because summaries cite +// numbers in prose too; user content is identical across legs, so the wider +// mask cannot hide a real divergence. +function citeNormBody(body: string): string { + return body.replace(/\bm\d{5}\b/g, ""); +} + /** Timestamps (ms-epoch magnitude) and UUIDs become placeholders; everything * else — refs, block ids, coverage sets, stats — must be byte-stable. */ // canonicalize/sha256 live in ./golden-canonical.ts (shared with the storage-corpus pins). @@ -585,15 +599,7 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: // order, so a harmless field-reorder commit (#3093) red all four // goldens. Deep-sort object keys harness-side only; array order is // preserved (foldAnchorOrder & co are ordered contracts). - const sortObjectKeys = (v: unknown): unknown => { - if (Array.isArray(v)) return v.map(sortObjectKeys); - if (v && typeof v === "object") { - const o: Record = {}; - for (const k of Object.keys(v as Record).sort()) o[k] = sortObjectKeys((v as Record)[k]); - return o; - } - return v; - }; + const lastView = JSON.stringify(canonicalize(sortObjectKeys({ state: session.state, blockContents: [...session.blockContents.entries()], @@ -606,6 +612,18 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: .replace(/127\.0\.0\.1:\d+/g, "127.0.0.1:") .split(JSON.stringify(sessionId).slice(1, -1)).join(""); const stateDigest = sha256(lastView); + // #4316 PR3/3: parallel digest modulo the one-time ledger alignment — + // cross-mode proofs compare states that may differ only in ref numbering. + const stateDigestAligned = sha256( + JSON.stringify(canonicalize(sortObjectKeys(stripOneTimeAlignment({ + state: session.state, + blockContents: [...session.blockContents.entries()], + stats: session.stats, + metadata: session.metadata, + })))) + .replace(/127\.0\.0\.1:\d+/g, "127.0.0\.1:") + .split(JSON.stringify(sessionId).slice(1, -1)).join(""), + ); // Localization aid for cross-machine digest drift: per-section and // per-top-level-state-key digests under the SAME normalization, so a // CI failure message pinpoints the divergent subtree without needing @@ -629,11 +647,15 @@ export async function driveWire(wire: Wire, sessionId = `golden-${wire}`, opts: const active = st.blocks?.filter((b) => b.active) ?? []; return { wire, - turns: bodies.map((b, i) => ({ i, sha: sha256(canonicalBody(b)), bytes: Buffer.byteLength(b, "utf8") })), + turns: bodies.map((b, i) => { + const cb = canonicalBody(b); + return { i, sha: sha256(cb), bytes: Buffer.byteLength(b, "utf8"), shaCiteNorm: sha256(canonicalBody(citeNormBody(b))) }; + }), foldAt, editedAt, summaryFrom: bodies.findIndex((b) => b.includes(SUMMARY_MARKER)), stateDigest, + stateDigestAligned, sectionDigests, stateKeyDigests, stateShape: { diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 5afa14b2c4..43e74c9457 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -4,88 +4,104 @@ { "i": 0, "sha": "daf2182687948ef53790608f8fe13f21f457552bd56c61e914699d71793a95d0", - "bytes": 22287 + "bytes": 22287, + "shaCiteNorm": "37a0f1ef30156301cae88e2dece8a03f8ba557440925d3f8fd4b6f654e9880aa" }, { "i": 1, "sha": "2894d8949694f953aef9b0a7d4c4ad03b3633b8cd90886852109468d10e72b45", - "bytes": 30997 + "bytes": 30997, + "shaCiteNorm": "ae251c0c242448c1d87f417194f38d42c901e563ca0234a616e22254d8d29352" }, { "i": 2, "sha": "911f0c635e145362329fef63920e7b4802f2f9c9528062173e867ad891832873", - "bytes": 36896 + "bytes": 36896, + "shaCiteNorm": "efbc1fcc13a7968e0dd8120bc1dc0e04b24d0aac997700fe52dd414e0270c351" }, { "i": 3, "sha": "425dee6b9938613b65f9c944aea28228eda5679915fd39597196876dda2915f6", - "bytes": 45569 + "bytes": 45569, + "shaCiteNorm": "0d5a15e5216f11dd47816ff0bc336532ffdaec00fecad51ff2c9995a4bb00254" }, { "i": 4, "sha": "e62cbddd7d1facbcf2efe3c781c09b14b5cf096067b7d7ae4818ee7a4639e0b6", - "bytes": 51431 + "bytes": 51431, + "shaCiteNorm": "43615d160671fcbd9408a80e0779f6cf122fdbde81b901e35a277d8e6ee8495a" }, { "i": 5, "sha": "03a6265741bdb3f74788ae479f5b1505e552e6082009dd3419605dff8143b5da", - "bytes": 60104 + "bytes": 60104, + "shaCiteNorm": "66411dc16ffa93bb1fbc28b167302372c7b170af10bfca6c95ef9466cf4b8628" }, { "i": 6, "sha": "c7f62d7a595006a08aaa71427f0bcdb3b620c8eac104723bc6ace7516aa3321c", - "bytes": 65966 + "bytes": 65966, + "shaCiteNorm": "c9d9ff30521f6d1fa5e3e94f9ed778a1491224fc6440cd8bc3dde9e6bd7f77e6" }, { "i": 7, "sha": "edb9fadc1d20813a24f45ea2d1e2ab0d92972d1cee22d45ac278d9dd1c17bd4c", - "bytes": 74639 + "bytes": 74639, + "shaCiteNorm": "4e4c14113531fb4152983af6c3596dd3944f30ca0ab4e720e92a31bd7a253a9d" }, { "i": 8, "sha": "d2be70c3953a492bf1b974ff4922001d74d0d61012446c59e732fec18852d713", - "bytes": 80501 + "bytes": 80501, + "shaCiteNorm": "ca749fa6f80100f404d828594314c8823dc2900aedb089264a057e72f54536fa" }, { "i": 9, "sha": "2ddad1b2186f7f0444543f1b72b2f4fe3dcfe6d8cf709211a54e820b7c3a6a4a", - "bytes": 49528 + "bytes": 49528, + "shaCiteNorm": "8008f1d0bbe5e604ca7b1201e6d60e04ab7a10c117373eda0412308319f5c6b2" }, { "i": 10, "sha": "7da7b1f5aed39a2af9df234e8ad97fff4fe1cf49623a3adc4a3c45cd22989be0", - "bytes": 57070 + "bytes": 57070, + "shaCiteNorm": "09e0e277a54ce862f28c41a40d4b1261a8b8198240a10191986f07ddb1b9ad34" }, { "i": 11, "sha": "fca40b7b4dae14bfa94542cb8d802ce7b097b7b2946f987437db98a1cbc0385c", - "bytes": 62892 + "bytes": 62892, + "shaCiteNorm": "58178314a73fd15478a8d1cef6e6f3f535e216ff3a27d5d5c0af85f3cc480fe5" }, { "i": 12, "sha": "8b5ff77e47604fe58cbb19234306863e3ec754c1d51a7a354bab40ffb12ef59a", - "bytes": 77030 + "bytes": 77030, + "shaCiteNorm": "0afc5df7cd478a9a127da7cb011c61a20cbf35ebe31409daad93e0e9c2944f24" }, { "i": 13, "sha": "375a80af22ca3da9763155fd5394cff9066e0695490105830137cdcd9432f7d1", - "bytes": 82815 + "bytes": 82815, + "shaCiteNorm": "c16a407ec4983f77cd62792e46a093c839853077d4b52cd799b81adb38a1ca20" }, { "i": 14, "sha": "3f23c183e08dfe04772ea8cf1b5196f8ad92a57868896d55aafc7eb9300ed53a", - "bytes": 91464 + "bytes": 91464, + "shaCiteNorm": "415619a3a81f1aad4f2790ffc63eaeb3a0410e27027e77609a01f0fbd1471abe" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "6727f42bd91879a7aa370e1ce0c78963bcba0df92e50390d4a8011f8f0da334d", + "stateDigest": "0a241435d456ba994c92b0072d2173bf2a6d00ee73223458fb4302e170efb1e3", + "stateDigestAligned": "c7079cd68bf9eb0075c3844a5cb1ae54ff6ecad8a43da424a7ffebd033eceb7f", "sectionDigests": { "state": "f93dc1c876c672d4", "blockContents": "dc7d13d250c94bb4", "stats": "c6f1d434975d047d", - "metadata": "f1ec4aab3aff777a" + "metadata": "16cad648ccf41950" }, "stateKeyDigests": { "blocks": "3a454b9a16e4f0b9", diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 46764f97db..79d5eb6b70 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -4,88 +4,104 @@ { "i": 0, "sha": "824973c12a04e4a9c6fc8a5b84555886597d1a41874e6b0da33a489fe5f9fe9b", - "bytes": 22397 + "bytes": 22397, + "shaCiteNorm": "40255e64416e7fbea076dfe1195d32eac811a33d6e48db44922f93bfd3dadc31" }, { "i": 1, "sha": "4261484dd0b52e37019fe840a217bf4940369c7cb3c6d1ac28cb6353d4a68a37", - "bytes": 30975 + "bytes": 30975, + "shaCiteNorm": "7630710b673b4aa2dd1a7aade6809f5a97bdcd525857a37e9ab253dffcb82f43" }, { "i": 2, "sha": "e8050d3ec8b0e3dd2f0ca67a4a0328a8e280e23b6f4a3e34380af58e8d38d678", - "bytes": 36787 + "bytes": 36787, + "shaCiteNorm": "ad6b3f4bda6cf0b9b779034ea1b0ceb715c7a763df6b33eab4d5df7862d84b14" }, { "i": 3, "sha": "73d69f8b8f1bd2899cfe2181b95bc2f3697695e9b85ad682d82f6cbd9e7355ff", - "bytes": 45365 + "bytes": 45365, + "shaCiteNorm": "d683426210363474b3723a9dfd4e0e58e9eaf8b747291106d0eb22b86d6b1466" }, { "i": 4, "sha": "d3a73b4a74f7975de837b7848f24d875866b1f2b0b15f4ef78064e3602da6665", - "bytes": 51177 + "bytes": 51177, + "shaCiteNorm": "7b8084737aabd65b5d9be87be1610819f658c14a632de7f3575b6fbf44860e0b" }, { "i": 5, "sha": "60ce463e424f124a2dc346479091cbd4b7a32c69696b798a1de6b1a3c0f3ef7b", - "bytes": 59755 + "bytes": 59755, + "shaCiteNorm": "7b96653787fc0adda16590f35f3cb70f47bbecfed5e8208bc9f9520f0c360330" }, { "i": 6, "sha": "1298d5f45146b76a42b5b75c712de8011fb091c5754f99d490a7df2e81ba2466", - "bytes": 65567 + "bytes": 65567, + "shaCiteNorm": "33e2e632f57a5aeec30801779d9e38fcc55247389ac83217e88e0c6500b6a783" }, { "i": 7, "sha": "b20457e685b4a1fecf0195f57b93e052a0d00a83feb8b1569ccb08d385663aa2", - "bytes": 74145 + "bytes": 74145, + "shaCiteNorm": "0c7a0b8156c69d5710d2e395b6b8c985878c9120de8ff6c8f1fbe957649b855d" }, { "i": 8, "sha": "a0ba24010c6faf10f11b66faca6f3aa915dde7ac006d66a434777cbd765328ca", - "bytes": 79957 + "bytes": 79957, + "shaCiteNorm": "180576856d481f74a2e1fd352cb1ee881644fe905fb3263f16b12c90a7f5b86a" }, { "i": 9, "sha": "f1e08e530a96ecb4996c0a63e840e3f374a82b095e285ffea94e70391eb32fe0", - "bytes": 49458 + "bytes": 49458, + "shaCiteNorm": "7f1a433d2f553cc122e877912c99464485726c3d0707c5ffd12acd2478475176" }, { "i": 10, "sha": "8f3e444b1951f67e4f86abf6e727162108ba39d389b1ee4c90eea69df15bada0", - "bytes": 56859 + "bytes": 56859, + "shaCiteNorm": "90d0ab02fca529d89c1e045ec2bb157872e5b70508975b6d1782fc6c0a525cc9" }, { "i": 11, "sha": "94a8e322c3c0822a18bccd6295c3d2bfb4c7961d897298361612f286fb6108fe", - "bytes": 62594 + "bytes": 62594, + "shaCiteNorm": "9ca56f6252eb19787f5577ed01833ea816d242299c4f7a3b24351b7fb767860e" }, { "i": 12, "sha": "2c0a965647d8832dc74ebe5edc8cfe59b67abc936714119e89c93410f985e1af", - "bytes": 76640 + "bytes": 76640, + "shaCiteNorm": "39c2ab72032ed2149fdbb66698cffc18f6e0d96f713fb37181e08cb0cadc993a" }, { "i": 13, "sha": "a7497d037a324f1fdb6f605db0b8161e4e183f1e2354bb68035fb9f1373ee0d0", - "bytes": 82375 + "bytes": 82375, + "shaCiteNorm": "001e2a63c5352cd3230a30bc82dd6f9fbb8c6421f42e1bc8dd87551f50b8ae9f" }, { "i": 14, "sha": "abd73cec691568d84e72384aa5facdc71c759eeafc865d62df7b4eca6683675a", - "bytes": 90929 + "bytes": 90929, + "shaCiteNorm": "0f6af62ddbc1c0a39009524d581f52f728e928d438e998a716be5d054a8c6609" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "e149c301f382438ee063a38f70dd794207b88e94bdf3c14c5191da0eeaa99bc9", + "stateDigest": "9f1b8258a916fcf63af14625def9ed0236eb8ec9c542f0027206985598d2a5b8", + "stateDigestAligned": "86e45900d61e81f0c8239bfe4b4019451b600c2a5258e89e53c68bc09fd2d5ca", "sectionDigests": { "state": "8390f21336885863", "blockContents": "94a58f96d7c5cfc3", "stats": "24d705b0e39bd13c", - "metadata": "55733fb81ccf7d69" + "metadata": "2198f47054c8acae" }, "stateKeyDigests": { "blocks": "d9770feee91dafc6", diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index ba999cadec..efa311863f 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -4,88 +4,104 @@ { "i": 0, "sha": "45998d29279880c0b19002f5ca40489e1e9dfcbdb53b9203a5ffe868ea3e51b8", - "bytes": 20510 + "bytes": 20510, + "shaCiteNorm": "a80634db66b3e2f5c96a66d48a7baf753f089d0af76e1c7370e44a22be3e2c50" }, { "i": 1, "sha": "f61275d77406c502c11238e2764ea84c8bf03c8302856749b8f5b73572fb5331", - "bytes": 29065 + "bytes": 29065, + "shaCiteNorm": "bbb40518ce7a359b6fb4d9ccebf356a433dee955daf8676d937fc2b1e2c493e0" }, { "i": 2, "sha": "98b77dc7c79575218c8e52ddec9386d5db7a7d30587dd992cb64c18536fb7667", - "bytes": 34891 + "bytes": 34891, + "shaCiteNorm": "3c18b9c31b2c19266a4f006c368008aa6dd0dbacdeb1bc00c86f3a76cd8fd8b5" }, { "i": 3, "sha": "2f3024ee57f441a1c0d1b06d38ccf1f4c74ef297aa3d58622bc29f69c21c93d6", - "bytes": 43446 + "bytes": 43446, + "shaCiteNorm": "8e074da69eef79658d01adab2bed04d55a232eebf3e437c0a94b4afefb52bb8c" }, { "i": 4, "sha": "27f6f223e7682e0e0790a69e43c982c3fe0e505153be7fa4d3ec6cf4532ff2e2", - "bytes": 49272 + "bytes": 49272, + "shaCiteNorm": "63b04fd0418ab224c41ee6e9ee2068a73b973c8df07e4e7f2ffbe6e19373ec7a" }, { "i": 5, "sha": "6a7e23920335f91762a8ac72b177a1cd68538ce1872325b6aa9e3b44191f3ca4", - "bytes": 57827 + "bytes": 57827, + "shaCiteNorm": "723ccf3a211d8b750959b479939ded61e4c9818473bde3a7364eae85ca9ad5da" }, { "i": 6, "sha": "5b770ce0c9456af96904cd5b007fbb07fb4194f2636db8b6ad2602ef71d3834d", - "bytes": 63653 + "bytes": 63653, + "shaCiteNorm": "4c20a67e2e4df13e061f8f094b92774d2c53c60a0711b6456731b943e510eec4" }, { "i": 7, "sha": "9bab6f659c0dcd05e6576e6d47b1d1d2378a6ace1a7effec721958f6f7476cd8", - "bytes": 72208 + "bytes": 72208, + "shaCiteNorm": "c807dfe30868d78848a846469e99e7d40fe7b866717974886c30ba8def051063" }, { "i": 8, "sha": "c179b940520413ba2a0e267165a9914712c3a9ff3c045761d61da876aeccedd7", - "bytes": 78034 + "bytes": 78034, + "shaCiteNorm": "2091c5306aba0a16a47547029fe8dd6b57cc8e31b3c6a2abea7d9a8ba7b63839" }, { "i": 9, "sha": "1b1448a075964dae98ad3b09565d31590cc9fdadb4d37300c4460030f67990e6", - "bytes": 50620 + "bytes": 50620, + "shaCiteNorm": "6ef485631895d0a448bb8d52235b9f3a00ba15143b587ac7253d119c83c61441" }, { "i": 10, "sha": "eddf6ad2081f5eb1e4822bf9805a7953eff3241269ed5f3eec346909c9da5bf2", - "bytes": 58023 + "bytes": 58023, + "shaCiteNorm": "caec6185725a235e023982a9d7f6b4fcb284bb6a2e21b89bb8f2b7e36bb1bf5c" }, { "i": 11, "sha": "0ae5c9b7b31dbb0d52c1b4a157bcc08c549ff435229383ec33fb0225b3c93f4e", - "bytes": 63772 + "bytes": 63772, + "shaCiteNorm": "894e9c4f827fb398b7a6c9bc91d430633e51bd552585fb5086c2cf3c18d6097f" }, { "i": 12, "sha": "d9a62f5c48f849bdc9514ae4586f2204ef5ac624aaf97c50dbc8ed2c1bee9706", - "bytes": 77777 + "bytes": 77777, + "shaCiteNorm": "2f1449ede69c9ee7db22f39c49584f64d35a29541097d78f524a4f22c5998acc" }, { "i": 13, "sha": "f60319947e77097d08edfcc5fdddd46057c44596a30d1d6bb33c5af4bad5001f", - "bytes": 83526 + "bytes": 83526, + "shaCiteNorm": "ee505218214a0e58875762c06c79e717164b89cfe29cdd61630fdd1fecb41a92" }, { "i": 14, "sha": "275f3b3e103dbfbf545448f8c421254cc7b025f8c081d941cd39bef4a90023be", - "bytes": 92056 + "bytes": 92056, + "shaCiteNorm": "c434408819f41c9e943e1cd60b80189f1844ba2c8f11898043f158f8485aee9b" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "80e83935e5fa7b052a3cb0f1d641bd81cb6ee59f91f3c1f2f31c204de29368a5", + "stateDigest": "dc673ce01ca301e10c015a3655663f58609166605a9d47c7ed633c9b0e8d3c06", + "stateDigestAligned": "39bfb4925b08d9208da2326c6ce291fca76911fec95d8693e98299ff05ee369e", "sectionDigests": { "state": "2e1def3a3234afdc", "blockContents": "473264d9df8ba338", "stats": "900481685c848462", - "metadata": "88cd02a467b4457e" + "metadata": "d827ffae0bef4182" }, "stateKeyDigests": { "blocks": "0cfd0b9c0fc53073", diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 7004d90435..3ece8b999d 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -4,88 +4,104 @@ { "i": 0, "sha": "57662dd5517499b0833b00df645aacd9ec25a1add3b6efa94703cc40006824a8", - "bytes": 22467 + "bytes": 22467, + "shaCiteNorm": "dd2d60a2354cb8bb76834fa946ad985964def531c400f15a9f0370ee948fd039" }, { "i": 1, "sha": "ade76466ddbffb25c3eba9e06896c110c147ce1daf7f1e231fe054d9d1832c4b", - "bytes": 31395 + "bytes": 31395, + "shaCiteNorm": "9a1ea902b7287c23566da611dbab379637998536dc37365d47cd8577fa5f6a8c" }, { "i": 2, "sha": "09ee3e5a15865e5cc5c10c28026c1501f565212134c64a03214fbf367f63b881", - "bytes": 37517 + "bytes": 37517, + "shaCiteNorm": "99c6597f2bc37b03f6935208eb4be452619d599ce21b06f118f15657c5494a0b" }, { "i": 3, "sha": "cd3e38d97e64660eb60f1e387a1e53de8723ae1aa916db4d0a7b8f7ec181c266", - "bytes": 46445 + "bytes": 46445, + "shaCiteNorm": "947f17b43bc46a8f1ad828172f394e003f499af7ca16196f907620662a3a7929" }, { "i": 4, "sha": "b8ff4f16ca15cc1b8589251a849e3e30ec7d7d82b92abc0bb262bd2f8422db38", - "bytes": 52567 + "bytes": 52567, + "shaCiteNorm": "c0ce07681a604168b19c4e585112d5644555b8283a7c92c620336d0ec1453806" }, { "i": 5, "sha": "70f002ab5e7c3301ec440fd6863b7635c38874842445794bf87229f783ce2124", - "bytes": 61495 + "bytes": 61495, + "shaCiteNorm": "20c8f20fe476b978fcb07d34f4311a2712e8ea04e829b8ee0912e7741d6f95d1" }, { "i": 6, "sha": "aa63e444c4b86e426d949d2030a2b2a4f84a1018ebb152f0dd429c9747f357b3", - "bytes": 67617 + "bytes": 67617, + "shaCiteNorm": "91c714ad3016518bef0e07f9a7d227c9adcd164b5c3b1e62ee24b20bc38524d2" }, { "i": 7, "sha": "4cef1346cac328fc82bd552007051118cc95751e62d4b3f10a4432297a6801e5", - "bytes": 76545 + "bytes": 76545, + "shaCiteNorm": "68db839f454054f55c426b3e173d9baa9d4120a6f963cfa4856d4bad2b11e318" }, { "i": 8, "sha": "adee16938078b6c6cac3ca423ffe427d2199493d6cbbbc01059b223de3bba077", - "bytes": 82667 + "bytes": 82667, + "shaCiteNorm": "f8ca121812fc632d1b2cb140ac8ec41ceeb4f82916256386aa308d2a4272b09f" }, { "i": 9, "sha": "a6cefd10bbb51de65d7c363ea6f299a2301f4d8104a5ce8b59f4de87e711130d", - "bytes": 50498 + "bytes": 50498, + "shaCiteNorm": "266499f0528988fed9680ffef231be1162e6501857511a3de6a5f95ba1564256" }, { "i": 10, "sha": "ed8590ef249a0756d077fcea24230dd83a7f805a03da6f91457c607b27351c5e", - "bytes": 58283 + "bytes": 58283, + "shaCiteNorm": "5ed641acfb444fed79d86b5104b176d11250d309a649a2230ba2f3f08950453c" }, { "i": 11, "sha": "461eb77a6cc41abbf8046fb3dc5858f18c8d848a8dff4a2c888a38a1d16df707", - "bytes": 64328 + "bytes": 64328, + "shaCiteNorm": "c674b6af035c1b786fd0f47c0c33abc4179337e3df595391f3fcf9b6973ef531" }, { "i": 12, "sha": "b356956d375e951d2d5a05fe55c1e28b09c616f9b2d7eca6fea5e62924dbe53e", - "bytes": 78746 + "bytes": 78746, + "shaCiteNorm": "f8d20e84c64dfd8e224496a6feb949d16edc9de40f5100243d0514ec86eeddec" }, { "i": 13, "sha": "1b3d93417fce2ba8f70c52d0425626398a571d7afbd287ffb89a6ba7ae37bd29", - "bytes": 84794 + "bytes": 84794, + "shaCiteNorm": "b24c9b065471e5a264cb73d58b2383d9f8fc5b4feeb1f7d0fea05aeb96d1ff01" }, { "i": 14, "sha": "9c82e4523129ef8df61cf7622bb637c7379cce153c78e3814bd3000853034323", - "bytes": 93703 + "bytes": 93703, + "shaCiteNorm": "cd5c51970f832bcaa5f27406e353c8e67a53706c2bc8249e35634b6339dc893a" } ], "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "08526d3552fca7ac7552cd16a221893a652fbe740c9b8dccac6db57b3d6302ee", + "stateDigest": "b3bddfa75ce3f8b94424be9a0164c1b9a183faa9daf705e6175cabdd671c95ff", + "stateDigestAligned": "ae78cdd8a1784474f1ffa5e9c8e8871871e4498b75df568b614a977e60af2339", "sectionDigests": { "state": "db313e8eba2d65fc", "blockContents": "b6701268a0bdcd8a", "stats": "327d3967b8371aa8", - "metadata": "b5c39d84db67f424" + "metadata": "a9eecd3816738158" }, "stateKeyDigests": { "blocks": "34b0dc17f8c24595", diff --git a/tests/issue4288-preflight-stale-baseline-fit-exit.test.ts b/tests/issue4288-preflight-stale-baseline-fit-exit.test.ts new file mode 100644 index 0000000000..926070fa27 --- /dev/null +++ b/tests/issue4288-preflight-stale-baseline-fit-exit.test.ts @@ -0,0 +1,173 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { listSessions } from "../src/session.ts"; + +// #4288: mid-session switch to a SMALLER-window model wedged the session. The +// stale usage baseline (session.lastInputTokens) was measured under the PRIOR +// wider-window model and sits above the new window, so the preflight TRIGGER +// fires on that baseline alone even though the actual payload FITS the new +// window. Inside preflight the round-boundary exit judged on `decisionTokens`, +// which is FLOORED by that same stale baseline — so `decisionTokens < +// textTarget` never trips and the loop burns the whole slow summarization +// ladder on an already-fitting payload; the client times out mid-ladder and +// the session appears stuck. +// +// Fix: the round-boundary exit judges on the payload's OWN floorless estimate +// (result.payloadEstimate) — the same quantity the caller's forward gate +// trusts (#1839/#1492). When the payload fits, preflight must stop IMMEDIATELY +// (zero summarization calls) and forward, instead of folding. +// +// Scenario (mirrors tests/preflight-compress.test.ts #247, inverted): +// 1. Request 1 on claude-big (400k window) creates the session; we then pin +// its usage baseline to 300k (source "usage") — the value a PRIOR +// wide-window model left behind, i.e. the #4288 precondition. +// 2. Request 2 switches to claude-small (SMALL_WINDOW) carrying an ~18k-token +// conversation that FITS the window, yet the stale 300k baseline still +// trips the trigger. Before the fix the ladder burned (2 summarization +// calls); after the fix preflight exits at round 1 with ZERO summarization +// calls and forwards a 200. + +const SUMMARY_TEXT = + "PREFLIGHT SUMMARY: the segment covered a multi-step debugging session."; + +function okSse(inputTokens: number): string { + return ( + `event: message_start\ndata: ${JSON.stringify({ type: "message_start", message: { id: "m1", role: "assistant", usage: { input_tokens: inputTokens } } })}\n\n` + + `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "text", text: "" } })}\n\n` + + `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } })}\n\n` + + `event: content_block_stop\ndata: ${JSON.stringify({ type: "content_block_stop", index: 0 })}\n\n` + + `event: message_delta\ndata: ${JSON.stringify({ type: "message_delta", delta: { stop_reason: "end_turn", stop_sequence: null }, usage: { output_tokens: 3 } })}\n\n` + + `event: message_stop\ndata: ${JSON.stringify({ type: "message_stop" })}\n\n` + ); +} + +function bigConversation(): Array<{ role: string; content: string }> { + const msgs: Array<{ role: string; content: string }> = []; + for (let i = 0; i < 12; i++) { + const role = i % 2 === 0 ? "user" : "assistant"; + const filler = `MARKER_${i}_content_`.repeat(250); + msgs.push({ role, content: `Message ${i} of the long conversation. ${filler}` }); + } + return msgs; +} + +// #4288 calibration: bigConversation() below is ~18k tokens, so it fits 40k +// with margin while staying far below the pinned 300k stale usage baseline. +const SMALL_WINDOW = 40_000; + +test("e2e #4288: model switch to a smaller window whose payload FITS → preflight exits with zero summarization (no ladder)", async () => { + const calls: Array<{ stream: boolean; body: string }> = []; + let streamCalls = 0; + const upstream = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + let parsed: { stream?: boolean } = {}; + try { parsed = JSON.parse(raw); } catch { /* keep {} */ } + calls.push({ stream: !!parsed.stream, body: raw }); + if (parsed.stream) { + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(okSse(streamCalls === 1 ? 300_000 : 1000)); + } else { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + id: "msg_summary", + type: "message", + role: "assistant", + model: "claude-small", + content: [{ type: "text", text: SUMMARY_TEXT }], + stop_reason: "end_turn", + usage: { input_tokens: 500, output_tokens: 50 }, + })); + } + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const proxy = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "claude-big": { context: 400_000 }, "claude-small": { context: SMALL_WINDOW } } } }, + modelContextLimit: 400_000, + kernelConfig: defaultConfig(400_000), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + } as ProxyOptions); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + + try { + const headers = { "content-type": "application/json", "x-acp-session": "issue4288-sess" }; + + // --- Request 1: big model creates the session --- + const r1 = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { + method: "POST", + headers, + body: JSON.stringify({ model: "claude-big", max_tokens: 1024, stream: true, messages: [{ role: "user", content: "hello" }] }), + }); + assert.equal(r1.status, 200); + await r1.text(); + const sessions = listSessions(); + assert.equal(sessions.length, 1, "exactly one session after request 1"); + const s1 = sessions[0]!; + // Simulate the #4288 precondition directly: the session carries a usage + // baseline (300k) measured under the PRIOR wide-window model. We set it + // explicitly rather than relying on request 1's SSE usage report, because + // the point under test is the preflight round-EXIT given a stale-high + // baseline + a fitting payload — the baseline-establishment path is + // orthogonal (and covered by other suites). + s1.stats.lastInputTokens = 300_000; + s1.stats.lastInputTokensSource = "usage"; + + // --- Request 2: switch to the SMALL_WINDOW model with the ~18k conversation --- + // The payload fits the window, but the stale 300k baseline still trips + // the trigger. The fix must make preflight exit at round 1 with ZERO + // summarization calls and forward a 200 (before the fix the ladder ran). + const r2 = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { + method: "POST", + headers, + body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: bigConversation() }), + }); + assert.equal(r2.status, 200, "the switched-model request succeeds (no context-full error, no mid-ladder abort)"); + await r2.text(); + + const summaryCalls = calls.filter((c) => !c.stream); + assert.equal(summaryCalls.length, 0, `payload fits the new window — preflight must NOT burn the summarization ladder (got ${summaryCalls.length} call(s); #4288)`); + } finally { + proxy.close(); + proxy.closeIdleConnections?.(); + await once(proxy, "close"); + upstream.close(); + upstream.closeIdleConnections?.(); + await once(upstream, "close"); + } +}); diff --git a/tests/issue4316-seq-bounds.test.ts b/tests/issue4316-seq-bounds.test.ts index e391b35d20..36e669b817 100644 --- a/tests/issue4316-seq-bounds.test.ts +++ b/tests/issue4316-seq-bounds.test.ts @@ -9,9 +9,9 @@ import type { CoreMessage, CompressionBlock } from "acp-kernel"; // #4316 layer 3 (owner ruling 2026-10-22): block coverage becomes a canonical // [start_seq, end_seq] interval so a client byte-rewrite can't churn held -// m-numbers. RED today — start_seq/end_seq are hard-wired to 0 at the single -// write site (src/storage/ingest.ts); GREEN once the store-layer fill resolves -// covered ids -> seq from the append-only transcript. +// m-numbers. Resolvable coverage carries the real first/last covered seq from +// the append-only transcript; unresolvable coverage stores the -1/-1 no-claim +// sentinel (planner/store convention, #4741) — never a degenerate [0,0]. function msg(id: string, role: "user" | "assistant"): CoreMessage { return { id, role, contentType: "text", text: `body ${id}` }; @@ -86,7 +86,7 @@ test("#4316: a block whose covered messages are present carries their real seq b } }); -test("#4316: a block whose covered ids are absent from the transcript keeps placeholder 0 (no crash, honest gap)", () => { +test("#4316: a block whose covered ids are absent from the transcript keeps the -1/-1 no-claim sentinel (no crash, honest gap)", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "bili-4316-gap-")); try { const store = UnifiedStore.open(path.join(dir, "index.db")); @@ -97,8 +97,8 @@ test("#4316: a block whose covered ids are absent from the transcript keeps plac assert.equal(seqOf(store, sid, "h_old"), undefined, "fixture sanity: h_old must NOT be a transcript row"); const b = store.getBlocks(sid).find((x) => x.blockId === "b1")!; - assert.equal(b.startSeq, 0, "unresolvable coverage keeps the 0 placeholder, never a fabricated bound"); - assert.equal(b.endSeq, 0, "unresolvable coverage keeps the 0 placeholder, never a fabricated bound"); + assert.equal(b.startSeq, -1, "unresolvable coverage keeps the -1/-1 no-claim sentinel, never a fabricated bound"); + assert.equal(b.endSeq, -1, "unresolvable coverage keeps the -1/-1 no-claim sentinel, never a fabricated bound"); } finally { try { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 }); } catch { /* Windows stray handle */ } } diff --git a/tests/issue4747-interleaved-rows.test.ts b/tests/issue4747-interleaved-rows.test.ts new file mode 100644 index 0000000000..51ba7120bf --- /dev/null +++ b/tests/issue4747-interleaved-rows.test.ts @@ -0,0 +1,29 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { resolveStoredDecompositions } from "../src/fold-decomposition-lookup.ts"; +import type { StoredRow, DecompositionSource } from "../src/fold-decomposition-lookup.ts"; +import type { Session } from "../src/session.ts"; +import type { FoldAnchor } from "../src/fold-reconcile.ts"; + +const mkSession = (anchorIds: string[]): Session => ({ + id: "s", meta: {}, stats: {}, metadata: { + foldAnchors: Object.fromEntries(anchorIds.map((id) => [id, { r: "assistant" }])), + foldStoredTried: {}, + }, + state: {} as never, createdAt: 1, lastSeen: 2, blockContents: new Map(), inFlight: 0, + persisted: false, pendingRetrievals: [], +} as never); + +test("#4747: interleaved rows (A1,B1,A2,B2) keep every id's LATEST row", () => { + const rows: StoredRow[] = [ + { seq: 10, msgId: "A", content: JSON.stringify({ id: "A", role: "assistant", contentType: "text", text: "A-old" }) }, + { seq: 11, msgId: "B", content: JSON.stringify({ id: "B", role: "assistant", contentType: "text", text: "B-1" }) }, + { seq: 12, msgId: "A", content: JSON.stringify({ id: "A", role: "assistant", contentType: "text", text: "A-new" }) }, + { seq: 13, msgId: "B", content: JSON.stringify({ id: "B", role: "assistant", contentType: "text", text: "B-2" }) }, + ]; + const src: DecompositionSource = { rows: (_sid: string, ids: readonly string[]) => (ids.includes("A") || ids.includes("B") ? rows : []) }; + const out = resolveStoredDecompositions(mkSession(["A", "B"]), "s", new Set(["wire-live"]), src); + assert.ok(out !== undefined, "rows returned"); + const texts = out!.map((m) => (m as { text?: string }).text); + assert.deepEqual(texts, ["A-new", "B-2"], "each id's LATEST row survives interleaving — old splice desync deleted A-new and duplicated B"); +}); diff --git a/tests/launcher-opencode-inline.test.ts b/tests/launcher-opencode-inline.test.ts index 7ff04b2ace..09f433c205 100644 --- a/tests/launcher-opencode-inline.test.ts +++ b/tests/launcher-opencode-inline.test.ts @@ -28,6 +28,7 @@ interface SpawnRecord { command: string; args: string[]; env: NodeJS.ProcessEnv | undefined; + verbatim: boolean | undefined; } // `bili opencode-inline` (#4669): the launcher must start NO proxy, hand the @@ -75,7 +76,7 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac const spawns: SpawnRecord[] = []; const spawnImpl: SpawnFn = (cmd, args, options) => { - spawns.push({ command: cmd, args: [...args], env: options?.env }); + spawns.push({ command: cmd, args: [...args], env: options?.env, verbatim: options?.windowsVerbatimArguments }); const child = makeFakeChild(0); const orig = child.on!.bind(child); (child as { on: SpawnChild["on"] }).on = (event, listener) => { @@ -109,7 +110,25 @@ test("runLaunch opencode-inline: no proxy spawned, BILI_OPENCODE_INLINE=1, attac { fetchImpl, fetchHealthInfo, spawnImpl, sleep: () => Promise.resolve() }, ); assert.equal(spawns.length, 1, "exactly one child (opencode itself) — no proxy spawn"); - assert.equal(spawns[0].command, fakeOc); + // #679: on Windows the .cmd shim rides comspec /d /s /c "" + // (CreateProcess cannot execute a batch file); elsewhere it spawns directly. + // Expectations derive from the child's captured env, not ambient + // process.env — concurrent tests in one CI shard rewrite globals mid-flight (#4599). + const cap = spawns[0]; + if (process.platform === "win32") { + // Mirror planClientSpawn's case-insensitive probe: sanitized env + // copies keep the OS key casing (`ComSpec`), plain objects don't (#4599). + const ce = cap.env ?? {}; + const childComspec = ce.COMSPEC ?? ce.ComSpec ?? ce.comspec; + const expectedComspec = typeof childComspec === "string" && childComspec.trim().length > 0 ? childComspec : "cmd.exe"; + assert.equal(cap.command, expectedComspec, "#679 comspec wrap uses the child env's COMSPEC"); + assert.deepEqual(cap.args.slice(0, 3), ["/d", "/s", "/c"]); + assert.equal(cap.verbatim, true, "#679 verbatim argv keeps cmd.exe from re-splitting"); + assert.match(cap.args[3] ?? "", /^".*\.cmd"$/, "comspec line wraps the quoted .cmd shim"); + } else { + assert.equal(cap.command, fakeOc); + assert.deepEqual(cap.args, []); + } const env = spawns[0].env!; assert.equal(env.BILI_OPENCODE_INLINE, "1"); assert.equal(env.BILLION_CONTEXT_PROXY, undefined, "inherited BILLION_CONTEXT_PROXY must be stripped (attach would win)"); @@ -179,7 +198,7 @@ test("runLaunch opencode-inline: refuses (exit 2) when the bili opencode plugin const spawns: SpawnRecord[] = []; const spawnImpl: SpawnFn = (cmd, args, options) => { - spawns.push({ command: cmd, args: [...args], env: options?.env }); + spawns.push({ command: cmd, args: [...args], env: options?.env, verbatim: options?.windowsVerbatimArguments }); return makeFakeChild(0); }; const prevExit = process.exit; diff --git a/tests/plugin-fork.test.ts b/tests/plugin-fork.test.ts index d15bc89fe9..1c4801377d 100644 --- a/tests/plugin-fork.test.ts +++ b/tests/plugin-fork.test.ts @@ -747,15 +747,26 @@ test("HTTP fork revision and originals survive a new proxy process", { timeout: let child: ReturnType | undefined; try { await compress(h); + // CI-only race (#4599): the compress fold lands in the live session + // state asynchronously after the tool response (#3573 replay + // semantics). On slow runners flushSync below would otherwise persist + // the pre-fold state and the cold load reports "0 block(s) + // (compressions=0)". Bounded-wait for the fold first. + for (let i = 0; i < 100 && !(resolveConversation("parent").session?.state.blocks?.length ?? 0); i++) { + await new Promise((r) => setTimeout(r, 50)); + } const parent = resolveConversation("parent").session!; parent.metadata.effectiveCcr = { ...DEFAULT_CCR_CONFIG, enabled: true }; parent.contentStore = storeOriginal(createContentStore(), { ref: "m00001", rawId: parent.pluginSnapshot![0].id, text: "CCR original retained independently after restart", kind: "original", tokens: 10, head: "CCR original" }); parent.contentStoreDirty = true; assert(h.store.flushSync(parent)); - // #4316 PR3/3: cold load runs the one-time ledger alignment — the loaded - // state now carries refLedgerAligned. Mirror it in the expectation. + // #4316 PR3/3: cold load runs the one-time ledger alignment, but only + // when the unified store is available — config-clean machines (CI) run + // legacy mode where the alignment is an honest skip and the loaded + // state never carries refLedgerAligned. Mirror the mode in the + // expectation instead of asserting the flag unconditionally. const persistedState = JSON.parse(JSON.stringify(parent.state)); - persistedState.refLedgerAligned = true; + if (h.store.replicasAvailable()) persistedState.refLedgerAligned = true; assert.deepEqual(JSON.parse(JSON.stringify(h.store.loadSync("parent")!.state)), persistedState); const snapshot = (await h.request("/__bili/plugin/snapshot?conversationId=parent")).body; const req = forkRequest(snapshot); diff --git a/tests/storage-engine-proofs.test.ts b/tests/storage-engine-proofs.test.ts index 118ca0bde7..d9eb3334b2 100644 --- a/tests/storage-engine-proofs.test.ts +++ b/tests/storage-engine-proofs.test.ts @@ -11,7 +11,11 @@ * render goldens, both with a mid-conversation restart * and with the legacy file DELETED at the restart (the * index alone carries the session — the migration's - * end-state promise). + * end-state promise). #4316 PR3/3 (380ef4dac): a + * full-mode cold load additionally re-homes citation + * numbers once (one-time ledger alignment) — comparisons + * run modulo that renumbering (citation-normalized + * digests), which is the ONLY permitted delta. * STALE REJECT — a multi-instance stale write that the guarded build * rejects must never reach index.db; the converged disk * record is what gets ingested. @@ -45,7 +49,7 @@ import { planIngest, loadLegacyView, toLegacyLike, type LegacySessionLike } from import { canonicalize, sha256 } from "./golden-canonical.ts"; import { driveWire, WIRES, type Wire } from "./golden-storage-drive.ts"; import { rmrf } from "./tmp-rm.ts"; -import { ENGPROOF_CHECKS, spawnCorruptIndexChild, viewDigest, viewStateDigest } from "./corrupt-index-child.ts"; +import { ENGPROOF_CHECKS, spawnCorruptIndexChild, stripOneTimeAlignment, viewDigest, viewStateDigest } from "./corrupt-index-child.ts"; process.env.BILI_PERSIST_ZSTD = "0"; // #3400: pin the legacy wire shape. This suite proves STORAGE restart @@ -145,11 +149,12 @@ function unifiedRows(store: UnifiedStore, id: string): ReturnType { + test(`storage engine: full-mode restart is invisible on the ${wire} wire (== legacy restart / committed goldens, modulo the #4316 one-time citation renumbering)`, { timeout: 240_000 }, async () => { const golden = JSON.parse(fs.readFileSync(path.join(RENDER_ROOT, `${wire}.json`), "utf8")) as { - turns: Array<{ i: number; sha: string }>; + turns: Array<{ i: number; sha: string; shaCiteNorm: string }>; foldAt: number; editedAt: number; summaryFrom: number; stateDigest: string; + stateDigestAligned: string; stateShape: Record; }; // The restart oracle: a legacy-only process restart. One caveat is @@ -162,16 +167,18 @@ for (const wire of WIRES) { const record = await driveWire(wire as Wire, `golden-${wire}`, { storageUnified: "full", reloadAfterEdit: leg }); assert.equal(record.turns.length, legacy.turns.length, `${wire}/${leg}: body count drifted vs legacy restart`); for (let i = 0; i < legacy.turns.length; i++) { - assert.equal(record.turns[i]!.sha, legacy.turns[i]!.sha, `${wire}/${leg}: outbound body #${i} digest drift vs legacy restart`); - assert.equal(record.turns[i]!.sha, golden.turns[i]!.sha, `${wire}/${leg}: outbound body #${i} digest drift vs committed goldens (restart must not change the wire)`); + // #4316 PR3/3 (380ef4dac): a full-mode cold load re-homes citation + // numbers once (one-time ledger alignment) — the ONLY permitted + // wire delta. Citation-normalized digests must match turn-for-turn. + assert.equal(record.turns[i]!.shaCiteNorm, legacy.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs legacy restart`); + assert.equal(record.turns[i]!.shaCiteNorm, golden.turns[i]!.shaCiteNorm, `${wire}/${leg}: outbound body #${i} drifts beyond citation renumbering vs committed goldens (restart-inherent bs-field loss aside)`); } assert.equal(record.foldAt, legacy.foldAt, `${wire}/${leg}: fold landmark drifted`); assert.equal(record.editedAt, legacy.editedAt, `${wire}/${leg}: edit landmark drifted`); assert.equal(record.summaryFrom, legacy.summaryFrom, `${wire}/${leg}: summary carrier landmark drifted`); - assert.equal(record.stateDigest, legacy.stateDigest, `${wire}/${leg}: persisted state digest drift vs legacy restart`); + assert.equal(record.stateDigestAligned, legacy.stateDigestAligned, `${wire}/${leg}: persisted state drifts beyond the one-time ledger alignment vs legacy restart`); + assert.equal(record.stateDigestAligned, golden.stateDigestAligned, `${wire}/${leg}: persisted state drifts beyond the one-time ledger alignment vs committed goldens`); assert.deepEqual(record.stateShape, legacy.stateShape, `${wire}/${leg}: state shape drift`); - assert.deepEqual(record.sectionDigests, legacy.sectionDigests, `${wire}/${leg}: state section digests drift`); - assert.deepEqual(record.stateKeyDigests, legacy.stateKeyDigests, `${wire}/${leg}: state key digests drift`); } }); } @@ -293,20 +300,25 @@ test("storage engine: rollback contract — unset the env and the legacy read pa // (The load-path digest is the stable oracle: an in-memory session vs // a loaded one differs benignly by the storageMode stamp + one-time // tail normalization, identically under both paths.) + // #3818 flip: absent now resolves to FULL, so the legacy read path has + // to be spelled out explicitly (the knob's one-line escape hatch) — + // merely deleting the env would read unified and vacate this leg. const storeFull = new SessionStore({ dir, debounceMs: 1, enabled: true }); const viaFull = storeFull.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); storeFull.closeQuietly(); assert.ok(viaFull, "full-mode reload returned null"); const digestFull = viewDigest(toLegacyLike(viaFull)); - delete process.env.BILI_STORAGE_UNIFIED; + process.env.BILI_STORAGE_UNIFIED = "legacy"; const store2 = new SessionStore({ dir, debounceMs: 1, enabled: true }); const legacy = store2.loadSync("corpus-folded", { protocol: "anthropic", upstreamOrigin: "http://127.0.0.1:8199" }); store2.closeQuietly(); assert.ok(legacy, "legacy-only reload returned null after rollback"); // #2960 ⑤: cross-mode equality is on the STATE proper — full mode // serves the raw table tail, legacy the envelope's folded view. - assert.equal(viewStateDigest(toLegacyLike(legacy)), viewStateDigest(toLegacyLike(viaFull)), "rollback: legacy-only load state digest drift vs full-mode load"); + // #4316 PR3/3: modulo the one-time ledger alignment (full-side cold + // load re-homes citation numbers; legacy keeps counter numbers). + assert.equal(viewStateDigest(stripOneTimeAlignment(toLegacyLike(legacy))), viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))), "rollback: legacy-only load state digest drift vs full-mode load"); assert.ok(viaFull.lastMessages && viaFull.lastMessages.length > 0, "⑤: full-mode load serves the table tail"); assert.ok(legacy.lastMessages && legacy.lastMessages.length > 0, "⑤: rollback keeps the envelope tail"); const disk = findWrittenRecord(dir, "corpus-folded"); @@ -381,7 +393,10 @@ test("storage engine: corrupt index.db degrades to legacy-only instead of bricki // executed in the child, verified in the parent). The child also runs // the #2781 mid-flow rename guard plus a post-close guard, so future // handle-leak drift trips there by name instead of at teardown. - const stateDigestClean = viewStateDigest(toLegacyLike(viaFull)); + // #4316 PR3/3: expected digest computed modulo the one-time ledger + // alignment — the clean read above already carried it in memory, and + // the child's fresh load applies it again; both sides must normalize. + const stateDigestClean = viewStateDigest(stripOneTimeAlignment(toLegacyLike(viaFull))); const childEnv: NodeJS.ProcessEnv = { ...process.env, BILI_STORAGE_UNIFIED: "full", @@ -509,5 +524,5 @@ test("storage engine: no SQLite engine -> SessionStore runs legacy-only, writes // Engine note: these legs run on the process-default engine; the corpus // suites (storage-unified.test.ts) cover BOTH engines at the state level, -// and the wire legs here assert against the same committed goldens the -// legacy baseline produced. +// and the wire legs here assert citation-normalized equality against the same +// committed goldens the legacy baseline produced. diff --git a/tests/storage-orphan-recovery.test.ts b/tests/storage-orphan-recovery.test.ts index c63f3f63fb..9fd6f13b0a 100644 --- a/tests/storage-orphan-recovery.test.ts +++ b/tests/storage-orphan-recovery.test.ts @@ -69,7 +69,12 @@ test("cold load recovers the pre-fix orphan freeze before processing a request", assert.equal(recovered.stats.requests, 876); assert.equal(recovered.state.nextBlockId, 4); assert.deepEqual(recovered.state.blocks.map(b => b.blockId), ["b3"]); - assert.equal(recovered.state.messageRefs.byRef.m00003, "raw-new"); + // #4316 PR3/3: a full-mode cold load re-homes citation numbers once + // (ledger alignment) — pin the content-addressable round-trip, not the + // counter-era number, so the assertion holds under any numbering scheme. + const alignedRef = recovered.state.messageRefs.byRaw["raw-new"]; + assert.ok(alignedRef, "raw-new must keep a citation number after recovery"); + assert.equal(recovered.state.messageRefs.byRef[alignedRef], "raw-new"); assert.equal(recovered.lastMessages?.at(-1)?.text, "new turn"); assert.equal(recovered.metadata.recoveryMarker, "fresh"); assert.equal(recovered.blockContents.get("b1")?.full.text, "retained original"); diff --git a/tests/web-blogfeed.test.ts b/tests/web-blogfeed.test.ts index 0d1d820907..46204193a9 100644 --- a/tests/web-blogfeed.test.ts +++ b/tests/web-blogfeed.test.ts @@ -144,6 +144,18 @@ test("a root without content degrades to an empty list instead of throwing", () assert.deepEqual(getBlogFeed(), { items: [] }); }); +test("CRLF frontmatter (Windows autocrlf checkout) still yields title/excerpt/date", () => { + const posts = join(pkgRoot, "website", "blog", "posts"); + writeFileSync(join(posts, "2026-10-08_crlf.md"), "---\r\ntitle: \"CRLF post\"\r\nexcerpt: \"EN excerpt.\"\r\n---\r\n\r\n## Body\r\n\r\nText.\r\n"); + _setBlogRootForTest(pkgRoot); + const data = getBlogFeed(); + const crlf = data.items.find((i) => i.id === "crlf"); + assert.ok(crlf, "crlf post listed"); + assert.equal(crlf.title, "CRLF post"); + assert.equal(crlf.excerpt, "EN excerpt."); + assert.equal(crlf.date, "2026-10-08"); +}); + test("default probing finds the repository tree from the TS source layout", () => { _setBlogRootForTest(null); const data = getBlogFeed();