From e6af249a2b65d5d74fb4ecef26371ce857d7cd1a Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 02:42:35 +0800 Subject: [PATCH 01/15] fix: mark structurally-exhausted tier-recovery deterministic (#5766) A bounded tier-distillation recovery that ends because span-math proved no legal two-block span (e.g. oversized inline media blowing every candidate span past the 60k source budget) is a structural impossibility: the identical payload can never recover regardless of upstream health. But the verdict was installed without the `deterministic` flag its sibling futile-round paths carry, so the server fail-fast degraded to 502 instead of 413 and status-code-driven hosts read it as transient and spun on doomed retries (field: 5+ in 8s). Propagate the recovery stopReason out of attemptTierRecovery and flag only the structurally-exhausted ending deterministic; cap/unavailable/no-shrink keep their 502-retryable contract per the existing fail-fast semantics (#3461/#3918). Behavior change: structurally-exhausted tier recovery now answers 413 (non-forwarded, non-retryable) instead of 502. Relay/budget-dependent endings are unchanged. Fixes #5766. (cherry picked from commit cf180606d0c01cbaa9c52ecfd83e7fad5f7fb712) --- src/preflight.ts | 11 +- ...5766-tier-exhaustion-deterministic.test.ts | 172 ++++++++++++++++++ 2 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 tests/issue5766-tier-exhaustion-deterministic.test.ts diff --git a/src/preflight.ts b/src/preflight.ts index c2f0177a0e..c37b10b8f9 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -1966,7 +1966,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa const active = deps.session.state.blocks.filter((b) => b.active); return active.filter((b) => b.tier === 1).length >= 2 || active.filter((b) => b.tier === 2).length >= 2; }; - type TierRecoveryResult = { recovered: boolean; callsUsed: number; netSaved: number; honestDetail: string }; + type TierRecoveryResult = { recovered: boolean; callsUsed: number; netSaved: number; honestDetail: string; stopReason: "fit" | "exhausted" | "cap" | "unavailable" | "no-shrink" }; const attemptTierRecovery = async (): Promise => { let callsUsed = 0; let netSaved = 0; @@ -2178,7 +2178,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa `futile round: folding compressible raw messages plus tier distillation (${callsUsed} summary call(s), net ~${netSaved} tokens saved) still leaves the payload ~${overBy} tokens over the window — ${why}. ` + `These figures are measured on the post-fold wire view; the separately displayed context total additionally floors on the upstream usage baseline, which also bills the system prompt and tool definitions. ` + `Raise the model context window or restart the session.`; - return { recovered, callsUsed, netSaved, honestDetail }; + return { recovered, callsUsed, netSaved, honestDetail, stopReason }; }; for (let round = 0; round < effectiveRoundCap; round++) { if (deps.signal?.aborted) { @@ -3228,7 +3228,12 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa const rec = await attemptTierRecovery(); deps.log("info", `[preflight] tier-distillation recovery finished: ${rec.recovered ? "recovered" : "not recovered"}; ${rec.callsUsed} summary call(s), net ~${rec.netSaved} tok saved (#2952)`); if (!rec.recovered) { - failure = { kind: "exhausted", detail: rec.honestDetail }; + // #5766/#3461: only the structurally-exhausted ending (span-math proved no legal + // two-block span / not enough positionable blocks — e.g. oversized inline media + // blowing every span past TIER_DISTILL_SOURCE_CHAR_BUDGET) is deterministic → + // 413 not-forwarded; cap/unavailable/no-shrink are relay/budget-dependent and MUST + // stay 502-retryable (server.ts fail-fast contract). + failure = { kind: "exhausted", deterministic: rec.stopReason === "exhausted", detail: rec.honestDetail }; } } if (currentTokens >= limit && !failure) { diff --git a/tests/issue5766-tier-exhaustion-deterministic.test.ts b/tests/issue5766-tier-exhaustion-deterministic.test.ts new file mode 100644 index 0000000000..6d507f7662 --- /dev/null +++ b/tests/issue5766-tier-exhaustion-deterministic.test.ts @@ -0,0 +1,172 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import test from "node:test"; +process.env.NODE_ENV = "test"; +import { createCore, createInitialState, defaultConfig, defaultPrompts, type CoreMessage } from "acp-kernel"; +import type { Session } from "../src/session.ts"; +import { preflightCompress, type PreflightDeps } from "../src/preflight.ts"; +import { ConfiguredSummaryPlan } from "../src/external-summary-runtime.js"; + +// #5766: a DSH session whose new turn carried several oversized inline items (six +// base64 images, ~130K chars each) made EVERY candidate two-block span cross the +// 60,000-char tier-distillation source budget, so bounded tier-recovery found no +// legal span and ended structurally exhausted. That verdict was installed WITHOUT +// the `deterministic` flag its sibling futile-round paths carry (#3461), so the +// server fail-fast degraded to 502 instead of 413 — and status-code-driven hosts +// read 502 as transient and spun on doomed retries (field: 5+ in 8s). +// +// These tests pin the repaired verdict classification at the preflight boundary: +// - the structurally-exhausted ending (span-math proved no legal span) IS +// marked deterministic — the identical payload can never recover regardless of +// upstream health; +// - the relay/budget-dependent endings (call-budget cap) are NOT marked +// deterministic — they keep their 502-retryable contract where the next round +// continues from a smaller baseline (#3918). +// The oversized-item shape is modeled with long-text blocks: `spanContent` measures +// the real rendered chars of a span, so an item whose content alone blows any span +// past the budget behaves identically whether it is inline media or prose. + +function filler(chars: number, tag: string): string { + const unit = `${tag} alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau upsilon phi chi psi omega `; + return unit.repeat(Math.ceil(chars / unit.length)).slice(0, chars); +} + +interface BuildOpts { limit: number; blocks: number[]; tailChars: number; } + +// One tier-1 block per entry of `blocks` (summary sized to the entry) plus a small +// foldable "early" message (defers the raw futility gate to tier recovery) and a +// protected tail. Block mass dominates the payload, so the raw pool alone cannot +// close any deficit. Mirrors tests/issue3646-tier-recovery-picker.test.ts. +function buildSession(o: BuildOpts) { + const core = createCore(); + const config = defaultConfig(o.limit); + config.preserveRecentMessages = 1; + config.preserveRecentTokens = 0; + config.compress.minCompressRange = 100; + config.compress.maxSummaryLength = 0; + const messages: CoreMessage[] = [ + { id: "early", role: "user", contentType: "text", text: filler(500, "EARLY") }, + ]; + for (let i = 0; i < o.blocks.length; i++) { + const chars = o.blocks[i]!; + messages.push({ id: `u${i}`, role: "user", contentType: "text", text: filler(chars, `HISTU${i}`) }); + messages.push({ id: `a${i}`, role: "assistant", contentType: "text", text: filler(120, `HISTA${i}`) }); + } + messages.push({ id: "tail", role: "user", contentType: "text", text: filler(o.tailChars, "TAIL") }); + const turn = core.processTurn({ messages, state: createInitialState(), config: { ...config, modelContextLimit: o.limit * 1000 }, tokenCount: o.limit * 1000, renderTags: "text-only" }); + const foldCfg = { ...config, preserveRecentMessages: 0, preserveRecentTokens: 0 }; + let state = turn.state; + for (let i = 0; i < o.blocks.length; i++) { + const ref = turn.state.messageRefs.byRaw[`u${i}`]; + const c = core.applyCompression({ messages, state, config: foldCfg, ranges: [{ startRef: ref, endRef: ref, summary: filler(o.blocks[i]!, `BLK${i}`) }] }); + state = c.state; + } + const seeded = state.blocks.filter((b) => b.active); + assert.equal(seeded.length, o.blocks.length, `fixture must seed ${o.blocks.length} tier-1 blocks (got ${seeded.length})`); + const session: Session = { + id: "issue5766-preflight", meta: {}, metadata: {}, state, + stats: { requests: 1, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: o.limit * 4, compressCreditTokens: 0, contextTokens: o.limit * 4, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, + pendingRetrievals: [], createdAt: Date.now(), lastSeen: Date.now(), blockContents: new Map(), inFlight: 0, persisted: false, + }; + return { core, config, messages, session }; +} + +async function startRecordingUpstream(body: object): Promise<{ server: http.Server; url: string; bodies: string[] }> { + const bodies: string[] = []; + const server = http.createServer((req, res) => { + let data = ""; + req.on("data", (c) => { data += c; }); + req.on("end", () => { + bodies.push(data); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); + }); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); + const addr = server.address() as { port: number }; + return { server, url: `http://127.0.0.1:${addr.port}`, bodies }; +} + +function openAiBody(content: string): object { + return { choices: [{ message: { role: "assistant", content }, finish_reason: "stop" }] }; +} + +function makePlan(url: string): ConfiguredSummaryPlan { + return new ConfiguredSummaryPlan({ enabled: true, targets: [{ name: "sm", protocol: "openai", url, model: "sm", apiKey: "k" }] }); +} + +function makeDeps(core: ReturnType, session: Session, config: ReturnType, url: string, logs: string[]): PreflightDeps { + return { + core, session, config, prompts: defaultPrompts, protocol: "responses", + // Port 9 refuses instantly: model-lane traffic must never reach the stub. + url: "http://127.0.0.1:9/v1/chat/completions", headers: {}, model: "test", + externalSummary: makePlan(url), log: (_l, m) => { logs.push(m); }, + }; +} + +test("#5766: a structurally-exhausted tier recovery (every span over budget) is marked deterministic", async () => { + // Three ~40k-char items: any two-block span (~80k) crosses the 60,000-char source + // budget, so the picker finds no legal candidate — the same gate six oversized + // inline images trip in production. Recovery must end structurally exhausted and + // the verdict MUST carry deterministic:true (→ 413 not-forwarded, no host spin). + const { core, config, messages, session } = buildSession({ limit: 10000, blocks: [40000, 40000, 40000], tailChars: 200 }); + const { server, url, bodies } = await startRecordingUpstream(openAiBody(filler(300, "CONDENSED"))); + try { + const logs: string[] = []; + const result = await preflightCompress(makeDeps(core, session, config, url, logs), messages); + assert.equal(result.fitsWindow, false, "a 120k-char payload cannot fit a 10k window"); + assert.equal(result.failure?.kind, "exhausted", `expected honest exhaustion, got ${JSON.stringify(result.failure)}`); + assert.match(result.failure?.detail ?? "", /no two-block span stayed under the 60,000-character source budget/, `diagnostic must name the source-budget blockage (got: ${result.failure?.detail})`); + // THE FIX: structural exhaustion is deterministic — the identical payload can + // never recover regardless of upstream health. Before the fix this was undefined. + assert.equal(result.failure?.deterministic, true, `structurally-exhausted recovery must be flagged deterministic (got ${JSON.stringify(result.failure)})`); + assert.equal(bodies.length, 0, "an all-oversized pool must not spend a summary call"); + } finally { + server.close(); + } +}); + +test("#5766: a single positionable block (below the two-block gate) is also marked deterministic", async () => { + // Only one active tier-1 block survives reconciliation — below the two-block gate, + // so no span can ever be assembled. Same structural truth as the over-budget case: + // deterministic, not a relay-dependent 502. + const { core, config, messages, session } = buildSession({ limit: 10000, blocks: [40000, 40000], tailChars: 200 }); + // Drop the second block's covered messages from the resent wire so reconciliation + // deactivates it, leaving exactly one active block. + const gone = new Set(["u1", "a1"]); + const resent = messages.filter((m) => !gone.has(m.id)); + const activeAfter = () => session.state.blocks.filter((b) => b.active).length; + const { server, url } = await startRecordingUpstream(openAiBody(filler(300, "CONDENSED"))); + try { + const logs: string[] = []; + const result = await preflightCompress(makeDeps(core, session, config, url, logs), resent); + assert.equal(result.failure?.kind, "exhausted", `expected honest exhaustion, got ${JSON.stringify(result.failure)}`); + assert.ok(activeAfter() <= 1 || /no tier held two or more active blocks|positionable/i.test(result.failure?.detail ?? ""), `expected a below-gate or unpositionable end state (active=${activeAfter()}, got: ${result.failure?.detail})`); + assert.equal(result.failure?.deterministic, true, `below-two-block-gate exhaustion must be flagged deterministic (got ${JSON.stringify(result.failure)})`); + } finally { + server.close(); + } +}); + +test("#5766 (negative gate): a call-budget-cap ending is NOT marked deterministic (keeps 502-retryable)", async () => { + // Twelve 25k blocks: every pair (~50k) fits, so recovery distills until it hits the + // six-call cap while the 60k-char tail floor alone still exceeds the window. That is + // a BUDGET ending, not a structural proof — it must stay non-deterministic so the + // server keeps its 502-retryable contract where the next round continues from the + // folded baseline (#3918). Over-marking this deterministic would wrongly 413 it. + const { core, config, messages, session } = buildSession({ limit: 12000, blocks: Array(12).fill(25000), tailChars: 60000 }); + const { server, url, bodies } = await startRecordingUpstream(openAiBody(filler(300, "CONDENSED"))); + try { + const logs: string[] = []; + const result = await preflightCompress(makeDeps(core, session, config, url, logs), messages); + const calls = bodies.filter((b) => b.includes("must be distilled")); + assert.equal(calls.length, 6, `recovery must run to the 6-call cap (used ${calls.length})`); + assert.equal(result.failure?.kind, "exhausted", `expected exhaustion at the cap, got ${JSON.stringify(result.failure)}`); + assert.match(result.failure?.detail ?? "", /bounded distillation call budget was reached after 6 call\(s\)/, `diagnostic must name the call-budget end (got: ${result.failure?.detail})`); + // THE CONTRAST: a budget ending is NOT a structural impossibility — it must not + // be flagged deterministic (would wrongly force a 413 and kill the continue-on-retry path). + assert.ok(!result.failure?.deterministic, `call-budget-cap ending must stay non-deterministic (got ${JSON.stringify(result.failure)})`); + } finally { + server.close(); + } +}); From 35922b373a144505e2ea248a71adf4dcc7b880bd Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 6 Nov 2026 04:10:18 +0800 Subject: [PATCH 02/15] test: reference GitHub issue #5764 in tier-exhaustion fix (was internal mirror number) (cherry picked from commit db8097825922067921096ba559d5ed615d59decb) --- src/preflight.ts | 2 +- ...5764-tier-exhaustion-deterministic.test.ts} | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) rename tests/{issue5766-tier-exhaustion-deterministic.test.ts => issue5764-tier-exhaustion-deterministic.test.ts} (93%) diff --git a/src/preflight.ts b/src/preflight.ts index c37b10b8f9..ee3e735d4a 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -3228,7 +3228,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa const rec = await attemptTierRecovery(); deps.log("info", `[preflight] tier-distillation recovery finished: ${rec.recovered ? "recovered" : "not recovered"}; ${rec.callsUsed} summary call(s), net ~${rec.netSaved} tok saved (#2952)`); if (!rec.recovered) { - // #5766/#3461: only the structurally-exhausted ending (span-math proved no legal + // #5764/#3461: only the structurally-exhausted ending (span-math proved no legal // two-block span / not enough positionable blocks — e.g. oversized inline media // blowing every span past TIER_DISTILL_SOURCE_CHAR_BUDGET) is deterministic → // 413 not-forwarded; cap/unavailable/no-shrink are relay/budget-dependent and MUST diff --git a/tests/issue5766-tier-exhaustion-deterministic.test.ts b/tests/issue5764-tier-exhaustion-deterministic.test.ts similarity index 93% rename from tests/issue5766-tier-exhaustion-deterministic.test.ts rename to tests/issue5764-tier-exhaustion-deterministic.test.ts index 6d507f7662..fb30f60f2b 100644 --- a/tests/issue5766-tier-exhaustion-deterministic.test.ts +++ b/tests/issue5764-tier-exhaustion-deterministic.test.ts @@ -7,7 +7,7 @@ import type { Session } from "../src/session.ts"; import { preflightCompress, type PreflightDeps } from "../src/preflight.ts"; import { ConfiguredSummaryPlan } from "../src/external-summary-runtime.js"; -// #5766: a DSH session whose new turn carried several oversized inline items (six +// #5764: a DSH session whose new turn carried several oversized inline items (six // base64 images, ~130K chars each) made EVERY candidate two-block span cross the // 60,000-char tier-distillation source budget, so bounded tier-recovery found no // legal span and ended structurally exhausted. That verdict was installed WITHOUT @@ -64,7 +64,7 @@ function buildSession(o: BuildOpts) { const seeded = state.blocks.filter((b) => b.active); assert.equal(seeded.length, o.blocks.length, `fixture must seed ${o.blocks.length} tier-1 blocks (got ${seeded.length})`); const session: Session = { - id: "issue5766-preflight", meta: {}, metadata: {}, state, + id: "issue5764-preflight", meta: {}, metadata: {}, state, stats: { requests: 1, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: o.limit * 4, compressCreditTokens: 0, contextTokens: o.limit * 4, retrieveCalls: 0, retrieveHits: 0, retrieveMisses: 0, storedBytes: 0, storeBytesSaved: 0, rangeRestores: 0 }, pendingRetrievals: [], createdAt: Date.now(), lastSeen: Date.now(), blockContents: new Map(), inFlight: 0, persisted: false, }; @@ -104,7 +104,7 @@ function makeDeps(core: ReturnType, session: Session, config: }; } -test("#5766: a structurally-exhausted tier recovery (every span over budget) is marked deterministic", async () => { +test("#5764: a structurally-exhausted tier recovery (every span over budget) is marked deterministic", async () => { // Three ~40k-char items: any two-block span (~80k) crosses the 60,000-char source // budget, so the picker finds no legal candidate — the same gate six oversized // inline images trip in production. Recovery must end structurally exhausted and @@ -126,10 +126,12 @@ test("#5766: a structurally-exhausted tier recovery (every span over budget) is } }); -test("#5766: a single positionable block (below the two-block gate) is also marked deterministic", async () => { - // Only one active tier-1 block survives reconciliation — below the two-block gate, - // so no span can ever be assembled. Same structural truth as the over-budget case: - // deterministic, not a relay-dependent 502. +test("#5764: a single positionable block (below the two-block gate) is also marked deterministic", async () => { + // Only one active tier-1 block survives reconciliation, so the recovery funnel is + // gated out entirely (eligibleForTierRecovery demands two) and the verdict lands on + // the #1841 futile-round classification that already carried the flag — this test + // pins the below-gate CONTRACT; the new stopReason propagation is pinned by the + // other two tests. Either way it is structural, not a relay-dependent 502. const { core, config, messages, session } = buildSession({ limit: 10000, blocks: [40000, 40000], tailChars: 200 }); // Drop the second block's covered messages from the resent wire so reconciliation // deactivates it, leaving exactly one active block. @@ -148,7 +150,7 @@ test("#5766: a single positionable block (below the two-block gate) is also mark } }); -test("#5766 (negative gate): a call-budget-cap ending is NOT marked deterministic (keeps 502-retryable)", async () => { +test("#5764 (negative gate): a call-budget-cap ending is NOT marked deterministic (keeps 502-retryable)", async () => { // Twelve 25k blocks: every pair (~50k) fits, so recovery distills until it hits the // six-call cap while the 60k-char tail floor alone still exceeds the window. That is // a BUDGET ending, not a structural proof — it must stay non-deterministic so the From e76f3713e97e0258f9797885bffc4823cfde0218 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 7 Nov 2026 13:08:40 +0800 Subject: [PATCH 03/15] fix: reanchor tag-changed shorter replays whose logical tail survived a host record repack (#5844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A host upgrade that changes BOTH the system prompt text and the message-record layout (dsh 0.2.1->0.2.2-rc.x: assistant text segments merged into adjacent records, some standalone assistant records dropped) destroys every byte-level continuation evidence at once: per-source-message fragment hashes are disjoint even though every logical turn is identical, so all four tagChanged escape rungs miss and the conversation forks onto an empty |sub: lane -> preflight folds from scratch -> resume dies with a fail-fast 413. Add a canonical-turn fingerprint rung at the #5686 site: when the replay is strictly shorter and kind-tail-aligned, reanchor onto the raw id if the last <=TAIL_WINDOW turns match the recorded skeleton turn-by-turn in CANONICAL form (neutralTurnCanonical — the identity chain's own hashing input), which survives record regrouping but not content mutation. The skeleton persists the canon column additively (absent on pre-change snapshots); a missing or malformed column degrades to exactly the pre-change byte-fragment path. The #5686 fragment reason string stays byte-identical. (cherry picked from commit 5e0dd2a3222fdcb8c41c343dbff954738c734108) --- src/session-resolve.ts | 56 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/src/session-resolve.ts b/src/session-resolve.ts index e68d9a6762..f0378945df 100644 --- a/src/session-resolve.ts +++ b/src/session-resolve.ts @@ -81,7 +81,7 @@ import { createHash } from "node:crypto"; // never lets a shallow side request clobber a deeper persisted entry. import { prefixAffinity } from "./prefix-affinity.js"; import { isClaudeLaneToken } from "./claude-lane-token.js"; -import { neutralTurns, type NeutralTurn } from "./affinity-neutral.js"; +import { neutralTurns, neutralTurnCanonical, type NeutralTurn } from "./affinity-neutral.js"; import { log as loggerLog } from "./logger.js"; // #3397: skeleton persistence hook (deferred-use ESM cycle with // src/order-persist.ts — each side calls the other only inside function @@ -210,7 +210,7 @@ export type SessionOrderResolution = { reason?: string; }; -type Recorded = { kinds: string[]; frags: string[]; tagHash: string; model?: string; lastSeen: number }; +type Recorded = { kinds: string[]; frags: string[]; canon?: string[]; tagHash: string; model?: string; lastSeen: number }; /** #3397: persona-tag identity is carried by its sha256 (the raw text can be * tens of KB of host system prompt; the persisted snapshot must not carry it @@ -218,6 +218,16 @@ type Recorded = { kinds: string[]; frags: string[]; tagHash: string; model?: str const tagHashOf = (tag: string): string => tag === "" ? "" : createHash("sha256").update(tag, "utf8").digest("hex"); +/** Per-turn canonical fingerprint (#5844): sha256 of neutralTurnCanonical — + * the identity chain's own hashing input (kind + logical text/calls), so it + * survives a host regrouping messages into different records while the + * per-source-message `frags` do not. */ +const canonFp = (turns: NeutralTurn[]): string[] => + turns.map((t) => createHash("sha256").update(neutralTurnCanonical(t), "utf8").digest("hex")); + +/** #5844: the persisted canon column's element shape (sha256 hex digest). */ +const CANON_FP_RE = /^[0-9a-f]{64}$/; + /** Order skeletons. #3397: no longer process-memory-only — every mutation is * debounced-persisted (src/order-persist.ts) and hydrated on boot, so a * restart can no longer re-run the cold-start claim race that flipped the @@ -229,13 +239,15 @@ export interface OrderSkeletonEntry { id: string; kinds: string[]; frags: string[]; + /** #5844: per-turn canonical fingerprints — absent on pre-change snapshots. */ + canon?: string[]; tagHash: string; model?: string; lastSeen: number; } function record(rawId: string, turns: NeutralTurn[], tagHash: string, model?: string): void { - skeletons.set(rawId, { kinds: turns.map((t) => t.kind), frags: fragsOf(turns), tagHash, model, lastSeen: Date.now() }); + skeletons.set(rawId, { kinds: turns.map((t) => t.kind), frags: fragsOf(turns), canon: canonFp(turns), tagHash, model, lastSeen: Date.now() }); scheduleOrderPersist(); } @@ -353,6 +365,21 @@ export function resolveSessionOrder(rawId: string, history: unknown, personaTag? const b = recorded.frags.slice(-w); return a.length === b.length && a.every((f, i) => f === b[i]); }; + // #5844: logical-turn twin of fragTailEquals — the last ≤TAIL_WINDOW turns + // compared by CANONICAL form (kind + logical text/calls, the identity + // chain's own hashing input) instead of per-source-message fragments. A + // host upgrade that regroups messages into different records (dsh + // 0.2.2-rc.1: assistant text segments merged into adjacent records) + // destroys the per-message granularity while every logical turn stays + // byte-identical; the canonical comparison sees through the repack. Old + // persisted skeletons carry no canon column → false → pre-change path. + const canonTailEquals = (n: number): boolean => { + if (recorded.canon === undefined) return false; + const w = Math.min(TAIL_WINDOW, n); + const a = canonFp(turns.slice(-w)); + const b = recorded.canon.slice(-w); + return a.length === b.length && a.every((c, i) => c === b[i]); + }; const tagChanged = tagHash !== "" && tagHash !== recorded.tagHash; // An EMPTY incoming tag (system-less / instructions-less request) is // the old ladder's non-anchoring path: it rides whatever the id holds @@ -486,9 +513,19 @@ export function resolveSessionOrder(rawId: string, history: unknown, personaTag? // mirror of the tag-LESS shorter-reanchor below (same two tests, same // TAIL_WINDOW, no extra floor): a byte-identical fragment-verified // tail cannot belong to a different conversation (MESSAGE-IDENTITY). - if (kinds.length > 0 && kinds.length < recorded.kinds.length && tailAligns(kinds, recorded.kinds) && fragTailEquals(kinds.length)) { - record(rawId, turns, tagHash, effectiveModel); - return { key: rawId, action: "reanchor", reason: "persona text changed but the shorter incoming keeps the recorded conversation's fragment-verified tail — head rewrite under a new system prompt, anchor kept (#5686)" }; + if (kinds.length > 0 && kinds.length < recorded.kinds.length && tailAligns(kinds, recorded.kinds)) { + const fragOk = fragTailEquals(kinds.length); + const canonOk = !fragOk && canonTailEquals(kinds.length); + if (fragOk || canonOk) { + record(rawId, turns, tagHash, effectiveModel); + return { + key: rawId, + action: "reanchor", + reason: canonOk + ? "persona text changed but the shorter incoming keeps the recorded conversation's content-verified tail (host repacked the message records, logical turns intact) — anchor kept (#5686/#5844)" + : "persona text changed but the shorter incoming keeps the recorded conversation's fragment-verified tail — head rewrite under a new system prompt, anchor kept (#5686)", + }; + } } // #5719: every byte-level escape needs EVIDENCE (fragments/cells/ // chain continuity) that the composite field failure destroys in the @@ -582,7 +619,7 @@ export function resetSessionOrderForTest(): void { /** #3397: persisted-snapshot surface (src/order-persist.ts wire shape). */ export function orderSnapshot(): OrderSkeletonEntry[] { - return [...skeletons.entries()].map(([id, r]) => ({ id, kinds: r.kinds, frags: r.frags, tagHash: r.tagHash, model: r.model, lastSeen: r.lastSeen })); + return [...skeletons.entries()].map(([id, r]) => ({ id, kinds: r.kinds, frags: r.frags, ...(r.canon !== undefined ? { canon: r.canon } : {}), tagHash: r.tagHash, model: r.model, lastSeen: r.lastSeen })); } /** Merge a hydrated snapshot: per-id newer lastSeen wins. Returns imported count. */ @@ -591,7 +628,10 @@ export function applyOrderSnapshot(entries: OrderSkeletonEntry[]): number { for (const e of entries) { const cur = skeletons.get(e.id); if (!cur || e.lastSeen > cur.lastSeen) { - skeletons.set(e.id, { kinds: e.kinds, frags: e.frags, tagHash: e.tagHash, model: e.model, lastSeen: e.lastSeen }); + // #5844: pre-change snapshots carry no canon column; a malformed + // one degrades to absent (byte-fragment path only), never trusted. + const canon = Array.isArray(e.canon) && e.canon.length > 0 && e.canon.every((c) => typeof c === "string" && CANON_FP_RE.test(c)) ? e.canon : undefined; + skeletons.set(e.id, { kinds: e.kinds, frags: e.frags, canon, tagHash: e.tagHash, model: e.model, lastSeen: e.lastSeen }); imported++; } } From 2ea662b11132b19aa32265e65d7e755d683781c0 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 7 Nov 2026 13:08:40 +0800 Subject: [PATCH 04/15] test: pin the #5844 repacked-tail reanchor (positive case, guardrails, degradation) Six pins: the bug shape reanchors with the new content-verified-tail reason; a mutated last user cell still forks derived-persona; a skeleton hydrated without the canon column forks exactly as before this change; the snapshot round-trip keeps the rescue alive across restarts; a malformed canon column degrades to the byte-fragment path; and a tag-invariant repacked shorter replay already reanchors through the pre-existing owner-rule branch (symmetry row). (cherry picked from commit 62aa540f861a4c78bda8350145ffc2505a97b1e9) --- .../issue5844-repacked-tail-reanchor.test.ts | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 tests/issue5844-repacked-tail-reanchor.test.ts diff --git a/tests/issue5844-repacked-tail-reanchor.test.ts b/tests/issue5844-repacked-tail-reanchor.test.ts new file mode 100644 index 0000000000..84d7528728 --- /dev/null +++ b/tests/issue5844-repacked-tail-reanchor.test.ts @@ -0,0 +1,162 @@ +// #5844: the host-upgrade REPACK shape — a dsh desktop upgrade (0.2.1-rc.3 → +// 0.2.2-rc.1) changed BOTH the system prompt (persona tagHash rotated, +// tagChanged) AND the message-record layout (assistant text segments stopped +// owning their own records; they were folded into the adjacent tool_use +// record). The long session's replay is then STRICTLY SHORTER than the +// recorded skeleton, head-diverged, with every logical turn content-identical +// but regrouped into different source messages. Every existing escape loses: +// head fragments broken (granularity changed), #4778 length-gated (shorter), +// and #5686's fragment tail is disjoint by construction — the old run hashed +// [frag(text), frag(calls)] per turn while the new run hashes one combined +// record, so the per-source-message `frags` never agree even though the +// logical conversation is byte-identical. Result (field, v0.1.212): the whole +// main conversation re-keys onto `|sub:` with ZERO blocks, preflight +// folds ~2.5M tokens from scratch, and the resume dies in a 413 retryable= +// false loop until the user hand-edits stateDir()/session-order.json. +// +// Fix (src/session-resolve.ts): the #5686 shorter-tail rung gains a parallel +// evidence class — the last ≤TAIL_WINDOW turns compared by CANONICAL form +// (kind + logical text/calls, the identity chain's own hashing input via +// neutralTurnCanonical) instead of per-source-message fragments. A repack +// changes the envelope, not the logical bytes; a content-verified tail +// cannot belong to a different conversation (MESSAGE-IDENTITY doctrine, +// exact comparison — no fuzzy similarity). Old persisted skeletons carry no +// canon column → the check degrades to absent → byte-fragment path only, so +// pre-change snapshots behave EXACTLY as before. +// +// Scope honesty: this rung rescues the repack shape when the removed/altered +// cells sit OUTSIDE the tail window. Cells mutated INSIDE the last eight +// positions still fork (the guardrail below) — safe direction, no substrate +// loss. The grown+shifted sibling (#5714) needs subsequence alignment and is +// tracked separately. + +import test from "node:test"; +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { applyOrderSnapshot, orderSkeletonOf, orderSnapshot, resetSessionOrderForTest, resolveSessionOrder, sessionOrderSkeletonOf } from "../src/session-resolve.ts"; + +process.env.NODE_ENV = "test"; +process.env.BILI_PERSIST = "0"; +const testRoot = mkdtempSync(join(tmpdir(), "bili-5844-")); +test.after(() => rmSync(testRoot, { recursive: true, force: true })); +for (const key of ["XDG_STATE_HOME", "XDG_DATA_HOME", "XDG_CACHE_HOME", "XDG_CONFIG_HOME"]) process.env[key] = testRoot; + +type M = { role: string; content?: unknown }; +const ID = "sess-5844-anchor"; +const S1 = "You are DeepSeek Harness.\nModel: GLM-A."; +const S2 = "You are DeepSeek Harness NEXT.\nModel: GLM-B.\nBudget day 2."; // upgrade re-stamp +const fp16 = (s: string): string => createHash("sha256").update(s, "utf8").digest("hex").slice(0, 16); + +/** Old-host turn: assistant prose is its OWN record, then a tool_use record. */ +function oldTurn(i: number): M[] { + return [ + { role: "user", content: `question ${i}. ` + "q".repeat(60) }, + { role: "assistant", content: `prose ${i}. ` + "p".repeat(80) }, + { role: "assistant", content: [{ type: "tool_use", id: `tu-${i}`, name: "read_file", input: { path: `f${i}.txt` } }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: `tu-${i}`, content: `result ${i}. ` + "r".repeat(60) }] }, + ]; +} + +/** New-host turn: prose MERGED into the tool_use record (same logical content). */ +function newTurn(i: number): M[] { + return [ + { role: "user", content: `question ${i}. ` + "q".repeat(60) }, + { role: "assistant", content: [ + { type: "text", text: `prose ${i}. ` + "p".repeat(80) }, + { type: "tool_use", id: `tu-${i}`, name: "read_file", input: { path: `f${i}.txt` } }, + ] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: `tu-${i}`, content: `result ${i}. ` + "r".repeat(60) }] }, + ]; +} + +/** Standalone commentary records the OLD host persisted; the NEW host no + * longer emits them (→ strictly shorter replay). Outside the tail window so + * the last eight logical turns stay intact. */ +const COMMENTARY_AT = [2, 5, 8, 11, 14]; +const TURNS = 30; + +const seed: M[] = [{ role: "system", content: S1 }]; +for (let i = 0; i < TURNS; i++) { + seed.push(...oldTurn(i)); + if (COMMENTARY_AT.includes(i)) seed.push({ role: "assistant", content: `interim commentary ${i}. ` + "c".repeat(80) }); +} +const incoming: M[] = [{ role: "system", content: S2 }]; +for (let i = 0; i < TURNS; i++) incoming.push(...newTurn(i)); // commentaries gone, records repacked + +const SEED_CELLS = orderSkeletonOf(seed).length; +const IN_CELLS = orderSkeletonOf(incoming).length; + +test("#5844: tag-changed STRICTLY-SHORTER repacked-tail replay re-anchors instead of forking", () => { + resetSessionOrderForTest(); + assert.ok(IN_CELLS < SEED_CELLS, "shape premise: the repack replay is strictly shorter"); + const claim = resolveSessionOrder(ID, seed, S1); + assert.equal(claim.action, "claim"); + assert.equal(claim.key, ID); + + const res = resolveSessionOrder(ID, incoming, S2); + assert.equal(res.key, ID, "anchor kept on the raw id — no |sub: fork"); + assert.equal(res.action, "reanchor"); + assert.match(res.reason ?? "", /#5844/, "the content-verified-tail path is distinguishable in the log surface"); + assert.equal(sessionOrderSkeletonOf(ID)?.length, IN_CELLS, "skeleton follows the living (repacked) end"); +}); + +test("#5844 guardrail: repacked replay with a MUTATED tail cell still forks (stranger wearing the skeleton)", () => { + resetSessionOrderForTest(); + resolveSessionOrder(ID, seed, S1); + const mutated: M[] = [...incoming]; + const lastUser = mutated.length - 2; // …, user(question N), assistant(text+tool_use) + mutated[lastUser] = { role: "user", content: "mutated final question — different conversation, same skeleton" }; + const res = resolveSessionOrder(ID, mutated, S2); + assert.equal(res.action, "derived-persona"); + assert.equal(res.key, `${ID}|sub:${fp16(S2)}`); +}); + +test("#5844 degradation: a skeleton hydrated WITHOUT the canon column behaves exactly pre-change (fork)", () => { + resetSessionOrderForTest(); + resolveSessionOrder(ID, seed, S1); + const snap = orderSnapshot(); + assert.ok(snap.some((e) => e.id === ID && Array.isArray(e.canon)), "live skeleton carries the canon column"); + const stripped = snap.map(({ canon: _dropped, ...rest }) => rest); + resetSessionOrderForTest(); + assert.equal(applyOrderSnapshot(stripped), 1); + const res = resolveSessionOrder(ID, incoming, S2); + assert.equal(res.action, "derived-persona", "no canon evidence → byte-fragment path only → fork, exactly as before this change"); + assert.equal(res.key, `${ID}|sub:${fp16(S2)}`); +}); + +test("#5844 persistence round-trip: the canon column survives snapshot → hydrate, so a restart keeps the rescue", () => { + resetSessionOrderForTest(); + resolveSessionOrder(ID, seed, S1); + const snap = orderSnapshot(); + resetSessionOrderForTest(); + assert.equal(applyOrderSnapshot(snap), 1); + const res = resolveSessionOrder(ID, incoming, S2); + assert.equal(res.key, ID); + assert.equal(res.action, "reanchor"); +}); + +test("#5844 hardening: a MALFORMED persisted canon degrades to absent (never trusted)", () => { + resetSessionOrderForTest(); + resolveSessionOrder(ID, seed, S1); + const snap = orderSnapshot(); + // Non-fingerprint elements (hand-edited or corrupted file) must drop the + // whole column: the repack shape falls back to the byte-fragment path. + const malformed = snap.map((e) => (e.id === ID ? { ...e, canon: ["not-a-fingerprint"] } : e)); + resetSessionOrderForTest(); + assert.equal(applyOrderSnapshot(malformed), 1); + const res = resolveSessionOrder(ID, incoming, S2); + assert.equal(res.action, "derived-persona"); + assert.equal(res.key, `${ID}|sub:${fp16(S2)}`); +}); + +test("#5844 symmetry regression: tag-INVARIANT repacked shorter replay re-anchors (pre-existing owner-rule-1 branch untouched)", () => { + resetSessionOrderForTest(); + resolveSessionOrder(ID, seed, S1); + const sameTagIncoming: M[] = [{ role: "system", content: S1 }, ...incoming.slice(1)]; + const res = resolveSessionOrder(ID, sameTagIncoming, S1); + assert.equal(res.key, ID); + assert.equal(res.action, "reanchor"); +}); From 5718af8795d1bc0d6ba92d623866111f6dfd457d Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 7 Nov 2026 13:08:40 +0800 Subject: [PATCH 05/15] =?UTF-8?q?docs:=20document=20persona-lane=20drift?= =?UTF-8?q?=20recovery=20=E2=80=94=20no=20supported=20attach=20command=20e?= =?UTF-8?q?xists=20(#5844)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SESSION-IDENTITY en/zh: state plainly that there is no supported command or config knob to re-attach a drifted |sub lane to its parent, and give the sanctioned manual procedure (stop the proxy first — the daemon silently rewrites session-order.json from memory; set the parent entry's tagHash to the sha256 hex digest of the host's current full system text or delete the entry; restart; bili sessions rm the orphan lane). Answers the reporter's section-8 question from #5844. (cherry picked from commit 676c2590f2a96f820a5925c29b0540a88a92aa6f) --- docs/session-identity.md | 35 ++++++++++++++++++++++++++++++++++ docs/session-identity.zh-CN.md | 13 +++++++++++++ 2 files changed, 48 insertions(+) diff --git a/docs/session-identity.md b/docs/session-identity.md index b1aaa5ba37..2255ef84cd 100644 --- a/docs/session-identity.md +++ b/docs/session-identity.md @@ -256,6 +256,41 @@ From then on: claude/codex/dsh need nothing here: they share one session id across subagents or have no child-session concept at all. +## Persona-lane drift: when re-anchoring is impossible (#5844) + +The order resolver keeps the live conversation on the raw id through persona +text changes whenever it can verify continuation (head continuity, +header-corroborated model switch, positional fragment runs, fragment- or +content-verified tails — `src/session-resolve.ts`). When the host destroys ALL +byte-level evidence in the same turn it rotates its system prompt — e.g. an +upgrade that changes both the system prompt and the message-record layout +(#5844), or a grown+shifted replay (#5714) — the conversation forks onto +`|sub:` with empty compression state. That fork is by design: +mis-attribution is the most expensive failure in this system (MESSAGE-IDENTITY). + +**There is no supported command to re-attach a derived lane to its parent, +and no config knob for it.** The resolution is decided per request from +evidence only; after a drift, the only sanctioned path is surgery on the +persisted skeleton: + +1. Stop the proxy first — the running daemon silently rewrites + `/session-order.json` from its in-memory skeletons, so any edit + made while it runs is lost (state dir = `$XDG_STATE_HOME/billion-context`, + default `~/.local/state/billion-context/`). +2. In `session-order.json`, set the parent entry's `tagHash` to the sha256 hex + digest of the host's CURRENT full system text (the field stores the full + digest, not a truncated fingerprint), or delete the entry entirely to let + the raw id be claimed fresh on the next request. +3. Restart the proxy; the next request re-anchors onto the raw id and the + fold state stored under it becomes live again. +4. Remove the orphaned derived lane: `bili sessions rm |sub:` + (its rows are otherwise inert but keep growing). + +Do NOT merge lanes when the replay content genuinely diverged (a different +conversation riding the same client id) — that is exactly what the fork +exists to prevent; the procedure above is for "same conversation, evidence +destroyed by the host". + ## Future direction The durable fix is a **client-side identity signal**: a stable diff --git a/docs/session-identity.zh-CN.md b/docs/session-identity.zh-CN.md index bfffdddb01..4ef655b2f0 100644 --- a/docs/session-identity.zh-CN.md +++ b/docs/session-identity.zh-CN.md @@ -144,6 +144,19 @@ claude/codex/dsh 在这里不需要任何信号:它们的 subagent 共享同一个会话 id,或根本没有子会话概念。 +## 派生车道漂移:无法重新锚定时怎么办(#5844) + +顺序解析器在能验证延续性时(头部延续、头部佐证的模型切换、逐位 fragment 连续段、fragment 或内容校验的尾部——`src/session-resolve.ts`)会把活会话留在原始 id 上。但当宿主在轮换 system prompt 的同一轮里销毁了**全部**字节级证据——例如一次升级同时改了 system prompt 和消息记录布局(#5844),或增长+位移的重放(#5714)——会话就会分叉到 `|sub:`,压缩状态为空。这是有意设计:误归属(mis-attribution)是本系统代价最高的失败(MESSAGE-IDENTITY)。 + +**目前没有任何受支持的命令能把派生车道重新挂回父会话,也没有对应的配置项。** 解析结果每轮只由证据决定;漂移发生后,唯一被认可的恢复路径是对持久化骨架做手术: + +1. 先停代理——运行中的守护进程会用内存里的骨架悄悄重写 `/session-order.json`,运行期间做的任何编辑都会丢失(state 目录 = `$XDG_STATE_HOME/billion-context`,默认 `~/.local/state/billion-context/`)。 +2. 在 `session-order.json` 里,把父条目的 `tagHash` 设为宿主**当前**完整 system 文本的 sha256 hex 摘要(该字段存的是完整摘要,不是截断指纹);或者直接删掉整个条目,让原始 id 在下一次请求时重新认领。 +3. 重启代理;下一次请求重新锚定到原始 id,其下已存储的折叠状态随之复活。 +4. 删除孤儿派生车道:`bili sessions rm |sub:`(其行数据虽不再被使用,但会持续增长)。 + +如果重放内容与记录**确实**分叉了(同一客户端 id 上跑着另一个对话),不要合并车道——这正是分叉机制要防止的事;上面的流程只适用于「同一对话、证据被宿主销毁」的情形。 + ## 未来方向 持久的修复是**客户端侧身份信号**:由客户端自身提供的稳定 per-conversation From 57d0ca72e1bb941c87b8d14199221925ab322009 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 7 Nov 2026 14:58:05 +0800 Subject: [PATCH 06/15] =?UTF-8?q?fix(#5844):=20review=20hardening=20?= =?UTF-8?q?=E2=80=94=20distrust=20a=20persisted=20canon=20column=20whose?= =?UTF-8?q?=20length=20drifts=20from=20kinds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The writers keep canon 1:1 with kinds (both derive from the same turn list in record()), so a length mismatch means the column is shifted/corrupted. Degrade to absent (byte-fragment path = pre-change behavior) instead of trusting misaligned fingerprints. (cherry picked from commit a1e0a7be9d962264f8dc6dea8aa18582d2638c91) --- src/session-resolve.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/session-resolve.ts b/src/session-resolve.ts index f0378945df..15881e98f0 100644 --- a/src/session-resolve.ts +++ b/src/session-resolve.ts @@ -630,7 +630,10 @@ export function applyOrderSnapshot(entries: OrderSkeletonEntry[]): number { if (!cur || e.lastSeen > cur.lastSeen) { // #5844: pre-change snapshots carry no canon column; a malformed // one degrades to absent (byte-fragment path only), never trusted. - const canon = Array.isArray(e.canon) && e.canon.length > 0 && e.canon.every((c) => typeof c === "string" && CANON_FP_RE.test(c)) ? e.canon : undefined; + // The writers keep canon 1:1 with kinds (both derive from the same + // turn list in record()), so a length mismatch means the column is + // shifted/corrupted — trust nothing from it. + const canon = Array.isArray(e.canon) && e.canon.length > 0 && e.canon.length === e.kinds.length && e.canon.every((c) => typeof c === "string" && CANON_FP_RE.test(c)) ? e.canon : undefined; skeletons.set(e.id, { kinds: e.kinds, frags: e.frags, canon, tagHash: e.tagHash, model: e.model, lastSeen: e.lastSeen }); imported++; } From 30b479b29a27170722884ef2680437b633412bab Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 7 Nov 2026 14:58:05 +0800 Subject: [PATCH 07/15] =?UTF-8?q?test(#5844):=20review=20fix=20=E2=80=94?= =?UTF-8?q?=20guardrail=20must=20exercise=20the=20canon=20path,=20not=20a?= =?UTF-8?q?=20kind=20break?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original guardrail replaced an assistant record with a user record, so the fork came from tailAligns' kind suffix comparison and never reached the canonical-fingerprint comparison it was meant to pin (it would pass even with the canon rung removed). Rewrite it to mutate the final tool result's text — layout and kinds unchanged — so the fork can only come from the canon mismatch itself. (cherry picked from commit 1d37f635ce5d67c226863100fe4b12c737d31f46) --- tests/issue5844-repacked-tail-reanchor.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/issue5844-repacked-tail-reanchor.test.ts b/tests/issue5844-repacked-tail-reanchor.test.ts index 84d7528728..30ff610b95 100644 --- a/tests/issue5844-repacked-tail-reanchor.test.ts +++ b/tests/issue5844-repacked-tail-reanchor.test.ts @@ -107,8 +107,16 @@ test("#5844 guardrail: repacked replay with a MUTATED tail cell still forks (str resetSessionOrderForTest(); resolveSessionOrder(ID, seed, S1); const mutated: M[] = [...incoming]; - const lastUser = mutated.length - 2; // …, user(question N), assistant(text+tool_use) - mutated[lastUser] = { role: "user", content: "mutated final question — different conversation, same skeleton" }; + // Mutate the FINAL tool result's text while keeping the repacked layout + // (same kinds, same record shape): the fork must come from the + // canonical-fingerprint mismatch itself, not from a kind break (a kind + // break dies earlier in tailAligns and would pass even without the canon + // comparison). A tool-result turn carries its text in the canonical form + // (unlike an assistant calls turn, whose canon is calls+role only for + // cross-wire neutrality — see affinity-neutral.ts header). + const lastTr = mutated.length - 1; + assert.equal((mutated[lastTr] as M).role, "user", "shape premise: the replay ends with a tool-result record"); + mutated[lastTr] = { role: "user", content: [{ type: "tool_result", tool_use_id: `tu-${TURNS - 1}`, content: "mutated final result — different conversation, same skeleton" }] }; const res = resolveSessionOrder(ID, mutated, S2); assert.equal(res.action, "derived-persona"); assert.equal(res.key, `${ID}|sub:${fp16(S2)}`); From dc0d5ee822791f1b6656b9133e22ca69e121a059 Mon Sep 17 00:00:00 2001 From: ci-check Date: Sat, 7 Nov 2026 14:58:05 +0800 Subject: [PATCH 08/15] docs(#5844): document the calls-turn prose boundary of the content-verified tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An assistant turn carrying tool calls hashes as calls+role only (cross-wire neutrality — OpenAI renders such turns with content:null), so a prose-only difference inside a calls turn is invisible to the #5844 rung exactly as it is to the rest of the identity chain that shares neutralTurnCanonical. State that boundary next to the manual re-attach procedure (en+zh). (cherry picked from commit c39dc843e1073d19c733a00dbf1e941d7b815850) --- docs/session-identity.md | 10 ++++++++++ docs/session-identity.zh-CN.md | 2 ++ 2 files changed, 12 insertions(+) diff --git a/docs/session-identity.md b/docs/session-identity.md index 2255ef84cd..8348184618 100644 --- a/docs/session-identity.md +++ b/docs/session-identity.md @@ -268,6 +268,16 @@ upgrade that changes both the system prompt and the message-record layout `|sub:` with empty compression state. That fork is by design: mis-attribution is the most expensive failure in this system (MESSAGE-IDENTITY). +The "content-verified" tail compares the last ≤8 turns in canonical form — +the identity chain's own hashing input (`neutralTurnCanonical`): kind plus +logical text/calls. For an assistant turn carrying tool calls the canonical +form is calls+role only (cross-wire neutrality — OpenAI renders such turns +with `content: null`), so a prose-only difference inside a calls turn is +invisible to this rung exactly as it is to the rest of the chain that shares +the same canonical form. A different conversation must still replay identical +user questions, tool results and tool calls across the whole tail window to be +accepted; record regrouping alone can never do that. + **There is no supported command to re-attach a derived lane to its parent, and no config knob for it.** The resolution is decided per request from evidence only; after a drift, the only sanctioned path is surgery on the diff --git a/docs/session-identity.zh-CN.md b/docs/session-identity.zh-CN.md index 4ef655b2f0..ad90d86801 100644 --- a/docs/session-identity.zh-CN.md +++ b/docs/session-identity.zh-CN.md @@ -148,6 +148,8 @@ claude/codex/dsh 在这里不需要任何信号:它们的 subagent 共享同一 顺序解析器在能验证延续性时(头部延续、头部佐证的模型切换、逐位 fragment 连续段、fragment 或内容校验的尾部——`src/session-resolve.ts`)会把活会话留在原始 id 上。但当宿主在轮换 system prompt 的同一轮里销毁了**全部**字节级证据——例如一次升级同时改了 system prompt 和消息记录布局(#5844),或增长+位移的重放(#5714)——会话就会分叉到 `|sub:`,压缩状态为空。这是有意设计:误归属(mis-attribution)是本系统代价最高的失败(MESSAGE-IDENTITY)。 +「内容校验」的尾部比较的是最后 ≤8 轮的**规范化形式**——身份链自有的 hash 输入(`neutralTurnCanonical`):kind 加逻辑文本/调用。对携带工具调用的 assistant 轮,规范化形式只含 calls+role(跨线中性——OpenAI 会把这类轮渲染为 `content: null`),因此 calls 轮内**仅散文不同**对本档不可见,这与共享同一规范化形式的身份链其余部分完全一致。另一个对话要能被接受,仍须在整段尾窗上重放逐字一致的用户提问、工具结果与工具调用;单纯的消息重打包做不到这一点。 + **目前没有任何受支持的命令能把派生车道重新挂回父会话,也没有对应的配置项。** 解析结果每轮只由证据决定;漂移发生后,唯一被认可的恢复路径是对持久化骨架做手术: 1. 先停代理——运行中的守护进程会用内存里的骨架悄悄重写 `/session-order.json`,运行期间做的任何编辑都会丢失(state 目录 = `$XDG_STATE_HOME/billion-context`,默认 `~/.local/state/billion-context/`)。 From 207a24df824c2674a9e8b99ceb1afdd55b8fe6bd Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 8 Nov 2026 01:15:17 +0800 Subject: [PATCH 09/15] =?UTF-8?q?fix:=20correct=20stale=20rung=20ordinal?= =?UTF-8?q?=20=E2=80=94=20#5686=20shorter-tail=20rung=20is=20the=20sixth?= =?UTF-8?q?=20reanchor=20rung?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 535bb92ccbdc9a724b1a6720386643f3dee2ad9d) --- src/session-resolve.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/session-resolve.ts b/src/session-resolve.ts index 15881e98f0..babc0958e7 100644 --- a/src/session-resolve.ts +++ b/src/session-resolve.ts @@ -500,7 +500,7 @@ export function resolveSessionOrder(rawId: string, history: unknown, personaTag? record(rawId, turns, tagHash, effectiveModel); return { key: rawId, action: "reanchor", reason: "persona text changed but the recorded history continues inside the incoming — anchor migrated (#4778)" }; } - // #5686: fifth rung — the composite field failure behind the + // #5686: sixth rung — the composite field failure behind the // |sub:-drift cluster (#5517/#5656 reports): a native-compaction // checkpoint LANDS (head cells replaced by one summary message) while // the host simultaneously rotates its per-turn system prompt (dsh NEXT From 0be5b7354cb3c4291e4039658effab29da69b1ad Mon Sep 17 00:00:00 2001 From: ci-check Date: Sun, 8 Nov 2026 00:24:45 +0800 Subject: [PATCH 10/15] fix: stop anthropic-wire sessions from sailing past the upstream wall (#5880) Two independent defects let a session bili believes fits request more than the upstream accepts on the Anthropic wire when the provider bills the completion against the same window as input (DeepSeek et al.): 1. The post-preflight-fold baseline anchor (server.ts) overwrote an ACTIVE overflow-arm evidence tag with an estimate-grade one whenever the fresh rebuilt-view estimate measured slightly above the netted arm value. Every successful fold silently disarmed the evidence floor; the next turn then fit-forwarded on a stale optimistic estimate until the 400 re-armed it - a fold -> forward -> 400 livelock. Extracted the write into applyPostRebuildBaselineAnchor (session.ts) with an overflow-arm guard, mirroring the armFailureShrink doctrine (relay.ts #2484). Unarmed and usage-grade behavior is unchanged (#1987/#604). 2. The outgoing max_tokens clamp had no call site on the anthropic lane (budget.ts wired openai/responses/google only). Added it, sized against the RAW native window exactly like the other lanes (#2096 semantics: nudge/preflight judge the headroom-adjusted window, egress guards the raw one). For canonical Anthropic (separate input/output limits) the requested budget already rides under the window and the clamp is a no-op. Tests: tests/issue5880-anthropic-clamp-arm.test.ts - unit pins for the arm guard (both directions) plus hermetic e2e through /v1/messages proving the clamp fires against the raw window near the wall and stays quiet when the request fits. Fixes #5880 (cherry picked from commit eba8a07dc83ffe4f0ba92c1b55ced85b06b5b611) --- src/server.ts | 7 +- src/server/budget.ts | 19 +- src/server/handle.ts | 2 +- src/server/prepare-anthropic.ts | 14 +- src/session.ts | 16 ++ tests/issue5880-anthropic-clamp-arm.test.ts | 253 ++++++++++++++++++++ 6 files changed, 294 insertions(+), 17 deletions(-) create mode 100644 tests/issue5880-anthropic-clamp-arm.test.ts diff --git a/src/server.ts b/src/server.ts index f06ee81a2b..c0a21c27fe 100644 --- a/src/server.ts +++ b/src/server.ts @@ -55,7 +55,7 @@ import { prepareResponses, prepareResponsesCompact } from "./server/prepare-resp import { FORCE_TEXT_PROTOCOL } from "./server/inject.js"; export { isChatGptCodexUpstream, isCodexResponsesLite, resolvePromptCacheKey, shouldInjectPromptCacheKey } from "./server/prepare-responses.js"; import { biliToolsDeclaredOnWire, countBiliToolUses, evaluateSelfHealRound, nudgeSuppressed, pluginLaneDegraded, pluginLaneRestore } from "./session-self-heal.js"; -import { getSession, hasProcessedState, listSessions, peekSession, type PendingRetrieval, type Session, initSessions, markDirty, flushAllSessions, acquireInFlight, releaseInFlight, totalInFlight, snapshotMessages, ensureCanonicalId, storeEffectiveConfig, foldCoverage, postRebuildAnchorTokens, setPostRebuildAnchor, tickPostRebuildAnchor, splitSessionWarnings, usageAnchorCycleStale, staleSmallAnchorFallback } from "./session.js"; +import { getSession, hasProcessedState, listSessions, peekSession, type PendingRetrieval, type Session, initSessions, markDirty, flushAllSessions, acquireInFlight, releaseInFlight, totalInFlight, snapshotMessages, ensureCanonicalId, storeEffectiveConfig, foldCoverage, postRebuildAnchorTokens, setPostRebuildAnchor, tickPostRebuildAnchor, applyPostRebuildBaselineAnchor, splitSessionWarnings, usageAnchorCycleStale, staleSmallAnchorFallback } from "./session.js"; import type { SeparatorManifest } from "./server/responses-ordering.js"; import { detectStaleInstall, isVersionNewer } from "./update.js"; import { decideEaddrinuse, eaddrinuseExitCode, isPortBusyError, postBindCanLadder, UPGRADE_REPLACEMENT_ENV, type FailFastReason } from "./listen-policy.js"; @@ -4372,10 +4372,7 @@ export async function preflightCompressIfNeeded( const rebuiltImageTokens = imageReserveFor(session, rebuilt.protocol, rebuilt.body, opts, upstreamUrl); remainingImageTokens = rebuiltImageTokens; reducedImageWire = result.compressedRanges > 0 && rebuiltImageTokens < imageTokens; - if (rebuiltTextSize > session.stats.lastInputTokens) { - session.stats.lastInputTokens = rebuiltTextSize; - session.stats.lastInputTokensSource = "estimate"; - } + applyPostRebuildBaselineAnchor(session, rebuiltTextSize); // Same calibrated caliber as the trigger above — gate, per-round exit // and this final fit must judge the payload on one scale (#1933 F1). // The baseline anchor above stays raw deliberately: it is a floor for diff --git a/src/server/budget.ts b/src/server/budget.ts index 014aaa23d2..b1e091e520 100644 --- a/src/server/budget.ts +++ b/src/server/budget.ts @@ -10,15 +10,16 @@ export { readOutputBudget }; // #453 hard backstop: cap the forwarded output budget so input+output can never // exceed the window on request-rebuilding upstreams (vLLM rejects an oversized -// total instead of clamping). Wired into the openai/responses/google prepare -// lanes only — canonical Anthropic routes enforce their input limit -// independently of max_tokens, so they get no clamp call site. #4205 boundary: -// wrapper providers on the anthropic wire (completion billed against the same -// window) DO reserve headroom at the band layer but keep no egress clamp here; -// their residual over-requests ride the overflow self-heal (#4093) instead. -// The proxy owns both sides of the sum, so capping output to (window - input - -// margin) makes the overflow impossible even when the agent ignores the -// compress nudge. +// total instead of clamping). Wired into ALL FOUR prepare lanes — openai, +// responses, google, and anthropic (#5880 added the anthropic site: wrapper +// providers on that wire bill completion against the SAME stated window as +// input — DeepSeek's 400 quotes "N in the messages, M in the completion" +// against one wall — which is exactly the shape this clamp exists for; for +// canonical Anthropic, whose input limit is enforced independently of +// max_tokens, the requested budget already rides under the window and the +// clamp is a no-op). The proxy owns both sides of the sum, so capping output +// to (window - input - margin) makes the overflow impossible even when the +// agent ignores the compress nudge. const OUTPUT_CLAMP_MARGIN_PCT = 0.05; const OUTPUT_CLAMP_MIN_MARGIN = 2048; // #5278: safety inflation applied to the planning estimate BEFORE sizing both diff --git a/src/server/handle.ts b/src/server/handle.ts index 0166ebde6b..643a31cb35 100644 --- a/src/server/handle.ts +++ b/src/server/handle.ts @@ -3036,7 +3036,7 @@ async function handleRequest( return await prepareGoogle(work as GoogleRequestBody, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, nativeWindow, googleModel ?? v1Model, googlePathKind(urlPath) === "stream-generate", visibilityMarkers, upstreamOrigin, req, decide); } const preparedOut = protocol === "anthropic" - ? await prepareAnthropic(work as AnthropicRequestBody, req, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, upstreamOrigin, reasoningCfg, visibilityMarkers, requestModel, decide) + ? await prepareAnthropic(work as AnthropicRequestBody, req, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, upstreamOrigin, nativeWindow, reasoningCfg, visibilityMarkers, requestModel, decide) : protocol === "openai" ? await prepareOpenai(work as OpenAIRequestBody, req, opts, core, reqConfig, reqPrompts, reqSurface, log, session, pluginMode, upstreamOrigin, nativeWindow, reasoningCfg, visibilityMarkers, route?.rewrittenUrl, decide, bodyText) : responsesCompact diff --git a/src/server/prepare-anthropic.ts b/src/server/prepare-anthropic.ts index a7b1908856..3449cb81ca 100644 --- a/src/server/prepare-anthropic.ts +++ b/src/server/prepare-anthropic.ts @@ -7,7 +7,8 @@ import { captureThreadsTurn, estimateThreadsTokens, isThreadsBody, threadsReforg import type { CompressReasoningConfig } from "../reasoning-drop.js"; import type { ProxyOptions } from "../config.js"; import { pluginAgentHeader } from "../plugin.js"; -import { applyReasoningDrop, dedupeAdjacentOutputConfig, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, type Prepared } from "../server.js"; +import { applyReasoningDrop, dedupeAdjacentOutputConfig, diagNudge, diagTagSummary, deriveTitle, effectiveTokenCount, imageBillingFor, imageReserveFor, imageTokenCapFor, isAutoInjectedNotification, planReasoningDrop, reapOrphansLogged, resolveKnownOutputCeiling, runNudgeDecision, stripKernelSummaries, warnAnthropicThinkingPairs, type Prepared } from "../server.js"; +import { currentCalibrationFactor } from "../util.js"; import { recordNudgeObservation } from "../context-analysis.js"; import { buildDecisionPrompt, buildDirectiveText, consumeFallback, ladderMode, resolveDecisionRange, stripDecisionReasoningFields, type DecideConfig } from "../nudge-decide.js"; import { excludeDeadRefRanges } from "../acp-status.js"; @@ -38,7 +39,7 @@ import { logRawStripCounts, markerArtifactCount, stripRawStage } from "../marker import { imageRescaleMaxDim, renderNone as knobRenderNone } from "../knobs.js"; import { applyImageRescalePass } from "../image-rescale.js"; import { isAutoModeClassifier } from "../auto-mode-forge.js"; -import { countSystemAndToolsTokens, emergencyNudge, emergencyNudgeArmed, projectThinkingMass } from "./budget.js"; +import { clampOutgoingOutput, countSystemAndToolsTokens, dshLedgerFloorTokens, emergencyNudge, emergencyNudgeArmed, projectThinkingMass } from "./budget.js"; import { effectiveAbsorbBlock } from "./prepare-responses.js"; import { effectiveBlockOpsEnabled } from "./prepare-responses.js"; import { injectSystem, injectTool } from "./inject.js"; @@ -61,6 +62,7 @@ export async function prepareAnthropic( session: Session, pluginMode: boolean, upstreamOrigin: string, + nativeWindow: number, reasoning: CompressReasoningConfig | undefined, visibilityMarkers: boolean, /** The route's resolved model, when the caller already has one. On the @@ -678,6 +680,14 @@ export async function prepareAnthropic( // main-scale tiebreak sees this request's own upper bound. probeRequestMembership(session, collectRequestIds(incomingMsgIds, originalMessages), session.stats.localInputEstimate ?? 0); if (upstreamOrigin) session.stats.lastLocalTextEstimateOrigin = upstreamOrigin; + // #5880: egress output-budget clamp, same call-site family as the openai/ + // responses/google lanes (prepare-openai.ts). Wrapper providers on this + // wire bill completion against the SAME stated window as input (DeepSeek's + // 400: "N in the messages, M in the completion" against one wall), so an + // input-only fit judgment must be backed by an output clamp here too; for + // canonical Anthropic (separate input/output limits) the requested + // max_tokens already rides under the window and the clamp is a no-op. + clampOutgoingOutput(rebuilt as Record, "max_tokens", { systemText: extractSystem(systemOut), tools: toolsOut, processedMessages, lastInputTokens: session.stats.lastInputTokens, lastInputTokensSource: session.stats.lastInputTokensSource, nativeWindow, headroomWindow: config.modelContextLimit, imageTokens: imageReserve, kFactor: currentCalibrationFactor(session.stats, session.metadata?.lastModel), kOrigin: session.stats.calibratedEstimateOrigin, origin: upstreamOrigin, ledgerFloorTokens: dshLedgerFloorTokens(session.metadata), knownOutputCeiling: resolveKnownOutputCeiling(req.headers, parsed as Record, opts.routes, upstreamOrigin, opts.sessionHeader, modelIdOf(parsed)) }, sessionId, log, (c) => { session.stats.lastOutputClamp = { ...c, at: Date.now() }; }); return { body: JSON.stringify(rebuilt), parsedBody: rebuilt, session, attachedRetrievals, attachedRetrievalNoteIds, processedMessages, originalMessages, judgeBaseline, anthropicSystem: parsed.system, anthropicBillingBlock, anthropicCacheMarks, anthropicClientCacheControls: clientCacheControls, systemNotes: sysNotes, protocol: "anthropic", stream, compressInjected: injectTools, pluginMode, mcpDialectNote: laneDecisions.mcpDialect, nudge, nudgeOutcome, prompts, surface, renderTags: renderStrategy, dropReasoning: (msgs: BiliMessage[]) => applyReasoningDrop(msgs, reasoningPlan, log, sessionId) } as Prepared; } diff --git a/src/session.ts b/src/session.ts index 33c9833c52..3be5be8ccb 100644 --- a/src/session.ts +++ b/src/session.ts @@ -1155,6 +1155,22 @@ export function setPostRebuildAnchor(session: Session, tokens: number): void { markDirty(session); } +// #1987 post-rebuild baseline anchor, hardened by #5880: an upstream-proven +// overflow-arm tag must NEVER be demoted to estimate-grade by a local payload +// estimate — demoting drops the evidence floor that keeps an armed session +// folding instead of trusting optimistic estimates (same doctrine as +// armFailureShrink, relay.ts #2484). In the #5880 incident the caliber delta +// between the netted arm value and a fresh estimate of the rebuilt view tipped +// the inequality on every fold turn, silently disarming the session so its +// estimate-only forwards 400'd turn after turn. +export function applyPostRebuildBaselineAnchor(session: Session, rebuiltTextSize: number): void { + if (session.stats.lastInputTokensSource === "overflow-arm") return; + if (rebuiltTextSize > session.stats.lastInputTokens) { + session.stats.lastInputTokens = rebuiltTextSize; + session.stats.lastInputTokensSource = "estimate"; + } +} + /** Metadata is persisted user-editable JSON — re-validate on read; a corrupt * stamp degrades to "no anchor" (legacy sizing) instead of poisoning the meter. */ export function postRebuildAnchorTokens(session: Session | undefined): number { diff --git a/tests/issue5880-anthropic-clamp-arm.test.ts b/tests/issue5880-anthropic-clamp-arm.test.ts new file mode 100644 index 0000000000..1d0c121f07 --- /dev/null +++ b/tests/issue5880-anthropic-clamp-arm.test.ts @@ -0,0 +1,253 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +// #5880: two independent defects let a session bili believes fits sail past the +// upstream wall on the ANTHROPIC wire (wrapper providers like DeepSeek bill the +// completion against the SAME window — their 400 says so): +// A. the post-preflight-fold anchor write (server.ts) overwrote an ACTIVE +// overflow-arm evidence tag with an estimate-grade one whenever the fresh +// rebuilt-view estimate measured slightly above the netted arm value — +// every successful fold silently disarmed the evidence floor, so the next +// turn fit-forwarded on a stale optimistic estimate until the 400 re-armed +// it: a livelock (fold → forward → 400 → re-arm → repeat). +// B. the outgoing max_tokens clamp had NO call site on the anthropic lane +// (budget.ts wired openai/responses/google only) — even with an honest +// estimate, requested input+output could exceed the native window. +// These tests pin both: the arm tag survives the anchor write (and unarmed +// sessions keep the #1987 behavior), and the anthropic lane clamps max_tokens +// against the RAW native window exactly like the other lanes do (#2096 +// semantics — the headroom-adjusted window feeds nudge/preflight, the raw one +// guards the egress budget). + +import { createInitialState } from "acp-kernel"; +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { setLogCapture } from "../src/logger.ts"; +import { applyPostRebuildBaselineAnchor } from "../src/session.ts"; +import type { Session } from "../src/session.ts"; + +// ---------- Part 1: unit — the post-rebuild anchor must not demote an arm ---------- + +function makeSession(lastInputTokens: number, source: Session["stats"]["lastInputTokensSource"]): Session { + const now = Date.now(); + return { + id: "s5880-unit", + meta: {}, + metadata: {}, + state: createInitialState(), + stats: { + requests: 1, + tokensSaved: 0, + inputTokens: 0, + cachedTokens: 0, + outputTokens: 0, + cacheSamples: 0, + lastInputTokens, + lastInputTokensSource: source, + compressCreditTokens: 0, + contextTokens: lastInputTokens, + retrieveCalls: 0, + retrieveHits: 0, + retrieveMisses: 0, + storedBytes: 0, + storeBytesSaved: 0, + rangeRestores: 0, + }, + pendingRetrievals: [], + createdAt: now, + lastSeen: now, + blockContents: new Map(), + inFlight: 0, + persisted: false, + } as unknown as Session & Record; +} + +test("#5880 unit: an active overflow-arm tag survives the post-rebuild anchor write", () => { + // Incident shape: the arm holds the netted measured value; the fresh + // rebuilt-view estimate measures slightly HIGHER (caliber delta) — the old + // inequality-based write demoted the tag to "estimate" here. + const s = makeSession(783_000, "overflow-arm"); + applyPostRebuildBaselineAnchor(s, 783_353); + assert.equal(s.stats.lastInputTokensSource, "overflow-arm", "armed tag must not be demoted"); + assert.equal(s.stats.lastInputTokens, 783_000, "arm value must not be overwritten by the estimate"); +}); + +test("#5880 unit: unarmed sessions keep the #1987 anchor behavior (both directions)", () => { + const up = makeSession(100_000, "estimate"); + applyPostRebuildBaselineAnchor(up, 120_000); + assert.equal(up.stats.lastInputTokens, 120_000, "larger rebuilt view must still anchor (#1987)"); + assert.equal(up.stats.lastInputTokensSource, "estimate"); + + const down = makeSession(100_000, "estimate"); + applyPostRebuildBaselineAnchor(down, 80_000); + assert.equal(down.stats.lastInputTokens, 100_000, "smaller rebuilt view must not lower the floor"); +}); + +test("#5880 unit: usage-grade baselines keep the pre-existing #604 overwrite behavior", () => { + // Only the overflow-arm tag is protected; a newer larger local measurement + // still retires an older usage sample (the relay path does the same). + const u = makeSession(100_000, "usage"); + applyPostRebuildBaselineAnchor(u, 120_000); + assert.equal(u.stats.lastInputTokens, 120_000); + assert.equal(u.stats.lastInputTokensSource, "estimate", "usage→estimate overwrite stays #604 behavior"); +}); + +// ---------- Part 2: e2e — the anthropic lane clamps max_tokens near the wall ---------- + +const WINDOW = 200_000; +const MODEL = "ds-flash"; + +// #5278/#2096 derivation kept here so the cap math stays testable end-to-end. +const expectedCap = (input: number, window: number): number => { + const safeEst = Math.ceil(input * 1.12); + const margin = Math.max(2048, Math.ceil(safeEst * 0.05)); + return Math.floor(window - safeEst - margin); +}; + +function filler(chars: number): string { + const unit = "the quick brown fox jumps over the lazy dog. "; + let s = ""; + while (s.length < chars) s += unit; + return s.slice(0, chars); +} + +function anthropicSse(event: string, data: unknown): string { + return `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`; +} + +const anthropicSseReply = (): string => + anthropicSse("message_start", { type: "message_start", message: { id: "m1", role: "assistant" } }) + + anthropicSse("content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }) + + anthropicSse("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "ok" } }) + + anthropicSse("content_block_stop", { type: "content_block_stop", index: 0 }) + + anthropicSse("message_delta", { type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 2 } }) + + anthropicSse("message_stop", { type: "message_stop" }); + +interface Rig { + proxyPort: number; + upstreamPort: number; + bodies: Array<{ url: string; json: Record }>; + closeAll(): Promise; +} + +async function startHarness(): Promise { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const bodies: Rig["bodies"] = []; + const upstream = http.createServer((req, res) => { + let raw = ""; + req.on("data", (c) => { raw += c; }); + req.on("end", () => { + try { bodies.push({ url: req.url ?? "", json: JSON.parse(raw) }); } catch { /* ignore */ } + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(anthropicSseReply()); + }); + }); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { [MODEL]: { context: WINDOW } } } }, + modelContextLimit: WINDOW, + kernelConfig: defaultConfig(WINDOW), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + await once(proxy, "listening"); + return { + proxyPort: (proxy.address() as { port: number }).port, + upstreamPort, + bodies, + closeAll: async () => { + proxy.close(); + proxy.closeIdleConnections?.(); + upstream.close(); + upstream.closeIdleConnections?.(); + await Promise.allSettled([once(proxy, "close"), once(upstream, "close")]); + }, + }; +} + +async function driveAnthropic(rig: Rig, session: string, body: Record): Promise { + const r = await fetch(`http://127.0.0.1:${rig.proxyPort}/bili/http://127.0.0.1:${rig.upstreamPort}/v1/messages`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": session }, + body: JSON.stringify(body), + }); + assert.equal(r.status, 200, `request must succeed, got ${r.status}: ${await r.text().catch(() => "")}`); + await r.body?.cancel().catch(() => { /* drained */ }); +} + +test("#5880 e2e: near-wall anthropic request gets max_tokens clamped against the RAW window", async () => { + const logs: string[] = []; + setLogCapture((level, msg) => { logs.push(`${level} ${msg}`); }); + const rig = await startHarness(); + try { + // ~560k ASCII chars ≈ 140k estimated tokens (non-CJK counts len/4) with + // max_tokens 60_000 (>25% of the window → headroom reservation shrinks + // the EFFECTIVE window to 150_000). The cap must be computed against + // the RAW 200_000, where it lands ~35k — below the request, so it fires; + // against the effective 150_000 it would go negative and surrender. + await driveAnthropic(rig, "s5880-clamp", { + model: MODEL, + max_tokens: 60_000, + stream: true, + system: "You are a helpful assistant.", + messages: [{ role: "user", content: filler(560_000) }], + }); + const sent = rig.bodies.at(-1)?.json; + assert.ok(sent, "upstream must have received the request"); + const sentMax: unknown = sent.max_tokens; + assert.ok(typeof sentMax === "number", `max_tokens must remain a number, got ${typeof sentMax}`); + assert.ok(sentMax < 60_000, `near-wall request must be clamped below 60000, got ${sentMax}`); + assert.ok(sentMax >= 1024, `clamp must stay above the surrender floor, got ${sentMax}`); + + const line = logs.find((l) => l.includes(`[s5880-clamp]`) && l.includes("output budget clamped")); + assert.ok(line, `a clamp log line must exist, got: ${JSON.stringify(logs.filter((l) => l.includes("clamped")))}`); + const m = /output budget clamped 60000 -> (\d+) \(input~(\d+), window=(\d+)\)/.exec(line!); + assert.ok(m, `clamp line must carry the cap math, got: ${line}`); + assert.equal(Number(m![3]), WINDOW, "cap must be computed against the RAW native window (#2096)"); + assert.equal(sentMax, expectedCap(Number(m![2]), WINDOW), "cap math must match the logged estimate"); + } finally { + setLogCapture(null); + await rig.closeAll(); + } +}); + +test("#5880 e2e: comfortably-fitting anthropic request keeps its max_tokens", async () => { + const logs: string[] = []; + setLogCapture((level, msg) => { logs.push(`${level} ${msg}`); }); + const rig = await startHarness(); + try { + await driveAnthropic(rig, "s5880-fit", { + model: MODEL, + max_tokens: 60_000, + stream: true, + system: "You are a helpful assistant.", + messages: [{ role: "user", content: "hello" }], + }); + const sent = rig.bodies.at(-1)?.json; + assert.ok(sent, "upstream must have received the request"); + assert.equal(sent?.max_tokens, 60_000, "fitting request must pass through unclamped"); + assert.ok(!logs.some((l) => l.includes("[s5880-fit]") && l.includes("output budget clamped")), "no clamp may fire for a fitting request"); + } finally { + setLogCapture(null); + await rig.closeAll(); + } +}); From ad7a2973c315505d9ea224147de1e4a66f998c6c Mon Sep 17 00:00:00 2001 From: ranxianglei Date: Tue, 10 Nov 2026 19:00:45 +0800 Subject: [PATCH 11/15] fix: drop dead Record from #5880 test session cast (cherry picked from commit d7dad51a0c1bd55dc0407ff4fd2f1f6c33c044bc) --- tests/issue5880-anthropic-clamp-arm.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/issue5880-anthropic-clamp-arm.test.ts b/tests/issue5880-anthropic-clamp-arm.test.ts index 1d0c121f07..d12ccb4aad 100644 --- a/tests/issue5880-anthropic-clamp-arm.test.ts +++ b/tests/issue5880-anthropic-clamp-arm.test.ts @@ -66,7 +66,7 @@ function makeSession(lastInputTokens: number, source: Session["stats"]["lastInpu blockContents: new Map(), inFlight: 0, persisted: false, - } as unknown as Session & Record; + } as unknown as Session; } test("#5880 unit: an active overflow-arm tag survives the post-rebuild anchor write", () => { From 9d18c77a554e50eddb2b77aca380ffbd8ec274cd Mon Sep 17 00:00:00 2001 From: ci-check Date: Fri, 13 Nov 2026 19:50:53 +0800 Subject: [PATCH 12/15] fix: keep overflow-armed tier exhaustion 502-retryable, not 413 (#6451) The #5764 fix marked structurally-exhausted tier recovery as deterministic (413 not-forwarded). But an overflow-armed baseline is a relay-billing artifact, not a structural doom: the upstream proved a smaller window, so the client may still recover by switching models or the upstream recovering. The preflight-image-budget upstream-overflow test (merged to master in 87082004b0) pins this: confirmed upstream overflow must stay 502-retryable, not 413. Refine the deterministic flag to exclude the overflow-arm case. --- src/preflight.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/preflight.ts b/src/preflight.ts index ee3e735d4a..70b9bcbab9 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -3233,7 +3233,11 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // blowing every span past TIER_DISTILL_SOURCE_CHAR_BUDGET) is deterministic → // 413 not-forwarded; cap/unavailable/no-shrink are relay/budget-dependent and MUST // stay 502-retryable (server.ts fail-fast contract). - failure = { kind: "exhausted", deterministic: rec.stopReason === "exhausted", detail: rec.honestDetail }; + // #6451: an overflow-armed baseline is a relay-billing artifact, not a structural + // doom — the upstream proved a smaller window, so the client may still recover by + // switching models or the upstream recovering; keep it 502-retryable. + const overflowArmed = deps.session.stats.lastInputTokensSource === "overflow-arm"; + failure = { kind: "exhausted", deterministic: rec.stopReason === "exhausted" && !overflowArmed, detail: rec.honestDetail }; } } if (currentTokens >= limit && !failure) { From 8ed56be997c292584844ab8b482c04f8fe77256f Mon Sep 17 00:00:00 2001 From: ci-check Date: Mon, 16 Nov 2026 20:42:39 +0800 Subject: [PATCH 13/15] docs(architecture): add src/responses-system-sections.ts to the module map (#6922) --- reference/architecture.md | 1 + 1 file changed, 1 insertion(+) diff --git a/reference/architecture.md b/reference/architecture.md index 484cd70bf1..9226ec7713 100644 --- a/reference/architecture.md +++ b/reference/architecture.md @@ -303,6 +303,7 @@ billion-context/ │ ├── remote-notices.ts # Remote announcement feed watcher: fail-open polling, last-known-good snapshot (#5548) │ ├── request-membership.ts # Settle-side membership probe: is this usage report ours (#4805) │ ├── resolver-wedge.ts # Persistent local DNS resolver wedge detector + alarm (#5221) +│ ├── responses-system-sections.ts # Single source of truth for the Responses-wire developer-section join: steady prepare + round-2 rebuild share one separator (#6922) │ ├── responses-tool-output.ts # Responses tool-output image parts: projection + patch helpers │ ├── rollout.ts # Feature rollout: per-install uuid, basis-point buckets, nested cohorts (#3062) │ ├── runaway-guard.ts # Intrinsic stream terminator for runaway tag/codepoint enumeration (#2346) From 245ecb1a9e5161ba97fa0b09865078025c9a2c8f Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 16 Nov 2026 22:51:46 +0800 Subject: [PATCH 14/15] fix(test): recalibrate #2929 probe anchor to nudge positions {3,5,11,14} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe anchor PROXY_TOOL_AT=6 was calibrated on 2026-10-31 for #4521 (schedule {4,6,8,9}). The nudge scheduling has shifted again — nudge now lands at {3,5,11,14}. Re-calibrate the anchor to the next nudge-carrying request pair (5 → tool round, 6 → fold round). The fold/proxy-tool-round mechanics under test are unchanged. Closes: master-red on test (windows-latest, 22, 3) and (ubuntu-latest, 22, 1). --- tests/issue2929-codex-fold-double-miss.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/issue2929-codex-fold-double-miss.test.ts b/tests/issue2929-codex-fold-double-miss.test.ts index 81b61ca727..83ee857801 100644 --- a/tests/issue2929-codex-fold-double-miss.test.ts +++ b/tests/issue2929-codex-fold-double-miss.test.ts @@ -38,11 +38,12 @@ const NUDGE_MARKER = "Compressible ranges ("; // and ~9KiB filler turns. Pre-#4521 (master) the injections landed on // requests {7,8,9}; since #4521 the armed-but-silent status line rides an // ephemeral carrier on the gated turns and its shown-marker feeds the nudge -// gate back, so the schedule shifts to {4,6,8,9} — the fold/proxy-tool-round -// mechanics under test are unchanged, only this probe's anchor re-calibrates -// to the next nudge-carrying request pair (6 → tool round, 7 → fold round). -const PROXY_TOOL_AT = 6; // upstream request index whose response = acp_status call (round 1) -const FOLD_AT = 7; // upstream request index whose response = compress call (round 2) +// gate back, so the schedule shifts — the fold/proxy-tool-round mechanics +// under test are unchanged, only this probe's anchor re-calibrates to the +// next nudge-carrying request pair (5 → tool round, 6 → fold round). +// Re-calibrated 2026-11-16: nudge now lands at {3,5,11,14} (was {4,6,8,9}). +const PROXY_TOOL_AT = 5; // upstream request index whose response = acp_status call (round 1) +const FOLD_AT = 6; // upstream request index whose response = compress call (round 2) // FOLD_AT+1 = post-fold refreshed re-request (round 3); FOLD_AT+2 = next steady client turn. const TOTAL_TURNS = 13; From 7247410976508f4dedf42afb89450c2041141c0b Mon Sep 17 00:00:00 2001 From: ework-agent Date: Tue, 17 Nov 2026 01:36:34 +0800 Subject: [PATCH 15/15] test(#2929): make fold probe schedule-independent + hermetic wire shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe's hardcoded request-index anchors (PROXY_TOOL_AT/FOLD_AT) broke every time the nudge gate's schedule shifted. The last recalibration (245ecb1a9) was measured on a direct local run (timestamps ON by default since #4313), but CI runs the suite preload pin BILI_MESSAGE_TIMESTAMPS="0" (timestamps OFF) — a different wire shape, a different schedule (no nudge at index 5) — so it passed locally and failed on every CI shard. Root-cause fix instead of another anchor whack-a-mole: - Arm the fold on the first request carrying the ephemeral nudge with >=11 tagged refs; fold on the re-request after the acp_status round. The probe now materializes the #2929 scenario wherever the schedule places it. - Pin compress.messageTimestamps:false so the probe's wire shape is hermetic regardless of how the suite is launched (matches the CI preload pin). Supersedes the 245ecb1a9 recalibration. Verified: passes with and without the preload pin, same deterministic schedule (nudge at inputs[7], fold at inputs[8]). --- .../issue2929-codex-fold-double-miss.test.ts | 67 +++++++++++-------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/tests/issue2929-codex-fold-double-miss.test.ts b/tests/issue2929-codex-fold-double-miss.test.ts index 83ee857801..7419629dc1 100644 --- a/tests/issue2929-codex-fold-double-miss.test.ts +++ b/tests/issue2929-codex-fold-double-miss.test.ts @@ -34,17 +34,17 @@ type Item = Record; const SUMMARY_MARKER = "[Compressed conversation section]"; const NUDGE_MARKER = "Compressible ranges ("; -// Indices verified against the kernel's nudge gate with nudgeGrowthTokens=2000 -// and ~9KiB filler turns. Pre-#4521 (master) the injections landed on -// requests {7,8,9}; since #4521 the armed-but-silent status line rides an -// ephemeral carrier on the gated turns and its shown-marker feeds the nudge -// gate back, so the schedule shifts — the fold/proxy-tool-round mechanics -// under test are unchanged, only this probe's anchor re-calibrates to the -// next nudge-carrying request pair (5 → tool round, 6 → fold round). -// Re-calibrated 2026-11-16: nudge now lands at {3,5,11,14} (was {4,6,8,9}). -const PROXY_TOOL_AT = 5; // upstream request index whose response = acp_status call (round 1) -const FOLD_AT = 6; // upstream request index whose response = compress call (round 2) -// FOLD_AT+1 = post-fold refreshed re-request (round 3); FOLD_AT+2 = next steady client turn. +// The fold turn is materialized SCHEDULE-INDEPENDENTLY: the fake upstream +// arms on the first request carrying the ephemeral nudge (with >=11 tagged +// refs to fold) and folds on the re-request that follows the acp_status +// round. A hardcoded request-index anchor broke every time the nudge gate's +// token schedule shifted — #4521's status-line feedback ({7,8,9} → {4,6,8,9}), +// #6928's status-line removal, and #4313's timestamp-rendering default flip +// (the suite preload pin BILI_MESSAGE_TIMESTAMPS="0" changes the wire shape +// and hence the schedule: OFF {4,13} vs ON {3,5,11,14} — direct local runs +// without the preload diverge from CI). The probe also pins +// compress.messageTimestamps:false so its wire shape is hermetic regardless +// of how the suite is launched. const TOTAL_TURNS = 13; const FILLER = (seed: number, kb: number): string => { @@ -121,6 +121,13 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl try { const captured: Array<{ url: string; body: string }> = []; + // Object (not bare lets) so the outer-scope reads aren't narrowed to + // the initial "idle" — the mutations happen inside the request callback. + const fold = { + state: "idle" as "idle" | "armed" | "folded", + armIdx: -1, // upstream request index that carried the nudge (acp_status round) + foldIdx: -1, // upstream request index that carried the compress call + }; upstream = http.createServer((req, res) => { const chunks: Buffer[] = []; req.on("data", (c: Buffer) => chunks.push(c)); @@ -128,15 +135,21 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl const idx = captured.length; const body = Buffer.concat(chunks).toString("utf8"); captured.push({ url: req.url ?? "", body }); - if (idx === PROXY_TOOL_AT) { - // Round 1 of the fold turn: model asks for proxy status — the - // loop executes it locally and re-requests (round 2). + const input = (JSON.parse(body) as { input?: Item[] }).input ?? []; + if (fold.state === "idle" && hasNudge(input) && refsOf(input).length >= 11) { + // Round 1 of the fold turn: the first nudge-carrying request + // with enough refs — model asks for proxy status, the loop + // executes it locally and re-requests (round 2). + fold.state = "armed"; + fold.armIdx = idx; functionCallSse(res, `resp_${idx}`, "acp_status", "{}", `call_st_${idx}`); return; } - if (idx === FOLD_AT) { - const prev = (JSON.parse(captured[idx - 1]!.body) as { input?: Item[] }).input ?? []; - const uniq = refsOf(prev); + if (fold.state === "armed") { + // Round 2: the re-request after the acp_status round — fold. + fold.state = "folded"; + fold.foldIdx = idx; + const uniq = refsOf(input); assert.ok(uniq.length >= 11, `fold round: expected >=11 tagged refs, saw ${uniq.length}`); const args = JSON.stringify({ content: [ { startId: uniq[3]!, endId: uniq[5]!, topic: "F1A", summary: `F1A summary: ${FILLER(1, 1)}` }, @@ -145,10 +158,9 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl functionCallSse(res, `resp_${idx}`, "compress", args, `call_cmp_${idx}`); return; } - const parsed = JSON.parse(body) as { input?: Item[] }; let label = "0"; - for (let i = (parsed.input ?? []).length - 1; i >= 0; i--) { - const it = parsed.input![i]!; + for (let i = input.length - 1; i >= 0; i--) { + const it = input[i]!; if (it.type === "message" && it.role === "user" && typeof it.content === "string") { const m = (it.content as string).match(/Turn (\d+):/); if (m) { label = m[1]!; break; } @@ -170,7 +182,7 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: 200_000 } } } }, modelContextLimit: 200_000, kernelConfig: defaultConfig(200_000), - compress: { injectTool: true, injectNudge: true, nudgeGrowthTokens: 2000 }, + compress: { injectTool: true, injectNudge: true, nudgeGrowthTokens: 2000, messageTimestamps: false }, promptCache: { routing: "auto" }, sessionHeader: "x-acp-session", log: true, @@ -224,17 +236,18 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl } const inputs = captured.map((c) => (JSON.parse(c.body) as { input?: Item[] }).input ?? []); - assert.ok(inputs.length >= FOLD_AT + 3, `expected >= ${FOLD_AT + 3} upstream requests, got ${inputs.length}`); + assert.ok(fold.state === "folded", `scenario miss: the nudge never fired on a request with >=11 refs — probe cannot materialize the #2929 trigger; adjust nudgeGrowthTokens/filler`); + assert.ok(inputs.length >= fold.foldIdx + 3, `expected >= ${fold.foldIdx + 3} upstream requests, got ${inputs.length}`); // (0) Scenario materialization: the nudge must have been injected into - // the PRE-FOLD original request (production shape: nudge armed -> model + // the arming (pre-fold) request (production shape: nudge armed -> model // calls proxy tool -> compress). Without it this probe does not cover // the #2929 trigger. - assert.ok(hasNudge(inputs[PROXY_TOOL_AT]!), `scenario miss: no ephemeral nudge in the pre-fold request (inputs[${PROXY_TOOL_AT}]) — probe does not cover the production trigger; adjust nudgeGrowthTokens/filler`); - console.log(`probe: nudge confirmed in pre-fold request; total upstream requests=${inputs.length}`); + assert.ok(hasNudge(inputs[fold.armIdx]!), `scenario miss: no ephemeral nudge in the arming request (inputs[${fold.armIdx}]) — probe does not cover the production trigger`); + console.log(`probe: nudge at inputs[${fold.armIdx}], fold at inputs[${fold.foldIdx}]; total upstream requests=${inputs.length}`); // (1) The post-fold re-request carries the in-place summary(s). - const rPost = inputs[FOLD_AT + 1]!; + const rPost = inputs[fold.foldIdx + 1]!; const postSums = summaryIndices(rPost); assert.ok(postSums.length >= 1, "post-fold re-request must carry the in-place summary"); @@ -242,7 +255,7 @@ test("#2929: nudge + proxy-tool-round fold keeps re-request/next-turn byte-stabl // the post-fold re-request up to and including the summary anchor. In // production this is where the chain breaks (next turn hits only the // instructions baseline). - const next = inputs[FOLD_AT + 2]!; + const next = inputs[fold.foldIdx + 2]!; const nextSums = summaryIndices(next); assert.ok(nextSums.length >= 1, `next steady turn lost the summary entirely (summaries: ${nextSums.length}) — fold did not survive to the client turn`); assert.equal(nextSums[0], postSums[0], `summary anchor drifted: re-request item[${postSums[0]}] vs next turn item[${nextSums[0]}]`);