diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index 2e205e9cd2..52407a1d7f 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -12,6 +12,7 @@ import { } from "acp-kernel"; import { log as loggerLog } from "./logger.js"; import { markDirty, reanchorNudgeOnUsageDrop, type Session } from "./session.js"; +import { normalizeUpstreamOrigin } from "./util.js"; // Render window for handleAcpCache's detail:"full" text view (#1489). The // ledger itself is unbounded — this only bounds how many lines the text @@ -628,9 +629,38 @@ export function recordCacheSample( * caller keeps its own settle-decision gate, log line, collapse watch and * outputTokens update; `reportedCached === null` means the provider reported * no cache tokens (recordCacheSample quarantines that sample). */ +// #1933 F1: estimator calibration constants. The local chars/4 estimate is a +// proxy whose ratio to real billing varies per upstream (observed 1.3–2.5× on +// one relay vs ~1.0× on another in the same session), so k̂ is learned per +// route and only applied on that route. Samples below MIN are noise (tiny +// requests), clamps bound a single pathological sample from wrecking the EMA +// and pin the correction to the deflate direction only (max 1, see below). +const CALIBRATION_MIN_ESTIMATE = 2000; +// Plausibility band for admitting a sample: outside it, the report and the +// payload it bills demonstrably don't correspond (placeholder billing, relay +// echo, mock upstreams) — a ratio there must never teach a factor. +const CALIBRATION_SAMPLE_MIN = 0.2; +const CALIBRATION_SAMPLE_MAX = 5; +// Final clamp on the published factor: bounds how far calibration can move +// any decision away from the raw estimate. One-way by design: the clamp max +// is 1, so a learned factor can only DEFLATE the estimate (fire later than +// the raw proxy would), never inflate it. Routes whose billing runs ABOVE +// the local estimate (samples >1) publish k̂=1 — legacy raw behavior — and +// stay covered by the overflow arm / learn-on-failure ladder instead. This +// eliminates the class "calibration itself causes an earlier trigger": the +// observed #1933 damage was over-triggering (37% window tax, fold churn), +// while the opposite error already has a backstop. Discussion: PR #1940. +const CALIBRATION_CLAMP_MIN = 0.25; +const CALIBRATION_CLAMP_MAX = 1; +// Evidence requirements: ≥2 recent same-route samples agreeing within ×2. +// One lucky/degenerate pair must not flip every estimate on the route. +const CALIBRATION_SAMPLE_WINDOW = 3; +const CALIBRATION_CONSISTENCY_RATIO = 2; + export function settleUsageReport( session: Session, s: { total: number; reportedCached: number | null; output?: number; protocol?: string; upstream?: string }, + localTextEstimate?: number, ): void { // #793: a zero-total sample carries no information (gateway placeholder or // relay echo) — it must not clobber the last trusted lastInputTokens. @@ -646,12 +676,48 @@ export function settleUsageReport( // compaction boundaries via resetSessionCompression (session.ts). session.stats.lastUsageGradeTokens = session.stats.lastInputTokens; session.stats.lastInputTokensSource = "usage"; + // #1933 F2: record which route measured this baseline — the gate uses + // it to demote the baseline when the current request routes elsewhere. + const settleOrigin = normalizeUpstreamOrigin(s.upstream); + if (settleOrigin !== undefined) session.stats.lastInputTokensOrigin = settleOrigin; + // #1933 F1: consume the pending pair — the local text estimate of the + // payload THIS report bills, recorded at prepare time. Same route + // required. Admitted samples (plausibility band) accumulate in a + // per-origin ring; the factor is published only once ≥2 recent samples + // agree within ×2, and cleared again when they stop agreeing — until + // then the raw estimate decides (legacy behavior). + const pendingEst = session.stats.lastLocalTextEstimate; + const pendingOrigin = normalizeUpstreamOrigin(session.stats.lastLocalTextEstimateOrigin); + if (pendingEst !== undefined && pendingEst >= CALIBRATION_MIN_ESTIMATE && settleOrigin !== undefined && pendingOrigin === settleOrigin) { + const rawSample = s.total / pendingEst; + let ring = session.stats.calibrationRing; + if (!ring || ring.origin !== settleOrigin) ring = { origin: settleOrigin, values: [] }; + if (rawSample >= CALIBRATION_SAMPLE_MIN && rawSample <= CALIBRATION_SAMPLE_MAX) { + ring.values.push(rawSample); + if (ring.values.length > CALIBRATION_SAMPLE_WINDOW) ring.values.shift(); + const spread = Math.max(...ring.values) / Math.min(...ring.values); + if (ring.values.length >= 2 && spread <= CALIBRATION_CONSISTENCY_RATIO) { + const mean = ring.values.reduce((a, b) => a + b, 0) / ring.values.length; + session.stats.calibratedEstimate = Math.min(CALIBRATION_CLAMP_MAX, Math.max(CALIBRATION_CLAMP_MIN, mean)); + session.stats.calibratedEstimateOrigin = settleOrigin; + } else { + delete session.stats.calibratedEstimate; + delete session.stats.calibratedEstimateOrigin; + } + } + session.stats.calibrationRing = ring; + } // #1110: a real usage report retires the one-shot overflow arm. delete session.stats.overflowArmTokens; // #1595: a real report landing far below a stale-high nudge reference // retires that reference too (one call covers all three lanes). reanchorNudgeOnUsageDrop(session); } + // #1933 F1: roll the pending pair forward to the current turn's value — + // it will be consumed by the NEXT report (or cleared when the lane + // provides none, so a stale pair can never be mispaired). + session.stats.lastLocalTextEstimate = localTextEstimate; + session.stats.lastLocalTextEstimateOrigin = localTextEstimate !== undefined ? normalizeUpstreamOrigin(s.upstream) : undefined; if (s.reportedCached !== null && s.total > 0) { session.stats.cachedTokens += s.reportedCached; session.stats.cacheSamples += 1; diff --git a/src/preflight.ts b/src/preflight.ts index dc77b952d2..74c501e80c 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -17,6 +17,7 @@ import { proxyDispatcher } from "./upstream-proxy.js"; import { lastCompressSuffix, type Session } from "./session.js"; import { peekRegistryOutputLimit } from "./registry.js"; import { safePrefix } from "./text-safe.js"; +import { applyEstimateCalibration } from "./util.js"; // #247: proactive pre-forward compression. When the session's real context // (previous turn's upstream input_tokens) exceeds the current model's window @@ -51,7 +52,7 @@ export const MAX_SUMMARY_CALLS_PER_PREFLIGHT = 16; // retries the span's only draw against a flaky summarizer kills the whole // preflight (one content_filter blip bricked an entire turn; the identical // payload summarized fine ~90s later on the user's manual retry). Retries -// count against MAX_SUMMARY_CALLS_PER_PREFLIGHT like any other call. +// count against the per-invocation summary budget like any other call. const TRANSIENT_EMPTY_SUMMARY_RETRIES = 2; // Per-protection-regime cap on wasted transient retries so a SYSTEMIC // (persistent) empty-summary failure degrades to today's behavior after a @@ -68,9 +69,10 @@ const TRANSIENT_EMPTY_RETRY_BUDGET = 4; // halving worklist can spend several calls on one range without completing a // fold. Beyond the bound the loop still exits cleanly — the fail-fast reports // the post-fold size and the remaining compressible-range count, so an -// operator sees exactly how far the budget ran out. 16 is tuned to the -// incident class behind #868 (a 1.39x-window payload); it is a fixed depth, -// not scaled to the overshoot — scaling it is a separate design question. +// operator sees exactly how far the budget ran out. #1933 made the depth +// dynamic: base 16 covers payloads up to ~1.4x the window (one ideal fold +// removes CHUNK_FRACTION x window); larger entry overshoots scale both +// budgets proportionally, capped at 2x the base (see preflightCompress). export type PreflightProtocol = "anthropic" | "openai" | "responses" | "google"; @@ -101,6 +103,8 @@ export interface PreflightDeps { * optimistic chars/4 estimator, which undercounts code/JSON replays by up * to ~4x and would let an over-window payload slip through uncompressed. */ unknownBaseline?: boolean; + /** #1933 F1: origin of the upstream this request routes to. When it matches the route that learned the session's estimate-calibration factor k̂, every per-round text estimate is scaled by k̂ so gate, per-round exit and final fit judge one payload on the same scale as the trigger that started this invocation; absent or mismatched → raw estimates (legacy behavior). */ + upstreamOrigin?: string; } export type PreflightFailureKind = "upstream" | "exhausted" | "aborted"; @@ -842,8 +846,13 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // kernel's own truncation/absorb behavior stays byte-identical. const imageReserve = deps.imageReserve ?? 0; const wireOverhead = deps.wireOverhead ?? 0; + // #1933 F1: capture k̂ once — it only mutates on usage settlement (outside + // this invocation), while every fit judgment in this loop must stay on + // one consistent scale with the gate that started it. + const kFactor = deps.session.stats.calibratedEstimate; + const kOrigin = deps.session.stats.calibratedEstimateOrigin; let textTarget = Math.max(0, Math.min(limit, deps.compressionTarget ?? limit) - imageReserve); - const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + imageReserve + wireOverhead, rangesRemaining: 0, fitsWindow: true }; + const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: applyEstimateCalibration(estimateCoreMessages(messages) + wireOverhead, kFactor, kOrigin, deps.upstreamOrigin) + imageReserve, rangesRemaining: 0, fitsWindow: true }; if (limit <= 0) return result; const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION)); // applyCompression rejects ranges below config.compress.minCompressRange @@ -890,7 +899,19 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa let transientRetryBudget = TRANSIENT_EMPTY_RETRY_BUDGET; let rangesTried = 0; let rangesRemaining = 0; - for (let round = 0; round < MAX_PREFLIGHT_ROUNDS; round++) { + // #1933 F3: scale the depth budgets with the entry overshoot (coverage-bound + // note atop the file). A raised budget is only a ceiling — well-behaved + // payloads exit early exactly as before; only genuinely huge payloads burn + // toward it before the fail-fast reports how far it ran out. + const entryLocal = (baselineKnown ? estimateCoreMessages(messages) : estimateCoreMessagesUpper(messages)) + imageReserve + wireOverhead; + const entryTokens = Math.max(baselineKnown ? deps.session.stats.lastInputTokens : 0, entryLocal); + const overshootRatio = limit > 0 && entryTokens > 0 ? entryTokens / limit : 1; + const summaryBudget = Math.min(MAX_SUMMARY_CALLS_PER_PREFLIGHT * 2, Math.max(MAX_SUMMARY_CALLS_PER_PREFLIGHT, Math.ceil(MAX_SUMMARY_CALLS_PER_PREFLIGHT * overshootRatio))); + const roundBudget = Math.min(MAX_PREFLIGHT_ROUNDS * 2, Math.max(MAX_PREFLIGHT_ROUNDS, Math.ceil(MAX_PREFLIGHT_ROUNDS * overshootRatio))); + if (summaryBudget > MAX_SUMMARY_CALLS_PER_PREFLIGHT) { + deps.log("warn", `[preflight] payload ~${entryTokens} tok vs window ${limit} (~${overshootRatio.toFixed(1)}x) — raising summarization budget ${MAX_SUMMARY_CALLS_PER_PREFLIGHT} -> ${summaryBudget}, rounds ${MAX_PREFLIGHT_ROUNDS} -> ${roundBudget}`); + } + for (let round = 0; round < roundBudget; round++) { if (deps.signal?.aborted) { failure = ABORTED_FAILURE; break; @@ -927,7 +948,11 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa const baselineFloor = messages.length > 0 ? ((deps.session.stats.lastInputTokensSource === "usage" || deps.session.stats.lastInputTokensSource === "overflow-arm") ? deps.session.stats.lastInputTokens : 0) : deps.session.stats.lastInputTokens; - const roundText = estimateCoreMessages(turn.messages) + wireOverhead; + const rawRoundText = estimateCoreMessages(turn.messages) + wireOverhead; + // #1933 F1: same calibrated caliber as the gate's trigger/fit checks — + // per-round exit and final fit must judge this payload identically to + // the trigger that started the invocation (no mid-loop scale drift). + const roundText = applyEstimateCalibration(rawRoundText, kFactor, kOrigin, deps.upstreamOrigin); currentTokens = Math.max(baselineFloor, roundText + imageReserve); // #1843: the TEXT-channel judgment quantity — the usage-grade baseline // bills images too, so project it onto the text channel by subtracting @@ -1116,7 +1141,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa const parts: string[] = []; const chunks = splitSummaryContent(content, budget, countText); for (const chunk of chunks) { - if (summaryCalls >= MAX_SUMMARY_CALLS_PER_PREFLIGHT) { + if (summaryCalls >= summaryBudget) { budgetHit = true; break; } @@ -1127,7 +1152,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa "unusable" in part && part.transient && transientTries < TRANSIENT_EMPTY_SUMMARY_RETRIES && transientRetryBudget > 0 && - summaryCalls < MAX_SUMMARY_CALLS_PER_PREFLIGHT && + summaryCalls < summaryBudget && !deps.signal?.aborted ) { transientTries += 1; @@ -1247,7 +1272,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa const unusableNote = lastUnusableDetail ? ` Last unusable summary: ${safePrefix(lastUnusableDetail, 300)}.` : ""; const skipNote = skipReasons.length > 0 ? ` Skipped: ${skipReasons.slice(0, 3).join(" | ")}.` : ""; if (budgetHit) { - failure = { kind: "exhausted", detail: `the preflight summarization budget (${MAX_SUMMARY_CALLS_PER_PREFLIGHT} calls per protection regime) was exhausted before the payload fit the window${unusableNote}${skipNote}` }; + failure = { kind: "exhausted", detail: `the preflight summarization budget (${summaryBudget} calls per protection regime) was exhausted before the payload fit the window${unusableNote}${skipNote}` }; } else if (relaxed && result.compressedRanges > 0) { failure = { kind: "exhausted", detail: `${relaxedExhaustedDetail}${skipNote}` }; } else if (result.compressedRanges === 0) { @@ -1258,7 +1283,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa ? `no viable range could be compressed: ${cause}${unusableNote}` : `no range could be compressed across ${rangesTried} viable range${rangesTried === 1 ? "" : "s"}: ${cause}${unusableNote}` }; } else { - failure = { kind: "exhausted", detail: `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds${unusableNote}${skipNote}` }; + failure = { kind: "exhausted", detail: `the compress budget was exhausted after ${roundBudget} rounds${unusableNote}${skipNote}` }; } } if (result.compressedRanges > 0) { diff --git a/src/server.ts b/src/server.ts index ec2d76ae1f..4a49c76a44 100644 --- a/src/server.ts +++ b/src/server.ts @@ -116,7 +116,7 @@ import { consumePluginRegisterFor, flushConversations, handlePluginCompact, hand import { setupMitm, readMitmUpstream, getBlindTunnelStats, liveBlindTunnels } from "./mitm.js"; import { evaluateChain, extractChainCarriers, stampOutbound, stripEmbeddedChainCarriers } from "./chain-checkpoint.js"; import type { BiliMessage } from "acp-kernel/wire"; -import { appendSystemText, BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, hardenOpenaiAssistantContent, isLoopbackAddress, inspectContextOverflow, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, systemToUser, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "./util.js"; +import { appendSystemText, applyEstimateCalibration, BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, hardenOpenaiAssistantContent, isLoopbackAddress, inspectContextOverflow, normalizeUpstreamOrigin, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, systemToUser, usageOutputTotal, usageTotals, type ContextOverflowInfo, type WireProtocol } from "./util.js"; import { safePrefix, safeSuffix } from "./text-safe.js"; import { BILI_TUNNEL_HEADER, checkTunnelDestination, classifyIp, localMachineIps, normalizeIpLiteral, parseIpLiteral, tunnelAllowlistFromEnv } from "./tunnel-guard.js"; @@ -3758,6 +3758,13 @@ async function prepareAnthropic( session.stats.localInputEstimate = estimateCoreMessagesUpper(processedMessages.length > 0 ? processedMessages : originalMessages) + countSystemAndToolsTokens(extractSystem(systemOut), toolsOut) + imageReserveFor(session, "anthropic", rebuilt, opts, upstreamOrigin); + // #1933 F1: billed-caliber twin of the row above (chars/4 instead of + // char-count upper bound) — settleUsageReport pairs it with this turn's + // usage report to learn the per-route estimate-calibration factor k̂. + session.stats.lastLocalTextEstimate = estimateCoreMessages(processedMessages.length > 0 ? processedMessages : originalMessages) + + countSystemAndToolsTokens(extractSystem(systemOut), toolsOut) + + imageReserveFor(session, "anthropic", rebuilt, opts, upstreamOrigin); + if (upstreamOrigin) session.stats.lastLocalTextEstimateOrigin = upstreamOrigin; return { body: JSON.stringify(rebuilt), session, attachedRetrievals, attachedRetrievalNoteIds, processedMessages, originalMessages, anthropicSystem: parsed.system, anthropicCacheMarks, systemNotes: sysNotes, protocol: "anthropic", stream, compressInjected: injectTools, pluginMode, nudge, prompts, surface, renderTags: process.env.ACP_RENDER_NONE ? "none" : "text-only", dropReasoning: stripReasoning } as Prepared; } @@ -4013,6 +4020,13 @@ async function prepareOpenai( session.stats.localInputEstimate = estimateCoreMessagesUpper(processedMessages.length > 0 ? processedMessages : originalMessages) + countSystemAndToolsTokens(openaiOutboundSystem || openaiSystemText, toolsOut) + imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin); + // #1933 F1: billed-caliber twin (chars/4) for the k̂ learning pair — + // see the anthropic-lane counterpart above. + session.stats.lastLocalTextEstimate = estimateCoreMessages(processedMessages.length > 0 ? processedMessages : originalMessages) + + countSystemAndToolsTokens(openaiOutboundSystem || openaiSystemText, toolsOut) + + imageReserveFor(session, "openai", rebuilt, opts, billingUpstream ?? upstreamOrigin); + const openaiPairOrigin = billingUpstream ?? upstreamOrigin; + if (openaiPairOrigin) session.stats.lastLocalTextEstimateOrigin = openaiPairOrigin; } snapshotMessages(session, originalMessages); markDirty(session); @@ -4631,6 +4645,13 @@ async function prepareResponses( session.stats.localInputEstimate = estimateCoreMessagesUpper(processedMessages.length > 0 ? processedMessages : originalMessages) + countSystemAndToolsTokens(responsesDevContent ?? "", toolsOut) + imageReserveFor(session, "responses", rebuilt, opts, billingUpstream ?? upstreamOrigin); + // #1933 F1: billed-caliber twin (chars/4) for the k̂ learning pair — + // see the anthropic-lane counterpart above. + session.stats.lastLocalTextEstimate = estimateCoreMessages(processedMessages.length > 0 ? processedMessages : originalMessages) + + countSystemAndToolsTokens(responsesDevContent ?? "", toolsOut) + + imageReserveFor(session, "responses", rebuilt, opts, billingUpstream ?? upstreamOrigin); + const responsesPairOrigin = billingUpstream ?? upstreamOrigin; + if (responsesPairOrigin) session.stats.lastLocalTextEstimateOrigin = responsesPairOrigin; } snapshotMessages(session, originalMessages); markDirty(session); @@ -5331,7 +5352,7 @@ async function preflightCompressIfNeeded( // single source of truth for that size (#1493) — armFailureShrink measures // the same quantity so a no-usage failure can't arm lastInputTokens to raw- // history scale and fire preflight on a payload that actually fits. - const { textEstimate, overheadEstimate, imageTokens, payloadEstimate } = outboundPayloadBreakdown(prepared, opts, route, req.url ?? ""); + const { textEstimate, overheadEstimate, imageTokens } = outboundPayloadBreakdown(prepared, opts, route, req.url ?? ""); // #553: anonymous requests resolve their session by prefix affinity. After // an ACP compression breaks the chain hash, the client's replay mints a NEW // session id (a fork) whose lastInputTokens is 0 — yet it carries the full @@ -5356,12 +5377,36 @@ async function preflightCompressIfNeeded( // (#604) on a measured (folded) payload describes a different view and // must not pull preflight into multi-minute runs over a payload whose own // post-fold estimate fits the window. - const baselineFloor = prepared.processedMessages.length > 0 + const baselineFloorRaw = prepared.processedMessages.length > 0 ? ((session.stats.lastInputTokensSource === "usage" || session.stats.lastInputTokensSource === "overflow-arm") ? session.stats.lastInputTokens : 0) : session.stats.lastInputTokens; + // #1933 F2: a usage baseline is only authoritative for the route that + // measured it — provider billing scales differ per upstream (the incident: + // ~257K local estimate vs 59-63% real usage on one route; after a mid- + // session model switch the stale cross-route baseline kept arming preflight + // on payloads the new upstream billed far below the window). Demote to + // untrusted when the request now routes elsewhere; the payload's own + // (calibrated) estimate then judges it. Unprovenanced baselines (sessions + // started before this field existed) keep the legacy behavior. + let baselineFloor = baselineFloorRaw; + const currentOrigin = normalizeUpstreamOrigin(route?.upstream); + const baselineOrigin = normalizeUpstreamOrigin(session.stats.lastInputTokensOrigin); + if (baselineFloor > 0 && currentOrigin !== undefined && baselineOrigin !== undefined && baselineOrigin !== currentOrigin) { + log("info", `[${session.id}] preflight usage-baseline ~${baselineFloor} tok was measured on ${baselineOrigin}, request now routes to ${currentOrigin} — demoting to untrusted, judging by this payload's own estimate (#1933)`); + baselineFloor = 0; + } + // #1933 F1: scale the local text estimate by the per-route calibration + // factor k̂ learned from this session's own usage reports (local estimate ÷ + // what upstream actually billed, EMA, clamped 0.25–1 — one-way, deflate + // only; see settleUsageReport). Unknown/mismatched origin → raw estimate, + // i.e. today's behavior. + const kFactor = session.stats.calibratedEstimate; + const kOrigin = session.stats.calibratedEstimateOrigin; + const calibratedText = applyEstimateCalibration(textEstimate + overheadEstimate, kFactor, kOrigin, currentOrigin); + const calibratedPayload = calibratedText + imageTokens; const tokenCount = unknownBaseline ? estimateCoreMessagesUpper(prepared.processedMessages) + overheadEstimate + imageTokens - : Math.max(baselineFloor, payloadEstimate); + : Math.max(baselineFloor, calibratedPayload); // #1843 dual-channel accounting: the trigger runs on the TEXT channel — // text vs `target − imageReserve`. Exact algebraic rewrite of the old // total-view trigger: subtracting the constant reserve from both sides of @@ -5374,12 +5419,12 @@ async function preflightCompressIfNeeded( // fits, nor keep it armed after the text has been folded down. const textChannel = unknownBaseline ? estimateCoreMessagesUpper(prepared.processedMessages) + overheadEstimate - : textEstimate + overheadEstimate; + : calibratedText; const textBudget = Math.max(0, compressionTarget - imageTokens); const decisionTrigger = Math.max(Math.max(0, baselineFloor - imageTokens), textChannel); const triggerFires = imageTokens >= compressionTarget || decisionTrigger >= textBudget; if (limit <= 0 || !model || !triggerFires) return prepared; - const payloadFitsWindow = (unknownBaseline ? tokenCount : payloadEstimate) < limit; + const payloadFitsWindow = (unknownBaseline ? tokenCount : calibratedPayload) < limit; // #496 forward-once-then-learn: the default image cost (base64/4) matches byte // relays (#488) but overestimates pixel-tile upstreams (a 400KB JPEG ≈ 1.6K real // tokens, not ~133K), so an image-dominated payload can clear the window on ESTIMATE @@ -5460,7 +5505,7 @@ async function preflightCompressIfNeeded( // Headroom or a stale baseline can trigger preflight on a fitting payload. // Anonymous sessions need the conservative upper bound to prove that fit. if (payloadFitsWindow) { - log("info", `[${session.id}] preflight target reached (~${tokenCount}) but the payload fits with no compressible ranges (~${payloadEstimate}/${limit}); forwarding as-is`); + log("info", `[${session.id}] preflight target reached (~${tokenCount}) but the payload fits with no compressible ranges (~${Math.round(calibratedPayload)}/${limit}${kFactor !== undefined ? `, k̂=${kFactor.toFixed(2)}` : ""}); forwarding as-is`); return prepared; } if (unknownBaseline) { @@ -5502,7 +5547,11 @@ async function preflightCompressIfNeeded( // nothing is foldable (no summarization call is spent in that case). The // old pre-check failed fast here on the normal-config compressibleRanges, // which excluded the soft zone — bricking the #330 livelock. - log("warn", `[${session.id}] context ${tokenCount} tokens reached preflight target ${compressionTarget} (model window ${limit}, model=${model}); preflight compressing before forward`); + // #1933 F4: the trigger line now carries both measurement scales — the + // provider-billed baseline and the (calibrated) local estimate — so a + // false trigger is diagnosable from the log alone instead of requiring a + // cross-reference between gate and nudge lines. + log("warn", `[${session.id}] context ${tokenCount} tokens reached preflight target ${compressionTarget} (model window ${limit}, model=${model}; usage-baseline=${baselineFloor > 0 ? baselineFloor : "none"} local-est=${Math.round(calibratedText)}${kFactor !== undefined ? ` raw=${Math.round(textEstimate + overheadEstimate)} k̂=${kFactor.toFixed(2)}` : ""}); preflight compressing before forward`); // #300: stamp the chain marker so a downstream bili skips these // summarization calls too (preflight always processes). const { upstreamUrl, headers, proxyUrl } = buildForwardTarget(req, opts, route, affinity, instanceId); @@ -5540,6 +5589,7 @@ async function preflightCompressIfNeeded( imageReserve: imageTokens, wireOverhead: overheadEstimate, unknownBaseline, + upstreamOrigin: currentOrigin, }, prepared.originalMessages, ); @@ -5566,9 +5616,11 @@ async function preflightCompressIfNeeded( // to this single client request. session.stats.requests -= 1; outbound = rebuilt; + // Same calibrated caliber as the trigger above — gate, per-round exit + // and this final fit must judge the payload on one scale (#1933 F1). const fits = unknownBaseline ? result.fitsWindow - : estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate + imageTokens < limit; + : applyEstimateCalibration(estimateCoreMessages(rebuilt.processedMessages) + overheadEstimate, kFactor, kOrigin, currentOrigin) + imageTokens < limit; if (fits) return rebuilt; // #1839: the two measurements disagree — preflight's own final view // (post-fold content + images + wire overhead) fits, but the fresh diff --git a/src/session.ts b/src/session.ts index 98602df2f2..7c97f0ea9d 100644 --- a/src/session.ts +++ b/src/session.ts @@ -168,6 +168,54 @@ export type Session = { * other baseline stats at native-compaction boundaries. Absent on * legacy session files → legacy path. */ lastUsageGradeTokens?: number; + /** #1933 F2: upstream origin (scheme://host[:port]) that measured + * lastInputTokens — written by settleUsageReport alongside "usage". + * The gate demotes a usage-grade baseline to untrusted when the + * CURRENT request routes elsewhere: a billing scale learned on one + * provider says nothing about whether another accepts the payload. + * Absent on legacy session files → baseline keeps legacy behavior. */ + lastInputTokensOrigin?: string; + /** #1933 F1: calibrated scale factor k̂ = mean of up to 3 recent + * consistent same-route samples of (upstream-billed input ÷ local + * text estimate), clamped to [0.25, 1] — one-way by design: the + * correction can only deflate the estimate (never fire earlier + * than the raw proxy); routes billing above the local estimate + * publish k̂=1 (legacy behavior, overflow arm covers them). A + * sample is only admitted + * when it falls in the plausibility band [0.2, 5] — outside it the + * report and the payload clearly don't correspond (placeholder + * billing, relay echo, mock upstreams) and must not teach anything. + * k̂ is published only once ≥2 admitted samples agree within ×2; + * disagreement clears it again (fall back to the conservative raw + * estimate). The chars/4 estimator is a proxy, not a measurement — + * its ratio to real billing varies per upstream (observed 1.3–2.5× + * on one relay, ~1.0× on another), and max()ing it against the + * usage baseline let the systematically-high proxy fire preflight + * while the provider billed 59–63% of the window. Applied ONLY + * where the raw estimate currently decides (trigger, zero-range + * fast-forward fit, folder exit, post-fold fit); absent or + * route-mismatched → raw estimate (today's behavior). */ + calibratedEstimate?: number; + /** #1933 F1: route the calibratedEstimate was learned on. A sample + * from a different origin starts a fresh ring instead of blending + * two providers' scales. */ + calibratedEstimateOrigin?: string; + /** #1933 F1: evidence ring behind calibratedEstimate — the recent + * admitted raw samples for ONE origin (max 3). Persisted so a + * restart doesn't re-arm the warmup delay; reset at native- + * compaction boundaries with the rest of the baseline stats. */ + calibrationRing?: { origin: string; values: number[] }; + /** #1933 F1: pending pairing input — local estimate of the LAST + * prepared outbound in BILLED caliber (estimateCoreMessages + + * system/tools overhead + image reserve, defaultCountTokens rate), + * recorded in prepare*. settleUsageReport pairs it with the NEXT + * usage report's billed total (same request) to sample k̂, then + * overwrites it with the current turn's value. In-memory only — a + * restart simply loses one pending pair. */ + lastLocalTextEstimate?: number; + /** #1933 F1: route of the pending lastLocalTextEstimate; the pair is + * only consumed when the settling report came from the same route. */ + lastLocalTextEstimateOrigin?: string; /** #1097 content store: total acp_retrieve calls issued this session. */ retrieveCalls: number; /** #1097: acp_retrieve calls that resolved to stored content. */ @@ -657,6 +705,15 @@ export function resetSessionCompression(session: Session): void { // #1569: pre-compaction billing evidence describes a payload lineage that // no longer exists — fall back to legacy sizing until a fresh report lands. delete session.stats.lastUsageGradeTokens; + // #1933: same rationale for the calibration pair and its provenance — k̂ was + // learned against the pre-compaction content class, and the baseline's + // measuring route no longer bounds this rebuilt session. + delete session.stats.lastInputTokensOrigin; + delete session.stats.calibratedEstimate; + delete session.stats.calibratedEstimateOrigin; + delete session.stats.calibrationRing; + delete session.stats.lastLocalTextEstimate; + delete session.stats.lastLocalTextEstimateOrigin; session.stats.contextTokens = 0; delete session.stats.contextTokensSource; session.metadata.nativeCompactionAt = Date.now(); diff --git a/src/util.ts b/src/util.ts index 841569e2ba..90809f8fa8 100644 --- a/src/util.ts +++ b/src/util.ts @@ -57,6 +57,37 @@ export function isLoopbackAddress(addr: string | undefined): boolean { return !!addr && (addr.startsWith("127.") || addr === "::1" || addr.startsWith("::ffff:127.")); } +/** #1933: reduce an upstream URL/origin to a comparable route key + * (scheme://host[:port]). Lanes report different shapes — `new URL().origin` + * from the forward path, full URLs with paths from resolveUpstream — and the + * baseline-provenance / calibration checks compare keys, so both must land on + * the same form. Unparseable input falls back to its trimmed raw value. */ +export function normalizeUpstreamOrigin(u: string | undefined): string | undefined { + if (!u) return undefined; + try { + return new URL(u).origin; + } catch { + const t = u.trim(); + return t || undefined; + } +} + +/** #1933 F1: apply the session's learned estimator scale k̂ to a raw local + * text estimate. The chars/4 estimator is a proxy whose ratio to real billing + * varies per upstream, so it may only decide on the route where k̂ was + * learned: both origins known and equal → scaled; either unknown or the + * routes differ → raw estimate unchanged (legacy behavior). */ +export function applyEstimateCalibration(raw: number, k: number | undefined, kOrigin: string | undefined, origin: string | undefined): number { + // Invalid factors (null/0/NaN/±Infinity — e.g. a corrupted persisted + // session.stats field) must degrade to no-correction, never zero or + // poison the reading: raw×0 would blind the estimate arm entirely. + if (k === undefined || !Number.isFinite(k) || k <= 0 || raw <= 0) return raw; + const a = normalizeUpstreamOrigin(origin); + const b = normalizeUpstreamOrigin(kOrigin); + if (a === undefined || b === undefined || a !== b) return raw; + return raw * k; +} + export type WireProtocol = "anthropic" | "openai" | "responses" | "google"; /** diff --git a/tests/issue1933-preflight-calibration.test.ts b/tests/issue1933-preflight-calibration.test.ts new file mode 100644 index 0000000000..8c04f537fa --- /dev/null +++ b/tests/issue1933-preflight-calibration.test.ts @@ -0,0 +1,376 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { settleUsageReport } from "../src/cache-ledger.ts"; +import { applyEstimateCalibration, normalizeUpstreamOrigin } from "../src/util.ts"; +import { resetSessionCompression, getSession, listSessions, type Session } from "../src/session.ts"; + +// #1933: the preflight trigger judged max(local estimate, provider usage +// baseline), but the local chars/4 estimator's ratio to real billing varies +// per upstream (observed 1.3–2.5x on one relay vs ~1.0x on another in the +// SAME session), so on over-billing relays the raw estimate alone crossed the +// target while the provider measured 59–63% of the window — preflight fired +// early every turn, and after a mid-session provider switch the stale +// cross-provider baseline fired it even on small payloads. Unit half pins the +// calibration math (k̂ learning in settleUsageReport, apply-on-same-route-only); +// the e2e half drives the real proxy against billing-factor mocks and pins the +// two sharp regressions: no false trigger when the calibrated estimate fits, +// and no stale cross-origin baseline after a route switch. + +let seq = 0; +function makeSession(): Session { + seq += 1; + return { + id: `i1933-${seq}`, + metadata: {}, + stats: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, tokenSnapshot: {} }, + } as unknown as Session; +} + +test("normalizeUpstreamOrigin: URL forms collapse to origin, garbage degrades safely", () => { + assert.equal(normalizeUpstreamOrigin("http://a:8080/v1/chat/completions?x=1"), "http://a:8080"); + assert.equal(normalizeUpstreamOrigin("https://a:8443"), "https://a:8443"); + assert.equal(normalizeUpstreamOrigin(" http://a "), "http://a"); + assert.equal(normalizeUpstreamOrigin(""), undefined); + assert.equal(normalizeUpstreamOrigin(undefined), undefined); + assert.equal(normalizeUpstreamOrigin("not a url"), "not a url"); +}); + +test("applyEstimateCalibration: scales only on the learned route, passes raw otherwise", () => { + assert.equal(applyEstimateCalibration(10000, 0.5, "http://a", "http://a/v1"), 5000); + assert.equal(applyEstimateCalibration(10000, undefined, "http://a", "http://a"), 10000); + assert.equal(applyEstimateCalibration(-5, 0.5, "http://a", "http://a"), -5); + assert.equal(applyEstimateCalibration(0, 0.5, "http://a", "http://a"), 0); + assert.equal(applyEstimateCalibration(10000, 0.5, undefined, "http://a"), 10000); + assert.equal(applyEstimateCalibration(10000, 0.5, "http://a", undefined), 10000); + assert.equal(applyEstimateCalibration(10000, 0.5, "http://a", "http://b"), 10000); + // Invalid factors (persisted garbage, null from JSON, 0, NaN, Infinity) + // degrade to no-correction — raw×0 would blind the estimate arm. + assert.equal(applyEstimateCalibration(10000, 0, "http://a", "http://a"), 10000); + assert.equal(applyEstimateCalibration(10000, null as unknown as number, "http://a", "http://a"), 10000); + assert.equal(applyEstimateCalibration(10000, Number.NaN, "http://a", "http://a"), 10000); + assert.equal(applyEstimateCalibration(10000, Number.POSITIVE_INFINITY, "http://a", "http://a"), 10000); + assert.equal(applyEstimateCalibration(10000, -0.5, "http://a", "http://a"), 10000); +}); + +// One settled pair (pending estimate + its usage report) on a route. +function pair(s: Session, est: number, origin: string, billed: number): void { + s.stats.lastLocalTextEstimate = est; + s.stats.lastLocalTextEstimateOrigin = origin; + settleUsageReport(s, { total: billed, reportedCached: null, upstream: origin }); +} + +test("settleUsageReport: k̂ publishes only after two consistent same-route samples", () => { + const s = makeSession(); + const st = s.stats; + + // First sample alone proves nothing — ring holds it, nothing published. + pair(s, 10000, "http://a", 5000); + assert.equal(st.calibratedEstimate, undefined, "one sample must not publish a factor"); + assert.deepEqual(st.calibrationRing?.values, [0.5]); + assert.equal(st.lastInputTokensOrigin, "http://a"); + assert.equal(st.lastLocalTextEstimate, undefined, "the pending pair is consumed, not reused"); + + // Second agreeing sample (11000/20000 = 0.55): publish the clamped mean. + pair(s, 20000, "http://a", 11000); + assert.ok(Math.abs((st.calibratedEstimate ?? 0) - 0.525) < 1e-9, `expected mean 0.525, got ${st.calibratedEstimate}`); + assert.equal(st.calibratedEstimateOrigin, "http://a"); +}); + +test("settleUsageReport: cross-route starts a fresh ring and clears the old factor", () => { + const s = makeSession(); + const st = s.stats; + pair(s, 10000, "http://a", 5000); + pair(s, 20000, "http://a", 11000); + assert.equal(st.calibratedEstimate, 0.525); + + // A first sample from another route must NOT blend two providers' scales + // and must retire the stale factor until its own evidence agrees. + pair(s, 10000, "http://b", 20000); + assert.equal(st.calibratedEstimate, undefined, "cross-route single sample clears the old k̂"); + assert.equal(st.calibratedEstimateOrigin, undefined); + assert.deepEqual(st.calibrationRing?.values, [2]); + assert.equal(st.lastInputTokensOrigin, "http://b"); + + pair(s, 10000, "http://b", 21000); + // 2.0/2.1 are consistent under-estimate evidence (billing ABOVE the local + // estimate), but calibration is one-way: the mean clamps to 1 = raw + // estimate, legacy behavior — it may never fire earlier than the proxy. + assert.equal(st.calibratedEstimate, 1, "under-estimate route clamps to k̂=1 (no inflation)"); + assert.equal(st.calibratedEstimateOrigin, "http://b"); +}); + +test("settleUsageReport: implausible pairs never enter the ring", () => { + const s = makeSession(); + const st = s.stats; + pair(s, 10000, "http://a", 5000); + pair(s, 20000, "http://a", 11000); + assert.equal(st.calibratedEstimate, 0.525); + + // Tiny estimates are noise — below the floor they must not touch the ring. + pair(s, 1000, "http://a", 2000); + assert.deepEqual(st.calibrationRing?.values, [0.5, 0.55], "tiny-estimate pairs are ignored"); + + // Out-of-band ratios (<0.2 / >5) mean the report does not correspond to + // this payload (placeholder billing, relay echo) — discarded, ring intact. + pair(s, 10000, "http://a", 100000); + pair(s, 10000, "http://a", 100); + assert.deepEqual(st.calibrationRing?.values, [0.5, 0.55], "out-of-band samples are discarded"); + assert.equal(st.calibratedEstimate, 0.525); + + // Disagreement (spread > x2 across the window) clears the factor back to raw. + pair(s, 10000, "http://a", 30000); + assert.equal(st.calibratedEstimate, undefined, "inconsistent evidence must clear k̂"); + assert.equal(st.calibratedEstimateOrigin, undefined); + + // The ring is bounded at three samples (oldest shifts out). + pair(s, 10000, "http://a", 4500); + pair(s, 10000, "http://a", 4600); + pair(s, 10000, "http://a", 4700); + assert.equal(st.calibrationRing?.values.length, 3, "ring capped at CALIBRATION_SAMPLE_WINDOW"); +}); + +test("settleUsageReport: published factors clamp to [0.25, 1] — one-way, deflate only", () => { + const hi = makeSession(); + pair(hi, 10000, "http://e", 45000); + pair(hi, 10000, "http://e", 50000); + assert.equal(hi.stats.calibratedEstimate, 1, "mean 4.75 clamps down to 1: an under-estimating route keeps legacy behavior, calibration never inflates a reading"); + + const lo = makeSession(); + pair(lo, 10000, "http://f", 2200); + pair(lo, 10000, "http://f", 2400); + assert.equal(lo.stats.calibratedEstimate, 0.25, "mean 0.23 clamps up to 0.25"); + + // The corrected reading can only drop BELOW the raw estimate, never rise: + // applying the clamped hi factor is a no-op (raw × 1), applying the lo + // one deflates — both bounded by the clamp range. + assert.equal(applyEstimateCalibration(10000, hi.stats.calibratedEstimate, "http://e", "http://e"), 10000); + assert.equal(applyEstimateCalibration(10000, lo.stats.calibratedEstimate, "http://f", "http://f"), 2500); +}); + +test("settleUsageReport: a report without a known upstream neither learns nor re-provenances", () => { + const s = makeSession(); + const st = s.stats; + pair(s, 10000, "http://a", 5000); + pair(s, 20000, "http://a", 11000); + assert.equal(st.lastInputTokensOrigin, "http://a"); + st.lastLocalTextEstimate = 10000; + st.lastLocalTextEstimateOrigin = "http://a"; + settleUsageReport(s, { total: 5000, reportedCached: null }); + assert.equal(st.calibratedEstimate, 0.525, "no-route reports must not disturb the ring"); + assert.equal(st.lastInputTokensOrigin, "http://a"); + assert.equal(st.lastLocalTextEstimate, undefined); +}); + +test("resetSessionCompression drops calibration + baseline provenance at the compaction boundary", () => { + _setStoreForTest(new SessionStore({ enabled: false })); + const s = getSession("i1933-reset"); + const st = s.stats; + st.lastInputTokens = 12345; + st.lastInputTokensSource = "usage"; + st.lastInputTokensOrigin = "http://a"; + st.calibratedEstimate = 0.7; + st.calibratedEstimateOrigin = "http://a"; + st.calibrationRing = { origin: "http://a", values: [0.7] }; + st.lastLocalTextEstimate = 999; + st.lastLocalTextEstimateOrigin = "http://a"; + resetSessionCompression(s); + assert.equal(st.lastInputTokensOrigin, undefined); + assert.equal(st.calibratedEstimate, undefined); + assert.equal(st.calibratedEstimateOrigin, undefined); + assert.equal(st.calibrationRing, undefined); + assert.equal(st.lastLocalTextEstimate, undefined); + assert.equal(st.lastLocalTextEstimateOrigin, undefined); +}); + +const WINDOW = 20_000; +const SUMMARY_TEXT = + "PREFLIGHT SUMMARY: this segment is a load-growth fixture whose every marker is derivable from its turn index."; + +type Call = { summary: boolean; stream: boolean }; + +function chatSse(text: string, usage?: { prompt_tokens: number; completion_tokens: number }): string { + const chunk = (delta: Record, finish: string | null, extra?: Record): string => + `data: ${JSON.stringify({ id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "gpt-test", choices: [{ index: 0, delta, finish_reason: finish }], ...extra })}\n\n`; + const u = usage ? { usage } : {}; + return chunk({ role: "assistant" }, null) + chunk({ content: text }, null) + chunk({}, "stop", u) + "data: [DONE]\n\n"; +} + +function longMessages(count: number, charsPerMsg: number): Array<{ role: string; content: string }> { + const pad = Math.max(1, charsPerMsg - 12); + const messages: Array<{ role: string; content: string }> = []; + for (let i = 0; i < count; i++) { + messages.push({ + role: i % 2 === 0 ? "user" : "assistant", + content: `turn ${i}: ` + "f".repeat(pad), + }); + } + return messages; +} + +// Bills what it receives: sum(messages[].content chars)/4 x factor — the same +// caliber bili estimates with, scaled to simulate a provider whose tokenizer +// counts more (factor > 1) or fewer (factor < 1) tokens than chars/4. +function makeUpstream(factor: number): { server: http.Server; calls: Call[] } { + const calls: Call[] = []; + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + const raw = Buffer.concat(chunks).toString("utf8"); + const isSummary = /TASK: The conversation segment below/.test(raw); + try { + const parsed = JSON.parse(raw) as { stream?: boolean; messages?: Array<{ content?: unknown }> }; + calls.push({ summary: isSummary, stream: !!parsed.stream }); + if (!isSummary) { + const chars = (parsed.messages ?? []).reduce((n, m) => n + (typeof m.content === "string" ? m.content.length : 0), 0); + const billed = Math.round((chars / 4) * factor); + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(chatSse("forwarded answer", { prompt_tokens: billed, completion_tokens: 3 })); + return; + } + } catch { /* fall through */ } + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ id: "chatcmpl-sum", object: "chat.completion", model: "gpt-test", choices: [{ index: 0, message: { role: "assistant", content: SUMMARY_TEXT }, finish_reason: "stop" }] })); + }); + }); + return { server, calls }; +} + +function proxyOptions(routes: Record): ProxyOptions { + return { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: routes as ProxyOptions["routes"], + modelContextLimit: WINDOW, + kernelConfig: defaultConfig(WINDOW, { + preserveRecentMessages: 2, + preserveRecentTokens: 2000, + compress: { minCompressRange: 1000, maxSummaryLength: 20000, minSummaryLength: 50 }, + }), + compress: { injectTool: true, injectNudge: true }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions; +} + +test("e2e F1: a relay billing below the local estimator teaches k̂<1 — the calibrated estimate keeps an over-raw payload under the trigger", async () => { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const { server: upstream, calls } = makeUpstream(0.5); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startServer(proxyOptions({ [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: WINDOW } } } })); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/chat/completions`; + const post = (messages: Array<{ role: string; content: string }>) => + fetch(url, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "i1933-f1" }, + body: JSON.stringify({ model: "gpt-test", stream: true, messages }), + }); + + try { + // Turns 1–2: growth traffic whose settled pairs teach the route its + // billing scale (each pair: billed = est x 0.5, ring needs two + // agreeing samples before a factor publishes). + const t1 = longMessages(4, 4000); + const r1 = await post(t1); + assert.equal(r1.status, 200, `turn 1 must forward: HTTP ${r1.status}`); + await r1.text(); + + const t2msgs = [...t1, ...longMessages(4, 4000)]; + const r2 = await post(t2msgs); + assert.equal(r2.status, 200, `turn 2 must forward: HTTP ${r2.status}`); + await r2.text(); + + const learned = listSessions().find((s) => s.id === "i1933-f1")?.stats?.calibratedEstimate; + assert.ok(learned !== undefined && learned > 0.3 && learned < 0.6, `two consistent samples must publish k̂≈0.4–0.5, got ${learned}`); + + // Turn 3: full-history resend sized so the RAW local estimate crosses + // the 20k target (~22k+) while the calibrated one stays far under + // (~9–10k). Pre-#1933 the raw estimate alone fired preflight here. + const t3msgs = [...t2msgs, ...longMessages(14, 4000)]; + const r3 = await post(t3msgs); + const body = await r3.text(); + assert.equal(r3.status, 200, `calibrated-fit payload must forward, not fold: HTTP ${r3.status} ${body.slice(0, 240)}`); + assert.ok(body.includes("forwarded answer"), "the model reply must reach the client"); + assert.equal(calls.filter((c) => c.summary).length, 0, `no preflight summarization may have run (got ${calls.filter((c) => c.summary).length})`); + assert.equal(calls.filter((c) => c.stream).length, 3, "all three turns were forwarded as-is"); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +}); + +test("e2e F2: a route switch demotes the stale usage-baseline — the small post-switch request is not folded", async () => { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const a = makeUpstream(1.4); + const b = makeUpstream(1.0); + a.server.listen(0, "127.0.0.1"); + await once(a.server, "listening"); + const portA = (a.server.address() as { port: number }).port; + b.server.listen(0, "127.0.0.1"); + await once(b.server, "listening"); + const portB = (b.server.address() as { port: number }).port; + const proxy = await startServer(proxyOptions({ + [`http://127.0.0.1:${portA}`]: { models: { "gpt-test": { context: WINDOW } } }, + [`http://127.0.0.1:${portB}`]: { models: { "gpt-test": { context: WINDOW } } }, + })); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + + try { + // Turn 1 on route A: the relay bills 1.4x the local estimate, so its + // usage report lands ABOVE the 20k window target even though the local + // estimate (~16k incl. overhead) stayed under it — no trigger, forward. + const r1 = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${portA}/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "i1933-f2" }, + body: JSON.stringify({ model: "gpt-test", stream: true, messages: longMessages(15, 4000) }), + }); + assert.equal(r1.status, 200, `turn 1 must forward: HTTP ${r1.status}`); + await r1.text(); + + // Turn 2 switches to route B with a SMALL payload. Pre-#1933 the + // baseline floor (route A's ~21k usage) alone crossed the target and + // folded this request; now the provenance mismatch demotes it. + const r2 = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${portB}/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": "i1933-f2" }, + body: JSON.stringify({ model: "gpt-test", stream: true, messages: longMessages(6, 4000) }), + }); + const body = await r2.text(); + assert.equal(r2.status, 200, `stale cross-route baseline must not fold this request: HTTP ${r2.status} ${body.slice(0, 240)}`); + assert.ok(body.includes("forwarded answer"), "the model reply must reach the client"); + assert.equal(b.calls.filter((c) => c.summary).length, 0, `no preflight summarization on the switched route (got ${b.calls.filter((c) => c.summary).length})`); + assert.equal(a.calls.filter((c) => c.summary).length, 0, "no preflight summarization on the original route either"); + } finally { + proxy.close(); + await once(proxy, "close"); + a.server.close(); + await once(a.server, "close"); + b.server.close(); + await once(b.server, "close"); + } +}); diff --git a/tests/preflight-multi-range.test.ts b/tests/preflight-multi-range.test.ts index 9c89f65e7f..d841e9de5c 100644 --- a/tests/preflight-multi-range.test.ts +++ b/tests/preflight-multi-range.test.ts @@ -183,7 +183,7 @@ test("#574 truthful exhaustion: every range's summary unusable → 502 only afte } }); -test("#574 budget cap: many unusable ranges → exactly MAX_SUMMARY_CALLS_PER_PREFLIGHT summary calls, budget-exhausted detail", async () => { +test("#574 budget cap: many unusable ranges → the raised #1933 cap (2x base) bounds the walk, budget-exhausted detail", async () => { const { server: upstream, calls } = makeUpstream(() => false); upstream.listen(0, "127.0.0.1"); await once(upstream, "listening"); @@ -197,7 +197,12 @@ test("#574 budget cap: many unusable ranges → exactly MAX_SUMMARY_CALLS_PER_PR const r = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { method: "POST", headers: { "content-type": "application/json", "x-acp-session": "multi-range-budget-sess" }, - body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: conversation(48, "t3-budget") }), + // #1933 raised the budget with the entry overshoot (~20x here, so + // the full 2x cap): the fixture must expose MORE viable ranges + // than the raised cap (200 turns -> ~55 ranges vs 32 calls) or the + // walk finishes its pass first and reports "no range could be + // compressed" instead of hitting the cap. + body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: conversation(200, "t3-budget") }), }); assert.equal(r.status, 502, "still over-window after the budget → fail-fast 502"); const json = JSON.parse(await r.text()) as { error?: { code?: string; retryable?: boolean; message?: string } }; @@ -206,8 +211,10 @@ test("#574 budget cap: many unusable ranges → exactly MAX_SUMMARY_CALLS_PER_PR assert.match(json.error?.message ?? "", /summarization budget/i, `the budget variant is reported (got: ${json.error?.message})`); assert.match(json.error?.message ?? "", /compressible range\(s\) still visible/, `reports how many compressible ranges remain (got: ${json.error?.message})`); + // #1933: the budget scales with entry overshoot up to the 2x cap, so + // the raised ceiling — not the base constant — bounds it. const summaryCalls = calls.filter((c) => !c.stream); - assert.equal(summaryCalls.length, MAX_SUMMARY_CALLS_PER_PREFLIGHT, `the call cap bounds the walk (got ${summaryCalls.length})`); + assert.equal(summaryCalls.length, MAX_SUMMARY_CALLS_PER_PREFLIGHT * 2, `the raised call cap bounds the walk (got ${summaryCalls.length})`); assert.equal(calls.filter((c) => c.stream).length, 0, "the over-window payload was NOT forwarded"); const s = listSessions()[0]; diff --git a/tests/preflight-round-budget.test.ts b/tests/preflight-round-budget.test.ts index 9aceb4387b..210858c652 100644 --- a/tests/preflight-round-budget.test.ts +++ b/tests/preflight-round-budget.test.ts @@ -19,7 +19,10 @@ import { MAX_PREFLIGHT_ROUNDS, MAX_SUMMARY_CALLS_PER_PREFLIGHT } from "../src/pr // that) needs a deep fold, and the per-invocation budget of 8 left a live // 1.39M-token Gemini session ~8k tokens short of fitting: it reported "the // compress budget was exhausted after 8 rounds" and dropped the turn even -// though every fold it attempted had succeeded. +// though every fold it attempted had succeeded. #1933 made the depth dynamic: +// base MAX_PREFLIGHT_ROUNDS scales with the entry overshoot (payload vs window), +// capped at 2x the base — an ~11x overshoot like the second fixture below gets +// the full raised budget of 32 rounds / 32 summary calls. const WINDOW = 30_000; const MESSAGES = 60; @@ -148,9 +151,11 @@ test("e2e preflight: beyond the round budget the fail-fast reports the post-fold const proxyPort = (proxy.address() as { port: number }).port; try { - // 400 small turns (~340k tokens vs a 30k window): each fold removes - // far less than the chunk budget, so even 16 successful rounds leave - // the payload well over the window — the walk stops at the round cap. + // 400 small turns (~384k tokens vs a 30k window, ~11x overshoot): each + // fold removes far less than the chunk budget, so even the raised + // #1933 budget of 32 successful rounds (base 16 scaled by overshoot, + // capped at 2x) leaves the payload well over the window — the walk + // stops at the raised round cap. const COUNT = 400; const REPEATS = 200; assert.ok(materialiseTokens(COUNT, REPEATS) > WINDOW * 3, `fixture must start well over the window (${materialiseTokens(COUNT, REPEATS)} vs ${WINDOW})`); @@ -164,12 +169,12 @@ test("e2e preflight: beyond the round budget the fail-fast reports the post-fold assert.equal(resp.status, 502, `beyond the round budget the turn must be refused: HTTP ${resp.status} ${body.slice(0, 240)}`); const json = JSON.parse(body) as { error?: { code?: string; message?: string } }; assert.equal(json.error?.code, "preflight_compress_failed"); - assert.match(json.error?.message ?? "", new RegExp(`exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds`, "i"), `names the round budget (got: ${json.error?.message})`); + assert.match(json.error?.message ?? "", new RegExp(`exhausted after ${MAX_PREFLIGHT_ROUNDS * 2} rounds`, "i"), `names the raised round budget (got: ${json.error?.message})`); assert.match(json.error?.message ?? "", /down from ~\d+ before preflight/, `quotes the post-fold size, not only the original (got: ${json.error?.message})`); assert.match(json.error?.message ?? "", /compressible range\(s\) still visible/, `reports how many compressible ranges remain (got: ${json.error?.message})`); const summaries = CALLS.filter((c) => c.summary); - assert.equal(summaries.length, MAX_SUMMARY_CALLS_PER_PREFLIGHT, `one fold per round, one call each (got ${summaries.length})`); + assert.equal(summaries.length, MAX_SUMMARY_CALLS_PER_PREFLIGHT * 2, `one fold per round, one call each, raised #1933 budget (got ${summaries.length})`); assert.equal(CALLS.filter((c) => c.raw.includes('"stream":true')).length, 0, "the over-window payload was NOT forwarded"); } finally { proxy.close();