From 2d4bc17f3fd73eeeee2d0892e0fcd59cc829a6f3 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 5 Oct 2026 11:13:15 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20separate=20calibrated=20context=20di?= =?UTF-8?q?splay=20from=20the=20safety=20bound;=20add=20model=20provenance?= =?UTF-8?q?=20to=20k=CC=82=20(#2117)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cache-ledger.ts | 11 +- src/preflight.ts | 6 +- src/server.ts | 69 +++- src/session.ts | 73 +++- src/util.ts | 17 + src/web/client.ts | 30 +- src/web/i18n.ts | 14 + src/web/sessions-data.ts | 37 +- ...ue2117-context-estimate-provenance.test.ts | 356 ++++++++++++++++++ 9 files changed, 588 insertions(+), 25 deletions(-) create mode 100644 tests/issue2117-context-estimate-provenance.test.ts diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index 33e71099c9..bc74551652 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -741,8 +741,15 @@ export function settleUsageReport( const pendingOrigin = normalizeUpstreamOrigin(session.stats.lastLocalTextEstimateOrigin); if (pendingEst !== undefined && pendingEst >= CALIBRATION_MIN_ESTIMATE && settleOrigin !== undefined && pendingOrigin === settleOrigin) { const rawSample = s.total / pendingEst; + // #2117 B: the ring is keyed by route AND model — tokenizers bill + // differently across models, so a mid-session model switch starts a + // FRESH ring instead of blending two models' billing scales into one + // factor. Legacy rings without a model key restart on the first + // known-model sample (discard-and-relearn: ≤2 samples until the old + // factor's evidence can no longer publish). + const settleModel = typeof session.metadata?.lastModel === "string" && session.metadata.lastModel !== "" ? session.metadata.lastModel : undefined; let ring = session.stats.calibrationRing; - if (!ring || ring.origin !== settleOrigin) ring = { origin: settleOrigin, values: [] }; + if (!ring || ring.origin !== settleOrigin || ring.model !== settleModel) ring = { origin: settleOrigin, model: settleModel, values: [] }; if (rawSample >= CALIBRATION_SAMPLE_MIN && rawSample <= CALIBRATION_SAMPLE_MAX) { ring.values.push(rawSample); if (ring.values.length > CALIBRATION_SAMPLE_WINDOW) ring.values.shift(); @@ -751,9 +758,11 @@ export function settleUsageReport( const mean = ring.values.reduce((a, b) => a + b, 0) / ring.values.length; session.stats.calibratedEstimate = Math.min(CALIBRATION_CLAMP_MAX, Math.max(CALIBRATION_CLAMP_MIN, mean)); session.stats.calibratedEstimateOrigin = settleOrigin; + session.stats.calibratedEstimateModel = settleModel; } else { delete session.stats.calibratedEstimate; delete session.stats.calibratedEstimateOrigin; + delete session.stats.calibratedEstimateModel; } } session.stats.calibrationRing = ring; diff --git a/src/preflight.ts b/src/preflight.ts index c03fb60e8f..a5b021b44b 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -18,7 +18,7 @@ import { proxyDispatcher } from "./upstream-proxy.js"; import { lastCompressSuffix, type Session } from "./session.js"; import { peekRegistryOutputLimit } from "./registry.js"; import { safePrefix } from "./text-safe.js"; -import { applyEstimateCalibration } from "./util.js"; +import { applyEstimateCalibration, currentCalibrationFactor } from "./util.js"; // #247: proactive pre-forward compression. When the session's real context // (previous turn's upstream input_tokens) exceeds the current model's window @@ -871,7 +871,9 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa // #1933 F1: capture k̂ once — it only mutates on usage settlement (outside // this invocation), while every fit judgment in this loop must stay on // one consistent scale with the gate that started it. - const kFactor = deps.session.stats.calibratedEstimate; + // #2117 B: model-provenance gate — a factor learned on another model acts + // as absent (raw estimate) rather than deciding with a cross-model scale. + const kFactor = currentCalibrationFactor(deps.session.stats, deps.session.metadata?.lastModel); const kOrigin = deps.session.stats.calibratedEstimateOrigin; let textTarget = Math.max(0, Math.min(limit, deps.compressionTarget ?? limit) - imageReserve); const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: applyEstimateCalibration(estimateCoreMessages(messages) + wireOverhead, kFactor, kOrigin, deps.upstreamOrigin) + imageReserve, rangesRemaining: 0, fitsWindow: true }; diff --git a/src/server.ts b/src/server.ts index 8d7a43664d..de1f0f2689 100644 --- a/src/server.ts +++ b/src/server.ts @@ -118,7 +118,7 @@ import { consumePluginRegisterFor, flushConversations, handlePluginCompact, hand import { setupMitm, readMitmUpstream, getBlindTunnelStats, liveBlindTunnels, MITM_RAW_SOCKET_KEY } from "./mitm.js"; import { evaluateChain, extractChainCarriers, stampOutbound, stripEmbeddedChainCarriers } from "./chain-checkpoint.js"; import type { BiliMessage } from "acp-kernel/wire"; -import { appendSystemText, applyEstimateCalibration, BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, hardenOpenaiAssistantContent, isLoopbackAddress, inspectContextOverflow, normalizeUpstreamOrigin, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, systemToUser, usageOutputTotal, usageTotals, strippedResponseIdWarning, type ContextOverflowInfo, type WireProtocol } from "./util.js"; +import { appendSystemText, applyEstimateCalibration, currentCalibrationFactor, BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, hardenOpenaiAssistantContent, isLoopbackAddress, inspectContextOverflow, normalizeUpstreamOrigin, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, systemToUser, usageOutputTotal, usageTotals, strippedResponseIdWarning, type ContextOverflowInfo, type WireProtocol } from "./util.js"; import { safePrefix, safeSuffix } from "./text-safe.js"; import { BILI_TUNNEL_HEADER, checkTunnelDestination, classifyIp, localMachineIps, normalizeIpLiteral, parseIpLiteral, tunnelAllowlistFromEnv } from "./tunnel-guard.js"; @@ -5594,13 +5594,33 @@ function outboundPayloadBreakdown( * project from it (numbers identical to re-parsing the same string), null = * caller already tried and failed → keep the prepared projection and skip the * doomed re-parses inside the helpers (they return 0 on unparseable input). */ -export function outboundContextEstimate( +/** #2117: both calibers of one outbound send, computed in ONE projection pass. + * upperBound is the char-count upper bound (every character counts as one + * token — never undershoots; the fail-closed decision/display caliber, and + * the legacy outboundContextEstimate value). textOverhead is the billing + * caliber preflight calibrates with k̂ (#1933 F1): CJK-aware + * estimateCoreMessages + system/tools wire overhead. imageTokens is the + * separate image term (learned per-route #1843/#1857), kept out of both so + * callers can recombine per channel. */ +export interface OutboundContextEstimates { + upperBound: number; + textOverhead: number; + imageTokens: number; +} + +/** #2078: `parsed` carries the caller's pre-parsed send body so forward() does + * not pay a second full JSON.parse of the largest payload in flight per + * request. Three states: undefined = parse here (legacy callers), object = + * project from it (numbers identical to re-parsing the same string), null = + * caller already tried and failed → keep the prepared projection and skip the + * doomed re-parses inside the helpers (they return 0 on unparseable input). */ +export function outboundContextEstimates( prepared: Prepared, wireBody: string, opts: ProxyOptions, upstream: string, parsed?: Record | null, -): number { +): OutboundContextEstimates { let msgs = prepared.processedMessages.length > 0 ? prepared.processedMessages : prepared.originalMessages; const project = (value: unknown): void => { switch (prepared.protocol) { @@ -5620,8 +5640,23 @@ export function outboundContextEstimate( } else { raw = ""; } - return estimateCoreMessagesUpper(msgs) + estimateWireOverhead(prepared.protocol, raw) - + imageReserveFor(prepared.session, prepared.protocol, raw, opts, upstream); + const overhead = estimateWireOverhead(prepared.protocol, raw); + const imageTokens = imageReserveFor(prepared.session, prepared.protocol, raw, opts, upstream); + return { + upperBound: estimateCoreMessagesUpper(msgs) + overhead + imageTokens, + textOverhead: estimateCoreMessages(msgs) + overhead, + imageTokens, + }; +} + +export function outboundContextEstimate( + prepared: Prepared, + wireBody: string, + opts: ProxyOptions, + upstream: string, + parsed?: Record | null, +): number { + return outboundContextEstimates(prepared, wireBody, opts, upstream, parsed).upperBound; } async function preflightCompressIfNeeded( @@ -5701,8 +5736,10 @@ async function preflightCompressIfNeeded( // factor k̂ learned from this session's own usage reports (local estimate ÷ // what upstream actually billed, EMA, clamped 0.25–1 — one-way, deflate // only; see settleUsageReport). Unknown/mismatched origin → raw estimate, - // i.e. today's behavior. - const kFactor = session.stats.calibratedEstimate; + // i.e. today's behavior. #2117 B: the model dimension gates too — a factor + // learned on another model acts as absent here rather than deciding with a + // cross-model billing scale (currentCalibrationFactor). + const kFactor = currentCalibrationFactor(session.stats, session.metadata?.lastModel); const kOrigin = session.stats.calibratedEstimateOrigin; const calibratedText = applyEstimateCalibration(textEstimate + overheadEstimate, kFactor, kOrigin, currentOrigin); const calibratedPayload = calibratedText + imageTokens; @@ -6350,9 +6387,21 @@ async function forward( sentParsed = null; } // Publish this send, not a historical usage baseline with a fresh timestamp. - const estimate = outboundContextEstimate(prepared, sentBody, opts, upstreamUrl, sentParsed); - prepared.session.stats.localInputEstimate = estimate; - prepared.session.stats.contextTokens = estimate; + const est = outboundContextEstimates(prepared, sentBody, opts, upstreamUrl, sentParsed); + prepared.session.stats.localInputEstimate = est.upperBound; + // The char-count upper bound stays the published context value: it is + // the fail-closed caliber decision paths and legacy displays trust for + // never-reporting upstreams (#553/#728/#1493). #2117 additionally + // publishes the billing-caliber estimate of THIS send — preflight's own + // formula (CJK-aware text + wire overhead), k̂-scaled only where the + // factor's route+model provenance matches (#1933 F1 / #2117 B), plus + // the image reserve — so display surfaces can show a calibrated reading + // instead of the ~2–3.5× over-counting bound. Display-only field. + const kFactor = currentCalibrationFactor(prepared.session.stats, prepared.session.metadata?.lastModel); + const scaledText = applyEstimateCalibration(est.textOverhead, kFactor, prepared.session.stats.calibratedEstimateOrigin, normalizeUpstreamOrigin(upstreamUrl)); + prepared.session.stats.contextEstimateTokens = Math.round(scaledText + est.imageTokens); + prepared.session.stats.contextEstimateCalibrated = scaledText !== est.textOverhead; + prepared.session.stats.contextTokens = est.upperBound; prepared.session.stats.contextTokensSource = "estimate"; noteForwardedBody(prepared.session, sentBody); } diff --git a/src/session.ts b/src/session.ts index 654f6d2f3a..d5db349b1f 100644 --- a/src/session.ts +++ b/src/session.ts @@ -156,6 +156,17 @@ export type Session = { * Cleared by resetSessionCompression (native-compaction boundary). * Persisted (survives restart like the rest of stats). */ localInputEstimate?: number; + /** #2117: billing-caliber estimate of the CURRENT outbound request — + * estimateCoreMessages + wire overhead, k̂-scaled only where the + * factor's route+model provenance matches (#1933 F1 / #2117 B), plus + * the image reserve. Written by forward() alongside the char-count + * upper bound above (which stays the fail-closed caliber for + * never-reporting upstreams, #553/#728). Display-only: no decision + * path reads it — preflight keeps its own per-invocation formula. */ + contextEstimateTokens?: number; + /** #2117: whether k̂ actually scaled contextEstimateTokens (false when + * the factor is absent or its provenance doesn't match). */ + contextEstimateCalibrated?: boolean; /** #1569: last netted input value written by a REAL upstream usage * report (the arming paths never touch it — they only pose as * usage-grade for lastInputTokens). While > 0, effectiveTokenCount @@ -201,11 +212,19 @@ export type Session = { * from a different origin starts a fresh ring instead of blending * two providers' scales. */ calibratedEstimateOrigin?: string; - /** #1933 F1: evidence ring behind calibratedEstimate — the recent - * admitted raw samples for ONE origin (max 3). Persisted so a - * restart doesn't re-arm the warmup delay; reset at native- - * compaction boundaries with the rest of the baseline stats. */ - calibrationRing?: { origin: string; values: number[] }; + /** #2117 B: model the calibratedEstimate was learned against — the + * second provenance dimension. The ring and every read gate on + * route AND model: a mid-session model switch invalidates the old + * factor instead of reusing a cross-model billing scale (dangerous + * in either direction — a stale k̂<1 deflates → trigger too late). + * Absent on factors published before this field existed → reads + * treat it as compatible and retire it through normal rollover. */ + calibratedEstimateModel?: string; + /** #1933 F1 / #2117 B: evidence ring behind calibratedEstimate — + * the recent admitted raw samples for ONE origin+model (max 3). + * Persisted so a restart doesn't re-arm the warmup delay; reset at + * native-compaction boundaries with the rest of the baseline stats. */ + calibrationRing?: { origin: string; model?: string; values: number[] }; /** #1933 F1: pending pairing input — local estimate of the LAST * prepared outbound in BILLED caliber (estimateCoreMessages + * system/tools overhead + image reserve, defaultCountTokens rate), @@ -430,6 +449,44 @@ export function statusInputBaseline(session: Session): number { : stats.lastUsageGradeTokens ?? 0; } +/** #2117: the honest best reading of "how full is this context right now" for + * DISPLAY surfaces, picked by provenance — never mixes calibers on one bar: + * usage — last upstream-measured input (billing grade); wins whenever present. + * estimate — the billing-caliber estimate of the CURRENT outbound request + * (contextEstimateTokens, k̂-scaled where its route+model + * provenance matches), shown only while a usage-grade anchor + * exists: a never-reporting upstream has no evidence that the + * optimistic rate holds, so it keeps the fail-closed upper bound + * (#553/#728 discipline). + * upper — char-count upper bound of the last send: never undershoots, + * but over-counts ASCII-heavy payloads up to ~3.5× — a BOUND, + * not a reading; over-window here does NOT mean actually + * over-window. + * Legacy sessions without contextEstimateTokens fall through to upper — + * exactly today's display. Read-only; decision paths are untouched. */ +export interface ContextBest { + tokens: number; + kind: "usage" | "estimate" | "upper"; + /** true when kind=="estimate" and k̂ actually scaled the value. */ + calibrated?: boolean; + /** wall-clock of the underlying measurement (usage reports stamp it). */ + at?: number; +} + +export function displayContextBest(session: Session): ContextBest | null { + const st = session.stats; + if (st.contextTokensSource === "usage" && st.contextTokens > 0) { + const at = typeof session.metadata?.contextTokensAt === "number" ? session.metadata.contextTokensAt : undefined; + return { tokens: st.contextTokens, kind: "usage", ...(at !== undefined ? { at } : {}) }; + } + const anchored = (st.lastUsageGradeTokens ?? 0) > 0 || st.lastInputTokensSource === "usage"; + if (anchored && typeof st.contextEstimateTokens === "number" && st.contextEstimateTokens > 0) { + return { tokens: st.contextEstimateTokens, kind: "estimate", ...(st.contextEstimateCalibrated ? { calibrated: true } : {}) }; + } + if (st.contextTokens > 0) return { tokens: st.contextTokens, kind: "upper" }; + return null; +} + /** Mirror of acp-kernel's resolveAdaptiveGrowth (not exported by the kernel): * min(growthCap, max(growthFloor, round(modelContextLimit × growthRatio))). * The per-request config stamped by storeEffectiveConfig is the same object @@ -791,9 +848,15 @@ export function resetSessionCompression(session: Session): void { delete session.stats.lastInputTokensOrigin; delete session.stats.calibratedEstimate; delete session.stats.calibratedEstimateOrigin; + // #2117 B: same boundary — the factor's model provenance dies with it. + delete session.stats.calibratedEstimateModel; delete session.stats.calibrationRing; delete session.stats.lastLocalTextEstimate; delete session.stats.lastLocalTextEstimateOrigin; + // #2117: the per-send estimate describes the pre-compaction payload too — + // let the next forward re-measure instead of displaying a stale reading. + delete session.stats.contextEstimateTokens; + delete session.stats.contextEstimateCalibrated; session.stats.contextTokens = 0; delete session.stats.contextTokensSource; session.metadata.nativeCompactionAt = Date.now(); diff --git a/src/util.ts b/src/util.ts index 6f88bfb8ba..fc7701feed 100644 --- a/src/util.ts +++ b/src/util.ts @@ -104,6 +104,23 @@ export function applyEstimateCalibration(raw: number, k: number | undefined, kOr return raw * k; } +/** #2117: read-side gate for the learned estimator scale — the model dimension + * of k̂ provenance. A factor published against model X must not decide on + * model Y: tokenizers bill differently, and a stale cross-model factor can be + * dangerously wrong in EITHER direction (k̂<1 deflates → trigger too LATE). + * Both models known (non-empty strings) and different → factor absent (raw + * estimate, legacy conservative behavior); either side unknown/empty/non- + * string → factor kept as-is, so factors published before this field existed + * (or requests without a model stamp) are never invalidated by missing + * information — they retire through the normal ring rollover when new samples + * land. Invalid factors degrade to absent. */ +export function currentCalibrationFactor(stats: { calibratedEstimate?: number; calibratedEstimateModel?: string }, lastModel?: unknown): number | undefined { + const k = stats.calibratedEstimate; + if (k === undefined || !Number.isFinite(k) || k <= 0) return undefined; + if (stats.calibratedEstimateModel !== undefined && typeof lastModel === "string" && lastModel !== "" && stats.calibratedEstimateModel !== lastModel) return undefined; + return k; +} + export type WireProtocol = "anthropic" | "openai" | "responses" | "google"; /** diff --git a/src/web/client.ts b/src/web/client.ts index ff789afda4..ca589cbf64 100644 --- a/src/web/client.ts +++ b/src/web/client.ts @@ -210,9 +210,9 @@ export const WEB_CLIENT = `(function () { const tr = document.createElement("tr"); tr.title = s.id; if (compact) { - tr.innerHTML = "" + sessionTitleCell(s) + '' + protoBadge(s.protocol) + '' + fmtW(s.contextTokens) + '' + savedTd(s) + '' + timeAgo(s.lastSeen) + ""; + tr.innerHTML = "" + sessionTitleCell(s) + '' + protoBadge(s.protocol) + '' + fmtW(s.contextBest ? s.contextBest.tokens : s.contextTokens) + '' + savedTd(s) + '' + timeAgo(s.lastSeen) + ""; } else { - tr.innerHTML = "" + sessionTitleCell(s) + '' + (s.clientHint ? '' + escapeHtml(s.clientHint) + "" : '' + t("common.none") + "") + '' + protoBadge(s.protocol) + '' + escapeHtml(hostOf(s.upstreamOrigin)) + '' + (s.requests ? s.requests : t("common.none")) + '' + fmtW(s.contextTokens) + '' + (s.inputTokens ? fmtW(s.inputTokens) : '' + t("common.none") + "") + "" + savedTd(s) + hitTd(s) + switchTd(s) + '' + (s.foldCount || 0) + '' + (s.blocks || 0) + '' + timeAgo(s.lastSeen) + ""; + tr.innerHTML = "" + sessionTitleCell(s) + '' + (s.clientHint ? '' + escapeHtml(s.clientHint) + "" : '' + t("common.none") + "") + '' + protoBadge(s.protocol) + '' + escapeHtml(hostOf(s.upstreamOrigin)) + '' + (s.requests ? s.requests : t("common.none")) + '' + fmtW(s.contextBest ? s.contextBest.tokens : s.contextTokens) + '' + (s.inputTokens ? fmtW(s.inputTokens) : '' + t("common.none") + "") + "" + savedTd(s) + hitTd(s) + switchTd(s) + '' + (s.foldCount || 0) + '' + (s.blocks || 0) + '' + timeAgo(s.lastSeen) + ""; } let navTimer = null; tr.addEventListener("click", (ev) => { @@ -804,13 +804,31 @@ export const WEB_CLIENT = `(function () { parts.push(""); // #1839: mark estimate-grade context numbers so a bounded local estimate // is never read as a measured value (the ghost-denominator incident). - const ctxEstMark = d.contextTokensSource === "estimate" ? ' ' + t("common.ctx_est") + "" : ""; + // #2117: drive the bar from contextBest — provenance-picked by the server + // (usage > calibrated estimate > char-count upper bound) — so one value + // never mixes calibers; the bound itself stays visible, labeled as a BOUND + // on its own sub-line, and the calibration behind the estimate is shown + // with its evidence instead of an unexplained number. + const best = d.contextBest || null; + const ctxVal = best ? best.tokens : (d.contextTokens || 0); + const ctxMark = best + ? ' ' + (best.kind === "usage" ? t("common.ctx_meas") : best.kind === "estimate" ? (best.calibrated ? t("common.ctx_est_calib") : t("common.ctx_est_raw")) : t("common.ctx_upper")) + "" + : (d.contextTokensSource === "estimate" ? ' ' + t("common.ctx_est") + "" : ""); if (d.contextWindow && d.contextWindow > 0) { - const pct = Math.min(100, Math.round((d.contextTokens / d.contextWindow) * 100)); + const pct = Math.min(100, Math.round((ctxVal / d.contextWindow) * 100)); const cls = pct >= 90 ? "bar-fill danger" : pct >= 70 ? "bar-fill warn" : "bar-fill"; - parts.push('
' + t("common.context") + ctxEstMark + " / " + t("common.window") + '
' + fmtW(d.contextTokens) + " / " + fmtW(d.contextWindow) + " (" + pct + "%)" + ctxEstMark + "
"); + parts.push('
' + t("common.context") + ctxMark + " / " + t("common.window") + '
' + fmtW(ctxVal) + " / " + fmtW(d.contextWindow) + " (" + pct + "%)" + ctxMark + "
"); + const ctxSubBits = []; + if ((d.contextUpperTokens || 0) > 0 && (!best || best.kind !== "upper") && d.contextUpperTokens !== ctxVal) { + ctxSubBits.push(t("det.upper_bound", { v: fmtW(d.contextUpperTokens) })); + } + if (d.estimateCalibration) { + const c = d.estimateCalibration; + ctxSubBits.push('' + t("det.calib_chip", { k: c.factor.toFixed(2), n: c.samples, spread: c.spread.toFixed(1), origin: c.origin ? hostOf(c.origin) : "" }) + ""); + } + if (ctxSubBits.length) parts.push('
' + ctxSubBits.join(" · ") + "
"); } else { - parts.push('
' + t("common.context") + ctxEstMark + ": " + fmtW(d.contextTokens || 0) + "
"); + parts.push('
' + t("common.context") + ctxMark + ": " + fmtW(ctxVal) + "
"); } if ((d.retrieveCalls || 0) > 0) parts.push('
' + t("det.ccr") + ' · ' + t("det.ccr_detail", { calls: d.retrieveCalls, hits: d.retrieveHits || 0, misses: d.retrieveMisses || 0 }) + "
"); if ((d.storedBytes || 0) > 0) parts.push('
' + t("det.store") + ' · ' + fmtB(d.storedBytes) + ((d.storeBytesSaved || 0) > 0 ? " / " + fmtB(d.storeBytesSaved) + " " + t("common.saved") : "") + "
"); diff --git a/src/web/i18n.ts b/src/web/i18n.ts index 0a259c869d..cec04a3c2c 100644 --- a/src/web/i18n.ts +++ b/src/web/i18n.ts @@ -35,6 +35,10 @@ export const MESSAGES: Record> = { "common.saved": "节省", "common.context": "上下文", "common.ctx_est": "估算", + "common.ctx_meas": "实测", + "common.ctx_est_calib": "估算·已校准", + "common.ctx_est_raw": "估算·未校准", + "common.ctx_upper": "估算·上界", "common.window": "窗口", "common.title": "标题", "common.protocol": "协议", @@ -136,6 +140,9 @@ export const MESSAGES: Record> = { "det.conflicts_cleared": "已清除", "det.usage": "用量统计", "det.last_input": "最近单次输入", + "det.upper_bound": "保守上界 {v}(字符数上界,只用于安全保护,不代表实际用量)", + "det.calib_chip": "k̂={k} · {n}样本 · 极差{spread}× · {origin}", + "det.calib_tip": "校准系数 k̂:本路由+模型下最近 {n} 个一致样本(极差 {spread}×)的均值,只向下修正估算;路由或模型变化后重新学习。来源 {origin}{model}。", "det.ccr": "CCR 内容检索", "det.ccr_detail": "{calls} 次调用 · {hits} 命中 · {misses} 未中", "det.store": "内容存储", @@ -348,6 +355,10 @@ export const MESSAGES: Record> = { "common.saved": "saved", "common.context": "context", "common.ctx_est": "estimated", + "common.ctx_meas": "measured", + "common.ctx_est_calib": "est.·calibrated", + "common.ctx_est_raw": "est.·uncalibrated", + "common.ctx_upper": "est.·upper bound", "common.window": "window", "common.title": "title", "common.protocol": "protocol", @@ -448,6 +459,9 @@ export const MESSAGES: Record> = { "det.conflicts_cleared": "Cleared", "det.usage": "Usage", "det.last_input": "Last single-turn input", + "det.upper_bound": "bound {v} (char-count upper bound — safety protection only, not actual usage)", + "det.calib_chip": "k̂={k} · {n} samples · spread {spread}× · {origin}", + "det.calib_tip": "Calibration factor k̂: mean of the last {n} consistent samples (spread {spread}×) on this route+model; deflates estimates only; relearned after a route or model change. Source {origin}{model}.", "det.ccr": "CCR content retrieval", "det.ccr_detail": "{calls} calls · {hits} hits · {misses} misses", "det.store": "Content store", diff --git a/src/web/sessions-data.ts b/src/web/sessions-data.ts index 25e439ad01..e62337fc4c 100644 --- a/src/web/sessions-data.ts +++ b/src/web/sessions-data.ts @@ -1,4 +1,4 @@ -import { listSessions, type Session } from "../session.js"; +import { listSessions, displayContextBest, type ContextBest, type Session } from "../session.js"; import { conflictEventsOf } from "../conflict-watch.js"; import { SessionStore } from "../persist.js"; import { renderHandoff } from "../export.js"; @@ -25,6 +25,12 @@ export interface WebSessionSummary { /** #1839: provenance of contextTokens — "usage" = last real usage report, * "estimate" = bounded local estimate (display should mark it as such). */ contextTokensSource?: "usage" | "estimate"; + /** #2117: the honest best reading of "how full is this context right now", + * picked by provenance (usage > calibrated estimate > char-count upper + * bound). Display surfaces drive their main bar from this so one value + * never mixes calibers; raw contextTokens stays for API compat + legacy + * clients. Absent when nothing has been observed yet. */ + contextBest?: ContextBest; tokensSaved: number; inputTokens: number; cachedTokens: number; @@ -124,6 +130,17 @@ export interface WebSessionDetail extends WebSessionSummary { /** Measured system-prompt size in tokens — the not-compressible baseline drawn * under the trajectory chart. */ systemPromptTokens?: number; + /** #2117: char-count upper bound of the LAST outbound send (the fail-closed + * caliber) — rendered beside the main bar as an explicit BOUND, never as a + * reading; over-window here does not mean actually over-window. */ + contextUpperTokens?: number; + /** #2117: route that measured the last usage-grade input (#1933 F2 provenance). */ + lastInputTokensOrigin?: string; + /** #2117: wall-clock when the last usage-grade context value settled. */ + contextMeasuredAt?: number; + /** #2117: the learned estimator calibration behind the displayed estimate — + * k̂ plus its evidence (samples admitted on this route+model, spread). */ + estimateCalibration?: { factor: number; origin?: string; model?: string; samples: number; spread: number }; ledger: ReturnType | null; /** Raw markdown of the handoff doc (handoffHtml rendered) — for the * copy-markdown / download buttons. */ @@ -233,8 +250,10 @@ function summaryOf(s: Session, live: boolean): WebSessionSummary { const clientHint = typeof metaRec["pluginAgent"] === "string" && metaRec["pluginAgent"] ? metaRec["pluginAgent"] as string : typeof metaRec["clientHint"] === "string" && metaRec["clientHint"] ? metaRec["clientHint"] as string : ""; + const best = displayContextBest(s); return { id: s.id, + ...(best ? { contextBest: best } : {}), ...(s.meta.title ? { title: s.meta.title } : {}), // #1426: meta.label is auto-stamped with the session id on many clients — // treat label === id as "no title" so lists/details show 无标题 + block hint. @@ -439,6 +458,22 @@ export async function buildSessionDetail(id: string): Promise 0 ? { systemPromptTokens: sysPrompt } : {}), + ...(typeof session.stats.localInputEstimate === "number" && session.stats.localInputEstimate > 0 + ? { contextUpperTokens: session.stats.localInputEstimate } : {}), + ...(session.stats.lastInputTokensOrigin ? { lastInputTokensOrigin: session.stats.lastInputTokensOrigin } : {}), + ...(typeof session.metadata["contextTokensAt"] === "number" ? { contextMeasuredAt: session.metadata["contextTokensAt"] as number } : {}), + ...(typeof session.stats.calibratedEstimate === "number" && session.stats.calibratedEstimate > 0 && (session.stats.calibrationRing?.values.length ?? 0) > 0 + ? (() => { + const vals = session.stats.calibrationRing!.values; + return { estimateCalibration: { + factor: session.stats.calibratedEstimate, + ...(session.stats.calibratedEstimateOrigin !== undefined ? { origin: session.stats.calibratedEstimateOrigin } : {}), + ...(session.stats.calibratedEstimateModel !== undefined ? { model: session.stats.calibratedEstimateModel } : {}), + samples: vals.length, + spread: Math.max(...vals) / Math.min(...vals), + } }; + })() + : {}), ledger: buildSessionCacheReport(session), handoffMd, handoffHtml: markdownToHtml(handoffMd), diff --git a/tests/issue2117-context-estimate-provenance.test.ts b/tests/issue2117-context-estimate-provenance.test.ts new file mode 100644 index 0000000000..811769ccc2 --- /dev/null +++ b/tests/issue2117-context-estimate-provenance.test.ts @@ -0,0 +1,356 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { defaultConfig } from "acp-kernel"; +import { startServer, outboundContextEstimate, outboundContextEstimates } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { settleUsageReport } from "../src/cache-ledger.ts"; +import { currentCalibrationFactor } from "../src/util.ts"; +import { displayContextBest, getSession, listSessions, resetSessionCompression, zeroStats, type Session } from "../src/session.ts"; + +// #2117: the admin bar showed the char-count UPPER BOUND of the in-flight send +// (every character = 1 token — up to ~3.5x over-count on ASCII-heavy tool +// output) as if it were the context size, then snapped to the real billed +// input when the usage report landed (repro: 206K "估算" → 98K measured). +// Fix A separates the calibers: forward() publishes the billing-caliber +// estimate of THIS send alongside the bound, display surfaces pick by +// provenance (usage > calibrated estimate > upper bound), and the bound stays +// visible only as a labeled bound. Fix B keys the learned estimator scale k̂ +// by route AND model so a mid-session model switch relearns instead of +// deciding with a cross-model billing scale. Decision paths keep their exact +// formulas; this pins the new seams and the refactored estimator split. + +let seq = 0; +function makeSession(): Session { + seq += 1; + return { + id: `i2117-${seq}`, + metadata: {}, + stats: {}, + state: { blocks: [], messageRefs: { byRaw: {}, byRef: {} }, tokenSnapshot: {} }, + } as unknown as Session; +} + +test("currentCalibrationFactor: invalid factors degrade to absent", () => { + assert.equal(currentCalibrationFactor({}, "m"), undefined); + assert.equal(currentCalibrationFactor({ calibratedEstimate: 0 }, "m"), undefined); + assert.equal(currentCalibrationFactor({ calibratedEstimate: Number.NaN }, "m"), undefined); + assert.equal(currentCalibrationFactor({ calibratedEstimate: Number.POSITIVE_INFINITY }, "m"), undefined); + assert.equal(currentCalibrationFactor({ calibratedEstimate: -0.5 }, "m"), undefined); +}); + +test("currentCalibrationFactor: cross-model factors act as absent, missing info keeps them", () => { + const st = { calibratedEstimate: 0.5 }; + // Legacy factor (never recorded a model) or unknown current model — missing + // information must NOT invalidate evidence; rollover retires it naturally. + assert.equal(currentCalibrationFactor(st, "gpt-x"), 0.5); + assert.equal(currentCalibrationFactor(st, undefined), 0.5); + assert.equal(currentCalibrationFactor({ ...st, calibratedEstimateModel: "gpt-x" }, undefined), 0.5); + // Both known and equal → applies. + assert.equal(currentCalibrationFactor({ ...st, calibratedEstimateModel: "gpt-x" }, "gpt-x"), 0.5); + // Both known and different → absent (the dangerous direction: stale k̂<1 + // would deflate the estimate and delay the trigger on the new model). + assert.equal(currentCalibrationFactor({ ...st, calibratedEstimateModel: "gpt-x" }, "gpt-y"), undefined); +}); + +type PreparedLike = Parameters[0]; +type OptsLike = Parameters[2]; +type Proto = Parameters[0]["protocol"]; + +const OPTS = { routes: {} } as unknown as OptsLike; +const UPSTREAM = "https://upstream.example/v1"; + +function makePrepared(protocol: Proto): PreparedLike { + return { + body: "", + session: { id: "t-2117" }, + processedMessages: [], + originalMessages: [], + protocol, + stream: false, + compressInjected: false, + } as unknown as PreparedLike; +} + +const FIXTURES: Array<[Proto, Record]> = [ + ["anthropic", { model: "m", max_tokens: 100, system: "S".repeat(4000), messages: [{ role: "user", content: [{ type: "text", text: "hello world" }] }], tools: [{ name: "t", description: "D".repeat(2000), input_schema: {} }] }], + ["openai", { model: "m", messages: [{ role: "system", content: "SYS ".repeat(300) }, { role: "user", content: "hello world" }], tools: [{ type: "function", function: { name: "t", description: "D".repeat(2000) } }] }], + ["responses", { model: "m", instructions: "I".repeat(4000), input: [{ type: "message", role: "user", content: "hello world" }], tools: [{ type: "function", name: "t", description: "D".repeat(2000) }] }], + ["google", { model: "m", systemInstruction: { parts: [{ text: "G".repeat(4000) }] }, contents: [{ role: "user", parts: [{ text: "hello world" }] }] }], +]; + +test("outboundContextEstimates: upperBound ≡ legacy outboundContextEstimate, all protocols", () => { + for (const [protocol, obj] of FIXTURES) { + const s = JSON.stringify(obj); + const p = makePrepared(protocol); + const legacy = outboundContextEstimate(p, s, OPTS, UPSTREAM, obj); + const e = outboundContextEstimates(p, s, OPTS, UPSTREAM, obj); + assert.equal(e.upperBound, legacy, `${protocol}: upperBound must equal the legacy value`); + assert.ok(e.upperBound >= e.textOverhead, `${protocol}: the bound must never undershoot the billing caliber`); + assert.ok(e.imageTokens === 0, `${protocol}: no images in the fixtures`); + } +}); + +test("outboundContextEstimates: ASCII-heavy payload — bound over-counts the billing caliber", () => { + const ascii = { model: "m", max_tokens: 100, system: "", messages: [{ role: "user", content: "f".repeat(4000) }] }; + const s = JSON.stringify(ascii); + const e = outboundContextEstimates(makePrepared("openai"), s, OPTS, UPSTREAM, ascii); + // Every character counts as one token in the bound (~4000+) while the + // billing caliber counts ~chars/4 — the repro's 2.1x gap, made sharp. + assert.ok(e.upperBound - e.textOverhead > 2000, `expected a wide gap, got ${e.upperBound} vs ${e.textOverhead}`); +}); + +test("displayContextBest: provenance picks usage > anchored estimate > upper bound", () => { + // Usage-grade reading wins whenever present, carrying its timestamp. + let s = makeSession(); + s.metadata = { contextTokensAt: 1234 }; + s.stats = { ...zeroStats(), contextTokens: 98_000, contextTokensSource: "usage", contextEstimateTokens: 206_000, contextEstimateCalibrated: true }; + assert.deepEqual(displayContextBest(s), { tokens: 98_000, kind: "usage", at: 1234 }); + + // Estimate-grade turn WITH a usage anchor: show the calibrated billing + // estimate of the current send, not the char-count bound. + s = makeSession(); + s.stats = { ...zeroStats(), contextTokens: 206_000, contextTokensSource: "estimate", contextEstimateTokens: 101_000, contextEstimateCalibrated: true, lastUsageGradeTokens: 98_000 }; + assert.deepEqual(displayContextBest(s), { tokens: 101_000, kind: "estimate", calibrated: true }); + + // Uncalibrated estimate keeps the uncalibrated flag. + s = makeSession(); + s.stats = { ...zeroStats(), contextTokens: 206_000, contextTokensSource: "estimate", contextEstimateTokens: 101_000, contextEstimateCalibrated: false, lastUsageGradeTokens: 98_000 }; + assert.deepEqual(displayContextBest(s), { tokens: 101_000, kind: "estimate" }); + + // NEVER-reporting upstream (no anchor): fail-closed — the upper bound + // stays the displayed value (#553/#728 discipline), labeled as a bound. + s = makeSession(); + s.stats = { ...zeroStats(), contextTokens: 206_000, contextTokensSource: "estimate", contextEstimateTokens: 101_000 }; + assert.deepEqual(displayContextBest(s), { tokens: 206_000, kind: "upper" }); + + // Legacy session files (field absent) fall through to today's behavior. + s = makeSession(); + s.stats = { ...zeroStats(), contextTokens: 206_000, contextTokensSource: "estimate" }; + assert.deepEqual(displayContextBest(s), { tokens: 206_000, kind: "upper" }); + + // Nothing observed yet. + s = makeSession(); + assert.equal(displayContextBest(s), null); + + // Usage source drained to zero does not win; the bound remains. + s = makeSession(); + s.stats = { ...zeroStats(), contextTokens: 0, contextTokensSource: "usage" }; + assert.equal(displayContextBest(s), null); +}); + +// One settled pair (pending estimate + its usage report) on a route+model. +function pair(s: Session, est: number, origin: string, billed: number, model?: string): void { + if (model !== undefined) s.metadata = { lastModel: model }; + s.stats.lastLocalTextEstimate = est; + s.stats.lastLocalTextEstimateOrigin = origin; + settleUsageReport(s, { total: billed, reportedCached: null, upstream: origin }); +} + +test("settleUsageReport B: same route+model publishes with model provenance", () => { + const s = makeSession(); + const st = s.stats; + pair(s, 10_000, "http://a", 5_000, "gpt-x"); + assert.equal(st.calibratedEstimate, undefined, "one sample must not publish"); + assert.equal(st.calibrationRing?.model, "gpt-x"); + pair(s, 20_000, "http://a", 11_000, "gpt-x"); + assert.ok(Math.abs((st.calibratedEstimate ?? 0) - 0.525) < 1e-9, `expected mean 0.525, got ${st.calibratedEstimate}`); + assert.equal(st.calibratedEstimateOrigin, "http://a"); + assert.equal(st.calibratedEstimateModel, "gpt-x"); +}); + +test("settleUsageReport B: a model switch starts a fresh ring and clears the old factor", () => { + const s = makeSession(); + const st = s.stats; + pair(s, 10_000, "http://a", 5_000, "gpt-x"); + pair(s, 20_000, "http://a", 11_000, "gpt-x"); + assert.equal(st.calibratedEstimate, 0.525); + + // First sample on the new model must NOT blend into the old evidence nor + // republish the cross-model factor — discard-and-relearn. + pair(s, 10_000, "http://a", 5_000, "gpt-y"); + assert.equal(st.calibratedEstimate, undefined, "cross-model single sample clears the old k̂"); + assert.equal(st.calibratedEstimateModel, undefined); + assert.deepEqual(st.calibrationRing?.values, [0.5]); + assert.equal(st.calibrationRing?.model, "gpt-y"); + + // Two agreeing samples on the new model republish it. + pair(s, 20_000, "http://a", 11_000, "gpt-y"); + assert.equal(st.calibratedEstimate, 0.525); + assert.equal(st.calibratedEstimateModel, "gpt-y"); + + // Switching BACK to gpt-x discards again — no cross-model memory either way. + pair(s, 10_000, "http://a", 5_000, "gpt-x"); + assert.equal(st.calibratedEstimate, undefined); + assert.equal(st.calibrationRing?.model, "gpt-x"); +}); + +test("settleUsageReport B: a legacy ring without a model key restarts on the first known-model sample", () => { + const s = makeSession(); + const st = s.stats; + // Pre-upgrade shape: factor + ring with no model dimension. + st.calibratedEstimate = 0.5; + st.calibratedEstimateOrigin = "http://a"; + st.calibrationRing = { origin: "http://a", values: [0.5, 0.55] }; + pair(s, 10_000, "http://a", 5_000, "gpt-x"); + assert.equal(st.calibratedEstimate, undefined, "legacy evidence must not blend with the new model's"); + assert.deepEqual(st.calibrationRing?.values, [0.5]); + assert.equal(st.calibrationRing?.model, "gpt-x"); +}); + +test("settleUsageReport B: unknown model keeps the legacy behavior (ring continues)", () => { + const s = makeSession(); + const st = s.stats; + // No model stamp anywhere (requests without a model id): rings stay keyed + // by origin alone, exactly as before this change. + pair(s, 10_000, "http://a", 5_000); + pair(s, 20_000, "http://a", 11_000); + assert.ok(Math.abs((st.calibratedEstimate ?? 0) - 0.525) < 1e-9); + assert.equal(st.calibratedEstimateModel, undefined); + assert.equal(st.calibrationRing?.model, undefined); + // A model-stamped sample arriving later restarts the ring. + pair(s, 10_000, "http://a", 5_000, "gpt-x"); + assert.equal(st.calibratedEstimate, undefined); + assert.equal(st.calibrationRing?.model, "gpt-x"); +}); + +test("resetSessionCompression drops the new provenance/display fields at the boundary", () => { + _setStoreForTest(new SessionStore({ enabled: false })); + const s = getSession("i2117-reset"); + const st = s.stats; + st.calibratedEstimate = 0.7; + st.calibratedEstimateOrigin = "http://a"; + st.calibratedEstimateModel = "gpt-x"; + st.calibrationRing = { origin: "http://a", model: "gpt-x", values: [0.7] }; + st.contextEstimateTokens = 12_000; + st.contextEstimateCalibrated = true; + resetSessionCompression(s); + assert.equal(st.calibratedEstimateModel, undefined); + assert.equal(st.calibrationRing, undefined); + assert.equal(st.contextEstimateTokens, undefined); + assert.equal(st.contextEstimateCalibrated, undefined); +}); + +const WINDOW = 20_000; + +function chatSse(text: string, usage?: { prompt_tokens: number; completion_tokens: number }): string { + const chunk = (delta: Record, finish: string | null, extra?: Record): string => + `data: ${JSON.stringify({ id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "gpt-test", choices: [{ index: 0, delta, finish_reason: finish }], ...extra })}\n\n`; + const u = usage ? { usage } : {}; + return chunk({ role: "assistant" }, null) + chunk({ content: text }, null) + chunk({}, "stop", u) + "data: [DONE]\n\n"; +} + +function longMessages(count: number, charsPerMsg: number): Array<{ role: string; content: string }> { + const pad = Math.max(1, charsPerMsg - 12); + const messages: Array<{ role: string; content: string }> = []; + for (let i = 0; i < count; i++) { + messages.push({ role: i % 2 === 0 ? "user" : "assistant", content: `turn ${i}: ` + "f".repeat(pad) }); + } + return messages; +} + +// Bills what it receives: sum(messages[].content chars)/4 x factor — the same +// caliber bili estimates with, scaled to simulate a relay whose tokenizer +// counts fewer tokens than chars/4 (factor < 1 → k̂ learns below 1). +function makeUpstream(factor: number): http.Server { + return http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + try { + const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { stream?: boolean; messages?: Array<{ content?: unknown }> }; + const chars = (parsed.messages ?? []).reduce((n, m) => n + (typeof m.content === "string" ? m.content.length : 0), 0); + const billed = Math.round((chars / 4) * factor); + res.writeHead(200, { "content-type": "text/event-stream" }); + res.end(chatSse("forwarded answer", { prompt_tokens: billed, completion_tokens: 3 })); + return; + } catch { /* fall through */ } + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ id: "chatcmpl-sum", object: "chat.completion", model: "gpt-test", choices: [{ index: 0, message: { role: "assistant", content: "ok" }, finish_reason: "stop" }] })); + }); + }); +} + +test("e2e: forward() publishes both calibers; usage wins the display after settle", async () => { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const upstream = makeUpstream(0.5); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startServer({ + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: WINDOW } } } } as ProxyOptions["routes"], + modelContextLimit: WINDOW, + kernelConfig: defaultConfig(WINDOW, { preserveRecentMessages: 2, preserveRecentTokens: 2000, compress: { minCompressRange: 1000, maxSummaryLength: 20000, minSummaryLength: 50 } }), + compress: { injectTool: true, injectNudge: true }, + sessionHeader: "x-acp-session", + log: false, + debug: false, + passthrough: false, + autoUpdate: false, + compat: { roles: {} }, + streamErrorShape: "protocol", + passthroughSource: null, + autoRestartOnUpdate: false, + updateTag: "latest", + advisoryCheck: false, + releaseNotesCheck: false, + promptCache: { routing: "auto" }, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + const url = `http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/chat/completions`; + const post = (messages: Array<{ role: string; content: string }>) => + fetch(url, { method: "POST", headers: { "content-type": "application/json", "x-acp-session": "i2117-e2e" }, body: JSON.stringify({ model: "gpt-test", stream: true, messages }) }); + + try { + const r1 = await post(longMessages(4, 4000)); + assert.equal(r1.status, 200, `turn 1 must forward: HTTP ${r1.status}`); + await r1.text(); + let s = listSessions().find((x) => x.id === "i2117-e2e"); + assert.ok(s, "session must exist"); + // Post-settle: the measured usage owns the context value... + assert.equal(s!.stats.contextTokensSource, "usage"); + assert.ok((s!.stats.lastInputTokens ?? 0) > 0); + assert.equal(s!.stats.contextTokens, s!.stats.lastInputTokens); + // ...while the per-send estimates survive alongside it. The billing + // caliber of THIS send was written pre-forward and must sit well + // BELOW the char-count bound for this ASCII-heavy payload. + assert.ok(typeof s!.stats.contextEstimateTokens === "number" && s!.stats.contextEstimateTokens > 0, "billing-caliber estimate must be published"); + assert.ok((s!.stats.localInputEstimate ?? 0) > s!.stats.contextEstimateTokens!, `bound (${s!.stats.localInputEstimate}) must exceed the billing caliber (${s!.stats.contextEstimateTokens})`); + assert.equal(s!.stats.contextEstimateCalibrated, false, "no k̂ after one sample"); + // Display: usage wins with its kind. + assert.deepEqual({ tokens: displayContextBest(s!)!.tokens, kind: displayContextBest(s!)!.kind }, { tokens: s!.stats.lastInputTokens, kind: "usage" }); + + // Turn 2 teaches the second sample → k̂ publishes (route+model gpt-test). + const r2 = await post([...longMessages(4, 4000), ...longMessages(4, 4000)]); + assert.equal(r2.status, 200); + await r2.text(); + s = listSessions().find((x) => x.id === "i2117-e2e"); + assert.ok(s!.stats.calibratedEstimate !== undefined && s!.stats.calibratedEstimate! < 1, `two consistent 0.5-ratio samples must publish k̂<1, got ${s!.stats.calibratedEstimate}`); + assert.equal(s!.stats.calibratedEstimateModel, "gpt-test"); + + // Turn 3: the forward-time estimate is scaled by the matching factor. + const r3 = await post([...longMessages(8, 4000), ...longMessages(4, 4000)]); + assert.equal(r3.status, 200); + await r3.text(); + s = listSessions().find((x) => x.id === "i2117-e2e"); + assert.equal(s!.stats.contextEstimateCalibrated, true, "matching route+model factor must scale the displayed estimate"); + assert.ok(s!.stats.contextEstimateTokens! < (s!.stats.localInputEstimate ?? 0) / 2, `calibrated estimate (${s!.stats.contextEstimateTokens}) must be far below the bound (${s!.stats.localInputEstimate})`); + } finally { + proxy.close(); + await once(proxy, "close"); + upstream.close(); + await once(upstream, "close"); + } +}); From df3337ba6287123bd8442bf7a62496cbfd1c1a74 Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 5 Oct 2026 11:36:19 +0800 Subject: [PATCH 2/3] fix: missing model info must not invalidate the calibration ring (#2117 B x #2129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merged-tree CI red: #2129's restart e2e 502s on this branch's head. The restore whitelist drops ring.model / calibratedEstimateModel (#2141 gap), so after a restart the first usage settle saw a model-less ring vs a known settle model, treated it as a cross-model switch, reset the ring, and the insufficient-samples branch deleted the just-restored k̂ — reintroducing the #2122/#2129 incident shape (over-raw resend folds instead of forwarding). The factor gate (currentCalibrationFactor) already follows "missing info never invalidates"; the ring condition now does too: a model-less ring continues and adopts the first known model; a known-vs-known mismatch still starts a fresh ring. Provenance stamps from the ring key so an unknown-model settle cannot blank an established one. Tests: re-pinned the two B-ring cases that asserted discard; the #2129 e2e (forward-without-folds after restart) is the regression pin — green on the merged tree again (68/68 targeted). --- src/cache-ledger.ts | 21 ++++++++++++++----- ...ue2117-context-estimate-provenance.test.ts | 19 +++++++++++------ 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index bc74551652..f6578cd782 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -744,12 +744,20 @@ export function settleUsageReport( // #2117 B: the ring is keyed by route AND model — tokenizers bill // differently across models, so a mid-session model switch starts a // FRESH ring instead of blending two models' billing scales into one - // factor. Legacy rings without a model key restart on the first - // known-model sample (discard-and-relearn: ≤2 samples until the old - // factor's evidence can no longer publish). + // factor. Missing model info NEVER invalidates (same discipline as + // currentCalibrationFactor): a ring without a model key — legacy + // pre-upgrade files included, and records restored after a restart + // whose load whitelist drops the model fields (#2141) — continues + // and ADOPTS the first known model. Discarding it would clear a + // provenance-matched published k̂ on the first post-restart settle + // (the #2129 regression class). const settleModel = typeof session.metadata?.lastModel === "string" && session.metadata.lastModel !== "" ? session.metadata.lastModel : undefined; let ring = session.stats.calibrationRing; - if (!ring || ring.origin !== settleOrigin || ring.model !== settleModel) ring = { origin: settleOrigin, model: settleModel, values: [] }; + if (!ring || ring.origin !== settleOrigin || (ring.model !== undefined && settleModel !== undefined && ring.model !== settleModel)) { + ring = { origin: settleOrigin, model: settleModel, values: [] }; + } else if (ring.model === undefined && settleModel !== undefined) { + ring.model = settleModel; + } if (rawSample >= CALIBRATION_SAMPLE_MIN && rawSample <= CALIBRATION_SAMPLE_MAX) { ring.values.push(rawSample); if (ring.values.length > CALIBRATION_SAMPLE_WINDOW) ring.values.shift(); @@ -758,7 +766,10 @@ export function settleUsageReport( const mean = ring.values.reduce((a, b) => a + b, 0) / ring.values.length; session.stats.calibratedEstimate = Math.min(CALIBRATION_CLAMP_MAX, Math.max(CALIBRATION_CLAMP_MIN, mean)); session.stats.calibratedEstimateOrigin = settleOrigin; - session.stats.calibratedEstimateModel = settleModel; + // Provenance is the ring's model key, not this sample's stamp — + // an unknown-model settle continues a model-keyed ring without + // blanking the established provenance. + session.stats.calibratedEstimateModel = ring.model; } else { delete session.stats.calibratedEstimate; delete session.stats.calibratedEstimateOrigin; diff --git a/tests/issue2117-context-estimate-provenance.test.ts b/tests/issue2117-context-estimate-provenance.test.ts index 811769ccc2..65bc3180b2 100644 --- a/tests/issue2117-context-estimate-provenance.test.ts +++ b/tests/issue2117-context-estimate-provenance.test.ts @@ -191,17 +191,22 @@ test("settleUsageReport B: a model switch starts a fresh ring and clears the old assert.equal(st.calibrationRing?.model, "gpt-x"); }); -test("settleUsageReport B: a legacy ring without a model key restarts on the first known-model sample", () => { +test("settleUsageReport B: a legacy ring without a model key continues and adopts the model (#2129)", () => { const s = makeSession(); const st = s.stats; - // Pre-upgrade shape: factor + ring with no model dimension. + // Pre-upgrade / post-restart shape: published factor + ring with no model + // dimension (the load whitelist drops the model fields — #2141). The k̂ is + // still provenance-matched to the session's model and must survive the + // first known-model settle instead of being discarded-and-cleared. st.calibratedEstimate = 0.5; st.calibratedEstimateOrigin = "http://a"; st.calibrationRing = { origin: "http://a", values: [0.5, 0.55] }; pair(s, 10_000, "http://a", 5_000, "gpt-x"); - assert.equal(st.calibratedEstimate, undefined, "legacy evidence must not blend with the new model's"); - assert.deepEqual(st.calibrationRing?.values, [0.5]); + assert.ok((st.calibratedEstimate ?? 0) > 0, "missing model info must not invalidate a published k̂"); + assert.ok(Math.abs((st.calibratedEstimate ?? 0) - ((0.5 + 0.55 + 0.5) / 3)) < 1e-9, `expected re-published mean, got ${st.calibratedEstimate}`); + assert.deepEqual(st.calibrationRing?.values, [0.5, 0.55, 0.5]); assert.equal(st.calibrationRing?.model, "gpt-x"); + assert.equal(st.calibratedEstimateModel, "gpt-x"); }); test("settleUsageReport B: unknown model keeps the legacy behavior (ring continues)", () => { @@ -214,10 +219,12 @@ test("settleUsageReport B: unknown model keeps the legacy behavior (ring continu assert.ok(Math.abs((st.calibratedEstimate ?? 0) - 0.525) < 1e-9); assert.equal(st.calibratedEstimateModel, undefined); assert.equal(st.calibrationRing?.model, undefined); - // A model-stamped sample arriving later restarts the ring. + // A model-stamped sample arriving later adopts the model on the existing + // ring — no discard (same missing-info discipline as the legacy case). pair(s, 10_000, "http://a", 5_000, "gpt-x"); - assert.equal(st.calibratedEstimate, undefined); + assert.ok(Math.abs((st.calibratedEstimate ?? 0) - ((0.5 + 0.55 + 0.5) / 3)) < 1e-9); assert.equal(st.calibrationRing?.model, "gpt-x"); + assert.equal(st.calibratedEstimateModel, "gpt-x"); }); test("resetSessionCompression drops the new provenance/display fields at the boundary", () => { From 88a008eba25854e367f057bbd1cc3f82c4e5c75a Mon Sep 17 00:00:00 2001 From: ework-agent Date: Mon, 5 Oct 2026 21:56:37 +0800 Subject: [PATCH 3/3] fix: treat missing sessions dir as empty index instead of 500 (#2152) --- src/web/sessions-data.ts | 21 +++++++++++++++++++-- tests/web-sessions.test.ts | 5 +++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/web/sessions-data.ts b/src/web/sessions-data.ts index 94b21af40e..fabb798053 100644 --- a/src/web/sessions-data.ts +++ b/src/web/sessions-data.ts @@ -270,13 +270,30 @@ function liveCoveredPaths(dir: string): Set { * new/changed files (mtime OR size moved) are decoded, one at a time with a * GC checkpoint between files, and the parsed record is dropped immediately * after summary extraction. Top-level walk failure: serve the previous - * snapshot when one exists, else propagate (→ HTTP 500, visible in UI). */ + * snapshot when one exists, else propagate (→ HTTP 500, visible in UI) — + * EXCEPT a MISSING dir with no snapshot yet: that is the legitimate + * "no data" state (fresh install / persistence never used) → empty index, + * mirroring session-gc's empty-result handling (#2152 CI: fresh runner had + * no sessions dir yet and /__bili/overview answered 500). A dir that VANISHES + * after a snapshot exists keeps serving the stale one (#1937). */ async function refreshIndex(): Promise { if (scanInFlight) return scanInFlight; const run = (async () => { const store = getDiskStore(); const dir = store.dir; - const files = await walkSessionFiles(dir); + let files: Array<{ abs: string; mtimeMs: number; size: number }>; + try { + files = await walkSessionFiles(dir); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + if (byFile) return; // vanished dir: keep serving the last known index (#1937) + // Never had one: nothing persisted yet (dir is created lazily on first + // save, persist.ts). Fresh installs must see an empty list, not a 500 + // (#2152 CI) — the old reject-loudly pin conflated "absent" with + // "misconfigured"; the trace now lives here, once per process. + log("warn", `[acp-web] sessions dir ${dir} does not exist yet — serving empty index`); + files = []; + } const prev = byFile ?? new Map(); const next = new Map(); const covered = liveCoveredPaths(dir); diff --git a/tests/web-sessions.test.ts b/tests/web-sessions.test.ts index be74b073d2..22eddb7c23 100644 --- a/tests/web-sessions.test.ts +++ b/tests/web-sessions.test.ts @@ -649,7 +649,7 @@ withSessionsDir("#1937 review: .tmp- mid-name (host label) stays indexed — ker assert.ok(det && det.id === "host-tmp", "detail resolves the same file instead of 404ing"); }); -withSessionsDir("#1937: dir breakage serves stale snapshot; missing dir rejects loudly", async (dir) => { +withSessionsDir("#1937: dir breakage serves stale snapshot; missing dir without snapshot → empty (#2152)", async (dir) => { const store = new SessionStore({ dir, debounceMs: 0, enabled: true }); await store.writeNow(makeSession("seed-1", { protocol: "openai" }, { requests: 2, inputTokens: 10, contextTokens: 9 })); _setStoreForTest(new SessionStore({ enabled: false })); @@ -662,7 +662,8 @@ withSessionsDir("#1937: dir breakage serves stale snapshot; missing dir rejects process.env.BILI_SESSIONS_DIR = path.join(dir, "gone-subdir"); _resetDiskCacheForTest(); - await assert.rejects(buildSessionList(), "no prior snapshot → reject (→ HTTP 500), never silent empty"); + const empty = await buildSessionList(); + assert.deepEqual(empty, [], "no prior snapshot → empty index (#2152), loudness moves to the warn log"); }); withSessionsDir("#1937: /__bili/sessions supports server-side paging & search", async (dir) => {