Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion src/cache-ledger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -920,8 +920,23 @@ export function settleUsageReport(
const pendingOrigin = normalizeUpstreamOrigin(session.stats.lastLocalTextEstimateOrigin);
if (pendingEst !== undefined && pendingEst >= CALIBRATION_MIN_ESTIMATE && settleOrigin !== undefined && pendingOrigin === settleOrigin) {
const rawSample = s.total / pendingEst;
// #2117 B: the ring is keyed by route AND model — tokenizers bill
// differently across models, so a mid-session model switch starts a
// FRESH ring instead of blending two models' billing scales into one
// factor. Missing model info NEVER invalidates (same discipline as
// currentCalibrationFactor): a ring without a model key — legacy
// pre-upgrade files included, and records restored after a restart
// whose load whitelist drops the model fields (#2141) — continues
// and ADOPTS the first known model. Discarding it would clear a
// provenance-matched published k̂ on the first post-restart settle
// (the #2129 regression class).
const settleModel = typeof session.metadata?.lastModel === "string" && session.metadata.lastModel !== "" ? session.metadata.lastModel : undefined;
let ring = session.stats.calibrationRing;
if (!ring || ring.origin !== settleOrigin) ring = { origin: settleOrigin, values: [] };
if (!ring || ring.origin !== settleOrigin || (ring.model !== undefined && settleModel !== undefined && ring.model !== settleModel)) {
ring = { origin: settleOrigin, model: settleModel, values: [] };
} else if (ring.model === undefined && settleModel !== undefined) {
ring.model = settleModel;
}
if (rawSample >= CALIBRATION_SAMPLE_MIN && rawSample <= CALIBRATION_SAMPLE_MAX) {
ring.values.push(rawSample);
if (ring.values.length > CALIBRATION_SAMPLE_WINDOW) ring.values.shift();
Expand All @@ -930,9 +945,14 @@ export function settleUsageReport(
const mean = ring.values.reduce((a, b) => a + b, 0) / ring.values.length;
session.stats.calibratedEstimate = Math.min(CALIBRATION_CLAMP_MAX, Math.max(CALIBRATION_CLAMP_MIN, mean));
session.stats.calibratedEstimateOrigin = settleOrigin;
// Provenance is the ring's model key, not this sample's stamp —
// an unknown-model settle continues a model-keyed ring without
// blanking the established provenance.
session.stats.calibratedEstimateModel = ring.model;
} else {
delete session.stats.calibratedEstimate;
delete session.stats.calibratedEstimateOrigin;
delete session.stats.calibratedEstimateModel;
}
}
session.stats.calibrationRing = ring;
Expand Down
6 changes: 4 additions & 2 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import { proxyDispatcher } from "./upstream-proxy.js";
import { lastCompressSuffix, type Session } from "./session.js";
import { peekRegistryOutputLimit } from "./registry.js";
import { safePrefix } from "./text-safe.js";
import { applyEstimateCalibration } from "./util.js";
import { applyEstimateCalibration, currentCalibrationFactor } from "./util.js";

// #247: proactive pre-forward compression. When the session's real context
// (previous turn's upstream input_tokens) exceeds the current model's window
Expand Down Expand Up @@ -929,7 +929,9 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
// #1933 F1: capture k̂ once — it only mutates on usage settlement (outside
// this invocation), while every fit judgment in this loop must stay on
// one consistent scale with the gate that started it.
const kFactor = deps.session.stats.calibratedEstimate;
// #2117 B: model-provenance gate — a factor learned on another model acts
// as absent (raw estimate) rather than deciding with a cross-model scale.
const kFactor = currentCalibrationFactor(deps.session.stats, deps.session.metadata?.lastModel);
const kOrigin = deps.session.stats.calibratedEstimateOrigin;
let textTarget = Math.max(0, Math.min(limit, deps.compressionTarget ?? limit) - imageReserve);
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: applyEstimateCalibration(estimateCoreMessages(messages) + wireOverhead, kFactor, kOrigin, deps.upstreamOrigin) + imageReserve, rangesRemaining: 0, fitsWindow: true };
Expand Down
69 changes: 59 additions & 10 deletions src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ import { consumePluginRegisterFor, flushConversations, handlePluginCompact, hand
import { setupMitm, readMitmUpstream, getBlindTunnelStats, liveBlindTunnels, MITM_RAW_SOCKET_KEY } from "./mitm.js";
import { evaluateChain, extractChainCarriers, stampOutbound, stripEmbeddedChainCarriers } from "./chain-checkpoint.js";
import type { BiliMessage } from "acp-kernel/wire";
import { appendSystemText, applyEstimateCalibration, BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, hardenOpenaiAssistantContent, isLoopbackAddress, inspectContextOverflow, normalizeUpstreamOrigin, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, systemToUser, usageOutputTotal, usageTotals, strippedResponseIdWarning, type ContextOverflowInfo, type WireProtocol } from "./util.js";
import { appendSystemText, applyEstimateCalibration, currentCalibrationFactor, BILI_PASSTHROUGH_HEADER, BILI_PLUGIN_BYPASS_HEADER, hardenOpenaiAssistantContent, isLoopbackAddress, inspectContextOverflow, normalizeUpstreamOrigin, reserveOutputHeadroom, resolveOutputHeadroomCap, shouldReserveOutputHeadroom, systemToUser, usageOutputTotal, usageTotals, strippedResponseIdWarning, type ContextOverflowInfo, type WireProtocol } from "./util.js";
import { safePrefix, safeSuffix } from "./text-safe.js";

import { BILI_TUNNEL_HEADER, checkTunnelDestination, classifyIp, localMachineIps, normalizeIpLiteral, parseIpLiteral, tunnelAllowlistFromEnv } from "./tunnel-guard.js";
Expand Down Expand Up @@ -5738,13 +5738,33 @@ function outboundPayloadBreakdown(
* project from it (numbers identical to re-parsing the same string), null =
* caller already tried and failed → keep the prepared projection and skip the
* doomed re-parses inside the helpers (they return 0 on unparseable input). */
export function outboundContextEstimate(
/** #2117: both calibers of one outbound send, computed in ONE projection pass.
* upperBound is the char-count upper bound (every character counts as one
* token — never undershoots; the fail-closed decision/display caliber, and
* the legacy outboundContextEstimate value). textOverhead is the billing
* caliber preflight calibrates with k̂ (#1933 F1): CJK-aware
* estimateCoreMessages + system/tools wire overhead. imageTokens is the
* separate image term (learned per-route #1843/#1857), kept out of both so
* callers can recombine per channel. */
export interface OutboundContextEstimates {
upperBound: number;
textOverhead: number;
imageTokens: number;
}

/** #2078: `parsed` carries the caller's pre-parsed send body so forward() does
* not pay a second full JSON.parse of the largest payload in flight per
* request. Three states: undefined = parse here (legacy callers), object =
* project from it (numbers identical to re-parsing the same string), null =
* caller already tried and failed → keep the prepared projection and skip the
* doomed re-parses inside the helpers (they return 0 on unparseable input). */
export function outboundContextEstimates(
prepared: Prepared,
wireBody: string,
opts: ProxyOptions,
upstream: string,
parsed?: Record<string, unknown> | null,
): number {
): OutboundContextEstimates {
let msgs = prepared.processedMessages.length > 0 ? prepared.processedMessages : prepared.originalMessages;
const project = (value: unknown): void => {
switch (prepared.protocol) {
Expand All @@ -5764,8 +5784,23 @@ export function outboundContextEstimate(
} else {
raw = "";
}
return estimateCoreMessagesUpper(msgs) + estimateWireOverhead(prepared.protocol, raw)
+ imageReserveFor(prepared.session, prepared.protocol, raw, opts, upstream);
const overhead = estimateWireOverhead(prepared.protocol, raw);
const imageTokens = imageReserveFor(prepared.session, prepared.protocol, raw, opts, upstream);
return {
upperBound: estimateCoreMessagesUpper(msgs) + overhead + imageTokens,
textOverhead: estimateCoreMessages(msgs) + overhead,
imageTokens,
};
}

export function outboundContextEstimate(
prepared: Prepared,
wireBody: string,
opts: ProxyOptions,
upstream: string,
parsed?: Record<string, unknown> | null,
): number {
return outboundContextEstimates(prepared, wireBody, opts, upstream, parsed).upperBound;
}

async function preflightCompressIfNeeded(
Expand Down Expand Up @@ -5845,8 +5880,10 @@ async function preflightCompressIfNeeded(
// factor k̂ learned from this session's own usage reports (local estimate ÷
// what upstream actually billed, EMA, clamped 0.25–1 — one-way, deflate
// only; see settleUsageReport). Unknown/mismatched origin → raw estimate,
// i.e. today's behavior.
const kFactor = session.stats.calibratedEstimate;
// i.e. today's behavior. #2117 B: the model dimension gates too — a factor
// learned on another model acts as absent here rather than deciding with a
// cross-model billing scale (currentCalibrationFactor).
const kFactor = currentCalibrationFactor(session.stats, session.metadata?.lastModel);
const kOrigin = session.stats.calibratedEstimateOrigin;
const calibratedText = applyEstimateCalibration(textEstimate + overheadEstimate, kFactor, kOrigin, currentOrigin);
const calibratedPayload = calibratedText + imageTokens;
Expand Down Expand Up @@ -6504,9 +6541,21 @@ async function forward(
sentParsed = null;
}
// Publish this send, not a historical usage baseline with a fresh timestamp.
const estimate = outboundContextEstimate(prepared, sentBody, opts, upstreamUrl, sentParsed);
prepared.session.stats.localInputEstimate = estimate;
prepared.session.stats.contextTokens = estimate;
const est = outboundContextEstimates(prepared, sentBody, opts, upstreamUrl, sentParsed);
prepared.session.stats.localInputEstimate = est.upperBound;
// The char-count upper bound stays the published context value: it is
// the fail-closed caliber decision paths and legacy displays trust for
// never-reporting upstreams (#553/#728/#1493). #2117 additionally
// publishes the billing-caliber estimate of THIS send — preflight's own
// formula (CJK-aware text + wire overhead), k̂-scaled only where the
// factor's route+model provenance matches (#1933 F1 / #2117 B), plus
// the image reserve — so display surfaces can show a calibrated reading
// instead of the ~2–3.5× over-counting bound. Display-only field.
const kFactor = currentCalibrationFactor(prepared.session.stats, prepared.session.metadata?.lastModel);
const scaledText = applyEstimateCalibration(est.textOverhead, kFactor, prepared.session.stats.calibratedEstimateOrigin, normalizeUpstreamOrigin(upstreamUrl));
prepared.session.stats.contextEstimateTokens = Math.round(scaledText + est.imageTokens);
prepared.session.stats.contextEstimateCalibrated = scaledText !== est.textOverhead;
prepared.session.stats.contextTokens = est.upperBound;
prepared.session.stats.contextTokensSource = "estimate";
// #2131: exact message count for seam forensics (sentParsed is the one
// shared parse above — Responses carries "input", Google native "contents").
Expand Down
78 changes: 73 additions & 5 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,17 @@ export type Session = {
* Cleared by resetSessionCompression (native-compaction boundary).
* Persisted (survives restart like the rest of stats). */
localInputEstimate?: number;
/** #2117: billing-caliber estimate of the CURRENT outbound request —
* estimateCoreMessages + wire overhead, k̂-scaled only where the
* factor's route+model provenance matches (#1933 F1 / #2117 B), plus
* the image reserve. Written by forward() alongside the char-count
* upper bound above (which stays the fail-closed caliber for
* never-reporting upstreams, #553/#728). Display-only: no decision
* path reads it — preflight keeps its own per-invocation formula. */
contextEstimateTokens?: number;
/** #2117: whether k̂ actually scaled contextEstimateTokens (false when
* the factor is absent or its provenance doesn't match). */
contextEstimateCalibrated?: boolean;
/** #1569: last netted input value written by a REAL upstream usage
* report (the arming paths never touch it — they only pose as
* usage-grade for lastInputTokens). While > 0, effectiveTokenCount
Expand Down Expand Up @@ -201,11 +212,19 @@ export type Session = {
* from a different origin starts a fresh ring instead of blending
* two providers' scales. */
calibratedEstimateOrigin?: string;
/** #1933 F1: evidence ring behind calibratedEstimate — the recent
* admitted raw samples for ONE origin (max 3). Persisted so a
* restart doesn't re-arm the warmup delay; reset at native-
* compaction boundaries with the rest of the baseline stats. */
calibrationRing?: { origin: string; values: number[] };
/** #2117 B: model the calibratedEstimate was learned against — the
* second provenance dimension. The ring and every read gate on
* route AND model: a mid-session model switch invalidates the old
* factor instead of reusing a cross-model billing scale (dangerous
* in either direction — a stale k̂<1 deflates → trigger too late).
* Absent on factors published before this field existed → reads
* treat it as compatible and retire it through normal rollover. */
calibratedEstimateModel?: string;
/** #1933 F1 / #2117 B: evidence ring behind calibratedEstimate —
* the recent admitted raw samples for ONE origin+model (max 3).
* Persisted so a restart doesn't re-arm the warmup delay; reset at
* native-compaction boundaries with the rest of the baseline stats. */
calibrationRing?: { origin: string; model?: string; values: number[] };
/** #1933 F1: pending pairing input — local estimate of the LAST
* prepared outbound in BILLED caliber (estimateCoreMessages +
* system/tools overhead + image reserve, defaultCountTokens rate),
Expand Down Expand Up @@ -430,6 +449,49 @@ export function statusInputBaseline(session: Session): number {
: stats.lastUsageGradeTokens ?? 0;
}

/** #2117: the honest best reading of "how full is this context right now" for
* DISPLAY surfaces, picked by provenance — never mixes calibers on one bar:
* usage — last upstream-measured input (billing grade); wins whenever present.
* estimate — the billing-caliber estimate of the CURRENT outbound request
* (contextEstimateTokens, k̂-scaled where its route+model
* provenance matches), shown only while a usage-grade anchor
* exists: a never-reporting upstream has no evidence that the
* optimistic rate holds, so it keeps the fail-closed upper bound
* (#553/#728 discipline).
* upper — char-count upper bound of the last send: never undershoots,
* but over-counts ASCII-heavy payloads up to ~3.5× — a BOUND,
* not a reading; over-window here does NOT mean actually
* over-window.
* Legacy sessions without contextEstimateTokens fall through to upper —
* exactly today's display. Read-only; decision paths are untouched. */
export interface ContextBest {
tokens: number;
kind: "usage" | "estimate" | "upper";
/** true when kind=="estimate" and k̂ actually scaled the value. */
calibrated?: boolean;
/** wall-clock of the underlying measurement (usage reports stamp it). */
at?: number;
}

// #2117 x #1937: minimal structural param — web list rows build from bounded
// summary sources (sessions-data.ts) that carry no full Session object.
export function displayContextBest(session: {
stats: Pick<Session["stats"], "contextTokens" | "contextTokensSource" | "lastUsageGradeTokens" | "lastInputTokensSource" | "contextEstimateTokens" | "contextEstimateCalibrated">;
metadata?: Record<string, unknown>;
}): ContextBest | null {
const st = session.stats;
if (st.contextTokensSource === "usage" && st.contextTokens > 0) {
const at = typeof session.metadata?.contextTokensAt === "number" ? session.metadata.contextTokensAt : undefined;
return { tokens: st.contextTokens, kind: "usage", ...(at !== undefined ? { at } : {}) };
}
const anchored = (st.lastUsageGradeTokens ?? 0) > 0 || st.lastInputTokensSource === "usage";
if (anchored && typeof st.contextEstimateTokens === "number" && st.contextEstimateTokens > 0) {
return { tokens: st.contextEstimateTokens, kind: "estimate", ...(st.contextEstimateCalibrated ? { calibrated: true } : {}) };
}
if (st.contextTokens > 0) return { tokens: st.contextTokens, kind: "upper" };
return null;
}

/** Mirror of acp-kernel's resolveAdaptiveGrowth (not exported by the kernel):
* min(growthCap, max(growthFloor, round(modelContextLimit × growthRatio))).
* The per-request config stamped by storeEffectiveConfig is the same object
Expand Down Expand Up @@ -833,9 +895,15 @@ export function resetSessionCompression(session: Session): void {
delete session.stats.lastInputTokensOrigin;
delete session.stats.calibratedEstimate;
delete session.stats.calibratedEstimateOrigin;
// #2117 B: same boundary — the factor's model provenance dies with it.
delete session.stats.calibratedEstimateModel;
delete session.stats.calibrationRing;
delete session.stats.lastLocalTextEstimate;
delete session.stats.lastLocalTextEstimateOrigin;
// #2117: the per-send estimate describes the pre-compaction payload too —
// let the next forward re-measure instead of displaying a stale reading.
delete session.stats.contextEstimateTokens;
delete session.stats.contextEstimateCalibrated;
session.stats.contextTokens = 0;
delete session.stats.contextTokensSource;
session.metadata.nativeCompactionAt = Date.now();
Expand Down
17 changes: 17 additions & 0 deletions src/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,23 @@ export function applyEstimateCalibration(raw: number, k: number | undefined, kOr
return raw * k;
}

/** #2117: read-side gate for the learned estimator scale — the model dimension
* of k̂ provenance. A factor published against model X must not decide on
* model Y: tokenizers bill differently, and a stale cross-model factor can be
* dangerously wrong in EITHER direction (k̂<1 deflates → trigger too LATE).
* Both models known (non-empty strings) and different → factor absent (raw
* estimate, legacy conservative behavior); either side unknown/empty/non-
* string → factor kept as-is, so factors published before this field existed
* (or requests without a model stamp) are never invalidated by missing
* information — they retire through the normal ring rollover when new samples
* land. Invalid factors degrade to absent. */
export function currentCalibrationFactor(stats: { calibratedEstimate?: number; calibratedEstimateModel?: string }, lastModel?: unknown): number | undefined {
const k = stats.calibratedEstimate;
if (k === undefined || !Number.isFinite(k) || k <= 0) return undefined;
if (stats.calibratedEstimateModel !== undefined && typeof lastModel === "string" && lastModel !== "" && stats.calibratedEstimateModel !== lastModel) return undefined;
return k;
}

export type WireProtocol = "anthropic" | "openai" | "responses" | "google";

/**
Expand Down
Loading
Loading