Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions src/cache-ledger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
} from "acp-kernel";
import { log as loggerLog } from "./logger.js";
import { markDirty, reanchorNudgeOnUsageDrop, type Session } from "./session.js";
import { normalizeUpstreamOrigin } from "./util.js";

// Render window for handleAcpCache's detail:"full" text view (#1489). The
// ledger itself is unbounded — this only bounds how many lines the text
Expand Down Expand Up @@ -628,9 +629,38 @@ export function recordCacheSample(
* caller keeps its own settle-decision gate, log line, collapse watch and
* outputTokens update; `reportedCached === null` means the provider reported
* no cache tokens (recordCacheSample quarantines that sample). */
// #1933 F1: estimator calibration constants. The local chars/4 estimate is a
// proxy whose ratio to real billing varies per upstream (observed 1.3–2.5× on
// one relay vs ~1.0× on another in the same session), so k̂ is learned per
// route and only applied on that route. Samples below MIN are noise (tiny
// requests), clamps bound a single pathological sample from wrecking the EMA
// and pin the correction to the deflate direction only (max 1, see below).
const CALIBRATION_MIN_ESTIMATE = 2000;
// Plausibility band for admitting a sample: outside it, the report and the
// payload it bills demonstrably don't correspond (placeholder billing, relay
// echo, mock upstreams) — a ratio there must never teach a factor.
const CALIBRATION_SAMPLE_MIN = 0.2;
const CALIBRATION_SAMPLE_MAX = 5;
// Final clamp on the published factor: bounds how far calibration can move
// any decision away from the raw estimate. One-way by design: the clamp max
// is 1, so a learned factor can only DEFLATE the estimate (fire later than
// the raw proxy would), never inflate it. Routes whose billing runs ABOVE
// the local estimate (samples >1) publish k̂=1 — legacy raw behavior — and
// stay covered by the overflow arm / learn-on-failure ladder instead. This
// eliminates the class "calibration itself causes an earlier trigger": the
// observed #1933 damage was over-triggering (37% window tax, fold churn),
// while the opposite error already has a backstop. Discussion: PR #1940.
const CALIBRATION_CLAMP_MIN = 0.25;
const CALIBRATION_CLAMP_MAX = 1;
// Evidence requirements: ≥2 recent same-route samples agreeing within ×2.
// One lucky/degenerate pair must not flip every estimate on the route.
const CALIBRATION_SAMPLE_WINDOW = 3;
const CALIBRATION_CONSISTENCY_RATIO = 2;

export function settleUsageReport(
session: Session,
s: { total: number; reportedCached: number | null; output?: number; protocol?: string; upstream?: string },
localTextEstimate?: number,
): void {
// #793: a zero-total sample carries no information (gateway placeholder or
// relay echo) — it must not clobber the last trusted lastInputTokens.
Expand All @@ -646,12 +676,48 @@ export function settleUsageReport(
// compaction boundaries via resetSessionCompression (session.ts).
session.stats.lastUsageGradeTokens = session.stats.lastInputTokens;
session.stats.lastInputTokensSource = "usage";
// #1933 F2: record which route measured this baseline — the gate uses
// it to demote the baseline when the current request routes elsewhere.
const settleOrigin = normalizeUpstreamOrigin(s.upstream);
if (settleOrigin !== undefined) session.stats.lastInputTokensOrigin = settleOrigin;
// #1933 F1: consume the pending pair — the local text estimate of the
// payload THIS report bills, recorded at prepare time. Same route
// required. Admitted samples (plausibility band) accumulate in a
// per-origin ring; the factor is published only once ≥2 recent samples
// agree within ×2, and cleared again when they stop agreeing — until
// then the raw estimate decides (legacy behavior).
const pendingEst = session.stats.lastLocalTextEstimate;
const pendingOrigin = normalizeUpstreamOrigin(session.stats.lastLocalTextEstimateOrigin);
if (pendingEst !== undefined && pendingEst >= CALIBRATION_MIN_ESTIMATE && settleOrigin !== undefined && pendingOrigin === settleOrigin) {
const rawSample = s.total / pendingEst;
let ring = session.stats.calibrationRing;
if (!ring || ring.origin !== settleOrigin) ring = { origin: settleOrigin, values: [] };
if (rawSample >= CALIBRATION_SAMPLE_MIN && rawSample <= CALIBRATION_SAMPLE_MAX) {
ring.values.push(rawSample);
if (ring.values.length > CALIBRATION_SAMPLE_WINDOW) ring.values.shift();
const spread = Math.max(...ring.values) / Math.min(...ring.values);
if (ring.values.length >= 2 && spread <= CALIBRATION_CONSISTENCY_RATIO) {
const mean = ring.values.reduce((a, b) => a + b, 0) / ring.values.length;
session.stats.calibratedEstimate = Math.min(CALIBRATION_CLAMP_MAX, Math.max(CALIBRATION_CLAMP_MIN, mean));
session.stats.calibratedEstimateOrigin = settleOrigin;
} else {
delete session.stats.calibratedEstimate;
delete session.stats.calibratedEstimateOrigin;
}
}
session.stats.calibrationRing = ring;
}
// #1110: a real usage report retires the one-shot overflow arm.
delete session.stats.overflowArmTokens;
// #1595: a real report landing far below a stale-high nudge reference
// retires that reference too (one call covers all three lanes).
reanchorNudgeOnUsageDrop(session);
}
// #1933 F1: roll the pending pair forward to the current turn's value —
// it will be consumed by the NEXT report (or cleared when the lane
// provides none, so a stale pair can never be mispaired).
session.stats.lastLocalTextEstimate = localTextEstimate;
session.stats.lastLocalTextEstimateOrigin = localTextEstimate !== undefined ? normalizeUpstreamOrigin(s.upstream) : undefined;
if (s.reportedCached !== null && s.total > 0) {
session.stats.cachedTokens += s.reportedCached;
session.stats.cacheSamples += 1;
Expand Down
47 changes: 36 additions & 11 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import { proxyDispatcher } from "./upstream-proxy.js";
import { lastCompressSuffix, type Session } from "./session.js";
import { peekRegistryOutputLimit } from "./registry.js";
import { safePrefix } from "./text-safe.js";
import { applyEstimateCalibration } from "./util.js";

// #247: proactive pre-forward compression. When the session's real context
// (previous turn's upstream input_tokens) exceeds the current model's window
Expand Down Expand Up @@ -51,7 +52,7 @@ export const MAX_SUMMARY_CALLS_PER_PREFLIGHT = 16;
// retries the span's only draw against a flaky summarizer kills the whole
// preflight (one content_filter blip bricked an entire turn; the identical
// payload summarized fine ~90s later on the user's manual retry). Retries
// count against MAX_SUMMARY_CALLS_PER_PREFLIGHT like any other call.
// count against the per-invocation summary budget like any other call.
const TRANSIENT_EMPTY_SUMMARY_RETRIES = 2;
// Per-protection-regime cap on wasted transient retries so a SYSTEMIC
// (persistent) empty-summary failure degrades to today's behavior after a
Expand All @@ -68,9 +69,10 @@ const TRANSIENT_EMPTY_RETRY_BUDGET = 4;
// halving worklist can spend several calls on one range without completing a
// fold. Beyond the bound the loop still exits cleanly — the fail-fast reports
// the post-fold size and the remaining compressible-range count, so an
// operator sees exactly how far the budget ran out. 16 is tuned to the
// incident class behind #868 (a 1.39x-window payload); it is a fixed depth,
// not scaled to the overshoot — scaling it is a separate design question.
// operator sees exactly how far the budget ran out. #1933 made the depth
// dynamic: base 16 covers payloads up to ~1.4x the window (one ideal fold
// removes CHUNK_FRACTION x window); larger entry overshoots scale both
// budgets proportionally, capped at 2x the base (see preflightCompress).

export type PreflightProtocol = "anthropic" | "openai" | "responses" | "google";

Expand Down Expand Up @@ -101,6 +103,8 @@ export interface PreflightDeps {
* optimistic chars/4 estimator, which undercounts code/JSON replays by up
* to ~4x and would let an over-window payload slip through uncompressed. */
unknownBaseline?: boolean;
/** #1933 F1: origin of the upstream this request routes to. When it matches the route that learned the session's estimate-calibration factor k̂, every per-round text estimate is scaled by k̂ so gate, per-round exit and final fit judge one payload on the same scale as the trigger that started this invocation; absent or mismatched → raw estimates (legacy behavior). */
upstreamOrigin?: string;
}

export type PreflightFailureKind = "upstream" | "exhausted" | "aborted";
Expand Down Expand Up @@ -842,8 +846,13 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
// kernel's own truncation/absorb behavior stays byte-identical.
const imageReserve = deps.imageReserve ?? 0;
const wireOverhead = deps.wireOverhead ?? 0;
// #1933 F1: capture k̂ once — it only mutates on usage settlement (outside
// this invocation), while every fit judgment in this loop must stay on
// one consistent scale with the gate that started it.
const kFactor = deps.session.stats.calibratedEstimate;
const kOrigin = deps.session.stats.calibratedEstimateOrigin;
let textTarget = Math.max(0, Math.min(limit, deps.compressionTarget ?? limit) - imageReserve);
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: estimateCoreMessages(messages) + imageReserve + wireOverhead, rangesRemaining: 0, fitsWindow: true };
const result: PreflightResult = { compressedRanges: 0, savedTokens: 0, payloadEstimate: applyEstimateCalibration(estimateCoreMessages(messages) + wireOverhead, kFactor, kOrigin, deps.upstreamOrigin) + imageReserve, rangesRemaining: 0, fitsWindow: true };
if (limit <= 0) return result;
const budget = Math.max(MIN_CHUNK_TOKENS, Math.floor(limit * CHUNK_FRACTION));
// applyCompression rejects ranges below config.compress.minCompressRange
Expand Down Expand Up @@ -890,7 +899,19 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
let transientRetryBudget = TRANSIENT_EMPTY_RETRY_BUDGET;
let rangesTried = 0;
let rangesRemaining = 0;
for (let round = 0; round < MAX_PREFLIGHT_ROUNDS; round++) {
// #1933 F3: scale the depth budgets with the entry overshoot (coverage-bound
// note atop the file). A raised budget is only a ceiling — well-behaved
// payloads exit early exactly as before; only genuinely huge payloads burn
// toward it before the fail-fast reports how far it ran out.
const entryLocal = (baselineKnown ? estimateCoreMessages(messages) : estimateCoreMessagesUpper(messages)) + imageReserve + wireOverhead;
const entryTokens = Math.max(baselineKnown ? deps.session.stats.lastInputTokens : 0, entryLocal);
const overshootRatio = limit > 0 && entryTokens > 0 ? entryTokens / limit : 1;
const summaryBudget = Math.min(MAX_SUMMARY_CALLS_PER_PREFLIGHT * 2, Math.max(MAX_SUMMARY_CALLS_PER_PREFLIGHT, Math.ceil(MAX_SUMMARY_CALLS_PER_PREFLIGHT * overshootRatio)));
const roundBudget = Math.min(MAX_PREFLIGHT_ROUNDS * 2, Math.max(MAX_PREFLIGHT_ROUNDS, Math.ceil(MAX_PREFLIGHT_ROUNDS * overshootRatio)));
if (summaryBudget > MAX_SUMMARY_CALLS_PER_PREFLIGHT) {
deps.log("warn", `[preflight] payload ~${entryTokens} tok vs window ${limit} (~${overshootRatio.toFixed(1)}x) — raising summarization budget ${MAX_SUMMARY_CALLS_PER_PREFLIGHT} -> ${summaryBudget}, rounds ${MAX_PREFLIGHT_ROUNDS} -> ${roundBudget}`);
}
for (let round = 0; round < roundBudget; round++) {
if (deps.signal?.aborted) {
failure = ABORTED_FAILURE;
break;
Expand Down Expand Up @@ -927,7 +948,11 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
const baselineFloor = messages.length > 0
? ((deps.session.stats.lastInputTokensSource === "usage" || deps.session.stats.lastInputTokensSource === "overflow-arm") ? deps.session.stats.lastInputTokens : 0)
: deps.session.stats.lastInputTokens;
const roundText = estimateCoreMessages(turn.messages) + wireOverhead;
const rawRoundText = estimateCoreMessages(turn.messages) + wireOverhead;
// #1933 F1: same calibrated caliber as the gate's trigger/fit checks —
// per-round exit and final fit must judge this payload identically to
// the trigger that started the invocation (no mid-loop scale drift).
const roundText = applyEstimateCalibration(rawRoundText, kFactor, kOrigin, deps.upstreamOrigin);
currentTokens = Math.max(baselineFloor, roundText + imageReserve);
// #1843: the TEXT-channel judgment quantity — the usage-grade baseline
// bills images too, so project it onto the text channel by subtracting
Expand Down Expand Up @@ -1116,7 +1141,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
const parts: string[] = [];
const chunks = splitSummaryContent(content, budget, countText);
for (const chunk of chunks) {
if (summaryCalls >= MAX_SUMMARY_CALLS_PER_PREFLIGHT) {
if (summaryCalls >= summaryBudget) {
budgetHit = true;
break;
}
Expand All @@ -1127,7 +1152,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
"unusable" in part && part.transient &&
transientTries < TRANSIENT_EMPTY_SUMMARY_RETRIES &&
transientRetryBudget > 0 &&
summaryCalls < MAX_SUMMARY_CALLS_PER_PREFLIGHT &&
summaryCalls < summaryBudget &&
!deps.signal?.aborted
) {
transientTries += 1;
Expand Down Expand Up @@ -1247,7 +1272,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
const unusableNote = lastUnusableDetail ? ` Last unusable summary: ${safePrefix(lastUnusableDetail, 300)}.` : "";
const skipNote = skipReasons.length > 0 ? ` Skipped: ${skipReasons.slice(0, 3).join(" | ")}.` : "";
if (budgetHit) {
failure = { kind: "exhausted", detail: `the preflight summarization budget (${MAX_SUMMARY_CALLS_PER_PREFLIGHT} calls per protection regime) was exhausted before the payload fit the window${unusableNote}${skipNote}` };
failure = { kind: "exhausted", detail: `the preflight summarization budget (${summaryBudget} calls per protection regime) was exhausted before the payload fit the window${unusableNote}${skipNote}` };
} else if (relaxed && result.compressedRanges > 0) {
failure = { kind: "exhausted", detail: `${relaxedExhaustedDetail}${skipNote}` };
} else if (result.compressedRanges === 0) {
Expand All @@ -1258,7 +1283,7 @@ export async function preflightCompress(deps: PreflightDeps, messages: CoreMessa
? `no viable range could be compressed: ${cause}${unusableNote}`
: `no range could be compressed across ${rangesTried} viable range${rangesTried === 1 ? "" : "s"}: ${cause}${unusableNote}` };
} else {
failure = { kind: "exhausted", detail: `the compress budget was exhausted after ${MAX_PREFLIGHT_ROUNDS} rounds${unusableNote}${skipNote}` };
failure = { kind: "exhausted", detail: `the compress budget was exhausted after ${roundBudget} rounds${unusableNote}${skipNote}` };
}
}
if (result.compressedRanges > 0) {
Expand Down
Loading
Loading