Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/cache-ledger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1980,6 +1980,13 @@ export function settleUsageReport(
// it to demote the baseline when the current request routes elsewhere.
const settleOrigin = normalizeUpstreamOrigin(s.upstream);
if (settleOrigin !== undefined) session.stats.lastInputTokensOrigin = settleOrigin;
// #6430: record which MODEL measured this baseline — the preflight gate
// uses it to demote the baseline when the current request runs on a
// different model (tokenizers bill differently across models),
// mirroring the origin demotion above and the k̂ read gate's model
// dimension. Same source as the k̂ settle model (ingestCalibrationPair).
const settleModel = typeof session.metadata?.lastModel === "string" && session.metadata.lastModel !== "" ? session.metadata.lastModel : undefined;
if (settleModel !== undefined) session.stats.lastInputTokensModel = settleModel;
// #1933 F1: consume the pending pair — the local text estimate of the
// payload THIS report bills, recorded at prepare time. Same route
// required. Admitted samples (plausibility band) accumulate in a
Expand Down
2 changes: 2 additions & 0 deletions src/persist.ts
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ interface PersistedSession {
emergencyShrinkAt?: number;
emergencyShrinkEstimate?: number;
lastInputTokensOrigin?: string;
lastInputTokensModel?: string;
overflowArmTokens?: number;
overflowArmCompletionReserve?: number;
overflowArmMismatchWarned?: boolean;
Expand Down Expand Up @@ -4297,6 +4298,7 @@ function buildSession(parsed: PersistedSession): Session {
localInputEstimate: typeof stats.localInputEstimate === "number" && Number.isFinite(stats.localInputEstimate) && stats.localInputEstimate > 0 ? stats.localInputEstimate : undefined,
lastUsageGradeTokens: typeof stats.lastUsageGradeTokens === "number" && Number.isFinite(stats.lastUsageGradeTokens) && stats.lastUsageGradeTokens >= 0 ? stats.lastUsageGradeTokens : undefined,
lastInputTokensOrigin: typeof stats.lastInputTokensOrigin === "string" && stats.lastInputTokensOrigin.trim().length > 0 ? stats.lastInputTokensOrigin : undefined,
lastInputTokensModel: typeof stats.lastInputTokensModel === "string" && stats.lastInputTokensModel.trim().length > 0 ? stats.lastInputTokensModel : undefined,
calibratedEstimate: restoreCalibratedEstimate(stats.calibratedEstimate),
calibratedEstimateOrigin: typeof stats.calibratedEstimateOrigin === "string" && stats.calibratedEstimateOrigin.trim().length > 0 ? stats.calibratedEstimateOrigin : undefined,
calibratedEstimateModel: typeof stats.calibratedEstimateModel === "string" && stats.calibratedEstimateModel.trim().length > 0 ? stats.calibratedEstimateModel : undefined,
Expand Down
34 changes: 33 additions & 1 deletion src/server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3957,6 +3957,22 @@ export async function preflightCompressIfNeeded(
log("info", `[${session.id}] preflight usage-baseline ~${baselineFloor} tok was measured on ${maskUrlForLog(baselineOrigin)}, request now routes to ${maskUrlForLog(currentOrigin)} — demoting to untrusted, judging by this payload's own estimate (#1933)`);
baselineFloor = 0;
}
// #6430: the baseline's MODEL provenance — a usage-grade baseline measured
// under a different model (same route) is not authoritative for this
// request's billing scale: tokenizers bill differently across models, so a
// cross-model baseline can sit far above (or below) the current model's true
// size (the incident: a 282K opus reading drove preflight on a fable payload
// that billed ~55K). The k̂ read gate (currentCalibrationFactor) already
// demotes on model mismatch; the baseline floor must match it, or a stale
// cross-model baseline keeps arming preflight on the same route. Mirrors the
// #1933 F2 origin demotion above. Absent model (legacy files / unprovenanced)
// keeps the legacy behavior — never invalidated by missing info.
const baselineModel = typeof session.stats.lastInputTokensModel === "string" && session.stats.lastInputTokensModel !== "" ? session.stats.lastInputTokensModel : undefined;
const currentModel = typeof session.metadata?.lastModel === "string" && session.metadata.lastModel !== "" ? session.metadata.lastModel : undefined;
if (baselineFloor > 0 && baselineModel !== undefined && currentModel !== undefined && baselineModel !== currentModel) {
log("info", `[${session.id}] preflight usage-baseline ~${baselineFloor} tok was measured under model ${baselineModel}, request now runs ${currentModel} — demoting to untrusted, judging by this payload's own estimate (#6430)`);
baselineFloor = 0;
}
// #1933 F1: scale the local text estimate by the per-route calibration
// factor k̂ learned from this session's own usage reports (local estimate ÷
// what upstream actually billed; clamped 0.25–4, two-way since #2366 — see
Expand All @@ -3966,7 +3982,23 @@ export async function preflightCompressIfNeeded(
// with a cross-model billing scale (currentCalibrationFactor).
const kFactor = currentCalibrationFactor(session.stats, session.metadata?.lastModel);
const kOrigin = session.stats.calibratedEstimateOrigin;
const calibratedText = applyEstimateCalibration(textEstimate + overheadEstimate, kFactor, kOrigin, currentOrigin);
// #6430: a fresh same-route+same-model usage baseline (not demoted above) is
// the most recent MEASURED reading of this lane's real billed size. When the
// current payload's RAW estimate is not bigger than that reading, the payload
// has not genuinely grown past the last measurement — any excess of the k̂-
// scaled estimate over the baseline is calibration over-correction (a stale
// k̂>1 inflating a payload the provider just billed below the target), not
// real growth. Cap the calibrated text at the measured baseline so a stale k̂
// cannot fire preflight below a fresh measurement: the measurement wins over
// the inflated estimate. Genuine growth (raw > baseline) leaves the
// calibrated estimate untouched, so a payload that really has crossed the
// target still folds; a missed fold here self-heals via the overflow arm on
// the next 400. The cap only applies while the baseline is trusted (same
// route+model); a demoted baseline floors at 0 and the estimate decides.
const rawTextEstimate = textEstimate + overheadEstimate;
const calibratedText = rawTextEstimate <= baselineFloor
? Math.min(applyEstimateCalibration(rawTextEstimate, kFactor, kOrigin, currentOrigin), baselineFloor)
: applyEstimateCalibration(rawTextEstimate, kFactor, kOrigin, currentOrigin);
const calibratedPayload = calibratedText + imageTokens;
const tokenCount = unknownBaseline
? estimateCoreMessagesUpper(prepared.processedMessages) + overheadEstimate + imageTokens
Expand Down
4 changes: 4 additions & 0 deletions src/server/relay.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1756,6 +1756,9 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare
// probe-forward gate) treat the arm as current-route evidence
// instead of demoting it against a stale settle origin.
s.stats.lastInputTokensOrigin = normalizeUpstreamOrigin(upstreamUrl);
// #6430: model provenance for the armed baseline — the preflight gate
// demotes it when the current request runs on a different model.
if (reqModel !== undefined) s.stats.lastInputTokensModel = reqModel;
// #1110: record the arm SEPARATELY so the side-request guard
// can read it without ever touching the nudge baseline. The guard
// keeps the CEILING even when the baseline books a measured or
Expand Down Expand Up @@ -1832,6 +1835,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare
s.stats.lastInputTokensSource = "overflow-arm";
s.stats.overflowArmTokens = arm; // #1110: guard reads this, not the baseline
s.stats.lastInputTokensOrigin = normalizeUpstreamOrigin(upstreamUrl); // #2313: same-route evidence, see the window branch above
if (reqModel !== undefined) s.stats.lastInputTokensModel = reqModel; // #6430: model provenance, see the window branch above
}
log("warn", `[${s.id}] upstream context overflow (window not parseable, model=${reqModel ?? "unknown"}) — armed emergency shrink at ~${arm} tokens (min of judging window ${judging} and payload estimate), nothing learned (#987): ${info.message}`);
}
Expand Down
14 changes: 14 additions & 0 deletions src/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -554,6 +554,18 @@ export type Session = {
* provider says nothing about whether another accepts the payload.
* Absent on legacy session files → baseline keeps legacy behavior. */
lastInputTokensOrigin?: string;
/** #6430: model id that measured lastInputTokens — written by
* settleUsageReport (and the overflow-arm paths) alongside the
* origin. The preflight gate demotes a usage-grade baseline to
* untrusted when the CURRENT request runs on a different model:
* tokenizers bill differently across models, so a cross-model
* baseline (e.g. a 282K opus reading on a fable lane) is not
* authoritative for this request's size. The k̂ read gate
* (currentCalibrationFactor) already demotes on model mismatch;
* the baseline floor must match it, or a stale cross-model
* baseline keeps arming preflight on the same route. Absent on
* legacy session files → baseline keeps legacy behavior. */
lastInputTokensModel?: string;
/** #1933 F1: calibrated scale factor k̂ = mean of up to 3 recent
* consistent same-route samples of (upstream-billed input ÷ local
* text estimate), clamped to [0.25, 4] — two-way since #2366: the
Expand Down Expand Up @@ -1807,6 +1819,8 @@ export function resetSessionCompression(session: Session): void {
// learned against the pre-compaction content class, and the baseline's
// measuring route no longer bounds this rebuilt session.
delete session.stats.lastInputTokensOrigin;
// #6430: the baseline's model provenance dies with the baseline.
delete session.stats.lastInputTokensModel;
delete session.stats.calibratedEstimate;
delete session.stats.calibratedEstimateOrigin;
// #2117 B: same boundary — the factor's model provenance dies with it.
Expand Down
6 changes: 3 additions & 3 deletions tests/golden/storage-render/anthropic.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@
"foldAt": 9,
"editedAt": 12,
"summaryFrom": 9,
"stateDigest": "ba0f75f6ddc882d00d5c456facd79166b62e3f2cacadd60744aa370d6b247105",
"stateDigestAligned": "66991d5cb8864f3ca029914aa9afc20cc8b9687cfd4391ffe200017c68057946",
"stateDigest": "24645d604a48c016a7c75e3bc60ce2e9e561c6079b24730f47cc9401b2d75092",
"stateDigestAligned": "a38b6f5cbc79779d6eb4955aaa46e0deb3e7183d966f2565c67c54fd3d0d8a34",
"sectionDigests": {
"state": "37b26b4f6898afff",
"blockContents": "dc7d13d250c94bb4",
"stats": "7caf981f99159be9",
"stats": "b679dfaeba151fb7",
"metadata": "ba51746c6749d0cf"
},
"stateKeyDigests": {
Expand Down
6 changes: 3 additions & 3 deletions tests/golden/storage-render/chat.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@
"foldAt": 9,
"editedAt": 12,
"summaryFrom": 9,
"stateDigest": "2c0b36206756617ae1e32cf0d221ddd8bf7f3e06994b49d11578bd1a35c90f26",
"stateDigestAligned": "3011800706759958cfeab6520289f1ffe257cf4f7c3275d846c03a991f9ed6b8",
"stateDigest": "7a6fdf15f2da6391812ccedd5a3addb3892a1005b73b299a3e7dd859ed961320",
"stateDigestAligned": "5b1a8b4a89ae359fa131b730a12111b5f0c39738524f366e016cd4acb369c9f9",
"sectionDigests": {
"state": "90857fbf82001a2b",
"blockContents": "94a58f96d7c5cfc3",
"stats": "7b6162aef3873d31",
"stats": "a41a71c5ff86bd68",
"metadata": "7218c1e3430c3d21"
},
"stateKeyDigests": {
Expand Down
6 changes: 3 additions & 3 deletions tests/golden/storage-render/google.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@
"foldAt": 9,
"editedAt": 12,
"summaryFrom": 9,
"stateDigest": "c5eefef6525f180ab54a7ea57c4ddf004a568db0fc2d6d227ede31dc960706db",
"stateDigestAligned": "5217aa02331256a419837cd91105d3e3172f2ff38f0f3914c0e01128e221f012",
"stateDigest": "aff30a310dbfeddfd9a22eade43ef9ebe82e5cffc19d10d4a26fb1b6db6e2f75",
"stateDigestAligned": "fbc2418aac73d5811ee7b230b16623facd546461e5f515439416c2979fb669e5",
"sectionDigests": {
"state": "c5243ba17364d3ac",
"blockContents": "473264d9df8ba338",
"stats": "c90671e484dc5d37",
"stats": "ff732e39f6f51bfd",
"metadata": "2980a8959aa0a39c"
},
"stateKeyDigests": {
Expand Down
6 changes: 3 additions & 3 deletions tests/golden/storage-render/responses.json
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,12 @@
"foldAt": 9,
"editedAt": 12,
"summaryFrom": 9,
"stateDigest": "ff092bc66e11c35b15ad98acf6206629115825b62168da2e5542e06297f6d47a",
"stateDigestAligned": "db5f08a7d57c14952c31571e45b4dbc6c2ed35fe8a97c58cda6fa854d411eb25",
"stateDigest": "22465866fd11ff6734f2d92677990084cc4f55da58170b2aed751b89be8c35c2",
"stateDigestAligned": "2c0184430ab851734090169ee860ca93f6fb97b06df272aa44604f8fedb4a024",
"sectionDigests": {
"state": "bee6e1ba0c6fba62",
"blockContents": "b6701268a0bdcd8a",
"stats": "1b56c993f03cd122",
"stats": "9e8f281283118864",
"metadata": "77714417e484ba89"
},
"stateKeyDigests": {
Expand Down
2 changes: 2 additions & 0 deletions tests/issue2129-restart-measurement-state.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ test("measurement-state stats survive a reload round-trip (#2129)", async () =>
st.localInputEstimate = 12345;
st.lastUsageGradeTokens = 9876;
st.lastInputTokensOrigin = "http://127.0.0.1:8787";
st.lastInputTokensModel = "claude-opus-5-5";
st.calibratedEstimate = 0.4375;
st.calibratedEstimateOrigin = "http://127.0.0.1:8787";
st.calibrationRing = { origin: "http://127.0.0.1:8787", values: [0.4, 0.45, 0.48] };
Expand All @@ -75,6 +76,7 @@ test("measurement-state stats survive a reload round-trip (#2129)", async () =>
assert.equal(back.stats.localInputEstimate, 12345, "localInputEstimate round-trips");
assert.equal(back.stats.lastUsageGradeTokens, 9876, "lastUsageGradeTokens round-trips");
assert.equal(back.stats.lastInputTokensOrigin, "http://127.0.0.1:8787", "lastInputTokensOrigin round-trips");
assert.equal(back.stats.lastInputTokensModel, "claude-opus-5-5", "lastInputTokensModel round-trips (#6430)");
assert.equal(back.stats.calibratedEstimate, 0.4375, "calibratedEstimate round-trips");
assert.equal(back.stats.calibratedEstimateOrigin, "http://127.0.0.1:8787", "calibratedEstimateOrigin round-trips");
assert.deepEqual(back.stats.calibrationRing, { origin: "http://127.0.0.1:8787", values: [0.4, 0.45, 0.48] }, "calibrationRing round-trips");
Expand Down
8 changes: 4 additions & 4 deletions tests/issue2189-preflight-billing-block.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ test("#2189 anthropic summary carries the client billing-attribution block at sy
const { proxy, upstream, proxyPort, upstreamPort } = await makePair(calls, 200, "");
try {
const r1 = await drive(proxyPort, upstreamPort, "s2189-main", {
model: "claude-big", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }],
model: "claude-small", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }],
});
assert.equal(r1.status, 200);
await r1.text();
Expand Down Expand Up @@ -233,7 +233,7 @@ test("#2189 legacy string system keeps the byte-identical summary shape", async
try {
const sys = "You are a helpful assistant.";
const r1 = await drive(proxyPort, upstreamPort, "s2189-legacy", {
model: "claude-big", max_tokens: 1024, stream: true, system: sys, messages: [{ role: "user", content: "hello" }],
model: "claude-small", max_tokens: 1024, stream: true, system: sys, messages: [{ role: "user", content: "hello" }],
});
assert.equal(r1.status, 200);
await r1.text();
Expand Down Expand Up @@ -268,7 +268,7 @@ test("#2189 bare-Error 429 fails fast after ONE attempt, retryable=false, retrie
const { proxy, upstream, proxyPort, upstreamPort } = await makePair(calls, 429, SHAPE_429_BODY, true);
try {
const r1 = await drive(proxyPort, upstreamPort, "s2189-shape", {
model: "claude-big", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }],
model: "claude-small", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }],
});
assert.equal(r1.status, 200);
await r1.text();
Expand Down Expand Up @@ -314,7 +314,7 @@ test("#2189 a genuine rate limit keeps the legacy 3-attempt replay and retryable
const { proxy, upstream, proxyPort, upstreamPort } = await makePair(calls, 429, GENUINE_429_BODY, true);
try {
const r1 = await drive(proxyPort, upstreamPort, "s2189-genuine", {
model: "claude-big", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }],
model: "claude-small", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }],
});
assert.equal(r1.status, 200);
await r1.text();
Expand Down
Loading
Loading