diff --git a/src/cache-ledger.ts b/src/cache-ledger.ts index abd5888910..0da552304a 100644 --- a/src/cache-ledger.ts +++ b/src/cache-ledger.ts @@ -1980,6 +1980,13 @@ export function settleUsageReport( // it to demote the baseline when the current request routes elsewhere. const settleOrigin = normalizeUpstreamOrigin(s.upstream); if (settleOrigin !== undefined) session.stats.lastInputTokensOrigin = settleOrigin; + // #6430: record which MODEL measured this baseline — the preflight gate + // uses it to demote the baseline when the current request runs on a + // different model (tokenizers bill differently across models), + // mirroring the origin demotion above and the k̂ read gate's model + // dimension. Same source as the k̂ settle model (ingestCalibrationPair). + const settleModel = typeof session.metadata?.lastModel === "string" && session.metadata.lastModel !== "" ? session.metadata.lastModel : undefined; + if (settleModel !== undefined) session.stats.lastInputTokensModel = settleModel; // #1933 F1: consume the pending pair — the local text estimate of the // payload THIS report bills, recorded at prepare time. Same route // required. Admitted samples (plausibility band) accumulate in a diff --git a/src/persist.ts b/src/persist.ts index cbc0059ea2..4612bd9183 100644 --- a/src/persist.ts +++ b/src/persist.ts @@ -205,6 +205,7 @@ interface PersistedSession { emergencyShrinkAt?: number; emergencyShrinkEstimate?: number; lastInputTokensOrigin?: string; + lastInputTokensModel?: string; overflowArmTokens?: number; overflowArmCompletionReserve?: number; overflowArmMismatchWarned?: boolean; @@ -4297,6 +4298,7 @@ function buildSession(parsed: PersistedSession): Session { localInputEstimate: typeof stats.localInputEstimate === "number" && Number.isFinite(stats.localInputEstimate) && stats.localInputEstimate > 0 ? stats.localInputEstimate : undefined, lastUsageGradeTokens: typeof stats.lastUsageGradeTokens === "number" && Number.isFinite(stats.lastUsageGradeTokens) && stats.lastUsageGradeTokens >= 0 ? stats.lastUsageGradeTokens : undefined, lastInputTokensOrigin: typeof stats.lastInputTokensOrigin === "string" && stats.lastInputTokensOrigin.trim().length > 0 ? stats.lastInputTokensOrigin : undefined, + lastInputTokensModel: typeof stats.lastInputTokensModel === "string" && stats.lastInputTokensModel.trim().length > 0 ? stats.lastInputTokensModel : undefined, calibratedEstimate: restoreCalibratedEstimate(stats.calibratedEstimate), calibratedEstimateOrigin: typeof stats.calibratedEstimateOrigin === "string" && stats.calibratedEstimateOrigin.trim().length > 0 ? stats.calibratedEstimateOrigin : undefined, calibratedEstimateModel: typeof stats.calibratedEstimateModel === "string" && stats.calibratedEstimateModel.trim().length > 0 ? stats.calibratedEstimateModel : undefined, diff --git a/src/server.ts b/src/server.ts index 3fb954b5ff..c1a896cad1 100644 --- a/src/server.ts +++ b/src/server.ts @@ -3957,6 +3957,22 @@ export async function preflightCompressIfNeeded( log("info", `[${session.id}] preflight usage-baseline ~${baselineFloor} tok was measured on ${maskUrlForLog(baselineOrigin)}, request now routes to ${maskUrlForLog(currentOrigin)} — demoting to untrusted, judging by this payload's own estimate (#1933)`); baselineFloor = 0; } + // #6430: the baseline's MODEL provenance — a usage-grade baseline measured + // under a different model (same route) is not authoritative for this + // request's billing scale: tokenizers bill differently across models, so a + // cross-model baseline can sit far above (or below) the current model's true + // size (the incident: a 282K opus reading drove preflight on a fable payload + // that billed ~55K). The k̂ read gate (currentCalibrationFactor) already + // demotes on model mismatch; the baseline floor must match it, or a stale + // cross-model baseline keeps arming preflight on the same route. Mirrors the + // #1933 F2 origin demotion above. Absent model (legacy files / unprovenanced) + // keeps the legacy behavior — never invalidated by missing info. + const baselineModel = typeof session.stats.lastInputTokensModel === "string" && session.stats.lastInputTokensModel !== "" ? session.stats.lastInputTokensModel : undefined; + const currentModel = typeof session.metadata?.lastModel === "string" && session.metadata.lastModel !== "" ? session.metadata.lastModel : undefined; + if (baselineFloor > 0 && baselineModel !== undefined && currentModel !== undefined && baselineModel !== currentModel) { + log("info", `[${session.id}] preflight usage-baseline ~${baselineFloor} tok was measured under model ${baselineModel}, request now runs ${currentModel} — demoting to untrusted, judging by this payload's own estimate (#6430)`); + baselineFloor = 0; + } // #1933 F1: scale the local text estimate by the per-route calibration // factor k̂ learned from this session's own usage reports (local estimate ÷ // what upstream actually billed; clamped 0.25–4, two-way since #2366 — see @@ -3966,7 +3982,23 @@ export async function preflightCompressIfNeeded( // with a cross-model billing scale (currentCalibrationFactor). const kFactor = currentCalibrationFactor(session.stats, session.metadata?.lastModel); const kOrigin = session.stats.calibratedEstimateOrigin; - const calibratedText = applyEstimateCalibration(textEstimate + overheadEstimate, kFactor, kOrigin, currentOrigin); + // #6430: a fresh same-route+same-model usage baseline (not demoted above) is + // the most recent MEASURED reading of this lane's real billed size. When the + // current payload's RAW estimate is not bigger than that reading, the payload + // has not genuinely grown past the last measurement — any excess of the k̂- + // scaled estimate over the baseline is calibration over-correction (a stale + // k̂>1 inflating a payload the provider just billed below the target), not + // real growth. Cap the calibrated text at the measured baseline so a stale k̂ + // cannot fire preflight below a fresh measurement: the measurement wins over + // the inflated estimate. Genuine growth (raw > baseline) leaves the + // calibrated estimate untouched, so a payload that really has crossed the + // target still folds; a missed fold here self-heals via the overflow arm on + // the next 400. The cap only applies while the baseline is trusted (same + // route+model); a demoted baseline floors at 0 and the estimate decides. + const rawTextEstimate = textEstimate + overheadEstimate; + const calibratedText = rawTextEstimate <= baselineFloor + ? Math.min(applyEstimateCalibration(rawTextEstimate, kFactor, kOrigin, currentOrigin), baselineFloor) + : applyEstimateCalibration(rawTextEstimate, kFactor, kOrigin, currentOrigin); const calibratedPayload = calibratedText + imageTokens; const tokenCount = unknownBaseline ? estimateCoreMessagesUpper(prepared.processedMessages) + overheadEstimate + imageTokens diff --git a/src/server/relay.ts b/src/server/relay.ts index 7ce093b91b..7029e623ee 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -1756,6 +1756,9 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare // probe-forward gate) treat the arm as current-route evidence // instead of demoting it against a stale settle origin. s.stats.lastInputTokensOrigin = normalizeUpstreamOrigin(upstreamUrl); + // #6430: model provenance for the armed baseline — the preflight gate + // demotes it when the current request runs on a different model. + if (reqModel !== undefined) s.stats.lastInputTokensModel = reqModel; // #1110: record the arm SEPARATELY so the side-request guard // can read it without ever touching the nudge baseline. The guard // keeps the CEILING even when the baseline books a measured or @@ -1832,6 +1835,7 @@ const reply = prepared && !prepared.sidePassthrough ? createHandoffReply(prepare s.stats.lastInputTokensSource = "overflow-arm"; s.stats.overflowArmTokens = arm; // #1110: guard reads this, not the baseline s.stats.lastInputTokensOrigin = normalizeUpstreamOrigin(upstreamUrl); // #2313: same-route evidence, see the window branch above + if (reqModel !== undefined) s.stats.lastInputTokensModel = reqModel; // #6430: model provenance, see the window branch above } log("warn", `[${s.id}] upstream context overflow (window not parseable, model=${reqModel ?? "unknown"}) — armed emergency shrink at ~${arm} tokens (min of judging window ${judging} and payload estimate), nothing learned (#987): ${info.message}`); } diff --git a/src/session.ts b/src/session.ts index ca5e7cddf3..262f3971f0 100644 --- a/src/session.ts +++ b/src/session.ts @@ -554,6 +554,18 @@ export type Session = { * provider says nothing about whether another accepts the payload. * Absent on legacy session files → baseline keeps legacy behavior. */ lastInputTokensOrigin?: string; + /** #6430: model id that measured lastInputTokens — written by + * settleUsageReport (and the overflow-arm paths) alongside the + * origin. The preflight gate demotes a usage-grade baseline to + * untrusted when the CURRENT request runs on a different model: + * tokenizers bill differently across models, so a cross-model + * baseline (e.g. a 282K opus reading on a fable lane) is not + * authoritative for this request's size. The k̂ read gate + * (currentCalibrationFactor) already demotes on model mismatch; + * the baseline floor must match it, or a stale cross-model + * baseline keeps arming preflight on the same route. Absent on + * legacy session files → baseline keeps legacy behavior. */ + lastInputTokensModel?: string; /** #1933 F1: calibrated scale factor k̂ = mean of up to 3 recent * consistent same-route samples of (upstream-billed input ÷ local * text estimate), clamped to [0.25, 4] — two-way since #2366: the @@ -1807,6 +1819,8 @@ export function resetSessionCompression(session: Session): void { // learned against the pre-compaction content class, and the baseline's // measuring route no longer bounds this rebuilt session. delete session.stats.lastInputTokensOrigin; + // #6430: the baseline's model provenance dies with the baseline. + delete session.stats.lastInputTokensModel; delete session.stats.calibratedEstimate; delete session.stats.calibratedEstimateOrigin; // #2117 B: same boundary — the factor's model provenance dies with it. diff --git a/tests/golden/storage-render/anthropic.json b/tests/golden/storage-render/anthropic.json index 7c01ccf504..caf3fa1c81 100644 --- a/tests/golden/storage-render/anthropic.json +++ b/tests/golden/storage-render/anthropic.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "ba0f75f6ddc882d00d5c456facd79166b62e3f2cacadd60744aa370d6b247105", - "stateDigestAligned": "66991d5cb8864f3ca029914aa9afc20cc8b9687cfd4391ffe200017c68057946", + "stateDigest": "24645d604a48c016a7c75e3bc60ce2e9e561c6079b24730f47cc9401b2d75092", + "stateDigestAligned": "a38b6f5cbc79779d6eb4955aaa46e0deb3e7183d966f2565c67c54fd3d0d8a34", "sectionDigests": { "state": "37b26b4f6898afff", "blockContents": "dc7d13d250c94bb4", - "stats": "7caf981f99159be9", + "stats": "b679dfaeba151fb7", "metadata": "ba51746c6749d0cf" }, "stateKeyDigests": { diff --git a/tests/golden/storage-render/chat.json b/tests/golden/storage-render/chat.json index 5ecde5722b..b887f2baa3 100644 --- a/tests/golden/storage-render/chat.json +++ b/tests/golden/storage-render/chat.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "2c0b36206756617ae1e32cf0d221ddd8bf7f3e06994b49d11578bd1a35c90f26", - "stateDigestAligned": "3011800706759958cfeab6520289f1ffe257cf4f7c3275d846c03a991f9ed6b8", + "stateDigest": "7a6fdf15f2da6391812ccedd5a3addb3892a1005b73b299a3e7dd859ed961320", + "stateDigestAligned": "5b1a8b4a89ae359fa131b730a12111b5f0c39738524f366e016cd4acb369c9f9", "sectionDigests": { "state": "90857fbf82001a2b", "blockContents": "94a58f96d7c5cfc3", - "stats": "7b6162aef3873d31", + "stats": "a41a71c5ff86bd68", "metadata": "7218c1e3430c3d21" }, "stateKeyDigests": { diff --git a/tests/golden/storage-render/google.json b/tests/golden/storage-render/google.json index 26c5f73b94..202f89f4f3 100644 --- a/tests/golden/storage-render/google.json +++ b/tests/golden/storage-render/google.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "c5eefef6525f180ab54a7ea57c4ddf004a568db0fc2d6d227ede31dc960706db", - "stateDigestAligned": "5217aa02331256a419837cd91105d3e3172f2ff38f0f3914c0e01128e221f012", + "stateDigest": "aff30a310dbfeddfd9a22eade43ef9ebe82e5cffc19d10d4a26fb1b6db6e2f75", + "stateDigestAligned": "fbc2418aac73d5811ee7b230b16623facd546461e5f515439416c2979fb669e5", "sectionDigests": { "state": "c5243ba17364d3ac", "blockContents": "473264d9df8ba338", - "stats": "c90671e484dc5d37", + "stats": "ff732e39f6f51bfd", "metadata": "2980a8959aa0a39c" }, "stateKeyDigests": { diff --git a/tests/golden/storage-render/responses.json b/tests/golden/storage-render/responses.json index 8041cd5155..0f577361a9 100644 --- a/tests/golden/storage-render/responses.json +++ b/tests/golden/storage-render/responses.json @@ -95,12 +95,12 @@ "foldAt": 9, "editedAt": 12, "summaryFrom": 9, - "stateDigest": "ff092bc66e11c35b15ad98acf6206629115825b62168da2e5542e06297f6d47a", - "stateDigestAligned": "db5f08a7d57c14952c31571e45b4dbc6c2ed35fe8a97c58cda6fa854d411eb25", + "stateDigest": "22465866fd11ff6734f2d92677990084cc4f55da58170b2aed751b89be8c35c2", + "stateDigestAligned": "2c0184430ab851734090169ee860ca93f6fb97b06df272aa44604f8fedb4a024", "sectionDigests": { "state": "bee6e1ba0c6fba62", "blockContents": "b6701268a0bdcd8a", - "stats": "1b56c993f03cd122", + "stats": "9e8f281283118864", "metadata": "77714417e484ba89" }, "stateKeyDigests": { diff --git a/tests/issue2129-restart-measurement-state.test.ts b/tests/issue2129-restart-measurement-state.test.ts index cc98a1f73f..5cccfe6005 100644 --- a/tests/issue2129-restart-measurement-state.test.ts +++ b/tests/issue2129-restart-measurement-state.test.ts @@ -62,6 +62,7 @@ test("measurement-state stats survive a reload round-trip (#2129)", async () => st.localInputEstimate = 12345; st.lastUsageGradeTokens = 9876; st.lastInputTokensOrigin = "http://127.0.0.1:8787"; + st.lastInputTokensModel = "claude-opus-5-5"; st.calibratedEstimate = 0.4375; st.calibratedEstimateOrigin = "http://127.0.0.1:8787"; st.calibrationRing = { origin: "http://127.0.0.1:8787", values: [0.4, 0.45, 0.48] }; @@ -75,6 +76,7 @@ test("measurement-state stats survive a reload round-trip (#2129)", async () => assert.equal(back.stats.localInputEstimate, 12345, "localInputEstimate round-trips"); assert.equal(back.stats.lastUsageGradeTokens, 9876, "lastUsageGradeTokens round-trips"); assert.equal(back.stats.lastInputTokensOrigin, "http://127.0.0.1:8787", "lastInputTokensOrigin round-trips"); + assert.equal(back.stats.lastInputTokensModel, "claude-opus-5-5", "lastInputTokensModel round-trips (#6430)"); assert.equal(back.stats.calibratedEstimate, 0.4375, "calibratedEstimate round-trips"); assert.equal(back.stats.calibratedEstimateOrigin, "http://127.0.0.1:8787", "calibratedEstimateOrigin round-trips"); assert.deepEqual(back.stats.calibrationRing, { origin: "http://127.0.0.1:8787", values: [0.4, 0.45, 0.48] }, "calibrationRing round-trips"); diff --git a/tests/issue2189-preflight-billing-block.test.ts b/tests/issue2189-preflight-billing-block.test.ts index 070cad16af..50d6e8ec41 100644 --- a/tests/issue2189-preflight-billing-block.test.ts +++ b/tests/issue2189-preflight-billing-block.test.ts @@ -191,7 +191,7 @@ test("#2189 anthropic summary carries the client billing-attribution block at sy const { proxy, upstream, proxyPort, upstreamPort } = await makePair(calls, 200, ""); try { const r1 = await drive(proxyPort, upstreamPort, "s2189-main", { - model: "claude-big", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }], + model: "claude-small", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }], }); assert.equal(r1.status, 200); await r1.text(); @@ -233,7 +233,7 @@ test("#2189 legacy string system keeps the byte-identical summary shape", async try { const sys = "You are a helpful assistant."; const r1 = await drive(proxyPort, upstreamPort, "s2189-legacy", { - model: "claude-big", max_tokens: 1024, stream: true, system: sys, messages: [{ role: "user", content: "hello" }], + model: "claude-small", max_tokens: 1024, stream: true, system: sys, messages: [{ role: "user", content: "hello" }], }); assert.equal(r1.status, 200); await r1.text(); @@ -268,7 +268,7 @@ test("#2189 bare-Error 429 fails fast after ONE attempt, retryable=false, retrie const { proxy, upstream, proxyPort, upstreamPort } = await makePair(calls, 429, SHAPE_429_BODY, true); try { const r1 = await drive(proxyPort, upstreamPort, "s2189-shape", { - model: "claude-big", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }], + model: "claude-small", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }], }); assert.equal(r1.status, 200); await r1.text(); @@ -314,7 +314,7 @@ test("#2189 a genuine rate limit keeps the legacy 3-attempt replay and retryable const { proxy, upstream, proxyPort, upstreamPort } = await makePair(calls, 429, GENUINE_429_BODY, true); try { const r1 = await drive(proxyPort, upstreamPort, "s2189-genuine", { - model: "claude-big", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }], + model: "claude-small", max_tokens: 1024, stream: true, system: ccSystem(), messages: [{ role: "user", content: "hello" }], }); assert.equal(r1.status, 200); await r1.text(); diff --git a/tests/issue6430-preflight-baseline-model.test.ts b/tests/issue6430-preflight-baseline-model.test.ts new file mode 100644 index 0000000000..f25bd62cf7 --- /dev/null +++ b/tests/issue6430-preflight-baseline-model.test.ts @@ -0,0 +1,209 @@ +import assert from "node:assert/strict"; +import http from "node:http"; +import { once } from "node:events"; +import test from "node:test"; + +process.env.NODE_ENV = "test"; + +import { defaultConfig } from "acp-kernel"; +import { startServer } from "../src/server.ts"; +import type { ProxyOptions } from "../src/config.ts"; +import { SessionStore, _setStoreForTest } from "../src/persist.ts"; +import { getSession } from "../src/session.ts"; +import { _setForTest as setRegistryForTest } from "../src/registry.ts"; +import { normalizeUpstreamOrigin } from "../src/util.ts"; +import { setLogCapture } from "../src/logger.ts"; + +// #6430: the preflight size model (local estimate × k̂ + usage baseline) diverges +// from the measured usage in BOTH directions, so preflight fires when it +// shouldn't: +// 1. OVERESTIMATE (incident case 1) — a fresh same-route+same-model usage +// baseline (167K, below the 200K target) is overridden by a stale k̂>1 that +// inflates the local estimate past the target (raw 151K × 1.47 = 223K) → +// spurious fold, 13.7s stall. A fresh measurement must win over a stale +// calibration: when the payload's raw estimate is not bigger than the +// baseline, the calibrated estimate is capped at the baseline. +// 2. CROSS-MODEL (incident case 2) — a usage baseline measured under a PRIOR +// model on the same route (282K under opus) carries no model provenance, so +// it survives the #1933 F2 route demotion and drives the trigger while the +// current payload's own estimate reads far below the target (~55K on a +// fable lane) → 303s preflight. The baseline floor must be scoped to the +// model that measured it, mirroring the k̂ read gate's model dimension. +// +// Both are one mechanism (the baseline floor and the k̂-scaled estimate must be +// scoped to the model that measured them), so they land in one PR. This file pins +// both directions plus the control (a same-model baseline above the window still +// drives preflight — no regression). The trigger's decision is asserted on the +// "reached preflight target" log line (present = fired). + +function textOfTokens(n: number): string { + return "x".repeat(n * 4); // ceil(4n / 4) = n tokens on the chars/4 estimator +} + +let LOGS: string[] = []; + +function mockUpstream(): http.Server { + return http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (c: Buffer) => chunks.push(c)); + req.on("end", () => { + res.writeHead(200, { "content-type": "text/event-stream" }); + const chunk = (delta: Record, finish: string | null): string => + `data: ${JSON.stringify({ id: "chatcmpl-1", object: "chat.completion.chunk", created: 1, model: "gpt-test", choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`; + res.end(chunk({ role: "assistant" }, null) + chunk({ content: "forwarded answer" }, null) + chunk({}, "stop") + "data: [DONE]\n\n"); + }); + }); +} + +function proxyOptions(upstreamPort: number, window: number): ProxyOptions { + return { + port: 0, + host: "127.0.0.1", + upstream: "http://127.0.0.1", + routes: { [`http://127.0.0.1:${upstreamPort}`]: { models: { "gpt-test": { context: window } } } } as ProxyOptions["routes"], + modelContextLimit: window, + kernelConfig: defaultConfig(window, { + preserveRecentMessages: 2, + preserveRecentTokens: 1000, + compress: { minCompressRange: 1000, maxSummaryLength: 20000, minSummaryLength: 50 }, + }), + compress: { injectTool: true, injectNudge: true }, + promptCache: { routing: "auto" }, + sessionHeader: "x-acp-session", + log: true, + debug: true, + passthrough: false, + autoUpdate: false, + mitm: { enabled: false, domains: [] }, + } as ProxyOptions; +} + +async function boot(window: number): Promise<{ proxyPort: number; upstreamPort: number; cleanup: () => Promise }> { + _setStoreForTest(new SessionStore({ enabled: false })); + setRegistryForTest({}); + const upstream = mockUpstream(); + upstream.listen(0, "127.0.0.1"); + await once(upstream, "listening"); + const upstreamPort = (upstream.address() as { port: number }).port; + const proxy = await startServer(proxyOptions(upstreamPort, window)); + await once(proxy, "listening"); + const proxyPort = (proxy.address() as { port: number }).port; + return { + proxyPort, + upstreamPort, + cleanup: async () => { + proxy.close(); + proxy.closeIdleConnections?.(); + await once(proxy, "close"); + upstream.close(); + upstream.closeIdleConnections?.(); + await once(upstream, "close"); + }, + }; +} + +function post(p: { proxyPort: number; upstreamPort: number }, session: string, model: string, messages: Record[]): Promise { + return fetch(`http://127.0.0.1:${p.proxyPort}/bili/http://127.0.0.1:${p.upstreamPort}/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json", "x-acp-session": session }, + body: JSON.stringify({ model, stream: true, messages }), + }); +} + +// Seed a settled usage baseline (what upstream last billed) with its provenance. +// `model` is the model that measured it; `khat` seeds a matching-origin/model +// calibration factor (the stale k̂>1 from incident case 1). +function seedBaseline(sessionId: string, upstreamPort: number, tokens: number, model: string | undefined, khat?: number): void { + const s = getSession(sessionId); + s.stats.lastInputTokens = tokens; + s.stats.lastInputTokensSource = "usage"; + s.stats.lastInputTokensOrigin = normalizeUpstreamOrigin(`http://127.0.0.1:${upstreamPort}`); + if (model !== undefined) s.stats.lastInputTokensModel = model; + if (khat !== undefined) { + s.stats.calibratedEstimate = khat; + s.stats.calibratedEstimateOrigin = normalizeUpstreamOrigin(`http://127.0.0.1:${upstreamPort}`); + s.stats.calibratedEstimateModel = model; + } +} + +// The trigger fired when EITHER log is present: "reached preflight target" +// (server.ts:4255, the fold is about to run) or "preflight target reached ... +// but the payload fits with no compressible ranges" (server.ts:4223, the +// trigger crossed the target but the fold found nothing to compress). +function preflightFired(): boolean { + return LOGS.some((l) => l.includes("reached preflight target") || l.includes("preflight target reached")); +} + +test("#6430 Fix A: cross-model usage baseline is demoted — no spurious preflight", async () => { + LOGS = []; + setLogCapture((_level, msg) => LOGS.push(msg)); + const p = await boot(100_000); + try { + // Baseline measured under model-A (110K, above the 100K window) on the + // SAME route. The current request runs model-B with a small payload + // (~10K, well below the window). Pre-fix: the baseline carried no model + // provenance, so it survived the #1933 F2 route demotion and floored the + // trigger at 110K >= 100K → spurious preflight. Post-fix: the model + // mismatch demotes the baseline → the payload's own estimate (10K) + // judges → no preflight. + const sessionId = "issue6430-fixA"; + seedBaseline(sessionId, p.upstreamPort, 110_000, "claude-opus-5-5"); + const resp = await post(p, sessionId, "gpt-test", [{ role: "user", content: textOfTokens(10_000) }]); + const body = await resp.text(); + assert.equal(resp.status, 200, `the small payload must forward: HTTP ${resp.status} ${body.slice(0, 240)}`); + assert.ok(body.includes("forwarded answer"), "the model reply must reach the client"); + assert.ok(!preflightFired(), `the cross-model baseline must not arm preflight (preflight logs: ${LOGS.filter((l) => l.includes("preflight")).join(" | ").slice(0, 400)})`); + assert.ok(LOGS.some((l) => l.includes("measured under model claude-opus-5-5") && l.includes("demoting to untrusted")), "the model demotion must be logged"); + } finally { + setLogCapture(null); + await p.cleanup(); + } +}); + +test("#6430 Fix B: a fresh same-model baseline caps a stale k̂ over-correction", async () => { + LOGS = []; + setLogCapture((_level, msg) => LOGS.push(msg)); + const p = await boot(40_000); + try { + // Same route + same model: the baseline (20K, below the 40K window) is + // trusted. A stale k̂=3.0 inflates the payload's raw estimate (~14K) to + // 42K >= 40K → pre-fix the trigger fires on the inflated estimate even + // though the provider just billed 20K below the target. Post-fix: the + // raw estimate (14K) is not bigger than the baseline (20K), so the + // calibrated estimate is capped at the baseline (20K < 40K) → no + // preflight. Genuine growth (raw > baseline) would leave the estimate + // untouched and still fold. + const sessionId = "issue6430-fixB"; + seedBaseline(sessionId, p.upstreamPort, 20_000, "gpt-test", 3.0); + const resp = await post(p, sessionId, "gpt-test", [{ role: "user", content: textOfTokens(14_000) }]); + const body = await resp.text(); + assert.equal(resp.status, 200, `the payload must forward (billed below the target): HTTP ${resp.status} ${body.slice(0, 240)}`); + assert.ok(body.includes("forwarded answer"), "the model reply must reach the client"); + assert.ok(!preflightFired(), `the stale k̂ must not override a fresh same-model baseline (preflight logs: ${LOGS.filter((l) => l.includes("preflight")).join(" | ").slice(0, 400)})`); + } finally { + setLogCapture(null); + await p.cleanup(); + } +}); + +test("#6430 control: a same-model baseline above the window still drives preflight", async () => { + LOGS = []; + setLogCapture((_level, msg) => LOGS.push(msg)); + const p = await boot(100_000); + try { + // Same route + same model, baseline (110K) ABOVE the 100K window: the + // baseline is trusted and must still arm preflight (the fix must not + // demote a legitimate same-model overflow). The payload is small, so the + // fold finds nothing to compress and forwards — but the trigger fired. + const sessionId = "issue6430-control"; + seedBaseline(sessionId, p.upstreamPort, 110_000, "gpt-test"); + const resp = await post(p, sessionId, "gpt-test", [{ role: "user", content: textOfTokens(10_000) }]); + const body = await resp.text(); + assert.equal(resp.status, 200, `the payload must forward after the (no-op) fold: HTTP ${resp.status} ${body.slice(0, 240)}`); + assert.ok(body.includes("forwarded answer"), "the model reply must reach the client"); + assert.ok(preflightFired(), `a same-model baseline above the window must still arm preflight (no regression). ALL LOGS:\n${LOGS.join("\n")}`); + } finally { + setLogCapture(null); + await p.cleanup(); + } +}); diff --git a/tests/preflight-compress.test.ts b/tests/preflight-compress.test.ts index 7d9d720688..4ff1b00049 100644 --- a/tests/preflight-compress.test.ts +++ b/tests/preflight-compress.test.ts @@ -24,14 +24,27 @@ import { listSessions } from "../src/session.ts"; // (dedicated summarization calls sized to fit the smaller window), then // rebuild and forward a payload that fits. // +// #6430: the usage baseline is scoped to the MODEL that measured it — a +// cross-model baseline (same route) is demoted, so after the switch the +// session carries no current-model upstream evidence. The #2313 forward-once +// escape then applies: the FIRST request after the switch is forwarded to +// acquire the new model's usage/overflow evidence (in the real world the +// over-window forward 400s and arms the overflow baseline instead), and the +// SECOND request folds. This test pins that two-request self-heal. +// // Scenario: // 1. Request 1 on claude-big (400k window), upstream reports -// input_tokens=300000 → session.lastInputTokens = 300000. -// 2. Request 2 on claude-small (12k window) with a ~13k-token -// conversation: the payload itself overflows → preflight fires: the -// oldest compressible ranges are summarized (non-streaming JSON calls -// to the upstream) and folded into blocks; the rebuilt payload (recent -// protected messages + summaries) is forwarded and gets a 200. +// input_tokens=300000 → session.lastInputTokens = 300000 (model +// claude-big). +// 2. Request 2 on claude-small (12k window) with a ~19k-token +// conversation: the cross-model 300k baseline is demoted (#6430) → the +// trigger fires on the payload's own estimate, but the #2313 +// forward-once escape forwards it for evidence — NO summarization call; +// the successful turn settles a fresh claude-small usage baseline. +// 3. Request 3 on claude-small with the same conversation: the fresh +// same-model baseline is trusted, preflight folds BEFORE forwarding +// (non-streaming JSON summarization calls, blocks), and the rebuilt +// payload (recent protected messages + summaries) is forwarded with 200. const SUMMARY_TEXT = "PREFLIGHT SUMMARY: the segment covered a multi-step debugging session. " + @@ -146,7 +159,10 @@ test("e2e: model switch to a smaller window → preflight compresses before forw assert.equal(s1.stats.lastInputTokens, 300_000, "session context is 300k tokens"); assert.equal(calls.filter((c) => !c.stream).length, 0, "no summarization on the first request"); - // --- Request 2: switch to the 260k model with a long conversation --- + // --- Request 2: switch to claude-small with a long conversation --- + // #6430: the cross-model 300k baseline is demoted → the trigger fires + // on the payload's own estimate, but the #2313 forward-once escape + // forwards it for evidence — NO summarization call this turn. const r2 = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { method: "POST", headers, @@ -155,14 +171,38 @@ test("e2e: model switch to a smaller window → preflight compresses before forw assert.equal(r2.status, 200, "the switched-model request succeeds (no context-full error)"); await r2.text(); + assert.equal(calls.filter((c) => !c.stream).length, 0, "forward-once: no summarization on the first switched-model request"); + const s2 = listSessions().find((x) => x.id === s1.id); + assert.equal((s2?.state.blocks ?? []).filter((b) => b.active).length, 0, "no compression block on the forward-once turn"); + // The forwarded payload is the ORIGINAL (not rebuilt): the oldest + // messages are still in the payload because nothing was folded. + const f2 = calls.filter((c) => c.stream); + assert.ok(f2[f2.length - 1]!.body.includes("MARKER_1_"), "forward-once sends the original payload (oldest messages intact)"); + // The successful turn settled a fresh claude-small usage baseline. + assert.equal(s2?.stats.lastInputTokens, 1000, "fresh claude-small baseline settled"); + assert.equal(s2?.stats.lastInputTokensModel, "claude-small", "baseline is scoped to the new model"); + + // --- Request 3: same model, same conversation → preflight folds --- + // The fresh same-model baseline is trusted; the trigger fires and the + // forward-once escape no longer applies (baselineFloor > 0) → the + // oldest compressible ranges are summarized and folded BEFORE the + // rebuilt payload is forwarded. + const r3 = await fetch(`http://127.0.0.1:${proxyPort}/bili/http://127.0.0.1:${upstreamPort}/v1/messages`, { + method: "POST", + headers, + body: JSON.stringify({ model: "claude-small", max_tokens: 1024, stream: true, messages: bigConversation() }), + }); + assert.equal(r3.status, 200, "the third request succeeds after the fold"); + await r3.text(); + const summaryCalls = calls.filter((c) => !c.stream); assert.ok(summaryCalls.length >= 1, `preflight made summarization call(s) before forwarding (got ${summaryCalls.length})`); for (const c of summaryCalls) { assert.ok(c.body.includes("Compression") || c.body.includes("compress"), "summarization call carries the compression rules"); } - const s2 = listSessions().find((x) => x.id === s1.id); - const activeBlocks = (s2?.state.blocks ?? []).filter((b) => b.active); + const s3 = listSessions().find((x) => x.id === s1.id); + const activeBlocks = (s3?.state.blocks ?? []).filter((b) => b.active); assert.ok(activeBlocks.length >= 1, `preflight created compression block(s) (got ${activeBlocks.length})`); assert.ok(activeBlocks.every((b) => b.summary === SUMMARY_TEXT), "blocks hold the upstream-written summaries"); @@ -179,8 +219,8 @@ test("e2e: model switch to a smaller window → preflight compresses before forw assert.ok(lastForward.body.includes("MARKER_11_"), "recent protected messages remain in the payload"); assert.ok(lastForward.body.includes(SUMMARY_TEXT), "the preflight summary is in the rebuilt payload"); - // The successful turn's usage report overwrote the 300k figure. - assert.equal(s2?.stats.lastInputTokens, 1000); + // The third turn's usage report overwrote the second turn's figure. + assert.equal(s3?.stats.lastInputTokens, 1000); } finally { proxy.close(); proxy.closeIdleConnections?.(); diff --git a/tests/preflight-max-output-tokens.test.ts b/tests/preflight-max-output-tokens.test.ts index e377b93fab..4aa44e0f1d 100644 --- a/tests/preflight-max-output-tokens.test.ts +++ b/tests/preflight-max-output-tokens.test.ts @@ -378,13 +378,27 @@ test("e2e #663 (model scoping): rejection learned for model A keeps model B's 32 `every model A retry must drop the cap, got ${JSON.stringify(astraSummaries)}`, ); - // Same session, different model: preflight must fire again (grown - // over-window) and model B's summary keeps the cap. + // Same session, different model: the cross-model usage baseline is + // demoted (#6430), so the trigger fires on the payload's own estimate + // and the #2313 forward-once escape acquires model B's evidence first — + // the request forwards WITHOUT folding, no summary on this turn. const callsBefore = calls.length; const r2 = await driveResponsesPreflight(proxyPort, upstreamPort, "s663-resp-d", "gpt-6-standard", longResponsesInput(24)); assert.equal(r2.status, 200, `second request (model B) must succeed, got ${r2.status}`); - const standardSummaries = calls.slice(callsBefore).filter((c) => c.summary); - assert.ok(standardSummaries.length >= 1, "second request triggered preflight summaries"); + assert.ok(calls.slice(callsBefore).length >= 1, "second request reached the upstream"); + assert.ok( + calls.slice(callsBefore).every((c) => !c.summary), + `cross-model request must forward once for evidence without folding (#6430 demotion → #2313), got ${JSON.stringify(calls.slice(callsBefore))}`, + ); + + // Third request (same model, still over-window): the fresh model-B + // usage baseline is trusted, preflight folds, and model B's summary + // keeps the cap. + const callsBefore3 = calls.length; + const r3 = await driveResponsesPreflight(proxyPort, upstreamPort, "s663-resp-d", "gpt-6-standard", longResponsesInput(24)); + assert.equal(r3.status, 200, `third request (model B) must succeed, got ${r3.status}`); + const standardSummaries = calls.slice(callsBefore3).filter((c) => c.summary); + assert.ok(standardSummaries.length >= 1, "third request triggered preflight summaries"); assert.ok( standardSummaries.every((c) => c.model === "gpt-6-standard" && typeof c.maxOutputTokens === "number" && c.maxOutputTokens <= 32768), `model B summaries must keep the output cap (<= 32768; #987 may clamp it), got ${JSON.stringify(standardSummaries)}`,