From ea10aefe17762a179e1da2a1901132305c859901 Mon Sep 17 00:00:00 2001 From: "itarun.p" Date: Mon, 24 Aug 2026 13:13:17 +0700 Subject: [PATCH 1/2] fix: mark unresolved auto-router cost as unpriced --- dashboard/src/lib/model-breakdown.ts | 2 +- src/lib/pricing/index.js | 18 +++++++++++++++++- test/model-breakdown.test.js | 15 ++++++++++++--- test/pricing-observability.test.js | 26 ++++++++++++++++++++++++++ test/pricing.test.js | 15 ++++++--------- 5 files changed, 62 insertions(+), 14 deletions(-) diff --git a/dashboard/src/lib/model-breakdown.ts b/dashboard/src/lib/model-breakdown.ts index 18b1ac671..70bc8900d 100644 --- a/dashboard/src/lib/model-breakdown.ts +++ b/dashboard/src/lib/model-breakdown.ts @@ -30,7 +30,7 @@ const FUZZY_PRICING_TIERS = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm: // all, but their tokens still count toward the total, so the same caveat applies // — they are excluded from the server's unpriced_models list (which exists to // name models needing a curated price) but must not silently read as priced. -const UNPRICED_PRICING_TIERS = new Set(["miss", "unattributed", "empty"]); +const UNPRICED_PRICING_TIERS = new Set(["miss", "unattributed", "empty", "routed-unresolved"]); function isKnownZeroCostModel(name: any) { const lower = String(name || "").toLowerCase(); diff --git a/src/lib/pricing/index.js b/src/lib/pricing/index.js index d4dc4416f..b52fec11e 100644 --- a/src/lib/pricing/index.js +++ b/src/lib/pricing/index.js @@ -56,6 +56,17 @@ const FUZZY_SOURCES = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix // Closed set on purpose: a real model id must never be silently un-priced here. const UNATTRIBUTED_MODEL_IDS = new Set(["unknown"]); +// These ids are logical routes that deliberately select a physical child model +// per request. Their names can include one tier name, but that is not evidence +// that every request used that tier. Do not run them through fuzzy pricing: a +// route-level estimate would silently charge every request at the wrong child +// rate. They stay visible as unpriced until per-request resolved-model or +// authoritative cost data is persisted. +const UNRESOLVED_LOGICAL_ROUTE_IDS = new Set([ + "claude-auto-pilot-fable-v1-canary", +]); +const UNPRICED_TIERS = new Set(["miss", "routed-unresolved"]); + // `last_refresh_error` is served over HTTP to the dashboard, so it is built // from CLOSED sets, never from an arbitrary value. A previous version accepted // anything symbol-shaped, which a QA pass broke immediately: a 32-character @@ -235,6 +246,11 @@ function getModelPricingMeta(model, opts = {}) { const lookupSource = resolveLookupSource(opts); const cacheKey = lookupSource ? `${lookupSource}\0${model}` : model; + if (UNRESOLVED_LOGICAL_ROUTE_IDS.has(String(model || "").trim())) { + state.tiers.set(cacheKey, { model, source: lookupSource, tier: "routed-unresolved" }); + return { pricing: ZERO_PRICING, tier: "routed-unresolved" }; + } + if (state.negativeCache.has(cacheKey)) { // Still unknown as of the current snapshot. If that snapshot has aged out, // a new model may have appeared upstream — refresh for the next caller. @@ -287,7 +303,7 @@ function getPricingDiagnostics() { const unpriced = new Set(); const fuzzy = []; for (const entry of state.tiers.values()) { - if (entry.tier === "miss") unpriced.add(entry.model); + if (UNPRICED_TIERS.has(entry.tier)) unpriced.add(entry.model); else if (FUZZY_SOURCES.has(entry.tier)) fuzzy.push({ model: entry.model, tier: entry.tier }); } return { diff --git a/test/model-breakdown.test.js b/test/model-breakdown.test.js index 61462e750..a1f1f9580 100644 --- a/test/model-breakdown.test.js +++ b/test/model-breakdown.test.js @@ -673,6 +673,12 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing", pricing_tier: "miss", totals: { billable_total_tokens: 1000, total_cost_usd: "0" }, }, + { + model: "claude-auto-pilot-fable-v1-canary", + model_id: "claude-auto-pilot-fable-v1-canary", + pricing_tier: "routed-unresolved", + totals: { billable_total_tokens: 1000, total_cost_usd: "0" }, + }, ], }, ], @@ -681,8 +687,8 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing", const [provider] = buildFleetData(modelBreakdown); assert.deepEqual( provider.missingPricingModels.map((m) => m.name), - ["brand-new-model"], - "only the tier=miss model is unpriced", + ["brand-new-model", "claude-auto-pilot-fable-v1-canary"], + "both a miss and an unresolved composite route are unpriced", ); assert.deepEqual( provider.fuzzyPricingModels.map((m) => m.name), @@ -691,7 +697,10 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing", ); const insights = buildUsageInsights(modelBreakdown); - assert.deepEqual(insights.missingPricingModels.map((m) => m.name), ["brand-new-model"]); + assert.deepEqual( + insights.missingPricingModels.map((m) => m.name), + ["brand-new-model", "claude-auto-pilot-fable-v1-canary"], + ); assert.deepEqual(insights.fuzzyPricingModels.map((m) => m.name), ["acme-9-turbo"]); }); diff --git a/test/pricing-observability.test.js b/test/pricing-observability.test.js index f096ddd49..ce1a94554 100644 --- a/test/pricing-observability.test.js +++ b/test/pricing-observability.test.js @@ -416,6 +416,32 @@ test("a real model that merely contains \"unknown\" is still priced or missed no assert.deepEqual(pricing.getPricingDiagnostics().unpriced_models, ["mystery-model"]); }); +test("a configured composite auto-router route is unpriced rather than fuzzy-matched as Fable", async () => { + const payload = { + current: { + "anthropic/claude-fable-5": entry(10e-6, 50e-6), + }, + }; + await loadWith(payload, tmpCachePath("composite-auto-router")); + + const meta = pricing.getModelPricingMeta("claude-auto-pilot-fable-v1-canary", { source: "hermes" }); + assert.equal(meta.tier, "routed-unresolved"); + assert.deepEqual(meta.pricing, pricing.ZERO_PRICING); + assert.equal( + pricing.computeRowCost({ + source: "hermes", + model: "claude-auto-pilot-fable-v1-canary", + cached_input_tokens: 1_000_000, + }), + 0, + ); + assert.deepEqual( + pricing.getPricingDiagnostics().unpriced_models, + ["claude-auto-pilot-fable-v1-canary"], + ); + assert.deepEqual(pricing.getPricingDiagnostics().fuzzy_priced_models, []); +}); + test("cost for an unattributed row is unchanged by the exemption", async () => { const payload = { current: { "acme-1": entry(1e-6, 2e-6) } }; await loadWith(payload, tmpCachePath("unattributed-cost")); diff --git a/test/pricing.test.js b/test/pricing.test.js index a36707b0f..602df7df1 100644 --- a/test/pricing.test.js +++ b/test/pricing.test.js @@ -569,15 +569,12 @@ test("index: getModelPricing pins Claude Fable/Mythos 5 pricing from curated ove assert.equal(pinned.cache_write, 12.5); } - // Regression (issue #187): auto-pilot/canary variants wrap the pinned name - // with extra prefix/suffix segments (e.g. "claude-auto-pilot-fable-v1-canary"), - // so "fable"/"mythos" is not a contiguous substring of "claude-fable-5"/ - // "claude-mythos-5" — every non-fuzzy resolution tier misses on these ids. - for (const model of ["claude-auto-pilot-fable-v1-canary", "claude-mythos-v1-canary"]) { - const pinned = pricing.getModelPricing(model); - assert.equal(pinned.input, 10, `${model} should resolve to the Fable/Mythos pin`); - assert.equal(pinned.output, 50, `${model} should resolve to the Fable/Mythos pin`); - } + // A simple renamed Mythos variant can still inherit the curated pin. The + // Fable auto-router label is intentionally excluded: it selects Sonnet, + // Opus, or Fable per request and cannot be priced as one child model. + const mythos = pricing.getModelPricing("claude-mythos-v1-canary"); + assert.equal(mythos.input, 10); + assert.equal(mythos.output, 50); assert.equal( pricing.computeRowCost({ From 9cfa9b3cac5c7e89685db754b5c032c0849fdeed Mon Sep 17 00:00:00 2001 From: "itarun.p" Date: Mon, 24 Aug 2026 13:19:01 +0700 Subject: [PATCH 2/2] fix: propagate unresolved route pricing to plan value --- dashboard/src/lib/plan-value.test.ts | 16 ++++++++++++++++ dashboard/src/lib/plan-value.ts | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/dashboard/src/lib/plan-value.test.ts b/dashboard/src/lib/plan-value.test.ts index dfd1a2c1d..bba1a260c 100644 --- a/dashboard/src/lib/plan-value.test.ts +++ b/dashboard/src/lib/plan-value.test.ts @@ -74,6 +74,22 @@ describe("computePlanValue", () => { expect(value.unpricedModels).toEqual(["unknown"]); }); + it("counts an unresolved composite route as unpriced, not exact free usage", () => { + const value = computePlanValue( + source({ + totals: { total_cost_usd: "0" }, + models: [{ + model: "claude-auto-pilot-fable-v1-canary", + pricing_tier: "routed-unresolved", + }], + }), + 20, + )!; + expect(value.listPriceUsd).toBe(0); + expect(value.confidence).toBe("floor"); + expect(value.unpricedModels).toEqual(["claude-auto-pilot-fable-v1-canary"]); + }); + it("is exact when every model priced exactly", () => { const value = computePlanValue( source({ diff --git a/dashboard/src/lib/plan-value.ts b/dashboard/src/lib/plan-value.ts index d86d520d0..e03972adf 100644 --- a/dashboard/src/lib/plan-value.ts +++ b/dashboard/src/lib/plan-value.ts @@ -61,7 +61,7 @@ const toNumber = (value: unknown): number => { // price at all; the fuzzy tiers matched by substring, which is a guess that can // be wrong in either direction. Kept as an explicit list rather than a // "not exact" rule so a NEW tier has to be classified deliberately. -const UNPRICED_TIERS = new Set(["miss", "empty", "unattributed"]); +const UNPRICED_TIERS = new Set(["miss", "empty", "unattributed", "routed-unresolved"]); const FUZZY_TIERS = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip", "litellm:strip"]); function modelName(model: SourceModel): string {