From 752e5d68ee001bacd5e18bd0014ca7f4a74b5a4f Mon Sep 17 00:00:00 2001 From: "itarun.p" Date: Tue, 25 Aug 2026 08:04:56 +0700 Subject: [PATCH 1/2] feat: estimate configured auto-router route costs --- dashboard/src/lib/model-breakdown.ts | 7 ++- dashboard/src/lib/plan-value.test.ts | 11 +++-- dashboard/src/lib/plan-value.ts | 8 +++- src/lib/pricing/index.js | 68 +++++++++++++++++++++++++++- test/model-breakdown.test.js | 19 ++++---- test/pricing-observability.test.js | 54 ++++++++++++++++------ 6 files changed, 135 insertions(+), 32 deletions(-) diff --git a/dashboard/src/lib/model-breakdown.ts b/dashboard/src/lib/model-breakdown.ts index 70bc8900..557fecc3 100644 --- a/dashboard/src/lib/model-breakdown.ts +++ b/dashboard/src/lib/model-breakdown.ts @@ -23,7 +23,12 @@ function resolveModelName(model: any, fallback: any) { // Server-side resolution tiers that mean the price was guessed from a partial // match rather than an exact model id (src/lib/pricing/index.js). A guessed // price is plausible and therefore never looks wrong — worth flagging. -const FUZZY_PRICING_TIERS = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip"]); +const FUZZY_PRICING_TIERS = new Set([ + "curated:fuzzy", + "litellm:fuzzy", + "litellm:prefix-strip", + "routed-estimated", +]); // Server tiers that mean "$0 because we have no price", as opposed to a model // that genuinely costs nothing. "unattributed"/"empty" rows carry no model id at diff --git a/dashboard/src/lib/plan-value.test.ts b/dashboard/src/lib/plan-value.test.ts index bba1a260..36e87ac6 100644 --- a/dashboard/src/lib/plan-value.test.ts +++ b/dashboard/src/lib/plan-value.test.ts @@ -74,20 +74,21 @@ describe("computePlanValue", () => { expect(value.unpricedModels).toEqual(["unknown"]); }); - it("counts an unresolved composite route as unpriced, not exact free usage", () => { + it("treats an approved composite-route blend as estimated, not exact", () => { const value = computePlanValue( source({ - totals: { total_cost_usd: "0" }, + totals: { total_cost_usd: "12" }, models: [{ model: "claude-auto-pilot-fable-v1-canary", - pricing_tier: "routed-unresolved", + pricing_tier: "routed-estimated", }], }), 20, )!; - expect(value.listPriceUsd).toBe(0); + expect(value.listPriceUsd).toBe(12); expect(value.confidence).toBe("floor"); - expect(value.unpricedModels).toEqual(["claude-auto-pilot-fable-v1-canary"]); + expect(value.unpricedModels).toEqual([]); + expect(value.fuzzyModels).toEqual(["claude-auto-pilot-fable-v1-canary"]); }); it("is exact when every model priced exactly", () => { diff --git a/dashboard/src/lib/plan-value.ts b/dashboard/src/lib/plan-value.ts index e03972ad..372e3b20 100644 --- a/dashboard/src/lib/plan-value.ts +++ b/dashboard/src/lib/plan-value.ts @@ -62,7 +62,13 @@ const toNumber = (value: unknown): number => { // be wrong in either direction. Kept as an explicit list rather than a // "not exact" rule so a NEW tier has to be classified deliberately. const UNPRICED_TIERS = new Set(["miss", "empty", "unattributed", "routed-unresolved"]); -const FUZZY_TIERS = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip", "litellm:strip"]); +const FUZZY_TIERS = new Set([ + "curated:fuzzy", + "litellm:fuzzy", + "litellm:prefix-strip", + "litellm:strip", + "routed-estimated", +]); function modelName(model: SourceModel): string { return String(model.model || model.model_id || "").trim(); diff --git a/src/lib/pricing/index.js b/src/lib/pricing/index.js index b52fec11..41f6c9fe 100644 --- a/src/lib/pricing/index.js +++ b/src/lib/pricing/index.js @@ -49,7 +49,12 @@ const RELOAD_COOLDOWN_MS = 5 * 60 * 1000; // curated fuzzy rule rather than an exact id, so the price is plausible but may // belong to a different model. Worth surfacing — a wrong price never looks // wrong, unlike a $0 one. -const FUZZY_SOURCES = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip"]); +const FUZZY_SOURCES = new Set([ + "curated:fuzzy", + "litellm:fuzzy", + "litellm:prefix-strip", + "routed-estimated", +]); // Placeholder ids that stand in for "this row has no model", so they resolve to // the "unattributed" tier instead of being looked up and recorded as a miss. @@ -67,6 +72,56 @@ const UNRESOLVED_LOGICAL_ROUTE_IDS = new Set([ ]); const UNPRICED_TIERS = new Set(["miss", "routed-unresolved"]); +// Owner-approved route mix estimates. The child-model ratios are a pricing +// policy, not routing telemetry: callers must surface `routed-estimated` as a +// non-exact price. Any configured child whose current price cannot be resolved +// makes the route unpriced rather than silently using a partial blend. +const ROUTED_ESTIMATE_POLICIES = new Map([ + ["claude-auto-pilot-fable-v1-canary", [ + { model: "anthropic/claude-sonnet-5", weight: 0.6 }, + { model: "anthropic/claude-opus-5", weight: 0.4 }, + ]], + ["gpt-5.6-auto-pilot", [ + { model: "gpt-5.6-terra", weight: 0.6 }, + { model: "gpt-5.6-sol", weight: 0.4 }, + ]], + ["gpt-5.6-auto-pilot-045-canary", [ + { model: "gpt-5.6-terra", weight: 0.6 }, + { model: "gpt-5.6-sol", weight: 0.4 }, + ]], + ["gpt-5.6-auto-pilot-055-v2", [ + { model: "gpt-5.6-terra", weight: 0.6 }, + { model: "gpt-5.6-sol", weight: 0.4 }, + ]], + ["gpt-5.6-auto-pilot-056-claude-reasoning-canary", [ + { model: "gpt-5.6-terra", weight: 0.6 }, + { model: "gpt-5.6-sol", weight: 0.4 }, + ]], +]); + +function isPotentialGptAutoRoute(model) { + return /^gpt-5\.6-auto/i.test(String(model || "").trim()); +} + +function resolveRoutedEstimate(model, lookupSource) { + const policy = ROUTED_ESTIMATE_POLICIES.get(String(model || "").trim()); + if (!policy) return null; + const mixed = { input: 0, output: 0, cache_read: 0, cache_write: 0 }; + for (const child of policy) { + const result = lookupPricing(child.model, { + curated: curatedOverrides, + litellm: state.litellmPerMillionMap, + source: lookupSource, + }); + if (!result.hit || !result.value) return null; + for (const field of Object.keys(mixed)) { + if (!Number.isFinite(result.value[field])) return null; + mixed[field] += child.weight * result.value[field]; + } + } + return mixed; +} + // `last_refresh_error` is served over HTTP to the dashboard, so it is built // from CLOSED sets, never from an arbitrary value. A previous version accepted // anything symbol-shaped, which a QA pass broke immediately: a 32-character @@ -246,7 +301,16 @@ function getModelPricingMeta(model, opts = {}) { const lookupSource = resolveLookupSource(opts); const cacheKey = lookupSource ? `${lookupSource}\0${model}` : model; - if (UNRESOLVED_LOGICAL_ROUTE_IDS.has(String(model || "").trim())) { + const routedEstimate = resolveRoutedEstimate(model, lookupSource); + if (routedEstimate) { + state.tiers.set(cacheKey, { model, source: lookupSource, tier: "routed-estimated" }); + return { pricing: routedEstimate, tier: "routed-estimated" }; + } + + if ( + UNRESOLVED_LOGICAL_ROUTE_IDS.has(String(model || "").trim()) + || isPotentialGptAutoRoute(model) + ) { state.tiers.set(cacheKey, { model, source: lookupSource, tier: "routed-unresolved" }); return { pricing: ZERO_PRICING, tier: "routed-unresolved" }; } diff --git a/test/model-breakdown.test.js b/test/model-breakdown.test.js index a1f1f958..e7336040 100644 --- a/test/model-breakdown.test.js +++ b/test/model-breakdown.test.js @@ -676,8 +676,8 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing", { model: "claude-auto-pilot-fable-v1-canary", model_id: "claude-auto-pilot-fable-v1-canary", - pricing_tier: "routed-unresolved", - totals: { billable_total_tokens: 1000, total_cost_usd: "0" }, + pricing_tier: "routed-estimated", + totals: { billable_total_tokens: 1000, total_cost_usd: "3" }, }, ], }, @@ -687,21 +687,24 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing", const [provider] = buildFleetData(modelBreakdown); assert.deepEqual( provider.missingPricingModels.map((m) => m.name), - ["brand-new-model", "claude-auto-pilot-fable-v1-canary"], - "both a miss and an unresolved composite route are unpriced", + ["brand-new-model"], + "only a true pricing miss is unpriced", ); assert.deepEqual( provider.fuzzyPricingModels.map((m) => m.name), - ["acme-9-turbo"], - "a substring-matched price is surfaced even though it is non-zero", + ["acme-9-turbo", "claude-auto-pilot-fable-v1-canary"], + "both substring and routed estimates are surfaced as non-exact pricing", ); const insights = buildUsageInsights(modelBreakdown); assert.deepEqual( insights.missingPricingModels.map((m) => m.name), - ["brand-new-model", "claude-auto-pilot-fable-v1-canary"], + ["brand-new-model"], + ); + assert.deepEqual( + insights.fuzzyPricingModels.map((m) => m.name), + ["acme-9-turbo", "claude-auto-pilot-fable-v1-canary"], ); - assert.deepEqual(insights.fuzzyPricingModels.map((m) => m.name), ["acme-9-turbo"]); }); test("dashboard model data keeps Hermes authoritative cost provenance distinct from fuzzy pricing", async () => { diff --git a/test/pricing-observability.test.js b/test/pricing-observability.test.js index ce1a9455..31f5366d 100644 --- a/test/pricing-observability.test.js +++ b/test/pricing-observability.test.js @@ -416,30 +416,54 @@ test("a real model that merely contains \"unknown\" is still priced or missed no assert.deepEqual(pricing.getPricingDiagnostics().unpriced_models, ["mystery-model"]); }); -test("a configured composite auto-router route is unpriced rather than fuzzy-matched as Fable", async () => { +test("configured auto-router routes use their approved child-model weighted estimates", async () => { const payload = { current: { - "anthropic/claude-fable-5": entry(10e-6, 50e-6), + "anthropic/claude-sonnet-5": { + input_cost_per_token: 2e-6, + output_cost_per_token: 10e-6, + cache_read_input_token_cost: 0.2e-6, + cache_creation_input_token_cost: 2.5e-6, + }, + "anthropic/claude-opus-5": { + input_cost_per_token: 5e-6, + output_cost_per_token: 25e-6, + cache_read_input_token_cost: 0.5e-6, + cache_creation_input_token_cost: 6.25e-6, + }, + "gpt-5.6-terra": { + input_cost_per_token: 2e-6, + output_cost_per_token: 12e-6, + cache_read_input_token_cost: 0.2e-6, + cache_creation_input_token_cost: 2.5e-6, + }, + "gpt-5.6-sol": { + input_cost_per_token: 4e-6, + output_cost_per_token: 20e-6, + cache_read_input_token_cost: 0.4e-6, + cache_creation_input_token_cost: 5e-6, + }, }, }; await loadWith(payload, tmpCachePath("composite-auto-router")); - const meta = pricing.getModelPricingMeta("claude-auto-pilot-fable-v1-canary", { source: "hermes" }); - assert.equal(meta.tier, "routed-unresolved"); - assert.deepEqual(meta.pricing, pricing.ZERO_PRICING); - assert.equal( - pricing.computeRowCost({ - source: "hermes", - model: "claude-auto-pilot-fable-v1-canary", - cached_input_tokens: 1_000_000, - }), - 0, - ); + const claude = pricing.getModelPricingMeta("claude-auto-pilot-fable-v1-canary", { source: "hermes" }); + assert.equal(claude.tier, "routed-estimated"); + assert.deepEqual(claude.pricing, { input: 3.2, output: 16, cache_read: 0.32, cache_write: 4 }); + const gpt = pricing.getModelPricingMeta("gpt-5.6-auto-pilot-055-v2", { source: "hermes" }); + assert.equal(gpt.tier, "routed-estimated"); + assert.deepEqual(gpt.pricing, { input: 2.8, output: 15.2, cache_read: 0.28, cache_write: 3.5 }); assert.deepEqual( pricing.getPricingDiagnostics().unpriced_models, - ["claude-auto-pilot-fable-v1-canary"], + [], + ); + assert.deepEqual( + pricing.getPricingDiagnostics().fuzzy_priced_models, + [ + { model: "claude-auto-pilot-fable-v1-canary", tier: "routed-estimated" }, + { model: "gpt-5.6-auto-pilot-055-v2", tier: "routed-estimated" }, + ], ); - assert.deepEqual(pricing.getPricingDiagnostics().fuzzy_priced_models, []); }); test("cost for an unattributed row is unchanged by the exemption", async () => { From f7f1f2725a7a03261796775efdee10844b5f4d01 Mon Sep 17 00:00:00 2001 From: "itarun.p" Date: Tue, 25 Aug 2026 08:10:28 +0700 Subject: [PATCH 2/2] fix: require exact child prices for route blends --- src/lib/pricing/index.js | 14 ++++++++---- test/pricing-observability.test.js | 35 ++++++++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/src/lib/pricing/index.js b/src/lib/pricing/index.js index 41f6c9fe..bef43aa4 100644 --- a/src/lib/pricing/index.js +++ b/src/lib/pricing/index.js @@ -78,8 +78,8 @@ const UNPRICED_TIERS = new Set(["miss", "routed-unresolved"]); // makes the route unpriced rather than silently using a partial blend. const ROUTED_ESTIMATE_POLICIES = new Map([ ["claude-auto-pilot-fable-v1-canary", [ - { model: "anthropic/claude-sonnet-5", weight: 0.6 }, - { model: "anthropic/claude-opus-5", weight: 0.4 }, + { model: "anthropic/claude-sonnet-5", pricingModel: "claude-sonnet-5", weight: 0.6 }, + { model: "anthropic/claude-opus-5", pricingModel: "claude-opus-5", weight: 0.4 }, ]], ["gpt-5.6-auto-pilot", [ { model: "gpt-5.6-terra", weight: 0.6 }, @@ -98,6 +98,12 @@ const ROUTED_ESTIMATE_POLICIES = new Map([ { model: "gpt-5.6-sol", weight: 0.4 }, ]], ]); +const ROUTE_CHILD_EXACT_TIERS = new Set([ + "curated:exact", + "curated:exact-dot", + "litellm:exact", + "litellm:exact-dot", +]); function isPotentialGptAutoRoute(model) { return /^gpt-5\.6-auto/i.test(String(model || "").trim()); @@ -108,12 +114,12 @@ function resolveRoutedEstimate(model, lookupSource) { if (!policy) return null; const mixed = { input: 0, output: 0, cache_read: 0, cache_write: 0 }; for (const child of policy) { - const result = lookupPricing(child.model, { + const result = lookupPricing(child.pricingModel || child.model, { curated: curatedOverrides, litellm: state.litellmPerMillionMap, source: lookupSource, }); - if (!result.hit || !result.value) return null; + if (!result.hit || !result.value || !ROUTE_CHILD_EXACT_TIERS.has(result.source)) return null; for (const field of Object.keys(mixed)) { if (!Number.isFinite(result.value[field])) return null; mixed[field] += child.weight * result.value[field]; diff --git a/test/pricing-observability.test.js b/test/pricing-observability.test.js index 31f5366d..3810cc58 100644 --- a/test/pricing-observability.test.js +++ b/test/pricing-observability.test.js @@ -419,13 +419,13 @@ test("a real model that merely contains \"unknown\" is still priced or missed no test("configured auto-router routes use their approved child-model weighted estimates", async () => { const payload = { current: { - "anthropic/claude-sonnet-5": { + "claude-sonnet-5": { input_cost_per_token: 2e-6, output_cost_per_token: 10e-6, cache_read_input_token_cost: 0.2e-6, cache_creation_input_token_cost: 2.5e-6, }, - "anthropic/claude-opus-5": { + "claude-opus-5": { input_cost_per_token: 5e-6, output_cost_per_token: 25e-6, cache_read_input_token_cost: 0.5e-6, @@ -466,6 +466,37 @@ test("configured auto-router routes use their approved child-model weighted esti ); }); +test("a routed estimate fails closed when any child price is only a fuzzy hit", async () => { + const payload = { + current: { + // The pricing policy asks for claude-sonnet-5. This shorter key would + // fuzzy-match it, but is not strong enough evidence for a weighted route. + "claude-sonnet": { + input_cost_per_token: 99e-6, + output_cost_per_token: 99e-6, + cache_read_input_token_cost: 99e-6, + cache_creation_input_token_cost: 99e-6, + }, + "claude-opus-5": { + input_cost_per_token: 5e-6, + output_cost_per_token: 25e-6, + cache_read_input_token_cost: 0.5e-6, + cache_creation_input_token_cost: 6.25e-6, + }, + }, + }; + await loadWith(payload, tmpCachePath("composite-child-fuzzy")); + + const meta = pricing.getModelPricingMeta("claude-auto-pilot-fable-v1-canary", { source: "hermes" }); + assert.equal(meta.tier, "routed-unresolved"); + assert.deepEqual(meta.pricing, pricing.ZERO_PRICING); + assert.deepEqual( + pricing.getPricingDiagnostics().unpriced_models, + ["claude-auto-pilot-fable-v1-canary"], + ); + assert.deepEqual(pricing.getPricingDiagnostics().fuzzy_priced_models, []); +}); + test("cost for an unattributed row is unchanged by the exemption", async () => { const payload = { current: { "acme-1": entry(1e-6, 2e-6) } }; await loadWith(payload, tmpCachePath("unattributed-cost"));