From 1c4cb6cd5fa12c6c88ae9aec64675bb02cf14dcf Mon Sep 17 00:00:00 2001 From: "itarun.p" Date: Sat, 22 Aug 2026 22:25:16 +0700 Subject: [PATCH] fix(pricing): general vendor-rename normalizer + fuzzy-pricing UI disclosure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups from the #187/#188 cross-verification review. 1. matcher.js: add a scoped denoise retry (issue #193). #188 fixed the fable/mythos canary regression with a per-name curated fuzzy rule, but left the structural gap open — any future auto-pilot/canary rename of an already-priced model would reproduce the same bug and require another one-line patch. The denoise retry strips the '-auto-pilot-' infix and trailing '-vN-canary'/'-canary' suffix and retries the full lookup ladder exactly once, only after every other tier has missed on the raw id, so a genuine exact match on the raw id (however unlikely) is never shadowed. Also narrows the #188 fuzzy rules from bare 'fable'/'mythos' substrings to 'claude-fable'/'claude-mythos'. Verified against the published pricing seed that a real collision risk existed: claude-mythos-preview is a genuine LiteLLM-priced model that happens to contain the substring 'mythos' — the bare rule would have silently mispriced any future model in the same shape that lacks its own exact LiteLLM/curated entry. 2. dashboard: wire the already-computed fuzzyPricingModels (from model-breakdown.ts) into ProviderBreakdownCard (issue #192). The provider card already discloses '{{count}} pricing missing' for unpriced models; it now also discloses '{{count}} pricing estimated' when any model resolved via substring/fuzzy match rather than an exact price, so a user sees when a total like $3,164.69 is a curated estimate, not a reconciled invoice figure. Verified: node --test test/pricing.test.js (39/39), dashboard ProviderBreakdownCard suite (10/10, incl. 2 new), npm run ci:local full gate (dashboard 320/320, product suite 1059/1059, all validators) — all green. Addresses #193, #192. --- dashboard/src/content/copy.csv | 1 + .../components/ProviderBreakdownCard.jsx | 9 ++ .../__tests__/ProviderBreakdownCard.test.jsx | 37 ++++++++ src/lib/pricing/curated-overrides.json | 4 +- src/lib/pricing/matcher.js | 37 +++++++- test/pricing.test.js | 94 +++++++++++++++++++ 6 files changed, 179 insertions(+), 3 deletions(-) diff --git a/dashboard/src/content/copy.csv b/dashboard/src/content/copy.csv index 3bd5e7185..ab14e4744 100644 --- a/dashboard/src/content/copy.csv +++ b/dashboard/src/content/copy.csv @@ -232,6 +232,7 @@ usage.overview.top_usage_model,ui,DashboardPage,UsageOverview,top_usage_model,To usage.overview.missing_pricing_count,ui,DashboardPage,UsageOverview,missing_pricing_count,{{count}} pricing missing,,active usage.overview.provider_top_cost,ui,DashboardPage,UsageOverview,provider_top_cost,Top cost: {{model}},,active usage.overview.provider_missing_pricing,ui,DashboardPage,UsageOverview,provider_missing_pricing,{{count}} pricing missing,,active +usage.overview.provider_fuzzy_pricing,ui,DashboardPage,UsageOverview,provider_fuzzy_pricing,{{count}} pricing estimated,Shown when any model in this provider resolved via substring/fuzzy match rather than an exact price,active usage.overview.provider_limit,ui,DashboardPage,UsageOverview,provider_limit,{{label}} · {{percent}}%,,active usage.overview.provider_limit_reset,ui,DashboardPage,UsageOverview,provider_limit_reset,{{label}} · {{percent}}% · resets in {{reset}},,active usage.overview.provider_limit_count,ui,DashboardPage,UsageOverview,provider_limit_count,{{used}}/{{limit}},,active diff --git a/dashboard/src/ui/dashboard/components/ProviderBreakdownCard.jsx b/dashboard/src/ui/dashboard/components/ProviderBreakdownCard.jsx index 348ac1c98..267ff8182 100644 --- a/dashboard/src/ui/dashboard/components/ProviderBreakdownCard.jsx +++ b/dashboard/src/ui/dashboard/components/ProviderBreakdownCard.jsx @@ -77,6 +77,8 @@ export function ProviderBreakdownCard({ fleetData = [], from, to, showInlineCont const isExpanded = expandedProvider === provider.label; const providerMissingPricingCount = provider.missingPricingModels?.length || 0; const hasProviderMissingPricing = Boolean(providerMissingPricingCount); + const providerFuzzyPricingCount = provider.fuzzyPricingModels?.length || 0; + const hasProviderFuzzyPricing = Boolean(providerFuzzyPricingCount); const visibleModels = getVisibleModels(provider); const hiddenModelCount = getHiddenModelCount(provider, visibleModels); @@ -180,6 +182,13 @@ export function ProviderBreakdownCard({ fleetData = [], from, to, showInlineCont })} ) : null} + {hasProviderFuzzyPricing ? ( +
+ {copy("usage.overview.provider_fuzzy_pricing", { + count: providerFuzzyPricingCount, + })} +
+ ) : null} ); })} diff --git a/dashboard/src/ui/dashboard/components/__tests__/ProviderBreakdownCard.test.jsx b/dashboard/src/ui/dashboard/components/__tests__/ProviderBreakdownCard.test.jsx index 6c270ec03..6e5ec0524 100644 --- a/dashboard/src/ui/dashboard/components/__tests__/ProviderBreakdownCard.test.jsx +++ b/dashboard/src/ui/dashboard/components/__tests__/ProviderBreakdownCard.test.jsx @@ -89,6 +89,43 @@ describe("ProviderBreakdownCard", () => { expect(screen.getByText("Top cost: fable-5")).toBeInTheDocument(); }); + it("shows a fuzzy-pricing caveat when a provider has fuzzy-matched models", () => { + render( + , + ); + + expect(screen.getByText("1 pricing estimated")).toBeInTheDocument(); + }); + + it("does not show the fuzzy-pricing caveat when every model resolved exactly", () => { + render(); + + expect(screen.queryByText(/pricing estimated/)).not.toBeInTheDocument(); + }); + const claudeFleet = [ { source: "claude", diff --git a/src/lib/pricing/curated-overrides.json b/src/lib/pricing/curated-overrides.json index 076a58c22..bacdf26a5 100644 --- a/src/lib/pricing/curated-overrides.json +++ b/src/lib/pricing/curated-overrides.json @@ -68,8 +68,8 @@ { "match": "kiro", "ref": "kiro-cli-agent" }, { "match": "hy3", "ref": "hy3-preview-agent" }, { "match": "composer", "ref": "composer-1" }, - { "match": "fable", "ref": "claude-fable-5" }, - { "match": "mythos", "ref": "claude-mythos-5" }, + { "match": "claude-fable", "ref": "claude-fable-5" }, + { "match": "claude-mythos", "ref": "claude-mythos-5" }, { "match": "minimax-m2.7-highspeed", "ref": "MiniMax-M2.7-highspeed" }, { "match": "minimax-m2.7", "ref": "MiniMax-M2.7" }, { "match": "deepseek-v4-flash", "ref": "deepseek-v4-flash" }, diff --git a/src/lib/pricing/matcher.js b/src/lib/pricing/matcher.js index 4b27ad613..6ee684d8e 100644 --- a/src/lib/pricing/matcher.js +++ b/src/lib/pricing/matcher.js @@ -100,7 +100,26 @@ function lookupContainedExactCaseInsensitive(table, model) { return null; } -function lookupPricing(model, { curated, litellm, source } = {}) { +// Strips known "vendor build-label" noise so an already-priced model hiding +// under an auto-pilot/canary wrapper name can still resolve without a +// hand-added curated entry for every wrapped variant (issue #193). Scoped to +// the exact shapes seen in production: an "-auto-pilot-" infix used as a +// build-label separator, and a trailing "-vN-canary" / "-canary" release +// marker. Deliberately does NOT touch "-preview"/"-beta"/reasoning-effort +// suffixes (handled separately by stripReasoningSuffix) since those can carry +// real pricing distinctions. +const AUTO_PILOT_INFIX = /-auto-pilot-/g; +const CANARY_SUFFIX = /-v\d+-canary$|-canary$/; + +function stripVendorRenameNoise(model) { + if (typeof model !== "string" || !model) return ""; + let stripped = model.replace(AUTO_PILOT_INFIX, "-"); + stripped = stripped.replace(CANARY_SUFFIX, ""); + return stripped; +} + +function lookupPricing(model, opts = {}) { + const { curated, litellm, source, _denoised = false } = opts; if (!model || typeof model !== "string") { return { hit: false, source: "empty", value: null }; } @@ -202,6 +221,21 @@ function lookupPricing(model, { curated, litellm, source } = {}) { } } + // 8. Vendor-rename denoise retry (issue #193). Runs LAST and only once (the + // `_denoised` guard prevents re-entering this branch on the recursive + // call), after every tier above has already had first crack at the raw + // model id — so a real curated/LiteLLM entry that happens to already + // contain "-auto-pilot-" or "-canary" is never shadowed by the stripped + // retry. Only fires when stripping actually changed the string, so it + // cannot loop or duplicate work for models with no vendor-rename noise. + if (!_denoised) { + const denoised = stripVendorRenameNoise(model); + if (denoised && denoised !== model) { + const retry = lookupPricing(denoised, { curated, litellm, source, _denoised: true }); + if (retry.hit) return retry; + } + } + return { hit: false, source: "miss", value: null }; } @@ -249,6 +283,7 @@ function buildLitellmPerMillionMap(rawData) { module.exports = { lookupPricing, stripReasoningSuffix, + stripVendorRenameNoise, normalizeAntigravityModel, convertLitellmEntry, buildLitellmPerMillionMap, diff --git a/test/pricing.test.js b/test/pricing.test.js index 5884df39e..a36707b0f 100644 --- a/test/pricing.test.js +++ b/test/pricing.test.js @@ -262,6 +262,100 @@ test("matcher: lookupPricing reverse-substring picks longest matching key", () = assert.equal(r.value.input, 2); }); +test("matcher: stripVendorRenameNoise removes auto-pilot infix and canary suffix", () => { + assert.equal( + matcher.stripVendorRenameNoise("claude-auto-pilot-fable-v1-canary"), + "claude-fable", + ); + assert.equal( + matcher.stripVendorRenameNoise("gpt-5.6-auto-pilot-055-v2"), + "gpt-5.6-055-v2", + ); + assert.equal( + matcher.stripVendorRenameNoise("claude-sonnet-4-6"), + "claude-sonnet-4-6", + "no vendor-rename noise present — string passes through unchanged", + ); + assert.equal(matcher.stripVendorRenameNoise(""), ""); + assert.equal(matcher.stripVendorRenameNoise(null), ""); +}); + +test("matcher: vendor-rename denoise retry (issue #193) resolves a renamed model with NO curated fuzzy rule needed", () => { + // The whole point of the denoise tier: a future auto-pilot/canary rename of + // an already-priced model must resolve WITHOUT anyone hand-adding a fuzzy + // rule for it, unlike the fable/mythos case that predates this tier. + const litellm = { "claude-opus-4-8": { input: 5, output: 25 } }; + const r = matcher.lookupPricing("claude-auto-pilot-opus-4-8-v1-canary", { + curated: { exact: {}, alias: {}, fuzzy: [] }, + litellm, + }); + assert.equal(r.hit, true); + assert.equal(r.source, "litellm:exact"); + assert.equal(r.value.input, 5); +}); + +test("matcher: vendor-rename denoise retry only fires after every other tier has missed on the raw id", () => { + // A model whose RAW id (with the auto-pilot/canary noise still present) is + // itself a real curated/LiteLLM entry must resolve on the raw id, not the + // stripped one — the denoise retry must never shadow a genuine exact match. + const curated = { + exact: { + "claude-auto-pilot-fable-v1-canary": { input: 999, output: 999 }, + }, + alias: {}, + fuzzy: [], + }; + const r = matcher.lookupPricing("claude-auto-pilot-fable-v1-canary", { + curated, + litellm: {}, + }); + assert.equal(r.hit, true); + assert.equal(r.source, "curated:exact"); + assert.equal(r.value.input, 999, "raw-id exact match must win over the denoised retry"); +}); + +test("matcher: vendor-rename denoise retry does not loop or hit twice for a string with no noise", () => { + const r = matcher.lookupPricing("totally-unknown-xyz-2099", { + curated: { exact: {}, alias: {}, fuzzy: [] }, + litellm: {}, + }); + assert.equal(r.hit, false); +}); + +test("matcher: fable/mythos fuzzy rules are scoped to a claude- prefix, not bare substring match (issue #193)", () => { + // Regression for the risk #193 flagged: a bare `{match:"fable"}` rule would + // silently price ANY future model containing that substring at the Opus-tier + // Fable 5 rate. The rule must require "claude-fable"/"claude-mythos", not + // just "fable"/"mythos", so an unrelated non-Claude model is left unpriced + // (miss) rather than mispriced. + const curated = { + exact: { "claude-fable-5": { input: 10, output: 50 } }, + alias: {}, + fuzzy: [ + { match: "claude-fable", ref: "claude-fable-5" }, + { match: "claude-mythos", ref: "claude-mythos-5" }, + ], + }; + const unrelated = matcher.lookupPricing("some-vendor-fable-mini", { + curated, + litellm: {}, + }); + assert.equal( + unrelated.hit, + false, + "a non-Claude model merely containing the substring 'fable' must not inherit the Fable 5 price", + ); + + // The actual affected id still resolves — the prefix scope does not break it. + const affected = matcher.lookupPricing("claude-auto-pilot-fable-v1-canary", { + curated, + litellm: {}, + }); + assert.equal(affected.hit, true); + assert.equal(affected.source, "curated:fuzzy"); + assert.equal(affected.value.input, 10); +}); + test("matcher: lookupPricing returns miss for completely unknown model", () => { const r = matcher.lookupPricing("totally-unknown-xyz-2099", { curated: { exact: {}, alias: {}, fuzzy: [] },