Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dashboard/src/content/copy.csv
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@ usage.overview.top_usage_model,ui,DashboardPage,UsageOverview,top_usage_model,To
usage.overview.missing_pricing_count,ui,DashboardPage,UsageOverview,missing_pricing_count,{{count}} pricing missing,,active
usage.overview.provider_top_cost,ui,DashboardPage,UsageOverview,provider_top_cost,Top cost: {{model}},,active
usage.overview.provider_missing_pricing,ui,DashboardPage,UsageOverview,provider_missing_pricing,{{count}} pricing missing,,active
usage.overview.provider_fuzzy_pricing,ui,DashboardPage,UsageOverview,provider_fuzzy_pricing,{{count}} pricing estimated,Shown when any model in this provider resolved via substring/fuzzy match rather than an exact price,active
usage.overview.provider_limit,ui,DashboardPage,UsageOverview,provider_limit,{{label}} · {{percent}}%,,active
usage.overview.provider_limit_reset,ui,DashboardPage,UsageOverview,provider_limit_reset,{{label}} · {{percent}}% · resets in {{reset}},,active
usage.overview.provider_limit_count,ui,DashboardPage,UsageOverview,provider_limit_count,{{used}}/{{limit}},,active
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ export function ProviderBreakdownCard({ fleetData = [], from, to, showInlineCont
const isExpanded = expandedProvider === provider.label;
const providerMissingPricingCount = provider.missingPricingModels?.length || 0;
const hasProviderMissingPricing = Boolean(providerMissingPricingCount);
const providerFuzzyPricingCount = provider.fuzzyPricingModels?.length || 0;
const hasProviderFuzzyPricing = Boolean(providerFuzzyPricingCount);
const visibleModels = getVisibleModels(provider);
const hiddenModelCount = getHiddenModelCount(provider, visibleModels);

Expand Down Expand Up @@ -180,6 +182,13 @@ export function ProviderBreakdownCard({ fleetData = [], from, to, showInlineCont
})}
</div>
) : null}
{hasProviderFuzzyPricing ? (
<div className="mt-1.5 truncate text-[10px] font-semibold text-amber-700 dark:text-amber-300">
{copy("usage.overview.provider_fuzzy_pricing", {
count: providerFuzzyPricingCount,
})}
</div>
) : null}
</button>
);
})}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,43 @@ describe("ProviderBreakdownCard", () => {
expect(screen.getByText("Top cost: fable-5")).toBeInTheDocument();
});

it("shows a fuzzy-pricing caveat when a provider has fuzzy-matched models", () => {
render(
<ProviderBreakdownCard
fleetData={[
{
source: "claude",
label: "CLAUDE",
totalPercent: "100.0",
usage: 26_000_000,
usd: 3164.69,
topCostModel: { name: "claude-auto-pilot-fable-v1-canary" },
missingPricingModels: [],
fuzzyPricingModels: [{ name: "claude-auto-pilot-fable-v1-canary" }],
models: [
{
id: "claude-auto-pilot-fable-v1-canary",
name: "claude-auto-pilot-fable-v1-canary",
share: 100,
usage: 26_000_000,
cost: 3164.69,
pricingTier: "curated:fuzzy",
},
],
},
]}
/>,
);

expect(screen.getByText("1 pricing estimated")).toBeInTheDocument();
});

it("does not show the fuzzy-pricing caveat when every model resolved exactly", () => {
render(<ProviderBreakdownCard fleetData={claudeFleet} />);

expect(screen.queryByText(/pricing estimated/)).not.toBeInTheDocument();
});

const claudeFleet = [
{
source: "claude",
Expand Down
4 changes: 2 additions & 2 deletions src/lib/pricing/curated-overrides.json
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,8 @@
{ "match": "kiro", "ref": "kiro-cli-agent" },
{ "match": "hy3", "ref": "hy3-preview-agent" },
{ "match": "composer", "ref": "composer-1" },
{ "match": "fable", "ref": "claude-fable-5" },
{ "match": "mythos", "ref": "claude-mythos-5" },
{ "match": "claude-fable", "ref": "claude-fable-5" },
{ "match": "claude-mythos", "ref": "claude-mythos-5" },
{ "match": "minimax-m2.7-highspeed", "ref": "MiniMax-M2.7-highspeed" },
{ "match": "minimax-m2.7", "ref": "MiniMax-M2.7" },
{ "match": "deepseek-v4-flash", "ref": "deepseek-v4-flash" },
Expand Down
37 changes: 36 additions & 1 deletion src/lib/pricing/matcher.js
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,26 @@ function lookupContainedExactCaseInsensitive(table, model) {
return null;
}

function lookupPricing(model, { curated, litellm, source } = {}) {
// Strips known "vendor build-label" noise so an already-priced model hiding
// under an auto-pilot/canary wrapper name can still resolve without a
// hand-added curated entry for every wrapped variant (issue #193). Scoped to
// the exact shapes seen in production: an "-auto-pilot-" infix used as a
// build-label separator, and a trailing "-vN-canary" / "-canary" release
// marker. Deliberately does NOT touch "-preview"/"-beta"/reasoning-effort
// suffixes (handled separately by stripReasoningSuffix) since those can carry
// real pricing distinctions.
const AUTO_PILOT_INFIX = /-auto-pilot-/g;
const CANARY_SUFFIX = /-v\d+-canary$|-canary$/;

function stripVendorRenameNoise(model) {
if (typeof model !== "string" || !model) return "";
let stripped = model.replace(AUTO_PILOT_INFIX, "-");
stripped = stripped.replace(CANARY_SUFFIX, "");
return stripped;
}

function lookupPricing(model, opts = {}) {
const { curated, litellm, source, _denoised = false } = opts;
if (!model || typeof model !== "string") {
return { hit: false, source: "empty", value: null };
}
Expand Down Expand Up @@ -202,6 +221,21 @@ function lookupPricing(model, { curated, litellm, source } = {}) {
}
}

// 8. Vendor-rename denoise retry (issue #193). Runs LAST and only once (the
// `_denoised` guard prevents re-entering this branch on the recursive
// call), after every tier above has already had first crack at the raw
// model id — so a real curated/LiteLLM entry that happens to already
// contain "-auto-pilot-" or "-canary" is never shadowed by the stripped
// retry. Only fires when stripping actually changed the string, so it
// cannot loop or duplicate work for models with no vendor-rename noise.
if (!_denoised) {
const denoised = stripVendorRenameNoise(model);
if (denoised && denoised !== model) {
const retry = lookupPricing(denoised, { curated, litellm, source, _denoised: true });
if (retry.hit) return retry;
}
}

return { hit: false, source: "miss", value: null };
}

Expand Down Expand Up @@ -249,6 +283,7 @@ function buildLitellmPerMillionMap(rawData) {
module.exports = {
lookupPricing,
stripReasoningSuffix,
stripVendorRenameNoise,
normalizeAntigravityModel,
convertLitellmEntry,
buildLitellmPerMillionMap,
Expand Down
94 changes: 94 additions & 0 deletions test/pricing.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,100 @@ test("matcher: lookupPricing reverse-substring picks longest matching key", () =
assert.equal(r.value.input, 2);
});

test("matcher: stripVendorRenameNoise removes auto-pilot infix and canary suffix", () => {
assert.equal(
matcher.stripVendorRenameNoise("claude-auto-pilot-fable-v1-canary"),
"claude-fable",
);
assert.equal(
matcher.stripVendorRenameNoise("gpt-5.6-auto-pilot-055-v2"),
"gpt-5.6-055-v2",
);
assert.equal(
matcher.stripVendorRenameNoise("claude-sonnet-4-6"),
"claude-sonnet-4-6",
"no vendor-rename noise present — string passes through unchanged",
);
assert.equal(matcher.stripVendorRenameNoise(""), "");
assert.equal(matcher.stripVendorRenameNoise(null), "");
});

test("matcher: vendor-rename denoise retry (issue #193) resolves a renamed model with NO curated fuzzy rule needed", () => {
// The whole point of the denoise tier: a future auto-pilot/canary rename of
// an already-priced model must resolve WITHOUT anyone hand-adding a fuzzy
// rule for it, unlike the fable/mythos case that predates this tier.
const litellm = { "claude-opus-4-8": { input: 5, output: 25 } };
const r = matcher.lookupPricing("claude-auto-pilot-opus-4-8-v1-canary", {
curated: { exact: {}, alias: {}, fuzzy: [] },
litellm,
});
assert.equal(r.hit, true);
assert.equal(r.source, "litellm:exact");
assert.equal(r.value.input, 5);
});

test("matcher: vendor-rename denoise retry only fires after every other tier has missed on the raw id", () => {
// A model whose RAW id (with the auto-pilot/canary noise still present) is
// itself a real curated/LiteLLM entry must resolve on the raw id, not the
// stripped one — the denoise retry must never shadow a genuine exact match.
const curated = {
exact: {
"claude-auto-pilot-fable-v1-canary": { input: 999, output: 999 },
},
alias: {},
fuzzy: [],
};
const r = matcher.lookupPricing("claude-auto-pilot-fable-v1-canary", {
curated,
litellm: {},
});
assert.equal(r.hit, true);
assert.equal(r.source, "curated:exact");
assert.equal(r.value.input, 999, "raw-id exact match must win over the denoised retry");
});

test("matcher: vendor-rename denoise retry does not loop or hit twice for a string with no noise", () => {
const r = matcher.lookupPricing("totally-unknown-xyz-2099", {
curated: { exact: {}, alias: {}, fuzzy: [] },
litellm: {},
});
assert.equal(r.hit, false);
});

test("matcher: fable/mythos fuzzy rules are scoped to a claude- prefix, not bare substring match (issue #193)", () => {
// Regression for the risk #193 flagged: a bare `{match:"fable"}` rule would
// silently price ANY future model containing that substring at the Opus-tier
// Fable 5 rate. The rule must require "claude-fable"/"claude-mythos", not
// just "fable"/"mythos", so an unrelated non-Claude model is left unpriced
// (miss) rather than mispriced.
const curated = {
exact: { "claude-fable-5": { input: 10, output: 50 } },
alias: {},
fuzzy: [
{ match: "claude-fable", ref: "claude-fable-5" },
{ match: "claude-mythos", ref: "claude-mythos-5" },
],
};
const unrelated = matcher.lookupPricing("some-vendor-fable-mini", {
curated,
litellm: {},
});
assert.equal(
unrelated.hit,
false,
"a non-Claude model merely containing the substring 'fable' must not inherit the Fable 5 price",
);

// The actual affected id still resolves — the prefix scope does not break it.
const affected = matcher.lookupPricing("claude-auto-pilot-fable-v1-canary", {
curated,
litellm: {},
});
assert.equal(affected.hit, true);
assert.equal(affected.source, "curated:fuzzy");
assert.equal(affected.value.input, 10);
});

test("matcher: lookupPricing returns miss for completely unknown model", () => {
const r = matcher.lookupPricing("totally-unknown-xyz-2099", {
curated: { exact: {}, alias: {}, fuzzy: [] },
Expand Down