Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion dashboard/src/lib/model-breakdown.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,12 @@ function resolveModelName(model: any, fallback: any) {
// Server-side resolution tiers that mean the price was guessed from a partial
// match rather than an exact model id (src/lib/pricing/index.js). A guessed
// price is plausible and therefore never looks wrong — worth flagging.
const FUZZY_PRICING_TIERS = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip"]);
const FUZZY_PRICING_TIERS = new Set([
"curated:fuzzy",
"litellm:fuzzy",
"litellm:prefix-strip",
"routed-estimated",
]);

// Server tiers that mean "$0 because we have no price", as opposed to a model
// that genuinely costs nothing. "unattributed"/"empty" rows carry no model id at
Expand Down
11 changes: 6 additions & 5 deletions dashboard/src/lib/plan-value.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,20 +74,21 @@ describe("computePlanValue", () => {
expect(value.unpricedModels).toEqual(["unknown"]);
});

it("counts an unresolved composite route as unpriced, not exact free usage", () => {
it("treats an approved composite-route blend as estimated, not exact", () => {
const value = computePlanValue(
source({
totals: { total_cost_usd: "0" },
totals: { total_cost_usd: "12" },
models: [{
model: "claude-auto-pilot-fable-v1-canary",
pricing_tier: "routed-unresolved",
pricing_tier: "routed-estimated",
}],
}),
20,
)!;
expect(value.listPriceUsd).toBe(0);
expect(value.listPriceUsd).toBe(12);
expect(value.confidence).toBe("floor");
expect(value.unpricedModels).toEqual(["claude-auto-pilot-fable-v1-canary"]);
expect(value.unpricedModels).toEqual([]);
expect(value.fuzzyModels).toEqual(["claude-auto-pilot-fable-v1-canary"]);
});

it("is exact when every model priced exactly", () => {
Expand Down
8 changes: 7 additions & 1 deletion dashboard/src/lib/plan-value.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,13 @@ const toNumber = (value: unknown): number => {
// be wrong in either direction. Kept as an explicit list rather than a
// "not exact" rule so a NEW tier has to be classified deliberately.
const UNPRICED_TIERS = new Set(["miss", "empty", "unattributed", "routed-unresolved"]);
const FUZZY_TIERS = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip", "litellm:strip"]);
const FUZZY_TIERS = new Set([
"curated:fuzzy",
"litellm:fuzzy",
"litellm:prefix-strip",
"litellm:strip",
"routed-estimated",
]);

function modelName(model: SourceModel): string {
return String(model.model || model.model_id || "").trim();
Expand Down
74 changes: 72 additions & 2 deletions src/lib/pricing/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,12 @@ const RELOAD_COOLDOWN_MS = 5 * 60 * 1000;
// curated fuzzy rule rather than an exact id, so the price is plausible but may
// belong to a different model. Worth surfacing — a wrong price never looks
// wrong, unlike a $0 one.
const FUZZY_SOURCES = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip"]);
const FUZZY_SOURCES = new Set([
"curated:fuzzy",
"litellm:fuzzy",
"litellm:prefix-strip",
"routed-estimated",
]);

// Placeholder ids that stand in for "this row has no model", so they resolve to
// the "unattributed" tier instead of being looked up and recorded as a miss.
Expand All @@ -67,6 +72,62 @@ const UNRESOLVED_LOGICAL_ROUTE_IDS = new Set([
]);
const UNPRICED_TIERS = new Set(["miss", "routed-unresolved"]);

// Owner-approved route mix estimates. The child-model ratios are a pricing
// policy, not routing telemetry: callers must surface `routed-estimated` as a
// non-exact price. Any configured child whose current price cannot be resolved
// makes the route unpriced rather than silently using a partial blend.
const ROUTED_ESTIMATE_POLICIES = new Map([
["claude-auto-pilot-fable-v1-canary", [
{ model: "anthropic/claude-sonnet-5", pricingModel: "claude-sonnet-5", weight: 0.6 },
{ model: "anthropic/claude-opus-5", pricingModel: "claude-opus-5", weight: 0.4 },
]],
["gpt-5.6-auto-pilot", [
{ model: "gpt-5.6-terra", weight: 0.6 },
{ model: "gpt-5.6-sol", weight: 0.4 },
]],
["gpt-5.6-auto-pilot-045-canary", [
{ model: "gpt-5.6-terra", weight: 0.6 },
{ model: "gpt-5.6-sol", weight: 0.4 },
]],
["gpt-5.6-auto-pilot-055-v2", [
{ model: "gpt-5.6-terra", weight: 0.6 },
{ model: "gpt-5.6-sol", weight: 0.4 },
]],
["gpt-5.6-auto-pilot-056-claude-reasoning-canary", [
{ model: "gpt-5.6-terra", weight: 0.6 },
{ model: "gpt-5.6-sol", weight: 0.4 },
]],
]);
const ROUTE_CHILD_EXACT_TIERS = new Set([
"curated:exact",
"curated:exact-dot",
"litellm:exact",
"litellm:exact-dot",
]);

function isPotentialGptAutoRoute(model) {
return /^gpt-5\.6-auto/i.test(String(model || "").trim());
}

function resolveRoutedEstimate(model, lookupSource) {
const policy = ROUTED_ESTIMATE_POLICIES.get(String(model || "").trim());
if (!policy) return null;
const mixed = { input: 0, output: 0, cache_read: 0, cache_write: 0 };
for (const child of policy) {
const result = lookupPricing(child.pricingModel || child.model, {
curated: curatedOverrides,
litellm: state.litellmPerMillionMap,
source: lookupSource,
});
if (!result.hit || !result.value || !ROUTE_CHILD_EXACT_TIERS.has(result.source)) return null;
for (const field of Object.keys(mixed)) {
if (!Number.isFinite(result.value[field])) return null;
mixed[field] += child.weight * result.value[field];
}
}
return mixed;
}

// `last_refresh_error` is served over HTTP to the dashboard, so it is built
// from CLOSED sets, never from an arbitrary value. A previous version accepted
// anything symbol-shaped, which a QA pass broke immediately: a 32-character
Expand Down Expand Up @@ -246,7 +307,16 @@ function getModelPricingMeta(model, opts = {}) {
const lookupSource = resolveLookupSource(opts);
const cacheKey = lookupSource ? `${lookupSource}\0${model}` : model;

if (UNRESOLVED_LOGICAL_ROUTE_IDS.has(String(model || "").trim())) {
const routedEstimate = resolveRoutedEstimate(model, lookupSource);
if (routedEstimate) {
state.tiers.set(cacheKey, { model, source: lookupSource, tier: "routed-estimated" });
return { pricing: routedEstimate, tier: "routed-estimated" };
}

if (
UNRESOLVED_LOGICAL_ROUTE_IDS.has(String(model || "").trim())
|| isPotentialGptAutoRoute(model)
) {
state.tiers.set(cacheKey, { model, source: lookupSource, tier: "routed-unresolved" });
return { pricing: ZERO_PRICING, tier: "routed-unresolved" };
}
Expand Down
19 changes: 11 additions & 8 deletions test/model-breakdown.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -676,8 +676,8 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing",
{
model: "claude-auto-pilot-fable-v1-canary",
model_id: "claude-auto-pilot-fable-v1-canary",
pricing_tier: "routed-unresolved",
totals: { billable_total_tokens: 1000, total_cost_usd: "0" },
pricing_tier: "routed-estimated",
totals: { billable_total_tokens: 1000, total_cost_usd: "3" },
},
],
},
Expand All @@ -687,21 +687,24 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing",
const [provider] = buildFleetData(modelBreakdown);
assert.deepEqual(
provider.missingPricingModels.map((m) => m.name),
["brand-new-model", "claude-auto-pilot-fable-v1-canary"],
"both a miss and an unresolved composite route are unpriced",
["brand-new-model"],
"only a true pricing miss is unpriced",
);
assert.deepEqual(
provider.fuzzyPricingModels.map((m) => m.name),
["acme-9-turbo"],
"a substring-matched price is surfaced even though it is non-zero",
["acme-9-turbo", "claude-auto-pilot-fable-v1-canary"],
"both substring and routed estimates are surfaced as non-exact pricing",
);

const insights = buildUsageInsights(modelBreakdown);
assert.deepEqual(
insights.missingPricingModels.map((m) => m.name),
["brand-new-model", "claude-auto-pilot-fable-v1-canary"],
["brand-new-model"],
);
assert.deepEqual(
insights.fuzzyPricingModels.map((m) => m.name),
["acme-9-turbo", "claude-auto-pilot-fable-v1-canary"],
);
assert.deepEqual(insights.fuzzyPricingModels.map((m) => m.name), ["acme-9-turbo"]);
});

test("dashboard model data keeps Hermes authoritative cost provenance distinct from fuzzy pricing", async () => {
Expand Down
75 changes: 65 additions & 10 deletions test/pricing-observability.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -416,25 +416,80 @@ test("a real model that merely contains \"unknown\" is still priced or missed no
assert.deepEqual(pricing.getPricingDiagnostics().unpriced_models, ["mystery-model"]);
});

test("a configured composite auto-router route is unpriced rather than fuzzy-matched as Fable", async () => {
test("configured auto-router routes use their approved child-model weighted estimates", async () => {
const payload = {
current: {
"anthropic/claude-fable-5": entry(10e-6, 50e-6),
"claude-sonnet-5": {
input_cost_per_token: 2e-6,
output_cost_per_token: 10e-6,
cache_read_input_token_cost: 0.2e-6,
cache_creation_input_token_cost: 2.5e-6,
},
"claude-opus-5": {
input_cost_per_token: 5e-6,
output_cost_per_token: 25e-6,
cache_read_input_token_cost: 0.5e-6,
cache_creation_input_token_cost: 6.25e-6,
},
"gpt-5.6-terra": {
input_cost_per_token: 2e-6,
output_cost_per_token: 12e-6,
cache_read_input_token_cost: 0.2e-6,
cache_creation_input_token_cost: 2.5e-6,
},
"gpt-5.6-sol": {
input_cost_per_token: 4e-6,
output_cost_per_token: 20e-6,
cache_read_input_token_cost: 0.4e-6,
cache_creation_input_token_cost: 5e-6,
},
},
};
await loadWith(payload, tmpCachePath("composite-auto-router"));

const claude = pricing.getModelPricingMeta("claude-auto-pilot-fable-v1-canary", { source: "hermes" });
assert.equal(claude.tier, "routed-estimated");
assert.deepEqual(claude.pricing, { input: 3.2, output: 16, cache_read: 0.32, cache_write: 4 });
const gpt = pricing.getModelPricingMeta("gpt-5.6-auto-pilot-055-v2", { source: "hermes" });
assert.equal(gpt.tier, "routed-estimated");
assert.deepEqual(gpt.pricing, { input: 2.8, output: 15.2, cache_read: 0.28, cache_write: 3.5 });
assert.deepEqual(
pricing.getPricingDiagnostics().unpriced_models,
[],
);
assert.deepEqual(
pricing.getPricingDiagnostics().fuzzy_priced_models,
[
{ model: "claude-auto-pilot-fable-v1-canary", tier: "routed-estimated" },
{ model: "gpt-5.6-auto-pilot-055-v2", tier: "routed-estimated" },
],
);
});

test("a routed estimate fails closed when any child price is only a fuzzy hit", async () => {
const payload = {
current: {
// The pricing policy asks for claude-sonnet-5. This shorter key would
// fuzzy-match it, but is not strong enough evidence for a weighted route.
"claude-sonnet": {
input_cost_per_token: 99e-6,
output_cost_per_token: 99e-6,
cache_read_input_token_cost: 99e-6,
cache_creation_input_token_cost: 99e-6,
},
"claude-opus-5": {
input_cost_per_token: 5e-6,
output_cost_per_token: 25e-6,
cache_read_input_token_cost: 0.5e-6,
cache_creation_input_token_cost: 6.25e-6,
},
},
};
await loadWith(payload, tmpCachePath("composite-child-fuzzy"));

const meta = pricing.getModelPricingMeta("claude-auto-pilot-fable-v1-canary", { source: "hermes" });
assert.equal(meta.tier, "routed-unresolved");
assert.deepEqual(meta.pricing, pricing.ZERO_PRICING);
assert.equal(
pricing.computeRowCost({
source: "hermes",
model: "claude-auto-pilot-fable-v1-canary",
cached_input_tokens: 1_000_000,
}),
0,
);
assert.deepEqual(
pricing.getPricingDiagnostics().unpriced_models,
["claude-auto-pilot-fable-v1-canary"],
Expand Down