Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dashboard/src/lib/model-breakdown.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ const FUZZY_PRICING_TIERS = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:
// all, but their tokens still count toward the total, so the same caveat applies
// — they are excluded from the server's unpriced_models list (which exists to
// name models needing a curated price) but must not silently read as priced.
const UNPRICED_PRICING_TIERS = new Set(["miss", "unattributed", "empty"]);
const UNPRICED_PRICING_TIERS = new Set(["miss", "unattributed", "empty", "routed-unresolved"]);

function isKnownZeroCostModel(name: any) {
const lower = String(name || "").toLowerCase();
Expand Down
16 changes: 16 additions & 0 deletions dashboard/src/lib/plan-value.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,22 @@ describe("computePlanValue", () => {
expect(value.unpricedModels).toEqual(["unknown"]);
});

it("counts an unresolved composite route as unpriced, not exact free usage", () => {
const value = computePlanValue(
source({
totals: { total_cost_usd: "0" },
models: [{
model: "claude-auto-pilot-fable-v1-canary",
pricing_tier: "routed-unresolved",
}],
}),
20,
)!;
expect(value.listPriceUsd).toBe(0);
expect(value.confidence).toBe("floor");
expect(value.unpricedModels).toEqual(["claude-auto-pilot-fable-v1-canary"]);
});

it("is exact when every model priced exactly", () => {
const value = computePlanValue(
source({
Expand Down
2 changes: 1 addition & 1 deletion dashboard/src/lib/plan-value.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ const toNumber = (value: unknown): number => {
// price at all; the fuzzy tiers matched by substring, which is a guess that can
// be wrong in either direction. Kept as an explicit list rather than a
// "not exact" rule so a NEW tier has to be classified deliberately.
const UNPRICED_TIERS = new Set(["miss", "empty", "unattributed"]);
const UNPRICED_TIERS = new Set(["miss", "empty", "unattributed", "routed-unresolved"]);
const FUZZY_TIERS = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix-strip", "litellm:strip"]);

function modelName(model: SourceModel): string {
Expand Down
18 changes: 17 additions & 1 deletion src/lib/pricing/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,17 @@ const FUZZY_SOURCES = new Set(["curated:fuzzy", "litellm:fuzzy", "litellm:prefix
// Closed set on purpose: a real model id must never be silently un-priced here.
const UNATTRIBUTED_MODEL_IDS = new Set(["unknown"]);

// These ids are logical routes that deliberately select a physical child model
// per request. Their names can include one tier name, but that is not evidence
// that every request used that tier. Do not run them through fuzzy pricing: a
// route-level estimate would silently charge every request at the wrong child
// rate. They stay visible as unpriced until per-request resolved-model or
// authoritative cost data is persisted.
const UNRESOLVED_LOGICAL_ROUTE_IDS = new Set([
"claude-auto-pilot-fable-v1-canary",
]);
const UNPRICED_TIERS = new Set(["miss", "routed-unresolved"]);

// `last_refresh_error` is served over HTTP to the dashboard, so it is built
// from CLOSED sets, never from an arbitrary value. A previous version accepted
// anything symbol-shaped, which a QA pass broke immediately: a 32-character
Expand Down Expand Up @@ -235,6 +246,11 @@ function getModelPricingMeta(model, opts = {}) {
const lookupSource = resolveLookupSource(opts);
const cacheKey = lookupSource ? `${lookupSource}\0${model}` : model;

if (UNRESOLVED_LOGICAL_ROUTE_IDS.has(String(model || "").trim())) {
state.tiers.set(cacheKey, { model, source: lookupSource, tier: "routed-unresolved" });
return { pricing: ZERO_PRICING, tier: "routed-unresolved" };
}

if (state.negativeCache.has(cacheKey)) {
// Still unknown as of the current snapshot. If that snapshot has aged out,
// a new model may have appeared upstream — refresh for the next caller.
Expand Down Expand Up @@ -287,7 +303,7 @@ function getPricingDiagnostics() {
const unpriced = new Set();
const fuzzy = [];
for (const entry of state.tiers.values()) {
if (entry.tier === "miss") unpriced.add(entry.model);
if (UNPRICED_TIERS.has(entry.tier)) unpriced.add(entry.model);
else if (FUZZY_SOURCES.has(entry.tier)) fuzzy.push({ model: entry.model, tier: entry.tier });
}
return {
Expand Down
15 changes: 12 additions & 3 deletions test/model-breakdown.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -673,6 +673,12 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing",
pricing_tier: "miss",
totals: { billable_total_tokens: 1000, total_cost_usd: "0" },
},
{
model: "claude-auto-pilot-fable-v1-canary",
model_id: "claude-auto-pilot-fable-v1-canary",
pricing_tier: "routed-unresolved",
totals: { billable_total_tokens: 1000, total_cost_usd: "0" },
},
],
},
],
Expand All @@ -681,8 +687,8 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing",
const [provider] = buildFleetData(modelBreakdown);
assert.deepEqual(
provider.missingPricingModels.map((m) => m.name),
["brand-new-model"],
"only the tier=miss model is unpriced",
["brand-new-model", "claude-auto-pilot-fable-v1-canary"],
"both a miss and an unresolved composite route are unpriced",
);
assert.deepEqual(
provider.fuzzyPricingModels.map((m) => m.name),
Expand All @@ -691,7 +697,10 @@ test("pricing_tier from the server beats the cost<=0 guess for missing pricing",
);

const insights = buildUsageInsights(modelBreakdown);
assert.deepEqual(insights.missingPricingModels.map((m) => m.name), ["brand-new-model"]);
assert.deepEqual(
insights.missingPricingModels.map((m) => m.name),
["brand-new-model", "claude-auto-pilot-fable-v1-canary"],
);
assert.deepEqual(insights.fuzzyPricingModels.map((m) => m.name), ["acme-9-turbo"]);
});

Expand Down
26 changes: 26 additions & 0 deletions test/pricing-observability.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,32 @@ test("a real model that merely contains \"unknown\" is still priced or missed no
assert.deepEqual(pricing.getPricingDiagnostics().unpriced_models, ["mystery-model"]);
});

test("a configured composite auto-router route is unpriced rather than fuzzy-matched as Fable", async () => {
const payload = {
current: {
"anthropic/claude-fable-5": entry(10e-6, 50e-6),
},
};
await loadWith(payload, tmpCachePath("composite-auto-router"));

const meta = pricing.getModelPricingMeta("claude-auto-pilot-fable-v1-canary", { source: "hermes" });
assert.equal(meta.tier, "routed-unresolved");
assert.deepEqual(meta.pricing, pricing.ZERO_PRICING);
assert.equal(
pricing.computeRowCost({
source: "hermes",
model: "claude-auto-pilot-fable-v1-canary",
cached_input_tokens: 1_000_000,
}),
0,
);
assert.deepEqual(
pricing.getPricingDiagnostics().unpriced_models,
["claude-auto-pilot-fable-v1-canary"],
);
assert.deepEqual(pricing.getPricingDiagnostics().fuzzy_priced_models, []);
});

test("cost for an unattributed row is unchanged by the exemption", async () => {
const payload = { current: { "acme-1": entry(1e-6, 2e-6) } };
await loadWith(payload, tmpCachePath("unattributed-cost"));
Expand Down
15 changes: 6 additions & 9 deletions test/pricing.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -569,15 +569,12 @@ test("index: getModelPricing pins Claude Fable/Mythos 5 pricing from curated ove
assert.equal(pinned.cache_write, 12.5);
}

// Regression (issue #187): auto-pilot/canary variants wrap the pinned name
// with extra prefix/suffix segments (e.g. "claude-auto-pilot-fable-v1-canary"),
// so "fable"/"mythos" is not a contiguous substring of "claude-fable-5"/
// "claude-mythos-5" — every non-fuzzy resolution tier misses on these ids.
for (const model of ["claude-auto-pilot-fable-v1-canary", "claude-mythos-v1-canary"]) {
const pinned = pricing.getModelPricing(model);
assert.equal(pinned.input, 10, `${model} should resolve to the Fable/Mythos pin`);
assert.equal(pinned.output, 50, `${model} should resolve to the Fable/Mythos pin`);
}
// A simple renamed Mythos variant can still inherit the curated pin. The
// Fable auto-router label is intentionally excluded: it selects Sonnet,
// Opus, or Fable per request and cannot be priced as one child model.
const mythos = pricing.getModelPricing("claude-mythos-v1-canary");
assert.equal(mythos.input, 10);
assert.equal(mythos.output, 50);

assert.equal(
pricing.computeRowCost({
Expand Down