Skip to content
7 changes: 4 additions & 3 deletions src/billing.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,9 +107,11 @@ function nonNegNumStrict(v: unknown): number | undefined | typeof REJECT {
* reload path funnels through it) and by nothing else. Keeps deep settle
* sites free of options plumbing, mirroring the registry costCache pattern. */
let overridePrices: Record<string, BillingPrice> = {};
let overrideCostRows: CostsShape | null = null;

export function setBillingOverrides(prices: Record<string, BillingPrice> | undefined): void {
overridePrices = prices ?? {};
overrideCostRows = overrideCosts();
}

export function billingOverrides(): Record<string, BillingPrice> {
Expand All @@ -130,9 +132,8 @@ function overrideCosts(): CostsShape | null {

export function resolveBillingPrice(model: string | undefined, host?: string): ResolvedBillingPrice | undefined {
if (!model) return undefined;
const costs = overrideCosts();
if (costs !== null) {
const hit = scanRegistryCostRow(costs, model, host, { conflictHint: "billing.prices" });
if (overrideCostRows !== null) {
const hit = scanRegistryCostRow(overrideCostRows, model, host, { conflictHint: "billing.prices" });
if (hit) return finalize(hit.row, "config");
}
const row = peekRegistryCostRow(model, host);
Expand Down
22 changes: 20 additions & 2 deletions src/preflight.ts
Original file line number Diff line number Diff line change
Expand Up @@ -464,10 +464,27 @@ export function clearEstimateCoreMessagesMemoForTests(): void {
// the #496/#1800 billing arbitration. OpenAI chat `image_url.url`, Anthropic
// `source.data` and Gemini `inline_data.data` were already excluded.
const NON_TEXT_BODY_KEYS = new Set(["data", "url", "b64_json", "file_data", "image_url"]);
// DSH official-API extension plugins attach transport-only top-level sibling fields
// (such as `dsh_session_log` up to 8 MiB and `dsh_plugin_packages`) that consume 0 model
// tokens and are stripped/ignored before inference, whereas nested fields inside
// `messages[]` remain model-visible and must still be counted.
const NON_MODEL_TOP_LEVEL_KEYS = new Set(["dsh_session_log", "dsh_plugin_packages"]);

function isNonModelTopLevelKey(key: string): boolean {
return NON_MODEL_TOP_LEVEL_KEYS.has(key) || key.startsWith("dsh_");
}

export function estimateRawBodyTokens(parsed: unknown): number {
let tokens = 0;
const pending: [unknown, string | undefined][] = [[parsed, undefined]];
const pending: [unknown, string | undefined][] = [];
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
if (isNonModelTopLevelKey(k)) continue;
pending.push([v, k]);
}
} else {
pending.push([parsed, undefined]);
}
while (pending.length > 0) {
const [value, key] = pending.pop()!;
if (typeof value === "string") {
Expand Down Expand Up @@ -503,8 +520,9 @@ export interface RawBodyBreakdown {
export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown {
const buckets: [number, number, number, number] = [0, 0, 0, 0]; // messages, tools, system, other
const pending: [unknown, string | undefined, number][] = [];
if (parsed && typeof parsed === "object") {
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
if (isNonModelTopLevelKey(k)) continue;
const bucket = k === "messages" ? 0 : k === "tools" ? 1 : k === "system" ? 2 : 3;
pending.push([v, k, bucket]);
}
Expand Down
60 changes: 52 additions & 8 deletions src/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,26 @@ type LoadedRegistry = { reg: RegistryShape; costs: CostsShape | null };
let cache: RegistryShape | null = null;
let costCache: CostsShape | null = null;
let loading: Promise<RegistryShape | null> | null = null;
const FAILED_FETCH_BACKOFF_MS = 10 * 60 * 1000;
let lastFailedFetchMs = 0;
let requestPathBudgetMs = 3_000;
export function _setRequestPathBudgetForTest(ms: number | undefined): void {
requestPathBudgetMs = ms ?? 3_000;
}
const warnedConflicts = new Set<string>();
const warnedPriceConflicts = new Set<string>();
// Case-folded key indexes for the roster maps (#2074): models.dev keys are
// mixed-case ("minimax/MiniMax-M3", "google/gemma-4-E2B-it") while relay ids
// often arrive lowercased — exact-key hits and suffix scans were case-
// sensitive on both sides, so any case drift silently fell back to the
// built-in table / default window. One slot per source object; identity-
// checked, so loadRegistry / _setForTest swaps invalidate automatically.
// built-in table / default window. Identity-checked, so loadRegistry /
// _setForTest swaps invalidate automatically. regLowerIdx is a single slot
// (one model registry at a time); costLowerIdx is a small Map because the
// billing path scans TWO cost tables per lookup (the overrides table, then the
// registry fallback) and a single slot evicted one on the other, rebuilding
// the large registry index on every overrides-miss lookup (#7278).
let regLowerIdx: { reg: RegistryShape; idx: LowerKeyIndex<ModelEntry> } | null = null;
let costLowerIdx: { costs: CostsShape; idx: LowerKeyIndex<CostRow> } | null = null;
let costLowerIdx: Map<CostsShape, LowerKeyIndex<CostRow>> | null = null;

/** Full models.dev snapshot committed at src/registry-snapshot.json
* (refresh with `npm run registry:snapshot`) and inlined into dist at build
Expand Down Expand Up @@ -242,6 +252,7 @@ export async function loadRegistry(): Promise<RegistryShape | null> {
if (loading) return loading;
const pending = (async () => {
const fresh = await fetchFresh();
lastFailedFetchMs = fresh ? 0 : Date.now();
if (fresh) {
cache = fresh.reg;
costCache = fresh.costs;
Expand Down Expand Up @@ -358,8 +369,23 @@ export function providerFromHost(host: string): string | undefined {
}

export async function contextFromRegistry(model: string, host?: string): Promise<number | undefined> {
const reg = await loadRegistry();
return registryLookup(reg, model, host);
// Request path (server/handle.ts awaits this for a model the warm cache
// misses). The fetch watchdog is an idle timer, so a models.dev that
// trickles bytes holds the load open far past its 15 s budget, and a
// failed fetch leaves `cache` on a fallback so loadRegistry() fetches
// again on every call. Wait at most requestPathBudgetMs (the load keeps
// warming the cache in the background) and skip the wait entirely during
// the backoff after a failure; loadRegistry() itself still retries per call.
if (cache && Date.now() - lastFailedFetchMs < FAILED_FETCH_BACKOFF_MS) return registryLookup(cache, model, host);
let timer: ReturnType<typeof setTimeout> | undefined;
const budget = new Promise<RegistryShape | null>((resolve) => {
timer = setTimeout(() => resolve(cache), requestPathBudgetMs);
});
try {
return registryLookup(await Promise.race([loadRegistry(), budget]), model, host);
} finally {
clearTimeout(timer);
}
}

/** Synchronous cache-only lookup: returns the window when the models.dev
Expand Down Expand Up @@ -582,10 +608,27 @@ function registryLowerIndex(reg: RegistryShape): LowerKeyIndex<ModelEntry> {
return regLowerIdx.idx;
}

// Cap is exactly the two cost tables that ever flow through the billing path
// (the overrides table and the registry snapshot); evict the least-recently-
// used on a new table so both stay hot across overrides-miss lookups (#7278).
const COST_IDX_CAP = 2;
function costLowerIndex(costs: CostsShape): LowerKeyIndex<CostRow> {
if (costLowerIdx && costLowerIdx.costs === costs) return costLowerIdx.idx;
costLowerIdx = { costs, idx: buildLowerKeyIndex(costs) };
return costLowerIdx.idx;
if (!costLowerIdx) costLowerIdx = new Map();
const cached = costLowerIdx.get(costs);
if (cached) {
// Refresh recency (Map preserves insertion order; re-set moves to end).
costLowerIdx.delete(costs);
costLowerIdx.set(costs, cached);
return cached;
}
const idx = buildLowerKeyIndex(costs);
costLowerIdx.set(costs, idx);
while (costLowerIdx.size > COST_IDX_CAP) {
const oldest = costLowerIdx.keys().next().value;
if (oldest === undefined) break;
costLowerIdx.delete(oldest);
}
return idx;
}

function registryLookup(reg: RegistryShape | null, model: string, host: string | undefined, field: "context" | "output"): number | undefined;
Expand Down Expand Up @@ -659,6 +702,7 @@ export function _resetForTest(): void {
cache = null;
costCache = null;
loading = null;
lastFailedFetchMs = 0;
regLowerIdx = null;
costLowerIdx = null;
warnedConflicts.clear();
Expand Down
38 changes: 38 additions & 0 deletions tests/billing-prices.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
setBillingOverrides,
stampBillingPrice,
stampedBillingPrice,
type BillingPrice,
} from "../src/billing.ts";
import { _setForTest as setRegistryForTest } from "../src/registry.ts";
import { _resetPluginStateForTest } from "../src/plugin.ts";
Expand Down Expand Up @@ -125,6 +126,43 @@ test("#4899 resolveBillingPrice: overrides win over the registry, matching model
assert.equal(resolveBillingPrice(""), undefined);
});

test("billing override reloads replace and clear normalized price rows", () => {
resetState();
setRegistryForTest({}, { "custom/model": { input: 2, output: 8 } });
setBillingOverrides({ "custom/model": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 } });
const initial = resolveBillingPrice("custom/model");
assert.deepEqual(initial, { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75, source: "config" });

setBillingOverrides({ "custom/other": { input: 4 } });
assert.deepEqual(resolveBillingPrice("custom/model"), { input: 2, output: 8, cacheRead: 2, cacheWrite: 2, source: "registry" });
assert.equal(resolveBillingPrice("custom/other")?.input, 4);
assert.equal(initial?.input, 3, "reloading leaves previously resolved prices unchanged");

setBillingOverrides({});
assert.equal(resolveBillingPrice("custom/other"), undefined);
assert.equal(resolveBillingPrice("custom/model")?.source, "registry");
setBillingOverrides({ "custom/model": { input: 6 } });
setBillingOverrides(undefined);
assert.equal(resolveBillingPrice("custom/model")?.input, 2);
assert.deepEqual(billingOverrides(), {});
});

test("billing override reloads refresh a reused table and optional rates", () => {
resetState();
const prices: Record<string, BillingPrice> = { "custom/model": { input: 3 } };
setBillingOverrides(prices);
assert.deepEqual(resolveBillingPrice("CUSTOM/MODEL"), { input: 3, output: 3, cacheRead: 3, cacheWrite: 3, source: "config" });

prices["custom/model"] = { input: 5, output: 0, cacheRead: 0, cacheWrite: 0 };
setBillingOverrides(prices);
assert.deepEqual(resolveBillingPrice("CUSTOM/MODEL"), { input: 5, output: 0, cacheRead: 0, cacheWrite: 0, source: "config" });
assert.equal(billingOverrides(), prices);

prices["custom/model"] = { input: 7 };
setBillingOverrides(prices);
assert.deepEqual(resolveBillingPrice("CUSTOM/MODEL"), { input: 7, output: 7, cacheRead: 7, cacheWrite: 7, source: "config" });
});

// ---------------------------------------------------------------------------
// Stamp + ledger math.
// ---------------------------------------------------------------------------
Expand Down
115 changes: 115 additions & 0 deletions tests/issue7278-registry-cost-index.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import test from "node:test";
import assert from "node:assert/strict";
import { _resetForTest, _setForTest, scanRegistryCostRow, type CostsShape } from "../src/registry.ts";

// #7278: the billing path scans TWO cost tables per lookup (the overrides
// table, then the registry fallback). The single-slot index cache evicted one
// on the other, so an overrides-miss lookup rebuilt the (large) registry cost
// index on EVERY call — O(registry rows) per lookup. The fix caps the index
// cache at the two tables that flow through the path and evicts the
// least-recently-used, so the registry index is built once and reused.
//
// Counting builds without production scaffolding: buildLowerKeyIndex calls
// Object.entries(table), which invokes the table's [[OwnPropertyKeys]] (the
// Proxy `ownKeys` trap) exactly once per build. A direct `table[key]` exact-
// key hit and the suffix scan (which iterates the prebuilt index, not the
// table) do NOT touch ownKeys — so the count is index builds, not lookups.

function makeCostTable(rows: number): CostsShape {
const t: CostsShape = {};
for (let i = 0; i < rows; i++) t[`model-${i}`] = { input: 1 };
return t;
}

test("#7278 registry cost index is built once across overrides-miss lookups", () => {
_resetForTest();
let registryBuilds = 0;
const registry = new Proxy(makeCostTable(4000), {
ownKeys(t) {
registryBuilds++;
return Reflect.ownKeys(t);
},
});
const overrides = makeCostTable(10);
_setForTest({}, registry);

const model = "absent-model";
for (let i = 0; i < 50; i++) {
scanRegistryCostRow(overrides, model); // overrides-miss
scanRegistryCostRow(registry, model); // registry fallback (miss)
}
// The registry table's index must be built ONCE (then reused), not rebuilt
// on every overrides-miss lookup (50× with the old single-slot cache).
assert.equal(registryBuilds, 1, `registry cost index rebuilt ${registryBuilds}× across 50 overrides-miss lookups (expected 1)`);
});

test("#7278 both cost tables stay cached when their identities are stable", () => {
_resetForTest();
let registryBuilds = 0;
let overridesBuilds = 0;
const registry = new Proxy(makeCostTable(4000), {
ownKeys(t) {
registryBuilds++;
return Reflect.ownKeys(t);
},
});
const overrides = new Proxy(makeCostTable(10), {
ownKeys(t) {
overridesBuilds++;
return Reflect.ownKeys(t);
},
});
_setForTest({}, registry);

const model = "absent-model";
for (let i = 0; i < 50; i++) {
scanRegistryCostRow(overrides, model);
scanRegistryCostRow(registry, model);
}
// With stable table identities (the #7276 shape), BOTH indexes are built
// exactly once and reused — no thrash at all.
assert.equal(registryBuilds, 1, `registry index rebuilt ${registryBuilds}× (expected 1)`);
assert.equal(overridesBuilds, 1, `overrides index rebuilt ${overridesBuilds}× (expected 1)`);
});

test("#7278 registry index stays hot when the overrides table is a fresh identity per lookup (real pre-#7276 path; pins LRU over FIFO)", () => {
_resetForTest();
let registryBuilds = 0;
const registry = new Proxy(makeCostTable(4000), {
ownKeys(t) {
registryBuilds++;
return Reflect.ownKeys(t);
},
});
_setForTest({}, registry);

const model = "absent-model";
for (let i = 0; i < 50; i++) {
// overrideCosts() hands scanRegistryCostRow a brand-new table identity on
// every lookup (the pre-#7276 path); the registry (costCache) is the same
// object throughout. The LRU must keep the stable registry index hot and
// evict the throwaway overrides indexes — a FIFO cache would evict the
// registry every other lookup and rebuild it (25× here), so this pins the
// recency-refresh the stable-identity test above cannot see.
scanRegistryCostRow(makeCostTable(10), model); // fresh overrides, miss
scanRegistryCostRow(registry, model); // stable registry fallback
}
assert.equal(registryBuilds, 1, `registry index rebuilt ${registryBuilds}× with a fresh overrides identity per lookup (expected 1 — LRU must keep the stable table hot; a FIFO cache evicts it every other lookup)`);
});

test("#7278 the cache still resolves a registry hit correctly (Proxy does not break lookup)", () => {
_resetForTest();
const registry: CostsShape = { "host-a/present-model": { input: 2, output: 8 } };
const overrides = makeCostTable(10);
_setForTest({}, registry);
// The resolveBillingPrice path: overrides-miss, then registry-hit.
const miss = scanRegistryCostRow(overrides, "present-model");
assert.equal(miss, undefined, "overrides-miss for a registry-only model");
const hit = scanRegistryCostRow(registry, "present-model");
assert.ok(hit, "registry hit resolves");
assert.equal(hit.key, "host-a/present-model");
assert.equal(hit.row.input, 2);
// A second lookup reuses the cached index and still resolves.
const hit2 = scanRegistryCostRow(registry, "present-model");
assert.ok(hit2 && hit2.row.input === 2, "second lookup still resolves");
});
Loading
Loading