diff --git a/src/billing.ts b/src/billing.ts index 66377b0032..a499cce44f 100644 --- a/src/billing.ts +++ b/src/billing.ts @@ -107,9 +107,11 @@ function nonNegNumStrict(v: unknown): number | undefined | typeof REJECT { * reload path funnels through it) and by nothing else. Keeps deep settle * sites free of options plumbing, mirroring the registry costCache pattern. */ let overridePrices: Record = {}; +let overrideCostRows: CostsShape | null = null; export function setBillingOverrides(prices: Record | undefined): void { overridePrices = prices ?? {}; + overrideCostRows = overrideCosts(); } export function billingOverrides(): Record { @@ -130,9 +132,8 @@ function overrideCosts(): CostsShape | null { export function resolveBillingPrice(model: string | undefined, host?: string): ResolvedBillingPrice | undefined { if (!model) return undefined; - const costs = overrideCosts(); - if (costs !== null) { - const hit = scanRegistryCostRow(costs, model, host, { conflictHint: "billing.prices" }); + if (overrideCostRows !== null) { + const hit = scanRegistryCostRow(overrideCostRows, model, host, { conflictHint: "billing.prices" }); if (hit) return finalize(hit.row, "config"); } const row = peekRegistryCostRow(model, host); diff --git a/src/preflight.ts b/src/preflight.ts index 44c053354a..ebe8d88b87 100644 --- a/src/preflight.ts +++ b/src/preflight.ts @@ -464,10 +464,27 @@ export function clearEstimateCoreMessagesMemoForTests(): void { // the #496/#1800 billing arbitration. OpenAI chat `image_url.url`, Anthropic // `source.data` and Gemini `inline_data.data` were already excluded. const NON_TEXT_BODY_KEYS = new Set(["data", "url", "b64_json", "file_data", "image_url"]); +// DSH official-API extension plugins attach transport-only top-level sibling fields +// (such as `dsh_session_log` up to 8 MiB and `dsh_plugin_packages`) that consume 0 model +// tokens and are stripped/ignored before inference, whereas nested fields inside +// `messages[]` remain model-visible and must still be counted. +const NON_MODEL_TOP_LEVEL_KEYS = new Set(["dsh_session_log", "dsh_plugin_packages"]); + +function isNonModelTopLevelKey(key: string): boolean { + return NON_MODEL_TOP_LEVEL_KEYS.has(key) || key.startsWith("dsh_"); +} export function estimateRawBodyTokens(parsed: unknown): number { let tokens = 0; - const pending: [unknown, string | undefined][] = [[parsed, undefined]]; + const pending: [unknown, string | undefined][] = []; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { + for (const [k, v] of Object.entries(parsed as Record)) { + if (isNonModelTopLevelKey(k)) continue; + pending.push([v, k]); + } + } else { + pending.push([parsed, undefined]); + } while (pending.length > 0) { const [value, key] = pending.pop()!; if (typeof value === "string") { @@ -503,8 +520,9 @@ export interface RawBodyBreakdown { export function estimateRawBodyBreakdown(parsed: unknown): RawBodyBreakdown { const buckets: [number, number, number, number] = [0, 0, 0, 0]; // messages, tools, system, other const pending: [unknown, string | undefined, number][] = []; - if (parsed && typeof parsed === "object") { + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { for (const [k, v] of Object.entries(parsed as Record)) { + if (isNonModelTopLevelKey(k)) continue; const bucket = k === "messages" ? 0 : k === "tools" ? 1 : k === "system" ? 2 : 3; pending.push([v, k, bucket]); } diff --git a/src/registry.ts b/src/registry.ts index b9acc7c476..7e2dcefda8 100644 --- a/src/registry.ts +++ b/src/registry.ts @@ -34,16 +34,26 @@ type LoadedRegistry = { reg: RegistryShape; costs: CostsShape | null }; let cache: RegistryShape | null = null; let costCache: CostsShape | null = null; let loading: Promise | null = null; +const FAILED_FETCH_BACKOFF_MS = 10 * 60 * 1000; +let lastFailedFetchMs = 0; +let requestPathBudgetMs = 3_000; +export function _setRequestPathBudgetForTest(ms: number | undefined): void { + requestPathBudgetMs = ms ?? 3_000; +} const warnedConflicts = new Set(); const warnedPriceConflicts = new Set(); // Case-folded key indexes for the roster maps (#2074): models.dev keys are // mixed-case ("minimax/MiniMax-M3", "google/gemma-4-E2B-it") while relay ids // often arrive lowercased — exact-key hits and suffix scans were case- // sensitive on both sides, so any case drift silently fell back to the -// built-in table / default window. One slot per source object; identity- -// checked, so loadRegistry / _setForTest swaps invalidate automatically. +// built-in table / default window. Identity-checked, so loadRegistry / +// _setForTest swaps invalidate automatically. regLowerIdx is a single slot +// (one model registry at a time); costLowerIdx is a small Map because the +// billing path scans TWO cost tables per lookup (the overrides table, then the +// registry fallback) and a single slot evicted one on the other, rebuilding +// the large registry index on every overrides-miss lookup (#7278). let regLowerIdx: { reg: RegistryShape; idx: LowerKeyIndex } | null = null; -let costLowerIdx: { costs: CostsShape; idx: LowerKeyIndex } | null = null; +let costLowerIdx: Map> | null = null; /** Full models.dev snapshot committed at src/registry-snapshot.json * (refresh with `npm run registry:snapshot`) and inlined into dist at build @@ -242,6 +252,7 @@ export async function loadRegistry(): Promise { if (loading) return loading; const pending = (async () => { const fresh = await fetchFresh(); + lastFailedFetchMs = fresh ? 0 : Date.now(); if (fresh) { cache = fresh.reg; costCache = fresh.costs; @@ -358,8 +369,23 @@ export function providerFromHost(host: string): string | undefined { } export async function contextFromRegistry(model: string, host?: string): Promise { - const reg = await loadRegistry(); - return registryLookup(reg, model, host); + // Request path (server/handle.ts awaits this for a model the warm cache + // misses). The fetch watchdog is an idle timer, so a models.dev that + // trickles bytes holds the load open far past its 15 s budget, and a + // failed fetch leaves `cache` on a fallback so loadRegistry() fetches + // again on every call. Wait at most requestPathBudgetMs (the load keeps + // warming the cache in the background) and skip the wait entirely during + // the backoff after a failure; loadRegistry() itself still retries per call. + if (cache && Date.now() - lastFailedFetchMs < FAILED_FETCH_BACKOFF_MS) return registryLookup(cache, model, host); + let timer: ReturnType | undefined; + const budget = new Promise((resolve) => { + timer = setTimeout(() => resolve(cache), requestPathBudgetMs); + }); + try { + return registryLookup(await Promise.race([loadRegistry(), budget]), model, host); + } finally { + clearTimeout(timer); + } } /** Synchronous cache-only lookup: returns the window when the models.dev @@ -582,10 +608,27 @@ function registryLowerIndex(reg: RegistryShape): LowerKeyIndex { return regLowerIdx.idx; } +// Cap is exactly the two cost tables that ever flow through the billing path +// (the overrides table and the registry snapshot); evict the least-recently- +// used on a new table so both stay hot across overrides-miss lookups (#7278). +const COST_IDX_CAP = 2; function costLowerIndex(costs: CostsShape): LowerKeyIndex { - if (costLowerIdx && costLowerIdx.costs === costs) return costLowerIdx.idx; - costLowerIdx = { costs, idx: buildLowerKeyIndex(costs) }; - return costLowerIdx.idx; + if (!costLowerIdx) costLowerIdx = new Map(); + const cached = costLowerIdx.get(costs); + if (cached) { + // Refresh recency (Map preserves insertion order; re-set moves to end). + costLowerIdx.delete(costs); + costLowerIdx.set(costs, cached); + return cached; + } + const idx = buildLowerKeyIndex(costs); + costLowerIdx.set(costs, idx); + while (costLowerIdx.size > COST_IDX_CAP) { + const oldest = costLowerIdx.keys().next().value; + if (oldest === undefined) break; + costLowerIdx.delete(oldest); + } + return idx; } function registryLookup(reg: RegistryShape | null, model: string, host: string | undefined, field: "context" | "output"): number | undefined; @@ -659,6 +702,7 @@ export function _resetForTest(): void { cache = null; costCache = null; loading = null; + lastFailedFetchMs = 0; regLowerIdx = null; costLowerIdx = null; warnedConflicts.clear(); diff --git a/tests/billing-prices.test.ts b/tests/billing-prices.test.ts index a66aaba3a0..f63c4c6dc2 100644 --- a/tests/billing-prices.test.ts +++ b/tests/billing-prices.test.ts @@ -24,6 +24,7 @@ import { setBillingOverrides, stampBillingPrice, stampedBillingPrice, + type BillingPrice, } from "../src/billing.ts"; import { _setForTest as setRegistryForTest } from "../src/registry.ts"; import { _resetPluginStateForTest } from "../src/plugin.ts"; @@ -125,6 +126,43 @@ test("#4899 resolveBillingPrice: overrides win over the registry, matching model assert.equal(resolveBillingPrice(""), undefined); }); +test("billing override reloads replace and clear normalized price rows", () => { + resetState(); + setRegistryForTest({}, { "custom/model": { input: 2, output: 8 } }); + setBillingOverrides({ "custom/model": { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75 } }); + const initial = resolveBillingPrice("custom/model"); + assert.deepEqual(initial, { input: 3, output: 15, cacheRead: 0.3, cacheWrite: 3.75, source: "config" }); + + setBillingOverrides({ "custom/other": { input: 4 } }); + assert.deepEqual(resolveBillingPrice("custom/model"), { input: 2, output: 8, cacheRead: 2, cacheWrite: 2, source: "registry" }); + assert.equal(resolveBillingPrice("custom/other")?.input, 4); + assert.equal(initial?.input, 3, "reloading leaves previously resolved prices unchanged"); + + setBillingOverrides({}); + assert.equal(resolveBillingPrice("custom/other"), undefined); + assert.equal(resolveBillingPrice("custom/model")?.source, "registry"); + setBillingOverrides({ "custom/model": { input: 6 } }); + setBillingOverrides(undefined); + assert.equal(resolveBillingPrice("custom/model")?.input, 2); + assert.deepEqual(billingOverrides(), {}); +}); + +test("billing override reloads refresh a reused table and optional rates", () => { + resetState(); + const prices: Record = { "custom/model": { input: 3 } }; + setBillingOverrides(prices); + assert.deepEqual(resolveBillingPrice("CUSTOM/MODEL"), { input: 3, output: 3, cacheRead: 3, cacheWrite: 3, source: "config" }); + + prices["custom/model"] = { input: 5, output: 0, cacheRead: 0, cacheWrite: 0 }; + setBillingOverrides(prices); + assert.deepEqual(resolveBillingPrice("CUSTOM/MODEL"), { input: 5, output: 0, cacheRead: 0, cacheWrite: 0, source: "config" }); + assert.equal(billingOverrides(), prices); + + prices["custom/model"] = { input: 7 }; + setBillingOverrides(prices); + assert.deepEqual(resolveBillingPrice("CUSTOM/MODEL"), { input: 7, output: 7, cacheRead: 7, cacheWrite: 7, source: "config" }); +}); + // --------------------------------------------------------------------------- // Stamp + ledger math. // --------------------------------------------------------------------------- diff --git a/tests/issue7278-registry-cost-index.test.ts b/tests/issue7278-registry-cost-index.test.ts new file mode 100644 index 0000000000..5fc3831a9f --- /dev/null +++ b/tests/issue7278-registry-cost-index.test.ts @@ -0,0 +1,115 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { _resetForTest, _setForTest, scanRegistryCostRow, type CostsShape } from "../src/registry.ts"; + +// #7278: the billing path scans TWO cost tables per lookup (the overrides +// table, then the registry fallback). The single-slot index cache evicted one +// on the other, so an overrides-miss lookup rebuilt the (large) registry cost +// index on EVERY call — O(registry rows) per lookup. The fix caps the index +// cache at the two tables that flow through the path and evicts the +// least-recently-used, so the registry index is built once and reused. +// +// Counting builds without production scaffolding: buildLowerKeyIndex calls +// Object.entries(table), which invokes the table's [[OwnPropertyKeys]] (the +// Proxy `ownKeys` trap) exactly once per build. A direct `table[key]` exact- +// key hit and the suffix scan (which iterates the prebuilt index, not the +// table) do NOT touch ownKeys — so the count is index builds, not lookups. + +function makeCostTable(rows: number): CostsShape { + const t: CostsShape = {}; + for (let i = 0; i < rows; i++) t[`model-${i}`] = { input: 1 }; + return t; +} + +test("#7278 registry cost index is built once across overrides-miss lookups", () => { + _resetForTest(); + let registryBuilds = 0; + const registry = new Proxy(makeCostTable(4000), { + ownKeys(t) { + registryBuilds++; + return Reflect.ownKeys(t); + }, + }); + const overrides = makeCostTable(10); + _setForTest({}, registry); + + const model = "absent-model"; + for (let i = 0; i < 50; i++) { + scanRegistryCostRow(overrides, model); // overrides-miss + scanRegistryCostRow(registry, model); // registry fallback (miss) + } + // The registry table's index must be built ONCE (then reused), not rebuilt + // on every overrides-miss lookup (50× with the old single-slot cache). + assert.equal(registryBuilds, 1, `registry cost index rebuilt ${registryBuilds}× across 50 overrides-miss lookups (expected 1)`); +}); + +test("#7278 both cost tables stay cached when their identities are stable", () => { + _resetForTest(); + let registryBuilds = 0; + let overridesBuilds = 0; + const registry = new Proxy(makeCostTable(4000), { + ownKeys(t) { + registryBuilds++; + return Reflect.ownKeys(t); + }, + }); + const overrides = new Proxy(makeCostTable(10), { + ownKeys(t) { + overridesBuilds++; + return Reflect.ownKeys(t); + }, + }); + _setForTest({}, registry); + + const model = "absent-model"; + for (let i = 0; i < 50; i++) { + scanRegistryCostRow(overrides, model); + scanRegistryCostRow(registry, model); + } + // With stable table identities (the #7276 shape), BOTH indexes are built + // exactly once and reused — no thrash at all. + assert.equal(registryBuilds, 1, `registry index rebuilt ${registryBuilds}× (expected 1)`); + assert.equal(overridesBuilds, 1, `overrides index rebuilt ${overridesBuilds}× (expected 1)`); +}); + +test("#7278 registry index stays hot when the overrides table is a fresh identity per lookup (real pre-#7276 path; pins LRU over FIFO)", () => { + _resetForTest(); + let registryBuilds = 0; + const registry = new Proxy(makeCostTable(4000), { + ownKeys(t) { + registryBuilds++; + return Reflect.ownKeys(t); + }, + }); + _setForTest({}, registry); + + const model = "absent-model"; + for (let i = 0; i < 50; i++) { + // overrideCosts() hands scanRegistryCostRow a brand-new table identity on + // every lookup (the pre-#7276 path); the registry (costCache) is the same + // object throughout. The LRU must keep the stable registry index hot and + // evict the throwaway overrides indexes — a FIFO cache would evict the + // registry every other lookup and rebuild it (25× here), so this pins the + // recency-refresh the stable-identity test above cannot see. + scanRegistryCostRow(makeCostTable(10), model); // fresh overrides, miss + scanRegistryCostRow(registry, model); // stable registry fallback + } + assert.equal(registryBuilds, 1, `registry index rebuilt ${registryBuilds}× with a fresh overrides identity per lookup (expected 1 — LRU must keep the stable table hot; a FIFO cache evicts it every other lookup)`); +}); + +test("#7278 the cache still resolves a registry hit correctly (Proxy does not break lookup)", () => { + _resetForTest(); + const registry: CostsShape = { "host-a/present-model": { input: 2, output: 8 } }; + const overrides = makeCostTable(10); + _setForTest({}, registry); + // The resolveBillingPrice path: overrides-miss, then registry-hit. + const miss = scanRegistryCostRow(overrides, "present-model"); + assert.equal(miss, undefined, "overrides-miss for a registry-only model"); + const hit = scanRegistryCostRow(registry, "present-model"); + assert.ok(hit, "registry hit resolves"); + assert.equal(hit.key, "host-a/present-model"); + assert.equal(hit.row.input, 2); + // A second lookup reuses the cached index and still resolves. + const hit2 = scanRegistryCostRow(registry, "present-model"); + assert.ok(hit2 && hit2.row.input === 2, "second lookup still resolves"); +}); diff --git a/tests/raw-body-dsh-extensions.test.ts b/tests/raw-body-dsh-extensions.test.ts new file mode 100644 index 0000000000..8c673d507f --- /dev/null +++ b/tests/raw-body-dsh-extensions.test.ts @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { estimateRawBodyBreakdown, estimateRawBodyTokens } from "../src/preflight.ts"; + +test("top-level non-model DSH extension fields contribute zero tokens (#dsh_session_log)", () => { + const baseline = { + model: "deepseek-chat", + system: "You are a helpful assistant.", + messages: [ + { role: "user", content: "Summarize the previous steps." }, + ], + }; + const baseTokens = estimateRawBodyTokens(baseline); + const baseBreakdown = estimateRawBodyBreakdown(baseline); + + // Multi-megabyte (>5.5 MiB) transport-only DSH extension payload attached at top level. + const hugeEvents = Array.from({ length: 1200 }, (_, i) => ({ + seq: i, + type: "assistant/text", + text: `event-${i}: ` + "x".repeat(4800), + })); + const withExtensions = { + ...baseline, + dsh_session_log: { + version: 1, + sessionId: "sess-123", + events: hugeEvents, + }, + dsh_plugin_packages: [ + { name: "@deepseek-ai/dsh-base", version: "0.2.2-rc.5" }, + { name: "billion-context", version: "0.1.216" }, + ], + dsh_custom_extension: { note: "y".repeat(50_000) }, + }; + + assert.ok(JSON.stringify(withExtensions).length > 5_500_000, "fixture exceeds 5.5 MiB"); + assert.equal(estimateRawBodyTokens(withExtensions), baseTokens); + assert.deepEqual(estimateRawBodyBreakdown(withExtensions), baseBreakdown); + assert.equal(estimateRawBodyBreakdown(withExtensions).other, baseBreakdown.other); +}); + +test("nested dsh_* fields inside messages[] are still counted", () => { + const withoutNested = { + model: "deepseek-chat", + messages: [{ role: "user", content: "hello" }], + }; + const withNested = { + model: "deepseek-chat", + messages: [ + { + role: "user", + content: "hello", + dsh_session_log: { note: "nested content inside user message should still count" }, + dsh_foo: "extra nested tokens", + }, + ], + }; + + assert.ok(estimateRawBodyTokens(withNested) > estimateRawBodyTokens(withoutNested)); + const bd = estimateRawBodyBreakdown(withNested); + assert.equal(bd.total, estimateRawBodyTokens(withNested)); + assert.ok(bd.messages > estimateRawBodyBreakdown(withoutNested).messages); +}); + +test("non-plain-object roots (arrays, strings, primitives) behave consistently", () => { + const arrayRoot = [{ dsh_session_log: "should still count when root is an array" }, "tail"]; + assert.equal(estimateRawBodyBreakdown(arrayRoot).total, estimateRawBodyTokens(arrayRoot)); + assert.ok(estimateRawBodyTokens(arrayRoot) > 0); + assert.equal(estimateRawBodyTokens("plain text"), estimateRawBodyBreakdown("plain text").total); + assert.equal(estimateRawBodyTokens(null), 0); +}); diff --git a/tests/registry-unsupported-proxy.test.ts b/tests/registry-unsupported-proxy.test.ts index 8dc1bcacd5..81f7836889 100644 --- a/tests/registry-unsupported-proxy.test.ts +++ b/tests/registry-unsupported-proxy.test.ts @@ -119,3 +119,60 @@ test("an unsupported proxy scheme neither rejects the loader nor poisons its mem await restore(); } }); + +// server/handle.ts awaits contextFromRegistry() on the request path for a model +// the warm cache misses. A failed fetch leaves the cache on the fallback, so +// before the backoff every such request refetched models.dev and stalled for +// the full fetch timeout when it was slow or unreachable. +test("after a failed fetch the request path serves the fallback instead of refetching", async () => { + const restore = await isolateLoaderEnv(); + let calls = 0; + let reg: typeof import("../src/registry.ts") | undefined; + try { + reg = await import("../src/registry.ts"); + reg._resetForTest(); + reg._setRegistryFetchForTest(async () => { + calls += 1; + throw new Error("network down"); + }); + + assert.equal(await reg.contextFromRegistry("no-such-model"), undefined); + assert.equal(calls, 1); + assert.equal(await reg.contextFromRegistry("no-such-model"), undefined); + assert.equal(calls, 1, "a second request inside the backoff must not refetch"); + + await reg.loadRegistry(); + assert.equal(calls, 2, "loadRegistry itself still retries per call"); + } finally { + reg?._setRegistryFetchForTest(undefined); + reg?._resetForTest(); + await restore(); + } +}); + +// The fetch watchdog is an idle timer: a models.dev that trickles bytes keeps +// the load open long past its 15 s budget. The request path must not wait for +// it — it answers from the current cache and lets the load finish behind it. +test("the request path does not wait on a stalled fetch beyond its budget", async () => { + const restore = await isolateLoaderEnv(); + let release: ((err: Error) => void) | undefined; + let reg: typeof import("../src/registry.ts") | undefined; + try { + reg = await import("../src/registry.ts"); + reg._resetForTest(); + reg._setRequestPathBudgetForTest(50); + reg._setRegistryFetchForTest(() => new Promise((_, reject) => { release = reject; })); + + const started = Date.now(); + assert.equal(await reg.contextFromRegistry("no-such-model"), undefined); + assert.ok(Date.now() - started < 1_000, "answered within the budget, not after the stalled fetch"); + } finally { + // Settle the in-flight load before resetting so it cannot write the + // fallback back into the module state after this test. + if (release) { release(new Error("network down")); await reg?.loadRegistry(); } + reg?._setRequestPathBudgetForTest(undefined); + reg?._setRegistryFetchForTest(undefined); + reg?._resetForTest(); + await restore(); + } +});