Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions scripts/bench-registry-suffix-index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
import { createHash } from "node:crypto";
import { performance } from "node:perf_hooks";
import { _resetForTest, _setForTest, peekRegistryContext, peekRegistryOutputLimit, scanRegistryCostRow, type CostsShape } from "../src/registry.js";
import snapshot from "../src/registry-snapshot.json" with { type: "json" };

// Run from the repository root with:
// node --expose-gc --import ./tests/preload-test-env.mjs --import tsx scripts/bench-registry-suffix-index.ts
// Use the bundled real roster; no network request or configuration mutation.
const costs = snapshot.costs as CostsShape;
const bareNames = [...new Set(Object.keys(costs).map((key) => key.slice(key.lastIndexOf("/") + 1)))];
const probes = bareNames.slice(0, 500);
const iterations = 5000;
function measure(label: string, names: string[]): void {
_resetForTest();
_setForTest(snapshot.models, costs);
global.gc?.();
const heapBefore = process.memoryUsage().heapUsed;
const coldStart = performance.now();
scanRegistryCostRow(costs, names[0], "relay.example");
peekRegistryContext(names[0], "relay.example");
peekRegistryOutputLimit(names[0], "relay.example");
const coldMs = performance.now() - coldStart;
global.gc?.();
const retainedBytes = process.memoryUsage().heapUsed - heapBefore;
// Warm conflict diagnostics before measurement; output is hashed below.
for (let i = 0; i < 5000; i++) {
const name = names[i % names.length];
scanRegistryCostRow(costs, name, "relay.example");
peekRegistryContext(name, "relay.example");
peekRegistryOutputLimit(name, "relay.example");
}
global.gc?.();
const outputs: unknown[] = [];
const cpuStart = process.cpuUsage();
const wallStart = performance.now();
for (let i = 0; i < iterations; i++) {
const name = names[i % names.length];
outputs.push([scanRegistryCostRow(costs, name, "relay.example"), peekRegistryContext(name, "relay.example"), peekRegistryOutputLimit(name, "relay.example")]);
}
const wallMs = performance.now() - wallStart;
const cpu = process.cpuUsage(cpuStart);
console.log(JSON.stringify({label, rows: Object.keys(costs).length, models: Object.keys(snapshot.models).length, names: names.length, iterations, coldMs, retainedBytes, cpuMs: (cpu.user + cpu.system) / 1000, wallMs, digest: createHash("sha256").update(JSON.stringify(outputs)).digest("hex")}));
}
measure("real-roster-relay-models", probes);
measure("unlisted-relay-models", Array.from({length: 100}, (_, i) => `unlisted-benchmark-model-${i}`));
40 changes: 30 additions & 10 deletions src/registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -446,8 +446,7 @@ export function scanRegistryCostRow(
let chosen: { row: CostRow; key: string; profile: PriceProfile } | undefined;
const seen = new Set<string>();
const parts: string[] = [];
for (const { lower, key, value: row } of idx.entries) {
if (!lower.endsWith(suffix)) continue;
for (const { key, value: row } of suffixEntries(idx, suffix)) {
const profile = priceProfileFromRow(row);
if (!profile) continue;
const sig = `${profile.w}|${profile.r}|${profile.q}`;
Expand Down Expand Up @@ -539,22 +538,44 @@ export function modelVariants(name: string): string[] {
// byLower keeps only the FIRST original key per lowercased form (case-only
// collisions in one roster are degenerate); entries keeps every key so the
// relay scan still sees all providers.
type LowerKeyEntry<T> = { lower: string; key: string; value: T };
type LowerKeyIndex<T> = {
byLower: Map<string, { key: string; value: T }>;
entries: Array<{ lower: string; key: string; value: T }>;
byLower: Map<string, LowerKeyEntry<T>>;
entries: Array<LowerKeyEntry<T>>;
bySuffix?: Map<string, Array<LowerKeyEntry<T>>>;
};

function buildLowerKeyIndex<T>(obj: Record<string, T>): LowerKeyIndex<T> {
const byLower = new Map<string, { key: string; value: T }>();
const entries: Array<{ lower: string; key: string; value: T }> = [];
const byLower = new Map<string, LowerKeyEntry<T>>();
const entries: Array<LowerKeyEntry<T>> = [];
for (const [key, value] of Object.entries(obj)) {
const lower = key.toLowerCase();
if (!byLower.has(lower)) byLower.set(lower, { key, value });
entries.push({ lower, key, value });
const entry = { lower, key, value };
if (!byLower.has(lower)) byLower.set(lower, entry);
entries.push(entry);
}
return { byLower, entries };
}

// Relay lookups match a slash-delimited suffix, including nested model ids.
// Build the index only when that fallback is used; known-provider/exact hits
// do not need it. Buckets retain every original row in snapshot key order.
function suffixEntries<T>(index: LowerKeyIndex<T>, suffix: string): ReadonlyArray<LowerKeyEntry<T>> {
if (!index.bySuffix) {
const bySuffix = new Map<string, Array<LowerKeyEntry<T>>>();
for (const entry of index.entries) {
for (let slash = entry.lower.indexOf("/"); slash >= 0; slash = entry.lower.indexOf("/", slash + 1)) {
const key = entry.lower.slice(slash);
const bucket = bySuffix.get(key);
if (bucket) bucket.push(entry);
else bySuffix.set(key, [entry]);
}
}
index.bySuffix = bySuffix;
}
return index.bySuffix.get(suffix) ?? [];
}

function registryLowerIndex(reg: RegistryShape): LowerKeyIndex<ModelEntry> {
if (regLowerIdx && regLowerIdx.reg === reg) return regLowerIdx.idx;
regLowerIdx = { reg, idx: buildLowerKeyIndex(reg) };
Expand Down Expand Up @@ -615,8 +636,7 @@ function registryLookup(reg: RegistryShape | null, model: string, host?: string,
let max: number | undefined;
const distinct = new Set<number>();
const parts: string[] = [];
for (const { lower, key, value: entry } of idx.entries) {
if (!lower.endsWith(suffix)) continue;
for (const { key, value: entry } of suffixEntries(idx, suffix)) {
const value = entry.limit?.[field];
if (typeof value !== "number" || value <= 0) continue;
if (max === undefined || value > max) max = value;
Expand Down
29 changes: 28 additions & 1 deletion tests/registry-price-profile.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import test from "node:test";
import assert from "node:assert/strict";
import { peekRegistryPriceProfile, providerFromHost, _resetForTest, _setForTest } from "../src/registry.ts";
import { peekRegistryPriceProfile, scanRegistryCostRow, providerFromHost, _resetForTest, _setForTest } from "../src/registry.ts";
import bundledSnapshot from "../src/registry-snapshot.json" with { type: "json" };

// #1279 follow-up: registry-derived default price profiles. The lookup is
Expand Down Expand Up @@ -122,3 +122,30 @@ test("bundled snapshot ships cost rows so the offline floor exercises pricing",
assert.equal(typeof snap.count, "number");
assert.equal(Object.keys(snap.models ?? {}).length, snap.count);
});


test("relay cost suffix lookup retains first usable row and full model-id precedence", () => {
_resetForTest();
const costs = {
"aaa/Family/Model": { input: 0, output: 0 },
"bbb/Other/Model": { input: 9, output: 90 },
"ccc/FAMILY/MODEL": { input: 2, output: 4, cache_read: 0 },
"ddd/family/model": { input: 3, output: 6 },
"EEE/FAMILY/MODEL": { input: 4, output: 8 },
"eee/family/model": { input: 5, output: 10 },
"fff/family/OtherModel": { input: 6, output: 12 },
};
for (let repeat = 0; repeat < 2; repeat++) {
const nested = scanRegistryCostRow(costs, "Family/Model", "relay.example");
assert.equal(nested?.key, "ccc/FAMILY/MODEL");
assert.deepEqual(nested?.profile, { w: 2, r: 0, q: 4 });
assert.equal(scanRegistryCostRow(costs, "family/model-thinking", "relay.example")?.key, "ccc/FAMILY/MODEL");
assert.equal(scanRegistryCostRow(costs, "MODEL", "relay.example")?.key, "bbb/Other/Model");
assert.equal(scanRegistryCostRow(costs, "odel", "relay.example"), undefined);
assert.equal(scanRegistryCostRow(costs, "model", "api.openai.com"), undefined);
assert.equal(scanRegistryCostRow(costs, "eee/family/model", "relay.example")?.row.input, 5);
}
const replacement = { "new/family/model": { input: 7, output: 14 } };
assert.equal(scanRegistryCostRow(replacement, "Family/Model", "relay.example")?.row.input, 7);
assert.equal(scanRegistryCostRow(costs, "Family/Model", "relay.example")?.row.input, 2);
});
28 changes: 27 additions & 1 deletion tests/registry.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import test from "node:test";
import assert from "node:assert/strict";
import { contextFromRegistry, peekRegistryContext, providerFromHost, newerFallback, bundledSnapshotLookup, bundledRegistryForTestsOnly, _resetForTest, _setForTest } from "../src/registry.ts";
import { contextFromRegistry, peekRegistryContext, peekRegistryOutputLimit, providerFromHost, newerFallback, bundledSnapshotLookup, bundledRegistryForTestsOnly, _resetForTest, _setForTest } from "../src/registry.ts";

test("providerFromHost maps minimax domains", () => {
assert.equal(providerFromHost("api.minimax.chat"), "minimax");
Expand Down Expand Up @@ -236,3 +236,29 @@ test("roster lookup is case-insensitive on exact keys and the relay scan (#2074)
// Genuinely absent models still miss.
assert.equal(peekRegistryContext("nope-xyz", "relay.example"), undefined);
});


test("relay suffix lookup preserves nested ids, case collisions and table replacement", () => {
_resetForTest();
const rows = {
"relayA/Family/Model": { limit: { context: 1000, output: 100 } },
"relayB/family/model": { limit: { context: 2000, output: 80 } },
"RELAYB/FAMILY/MODEL": { limit: { context: 3000, output: 90 } },
"relayC/Other/Model": { limit: { context: 4000, output: 200 } },
"relayD/Family/OtherModel": { limit: { context: 9000, output: 900 } },
"relayE/Family/Modeling": { limit: { context: 8000, output: 800 } },
};
_setForTest(rows);
for (let repeat = 0; repeat < 2; repeat++) {
assert.equal(peekRegistryContext("Family/Model", "relay.example"), 3000);
assert.equal(peekRegistryOutputLimit("FAMILY/MODEL", "relay.example"), 100);
assert.equal(peekRegistryContext("model-thinking", "relay.example"), 4000);
assert.equal(peekRegistryOutputLimit("model", "relay.example"), 200);
assert.equal(peekRegistryContext("odel", "relay.example"), undefined);
assert.equal(peekRegistryContext("Family/Model", "api.openai.com"), undefined);
assert.equal(peekRegistryContext("relayB/family/model", "relay.example"), 2000);
}
_setForTest({ "new/Family/Model": { limit: { context: 7000, output: 700 } } });
assert.equal(peekRegistryContext("family/model", "relay.example"), 7000);
assert.equal(peekRegistryOutputLimit("model", "relay.example"), 700);
});
Loading