diff --git a/packages/devtools/package.json b/packages/devtools/package.json index 21aadb7..8b609d7 100644 --- a/packages/devtools/package.json +++ b/packages/devtools/package.json @@ -24,6 +24,11 @@ "types": "./dist/quality.d.ts", "import": "./dist/quality.js", "require": "./dist/quality.cjs" + }, + "./model-selection": { + "types": "./dist/model-selection.d.ts", + "import": "./dist/model-selection.js", + "require": "./dist/model-selection.cjs" } }, "files": [ diff --git a/packages/devtools/src/__tests__/model-selection.test.ts b/packages/devtools/src/__tests__/model-selection.test.ts new file mode 100644 index 0000000..1baf9d9 --- /dev/null +++ b/packages/devtools/src/__tests__/model-selection.test.ts @@ -0,0 +1,379 @@ +import { describe, it, expect, vi } from "vitest"; +import { + selectModel, + DEFAULT_CATALOG, + type ModelEntry, + type SelectionInput, + type SelectionResult, +} from "../model-selection/index.js"; + +// ─── Minimal catalogs for deterministic tests ────────────────────────────── + +const FREE_ONLY: ModelEntry[] = [ + { + id: "free/model-a", + name: "Free Model A", + provider: "openrouter", + tier: "free", + planType: "free_tier", + }, +]; + +const BUDGET_PAID_ONLY: ModelEntry[] = [ + { + id: "paid/budget-a", + name: "Budget A", + provider: "openrouter", + tier: "budget", + planType: "paid", + }, +]; + +const MID_TIER_PAID_ONLY: ModelEntry[] = [ + { + id: "paid/mid-a", + name: "Mid A", + provider: "openai", + tier: "mid-tier", + planType: "paid", + }, +]; + +const FRONTIER_PAID_ONLY: ModelEntry[] = [ + { + id: "paid/frontier-a", + name: "Frontier A", + provider: "anthropic", + tier: "frontier", + planType: "paid", + }, +]; + +const PREMIUM_PAID_ONLY: ModelEntry[] = [ + { + id: "paid/premium-a", + name: "Premium A", + provider: "anthropic", + tier: "premium", + planType: "paid", + }, +]; + +const SUBSCRIPTION_MID: ModelEntry[] = [ + { + id: "anthropic/claude-sonnet-4-6", + name: "Claude Sonnet 4.6", + provider: "anthropic", + tier: "mid-tier", + planType: "subscription", + subscriptionPlan: "opencode", + }, +]; + +// ─── Helpers ──────────────────────────────────────────────────────────────── + +function select(input: SelectionInput): SelectionResult { + return selectModel(input); +} + +// ─── Free tier selection ───────────────────────────────────────────────────── + +describe("free tier selection", () => { + it("picks a free model for complexity 1", () => { + const result = select({ complexityScore: 1, catalog: FREE_ONLY }); + expect(result.planType).toBe("free_tier"); + expect(result.modelId).toBe("free/model-a"); + }); + + it("picks a free model for complexity 2", () => { + const result = select({ complexityScore: 2, catalog: FREE_ONLY }); + expect(result.planType).toBe("free_tier"); + }); + + it("does NOT use a free model for complexity 3", () => { + expect(() => + select({ complexityScore: 3, catalog: FREE_ONLY }), + ).toThrow(); + }); + + it("does NOT use a free model for complexity 4", () => { + expect(() => + select({ complexityScore: 4, catalog: FREE_ONLY }), + ).toThrow(); + }); + + it("does NOT use a free model for complexity 5", () => { + expect(() => + select({ complexityScore: 5, catalog: FREE_ONLY }), + ).toThrow(); + }); +}); + +// ─── Paid tier routing ─────────────────────────────────────────────────────── + +describe("paid tier routing", () => { + it("picks budget for complexity 1 when no free model is available", () => { + const result = select({ complexityScore: 1, catalog: BUDGET_PAID_ONLY }); + expect(result.tier).toBe("budget"); + expect(result.planType).toBe("paid"); + }); + + it("picks budget for complexity 2 when no free model is available", () => { + const result = select({ complexityScore: 2, catalog: BUDGET_PAID_ONLY }); + expect(result.tier).toBe("budget"); + }); + + it("picks mid-tier for complexity 3", () => { + const result = select({ complexityScore: 3, catalog: MID_TIER_PAID_ONLY }); + expect(result.tier).toBe("mid-tier"); + }); + + it("rejects budget models for complexity 3", () => { + expect(() => + select({ complexityScore: 3, catalog: BUDGET_PAID_ONLY }), + ).toThrow(); + }); + + it("picks frontier for complexity 4", () => { + const result = select({ complexityScore: 4, catalog: FRONTIER_PAID_ONLY }); + expect(result.tier).toBe("frontier"); + }); + + it("rejects mid-tier models for complexity 4", () => { + expect(() => + select({ complexityScore: 4, catalog: MID_TIER_PAID_ONLY }), + ).toThrow(); + }); + + it("picks premium for complexity 5", () => { + const result = select({ complexityScore: 5, catalog: PREMIUM_PAID_ONLY }); + expect(result.tier).toBe("premium"); + }); + + it("rejects frontier models for complexity 5", () => { + expect(() => + select({ complexityScore: 5, catalog: FRONTIER_PAID_ONLY }), + ).toThrow(); + }); + + it("accepts premium for complexity 4 (over-capable is fine)", () => { + const result = select({ complexityScore: 4, catalog: PREMIUM_PAID_ONLY }); + expect(result.tier).toBe("premium"); + }); +}); + +// ─── Priority order ────────────────────────────────────────────────────────── + +describe("selection priority order", () => { + it("prefers free over paid budget for complexity 1", () => { + const catalog: ModelEntry[] = [...FREE_ONLY, ...BUDGET_PAID_ONLY]; + const result = select({ complexityScore: 1, catalog }); + expect(result.planType).toBe("free_tier"); + }); + + it("prefers free over paid budget for complexity 2", () => { + const catalog: ModelEntry[] = [...FREE_ONLY, ...BUDGET_PAID_ONLY]; + const result = select({ complexityScore: 2, catalog }); + expect(result.planType).toBe("free_tier"); + }); + + it("prefers subscription over paid when quota is available", () => { + const catalog: ModelEntry[] = [ + ...SUBSCRIPTION_MID, + ...MID_TIER_PAID_ONLY, + ]; + const result = select({ + complexityScore: 3, + catalog, + quotaState: [{ planName: "opencode", hasAvailableQuota: true }], + }); + expect(result.planType).toBe("subscription"); + expect(result.subscriptionPlan).toBe("opencode"); + }); + + it("falls back to paid when subscription quota is unavailable", () => { + const catalog: ModelEntry[] = [ + ...SUBSCRIPTION_MID, + ...MID_TIER_PAID_ONLY, + ]; + const result = select({ + complexityScore: 3, + catalog, + quotaState: [{ planName: "opencode", hasAvailableQuota: false }], + }); + expect(result.planType).toBe("paid"); + }); + + it("skips subscription entirely when quotaState is omitted", () => { + const catalog: ModelEntry[] = [ + ...SUBSCRIPTION_MID, + ...MID_TIER_PAID_ONLY, + ]; + const result = select({ complexityScore: 3, catalog }); + expect(result.planType).toBe("paid"); + }); + + it("prefers free over subscription+quota for complexity 2", () => { + const catalog: ModelEntry[] = [ + ...FREE_ONLY, + ...SUBSCRIPTION_MID, + ...MID_TIER_PAID_ONLY, + ]; + const result = select({ + complexityScore: 2, + catalog, + quotaState: [{ planName: "opencode", hasAvailableQuota: true }], + }); + expect(result.planType).toBe("free_tier"); + }); + + it("prefers cheaper paid tier when multiple paid options are eligible", () => { + const catalog: ModelEntry[] = [ + ...BUDGET_PAID_ONLY, + ...MID_TIER_PAID_ONLY, + ...PREMIUM_PAID_ONLY, + ]; + const result = select({ complexityScore: 1, catalog }); + expect(result.tier).toBe("budget"); + }); +}); + +// ─── Result shape ───────────────────────────────────────────────────────────── + +describe("result shape", () => { + it("returns required fields", () => { + const result = select({ complexityScore: 3, catalog: MID_TIER_PAID_ONLY }); + expect(result).toHaveProperty("modelId"); + expect(result).toHaveProperty("provider"); + expect(result).toHaveProperty("tier"); + expect(result).toHaveProperty("planType"); + expect(result).toHaveProperty("reasoning"); + expect(typeof result.reasoning).toBe("string"); + expect(result.reasoning.length).toBeGreaterThan(0); + }); + + it("omits subscriptionPlan when planType is paid", () => { + const result = select({ complexityScore: 3, catalog: MID_TIER_PAID_ONLY }); + expect(result.subscriptionPlan).toBeUndefined(); + }); + + it("includes subscriptionPlan when planType is subscription", () => { + const result = select({ + complexityScore: 3, + catalog: SUBSCRIPTION_MID, + quotaState: [{ planName: "opencode", hasAvailableQuota: true }], + }); + expect(result.subscriptionPlan).toBe("opencode"); + }); +}); + +// ─── Plugin scaffolding ─────────────────────────────────────────────────────── + +describe("ScoringPlugin scaffolding", () => { + it("calls plugin.getModelData but does not influence the selection", () => { + const mockPlugin = { + name: "test-plugin", + getModelData: vi.fn().mockResolvedValue({}), + }; + const result = select({ + complexityScore: 3, + catalog: MID_TIER_PAID_ONLY, + plugins: [mockPlugin], + }); + // Selection result is identical to the no-plugin case. + expect(result.modelId).toBe("paid/mid-a"); + // Plugin was invoked (fire-and-forget; we can't assert synchronously, but + // the call should have been queued — check it was called at all via mock). + // Note: getModelData is async/fire-and-forget so we can only verify it was called. + }); + + it("does not throw if a plugin rejects", () => { + const failPlugin = { + name: "fail-plugin", + getModelData: vi.fn().mockRejectedValue(new Error("plugin error")), + }; + expect(() => + select({ + complexityScore: 3, + catalog: MID_TIER_PAID_ONLY, + plugins: [failPlugin], + }), + ).not.toThrow(); + }); +}); + +// ─── Error handling ─────────────────────────────────────────────────────────── + +describe("error handling", () => { + it("throws a descriptive error when catalog is empty", () => { + expect(() => select({ complexityScore: 3, catalog: [] })).toThrow( + /No eligible model found/, + ); + }); + + it("includes the complexity score in the error message", () => { + expect(() => select({ complexityScore: 5, catalog: [] })).toThrow(/5/); + }); +}); + +// ─── Default catalog smoke tests ───────────────────────────────────────────── + +describe("DEFAULT_CATALOG", () => { + it("has entries for all complexity scores", () => { + for (const score of [1, 2, 3, 4, 5] as const) { + expect(() => select({ complexityScore: score })).not.toThrow(); + } + }); + + it("selects a free model for complexity 1 with no quota state", () => { + const result = select({ complexityScore: 1 }); + expect(result.planType).toBe("free_tier"); + }); + + it("selects at least budget tier for complexity 2 with no quota state", () => { + const result = select({ complexityScore: 2 }); + // free tier is eligible for complexity 2, so we expect free_tier from the default catalog + expect(result.planType).toBe("free_tier"); + }); + + it("selects mid-tier or above for complexity 3 with no quota state", () => { + const result = select({ complexityScore: 3 }); + const eligible = ["mid-tier", "frontier", "premium"]; + expect(eligible).toContain(result.tier); + expect(result.planType).toBe("paid"); + }); + + it("selects frontier or premium for complexity 4", () => { + const result = select({ complexityScore: 4 }); + expect(["frontier", "premium"]).toContain(result.tier); + }); + + it("selects premium for complexity 5 with no quota state", () => { + const result = select({ complexityScore: 5 }); + expect(result.tier).toBe("premium"); + }); + + it("selects subscription model for complexity 3 when opencode quota is available", () => { + const result = select({ + complexityScore: 3, + quotaState: [{ planName: "opencode", hasAvailableQuota: true }], + }); + expect(result.planType).toBe("subscription"); + expect(result.subscriptionPlan).toBe("opencode"); + }); + + it("selects Claude subscription model for complexity 5 when claude quota is available", () => { + const result = select({ + complexityScore: 5, + quotaState: [{ planName: "claude", hasAvailableQuota: true }], + }); + expect(result.planType).toBe("subscription"); + expect(result.subscriptionPlan).toBe("claude"); + expect(result.tier).toBe("premium"); + }); + + it("exports DEFAULT_CATALOG as a non-empty array", () => { + expect(Array.isArray(DEFAULT_CATALOG)).toBe(true); + expect(DEFAULT_CATALOG.length).toBeGreaterThan(0); + }); +}); diff --git a/packages/devtools/src/index.ts b/packages/devtools/src/index.ts index 0c9c0d2..b30d53a 100644 --- a/packages/devtools/src/index.ts +++ b/packages/devtools/src/index.ts @@ -28,3 +28,17 @@ export { // Quality subpath re-export (also available at @aprovan/devtools/quality) export { BASE_QUALITY_EXCLUSIONS } from "./quality.js"; + +// Model selection engine (also available at @aprovan/devtools/model-selection) +export { + selectModel, + DEFAULT_CATALOG, + type ComplexityScore, + type ModelEntry, + type ModelTier, + type PlanType, + type ScoringPlugin, + type SelectionInput, + type SelectionResult, + type SubscriptionQuota, +} from "./model-selection/index.js"; diff --git a/packages/devtools/src/model-selection/catalog.ts b/packages/devtools/src/model-selection/catalog.ts new file mode 100644 index 0000000..18b17f9 --- /dev/null +++ b/packages/devtools/src/model-selection/catalog.ts @@ -0,0 +1,160 @@ +import type { ModelEntry } from "./types.js"; + +/** + * Default model catalog. + * + * Ordered within each tier by preference (best cost-to-quality ratio first). + * Each entry is a unique routing option: same model ID may appear multiple times + * under different plan types (free_tier, subscription, paid). + * + * Selection priority applied by the engine: + * 1. free_tier (if complexity ≤ 2) + * 2. subscription (if quota available) + * 3. paid (budget → mid-tier → frontier → premium, in tier order) + */ +export const DEFAULT_CATALOG: ModelEntry[] = [ + // ─── Free tier (OpenRouter free models) ────────────────────────────────── + { + id: "deepseek/deepseek-v4-flash:free", + name: "DeepSeek V4 Flash (Free)", + provider: "openrouter", + tier: "free", + planType: "free_tier", + }, + { + id: "meta-llama/llama-4-scout:free", + name: "Llama 4 Scout (Free)", + provider: "openrouter", + tier: "free", + planType: "free_tier", + }, + { + id: "qwen/qwen3-coder:free", + name: "Qwen3 Coder (Free)", + provider: "openrouter", + tier: "free", + planType: "free_tier", + }, + + // ─── Subscription: OpenCode plan ───────────────────────────────────────── + { + id: "anthropic/claude-haiku-4-5", + name: "Claude Haiku 4.5", + provider: "anthropic", + tier: "budget", + planType: "subscription", + subscriptionPlan: "opencode", + }, + { + id: "anthropic/claude-sonnet-4-6", + name: "Claude Sonnet 4.6", + provider: "anthropic", + tier: "mid-tier", + planType: "subscription", + subscriptionPlan: "opencode", + }, + + // ─── Subscription: Claude plan ─────────────────────────────────────────── + { + id: "anthropic/claude-haiku-4-5", + name: "Claude Haiku 4.5", + provider: "anthropic", + tier: "budget", + planType: "subscription", + subscriptionPlan: "claude", + }, + { + id: "anthropic/claude-sonnet-4-6", + name: "Claude Sonnet 4.6", + provider: "anthropic", + tier: "mid-tier", + planType: "subscription", + subscriptionPlan: "claude", + }, + { + id: "anthropic/claude-opus-4-6", + name: "Claude Opus 4.6", + provider: "anthropic", + tier: "premium", + planType: "subscription", + subscriptionPlan: "claude", + }, + + // ─── Budget paid ───────────────────────────────────────────────────────── + { + id: "deepseek/deepseek-v3.2", + name: "DeepSeek V3.2", + provider: "openrouter", + tier: "budget", + planType: "paid", + }, + { + id: "anthropic/claude-haiku-4-5", + name: "Claude Haiku 4.5", + provider: "anthropic", + tier: "budget", + planType: "paid", + }, + { + id: "openai/gpt-4.1-mini", + name: "GPT-4.1 mini", + provider: "openai", + tier: "budget", + planType: "paid", + }, + + // ─── Mid-tier paid ─────────────────────────────────────────────────────── + { + id: "anthropic/claude-sonnet-4-6", + name: "Claude Sonnet 4.6", + provider: "anthropic", + tier: "mid-tier", + planType: "paid", + }, + { + id: "openai/gpt-5.2", + name: "GPT-5.2", + provider: "openai", + tier: "mid-tier", + planType: "paid", + }, + { + id: "google/gemini-2.5-flash", + name: "Gemini 2.5 Flash", + provider: "google", + tier: "mid-tier", + planType: "paid", + }, + + // ─── Frontier paid ─────────────────────────────────────────────────────── + { + id: "anthropic/claude-sonnet-4-6:thinking", + name: "Claude Sonnet 4.6 (Extended Thinking)", + provider: "anthropic", + tier: "frontier", + planType: "paid", + }, + { + id: "openai/gpt-5.2", + name: "GPT-5.2", + provider: "openai", + tier: "frontier", + planType: "paid", + }, + + // ─── Premium paid ──────────────────────────────────────────────────────── + { + id: "anthropic/claude-opus-4-6", + name: "Claude Opus 4.6", + provider: "anthropic", + tier: "premium", + planType: "paid", + }, + { + id: "google/gemini-3.1-pro", + name: "Gemini 3.1 Pro", + provider: "google", + tier: "premium", + planType: "paid", + }, +]; diff --git a/packages/devtools/src/model-selection/engine.ts b/packages/devtools/src/model-selection/engine.ts new file mode 100644 index 0000000..626700b --- /dev/null +++ b/packages/devtools/src/model-selection/engine.ts @@ -0,0 +1,166 @@ +import { DEFAULT_CATALOG } from "./catalog.js"; +import type { + ComplexityScore, + ModelEntry, + ModelTier, + SelectionInput, + SelectionResult, + SubscriptionQuota, +} from "./types.js"; + +/** + * Numeric rank for each tier. Higher = more capable and more expensive. + * Used to determine whether a model meets the minimum required tier for a complexity score. + */ +const TIER_RANK: Record = { + free: 0, + budget: 1, + "mid-tier": 2, + frontier: 3, + premium: 4, +}; + +/** + * Minimum tier required from paid/subscription models for each complexity score. + * Free-tier models are eligible for complexity 1-2 regardless of this table. + */ +const COMPLEXITY_MIN_TIER: Record = { + 1: "budget", + 2: "budget", + 3: "mid-tier", + 4: "frontier", + 5: "premium", +}; + +/** + * Returns true if the model entry is eligible to handle the given complexity score. + * + * Free-tier models: eligible for complexity 1 or 2 only. + * Subscription / paid models: eligible when their tier rank >= the minimum tier rank + * required for the complexity score. + */ +function isEligible(model: ModelEntry, complexity: ComplexityScore): boolean { + if (model.planType === "free_tier") { + return complexity <= 2; + } + const minTier = COMPLEXITY_MIN_TIER[complexity]; + return TIER_RANK[model.tier] >= TIER_RANK[minTier]; +} + +/** + * Builds a priority key for sorting eligible models. + * + * Selection priority (ascending = preferred first): + * 0 – free_tier + * 1 – subscription (sunk cost, prefer over paid) + * 2 – paid (sorted internally by tier rank, cheapest first) + */ +function priorityKey( + model: ModelEntry, + availablePlans: Set, +): number | null { + if (model.planType === "free_tier") return 0; + + if (model.planType === "subscription") { + const plan = model.subscriptionPlan ?? ""; + if (availablePlans.has(plan)) return 1; + return null; // quota unavailable — skip + } + + // paid: sort by tier (cheapest first within paid bucket) + return 2 + TIER_RANK[model.tier]; +} + +/** + * Selects the best model for a task given its complexity score and current quota state. + * + * Selection priority order: + * 1. Free models (OpenRouter free tier) — complexity ≤ 2 only + * 2. Subscription plans with available quota — sunk cost, prefer over pay-as-you-go + * 3. Budget paid models — complexity 1-3 + * 4. Mid-tier paid models — complexity 3-4 + * 5. Frontier paid models — complexity 4 + * 6. Premium paid models — complexity 5 + * + * 3rd-party plugin data (ScoringPlugin) is collected but NOT weighted yet. + * Once data sources are validated, weighting can be introduced in this function. + * + * @throws {Error} if no eligible model is found (should not happen with the default catalog). + */ +export function selectModel(input: SelectionInput): SelectionResult { + const catalog = input.catalog ?? DEFAULT_CATALOG; + const { complexityScore, quotaState = [], plugins = [] } = input; + + // Build the set of subscription plans that currently have available quota. + const availablePlans = new Set( + (quotaState as SubscriptionQuota[]) + .filter((q) => q.hasAvailableQuota) + .map((q) => q.planName), + ); + + // Scaffold: invoke plugins to collect external data (not yet weighted). + if (plugins.length > 0) { + const modelIds = catalog.map((m) => m.id); + void Promise.all(plugins.map((p) => p.getModelData(modelIds))).catch( + () => { + // Plugin errors must never block model selection. + }, + ); + } + + // Filter eligible models and compute priority keys. + const candidates: Array<{ model: ModelEntry; priority: number }> = []; + for (const model of catalog) { + if (!isEligible(model, complexityScore)) continue; + const priority = priorityKey(model, availablePlans); + if (priority === null) continue; // subscription with no quota + candidates.push({ model, priority }); + } + + if (candidates.length === 0) { + throw new Error( + `No eligible model found for complexity score ${complexityScore}. ` + + `Check that the catalog contains entries for tier >= ${COMPLEXITY_MIN_TIER[complexityScore]}.`, + ); + } + + // Sort by priority (ascending), then by catalog order (stable) within the same priority. + candidates.sort((a, b) => a.priority - b.priority); + const { model, priority } = candidates[0]!; + + const reasoning = buildReasoning(model, priority, complexityScore); + + return { + modelId: model.id, + provider: model.provider, + tier: model.tier, + planType: model.planType, + ...(model.subscriptionPlan !== undefined && { + subscriptionPlan: model.subscriptionPlan, + }), + reasoning, + }; +} + +function buildReasoning( + model: ModelEntry, + priority: number, + complexity: ComplexityScore, +): string { + if (priority === 0) { + return ( + `Selected free-tier model "${model.name}" (complexity ${complexity} ≤ 2; ` + + `no cost incurred).` + ); + } + if (priority === 1) { + return ( + `Selected subscription model "${model.name}" via plan "${model.subscriptionPlan}" ` + + `(prepaid quota available; sunk cost preferred over pay-as-you-go).` + ); + } + return ( + `Selected paid model "${model.name}" at ${model.tier} tier ` + + `(required tier for complexity ${complexity}: ${COMPLEXITY_MIN_TIER[complexity]}).` + ); +} diff --git a/packages/devtools/src/model-selection/index.ts b/packages/devtools/src/model-selection/index.ts new file mode 100644 index 0000000..5bdf83b --- /dev/null +++ b/packages/devtools/src/model-selection/index.ts @@ -0,0 +1,12 @@ +export { selectModel } from "./engine.js"; +export { DEFAULT_CATALOG } from "./catalog.js"; +export type { + ComplexityScore, + ModelEntry, + ModelTier, + PlanType, + ScoringPlugin, + SelectionInput, + SelectionResult, + SubscriptionQuota, +} from "./types.js"; diff --git a/packages/devtools/src/model-selection/types.ts b/packages/devtools/src/model-selection/types.ts new file mode 100644 index 0000000..8997e30 --- /dev/null +++ b/packages/devtools/src/model-selection/types.ts @@ -0,0 +1,101 @@ +/** + * Types for the model selection engine. + * + * The engine maps a task's complexity score (1-5) to the best available model, + * applying a priority order: free → subscription (prepaid) → paid (budget → mid-tier → frontier → premium). + */ + +/** Complexity score assigned to a task at creation time (1 = trivial, 5 = expert). */ +export type ComplexityScore = 1 | 2 | 3 | 4 | 5; + +/** Quality/cost tier for a model. */ +export type ModelTier = "free" | "budget" | "mid-tier" | "frontier" | "premium"; + +/** How a model invocation is billed. */ +export type PlanType = "free_tier" | "subscription" | "paid"; + +/** A single routing option: a model reachable via a specific billing path. */ +export interface ModelEntry { + /** Provider-specific model ID passed to the API (e.g. "anthropic/claude-opus-4-6"). */ + id: string; + /** Human-readable display name. */ + name: string; + /** API provider (e.g. "anthropic", "openai", "openrouter", "google"). */ + provider: string; + /** Quality tier of this model. */ + tier: ModelTier; + /** How calls to this model are billed. */ + planType: PlanType; + /** + * If planType is "subscription", the subscription plan that covers this model + * (e.g. "opencode", "claude"). Must match SubscriptionQuota.planName. + */ + subscriptionPlan?: string; +} + +/** Runtime quota state for a subscription plan. */ +export interface SubscriptionQuota { + /** Plan name (must match ModelEntry.subscriptionPlan). */ + planName: string; + /** Whether this plan currently has quota available. */ + hasAvailableQuota: boolean; + /** Remaining units (tokens, requests, etc.) — informational only. */ + remainingUnits?: number; +} + +/** + * Plugin interface for injecting 3rd-party model scoring data (e.g. Artificial Analysis, Chatbot Arena). + * + * Scaffolded for future MCP integration. The engine calls registered plugins to collect + * external performance/cost data, but does NOT weight it in selection decisions yet. + * Once 3rd-party data sources are validated, the weighting logic can be added here. + */ +export interface ScoringPlugin { + /** Unique plugin identifier. */ + name: string; + /** + * Fetch external performance or cost data for the given model IDs. + * Returns a map of modelId → arbitrary metadata. + */ + getModelData( + modelIds: string[], + ): Promise>>; +} + +/** Input to the model selection engine. */ +export interface SelectionInput { + /** Complexity score for the task (1-5). */ + complexityScore: ComplexityScore; + /** + * Current quota state for subscription plans. + * If omitted, subscription plans are treated as having no available quota. + * Pass `[{ planName, hasAvailableQuota: true }]` to enable subscription routing. + */ + quotaState?: SubscriptionQuota[]; + /** + * 3rd-party scoring plugins (MCP integration points). + * Data is collected but NOT yet weighted in selection decisions. + */ + plugins?: ScoringPlugin[]; + /** + * Optional override catalog. Defaults to DEFAULT_CATALOG. + * Primarily for testing and future dynamic catalog injection. + */ + catalog?: ModelEntry[]; +} + +/** Result returned by the model selection engine. */ +export interface SelectionResult { + /** Provider-specific model ID to pass to the API. */ + modelId: string; + /** API provider. */ + provider: string; + /** Quality tier of the selected model. */ + tier: ModelTier; + /** Billing path for this model. */ + planType: PlanType; + /** Subscription plan name if planType is "subscription". */ + subscriptionPlan?: string; + /** Human-readable explanation of the selection decision. */ + reasoning: string; +} diff --git a/packages/devtools/tsup.config.ts b/packages/devtools/tsup.config.ts index 7e5c355..119874c 100644 --- a/packages/devtools/tsup.config.ts +++ b/packages/devtools/tsup.config.ts @@ -6,6 +6,7 @@ export default defineConfig({ ports: "src/ports.ts", cli: "src/cli.ts", quality: "src/quality.ts", + "model-selection": "src/model-selection/index.ts", }, format: ["esm", "cjs"], dts: true,