diff --git a/jest.config.js b/jest.config.js index 21323931..c5afcc8f 100644 --- a/jest.config.js +++ b/jest.config.js @@ -22,5 +22,32 @@ const customJestConfig = { collectCoverageFrom: ['lib/**/*.{ts,tsx}', '!lib/**/*.d.ts', '!lib/**/index.ts'], }; -// createJestConfig is exported this way to ensure that next/jest can load the Next.js config which is async -module.exports = createJestConfig(customJestConfig); +/** + * Packages that must be transformed rather than required. + * + * `ai-kit` is ESM-only ("type": "module", no require condition). Jest runs CJS, + * so without this it dies on `Unexpected token 'export'` the moment anything + * imports it. + * + * This list is for packages this repo imports DIRECTLY and on purpose. If an + * entry is ever needed for something no file here imports, that is a dependency + * leaking through someone's re-export, and the fix belongs in that package + * rather than in this list — `ai-kit` shipped exactly that bug once, dragging + * `ai-forms` in behind its root export, and it was fixed there. + */ +const ESM_PACKAGES = ['ai-kit']; + +// createJestConfig is exported this way to ensure that next/jest can load the +// Next.js config which is async. The transformIgnorePatterns override has to +// happen AFTER it resolves: next/jest builds its own, and anything set in +// customJestConfig above is replaced wholesale rather than merged. +module.exports = async () => { + const config = await createJestConfig(customJestConfig)(); + + config.transformIgnorePatterns = [ + `/node_modules/(?!(${ESM_PACKAGES.join('|')})/)`, + ...(config.transformIgnorePatterns ?? []).filter((p) => !/node_modules/.test(p)), + ]; + + return config; +}; diff --git a/lib/constants.ts b/lib/constants.ts index 3ae8f722..1023fba8 100644 --- a/lib/constants.ts +++ b/lib/constants.ts @@ -48,25 +48,29 @@ export const DOMAIN_ERRORS = { } as const; // API Configuration +// +// NO MODEL IDS HERE. They used to live in this object — `llama-3.1-8b-instant` +// for Groq and `openai/gpt-oss-20b:free` for OpenRouter — and both have since +// been retired by their vendors. The comment they carried said "free catalogues +// rot — when it does, replace it with another `:free` id", which was correct +// and is exactly the maintenance this repo then did not do, because nothing +// tells a constant when it has stopped being true. +// +// So `lib/llm-client.ts` takes them from `ai-kit`, which holds one list for the +// whole fleet, re-probes it against the live catalogues, and is checked daily by +// dotfiles/scripts/ci/model-pin-audit.mjs. Pasting an id back into this file +// removes it from that coverage. +// +// The cost rule that lived here still holds and is still tested: OpenRouter is +// only reached when Groq's free tier is spent — i.e. when nobody is watching — +// so a paid id there turns an outage into a bill. `ai-kit`'s `modelCost` is now +// what enforces it, over the whole list rather than over one default. export const API_CONFIG = { // Groq GROQ_API_URL: 'https://api.groq.com/openai/v1/chat/completions', - GROQ_MODEL: 'llama-3.1-8b-instant', // OpenRouter OPENROUTER_API_URL: 'https://openrouter.ai/api/v1/chat/completions', - // Must be a FREE id. OpenRouter is the FALLBACK here — reached when Groq's - // free tier is spent, i.e. only when nobody is watching. This default was - // `anthropic/claude-sonnet-5`, a premium paid model, so "the free tier ran - // out" produced a bill rather than a degraded answer — a spending decision - // made by an outage instead of by a person. (It also broke the standing rule - // that Anthropic is never a primary or fallback provider in this fleet.) - // - // `openai/gpt-oss-20b:free` reports pricing.prompt = 0 in OpenRouter's own - // catalogue (checked 2026-08-16) and was probed live for tool support on - // 2026-08-15. Free catalogues rot — when it does, replace it with another - // `:free` id, never by dropping the suffix. - OPENROUTER_DEFAULT_MODEL: 'openai/gpt-oss-20b:free', // Token limits MAX_CONTEXT_CHARS: 8000, diff --git a/lib/llm-client.ts b/lib/llm-client.ts index 3d6ce18d..99a497e2 100644 --- a/lib/llm-client.ts +++ b/lib/llm-client.ts @@ -7,10 +7,27 @@ * - Ollama (local) */ +import { freeChain, providerModels } from 'ai-kit'; import { API_CONFIG } from '@/lib/constants'; import { getServerEnv, getClientEnv } from '@/lib/config/env'; import { logger } from './logger'; +/** + * The models to try at each vendor, in order, from `ai-kit`. + * + * A list rather than a name, because the previous single ids were retired out + * from under this app and there was nothing between that and total failure: + * `generateWithBestProvider` picks ONE provider and calls it once. A retired id + * was a dead chatbot with a valid key. + * + * The lists cross models, not vendors — vendor selection above stays exactly as + * it was. That is the smaller half of the protection (a spent daily budget is + * org-wide, so every model at the same vendor dies together), but it is the + * half that covers rot, which is what actually happened here twice. + */ +const groqModels = () => providerModels(freeChain('BOTSMANN')[0]); +const openRouterModels = () => providerModels(freeChain('BOTSMANN')[1]); + export type ModelProvider = 'groq' | 'openrouter' | 'ollama'; interface LLMMessage { @@ -75,32 +92,44 @@ async function generateWithGroq( throw new Error('Groq API key not configured'); } - const response = await fetch(API_CONFIG.GROQ_API_URL, { - method: 'POST', - headers: { - Authorization: `Bearer ${key}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - model: API_CONFIG.GROQ_MODEL, - messages, - temperature, - max_tokens: maxTokens, - }), - }); - - if (!response.ok) { - const error = await response.text(); - logger.error(`Groq API error: ${response.status}`, error); - throw new Error(`Groq API error: ${response.status}`); + const models = groqModels(); + let lastStatus = 0; + let lastError = ''; + + for (const model of models) { + const response = await fetch(API_CONFIG.GROQ_API_URL, { + method: 'POST', + headers: { + Authorization: `Bearer ${key}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model, + messages, + temperature, + max_tokens: maxTokens, + }), + }); + + if (!response.ok) { + lastStatus = response.status; + lastError = await response.text(); + // A 404 here means the id was retired, which is the whole reason this is + // a loop; a 429 means this model is busy or spent. Either way the next id + // is a different model and worth asking. + logger.error(`Groq API error: ${response.status} (model ${model})`, lastError); + continue; + } + + const data = await response.json(); + return { + content: data.choices[0]?.message?.content || '', + provider: 'groq', + model, + }; } - const data = await response.json(); - return { - content: data.choices[0]?.message?.content || '', - provider: 'groq', - model: API_CONFIG.GROQ_MODEL, - }; + throw new Error(`Groq API error: ${lastStatus} — all ${models.length} model(s) failed`); } /** @@ -118,36 +147,47 @@ async function generateWithOpenRouter( throw new Error('OpenRouter API key required'); } - const selectedModel = model || API_CONFIG.OPENROUTER_DEFAULT_MODEL; - - const response = await fetch(API_CONFIG.OPENROUTER_API_URL, { - method: 'POST', - headers: { - Authorization: `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - 'HTTP-Referer': getClientEnv().NEXT_PUBLIC_APP_URL, - 'X-Title': 'Botsmann', - }, - body: JSON.stringify({ + // An explicit caller override is honoured as-is and alone: if someone names a + // model, silently answering from a different one is worse than failing. + const models = model ? [model] : openRouterModels(); + let lastStatus = 0; + let lastError = ''; + + for (const selectedModel of models) { + const response = await fetch(API_CONFIG.OPENROUTER_API_URL, { + method: 'POST', + headers: { + Authorization: `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + 'HTTP-Referer': getClientEnv().NEXT_PUBLIC_APP_URL, + 'X-Title': 'Botsmann', + }, + body: JSON.stringify({ + model: selectedModel, + messages, + temperature, + max_tokens: maxTokens, + }), + }); + + if (!response.ok) { + lastStatus = response.status; + lastError = await response.text(); + logger.error(`OpenRouter API error: ${response.status} (model ${selectedModel})`, lastError); + continue; + } + + const data = await response.json(); + return { + content: data.choices[0]?.message?.content || '', + provider: 'openrouter', model: selectedModel, - messages, - temperature, - max_tokens: maxTokens, - }), - }); - - if (!response.ok) { - const error = await response.text(); - logger.error('OpenRouter API error:', error); - throw new Error('OpenRouter API request failed'); + }; } - const data = await response.json(); - return { - content: data.choices[0]?.message?.content || '', - provider: 'openrouter', - model: selectedModel, - }; + throw new Error( + `OpenRouter API request failed: ${lastStatus} — all ${models.length} model(s) failed`, + ); } /** diff --git a/package-lock.json b/package-lock.json index f7a743ac..b7f1bead 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@tailwindcss/forms": "^0.5.7", "@tailwindcss/typography": "^0.5.16", "@xenova/transformers": "^2.17.2", + "ai-kit": "github:maonakamoto/ai-kit#v0.4.0", "autoprefixer": "^10.4.17", "date-fns": "^4.1.0", "gray-matter": "^4.0.3", @@ -5437,6 +5438,42 @@ "node": ">= 14" } }, + "node_modules/ai-forms": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ai-forms/-/ai-forms-0.1.2.tgz", + "integrity": "sha512-agadca4pN0iGlZTlNy1Vo2SmnQB+0dNkrQSDE3wCJYeaVxKDs5KaxiwGj5+GL6mVYcI1xZ8pwnT9kL9WcxnBHA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": ">=18" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, + "node_modules/ai-kit": { + "version": "0.4.0", + "resolved": "git+ssh://git@github.com/maonakamoto/ai-kit.git#44f6bcaa69350c324b419bfd7026053dac1eb187", + "license": "MIT", + "dependencies": { + "ai-forms": "^0.1.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": ">=18" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, "node_modules/ajv": { "version": "6.15.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", diff --git a/package.json b/package.json index 06336ee5..dabdb0c7 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "@tailwindcss/forms": "^0.5.7", "@tailwindcss/typography": "^0.5.16", "@xenova/transformers": "^2.17.2", + "ai-kit": "github:maonakamoto/ai-kit#v0.4.0", "autoprefixer": "^10.4.17", "date-fns": "^4.1.0", "gray-matter": "^4.0.3", diff --git a/tests/__tests__/lib/free-fallback.test.ts b/tests/__tests__/lib/free-fallback.test.ts index 2f4e9632..4c6f3116 100644 --- a/tests/__tests__/lib/free-fallback.test.ts +++ b/tests/__tests__/lib/free-fallback.test.ts @@ -4,24 +4,47 @@ * Pinned as a test rather than trusted to review because this path is invisible * in normal operation: OpenRouter is only reached when Groq's free tier is * spent, so a paid id there bills precisely when nobody is watching. The - * previous default was `anthropic/claude-sonnet-5` — a premium model, and a + * original default was `anthropic/claude-sonnet-5` — a premium model, and a * breach of the standing fleet rule that Anthropic is never a primary or * fallback provider. * - * The rule mirrors `modelCost` in the shared `ai-ration` package: a routed id - * (`vendor/model`) is free only with the `:free` suffix. That suffix is the - * entire difference between free routing and a per-call charge for identical - * weights, which is why an id one token away from correct kept slipping through. + * The rule itself is unchanged. What changed is what it is checked against. + * + * It used to assert a regex on ONE constant in this repo, + * `OPENROUTER_DEFAULT_MODEL`. That constant is gone: it named + * `openai/gpt-oss-20b:free`, which the vendor has since retired, and the Groq id + * beside it went the same way. The ids now come from `ai-kit`, so the guarantee + * has to cover the whole list this app might actually send — every model, not + * just the first — and it uses `modelCost`, the function the rest of the fleet + * reasons about price with, rather than a regex re-derived here. + * + * `modelCost` is the right check and not merely the convenient one: for a routed + * id (`vendor/model`) the `:free` suffix is the entire difference between free + * routing and a per-call charge for identical weights, which is why an id one + * token away from correct kept slipping through review. */ -import { API_CONFIG } from '@/lib/constants'; +import { freeChain, providerModels, modelCost } from 'ai-kit'; + +const openRouter = freeChain('BOTSMANN')[1]; describe('free-first fallback', () => { - it('uses a :free OpenRouter model as the fallback default', () => { - expect(API_CONFIG.OPENROUTER_DEFAULT_MODEL).toMatch(/:free$/); + it('has an OpenRouter list to fall back to at all', () => { + // Guards the indexing above. If the chain is ever reordered so that [1] is + // not OpenRouter, every assertion below would pass vacuously against the + // wrong vendor. + expect(openRouter.id).toBe('openrouter'); + expect(providerModels(openRouter).length).toBeGreaterThan(0); + }); + + it('offers only free models as the fallback — every one of them', () => { + const paid = providerModels(openRouter).filter((m) => modelCost(m) !== 'free'); + // Named, not counted: the failure should say which id would bill. + expect(paid).toEqual([]); }); it('never falls back to Anthropic', () => { // Standing fleet rule: Anthropic is paid and is not a fallback anywhere. - expect(API_CONFIG.OPENROUTER_DEFAULT_MODEL).not.toMatch(/anthropic/i); + const anthropic = providerModels(openRouter).filter((m) => /anthropic/i.test(m)); + expect(anthropic).toEqual([]); }); }); diff --git a/tests/__tests__/lib/llm-client.test.ts b/tests/__tests__/lib/llm-client.test.ts index 1fb2afc1..37862712 100644 --- a/tests/__tests__/lib/llm-client.test.ts +++ b/tests/__tests__/lib/llm-client.test.ts @@ -1,15 +1,22 @@ import { generateLLMResponse, isOllamaAvailable, getBestProvider } from '@/lib/llm-client'; // Mock dependencies +// The model ids are deliberately absent: they come from `ai-kit` now, not from +// this repo. Asserting them literally here is what made these tests agree with +// a production outage — they mocked `llama-3.1-8b-instant` and passed happily +// for as long as Groq had been refusing that id in production. jest.mock('@/lib/constants', () => ({ API_CONFIG: { GROQ_API_URL: 'https://api.groq.com/openai/v1/chat/completions', - GROQ_MODEL: 'llama-3.1-8b-instant', OPENROUTER_API_URL: 'https://openrouter.ai/api/v1/chat/completions', - OPENROUTER_DEFAULT_MODEL: 'anthropic/claude-sonnet-5', }, })); +import { freeChain, providerModels } from 'ai-kit'; + +const GROQ_MODELS = providerModels(freeChain('BOTSMANN')[0]); +const OPENROUTER_MODELS = providerModels(freeChain('BOTSMANN')[1]); + jest.mock('@/lib/config/env', () => ({ getServerEnv: jest.fn(() => ({ GROQ_API_KEY: 'test-groq-key', @@ -78,14 +85,53 @@ describe('generateLLMResponse', () => { headers: expect.objectContaining({ Authorization: 'Bearer test-groq-key', }), - body: expect.stringContaining('"model":"llama-3.1-8b-instant"'), + body: expect.stringContaining(`"model":"${GROQ_MODELS[0]}"`), }), ); expect(result).toEqual({ content: 'Hello back!', provider: 'groq', - model: 'llama-3.1-8b-instant', + model: GROQ_MODELS[0], + }); + // The id must come from the maintained list, not from a constant here. + // `llama-3.*` is what this repo used to hardcode and what Groq retired. + expect(result.model).not.toMatch(/^llama-3/); + }); + + it('steps to the next model when the vendor has retired the first', async () => { + // The actual outage: HTTP 404 model_not_found, with a perfectly valid key. + // Before this, `generateWithBestProvider` picked one provider and called + // it once, so a retired id was simply a dead chatbot. + mockFetch + .mockResolvedValueOnce({ + ok: false, + status: 404, + text: async () => '{"error":{"code":"model_not_found"}}', + }) + .mockResolvedValueOnce({ + ok: true, + json: async () => ({ choices: [{ message: { content: 'second model' } }] }), + }); + + const result = await generateLLMResponse(testMessages, { provider: 'groq' }); + + expect(result.content).toBe('second model'); + expect(mockFetch).toHaveBeenCalledTimes(2); + // A retry that sends the SAME id is not a fallback. + expect(result.model).toBe(GROQ_MODELS[1]); + }); + + it('reports the whole list when every model fails', async () => { + mockFetch.mockResolvedValue({ + ok: false, + status: 429, + text: async () => 'rate limit exceeded', }); + + await expect(generateLLMResponse(testMessages, { provider: 'groq' })).rejects.toThrow( + /all \d+ model\(s\) failed/, + ); + expect(mockFetch).toHaveBeenCalledTimes(GROQ_MODELS.length); }); it('uses provided API key over server key', async () => { @@ -162,10 +208,15 @@ describe('generateLLMResponse', () => { expect(mockFetch).toHaveBeenCalledWith( 'https://openrouter.ai/api/v1/chat/completions', expect.objectContaining({ - body: expect.stringContaining('"model":"anthropic/claude-sonnet-5"'), + body: expect.stringContaining(`"model":"${OPENROUTER_MODELS[0]}"`), }), ); - expect(result.model).toBe('anthropic/claude-sonnet-5'); + expect(result.model).toBe(OPENROUTER_MODELS[0]); + // This test used to assert `anthropic/claude-sonnet-5` — a PAID model, on + // the path reached only when Groq's free tier is spent. It passed, which + // is how the fleet's no-Anthropic-fallback rule got broken in the first + // place. See free-fallback.test.ts for the cost guarantee itself. + expect(result.model).not.toMatch(/anthropic/i); }); it('uses custom model when specified', async () => {