From 2060499ea1a64bbddf7e4a407bc354bacc850487 Mon Sep 17 00:00:00 2001 From: Mao Nakamoto <41178744+maonakamoto@users.noreply.github.com> Date: Fri, 28 Aug 2026 18:06:37 +0200 Subject: [PATCH] fix(ai): a revoked key on the first provider took the whole chain down generateWithBestProvider picked ONE provider and called it once. Being configured counted as being available, so a present-but-revoked key was indistinguishable from a working one: botsmann's Groq key started returning 401 and the entire AI layer went down, with no attempt at anything else. kivvi survives the same dead-Groq situation on this box because it has an OpenRouter key and reaches it. botsmann would not have, even after adding one -- Groq is chosen first for merely existing, and its 401 was terminal. getProviderChain() every configured provider, in preference order (Ollama, Groq, OpenRouter) rather than the first generateWithBestProvider() walks that chain, demoting on failure, and only throws once every provider has been tried generateLLMResponse already walks the model list within a provider (#137), so this is the layer above: models, then providers. The error now names each provider that failed and why, instead of surfacing whichever one happened to be first. This does NOT restore botsmann's AI. Every working LLM key on the box belongs to a client -- AOZ's and RevampIT's -- and botsmann is a bitbaum product, so pointing it at one would bill a client for our usage. It needs its own key. What this does is make sure that once a second key exists, one dead key can never take the product down again. verify: format, lint, typecheck, 272 tests, build -- all green. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01HVwg8DKHQktxJuHeLM3xpG --- lib/llm-client.ts | 74 +++++++++++++++++++++----- tests/__tests__/lib/llm-client.test.ts | 73 ++++++++++++++++++++++++- 2 files changed, 132 insertions(+), 15 deletions(-) diff --git a/lib/llm-client.ts b/lib/llm-client.ts index 99a497e2..1f52cf77 100644 --- a/lib/llm-client.ts +++ b/lib/llm-client.ts @@ -323,26 +323,72 @@ export async function getBestProvider(): Promise<{ /** * Generate a response using the best available provider */ +/** + * Every provider that is configured, in preference order. + * + * Ollama first (local, private, free), then Groq (free tier), then OpenRouter + * (paid). Being configured is not the same as working -- a key can be present + * and revoked -- so this returns the whole chain and lets the caller demote. + */ +export async function getProviderChain(): Promise< + Array<{ provider: ModelProvider; reason: string }> +> { + const chain: Array<{ provider: ModelProvider; reason: string }> = []; + const env = getServerEnv(); + + if (await isOllamaAvailable()) { + chain.push({ provider: 'ollama', reason: 'Local Ollama running' }); + } + if (env.GROQ_API_KEY) { + chain.push({ provider: 'groq', reason: 'Groq API key configured' }); + } + if (env.OPENROUTER_API_KEY) { + chain.push({ provider: 'openrouter', reason: 'OpenRouter API key configured' }); + } + + return chain; +} + +/** + * Generate using the first provider that actually answers. + * + * This used to pick one provider and call it once, so a configured-but-revoked + * key was indistinguishable from having no provider at all: botsmann's Groq key + * started returning 401 and the whole AI layer went down while an OpenRouter + * key sat unused. Being chosen must not mean being trusted -- each provider + * gets demoted on failure and the next one is tried. + * + * generateLLMResponse already walks the model list within a provider, so this + * is the layer above that: models, then providers. + */ export async function generateWithBestProvider( messages: LLMMessage[], options?: Partial>, ): Promise { - const { provider, available, reason } = await getBestProvider(); + const chain = await getProviderChain(); - if (!available) { - throw new Error(reason); + if (chain.length === 0) { + throw new Error('No LLM provider available. Start Ollama or configure API keys.'); } - const fullOptions: LLMOptions = { - provider, - apiKey: provider === 'groq' ? getServerEnv().GROQ_API_KEY : getServerEnv().OPENROUTER_API_KEY, - ollamaUrl: getServerEnv().OLLAMA_URL, - ...options, - }; + const env = getServerEnv(); + const failures: string[] = []; + + for (const { provider, reason } of chain) { + try { + const response = await generateLLMResponse(messages, { + provider, + apiKey: provider === 'groq' ? env.GROQ_API_KEY : env.OPENROUTER_API_KEY, + ollamaUrl: env.OLLAMA_URL, + ...options, + }); + return { ...response, providerInfo: reason }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + failures.push(`${provider}: ${message}`); + logger.warn(`[LLM] ${provider} failed, trying next provider`, { error: message }); + } + } - const response = await generateLLMResponse(messages, fullOptions); - return { - ...response, - providerInfo: reason, - }; + throw new Error(`All ${chain.length} provider(s) failed \u2014 ${failures.join('; ')}`); } diff --git a/tests/__tests__/lib/llm-client.test.ts b/tests/__tests__/lib/llm-client.test.ts index 37862712..e117eecb 100644 --- a/tests/__tests__/lib/llm-client.test.ts +++ b/tests/__tests__/lib/llm-client.test.ts @@ -1,4 +1,9 @@ -import { generateLLMResponse, isOllamaAvailable, getBestProvider } from '@/lib/llm-client'; +import { + generateLLMResponse, + isOllamaAvailable, + getBestProvider, + generateWithBestProvider, +} from '@/lib/llm-client'; // Mock dependencies // The model ids are deliberately absent: they come from `ai-kit` now, not from @@ -384,3 +389,69 @@ describe('getBestProvider', () => { expect(result.reason).toContain('No LLM provider available'); }); }); + +describe('generateWithBestProvider — provider-level failover', () => { + const messages = [{ role: 'user' as const, content: 'hi' }]; + + /** + * The real outage: botsmann's Groq key started returning 401 while an + * OpenRouter key sat unused, and the whole AI layer went down. Being + * configured is not the same as working, so a provider must be demoted on + * failure rather than trusted because it was listed first. + */ + it('falls through to the next provider when the first has a revoked key', async () => { + (getServerEnv as jest.Mock).mockReturnValue({ + ...defaultEnv, + GROQ_API_KEY: 'gsk_revoked', + OPENROUTER_API_KEY: 'sk-or-working', + OLLAMA_URL: '', + }); + + mockFetch.mockImplementation(async (url: string) => { + const target = String(url); + if (target.includes('11434')) throw new Error('connection refused'); // no Ollama + if (target.includes('groq.com')) { + return { ok: false, status: 401, text: async () => '{"error":{"code":"invalid_api_key"}}' }; + } + return { + ok: true, + json: async () => ({ choices: [{ message: { content: 'from openrouter' } }] }), + }; + }); + + const result = await generateWithBestProvider(messages); + + expect(result.content).toBe('from openrouter'); + expect(result.provider).toBe('openrouter'); + }); + + it('reports every provider it tried when they all fail', async () => { + (getServerEnv as jest.Mock).mockReturnValue({ + ...defaultEnv, + GROQ_API_KEY: 'gsk_revoked', + OPENROUTER_API_KEY: 'sk-or-also-revoked', + OLLAMA_URL: '', + }); + + mockFetch.mockImplementation(async (url: string) => { + if (String(url).includes('11434')) throw new Error('connection refused'); + return { ok: false, status: 401, text: async () => 'invalid_api_key' }; + }); + + await expect(generateWithBestProvider(messages)).rejects.toThrow( + /All \d+ provider\(s\) failed/, + ); + }); + + it('says so plainly when nothing is configured at all', async () => { + (getServerEnv as jest.Mock).mockReturnValue({ + ...defaultEnv, + GROQ_API_KEY: '', + OPENROUTER_API_KEY: '', + OLLAMA_URL: '', + }); + mockFetch.mockRejectedValue(new Error('connection refused')); + + await expect(generateWithBestProvider(messages)).rejects.toThrow(/No LLM provider available/); + }); +});