diff --git a/lib/llm-client.ts b/lib/llm-client.ts index 99a497e2..1f52cf77 100644 --- a/lib/llm-client.ts +++ b/lib/llm-client.ts @@ -323,26 +323,72 @@ export async function getBestProvider(): Promise<{ /** * Generate a response using the best available provider */ +/** + * Every provider that is configured, in preference order. + * + * Ollama first (local, private, free), then Groq (free tier), then OpenRouter + * (paid). Being configured is not the same as working -- a key can be present + * and revoked -- so this returns the whole chain and lets the caller demote. + */ +export async function getProviderChain(): Promise< + Array<{ provider: ModelProvider; reason: string }> +> { + const chain: Array<{ provider: ModelProvider; reason: string }> = []; + const env = getServerEnv(); + + if (await isOllamaAvailable()) { + chain.push({ provider: 'ollama', reason: 'Local Ollama running' }); + } + if (env.GROQ_API_KEY) { + chain.push({ provider: 'groq', reason: 'Groq API key configured' }); + } + if (env.OPENROUTER_API_KEY) { + chain.push({ provider: 'openrouter', reason: 'OpenRouter API key configured' }); + } + + return chain; +} + +/** + * Generate using the first provider that actually answers. + * + * This used to pick one provider and call it once, so a configured-but-revoked + * key was indistinguishable from having no provider at all: botsmann's Groq key + * started returning 401 and the whole AI layer went down while an OpenRouter + * key sat unused. Being chosen must not mean being trusted -- each provider + * gets demoted on failure and the next one is tried. + * + * generateLLMResponse already walks the model list within a provider, so this + * is the layer above that: models, then providers. + */ export async function generateWithBestProvider( messages: LLMMessage[], options?: Partial>, ): Promise { - const { provider, available, reason } = await getBestProvider(); + const chain = await getProviderChain(); - if (!available) { - throw new Error(reason); + if (chain.length === 0) { + throw new Error('No LLM provider available. Start Ollama or configure API keys.'); } - const fullOptions: LLMOptions = { - provider, - apiKey: provider === 'groq' ? getServerEnv().GROQ_API_KEY : getServerEnv().OPENROUTER_API_KEY, - ollamaUrl: getServerEnv().OLLAMA_URL, - ...options, - }; + const env = getServerEnv(); + const failures: string[] = []; + + for (const { provider, reason } of chain) { + try { + const response = await generateLLMResponse(messages, { + provider, + apiKey: provider === 'groq' ? env.GROQ_API_KEY : env.OPENROUTER_API_KEY, + ollamaUrl: env.OLLAMA_URL, + ...options, + }); + return { ...response, providerInfo: reason }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + failures.push(`${provider}: ${message}`); + logger.warn(`[LLM] ${provider} failed, trying next provider`, { error: message }); + } + } - const response = await generateLLMResponse(messages, fullOptions); - return { - ...response, - providerInfo: reason, - }; + throw new Error(`All ${chain.length} provider(s) failed \u2014 ${failures.join('; ')}`); } diff --git a/tests/__tests__/lib/llm-client.test.ts b/tests/__tests__/lib/llm-client.test.ts index 37862712..e117eecb 100644 --- a/tests/__tests__/lib/llm-client.test.ts +++ b/tests/__tests__/lib/llm-client.test.ts @@ -1,4 +1,9 @@ -import { generateLLMResponse, isOllamaAvailable, getBestProvider } from '@/lib/llm-client'; +import { + generateLLMResponse, + isOllamaAvailable, + getBestProvider, + generateWithBestProvider, +} from '@/lib/llm-client'; // Mock dependencies // The model ids are deliberately absent: they come from `ai-kit` now, not from @@ -384,3 +389,69 @@ describe('getBestProvider', () => { expect(result.reason).toContain('No LLM provider available'); }); }); + +describe('generateWithBestProvider — provider-level failover', () => { + const messages = [{ role: 'user' as const, content: 'hi' }]; + + /** + * The real outage: botsmann's Groq key started returning 401 while an + * OpenRouter key sat unused, and the whole AI layer went down. Being + * configured is not the same as working, so a provider must be demoted on + * failure rather than trusted because it was listed first. + */ + it('falls through to the next provider when the first has a revoked key', async () => { + (getServerEnv as jest.Mock).mockReturnValue({ + ...defaultEnv, + GROQ_API_KEY: 'gsk_revoked', + OPENROUTER_API_KEY: 'sk-or-working', + OLLAMA_URL: '', + }); + + mockFetch.mockImplementation(async (url: string) => { + const target = String(url); + if (target.includes('11434')) throw new Error('connection refused'); // no Ollama + if (target.includes('groq.com')) { + return { ok: false, status: 401, text: async () => '{"error":{"code":"invalid_api_key"}}' }; + } + return { + ok: true, + json: async () => ({ choices: [{ message: { content: 'from openrouter' } }] }), + }; + }); + + const result = await generateWithBestProvider(messages); + + expect(result.content).toBe('from openrouter'); + expect(result.provider).toBe('openrouter'); + }); + + it('reports every provider it tried when they all fail', async () => { + (getServerEnv as jest.Mock).mockReturnValue({ + ...defaultEnv, + GROQ_API_KEY: 'gsk_revoked', + OPENROUTER_API_KEY: 'sk-or-also-revoked', + OLLAMA_URL: '', + }); + + mockFetch.mockImplementation(async (url: string) => { + if (String(url).includes('11434')) throw new Error('connection refused'); + return { ok: false, status: 401, text: async () => 'invalid_api_key' }; + }); + + await expect(generateWithBestProvider(messages)).rejects.toThrow( + /All \d+ provider\(s\) failed/, + ); + }); + + it('says so plainly when nothing is configured at all', async () => { + (getServerEnv as jest.Mock).mockReturnValue({ + ...defaultEnv, + GROQ_API_KEY: '', + OPENROUTER_API_KEY: '', + OLLAMA_URL: '', + }); + mockFetch.mockRejectedValue(new Error('connection refused')); + + await expect(generateWithBestProvider(messages)).rejects.toThrow(/No LLM provider available/); + }); +});