fix(ai): a revoked key on the first provider took the whole chain down

lib/llm-client.ts

@@ -323,26 +323,72 @@ export async function getBestProvider(): Promise<{
 /**
  * Generate a response using the best available provider
  */
+/**
+ * Every provider that is configured, in preference order.
+ *
+ * Ollama first (local, private, free), then Groq (free tier), then OpenRouter
+ * (paid). Being configured is not the same as working -- a key can be present
+ * and revoked -- so this returns the whole chain and lets the caller demote.
+ */
+export async function getProviderChain(): Promise<
+  Array<{ provider: ModelProvider; reason: string }>
+> {
+  const chain: Array<{ provider: ModelProvider; reason: string }> = [];
+  const env = getServerEnv();
+
+  if (await isOllamaAvailable()) {
+    chain.push({ provider: 'ollama', reason: 'Local Ollama running' });
+  }
+  if (env.GROQ_API_KEY) {
+    chain.push({ provider: 'groq', reason: 'Groq API key configured' });
+  }
+  if (env.OPENROUTER_API_KEY) {
+    chain.push({ provider: 'openrouter', reason: 'OpenRouter API key configured' });
+  }
+
+  return chain;
+}
+
+/**
+ * Generate using the first provider that actually answers.
+ *
+ * This used to pick one provider and call it once, so a configured-but-revoked
+ * key was indistinguishable from having no provider at all: botsmann's Groq key
+ * started returning 401 and the whole AI layer went down while an OpenRouter
+ * key sat unused. Being chosen must not mean being trusted -- each provider
+ * gets demoted on failure and the next one is tried.
+ *
+ * generateLLMResponse already walks the model list within a provider, so this
+ * is the layer above that: models, then providers.
+ */
 export async function generateWithBestProvider(
   messages: LLMMessage[],
   options?: Partial<Omit<LLMOptions, 'provider'>>,
 ): Promise<LLMResponse & { providerInfo: string }> {
-  const { provider, available, reason } = await getBestProvider();
+  const chain = await getProviderChain();
 
-  if (!available) {
-    throw new Error(reason);
+  if (chain.length === 0) {
+    throw new Error('No LLM provider available. Start Ollama or configure API keys.');
   }
 
-  const fullOptions: LLMOptions = {
-    provider,
-    apiKey: provider === 'groq' ? getServerEnv().GROQ_API_KEY : getServerEnv().OPENROUTER_API_KEY,
-    ollamaUrl: getServerEnv().OLLAMA_URL,
-    ...options,
-  };
+  const env = getServerEnv();
+  const failures: string[] = [];
+
+  for (const { provider, reason } of chain) {
+    try {
+      const response = await generateLLMResponse(messages, {
+        provider,
+        apiKey: provider === 'groq' ? env.GROQ_API_KEY : env.OPENROUTER_API_KEY,
+        ollamaUrl: env.OLLAMA_URL,
+        ...options,
+      });
+      return { ...response, providerInfo: reason };
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      failures.push(`${provider}: ${message}`);
+      logger.warn(`[LLM] ${provider} failed, trying next provider`, { error: message });
+    }
+  }
 
-  const response = await generateLLMResponse(messages, fullOptions);
-  return {
-    ...response,
-    providerInfo: reason,
-  };
+  throw new Error(`All ${chain.length} provider(s) failed \u2014 ${failures.join('; ')}`);
 }

tests/__tests__/lib/llm-client.test.ts

@@ -1,4 +1,9 @@
-import { generateLLMResponse, isOllamaAvailable, getBestProvider } from '@/lib/llm-client';
+import {
+  generateLLMResponse,
+  isOllamaAvailable,
+  getBestProvider,
+  generateWithBestProvider,
+} from '@/lib/llm-client';
 
 // Mock dependencies
 // The model ids are deliberately absent: they come from `ai-kit` now, not from
@@ -384,3 +389,69 @@ describe('getBestProvider', () => {
     expect(result.reason).toContain('No LLM provider available');
   });
 });
+
+describe('generateWithBestProvider — provider-level failover', () => {
+  const messages = [{ role: 'user' as const, content: 'hi' }];
+
+  /**
+   * The real outage: botsmann's Groq key started returning 401 while an
+   * OpenRouter key sat unused, and the whole AI layer went down. Being
+   * configured is not the same as working, so a provider must be demoted on
+   * failure rather than trusted because it was listed first.
+   */
+  it('falls through to the next provider when the first has a revoked key', async () => {
+    (getServerEnv as jest.Mock).mockReturnValue({
+      ...defaultEnv,
+      GROQ_API_KEY: 'gsk_revoked',
+      OPENROUTER_API_KEY: 'sk-or-working',
+      OLLAMA_URL: '',
+    });
+
+    mockFetch.mockImplementation(async (url: string) => {
+      const target = String(url);
+      if (target.includes('11434')) throw new Error('connection refused'); // no Ollama
+      if (target.includes('groq.com')) {
+        return { ok: false, status: 401, text: async () => '{"error":{"code":"invalid_api_key"}}' };
+      }
+      return {
+        ok: true,
+        json: async () => ({ choices: [{ message: { content: 'from openrouter' } }] }),
+      };
+    });
+
+    const result = await generateWithBestProvider(messages);
+
+    expect(result.content).toBe('from openrouter');
+    expect(result.provider).toBe('openrouter');
+  });
+
+  it('reports every provider it tried when they all fail', async () => {
+    (getServerEnv as jest.Mock).mockReturnValue({
+      ...defaultEnv,
+      GROQ_API_KEY: 'gsk_revoked',
+      OPENROUTER_API_KEY: 'sk-or-also-revoked',
+      OLLAMA_URL: '',
+    });
+
+    mockFetch.mockImplementation(async (url: string) => {
+      if (String(url).includes('11434')) throw new Error('connection refused');
+      return { ok: false, status: 401, text: async () => 'invalid_api_key' };
+    });
+
+    await expect(generateWithBestProvider(messages)).rejects.toThrow(
+      /All \d+ provider\(s\) failed/,
+    );
+  });
+
+  it('says so plainly when nothing is configured at all', async () => {
+    (getServerEnv as jest.Mock).mockReturnValue({
+      ...defaultEnv,
+      GROQ_API_KEY: '',
+      OPENROUTER_API_KEY: '',
+      OLLAMA_URL: '',
+    });
+    mockFetch.mockRejectedValue(new Error('connection refused'));
+
+    await expect(generateWithBestProvider(messages)).rejects.toThrow(/No LLM provider available/);
+  });
+});