Fall back system llm

backend/.env.example

@@ -12,6 +12,12 @@ SECRET_KEY="your-secret-key-change-in-production"
 ALGORITHM="HS256"
 ACCESS_TOKEN_EXPIRE_MINUTES=3000
 
+# LLM
+LLM_MODEL_NAME=groq/openai/gpt-oss-120b
+GROQ_API_KEY=
+LLM_PROVIDER=litellm
+FALLBACK_LLM_PROVIDER=
+LITELLM_API_KEY=
 # Piston (code execution) - runs via docker-compose
 # Use http://piston:2000 inside docker-compose, http://localhost:2000 for local dev
 PISTON_URL=http://localhost:2000

backend/README.md

@@ -138,9 +138,15 @@ ACCESS_TOKEN_EXPIRE_MINUTES=30000
 # TaskIQ / Redis
 REDIS_URL=redis://localhost:6379/0
 
-# LLM (Groq)
+# LLM — primary provider
 LLM_MODEL_NAME=groq/openai/gpt-oss-120b
 GROQ_API_KEY=your-groq-api-key
+LLM_PROVIDER=litellm
+
+# LLM — fallback provider (optional; leave blank to disable)
+# Set to any LiteLLM-compatible model string, e.g. gemini/gemini-2.5-flash
+FALLBACK_LLM_PROVIDER=
+LITELLM_API_KEY=
 
 # Supabase Storage
 SUPABASE_URL=https://your-project.supabase.co
@@ -259,7 +265,7 @@ app/interfaces/encrypter.py         →  app/utils/ (JwtEncrypter)
 app/interfaces/base_agent.py        →  app/ai/resume_evaluator.py
 ```
 
-This makes it straightforward to swap providers (e.g., Groq → OpenAI, Supabase → S3) without touching business logic.
+This makes it straightforward to swap providers (e.g., Groq → OpenAI, Supabase → S3) without touching business logic. The active provider is selected at startup via `LLM_PROVIDER`; a fallback is instantiated from `FALLBACK_LLM_PROVIDER` and activated automatically on `AIProviderError` or `AITimeoutError` during resume evaluation.
 
 ### Model Structure
 

backend/app/background/taskiq/tasks/resume_processing.py

@@ -2,15 +2,19 @@
 
 from sqlalchemy import select
 
-from app.ai.lite_llm import LiteLLMProvider
 from app.ai.resume_evaluator import ResumeEvaluator
 from app.ai.schema import ResumeEvaluatorRequest
 from app.background.taskiq.taskiq import broker
 from app.database import AsyncSessionLocal
+from app.exceptions.ai import AIProviderError, AITimeoutError
 from app.logger import get_logger
 from app.models.application import Application
 from app.models.interview import CustomInterview
-from app.utils.default_providers import default_storage_provider
+from app.utils.default_providers import (
+    default_fallback_llm_provider,
+    default_llm_provider,
+    default_storage_provider,
+)
 from app.utils.pdf import extract_pdf_content
 
 logger = get_logger(__name__)
@@ -22,6 +26,8 @@ async def process_resume_task(file_bytes_b64: str, file_name: str, application_i
 
     file_bytes = base64.b64decode(file_bytes_b64)
     provider = default_storage_provider()
+    llm_provider = default_llm_provider()
+    fallback_llm_provider = default_fallback_llm_provider()
 
     async with AsyncSessionLocal() as session:
         app_to_update = await session.get(Application, application_id)
@@ -41,7 +47,7 @@ async def process_resume_task(file_bytes_b64: str, file_name: str, application_i
                 raise ValueError(f"Interview not found for application {application_id}.")
 
             extracted_text = extract_pdf_content(file_bytes)
-            evaluator = ResumeEvaluator(llm_provider=LiteLLMProvider())
+            evaluator = ResumeEvaluator(llm_provider=llm_provider)
 
             req = ResumeEvaluatorRequest(
                 resume_text=extracted_text,
@@ -51,7 +57,17 @@ async def process_resume_task(file_bytes_b64: str, file_name: str, application_i
             )
 
             logger.info("Starting resume evaluation for application %d...", application_id)
-            res = await evaluator.evaluate(req)
+            try:
+                res = await evaluator.evaluate(req)
+            except (AIProviderError, AITimeoutError):
+                if fallback_llm_provider is None:
+                    raise
+                logger.warning(
+                    "Primary LLM provider failed, retrying with fallback for application %d",
+                    application_id,
+                )
+                fallback_evaluator = ResumeEvaluator(llm_provider=fallback_llm_provider)
+                res = await fallback_evaluator.evaluate(req)
 
             app_to_update.resume = public_url
             app_to_update.extracted_resume = res.extracted_standardized_resume

backend/app/config.py

@@ -20,6 +20,9 @@ class Settings(BaseSettings):
     # LLM
     LLM_MODEL_NAME: str = "groq/openai/gpt-oss-120b"
     GROQ_API_KEY: str = ""
+    LLM_PROVIDER: str = "litellm"
+    FALLBACK_LLM_PROVIDER: str = "gemini/gemini-2.5-flash"
+    LITELLM_API_KEY: str = ""
     CHATGROQ_API_KEY: str = ""
     CHATGROQ_MODEL_NAME: str = "llama3-8b-8192"
     ANTHROPIC_API_KEY: str = ""

backend/app/utils/default_providers.py

@@ -1,5 +1,6 @@
 from app.config import settings
 from app.interfaces.background_worker import BackgroundWorkerInterface
+from app.interfaces.llm_provider import LLMProviderInterface
 from app.interfaces.email_provider import EmailProvider
 from app.interfaces.storage_proivder import StorageProviderInterface
 
@@ -22,6 +23,24 @@ def default_worker_provider() -> BackgroundWorkerInterface:
     raise ValueError(f"Unknown background worker: '{settings.BACKGROUND_WORKER}'")
 
 
+def default_llm_provider() -> LLMProviderInterface:
+    if settings.LLM_PROVIDER == "litellm":
+        from app.ai.lite_llm import LiteLLMProvider
+
+        return LiteLLMProvider()
+
+    raise ValueError(f"Unknown LLM provider: '{settings.LLM_PROVIDER}'")
+
+
+def default_fallback_llm_provider() -> LLMProviderInterface | None:
+    if not settings.FALLBACK_LLM_PROVIDER:
+        return None
+
+    from app.ai.lite_llm import LiteLLMProvider
+
+    return LiteLLMProvider(
+        model_name=settings.FALLBACK_LLM_PROVIDER, api_key=settings.LITELLM_API_KEY
+    )
 def default_email_provider() -> EmailProvider:
     if settings.EMAIL_PROVIDER == "smtp":
         from app.utils.smtp_provider import SmtpEmailProvider