diff --git a/backend/.env.example b/backend/.env.example index 22ed350..7a30b40 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -12,6 +12,12 @@ SECRET_KEY="your-secret-key-change-in-production" ALGORITHM="HS256" ACCESS_TOKEN_EXPIRE_MINUTES=3000 +# LLM +LLM_MODEL_NAME=groq/openai/gpt-oss-120b +GROQ_API_KEY= +LLM_PROVIDER=litellm +FALLBACK_LLM_PROVIDER= +LITELLM_API_KEY= # Piston (code execution) - runs via docker-compose # Use http://piston:2000 inside docker-compose, http://localhost:2000 for local dev PISTON_URL=http://localhost:2000 diff --git a/backend/README.md b/backend/README.md index 587fb9a..b5dd7c6 100644 --- a/backend/README.md +++ b/backend/README.md @@ -138,9 +138,15 @@ ACCESS_TOKEN_EXPIRE_MINUTES=30000 # TaskIQ / Redis REDIS_URL=redis://localhost:6379/0 -# LLM (Groq) +# LLM — primary provider LLM_MODEL_NAME=groq/openai/gpt-oss-120b GROQ_API_KEY=your-groq-api-key +LLM_PROVIDER=litellm + +# LLM — fallback provider (optional; leave blank to disable) +# Set to any LiteLLM-compatible model string, e.g. gemini/gemini-2.5-flash +FALLBACK_LLM_PROVIDER= +LITELLM_API_KEY= # Supabase Storage SUPABASE_URL=https://your-project.supabase.co @@ -259,7 +265,7 @@ app/interfaces/encrypter.py → app/utils/ (JwtEncrypter) app/interfaces/base_agent.py → app/ai/resume_evaluator.py ``` -This makes it straightforward to swap providers (e.g., Groq → OpenAI, Supabase → S3) without touching business logic. +This makes it straightforward to swap providers (e.g., Groq → OpenAI, Supabase → S3) without touching business logic. The active provider is selected at startup via `LLM_PROVIDER`; a fallback is instantiated from `FALLBACK_LLM_PROVIDER` and activated automatically on `AIProviderError` or `AITimeoutError` during resume evaluation. ### Model Structure diff --git a/backend/app/background/taskiq/tasks/resume_processing.py b/backend/app/background/taskiq/tasks/resume_processing.py index 84d72d9..20aaa1a 100644 --- a/backend/app/background/taskiq/tasks/resume_processing.py +++ b/backend/app/background/taskiq/tasks/resume_processing.py @@ -2,15 +2,19 @@ from sqlalchemy import select -from app.ai.lite_llm import LiteLLMProvider from app.ai.resume_evaluator import ResumeEvaluator from app.ai.schema import ResumeEvaluatorRequest from app.background.taskiq.taskiq import broker from app.database import AsyncSessionLocal +from app.exceptions.ai import AIProviderError, AITimeoutError from app.logger import get_logger from app.models.application import Application from app.models.interview import CustomInterview -from app.utils.default_providers import default_storage_provider +from app.utils.default_providers import ( + default_fallback_llm_provider, + default_llm_provider, + default_storage_provider, +) from app.utils.pdf import extract_pdf_content logger = get_logger(__name__) @@ -22,6 +26,8 @@ async def process_resume_task(file_bytes_b64: str, file_name: str, application_i file_bytes = base64.b64decode(file_bytes_b64) provider = default_storage_provider() + llm_provider = default_llm_provider() + fallback_llm_provider = default_fallback_llm_provider() async with AsyncSessionLocal() as session: app_to_update = await session.get(Application, application_id) @@ -41,7 +47,7 @@ async def process_resume_task(file_bytes_b64: str, file_name: str, application_i raise ValueError(f"Interview not found for application {application_id}.") extracted_text = extract_pdf_content(file_bytes) - evaluator = ResumeEvaluator(llm_provider=LiteLLMProvider()) + evaluator = ResumeEvaluator(llm_provider=llm_provider) req = ResumeEvaluatorRequest( resume_text=extracted_text, @@ -51,7 +57,17 @@ async def process_resume_task(file_bytes_b64: str, file_name: str, application_i ) logger.info("Starting resume evaluation for application %d...", application_id) - res = await evaluator.evaluate(req) + try: + res = await evaluator.evaluate(req) + except (AIProviderError, AITimeoutError): + if fallback_llm_provider is None: + raise + logger.warning( + "Primary LLM provider failed, retrying with fallback for application %d", + application_id, + ) + fallback_evaluator = ResumeEvaluator(llm_provider=fallback_llm_provider) + res = await fallback_evaluator.evaluate(req) app_to_update.resume = public_url app_to_update.extracted_resume = res.extracted_standardized_resume diff --git a/backend/app/config.py b/backend/app/config.py index dea15df..80baffe 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -20,6 +20,9 @@ class Settings(BaseSettings): # LLM LLM_MODEL_NAME: str = "groq/openai/gpt-oss-120b" GROQ_API_KEY: str = "" + LLM_PROVIDER: str = "litellm" + FALLBACK_LLM_PROVIDER: str = "gemini/gemini-2.5-flash" + LITELLM_API_KEY: str = "" CHATGROQ_API_KEY: str = "" CHATGROQ_MODEL_NAME: str = "llama3-8b-8192" ANTHROPIC_API_KEY: str = "" diff --git a/backend/app/utils/default_providers.py b/backend/app/utils/default_providers.py index 1e7fb7e..c6ac77e 100644 --- a/backend/app/utils/default_providers.py +++ b/backend/app/utils/default_providers.py @@ -1,5 +1,6 @@ from app.config import settings from app.interfaces.background_worker import BackgroundWorkerInterface +from app.interfaces.llm_provider import LLMProviderInterface from app.interfaces.email_provider import EmailProvider from app.interfaces.storage_proivder import StorageProviderInterface @@ -22,6 +23,24 @@ def default_worker_provider() -> BackgroundWorkerInterface: raise ValueError(f"Unknown background worker: '{settings.BACKGROUND_WORKER}'") +def default_llm_provider() -> LLMProviderInterface: + if settings.LLM_PROVIDER == "litellm": + from app.ai.lite_llm import LiteLLMProvider + + return LiteLLMProvider() + + raise ValueError(f"Unknown LLM provider: '{settings.LLM_PROVIDER}'") + + +def default_fallback_llm_provider() -> LLMProviderInterface | None: + if not settings.FALLBACK_LLM_PROVIDER: + return None + + from app.ai.lite_llm import LiteLLMProvider + + return LiteLLMProvider( + model_name=settings.FALLBACK_LLM_PROVIDER, api_key=settings.LITELLM_API_KEY + ) def default_email_provider() -> EmailProvider: if settings.EMAIL_PROVIDER == "smtp": from app.utils.smtp_provider import SmtpEmailProvider