feat: add real-time repository analysis and improve AI retrieval

.github/workflows/ci.yml

@@ -0,0 +1,61 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+      - master
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  frontend:
+    name: Frontend checks
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: frontend
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Build
+        run: npm run build
+
+  backend:
+    name: Backend checks
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: backend
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: backend/requirements.txt
+
+      - name: Install dependencies
+        run: python -m pip install -r requirements.txt
+
+      - name: Compile backend
+        run: python -m compileall -q app

backend/app/ai/context.py

@@ -11,12 +11,12 @@
 Keep answers useful and concise, with short headings or bullets when appropriate.
 Format every answer for a human reader:
 1. Start with a direct answer in one or two sentences.
-2. Use short Markdown headings beginning with ### when sections are useful.
+2. Use short Markdown headings beginning with -> when sections are useful.
 3. Use simple bullet points for evidence and file paths in backticks.
 4. End with a brief limitation statement only when the supplied context is incomplete.
 Do not return JSON, XML, provider metadata, or internal reasoning."""
 
-MAX_CONTEXT_CHARS = 18000
+MAX_CONTEXT_CHARS = 12000
 
 
 def _compact_analysis(analysis: dict):
@@ -35,16 +35,21 @@ def _compact_analysis(analysis: dict):
 
 
 def repository_context(analysis: dict, chunks: list[dict]):
+    compact = _compact_analysis(analysis)
+    architecture = compact.get("architecture", {})
+    architecture["module_graph"] = architecture.get("module_graph", [])[:20]
+    compact["source_analysis"] = compact.get("source_analysis", [])[:25]
+    compact["architecture"] = architecture
     return {
-        "analysis": _compact_analysis(analysis),
+        "analysis": compact,
         "retrieved_code_chunks": [
             {
                 "file": chunk["file_path"],
                 "language": chunk["language"],
                 "chunk_index": chunk["chunk_index"],
                 "content": chunk["content"][:2400]
             }
-            for chunk in chunks[:4]
+            for chunk in chunks[:2]
         ]
     }
 
@@ -77,7 +82,17 @@ def profile_context(repositories: list[dict]):
 def build_messages(question: str, context: dict):
     context_text = json.dumps(context, separators=(",", ":"), default=str)
     if len(context_text) > MAX_CONTEXT_CHARS:
-        context_text = context_text[:MAX_CONTEXT_CHARS] + "\n[context truncated]"
+        context_text = json.dumps(
+            {
+                **context,
+                "retrieved_code_chunks": context.get("retrieved_code_chunks", [])[:1],
+                "context_note": "Some low-priority context was omitted to preserve response speed."
+            },
+            separators=(",", ":"),
+            default=str
+        )
+        if len(context_text) > MAX_CONTEXT_CHARS:
+            context_text = context_text[:MAX_CONTEXT_CHARS] + "\n[context truncated]"
 
     return [
         {"role": "system", "content": SYSTEM_PROMPT},

backend/app/ai/llm.py

@@ -39,11 +39,12 @@ async def _request_provider(messages: list[dict], provider_name: str):
     if not settings["key"] or settings["key"].startswith("replace-after"):
         raise AIProviderError(f"{provider_name} API key is not configured")
 
+    configured_model = AI_MODEL if provider_name == AI_PROVIDER else None
     payload = {
-        "model": AI_MODEL or settings["default_model"],
+        "model": configured_model or settings["default_model"],
         "messages": messages,
         "temperature": 0.2,
-        "max_tokens": 1200
+        "max_tokens": 900
     }
     headers = {
         "Authorization": f"Bearer {settings['key']}",
@@ -74,6 +75,11 @@ async def ask_llm(messages: list[dict], provider: str | None = None):
         return await _request_provider(messages, provider_name)
     except AIProviderError as error:
         error_text = str(error).lower()
-        if provider is None and provider_name == "groq" and "context_length" in error_text:
+        if (
+            provider is None
+            and provider_name == "groq"
+            and ("context_length" in error_text or "rate_limit" in error_text)
+            and MISTRAL_API_KEY
+        ):
             return await _request_provider(messages, "mistral")
         raise

backend/app/ai/retrieval.py

@@ -38,18 +38,23 @@
 
 async def retrieve_chunks(access_token: str, owner: str, repo: str, question: str, limit: int = 3):
     full_name = f"{owner}/{repo}"
-    chunks = get_code_chunks(full_name)
+    repository = await get_repository(access_token, owner, repo)
+    source_version = repository.get("pushed_at") or repository.get("updated_at")
+    chunks = get_code_chunks(full_name, source_version)
     if not chunks:
         await index_repository(access_token, owner, repo)
-        chunks = get_code_chunks(full_name)
+        chunks = get_code_chunks(full_name, source_version)
 
     query_embedding = await embed_texts([question])
     question_terms = _terms(question)
     scored = []
     for chunk in chunks:
         score = 0.0
         if query_embedding and chunk.get("embedding"):
-            score = cosine_similarity(query_embedding[0], json.loads(chunk["embedding"]))
+            try:
+                score = cosine_similarity(query_embedding[0], json.loads(chunk["embedding"]))
+            except (TypeError, ValueError, json.JSONDecodeError):
+                score = 0.0
         else:
             content_terms = _terms(chunk["file_path"] + " " + chunk["content"])
             score = len(question_terms & content_terms) / max(len(question_terms), 1)

backend/app/analyzers/architecture_analyzer.py

@@ -1,4 +1,44 @@
-def analyze_architecture(files: list[str], technologies: list[str]):
+from pathlib import PurePosixPath
+
+
+def _module_candidates(import_name: str, source_file: str):
+    if not import_name.startswith("."):
+        return []
+
+    base = PurePosixPath(source_file).parent
+    relative = import_name.lstrip(".").replace(".", "/")
+    if import_name.startswith(".."):
+        base = base.parent
+    target = str(base / relative).replace("\\", "/").lstrip("./")
+    return [
+        target,
+        *[f"{target}.{extension}" for extension in ("py", "js", "jsx", "ts", "tsx")],
+        f"{target}/index.js",
+        f"{target}/index.ts",
+        f"{target}/__init__.py"
+    ]
+
+
+def _build_module_graph(source_analysis: list[dict]):
+    known_files = {item["file"] for item in source_analysis}
+    graph = []
+    for item in source_analysis:
+        for import_name in item.get("imports", []):
+            target = next(
+                (candidate for candidate in _module_candidates(import_name, item["file"])
+                 if candidate in known_files),
+                None
+            )
+            if target:
+                graph.append({"from": item["file"], "to": target, "import": import_name})
+    return graph[:100]
+
+
+def analyze_architecture(
+    files: list[str],
+    technologies: list[str],
+    source_analysis: list[dict] | None = None
+):
     lower_files = [path.lower() for path in files]
     lower_technologies = {technology.lower() for technology in technologies}
     layers = []
@@ -28,10 +68,27 @@ def analyze_architecture(files: list[str], technologies: list[str]):
     else:
         architecture_type = "Unclassified"
 
+    source_analysis = source_analysis or []
+    module_graph = _build_module_graph(source_analysis)
+    entry_points = [
+        path for path in files
+        if PurePosixPath(path).name.lower() in {
+            "main.py", "app.py", "server.py", "index.js", "main.jsx", "main.tsx"
+        }
+    ]
+
     return {
         "architecture_type": architecture_type,
         "frontend": next((name for name in ("React", "Vue", "Angular", "Next.js") if name.lower() in lower_technologies), None),
         "backend": next((name for name in ("FastAPI", "Flask", "Django", "Node.js") if name.lower() in lower_technologies), None),
         "external_services": ["GitHub API"] if any("github" in path for path in lower_files) else [],
-        "layers": layers
+        "layers": layers,
+        "entry_points": entry_points,
+        "module_graph": module_graph,
+        "module_relationships": len(module_graph),
+        "analyzed_source_files": len(source_analysis),
+        "analysis_note": (
+            "Architecture is inferred from repository paths, detected technologies, "
+            "and imports in the analyzed source files."
+        )
     }

backend/app/analyzers/file_detector.py

@@ -13,21 +13,29 @@
     "docker-compose.yml", "docker-compose.yaml", ".env.example", "README.md"
 }
 
+SOURCE_EXTENSIONS = {"py", "js", "jsx", "ts", "tsx"}
+MAX_ANALYSIS_FILES = 40
+
 
 def detect_important_files(files: list[str]):
     entry_points = []
     config_files = []
+    source_files = []
 
     for path in files:
         name = PurePosixPath(path).name
         if name in ENTRY_POINT_NAMES:
             entry_points.append(path)
         if name in CONFIG_FILE_NAMES or name.startswith("config."):
             config_files.append(path)
+        if PurePosixPath(path).suffix.lower().lstrip(".") in SOURCE_EXTENSIONS:
+            source_files.append(path)
 
-    important_files = list(dict.fromkeys(entry_points + config_files))
+    analysis_files = list(dict.fromkeys(entry_points + config_files + source_files))
     return {
         "entry_points": entry_points,
         "config_files": config_files,
-        "important_files": important_files
+        "source_files": source_files,
+        "important_files": analysis_files[:MAX_ANALYSIS_FILES],
+        "total_candidates": len(analysis_files)
     }

backend/app/database/repositories.py

@@ -27,12 +27,18 @@ def replace_code_chunks(repository: str, source_version: str | None, chunks: lis
     connection.close()
 
 
-def get_code_chunks(repository: str):
+def get_code_chunks(repository: str, source_version: str | None = None):
     connection = get_connection()
-    rows = connection.execute(
-        "SELECT * FROM code_chunks WHERE repository = ?",
-        (repository,)
-    ).fetchall()
+    if source_version is None:
+        rows = connection.execute(
+            "SELECT * FROM code_chunks WHERE repository = ?",
+            (repository,)
+        ).fetchall()
+    else:
+        rows = connection.execute(
+            "SELECT * FROM code_chunks WHERE repository = ? AND source_version = ?",
+            (repository, source_version)
+        ).fetchall()
     connection.close()
     return [dict(row) for row in rows]
 

backend/app/services/repo_analyzer.py

@@ -1,3 +1,5 @@
+import asyncio
+
 from app.services.github_api import (
     get_repository,
     get_repository_languages,
@@ -85,15 +87,16 @@ async def analyze_repository(
     selected_files = important_file_analysis["important_files"]
     source_contents = {}
 
-    for path in selected_files:
+    async def fetch_source(path: str):
         try:
-            content = await get_repository_file(
-                access_token,
-                owner,
-                repo,
-                path
-            )
+            return path, await get_repository_file(access_token, owner, repo, path)
         except Exception:
+            return path, None
+
+    fetched_files = await asyncio.gather(*(fetch_source(path) for path in selected_files))
+
+    for path, content in fetched_files:
+        if content is None:
             continue
 
         if path.endswith("package.json"):
@@ -142,7 +145,7 @@ async def analyze_repository(
         "readme": analyze_readme(readme_content),
         "important_files": important_file_analysis,
         "source_analysis": source_analysis,
-        "architecture": analyze_architecture(files, technologies)
+        "architecture": analyze_architecture(files, technologies, source_analysis)
     }