diff --git a/.env.example b/.env.example index 278c447..8d4d321 100644 --- a/.env.example +++ b/.env.example @@ -32,6 +32,10 @@ SERVER_CORS_ALLOW_CREDENTIALS=true # ============================================================================= # Model Settings # ============================================================================= +# Provider profile: "cloud", "hybrid", "private", or "retrieval-only". +# This M1 setting is a service-level foundation; retrieval-only disables generation-provider requirements. +MODEL_PROFILE=cloud + # LLM Provider: "local", "openai", "groq", "anthropic", "openrouter", "together" # Default: "groq" (free tier, no GPU required). Use "local" for on-device inference (requires GPU) MODEL_LLM_PROVIDER=groq diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e59139..cee8cc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,19 +23,19 @@ jobs: python -m pip install --upgrade pip python -m pip install build python -m pip install -e . --no-deps - python -m pip install pytest pydantic pydantic-settings click PyYAML fastapi structlog ruff mypy + python -m pip install pytest pytest-asyncio pydantic pydantic-settings click PyYAML fastapi structlog ruff mypy mcp httpx - - name: Run Milestone 0 tests - run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py + - name: Run Milestone 0 and 1 tests + run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py - - name: Run Milestone 0 lint baseline - run: ruff check tests/test_m0_baseline.py scripts/evalfly_smoke.py + - name: Run Milestone 0 and 1 lint baseline + run: ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py - - name: Run Milestone 0 type baseline - run: PYTHONPATH=src mypy tests/test_m0_baseline.py scripts/evalfly_smoke.py + - name: Run Milestone 0 and 1 type baseline + run: PYTHONPATH=src mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py scripts/evalfly_smoke.py - name: Compile changed runtime modules - run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/generator.py tests/test_m0_baseline.py scripts/evalfly_smoke.py + run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/generator.py src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py - name: Build package run: python -m build diff --git a/.pi/.specsafe-state.json b/.pi/.specsafe-state.json index c57a86e..e4bda32 100644 --- a/.pi/.specsafe-state.json +++ b/.pi/.specsafe-state.json @@ -60,6 +60,26 @@ "turns": 0 } } + }, + { + "sliceId": "CRAG-20260627-M1-SERVICE-SEAM", + "workspaceId": "CodeRAG", + "sessionId": "omp-session", + "beganAt": "2026-06-27T21:04:01.773Z", + "endedAt": "2026-06-27T22:02:12.981Z", + "outcome": "PASS", + "costSummary": { + "externalMemoryCalls": 0, + "externalMemoryCost": 0, + "subagentTokens": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "cost": 0, + "turns": 0 + } + } } ] } diff --git a/docs/superpowers/plans/2026-06-27-m1-service-seam-contract.md b/docs/superpowers/plans/2026-06-27-m1-service-seam-contract.md new file mode 100644 index 0000000..01a4323 --- /dev/null +++ b/docs/superpowers/plans/2026-06-27-m1-service-seam-contract.md @@ -0,0 +1,131 @@ +# CodeRAG Milestone 1 Service Seam Contract + +Slice: `CRAG-20260627-M1-SERVICE-SEAM` +Parent plan: `docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md` +Branch: `feat/m1-service-seam` + +## Scope + +Milestone 1 extracts shared core services that preserve current behavior while removing duplicated repository metadata, indexing, retrieval/query, and deletion orchestration from API/MCP/CLI-facing handlers. This milestone is a seam extraction, not a retrieval-quality milestone: it must not introduce hybrid retrieval, SQLite registry, verified citation semantics, graph extraction, or autonomous editing behavior. + +## Source of truth + +1. Roadmap Milestone 1 (`docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md`). +2. Current implementation on `main` after PR #1 merge. +3. Existing public behavior of API routes, MCP tools, and CLI commands. +4. Existing JSON repository metadata shape in `settings.data_dir / "repositories.json"` until Milestone 2 replaces it with SQLite. + +## Frozen acceptance criteria + +### M1-AC1: Shared JSON-backed repository registry seam + +Pass requires: +- A `RepositoryRegistry` service owns loading, saving, listing, partial-id resolution, get, add/update, remove, and reloading of repository metadata. +- API routes and MCP handlers no longer each own separate `repositories` dictionaries and separate `repositories.json` load/save logic. +- The registry preserves the current JSON file format (`list[Repository.to_dict()]`) so existing alpha users are not forced into a migration in M1. + +Fail if: +- API and MCP can still report conflicting repository metadata because each maintains independent JSON state code. +- Repository ID partial matching semantics disappear. +- JSON format changes without a migration/reset decision. + +Required evidence: +- Unit tests for registry load/save/list/partial-id/remove behavior using a temp data directory. +- Regression tests proving API and MCP can share the same registry-backed metadata file. + +### M1-AC2: Shared indexing service seam + +Pass requires: +- An `IndexingService` owns repository validation/parsing, clone, file filtering, batching, embedding, vector-store writes, status transitions, error recording, commit SHA recording, and repo deletion cache cleanup where applicable. +- MCP handlers delegate full index, update, and delete operations to the service. +- API background indexing delegates to the same service path for full indexing. +- Existing response shapes for API and MCP are preserved. + +Fail if: +- Full indexing logic remains duplicated in both API routes and MCP handlers. +- Error status is not persisted on failed indexing. +- Embedding resources are not unloaded/released on the full-index path. + +Required evidence: +- Unit tests with fake dependencies proving full indexing status transitions and batch writes. +- Unit tests proving failed indexing persists `error` status and message. +- Unit tests or existing MCP regression tests proving handler response compatibility. + +### M1-AC3: Shared retrieval/query service seam + +Pass requires: +- A `RetrievalService` or `QueryService` owns readiness checks and response generation through `ResponseGenerator`. +- API query and MCP query delegate to the shared service instead of duplicating ready/not-found checks and generator orchestration. +- A search-only service path owns semantic search formatting for MCP `search_code` without invoking the LLM. +- Existing API `QueryResponse` and MCP dict shapes are preserved. + +Fail if: +- API and MCP still independently implement repository readiness checks and generator construction. +- Missing repo and not-ready errors become uncaught exceptions in MCP/CLI flows. + +Required evidence: +- Unit tests for query success, missing repo, and not-ready repo using fake generator/retriever dependencies. +- Existing MCP handler tests pass unchanged or with strictly seam-related fixture updates. + +### M1-AC4: Provider/profile config foundation + +Pass requires: +- A `ProviderConfigService` exposes the selected profile: `cloud`, `hybrid`, `private`, or `retrieval-only`. +- Invalid profiles fail configuration validation. +- Retrieval-only profile can be detected without requiring an LLM API key. +- Existing default remains low-friction cloud-oriented behavior and does not require a local generative model. + +Fail if: +- Default profile implies private/local-only hardware requirements. +- Retrieval-only mode still attempts to initialize/generate with an LLM in service-level validation paths or the existing UI query path. + +Required evidence: +- Config/service tests for valid profiles, invalid profile rejection, documented `.env` wiring, retrieval-only capability flags, and a static UI guard regression. + +### M1-AC5: Governance and evidence + +Pass requires: +- SpecSafe slice is open before implementation edits and closed only after verification. +- EvalFly smoke config is extended with M1 deterministic file/service checks, or a committed note explains why a deeper M1 EvalFly suite waits for fixture repos. +- CI remains green for the M0 gate and includes M1 unit tests in its targeted pytest step. +- Work is committed on `feat/m1-service-seam`, pushed, and opened as a PR after local verification. + +Fail if: +- Tests/evals are weakened, deleted, bypassed, or relabeled to pass. +- M1 is declared production-ready or claims SQLite/hybrid/citation/graph capabilities. + +Required evidence: +- Targeted pytest output for M0 + M1 tests. +- Ruff and mypy output for touched tests/scripts/service modules. +- `scripts/evalfly_smoke.py --suite smoke` output. +- Package build output. +- Post-implementation review against this contract. + +## Implementation boundaries + +Allowed: +- Add `src/coderag/services/` modules. +- Refactor API and MCP handlers to delegate to services. +- Update CLI only where it consumes MCP handler behavior. +- Keep JSON registry as the M1 backing store. +- Keep ChromaDB and current vector-only retrieval behind the new service seam. + +Not allowed in M1: +- SQLite registry schema or migrations. +- Hybrid/BM25 retrieval. +- Verified citation semantics or `grounded=True` behavior changes. +- Code graph extraction. +- New autonomous agent behavior. + +## Evidence commands + +Run before PR: + +```bash +PYTHONPATH=src .venv/bin/python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py +.venv/bin/python -m ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py tests/test_m1_services.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py +PYTHONPATH=src .venv/bin/python -m mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py tests/test_m1_services.py tests/test_api_routes.py scripts/evalfly_smoke.py +.venv/bin/python scripts/evalfly_smoke.py --suite smoke +.venv/bin/python -m py_compile src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m1_services.py tests/test_api_routes.py tests/test_ui_handlers.py +.venv/bin/python -m build +``` diff --git a/evals/config.json b/evals/config.json index 922a725..9d78fa9 100644 --- a/evals/config.json +++ b/evals/config.json @@ -121,6 +121,186 @@ } ] } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m1-service-contract-exists", + "title": "Milestone 1 service seam contract exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "delivery_governance", + "source": { + "kind": "acceptance_criteria", + "criteria": "M1-AC5" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "docs/superpowers/plans/2026-06-27-m1-service-seam-contract.md is present for Milestone 1 governance." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "docs/superpowers/plans/2026-06-27-m1-service-seam-contract.md" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m1-registry-service-exists", + "title": "Repository registry service exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "service_seam", + "source": { + "kind": "acceptance_criteria", + "criteria": "M1-AC1" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "src/coderag/services/registry.py is present for Milestone 1 governance." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "src/coderag/services/registry.py" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m1-indexing-service-exists", + "title": "Indexing service exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "service_seam", + "source": { + "kind": "acceptance_criteria", + "criteria": "M1-AC2" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "src/coderag/services/indexing.py is present for Milestone 1 governance." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "src/coderag/services/indexing.py" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m1-retrieval-service-exists", + "title": "Retrieval service exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "service_seam", + "source": { + "kind": "acceptance_criteria", + "criteria": "M1-AC3" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "src/coderag/services/retrieval.py is present for Milestone 1 governance." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "src/coderag/services/retrieval.py" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m1-service-tests-exist", + "title": "Milestone 1 service tests exist", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "test_coverage", + "source": { + "kind": "acceptance_criteria", + "criteria": "M1-AC1 M1-AC2 M1-AC3 M1-AC4" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "tests/test_m1_services.py is present for Milestone 1 governance." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "tests/test_m1_services.py" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m1-api-route-tests-exist", + "title": "Milestone 1 API route seam tests exist", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "test_coverage", + "source": { + "kind": "acceptance_criteria", + "criteria": "M1-AC1 M1-AC2 M1-AC3" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "tests/test_api_routes.py is present for Milestone 1 governance." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "tests/test_api_routes.py" + } + ] + } } ] } diff --git a/src/coderag/api/routes.py b/src/coderag/api/routes.py index 67360ed..ff9c243 100644 --- a/src/coderag/api/routes.py +++ b/src/coderag/api/routes.py @@ -1,178 +1,58 @@ """REST API routes.""" -import json -from datetime import datetime -from typing import Optional - -from fastapi import APIRouter, HTTPException, BackgroundTasks -from fastapi.responses import JSONResponse +from fastapi import APIRouter, BackgroundTasks, HTTPException from coderag.api.schemas import ( + CitationResponse, IndexRepositoryRequest, IndexRepositoryResponse, + ListRepositoriesResponse, QueryRequest, QueryResponse, - ListRepositoriesResponse, RepositoryInfo, - CitationResponse, RetrievedChunkResponse, - ErrorResponse, ) -from coderag.config import get_settings -from coderag.generation.generator import ResponseGenerator -from coderag.indexing.embeddings import EmbeddingGenerator -from coderag.indexing.vectorstore import VectorStore -from coderag.ingestion.chunker import CodeChunker -from coderag.ingestion.filter import FileFilter -from coderag.ingestion.loader import RepositoryLoader -from coderag.ingestion.validator import GitHubURLValidator, ValidationError from coderag.logging import get_logger -from coderag.models.document import Document -from coderag.models.query import Query as QueryModel -from coderag.models.repository import Repository, RepositoryStatus +from coderag.services.indexing import IndexingOptions, IndexingService +from coderag.services.registry import RepositoryRegistry +from coderag.services.retrieval import RetrievalService logger = get_logger(__name__) router = APIRouter() -# Global state (in production, use a proper database) -settings = get_settings() -repos_file = settings.data_dir / "repositories.json" -repositories: dict[str, Repository] = {} - - -def load_repositories() -> None: - """Load repositories from disk.""" - global repositories - if repos_file.exists(): - try: - data = json.loads(repos_file.read_text()) - repositories = {r["id"]: Repository.from_dict(r) for r in data} - except Exception as e: - logger.error("Failed to load repositories", error=str(e)) - - -def save_repositories() -> None: - """Save repositories to disk.""" - repos_file.parent.mkdir(parents=True, exist_ok=True) - data = [r.to_dict() for r in repositories.values()] - repos_file.write_text(json.dumps(data, indent=2)) - - -# Load on startup -load_repositories() - - -def resolve_repo_id(partial_id: str) -> Optional[str]: - """Resolve a partial repository ID to a full ID. - - Supports both full UUIDs and partial IDs (first 8+ characters). - Returns None if no match or multiple matches found. - """ - # First try exact match - if partial_id in repositories: - return partial_id +registry = RepositoryRegistry() +indexing_service = IndexingService(registry=registry) +retrieval_service = RetrievalService(registry=registry) - # Try prefix match (minimum 8 characters recommended) - matches = [rid for rid in repositories.keys() if rid.startswith(partial_id)] - if len(matches) == 1: - return matches[0] - - return None - - -def get_repo_or_404(repo_id: str) -> Repository: - """Get a repository by ID (full or partial), raising 404 if not found.""" - full_id = resolve_repo_id(repo_id) - if full_id is None: +def get_repo_or_404(repo_id: str): + """Get a repository by full ID or unambiguous prefix, raising 404 if not found.""" + repo = registry.get_unique(repo_id) + if repo is None: raise HTTPException(status_code=404, detail="Repository not found") - return repositories[full_id] + return repo async def index_repository_task( url: str, repo_id: str, - branch: Optional[str], - include_patterns: Optional[list[str]], - exclude_patterns: Optional[list[str]], + branch: str | None, + include_patterns: list[str] | None, + exclude_patterns: list[str] | None, ) -> None: - """Background task to index a repository.""" - import torch - - repo = repositories[repo_id] - embedder = None - + """Background task to index a repository through the shared service seam.""" try: - # Validate and clone - validator = GitHubURLValidator() - repo_info = await validator.validate_repository(url) - branch = branch or repo_info.branch or "main" - - loader = RepositoryLoader() - repo_path = loader.clone_repository(repo_info, branch) - - repo.clone_path = repo_path - repo.status = RepositoryStatus.INDEXING - save_repositories() - - # Filter files - file_filter = FileFilter( - include_patterns=include_patterns, - exclude_patterns=exclude_patterns, + indexing_service.index_repository_record( + repo_id, + url, + IndexingOptions( + branch=branch, + include_patterns=include_patterns, + exclude_patterns=exclude_patterns, + ), ) - files = list(file_filter.filter_files(repo_path)) - - # Process in batches to avoid CUDA OOM - vectorstore = VectorStore() - vectorstore.delete_repo_chunks(repo.id) - embedder = EmbeddingGenerator() - chunker = CodeChunker() - - batch_size = settings.ingestion.batch_size - total_chunks = 0 - batch: list = [] - - for file_path in files: - try: - doc = Document.from_file(file_path, repo_path, repo.id) - for chunk in chunker.chunk_document(doc): - chunk.repo_id = repo.id - batch.append(chunk) - - if len(batch) >= batch_size: - embedded = embedder.embed_chunks(batch, show_progress=False) - vectorstore.add_chunks(embedded) - total_chunks += len(batch) - del embedded - if torch.cuda.is_available(): - torch.cuda.empty_cache() - batch = [] - except Exception as e: - logger.warning("Failed to process file", path=str(file_path), error=str(e)) - - # Process final batch - if batch: - embedded = embedder.embed_chunks(batch, show_progress=False) - vectorstore.add_chunks(embedded) - total_chunks += len(batch) - del embedded - - # Update status - repo.chunk_count = total_chunks - repo.indexed_at = datetime.now() - repo.status = RepositoryStatus.READY - save_repositories() - - logger.info("Repository indexed", repo_id=repo_id, chunks=total_chunks) - - except Exception as e: - logger.error("Indexing failed", repo_id=repo_id, error=str(e)) - repo.status = RepositoryStatus.ERROR - repo.error_message = str(e) - save_repositories() - finally: - if embedder is not None: - embedder.unload() + except Exception as exc: + logger.error("Indexing failed", repo_id=repo_id, error=str(exc)) @router.post("/repos/index", response_model=IndexRepositoryResponse, status_code=202) @@ -181,16 +61,18 @@ async def index_repository( background_tasks: BackgroundTasks, ) -> IndexRepositoryResponse: """Index a GitHub repository.""" - # Create repository record - repo = Repository( - url=request.url, - branch=request.branch or "main", - status=RepositoryStatus.PENDING, - ) - repositories[repo.id] = repo - save_repositories() + try: + repo = indexing_service.create_repository_record( + request.url, + IndexingOptions( + branch=request.branch, + include_patterns=request.include_patterns, + exclude_patterns=request.exclude_patterns, + ), + ) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc - # Start background indexing background_tasks.add_task( index_repository_task, request.url, @@ -209,60 +91,50 @@ async def index_repository( @router.post("/query", response_model=QueryResponse) async def query_repository(request: QueryRequest) -> QueryResponse: - """Query a repository. - - Supports both full repository IDs and partial IDs (first 8+ characters). - """ - # Check repository exists (supports partial IDs) + """Query a repository by full ID or prefix.""" repo = get_repo_or_404(request.repo_id) - - if repo.status != RepositoryStatus.READY: - raise HTTPException( - status_code=400, - detail=f"Repository not ready (status: {repo.status.value})", - ) - - try: - # Generate response (use resolved repo.id for consistency) - generator = ResponseGenerator() - query = QueryModel( - question=request.question, - repo_id=repo.id, # Use resolved full ID - top_k=request.top_k, - ) - response = generator.generate(query) - - # Convert to API schema - return QueryResponse( - answer=response.answer, - citations=[ - CitationResponse( - file_path=c.file_path, - start_line=c.start_line, - end_line=c.end_line, - ) - for c in response.citations - ], - retrieved_chunks=[ - RetrievedChunkResponse( - chunk_id=c.chunk_id, - file_path=c.file_path, - start_line=c.start_line, - end_line=c.end_line, - relevance_score=c.relevance_score, - chunk_type=c.chunk_type, - name=c.name, - content=c.content, - ) - for c in response.retrieved_chunks - ], - grounded=response.grounded, - query_id=response.query_id, - ) - - except Exception as e: - logger.error("Query failed", error=str(e)) - raise HTTPException(status_code=500, detail=str(e)) + result = retrieval_service.query_code(repo.id, request.question, request.top_k) + if result.error is not None or result.response is None: + if result.error and result.error.startswith("Repository not found"): + raise HTTPException(status_code=404, detail="Repository not found") + if result.error and result.error.startswith("Repository not ready"): + raise HTTPException( + status_code=400, + detail=f"Repository not ready (status: {result.repo.status.value})" if result.repo else result.error, + ) + if result.error and ( + result.error.startswith("LLM generation is disabled") or result.error.startswith("Private profile requires") + ): + raise HTTPException(status_code=400, detail=result.error) + raise HTTPException(status_code=500, detail=result.error or "Query failed") + + response = result.response + return QueryResponse( + answer=response.answer, + citations=[ + CitationResponse( + file_path=c.file_path, + start_line=c.start_line, + end_line=c.end_line, + ) + for c in response.citations + ], + retrieved_chunks=[ + RetrievedChunkResponse( + chunk_id=c.chunk_id, + file_path=c.file_path, + start_line=c.start_line, + end_line=c.end_line, + relevance_score=c.relevance_score, + chunk_type=c.chunk_type, + name=c.name, + content=c.content, + ) + for c in response.retrieved_chunks + ], + grounded=response.grounded, + query_id=response.query_id, + ) @router.get("/repos", response_model=ListRepositoriesResponse) @@ -279,17 +151,14 @@ async def list_repositories() -> ListRepositoriesResponse: indexed_at=repo.indexed_at, error_message=repo.error_message, ) - for repo in repositories.values() + for repo in registry.list() ] ) @router.get("/repos/{repo_id}", response_model=RepositoryInfo) async def get_repository(repo_id: str) -> RepositoryInfo: - """Get repository details. - - Supports both full repository IDs and partial IDs (first 8+ characters). - """ + """Get repository details by full ID or prefix.""" repo = get_repo_or_404(repo_id) return RepositoryInfo( id=repo.id, @@ -304,23 +173,13 @@ async def get_repository(repo_id: str) -> RepositoryInfo: @router.delete("/repos/{repo_id}") async def delete_repository(repo_id: str) -> dict: - """Delete a repository. - - Supports both full repository IDs and partial IDs (first 8+ characters). - """ + """Delete a repository by full ID or prefix.""" repo = get_repo_or_404(repo_id) - try: - # Delete from vector store (use resolved full ID) - vectorstore = VectorStore() - vectorstore.delete_repo_chunks(repo.id) - - # Delete from records (use resolved full ID) - del repositories[repo.id] - save_repositories() - - return {"message": f"Repository {repo.full_name} deleted"} - - except Exception as e: - logger.error("Delete failed", error=str(e)) - raise HTTPException(status_code=500, detail=str(e)) + result = indexing_service.delete_repository(repo.id) + except Exception as exc: + logger.error("Delete failed", error=str(exc)) + raise HTTPException(status_code=500, detail=str(exc)) from exc + if result is None: + raise HTTPException(status_code=404, detail="Repository not found") + return {"message": f"Repository {result.repo.full_name} deleted"} diff --git a/src/coderag/config.py b/src/coderag/config.py index 0362242..2654965 100644 --- a/src/coderag/config.py +++ b/src/coderag/config.py @@ -1,25 +1,34 @@ """Application configuration using pydantic-settings.""" from pathlib import Path -from typing import Optional +from typing import Literal from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict + from coderag import __version__ class ModelSettings(BaseSettings): """LLM and embedding model configuration.""" - model_config = SettingsConfigDict(env_prefix="MODEL_") + model_config = SettingsConfigDict( + env_prefix="MODEL_", + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + ) + + # Privacy/provider profile: cloud is low-friction default; local-only is opt-in. + profile: Literal["cloud", "hybrid", "private", "retrieval-only"] = "cloud" # LLM Provider: "local", "openai", "groq", "anthropic", "openrouter" # Default to "groq" (free tier available, no GPU required) llm_provider: str = "groq" # API settings (for remote providers) - llm_api_key: Optional[str] = None - llm_api_base: Optional[str] = None # Custom API base URL + llm_api_key: str | None = None + llm_api_base: str | None = None # Custom API base URL # Model name (local or remote) llm_name: str = "Qwen/Qwen2.5-Coder-3B-Instruct" @@ -148,7 +157,7 @@ def ensure_directories(self) -> None: self.ingestion.repos_cache_dir.mkdir(parents=True, exist_ok=True) -_settings: Optional[Settings] = None +_settings: Settings | None = None def get_settings() -> Settings: diff --git a/src/coderag/mcp/__init__.py b/src/coderag/mcp/__init__.py index 815468f..8c9348d 100644 --- a/src/coderag/mcp/__init__.py +++ b/src/coderag/mcp/__init__.py @@ -1,7 +1,8 @@ """MCP (Model Context Protocol) server for CodeRAG.""" +from typing import Any + from coderag.mcp.handlers import MCPHandlers, get_mcp_handlers -from coderag.mcp.server import create_mcp_server, mcp __all__ = [ "MCPHandlers", @@ -9,3 +10,15 @@ "create_mcp_server", "mcp", ] + + +def __getattr__(name: str) -> Any: + if name in {"create_mcp_server", "mcp"}: + from coderag.mcp import server + + return getattr(server, name) + if name in {"tools", "resources", "prompts"}: + import importlib + + return importlib.import_module(f"coderag.mcp.{name}") + raise AttributeError(name) diff --git a/src/coderag/mcp/handlers.py b/src/coderag/mcp/handlers.py index 31d3c07..7f1f966 100644 --- a/src/coderag/mcp/handlers.py +++ b/src/coderag/mcp/handlers.py @@ -1,224 +1,73 @@ -"""MCP handlers for CodeRAG - non-streaming versions of UI handlers.""" +"""MCP handlers for CodeRAG.""" -import json -from datetime import datetime -from pathlib import Path -from typing import Any, Optional +from typing import Any -import torch - -from coderag.config import get_settings -from coderag.generation.generator import ResponseGenerator -from coderag.indexing.embeddings import EmbeddingGenerator -from coderag.indexing.vectorstore import VectorStore -from coderag.ingestion.chunker import CodeChunker -from coderag.ingestion.filter import FileFilter -from coderag.ingestion.loader import RepositoryLoader -from coderag.ingestion.validator import GitHubURLValidator, ValidationError from coderag.logging import get_logger -from coderag.models.chunk import Chunk -from coderag.models.document import Document -from coderag.models.query import Query -from coderag.models.repository import Repository, RepositoryStatus +from coderag.models.repository import Repository +from coderag.services.indexing import IndexingOptions, IndexingService +from coderag.services.registry import RepositoryRegistry +from coderag.services.retrieval import RetrievalService logger = get_logger(__name__) class MCPHandlers: - """Handlers for MCP tools - non-streaming versions.""" - - def __init__(self) -> None: - self.settings = get_settings() - self.validator = GitHubURLValidator() - self.loader = RepositoryLoader() - self.filter = FileFilter() - self.chunker = CodeChunker() - self.embedder = EmbeddingGenerator() - self.vectorstore = VectorStore() - self.generator: Optional[ResponseGenerator] = None - - # Repository metadata storage (shared with UIHandlers) - self.repos_file = self.settings.data_dir / "repositories.json" - self.repositories: dict[str, Repository] = self._load_repositories() - - def _load_repositories(self) -> dict[str, Repository]: - """Load repositories from JSON file.""" - if self.repos_file.exists(): - try: - data = json.loads(self.repos_file.read_text()) - return {r["id"]: Repository.from_dict(r) for r in data} - except Exception as e: - logger.error("Failed to load repositories", error=str(e)) - return {} - - def _save_repositories(self) -> None: - """Save repositories to JSON file.""" - self.repos_file.parent.mkdir(parents=True, exist_ok=True) - data = [r.to_dict() for r in self.repositories.values()] - self.repos_file.write_text(json.dumps(data, indent=2)) - - def _reload_repositories(self) -> None: - """Reload repositories from disk (for consistency with UIHandlers).""" - self.repositories = self._load_repositories() - - def _find_repository(self, repo_id: str) -> Optional[Repository]: - """Find repository by full or partial ID.""" - self._reload_repositories() - for rid, repo in self.repositories.items(): - if rid == repo_id or rid.startswith(repo_id): - return repo - return None + """Handlers for MCP tools backed by shared application services.""" - def _process_batch(self, chunks: list[Chunk]) -> int: - """Process a batch: embed + store + release memory.""" - if not chunks: - return 0 - - try: - embedded = self.embedder.embed_chunks(chunks, show_progress=False) - self.vectorstore.add_chunks(embedded) - finally: - # Always release memory, even on partial failure - if torch.cuda.is_available(): - torch.cuda.empty_cache() - - return len(chunks) - - def _get_current_commit(self, repo_path: Path) -> str: - """Get the SHA of the current commit.""" - from git import Repo - - git_repo = Repo(repo_path) - return git_repo.head.commit.hexsha - - def _get_changed_files( + def __init__( self, - repo_path: Path, - last_commit: str, - current_commit: str, - ) -> tuple[set[str], set[str], set[str]]: - """Get files that were added, modified, or deleted between commits.""" - from git import Repo - - git_repo = Repo(repo_path) - diff = git_repo.commit(last_commit).diff(current_commit) - - added: set[str] = set() - modified: set[str] = set() - deleted: set[str] = set() + registry: RepositoryRegistry | None = None, + indexing_service: IndexingService | None = None, + retrieval_service: RetrievalService | None = None, + ) -> None: + self.registry = registry or RepositoryRegistry() + self.indexing_service = indexing_service or IndexingService(registry=self.registry) + self.retrieval_service = retrieval_service or RetrievalService(registry=self.registry) + + @property + def repositories(self) -> dict[str, Repository]: + """Compatibility view for existing tests and adapters.""" + return {repo.id: repo for repo in self.registry.list()} + + @repositories.setter + def repositories(self, repositories: dict[str, Repository]) -> None: + self.registry.save(repositories) - for d in diff: - if d.new_file: - added.add(d.b_path) - elif d.deleted_file: - deleted.add(d.a_path) - elif d.renamed: - deleted.add(d.a_path) - added.add(d.b_path) - else: - modified.add(d.b_path or d.a_path) + def _reload_repositories(self) -> None: + """Compatibility no-op; registry reads latest data on demand.""" + self.registry.load() - return added, modified, deleted + def _find_repository(self, repo_id: str) -> Repository | None: + return self.registry.get(repo_id) async def index_repository( self, url: str, branch: str = "", - include_patterns: Optional[list[str]] = None, - exclude_patterns: Optional[list[str]] = None, + include_patterns: list[str] | None = None, + exclude_patterns: list[str] | None = None, ) -> dict[str, Any]: - """Index a GitHub repository (non-streaming version).""" + """Index a GitHub repository.""" try: - logger.info("MCP: Starting indexing", url=url, branch=branch) - repo_info = self.validator.parse_url(url) - branch = branch.strip() if branch else repo_info.branch or "main" - - # Create repository record - repo = Repository( - url=repo_info.url, - branch=branch, - status=RepositoryStatus.CLONING, - ) - self.repositories[repo.id] = repo - self._save_repositories() - - # Clone repository - logger.info("MCP: Cloning repository", url=url, branch=branch) - repo_path = self.loader.clone_repository(repo_info, branch) - repo.clone_path = repo_path - repo.status = RepositoryStatus.INDEXING - self._save_repositories() - - # Setup filter with custom patterns - file_filter = FileFilter( - include_patterns=include_patterns, - exclude_patterns=exclude_patterns, + result = await self.indexing_service.index_repository( + url, + IndexingOptions( + branch=branch or None, + include_patterns=include_patterns, + exclude_patterns=exclude_patterns, + ), ) - - # Process files - logger.info("MCP: Filtering files", repo_path=str(repo_path)) - files = list(file_filter.filter_files(repo_path)) - file_count = len(files) - logger.info("MCP: Files to process", count=file_count) - - # Delete existing chunks for this repo (re-indexing) - self.vectorstore.delete_repo_chunks(repo.id) - - # Index all files - total_chunks = 0 - batch: list[Chunk] = [] - batch_size = self.settings.ingestion.batch_size - - for file_path in files: - try: - doc = Document.from_file(file_path, repo_path, repo.id) - for chunk in self.chunker.chunk_document(doc): - chunk.repo_id = repo.id - batch.append(chunk) - - if len(batch) >= batch_size: - total_chunks += self._process_batch(batch) - batch = [] - except Exception as e: - logger.warning("Failed to process file", path=str(file_path), error=str(e)) - - # Process final batch - if batch: - total_chunks += self._process_batch(batch) - - # Save commit for incremental updates - try: - repo.last_commit = self._get_current_commit(repo_path) - except Exception: - repo.last_commit = None - - # Update repository status - repo.chunk_count = total_chunks - repo.indexed_at = datetime.now() - repo.status = RepositoryStatus.READY - self._save_repositories() - - logger.info("MCP: Indexing complete", repo_id=repo.id, chunks=total_chunks) - return { "success": True, - "repo_id": repo.id, - "name": repo.full_name, - "branch": repo.branch, - "files_processed": file_count, - "chunks_indexed": total_chunks, + "repo_id": result.repo.id, + "name": result.repo.full_name, + "branch": result.repo.branch, + "files_processed": result.files_processed, + "chunks_indexed": result.chunks_indexed, } - - except ValidationError as e: - logger.error("MCP: Validation error", error=str(e)) - return {"success": False, "error": f"Validation error: {str(e)}"} - except Exception as e: - logger.error("MCP: Indexing failed", error=str(e), exc_info=True) - if "repo" in locals(): - repo.status = RepositoryStatus.ERROR - repo.error_message = str(e) - self._save_repositories() - return {"success": False, "error": str(e)} + except Exception as exc: + logger.error("MCP: Indexing failed", error=str(exc), exc_info=True) + return {"success": False, "error": str(exc)} async def query_code( self, @@ -227,96 +76,57 @@ async def query_code( top_k: int = 5, ) -> dict[str, Any]: """Ask a question about a repository.""" - repo = self._find_repository(repo_id) - - if not repo: + result = self.retrieval_service.query_code(repo_id, question, top_k) + if result.error is not None or result.response is None: return { "answer": "", "citations": [], "evidence": [], "grounded": False, - "error": f"Repository not found: {repo_id}", - } - - if repo.status != RepositoryStatus.READY: - return { - "answer": "", - "citations": [], - "evidence": [], - "grounded": False, - "error": f"Repository not ready: status is {repo.status.value}", - } - - try: - # Lazy load generator - if self.generator is None: - self.generator = ResponseGenerator() - - query = Query( - question=question.strip(), - repo_id=repo.id, - top_k=int(top_k), - ) - - response = self.generator.generate(query) - - return { - "answer": response.answer, - "citations": response.citations, - "evidence": [ - { - "file": chunk.file_path, - "start_line": chunk.start_line, - "end_line": chunk.end_line, - "content": chunk.content[:500], # Truncate for MCP response - "relevance": round(chunk.relevance_score or 0, 3), - } - for chunk in response.retrieved_chunks - ], - "grounded": response.grounded, + "error": result.error or "Query failed", } - except Exception as e: - logger.error("MCP: Query failed", error=str(e)) - return { - "answer": "", - "citations": [], - "evidence": [], - "grounded": False, - "error": str(e), - } + response = result.response + return { + "answer": response.answer, + "citations": response.citations, + "evidence": [ + { + "file": chunk.file_path, + "start_line": chunk.start_line, + "end_line": chunk.end_line, + "content": chunk.content[:500], + "relevance": round(chunk.relevance_score or 0, 3), + } + for chunk in response.retrieved_chunks + ], + "grounded": response.grounded, + } async def list_repositories(self) -> dict[str, Any]: """List all indexed repositories.""" - self._reload_repositories() - - repos = [] - for repo in self.repositories.values(): - repos.append({ + repos = [ + { "id": repo.id, "name": repo.full_name, "branch": repo.branch, "status": repo.status.value, "chunk_count": repo.chunk_count, "indexed_at": repo.indexed_at.isoformat() if repo.indexed_at else None, - }) - - return { - "repositories": repos, - "count": len(repos), - } + } + for repo in self.registry.list() + ] + return {"repositories": repos, "count": len(repos)} async def get_repository_info(self, repo_id: str) -> dict[str, Any]: """Get detailed repository information.""" - repo = self._find_repository(repo_id) - + repo = self.registry.get(repo_id) if not repo: return {"error": f"Repository not found: {repo_id}"} - # Get indexed files from vectorstore indexed_files: list[str] = [] try: - files = self.vectorstore.get_indexed_files(repo.id) + files = self.indexing_service.get_indexed_files(repo.id) indexed_files = list(files) if files else [] except Exception: pass @@ -337,69 +147,21 @@ async def get_repository_info(self, repo_id: str) -> dict[str, Any]: async def delete_repository(self, repo_id: str) -> dict[str, Any]: """Delete an indexed repository.""" - repo = self._find_repository(repo_id) - - if not repo: + result = self.indexing_service.delete_repository(repo_id) + if result is None: return {"success": False, "error": f"Repository not found: {repo_id}"} - - try: - # Get chunk count before deletion - chunk_count = self.vectorstore.get_repo_chunk_count(repo.id) - - # Delete from vector store - self.vectorstore.delete_repo_chunks(repo.id) - - # Delete cached repo - try: - self.loader.delete_cache( - type("RepoInfo", (), {"owner": repo.owner, "name": repo.name})() - ) - except Exception: - pass - - # Remove from records - del self.repositories[repo.id] - self._save_repositories() - - logger.info("MCP: Repository deleted", repo_id=repo.id) - - return { - "success": True, - "repo_id": repo.id, - "name": repo.full_name, - "chunks_deleted": chunk_count, - } - - except Exception as e: - logger.error("MCP: Delete failed", error=str(e)) - return {"success": False, "error": str(e)} + return { + "success": True, + "repo_id": result.repo.id, + "name": result.repo.full_name, + "chunks_deleted": result.chunks_deleted, + } async def update_repository(self, repo_id: str) -> dict[str, Any]: """Incremental update of a repository.""" - repo = self._find_repository(repo_id) - - if not repo: - return {"success": False, "error": f"Repository not found: {repo_id}"} - - if not repo.last_commit: - return { - "success": False, - "error": "No previous indexing found. Please re-index the full repository.", - } - - if not repo.clone_path or not Path(repo.clone_path).exists(): - return {"success": False, "error": "Repository cache not found. Please re-index."} - try: - repo_path = Path(repo.clone_path) - - # Update local repository - logger.info("MCP: Updating local repository", repo_id=repo.id) - self.loader._update_repository(repo_path, repo.branch, None) - - current_commit = self._get_current_commit(repo_path) - - if current_commit == repo.last_commit: + result = self.indexing_service.update_repository(repo_id) + if result.already_up_to_date: return { "success": True, "message": "Repository is already up to date", @@ -407,112 +169,30 @@ async def update_repository(self, repo_id: str) -> dict[str, Any]: "chunks_added": 0, "chunks_deleted": 0, } - - added, modified, deleted = self._get_changed_files( - repo_path, repo.last_commit, current_commit - ) - - logger.info( - "MCP: Changes detected", - added=len(added), - modified=len(modified), - deleted=len(deleted), - ) - - # Delete chunks for deleted/modified files - chunks_deleted = 0 - for file_path in deleted | modified: - count = self.vectorstore.delete_file_chunks(repo.id, file_path) - chunks_deleted += count if count else 0 - - # Index new/modified files - files_to_index = [] - file_filter = FileFilter() - for file_path in added | modified: - full_path = repo_path / file_path - if full_path.exists() and file_filter.should_include(full_path, repo_path): - files_to_index.append(full_path) - - new_chunks = 0 - if files_to_index: - batch_size = self.settings.ingestion.batch_size - batch: list[Chunk] = [] - - for file_path in files_to_index: - try: - doc = Document.from_file(file_path, repo_path, repo.id) - for chunk in self.chunker.chunk_document(doc): - chunk.repo_id = repo.id - batch.append(chunk) - - if len(batch) >= batch_size: - new_chunks += self._process_batch(batch) - batch = [] - except Exception as e: - logger.warning("Failed to process file", path=str(file_path), error=str(e)) - - if batch: - new_chunks += self._process_batch(batch) - - # Update metadata - repo.last_commit = current_commit - repo.indexed_at = datetime.now() - repo.chunk_count = self.vectorstore.get_repo_chunk_count(repo.id) - self._save_repositories() - return { "success": True, - "files_changed": len(added | modified | deleted), - "files_added": len(added), - "files_modified": len(modified), - "files_deleted": len(deleted), - "chunks_added": new_chunks, - "chunks_deleted": chunks_deleted, - "total_chunks": repo.chunk_count, + "files_changed": result.files_changed, + "files_added": result.files_added, + "files_modified": result.files_modified, + "files_deleted": result.files_deleted, + "chunks_added": result.chunks_added, + "chunks_deleted": result.chunks_deleted, + "total_chunks": result.total_chunks, } - - except Exception as e: - logger.error("MCP: Incremental update failed", error=str(e), exc_info=True) - return {"success": False, "error": str(e)} + except Exception as exc: + return {"success": False, "error": str(exc)} async def search_code( self, repo_id: str, query: str, top_k: int = 10, - file_filter: Optional[str] = None, - chunk_type: Optional[str] = None, + file_filter: str | None = None, + chunk_type: str | None = None, ) -> dict[str, Any]: """Semantic code search without LLM generation.""" - repo = self._find_repository(repo_id) - - if not repo: - return {"results": [], "error": f"Repository not found: {repo_id}"} - - if repo.status != RepositoryStatus.READY: - return {"results": [], "error": f"Repository not ready: status is {repo.status.value}"} - try: - # Generate query embedding - query_embedding = self.embedder.generate_embedding(query, is_query=True) - - # Search vectorstore (query returns list of (Chunk, score) tuples) - results = self.vectorstore.query( - query_embedding=query_embedding, - repo_id=repo.id, - top_k=top_k, - ) - - # Filter by file pattern if provided - if file_filter: - import fnmatch - - results = [(chunk, score) for chunk, score in results if fnmatch.fnmatch(chunk.file_path, file_filter)] - - # Filter by chunk type if provided - if chunk_type: - results = [(chunk, score) for chunk, score in results if chunk.chunk_type == chunk_type] - + results = self.retrieval_service.search_code(repo_id, query, top_k, file_filter, chunk_type) return { "results": [ { @@ -523,18 +203,16 @@ async def search_code( "content": chunk.content, "relevance_score": round(score, 3), } - for chunk, score in results[:top_k] + for chunk, score in results ], "count": len(results), } - - except Exception as e: - logger.error("MCP: Search failed", error=str(e)) - return {"results": [], "error": str(e)} + except Exception as exc: + logger.error("MCP: Search failed", error=str(exc)) + return {"results": [], "error": str(exc)} -# Singleton pattern -_mcp_handlers: Optional[MCPHandlers] = None +_mcp_handlers: MCPHandlers | None = None def get_mcp_handlers() -> MCPHandlers: diff --git a/src/coderag/mcp/tools.py b/src/coderag/mcp/tools.py index 1f080c1..1824454 100644 --- a/src/coderag/mcp/tools.py +++ b/src/coderag/mcp/tools.py @@ -1,6 +1,5 @@ """MCP tool definitions for CodeRAG.""" -from typing import Optional from coderag.mcp.handlers import get_mcp_handlers from coderag.mcp.server import mcp @@ -10,8 +9,8 @@ async def index_repository( url: str, branch: str = "", - include_patterns: Optional[list[str]] = None, - exclude_patterns: Optional[list[str]] = None, + include_patterns: list[str] | None = None, + exclude_patterns: list[str] | None = None, ) -> dict: """Index a GitHub repository for code Q&A. @@ -115,8 +114,8 @@ async def search_code( repo_id: str, query: str, top_k: int = 10, - file_filter: Optional[str] = None, - chunk_type: Optional[str] = None, + file_filter: str | None = None, + chunk_type: str | None = None, ) -> dict: """Semantic code search without LLM generation. diff --git a/src/coderag/services/__init__.py b/src/coderag/services/__init__.py new file mode 100644 index 0000000..0cb32e5 --- /dev/null +++ b/src/coderag/services/__init__.py @@ -0,0 +1,23 @@ +"""Shared CodeRAG application services.""" + +from coderag.services.indexing import ( + IncrementalIndexingResult, + IndexingOptions, + IndexingResult, + IndexingService, +) +from coderag.services.providers import ProviderConfigService +from coderag.services.registry import RepositoryRegistry, RepositoryRegistryError +from coderag.services.retrieval import QueryServiceResult, RetrievalService + +__all__ = [ + "IndexingOptions", + "IncrementalIndexingResult", + "IndexingResult", + "IndexingService", + "ProviderConfigService", + "QueryServiceResult", + "RepositoryRegistry", + "RepositoryRegistryError", + "RetrievalService", +] diff --git a/src/coderag/services/indexing.py b/src/coderag/services/indexing.py new file mode 100644 index 0000000..37948f0 --- /dev/null +++ b/src/coderag/services/indexing.py @@ -0,0 +1,371 @@ +"""Repository indexing orchestration service.""" + +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, cast + +from coderag.config import get_settings +from coderag.logging import get_logger +from coderag.models.chunk import Chunk +from coderag.models.document import Document +from coderag.models.repository import Repository, RepositoryStatus +from coderag.services.registry import RepositoryRegistry + +logger = get_logger(__name__) + + +@dataclass(frozen=True) +class IndexingOptions: + """Options for a full repository indexing run.""" + + branch: str | None = None + include_patterns: list[str] | None = None + exclude_patterns: list[str] | None = None + + +@dataclass(frozen=True) +class IndexingResult: + """Result of a full repository indexing run.""" + + repo: Repository + files_processed: int + chunks_indexed: int + last_commit: str | None + + +@dataclass(frozen=True) +class IncrementalIndexingResult: + """Result of an incremental repository update.""" + + repo: Repository + files_changed: int + files_added: int + files_modified: int + files_deleted: int + chunks_added: int + chunks_deleted: int + total_chunks: int + already_up_to_date: bool = False + + +@dataclass(frozen=True) +class RepositoryDeleteResult: + """Result of deleting a repository from metadata, cache, and vector index.""" + + repo: Repository + chunks_deleted: int + + +class IndexingService: + """Shared indexing service for API, MCP, CLI, and UI adapters.""" + + def __init__( + self, + registry: RepositoryRegistry | None = None, + validator: Any | None = None, + loader: Any | None = None, + file_filter_factory: Callable[..., Any] | None = None, + chunker: Any | None = None, + embedder: Any | None = None, + vectorstore: Any | None = None, + commit_resolver: Callable[[Path], str | None] | None = None, + ) -> None: + self.settings = get_settings() + self.registry = registry or RepositoryRegistry() + self.validator = validator + self.loader = loader + self.file_filter_factory = file_filter_factory + self.chunker = chunker + self.embedder = embedder + self.vectorstore = vectorstore + self.commit_resolver = commit_resolver or self._get_current_commit + + async def index_repository(self, url: str, options: IndexingOptions | None = None) -> IndexingResult: + """Async-compatible full indexing entrypoint.""" + return self.index_repository_sync(url, options) + + def create_repository_record( + self, + url: str, + options: IndexingOptions | None = None, + status: RepositoryStatus = RepositoryStatus.PENDING, + ) -> Repository: + """Create and persist a repository record before background indexing.""" + options = options or IndexingOptions() + repo_info = self._get_validator().parse_url(url) + branch = (options.branch or repo_info.branch or "main").strip() or "main" + repo = Repository(url=repo_info.url, branch=branch, status=status) + self.registry.add(repo) + return repo + + def index_repository_record( + self, + repo_id: str, + url: str, + options: IndexingOptions | None = None, + ) -> IndexingResult: + """Index an adapter-created repository record by id.""" + options = options or IndexingOptions() + repo_info = self._get_validator().parse_url(url) + branch = (options.branch or repo_info.branch or "main").strip() or "main" + repo = self.registry.get(repo_id) + if repo is None: + raise ValueError(f"Repository not found: {repo_id}") + return self.index_existing_repository(repo, repo_info, branch, options) + + def index_repository_sync(self, url: str, options: IndexingOptions | None = None) -> IndexingResult: + """Clone, chunk, embed, and persist a repository index.""" + options = options or IndexingOptions() + repo_info = self._get_validator().parse_url(url) + branch = (options.branch or repo_info.branch or "main").strip() or "main" + repo = Repository(url=repo_info.url, branch=branch, status=RepositoryStatus.CLONING) + self.registry.add(repo) + return self.index_existing_repository(repo, repo_info, branch, options) + + def index_existing_repository( + self, + repo: Repository, + repo_info: Any, + branch: str, + options: IndexingOptions | None = None, + ) -> IndexingResult: + """Index into an existing repository record created by an adapter.""" + options = options or IndexingOptions() + try: + logger.info("Indexing service: cloning repository", url=repo.url, branch=branch) + repo.status = RepositoryStatus.CLONING + self.registry.update(repo) + repo_path = self._get_loader().clone_repository(repo_info, branch) + repo.clone_path = repo_path + repo.branch = branch + repo.status = RepositoryStatus.INDEXING + self.registry.update(repo) + + file_filter = self._get_file_filter_factory()( + include_patterns=options.include_patterns, + exclude_patterns=options.exclude_patterns, + ) + files = list(file_filter.filter_files(repo_path)) + + self._get_vectorstore().delete_repo_chunks(repo.id) + chunks_indexed = self.index_files(files, repo_path, repo.id) + last_commit = self._resolve_commit(repo_path) + + repo.chunk_count = chunks_indexed + repo.indexed_at = datetime.now() + repo.status = RepositoryStatus.READY + repo.last_commit = last_commit + self.registry.update(repo) + return IndexingResult(repo=repo, files_processed=len(files), chunks_indexed=chunks_indexed, last_commit=last_commit) + except Exception as exc: + repo.status = RepositoryStatus.ERROR + repo.error_message = str(exc) + self.registry.update(repo) + logger.error("Indexing service: indexing failed", repo_id=repo.id, error=str(exc)) + raise + finally: + unload = getattr(self.embedder, "unload", None) + if callable(unload): + unload() + + def index_files(self, files: Iterable[Path], repo_path: Path, repo_id: str) -> int: + """Chunk/embed/store a collection of files for one repository.""" + total_chunks = 0 + batch: list[Chunk] = [] + batch_size = self.settings.ingestion.batch_size + + for file_path in files: + try: + document = Document.from_file(file_path, repo_path, repo_id) + for chunk in self._get_chunker().chunk_document(document): + chunk.repo_id = repo_id + batch.append(chunk) + if len(batch) >= batch_size: + total_chunks += self._process_batch(batch) + batch = [] + except Exception as exc: + logger.warning("Failed to process file", path=str(file_path), error=str(exc)) + + if batch: + total_chunks += self._process_batch(batch) + + return total_chunks + + def delete_repository(self, repo_id: str) -> RepositoryDeleteResult | None: + """Delete repository chunks, cache, and metadata.""" + repo = self.registry.get(repo_id) + if repo is None: + return None + + vectorstore = self._get_vectorstore() + chunks_deleted = vectorstore.get_repo_chunk_count(repo.id) + vectorstore.delete_repo_chunks(repo.id) + + try: + self._get_loader().delete_cache(type("RepoInfo", (), {"owner": repo.owner, "name": repo.name})()) + except Exception: + logger.debug("Repository cache delete skipped", repo_id=repo.id) + + removed = self.registry.remove(repo.id) + if removed is None: + return None + return RepositoryDeleteResult(repo=removed, chunks_deleted=chunks_deleted) + + def update_repository(self, repo_id: str) -> IncrementalIndexingResult: + """Incrementally update an indexed repository from its cached clone.""" + repo = self.registry.get(repo_id) + if repo is None: + raise ValueError(f"Repository not found: {repo_id}") + if not repo.last_commit: + raise ValueError("No previous indexing found. Please re-index the full repository.") + if not repo.clone_path or not Path(repo.clone_path).exists(): + raise ValueError("Repository cache not found. Please re-index.") + + repo_path = Path(repo.clone_path) + self._get_loader()._update_repository(repo_path, repo.branch, None) + current_commit = self._resolve_commit(repo_path) + if current_commit == repo.last_commit: + return IncrementalIndexingResult( + repo=repo, + files_changed=0, + files_added=0, + files_modified=0, + files_deleted=0, + chunks_added=0, + chunks_deleted=0, + total_chunks=repo.chunk_count, + already_up_to_date=True, + ) + + added, modified, deleted = self._get_changed_files(repo_path, repo.last_commit, current_commit or repo.last_commit) + chunks_deleted = 0 + for file_path in deleted | modified: + count = self._get_vectorstore().delete_file_chunks(repo.id, file_path) + chunks_deleted += count or 0 + + file_filter = self._get_file_filter_factory()() + files_to_index = [] + for file_path in added | modified: + full_path = repo_path / file_path + if full_path.exists() and file_filter.should_include(full_path, repo_path): + files_to_index.append(full_path) + + chunks_added = self.index_files(files_to_index, repo_path, repo.id) if files_to_index else 0 + repo.last_commit = current_commit + repo.indexed_at = datetime.now() + repo.chunk_count = self._get_vectorstore().get_repo_chunk_count(repo.id) + self.registry.update(repo) + + return IncrementalIndexingResult( + repo=repo, + files_changed=len(added | modified | deleted), + files_added=len(added), + files_modified=len(modified), + files_deleted=len(deleted), + chunks_added=chunks_added, + chunks_deleted=chunks_deleted, + total_chunks=repo.chunk_count, + ) + + def get_indexed_files(self, repo_id: str) -> set[str]: + return cast(set[str], self._get_vectorstore().get_indexed_files(repo_id)) + + def get_chunk_count(self, repo_id: str) -> int: + return int(self._get_vectorstore().get_repo_chunk_count(repo_id)) + + def _process_batch(self, chunks: list[Chunk]) -> int: + if not chunks: + return 0 + try: + embedded = self._get_embedder().embed_chunks(chunks, show_progress=False) + self._get_vectorstore().add_chunks(embedded) + return len(chunks) + finally: + try: + import torch + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + except ImportError: + pass + + def _resolve_commit(self, repo_path: Path) -> str | None: + try: + return self.commit_resolver(repo_path) + except Exception: + return None + + def _get_current_commit(self, repo_path: Path) -> str: + from git import Repo + + git_repo = Repo(repo_path) + return cast(str, git_repo.head.commit.hexsha) + + def _get_changed_files( + self, + repo_path: Path, + last_commit: str, + current_commit: str, + ) -> tuple[set[str], set[str], set[str]]: + from git import Repo + + git_repo = Repo(repo_path) + diff = git_repo.commit(last_commit).diff(current_commit) + added: set[str] = set() + modified: set[str] = set() + deleted: set[str] = set() + for item in diff: + if item.new_file: + added.add(item.b_path) + elif item.deleted_file: + deleted.add(item.a_path) + elif item.renamed: + deleted.add(item.a_path) + added.add(item.b_path) + else: + modified.add(item.b_path or item.a_path) + return added, modified, deleted + + def _get_validator(self) -> Any: + if self.validator is None: + from coderag.ingestion.validator import GitHubURLValidator + + self.validator = GitHubURLValidator() + return self.validator + + def _get_loader(self) -> Any: + if self.loader is None: + from coderag.ingestion.loader import RepositoryLoader + + self.loader = RepositoryLoader() + return self.loader + + def _get_file_filter_factory(self) -> Callable[..., Any]: + if self.file_filter_factory is None: + from coderag.ingestion.filter import FileFilter + + self.file_filter_factory = FileFilter + return self.file_filter_factory + + def _get_chunker(self) -> Any: + if self.chunker is None: + from coderag.ingestion.chunker import CodeChunker + + self.chunker = CodeChunker() + return self.chunker + + def _get_embedder(self) -> Any: + if self.embedder is None: + from coderag.indexing.embeddings import EmbeddingGenerator + + self.embedder = EmbeddingGenerator() + return self.embedder + + def _get_vectorstore(self) -> Any: + if self.vectorstore is None: + from coderag.indexing.vectorstore import VectorStore + + self.vectorstore = VectorStore() + return self.vectorstore diff --git a/src/coderag/services/providers.py b/src/coderag/services/providers.py new file mode 100644 index 0000000..ccd6d8a --- /dev/null +++ b/src/coderag/services/providers.py @@ -0,0 +1,34 @@ +"""Provider/profile configuration service.""" + +from coderag.config import ModelSettings, get_settings + + +class ProviderConfigService: + """Expose model/profile capability flags without constructing providers.""" + + def __init__(self, model_settings: ModelSettings | None = None) -> None: + self.model_settings = model_settings or get_settings().models + + @property + def profile(self) -> str: + return self.model_settings.profile + + @property + def requires_generation_provider(self) -> bool: + return self.profile != "retrieval-only" + + @property + def requires_local_llm(self) -> bool: + return self.profile == "private" + + @property + def uses_cloud_generation(self) -> bool: + return self.requires_generation_provider and self.model_settings.llm_provider.lower() != "local" + + def generation_block_reason(self) -> str | None: + """Return why LLM generation is disallowed for the current profile.""" + if self.profile == "retrieval-only": + return "LLM generation is disabled by the retrieval-only profile" + if self.profile == "private" and self.model_settings.llm_provider.lower() != "local": + return "Private profile requires MODEL_LLM_PROVIDER=local" + return None diff --git a/src/coderag/services/registry.py b/src/coderag/services/registry.py new file mode 100644 index 0000000..c6876c6 --- /dev/null +++ b/src/coderag/services/registry.py @@ -0,0 +1,113 @@ +"""Repository metadata registry service.""" + +import json +from pathlib import Path + +from coderag.config import get_settings +from coderag.logging import get_logger +from coderag.models.repository import Repository + +logger = get_logger(__name__) + + +class RepositoryRegistryError(RuntimeError): + """Raised when existing repository metadata cannot be safely read.""" + + +class RepositoryRegistry: + """JSON-backed repository metadata registry. + + This preserves the pre-Milestone-1 `repositories.json` file shape while + centralizing load/save/partial-id semantics for API, MCP, CLI, and UI + adapters. Milestone 2 can replace this backing store with SQLite behind the + same interface. + """ + + def __init__(self, repos_file: Path | None = None) -> None: + settings = get_settings() + self.repos_file = repos_file or settings.data_dir / "repositories.json" + + def load(self) -> dict[str, Repository]: + """Load all repositories keyed by id.""" + if not self.repos_file.exists(): + return {} + + try: + data = json.loads(self.repos_file.read_text()) + return {item["id"]: Repository.from_dict(item) for item in data} + except Exception as exc: + logger.error("Failed to load repositories", path=str(self.repos_file), error=str(exc)) + raise RepositoryRegistryError(f"Failed to load repositories from {self.repos_file}") from exc + + def save(self, repositories: dict[str, Repository]) -> None: + """Persist repositories using the existing JSON list shape.""" + self.repos_file.parent.mkdir(parents=True, exist_ok=True) + data = [repo.to_dict() for repo in repositories.values()] + self.repos_file.write_text(json.dumps(data, indent=2)) + + def list(self) -> list[Repository]: + """Return repositories from the latest registry snapshot.""" + return list(self.load().values()) + + def resolve_id(self, partial_id: str) -> str | None: + """Resolve a full or prefix repository id using first-match compatibility.""" + repositories = self.load() + if partial_id in repositories: + return partial_id + + for repo_id in repositories: + if repo_id.startswith(partial_id): + return repo_id + return None + + def resolve_unique_id(self, partial_id: str) -> str | None: + """Resolve a full or unambiguous prefix repository id.""" + repositories = self.load() + if partial_id in repositories: + return partial_id + + matches = [repo_id for repo_id in repositories if repo_id.startswith(partial_id)] + if len(matches) == 1: + return matches[0] + return None + + def get(self, repo_id: str) -> Repository | None: + """Get a repository by full id or first prefix match.""" + repositories = self.load() + resolved_id = repo_id if repo_id in repositories else self.resolve_id(repo_id) + if resolved_id is None: + return None + return repositories.get(resolved_id) + + def get_unique(self, repo_id: str) -> Repository | None: + """Get a repository by full id or unambiguous prefix.""" + repositories = self.load() + resolved_id = repo_id if repo_id in repositories else self.resolve_unique_id(repo_id) + if resolved_id is None: + return None + return repositories.get(resolved_id) + + def add(self, repo: Repository) -> Repository: + """Add a repository and persist it.""" + repositories = self.load() + repositories[repo.id] = repo + self.save(repositories) + return repo + + def update(self, repo: Repository) -> Repository: + """Update a repository and persist it.""" + repositories = self.load() + repositories[repo.id] = repo + self.save(repositories) + return repo + + def remove(self, repo_id: str) -> Repository | None: + """Remove a repository by full id or prefix and persist the registry.""" + repositories = self.load() + resolved_id = repo_id if repo_id in repositories else self.resolve_id(repo_id) + if resolved_id is None: + return None + + repo = repositories.pop(resolved_id) + self.save(repositories) + return repo diff --git a/src/coderag/services/retrieval.py b/src/coderag/services/retrieval.py new file mode 100644 index 0000000..0c38101 --- /dev/null +++ b/src/coderag/services/retrieval.py @@ -0,0 +1,133 @@ +"""Shared retrieval and query orchestration service.""" + +import fnmatch +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, cast + +from coderag.config import get_settings +from coderag.logging import get_logger +from coderag.models.chunk import Chunk +from coderag.models.query import Query +from coderag.models.repository import Repository, RepositoryStatus +from coderag.models.response import Response +from coderag.services.providers import ProviderConfigService +from coderag.services.registry import RepositoryRegistry + +logger = get_logger(__name__) + + +@dataclass(frozen=True) +class QueryServiceResult: + """Service-level query result with typed adapter-friendly errors.""" + + response: Response | None = None + repo: Repository | None = None + error: str | None = None + + +class RetrievalService: + """Shared retrieval/query service for API, MCP, CLI, and UI adapters.""" + + def __init__( + self, + registry: RepositoryRegistry | None = None, + generator_factory: Callable[[], Any] | None = None, + embedder: Any | None = None, + vectorstore: Any | None = None, + provider_config: ProviderConfigService | None = None, + ) -> None: + self.registry = registry or RepositoryRegistry() + self.generator_factory = generator_factory or self._default_generator_factory + self.embedder = embedder + self.vectorstore = vectorstore + self._generator: Any | None = None + self.provider_config = provider_config or ProviderConfigService() + self.retrieval_settings = get_settings().retrieval + + def require_ready_repository(self, repo_id: str) -> tuple[Repository | None, str | None]: + """Return a ready repository or a public adapter-safe error message.""" + repo = self.registry.get(repo_id) + if repo is None: + return None, f"Repository not found: {repo_id}" + if repo.status != RepositoryStatus.READY: + return repo, f"Repository not ready: status is {repo.status.value}" + return repo, None + + def query_code(self, repo_id: str, question: str, top_k: int = 5) -> QueryServiceResult: + """Generate an answer for a ready repository.""" + repo, error = self.require_ready_repository(repo_id) + if error is not None: + return QueryServiceResult(repo=repo, error=error) + assert repo is not None + block_reason = self.provider_config.generation_block_reason() + if block_reason is not None: + return QueryServiceResult(repo=repo, error=block_reason) + top_k = self._clamp_top_k(top_k) + + try: + query = Query(question=question.strip(), repo_id=repo.id, top_k=int(top_k)) + response = self._get_generator().generate(query) + return QueryServiceResult(response=response, repo=repo) + except Exception as exc: + logger.error("Retrieval service: query failed", repo_id=repo.id if repo else repo_id, error=str(exc)) + return QueryServiceResult(repo=repo, error=str(exc)) + + def search_code( + self, + repo_id: str, + query: str, + top_k: int = 10, + file_filter: str | None = None, + chunk_type: str | None = None, + ) -> list[tuple[Chunk, float]]: + """Semantic code search without LLM generation.""" + repo, error = self.require_ready_repository(repo_id) + if error is not None: + raise ValueError(error) + assert repo is not None + + top_k = self._clamp_top_k(top_k) + embedder = self._get_embedder() + vectorstore = self._get_vectorstore() + query_embedding = embedder.generate_embedding(query, is_query=True) + results = vectorstore.query( + query_embedding=query_embedding, + repo_id=repo.id, + top_k=top_k, + similarity_threshold=self.retrieval_settings.similarity_threshold, + ) + + if file_filter: + results = [(chunk, score) for chunk, score in results if fnmatch.fnmatch(chunk.file_path, file_filter)] + if chunk_type: + results = [(chunk, score) for chunk, score in results if chunk.chunk_type.value == chunk_type or chunk.chunk_type == chunk_type] + + return cast(list[tuple[Chunk, float]], results[:top_k]) + + def _clamp_top_k(self, top_k: int) -> int: + return min(max(int(top_k), 1), self.retrieval_settings.max_top_k) + + def _default_generator_factory(self) -> Any: + from coderag.generation.generator import ResponseGenerator + + return ResponseGenerator() + + def _get_embedder(self) -> Any: + if self.embedder is None: + from coderag.indexing.embeddings import EmbeddingGenerator + + self.embedder = EmbeddingGenerator() + return self.embedder + + def _get_vectorstore(self) -> Any: + if self.vectorstore is None: + from coderag.indexing.vectorstore import VectorStore + + self.vectorstore = VectorStore() + return self.vectorstore + + def _get_generator(self) -> Any: + if self._generator is None: + self._generator = self.generator_factory() + return self._generator diff --git a/src/coderag/ui/handlers.py b/src/coderag/ui/handlers.py index 2628337..41d7cea 100644 --- a/src/coderag/ui/handlers.py +++ b/src/coderag/ui/handlers.py @@ -1,6 +1,5 @@ """UI event handlers for Gradio interface.""" -import json from datetime import datetime from pathlib import Path from typing import Iterator, Optional @@ -20,6 +19,8 @@ from coderag.models.document import Document from coderag.models.query import Query from coderag.models.repository import Repository, RepositoryStatus +from coderag.services.providers import ProviderConfigService +from coderag.ui.repository_metadata import load_repositories, save_repositories logger = get_logger(__name__) @@ -36,24 +37,21 @@ def __init__(self) -> None: self.embedder = EmbeddingGenerator() self.vectorstore = VectorStore() self.generator: Optional[ResponseGenerator] = None + self.provider_config = ProviderConfigService(self.settings.models) # Repository metadata storage self.repos_file = self.settings.data_dir / "repositories.json" self.repositories: dict[str, Repository] = self._load_repositories() def _load_repositories(self) -> dict[str, Repository]: - if self.repos_file.exists(): - try: - data = json.loads(self.repos_file.read_text()) - return {r["id"]: Repository.from_dict(r) for r in data} - except Exception as e: - logger.error("Failed to load repositories", error=str(e)) - return {} + try: + return load_repositories(self.repos_file) + except Exception as e: + logger.error("Failed to load repositories", error=str(e)) + raise def _save_repositories(self) -> None: - self.repos_file.parent.mkdir(parents=True, exist_ok=True) - data = [r.to_dict() for r in self.repositories.values()] - self.repos_file.write_text(json.dumps(data, indent=2)) + save_repositories(self.repos_file, self.repositories) # ========================================================================= # Streaming Methods (Nivel 1) @@ -411,6 +409,10 @@ def ask_question( if not question.strip(): return "", "", "Please enter a question" + block_reason = self.provider_config.generation_block_reason() + if block_reason is not None: + return "", "", f"Error: {block_reason}" + try: # Lazy load generator if self.generator is None: diff --git a/src/coderag/ui/repository_metadata.py b/src/coderag/ui/repository_metadata.py new file mode 100644 index 0000000..1a014f1 --- /dev/null +++ b/src/coderag/ui/repository_metadata.py @@ -0,0 +1,22 @@ +"""Repository metadata persistence helpers for the Gradio UI.""" + +import json +from pathlib import Path + +from coderag.models.repository import Repository + + +def load_repositories(repos_file: Path) -> dict[str, Repository]: + """Load UI repository metadata, failing closed on corrupt existing data.""" + if not repos_file.exists(): + return {} + + data = json.loads(repos_file.read_text()) + return {repo_data["id"]: Repository.from_dict(repo_data) for repo_data in data} + + +def save_repositories(repos_file: Path, repositories: dict[str, Repository]) -> None: + """Persist UI repository metadata using the existing list-of-dicts JSON shape.""" + repos_file.parent.mkdir(parents=True, exist_ok=True) + data = [repo.to_dict() for repo in repositories.values()] + repos_file.write_text(json.dumps(data, indent=2)) diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py new file mode 100644 index 0000000..d22ea05 --- /dev/null +++ b/tests/test_api_routes.py @@ -0,0 +1,229 @@ +"""REST API route tests for the Milestone 1 service seam.""" +# mypy: disable-error-code="no-untyped-def,type-arg,var-annotated,assignment,arg-type,call-arg,attr-defined,union-attr" + +from dataclasses import dataclass + +import pytest + +from coderag.api import routes +from coderag.api.schemas import IndexRepositoryRequest, QueryRequest +from coderag.models.repository import Repository, RepositoryStatus +from coderag.services.indexing import RepositoryDeleteResult +from coderag.services.retrieval import QueryServiceResult + + +class FakeBackgroundTasks: + def __init__(self) -> None: + self.tasks = [] + + def add_task(self, func, *args, **kwargs) -> None: + self.tasks.append((func, args, kwargs)) + + +class FakeIndexingService: + def __init__(self) -> None: + self.created_options = None + self.deleted_repo_id = None + self.repo = Repository( + id="repo-123456", + url="https://github.com/acme/demo", + branch="main", + status=RepositoryStatus.PENDING, + ) + + def create_repository_record(self, url, options): + self.created_options = options + self.repo.url = url + self.repo.branch = options.branch or "main" + return self.repo + + def delete_repository(self, repo_id): + self.deleted_repo_id = repo_id + if repo_id == self.repo.id: + return RepositoryDeleteResult(repo=self.repo, chunks_deleted=4) + return None + + +class FakeRegistry: + def __init__(self, repo: Repository) -> None: + self.repo = repo + + def get(self, repo_id: str): + if self.repo.id.startswith(repo_id): + return self.repo + return None + + def get_unique(self, repo_id: str): + if self.repo.id.startswith(repo_id): + return self.repo + return None + + def list(self): + return [self.repo] + + + +class AmbiguousRegistry: + def __init__(self) -> None: + self.repos = [ + Repository(id="abcdef111111", url="https://github.com/acme/one", status=RepositoryStatus.READY), + Repository(id="abcdef222222", url="https://github.com/acme/two", status=RepositoryStatus.READY), + ] + + def get_unique(self, repo_id: str): + matches = [repo for repo in self.repos if repo.id.startswith(repo_id)] + if len(matches) == 1: + return matches[0] + return None + + def list(self): + return self.repos + +@dataclass +class FakeGeneratedResponse: + answer: str = "shared answer" + citations: list = None + retrieved_chunks: list = None + grounded: bool = False + query_id: str = "query-1" + + def __post_init__(self) -> None: + self.citations = [] if self.citations is None else self.citations + self.retrieved_chunks = [] if self.retrieved_chunks is None else self.retrieved_chunks + + +class FakeRetrievalService: + def __init__(self, repo: Repository) -> None: + self.repo = repo + self.calls = [] + + def query_code(self, repo_id: str, question: str, top_k: int): + self.calls.append((repo_id, question, top_k)) + if repo_id == "missing": + return QueryServiceResult(error="Repository not found: missing") + if self.repo.status is not RepositoryStatus.READY: + return QueryServiceResult(repo=self.repo, error=f"Repository not ready: status is {self.repo.status.value}") + return QueryServiceResult(repo=self.repo, response=FakeGeneratedResponse()) + + +@pytest.mark.asyncio +async def test_index_repository_route_creates_record_through_indexing_service(monkeypatch): + indexing = FakeIndexingService() + monkeypatch.setattr(routes, "indexing_service", indexing) + background = FakeBackgroundTasks() + + response = await routes.index_repository( + IndexRepositoryRequest(url="https://github.com/acme/demo", branch="main", include_patterns=["*.py"]), + background, + ) + + assert response.repo_id == "repo-123456" + assert response.status == "pending" + assert indexing.created_options.include_patterns == ["*.py"] + assert background.tasks[0][0] is routes.index_repository_task + assert background.tasks[0][1][:3] == ("https://github.com/acme/demo", "repo-123456", "main") + + +@pytest.mark.asyncio +async def test_query_route_uses_retrieval_service_and_preserves_response(monkeypatch): + repo = Repository(id="repo-ready", url="https://github.com/acme/demo", status=RepositoryStatus.READY) + retrieval = FakeRetrievalService(repo) + monkeypatch.setattr(routes, "registry", FakeRegistry(repo)) + monkeypatch.setattr(routes, "retrieval_service", retrieval) + + response = await routes.query_repository(QueryRequest(repo_id="repo-ready", question="How?", top_k=3)) + + assert response.answer == "shared answer" + assert response.query_id == "query-1" + assert retrieval.calls == [("repo-ready", "How?", 3)] + + +@pytest.mark.asyncio +async def test_query_route_maps_missing_and_not_ready_errors(monkeypatch): + repo = Repository(id="repo-pending", url="https://github.com/acme/demo", status=RepositoryStatus.INDEXING) + retrieval = FakeRetrievalService(repo) + monkeypatch.setattr(routes, "registry", FakeRegistry(repo)) + monkeypatch.setattr(routes, "retrieval_service", retrieval) + + with pytest.raises(routes.HTTPException) as missing: + await routes.query_repository(QueryRequest(repo_id="missing", question="How?")) + with pytest.raises(routes.HTTPException) as not_ready: + await routes.query_repository(QueryRequest(repo_id=repo.id, question="How?")) + + assert missing.value.status_code == 404 + assert not_ready.value.status_code == 400 + assert not_ready.value.detail == "Repository not ready (status: indexing)" + + +@pytest.mark.asyncio +async def test_delete_route_maps_missing_repository(monkeypatch): + indexing = FakeIndexingService() + monkeypatch.setattr(routes, "registry", FakeRegistry(indexing.repo)) + monkeypatch.setattr(routes, "indexing_service", indexing) + + with pytest.raises(routes.HTTPException) as exc_info: + await routes.delete_repository("missing") + + assert exc_info.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_route_repository_list_get_and_delete_share_registry_and_indexing_service(monkeypatch): + indexing = FakeIndexingService() + registry = FakeRegistry(indexing.repo) + monkeypatch.setattr(routes, "indexing_service", indexing) + monkeypatch.setattr(routes, "registry", registry) + + listed = await routes.list_repositories() + fetched = await routes.get_repository("repo-123") + deleted = await routes.delete_repository("repo-123") + + assert listed.repositories[0].id == "repo-123456" + assert fetched.id == "repo-123456" + assert deleted == {"message": "Repository acme/demo deleted"} + assert indexing.deleted_repo_id == "repo-123456" + + + +@pytest.mark.asyncio +async def test_delete_route_rejects_ambiguous_partial_id_without_deleting(monkeypatch): + indexing = FakeIndexingService() + monkeypatch.setattr(routes, "registry", AmbiguousRegistry()) + monkeypatch.setattr(routes, "indexing_service", indexing) + + with pytest.raises(routes.HTTPException) as exc_info: + await routes.delete_repository("abcdef") + + assert exc_info.value.status_code == 404 + assert indexing.deleted_repo_id is None + +@pytest.mark.asyncio +async def test_api_and_mcp_handlers_share_registry_backed_metadata_file(tmp_path, monkeypatch): + from coderag.mcp.handlers import MCPHandlers + from coderag.services.registry import RepositoryRegistry + + repos_file = tmp_path / "repositories.json" + api_registry = RepositoryRegistry(repos_file) + repo = api_registry.add( + Repository( + id="shared-repo", + url="https://github.com/acme/demo", + branch="main", + status=RepositoryStatus.READY, + chunk_count=7, + ) + ) + monkeypatch.setattr(routes, "registry", api_registry) + + mcp = MCPHandlers( + registry=RepositoryRegistry(repos_file), + indexing_service=FakeIndexingService(), + retrieval_service=FakeRetrievalService(repo), + ) + + api_list = await routes.list_repositories() + mcp_list = await mcp.list_repositories() + + assert api_list.repositories[0].id == "shared-repo" + assert mcp_list["repositories"][0]["id"] == "shared-repo" + assert mcp_list["repositories"][0]["chunk_count"] == 7 diff --git a/tests/test_m1_services.py b/tests/test_m1_services.py new file mode 100644 index 0000000..505faef --- /dev/null +++ b/tests/test_m1_services.py @@ -0,0 +1,342 @@ +"""Milestone 1 service seam tests.""" +# mypy: disable-error-code="no-untyped-def,type-arg,var-annotated,attr-defined,assignment,union-attr,arg-type" + +from dataclasses import dataclass +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from coderag import config as config_module +from coderag.config import ModelSettings +from coderag.models.chunk import Chunk, ChunkMetadata, ChunkType +from coderag.models.repository import Repository, RepositoryStatus +from coderag.services.indexing import IndexingOptions, IndexingService +from coderag.services.providers import ProviderConfigService +from coderag.services.registry import RepositoryRegistry, RepositoryRegistryError +from coderag.services.retrieval import RetrievalService + + +@dataclass +class FakeRepoInfo: + url: str = "https://github.com/acme/demo" + owner: str = "acme" + name: str = "demo" + branch: str = "main" + + @property + def full_name(self) -> str: + return f"{self.owner}/{self.name}" + + +class FakeValidator: + def parse_url(self, url: str) -> FakeRepoInfo: + return FakeRepoInfo(url=url) + + +class FakeLoader: + def __init__(self, repo_path: Path) -> None: + self.repo_path = repo_path + self.deleted: list[str] = [] + + def clone_repository(self, _repo_info: FakeRepoInfo, _branch: str) -> Path: + return self.repo_path + + def delete_cache(self, repo_info: object) -> None: + self.deleted.append(f"{repo_info.owner}/{repo_info.name}") + + +class FakeFilter: + def __init__(self, include_patterns=None, exclude_patterns=None) -> None: + self.include_patterns = include_patterns + self.exclude_patterns = exclude_patterns + + def filter_files(self, repo_path: Path): + yield repo_path / "app.py" + + def should_include(self, _file_path: Path, _repo_path: Path) -> bool: + return True + + +class FakeChunker: + def chunk_document(self, document): + yield Chunk( + content=document.content, + repo_id=document.repo_id, + metadata=ChunkMetadata( + file_path=document.file_path, + start_line=1, + end_line=1, + chunk_type=ChunkType.TEXT, + language="python", + name=None, + ), + ) + + +class FakeEmbedder: + def __init__(self) -> None: + self.unloaded = False + + def embed_chunks(self, chunks: list[Chunk], show_progress: bool = False) -> list[Chunk]: + _ = show_progress + for chunk in chunks: + chunk.embedding = [0.1, 0.2] + return chunks + + def generate_embedding(self, text: str, is_query: bool = False) -> list[float]: + _ = (text, is_query) + return [0.1, 0.2] + + def unload(self) -> None: + self.unloaded = True + + +class FakeVectorStore: + def __init__(self) -> None: + self.added: list[Chunk] = [] + self.deleted_repos: list[str] = [] + self.queries: list[dict] = [] + + def add_chunks(self, chunks: list[Chunk]) -> int: + self.added.extend(chunks) + return len(chunks) + + def delete_repo_chunks(self, repo_id: str) -> int: + self.deleted_repos.append(repo_id) + return 3 + + def delete_file_chunks(self, _repo_id: str, _file_path: str) -> int: + return 1 + + def get_repo_chunk_count(self, repo_id: str) -> int: + return len([chunk for chunk in self.added if chunk.repo_id == repo_id]) + + def get_indexed_files(self, repo_id: str) -> set[str]: + return {chunk.file_path for chunk in self.added if chunk.repo_id == repo_id} + + def query(self, query_embedding, repo_id: str, top_k: int = 5, similarity_threshold: float = 0.0): + _ = query_embedding + self.queries.append({"repo_id": repo_id, "top_k": top_k, "threshold": similarity_threshold}) + return [(chunk, 0.87) for chunk in self.added if chunk.repo_id == repo_id][:top_k] + + +class FakeResponse: + answer = "shared answer" + citations = [] + retrieved_chunks = [] + grounded = False + query_id = "query-1" + + +class FakeGenerator: + def __init__(self) -> None: + self.queries = [] + + def generate(self, query): + self.queries.append(query) + return FakeResponse() + + +def test_repository_registry_preserves_json_shape_and_partial_ids(tmp_path): + registry = RepositoryRegistry(tmp_path / "repositories.json") + repo = Repository( + id="abcdef123456", + url="https://github.com/acme/demo", + branch="main", + status=RepositoryStatus.READY, + chunk_count=7, + ) + + registry.add(repo) + + assert registry.resolve_id("abcdef12") == repo.id + assert registry.get("abcdef12") == repo + raw = (tmp_path / "repositories.json").read_text() + assert raw.startswith("[") + assert '"url": "https://github.com/acme/demo"' in raw + + reloaded = RepositoryRegistry(tmp_path / "repositories.json") + assert reloaded.list()[0].id == repo.id + + removed = reloaded.remove("abcdef12") + assert removed == repo + assert reloaded.list() == [] + + +def test_repository_registry_corrupt_json_fails_closed_without_overwriting(tmp_path): + repos_file = tmp_path / "repositories.json" + repos_file.write_text("{not-json") + registry = RepositoryRegistry(repos_file) + + with pytest.raises(RepositoryRegistryError): + registry.list() + assert repos_file.read_text() == "{not-json" + + +def test_indexing_service_full_index_updates_shared_registry_and_vector_store(tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "app.py").write_text("print('hello')\n") + registry = RepositoryRegistry(tmp_path / "repositories.json") + vectorstore = FakeVectorStore() + embedder = FakeEmbedder() + service = IndexingService( + registry=registry, + validator=FakeValidator(), + loader=FakeLoader(repo_path), + file_filter_factory=FakeFilter, + chunker=FakeChunker(), + embedder=embedder, + vectorstore=vectorstore, + commit_resolver=lambda _: "abc123", + ) + + result = service.index_repository_sync("https://github.com/acme/demo", IndexingOptions(branch="main")) + + assert result.repo.status is RepositoryStatus.READY + assert result.files_processed == 1 + assert result.chunks_indexed == 1 + assert result.repo.last_commit == "abc123" + assert registry.get(result.repo.id).chunk_count == 1 + assert vectorstore.deleted_repos == [result.repo.id] + assert vectorstore.added[0].repo_id == result.repo.id + assert embedder.unloaded is True + + +def test_indexing_service_persists_error_status_on_failure(tmp_path): + class ExplodingLoader(FakeLoader): + def clone_repository(self, _repo_info: FakeRepoInfo, _branch: str) -> Path: + raise RuntimeError("clone failed") + + registry = RepositoryRegistry(tmp_path / "repositories.json") + service = IndexingService( + registry=registry, + validator=FakeValidator(), + loader=ExplodingLoader(tmp_path / "missing"), + file_filter_factory=FakeFilter, + chunker=FakeChunker(), + embedder=FakeEmbedder(), + vectorstore=FakeVectorStore(), + ) + + with pytest.raises(RuntimeError, match="clone failed"): + service.index_repository_sync("https://github.com/acme/demo") + + [repo] = registry.list() + assert repo.status is RepositoryStatus.ERROR + assert repo.error_message == "clone failed" + + +def test_retrieval_service_checks_repository_readiness_before_generation(tmp_path): + registry = RepositoryRegistry(tmp_path / "repositories.json") + repo = registry.add(Repository(id="repo-ready", url="https://github.com/acme/demo", status=RepositoryStatus.READY)) + generator = FakeGenerator() + service = RetrievalService(registry=registry, generator_factory=lambda: generator) + + result = service.query_code(repo.id, "How?", top_k=3) + + assert result.error is None + assert result.response.answer == "shared answer" + assert generator.queries[0].repo_id == repo.id + assert generator.queries[0].top_k == 3 + + +def test_retrieval_service_returns_typed_errors_for_missing_and_not_ready_repos(tmp_path): + registry = RepositoryRegistry(tmp_path / "repositories.json") + pending = registry.add(Repository(id="pending-repo", url="https://github.com/acme/demo", status=RepositoryStatus.INDEXING)) + service = RetrievalService(registry=registry, generator_factory=lambda: FakeGenerator()) + + missing = service.query_code("missing", "How?") + not_ready = service.query_code(pending.id, "How?") + assert missing.error == "Repository not found: missing" + assert not_ready.error == "Repository not ready: status is indexing" + + +def test_retrieval_service_blocks_generation_for_retrieval_only_profile(tmp_path): + registry = RepositoryRegistry(tmp_path / "repositories.json") + repo = registry.add(Repository(id="repo-ready", url="https://github.com/acme/demo", status=RepositoryStatus.READY)) + generator = FakeGenerator() + service = RetrievalService( + registry=registry, + generator_factory=lambda: generator, + provider_config=ProviderConfigService(ModelSettings(profile="retrieval-only", llm_provider="groq")), + ) + + result = service.query_code(repo.id, "How?") + + assert result.error == "LLM generation is disabled by the retrieval-only profile" + assert generator.queries == [] + + +def test_retrieval_service_blocks_private_profile_with_remote_provider(tmp_path): + registry = RepositoryRegistry(tmp_path / "repositories.json") + repo = registry.add(Repository(id="repo-ready", url="https://github.com/acme/demo", status=RepositoryStatus.READY)) + service = RetrievalService( + registry=registry, + generator_factory=lambda: FakeGenerator(), + provider_config=ProviderConfigService(ModelSettings(profile="private", llm_provider="groq")), + ) + + result = service.query_code(repo.id, "How?") + + assert result.error == "Private profile requires MODEL_LLM_PROVIDER=local" + + + +def test_retrieval_service_search_code_preserves_vector_results_and_filters(tmp_path): + registry = RepositoryRegistry(tmp_path / "repositories.json") + repo = registry.add(Repository(id="repo-ready", url="https://github.com/acme/demo", status=RepositoryStatus.READY)) + vectorstore = FakeVectorStore() + chunk = Chunk( + content="def hello(): pass", + repo_id=repo.id, + metadata=ChunkMetadata( + file_path="src/app.py", + start_line=1, + end_line=1, + chunk_type=ChunkType.FUNCTION, + language="python", + name="hello", + ), + ) + vectorstore.add_chunks([chunk]) + service = RetrievalService(registry=registry, embedder=FakeEmbedder(), vectorstore=vectorstore) + + results = service.search_code(repo.id, "hello", top_k=999, file_filter="src/*.py", chunk_type="function") + + assert results[0][0] == chunk + assert results[0][1] == 0.87 + assert vectorstore.queries[0] == {"repo_id": repo.id, "top_k": 20, "threshold": 0.3} + + +def test_provider_config_service_exposes_profiles_without_requiring_local_llm(): + cloud = ProviderConfigService(ModelSettings(profile="cloud", llm_provider="groq")) + private_remote = ProviderConfigService(ModelSettings(profile="private", llm_provider="groq")) + retrieval_only = ProviderConfigService(ModelSettings(profile="retrieval-only", llm_provider="groq")) + + assert cloud.profile == "cloud" + assert cloud.requires_generation_provider is True + assert cloud.requires_local_llm is False + assert cloud.uses_cloud_generation is True + assert private_remote.requires_local_llm is True + assert private_remote.generation_block_reason() == "Private profile requires MODEL_LLM_PROVIDER=local" + assert retrieval_only.profile == "retrieval-only" + assert retrieval_only.requires_generation_provider is False + + with pytest.raises(ValidationError): + ModelSettings(profile="invalid") + + +def test_provider_config_service_reads_documented_model_profile_from_env_file(tmp_path, monkeypatch): + (tmp_path / ".env").write_text("MODEL_PROFILE=retrieval-only\nMODEL_LLM_PROVIDER=groq\n") + monkeypatch.chdir(tmp_path) + config_module._settings = None + + try: + provider_config = ProviderConfigService() + + assert provider_config.profile == "retrieval-only" + assert provider_config.generation_block_reason() == "LLM generation is disabled by the retrieval-only profile" + finally: + config_module._settings = None diff --git a/tests/test_mcp_handlers.py b/tests/test_mcp_handlers.py index b8b3ea8..fdad52b 100644 --- a/tests/test_mcp_handlers.py +++ b/tests/test_mcp_handlers.py @@ -1,244 +1,289 @@ -"""Unit tests for MCP handlers.""" +"""Unit tests for service-backed MCP handlers.""" from datetime import datetime -from unittest.mock import AsyncMock, MagicMock, patch -import sys +from types import SimpleNamespace import pytest +from coderag.mcp.handlers import MCPHandlers, get_mcp_handlers from coderag.models.repository import Repository, RepositoryStatus +from coderag.services.registry import RepositoryRegistry + + +class FakeIndexingService: + def __init__(self, registry: RepositoryRegistry) -> None: + self.registry = registry + self.deleted: list[str] = [] + + async def index_repository(self, url, options): + repo = self.registry.add(Repository(id="indexed-repo", url=url, branch=options.branch or "main", status=RepositoryStatus.READY)) + return SimpleNamespace(repo=repo, files_processed=2, chunks_indexed=3) + + def delete_repository(self, repo_id: str): + repo = self.registry.remove(repo_id) + if repo is None: + return None + self.deleted.append(repo.id) + return SimpleNamespace(repo=repo, chunks_deleted=3) + + def update_repository(self, repo_id: str): + repo = self.registry.get(repo_id) + if repo is None: + raise ValueError(f"Repository not found: {repo_id}") + if not repo.last_commit: + raise ValueError("No previous indexing found. Please re-index the full repository.") + return SimpleNamespace( + already_up_to_date=True, + files_changed=0, + files_added=0, + files_modified=0, + files_deleted=0, + chunks_added=0, + chunks_deleted=0, + total_chunks=repo.chunk_count, + ) + + def get_indexed_files(self, repo_id: str): + _ = repo_id + return {"src/app.py"} + + +class FakeRetrievalService: + def __init__(self, registry: RepositoryRegistry) -> None: + self.registry = registry + + def query_code(self, repo_id: str, question: str, top_k: int = 5): + _ = (question, top_k) + repo = self.registry.get(repo_id) + if repo is None: + return SimpleNamespace(response=None, repo=None, error=f"Repository not found: {repo_id}") + if repo.status is not RepositoryStatus.READY: + return SimpleNamespace(response=None, repo=repo, error=f"Repository not ready: status is {repo.status.value}") + response = SimpleNamespace(answer="Test answer", citations=[], retrieved_chunks=[], grounded=False) + return SimpleNamespace(response=response, repo=repo, error=None) + + def search_code(self, repo_id: str, query: str, top_k: int = 10, file_filter=None, chunk_type=None): + _ = (query, top_k, file_filter, chunk_type) + repo = self.registry.get(repo_id) + if repo is None: + raise ValueError(f"Repository not found: {repo_id}") + return [ + ( + SimpleNamespace( + file_path="src/app.py", + start_line=1, + end_line=3, + chunk_type="function", + content="def app(): pass", + ), + 0.876, + ) + ] @pytest.fixture -def mock_settings(): - """Mock settings for testing.""" - settings = MagicMock() - settings.data_dir = MagicMock() - settings.data_dir.__truediv__ = MagicMock(return_value=MagicMock()) - settings.ingestion.batch_size = 10 - settings.ingestion.chunk_size = 1000 - return settings +def registry(tmp_path): + return RepositoryRegistry(tmp_path / "repositories.json") @pytest.fixture def sample_repository(): - """Create a sample repository for testing.""" return Repository( - id="test-repo-id-12345", + id="test-repo-123456", url="https://github.com/owner/repo", branch="main", status=RepositoryStatus.READY, - chunk_count=100, - indexed_at=datetime.now(), - last_commit="abc123", + chunk_count=42, + indexed_at=datetime(2024, 1, 1, 12, 0, 0), ) @pytest.fixture -def mock_mcp_handlers(): - """Create a mock MCPHandlers instance with all dependencies mocked.""" - with patch.dict(sys.modules, {}): - # Mock all the heavy dependencies - with patch("coderag.config.get_settings") as mock_get_settings, \ - patch("coderag.mcp.handlers.GitHubURLValidator"), \ - patch("coderag.mcp.handlers.RepositoryLoader"), \ - patch("coderag.mcp.handlers.FileFilter"), \ - patch("coderag.mcp.handlers.CodeChunker"), \ - patch("coderag.mcp.handlers.EmbeddingGenerator"), \ - patch("coderag.mcp.handlers.VectorStore"): - - mock_settings = MagicMock() - mock_settings.data_dir = MagicMock() - mock_file = MagicMock() - mock_file.exists.return_value = False - mock_file.parent = MagicMock() - mock_settings.data_dir.__truediv__ = MagicMock(return_value=mock_file) - mock_settings.ingestion.batch_size = 10 - mock_get_settings.return_value = mock_settings - - from coderag.mcp.handlers import MCPHandlers - handlers = MCPHandlers() - yield handlers +def mcp_handlers(registry): + return MCPHandlers( + registry=registry, + indexing_service=FakeIndexingService(registry), + retrieval_service=FakeRetrievalService(registry), + ) class TestMCPHandlersSingleton: - """Tests for MCPHandlers singleton pattern.""" - def test_get_mcp_handlers_returns_same_instance(self): - """Test that get_mcp_handlers returns the same instance.""" - with patch("coderag.config.get_settings") as mock_get_settings, \ - patch("coderag.mcp.handlers.GitHubURLValidator"), \ - patch("coderag.mcp.handlers.RepositoryLoader"), \ - patch("coderag.mcp.handlers.FileFilter"), \ - patch("coderag.mcp.handlers.CodeChunker"), \ - patch("coderag.mcp.handlers.EmbeddingGenerator"), \ - patch("coderag.mcp.handlers.VectorStore"): - - mock_settings = MagicMock() - mock_settings.data_dir = MagicMock() - mock_file = MagicMock() - mock_file.exists.return_value = False - mock_settings.data_dir.__truediv__ = MagicMock(return_value=mock_file) - mock_get_settings.return_value = mock_settings - - # Reset singleton - import coderag.mcp.handlers as handlers_module - handlers_module._mcp_handlers = None + import coderag.mcp.handlers as handlers_module - from coderag.mcp.handlers import get_mcp_handlers + handlers_module._mcp_handlers = None + handler1 = get_mcp_handlers() + handler2 = get_mcp_handlers() - handler1 = get_mcp_handlers() - handler2 = get_mcp_handlers() - - assert handler1 is handler2 + assert handler1 is handler2 class TestListRepositories: - """Tests for list_repositories handler.""" - @pytest.mark.asyncio - async def test_list_repositories_empty(self, mock_mcp_handlers): - """Test listing repositories when none exist.""" - result = await mock_mcp_handlers.list_repositories() + async def test_list_repositories_empty(self, mcp_handlers): + result = await mcp_handlers.list_repositories() assert result["count"] == 0 assert result["repositories"] == [] @pytest.mark.asyncio - async def test_list_repositories_with_repos(self, mock_mcp_handlers, sample_repository): - """Test listing repositories when some exist.""" - mock_mcp_handlers.repositories = {sample_repository.id: sample_repository} - # Prevent reload from disk - mock_mcp_handlers._reload_repositories = MagicMock() + async def test_list_repositories_with_repos(self, mcp_handlers, registry, sample_repository): + registry.add(sample_repository) - result = await mock_mcp_handlers.list_repositories() + result = await mcp_handlers.list_repositories() assert result["count"] == 1 - assert len(result["repositories"]) == 1 assert result["repositories"][0]["id"] == sample_repository.id assert result["repositories"][0]["name"] == sample_repository.full_name class TestGetRepositoryInfo: - """Tests for get_repository_info handler.""" - @pytest.mark.asyncio - async def test_get_repository_info_found(self, mock_mcp_handlers, sample_repository): - """Test getting info for an existing repository.""" - mock_mcp_handlers.repositories = {sample_repository.id: sample_repository} - mock_mcp_handlers._reload_repositories = MagicMock() - mock_mcp_handlers.vectorstore.get_repo_files.return_value = ["file1.py", "file2.py"] + async def test_get_repository_info_found(self, mcp_handlers, registry, sample_repository): + registry.add(sample_repository) - result = await mock_mcp_handlers.get_repository_info(sample_repository.id) + result = await mcp_handlers.get_repository_info(sample_repository.id) assert result["id"] == sample_repository.id - assert result["name"] == sample_repository.name - assert result["url"] == sample_repository.url - assert "indexed_files" in result + assert result["full_name"] == sample_repository.full_name + assert result["indexed_files"] == ["src/app.py"] @pytest.mark.asyncio - async def test_get_repository_info_partial_id(self, mock_mcp_handlers, sample_repository): - """Test getting info using partial repository ID.""" - mock_mcp_handlers.repositories = {sample_repository.id: sample_repository} - mock_mcp_handlers._reload_repositories = MagicMock() - mock_mcp_handlers.vectorstore.get_repo_files.return_value = [] + async def test_get_repository_info_partial_id(self, mcp_handlers, registry, sample_repository): + registry.add(sample_repository) - # Use first 8 characters - result = await mock_mcp_handlers.get_repository_info("test-rep") + result = await mcp_handlers.get_repository_info("test-repo") assert result["id"] == sample_repository.id @pytest.mark.asyncio - async def test_get_repository_info_not_found(self, mock_mcp_handlers): - """Test getting info for non-existent repository.""" - result = await mock_mcp_handlers.get_repository_info("nonexistent") + async def test_get_repository_info_not_found(self, mcp_handlers): + result = await mcp_handlers.get_repository_info("missing") assert "error" in result class TestDeleteRepository: - """Tests for delete_repository handler.""" - @pytest.mark.asyncio - async def test_delete_repository_success(self, mock_mcp_handlers, sample_repository): - """Test successfully deleting a repository.""" - mock_mcp_handlers.repositories = {sample_repository.id: sample_repository} - mock_mcp_handlers._reload_repositories = MagicMock() - mock_mcp_handlers._save_repositories = MagicMock() - mock_mcp_handlers.vectorstore.get_repo_chunk_count.return_value = 100 - mock_mcp_handlers.vectorstore.delete_repo_chunks = MagicMock() + async def test_delete_repository_success(self, mcp_handlers, registry, sample_repository): + registry.add(sample_repository) - result = await mock_mcp_handlers.delete_repository(sample_repository.id) + result = await mcp_handlers.delete_repository(sample_repository.id) assert result["success"] is True - assert result["chunks_deleted"] == 100 - assert sample_repository.id not in mock_mcp_handlers.repositories + assert result["chunks_deleted"] == 3 + assert registry.get(sample_repository.id) is None @pytest.mark.asyncio - async def test_delete_repository_not_found(self, mock_mcp_handlers): - """Test deleting non-existent repository.""" - result = await mock_mcp_handlers.delete_repository("nonexistent") + async def test_delete_repository_not_found(self, mcp_handlers): + result = await mcp_handlers.delete_repository("missing") assert result["success"] is False - assert "error" in result + assert "not found" in result["error"] -class TestQueryCode: - """Tests for query_code handler.""" +class TestIndexRepository: + @pytest.mark.asyncio + async def test_index_repository_success_shape(self, mcp_handlers): + result = await mcp_handlers.index_repository("https://github.com/owner/repo", branch="main") + assert result == { + "success": True, + "repo_id": "indexed-repo", + "name": "owner/repo", + "branch": "main", + "files_processed": 2, + "chunks_indexed": 3, + } + + +class TestQueryCode: @pytest.mark.asyncio - async def test_query_code_repo_not_found(self, mock_mcp_handlers): - """Test querying non-existent repository.""" - result = await mock_mcp_handlers.query_code("nonexistent", "What does this do?") + async def test_query_code_repo_not_found(self, mcp_handlers): + result = await mcp_handlers.query_code("missing", "What does this do?") assert result["grounded"] is False assert result["answer"] == "" assert "error" in result @pytest.mark.asyncio - async def test_query_code_repo_not_ready(self, mock_mcp_handlers, sample_repository): - """Test querying repository that's not ready.""" + async def test_query_code_repo_not_ready(self, mcp_handlers, registry, sample_repository): sample_repository.status = RepositoryStatus.INDEXING - mock_mcp_handlers.repositories = {sample_repository.id: sample_repository} - mock_mcp_handlers._reload_repositories = MagicMock() + registry.add(sample_repository) - result = await mock_mcp_handlers.query_code(sample_repository.id, "What does this do?") + result = await mcp_handlers.query_code(sample_repository.id, "What does this do?") assert result["grounded"] is False assert "not ready" in result["error"] + @pytest.mark.asyncio + async def test_query_code_success_shape(self, mcp_handlers, registry, sample_repository): + registry.add(sample_repository) + + result = await mcp_handlers.query_code(sample_repository.id, "What does this do?", top_k=3) + + assert result["answer"] == "Test answer" + assert result["citations"] == [] + assert result["evidence"] == [] + assert result["grounded"] is False -class TestSearchCode: - """Tests for search_code handler.""" + +class TestSearchCode: @pytest.mark.asyncio - async def test_search_code_repo_not_found(self, mock_mcp_handlers): - """Test searching non-existent repository.""" - result = await mock_mcp_handlers.search_code("nonexistent", "function") + async def test_search_code_repo_not_found(self, mcp_handlers): + result = await mcp_handlers.search_code("missing", "function") assert result["results"] == [] assert "error" in result + @pytest.mark.asyncio + async def test_search_code_success_shape(self, mcp_handlers, registry, sample_repository): + registry.add(sample_repository) -class TestUpdateRepository: - """Tests for update_repository handler.""" + result = await mcp_handlers.search_code(sample_repository.id, "function", top_k=1) + assert result["count"] == 1 + assert result["results"][0] == { + "file_path": "src/app.py", + "start_line": 1, + "end_line": 3, + "chunk_type": "function", + "content": "def app(): pass", + "relevance_score": 0.876, + } + + + +class TestUpdateRepository: @pytest.mark.asyncio - async def test_update_repository_not_found(self, mock_mcp_handlers): - """Test updating non-existent repository.""" - result = await mock_mcp_handlers.update_repository("nonexistent") + async def test_update_repository_not_found(self, mcp_handlers): + result = await mcp_handlers.update_repository("missing") assert result["success"] is False assert "not found" in result["error"] @pytest.mark.asyncio - async def test_update_repository_no_previous_commit(self, mock_mcp_handlers, sample_repository): - """Test updating repository without previous indexing.""" + async def test_update_repository_no_previous_commit(self, mcp_handlers, registry, sample_repository): sample_repository.last_commit = None - mock_mcp_handlers.repositories = {sample_repository.id: sample_repository} - mock_mcp_handlers._reload_repositories = MagicMock() + registry.add(sample_repository) - result = await mock_mcp_handlers.update_repository(sample_repository.id) + result = await mcp_handlers.update_repository(sample_repository.id) assert result["success"] is False assert "No previous indexing" in result["error"] + + @pytest.mark.asyncio + async def test_update_repository_already_up_to_date_shape(self, mcp_handlers, registry, sample_repository): + sample_repository.last_commit = "abc123" + registry.add(sample_repository) + + result = await mcp_handlers.update_repository(sample_repository.id) + + assert result == { + "success": True, + "message": "Repository is already up to date", + "files_changed": 0, + "chunks_added": 0, + "chunks_deleted": 0, + } diff --git a/tests/test_mcp_tools.py b/tests/test_mcp_tools.py index 8aef5a0..e8e9d62 100644 --- a/tests/test_mcp_tools.py +++ b/tests/test_mcp_tools.py @@ -10,14 +10,7 @@ class TestToolRegistration: def test_tools_are_registered(self): """Test that all tools are registered with the MCP server.""" - with patch("coderag.config.get_settings") as mock_get_settings, \ - patch("coderag.mcp.handlers.GitHubURLValidator"), \ - patch("coderag.mcp.handlers.RepositoryLoader"), \ - patch("coderag.mcp.handlers.FileFilter"), \ - patch("coderag.mcp.handlers.CodeChunker"), \ - patch("coderag.mcp.handlers.EmbeddingGenerator"), \ - patch("coderag.mcp.handlers.VectorStore"): - + with patch("coderag.config.get_settings") as mock_get_settings: mock_settings = MagicMock() mock_settings.data_dir = MagicMock() mock_file = MagicMock() diff --git a/tests/test_ui_handlers.py b/tests/test_ui_handlers.py new file mode 100644 index 0000000..0074069 --- /dev/null +++ b/tests/test_ui_handlers.py @@ -0,0 +1,64 @@ +"""UI privacy/profile guard tests.""" + +import ast +import importlib.util +import json +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +HANDLERS_PATH = REPO_ROOT / "src" / "coderag" / "ui" / "handlers.py" +METADATA_PATH = REPO_ROOT / "src" / "coderag" / "ui" / "repository_metadata.py" + + +def _load_metadata_module(): + spec = importlib.util.spec_from_file_location("coderag_ui_repository_metadata_under_test", METADATA_PATH) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +def _call_name(node: ast.AST) -> str | None: + if not isinstance(node, ast.Call): + return None + func = node.func + if isinstance(func, ast.Name): + return func.id + if isinstance(func, ast.Attribute): + return func.attr + return None + + +def _function(module: ast.Module, name: str) -> ast.FunctionDef: + return next( + node for node in ast.walk(module) if isinstance(node, ast.FunctionDef) and node.name == name + ) + + +def test_ui_ask_question_checks_provider_profile_before_generation_setup(): + module = ast.parse(HANDLERS_PATH.read_text()) + ask_question = _function(module, "ask_question") + + call_names = [name for node in ast.walk(ask_question) if (name := _call_name(node)) is not None] + + assert "generation_block_reason" in call_names + assert "ResponseGenerator" in call_names + assert "generate" in call_names + assert call_names.index("generation_block_reason") < call_names.index("ResponseGenerator") + assert call_names.index("generation_block_reason") < call_names.index("generate") + + +def test_ui_repository_loader_fails_closed_on_corrupt_metadata(tmp_path): + metadata = _load_metadata_module() + repos_file = tmp_path / "repositories.json" + repos_file.write_text("{not-json") + + try: + metadata.load_repositories(repos_file) + except json.JSONDecodeError: + pass + else: + raise AssertionError("corrupt repositories.json must not load as an empty registry") + + assert repos_file.read_text() == "{not-json"