diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3176a73..e1d355a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,17 +25,17 @@ jobs: python -m pip install -e . --no-deps python -m pip install pytest pytest-asyncio pydantic pydantic-settings click PyYAML fastapi structlog ruff mypy mcp httpx - - name: Run Milestone 0-3 tests - run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py + - name: Run Milestone 0-4 tests + run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py tests/test_m4_hybrid_retrieval.py - - name: Run Milestone 0-3 lint baseline - run: ruff check src/coderag/generation src/coderag/models src/coderag/services src/coderag/config.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py + - name: Run Milestone 0-4 lint baseline + run: ruff check src/coderag/generation src/coderag/indexing/vectorstore.py src/coderag/models src/coderag/retrieval src/coderag/services src/coderag/config.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/cli.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py tests/test_m4_hybrid_retrieval.py scripts/evalfly_smoke.py - - name: Run Milestone 0-3 type baseline - run: PYTHONPATH=src mypy src/coderag/generation src/coderag/models src/coderag/services src/coderag/config.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py scripts/evalfly_smoke.py + - name: Run Milestone 0-4 type baseline + run: PYTHONPATH=src mypy src/coderag/generation src/coderag/indexing/vectorstore.py src/coderag/models src/coderag/retrieval src/coderag/services src/coderag/config.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py tests/test_m4_hybrid_retrieval.py scripts/evalfly_smoke.py - name: Compile changed runtime modules - run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/*.py src/coderag/models/*.py src/coderag/services/*.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py + run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/indexing/vectorstore.py src/coderag/generation/*.py src/coderag/models/*.py src/coderag/retrieval/*.py src/coderag/services/*.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_ui_handlers.py tests/test_m4_hybrid_retrieval.py scripts/evalfly_smoke.py - name: Build package run: python -m build diff --git a/.pi/.specsafe-state.json b/.pi/.specsafe-state.json index a945447..df81ffa 100644 --- a/.pi/.specsafe-state.json +++ b/.pi/.specsafe-state.json @@ -120,6 +120,26 @@ "turns": 0 } } + }, + { + "sliceId": "CRAG-20260630-M4-HYBRID-CONTEXT-PACKS", + "workspaceId": "CodeRAG", + "sessionId": "omp-session", + "beganAt": "2026-06-30T20:39:16.458Z", + "endedAt": "2026-06-30T20:58:56.662Z", + "outcome": "PASS", + "costSummary": { + "externalMemoryCalls": 0, + "externalMemoryCost": 0, + "subagentTokens": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "cost": 0, + "turns": 0 + } + } } ] } diff --git a/docs/superpowers/plans/2026-06-30-m4-hybrid-retrieval-context-packs-contract.md b/docs/superpowers/plans/2026-06-30-m4-hybrid-retrieval-context-packs-contract.md new file mode 100644 index 0000000..40a88a3 --- /dev/null +++ b/docs/superpowers/plans/2026-06-30-m4-hybrid-retrieval-context-packs-contract.md @@ -0,0 +1,135 @@ +# Milestone 4 Contract: Hybrid Retrieval and Context Packs + +**Slice:** `CRAG-20260630-M4-HYBRID-CONTEXT-PACKS` + +**Branch:** `feat/m4-hybrid-retrieval-context-packs` + +**Goal:** Modernize CodeRAG beyond vector-only retrieval by adding explainable hybrid retrieval and structured context packs that work without LLM generation. + +## Scope + +- Add local lexical retrieval over indexed chunks. +- Preserve existing semantic vector retrieval and query behavior. +- Merge vector + lexical candidates with score components. +- Add path/name metadata boosting. +- Deduplicate same-file overlapping ranges. +- Add deterministic token/context budgeting. +- Add `ContextPack` schema and shared service method. +- Expose context packs via MCP, REST, and CLI. +- Extend deterministic tests, EvalFly smoke checks, and CI M4 gates. + +## Non-goals + +- No code graph MVP in M4. +- No autonomous editing behavior. +- No new model/provider abstraction beyond using existing embeddings/vector store. +- No production-ready claim. +- No broad README rewrite beyond later M6. + +## Required Interfaces + +```python +@dataclass +class ScoreBreakdown: + vector: float | None + lexical: float | None + path: float + name: float + combined: float + +@dataclass +class RetrievalBudget: + max_chunks: int + max_tokens: int + max_chunks_per_file: int + +class HybridRetriever: + def retrieve( + self, + query: str, + repo_id: str, + top_k: int, + budget: RetrievalBudget | None = None, + ) -> list[RetrievedChunk]: ... +``` + +`RetrievedChunk` may be extended with optional metadata fields for score components, token estimate, and retrieval sources without breaking existing callers. + +```python +class RetrievalService: + def search_hybrid(self, repo_id: str, query: str, top_k: int = 10) -> list[RetrievedChunk]: ... + def get_context_pack(self, repo_id: str, query: str, top_k: int = 8, max_tokens: int = 4000) -> ContextPack: ... +``` + +## Acceptance Criteria + +### AC-M4.1 Hybrid retrieval + +**Pass requires** +- Semantic vector candidates remain included when embeddings/vectorstore are available. +- Lexical candidates are computed locally from indexed chunk text. +- Combined score includes vector, lexical, path, name, and combined components. +- Ranking order is explainable from score components. + +**Fail if** +- Retrieval remains vector-only. +- Lexical-only exact matches cannot surface without semantic score. +- Results have no score explanation. + +### AC-M4.2 Dedupe and budget + +**Pass requires** +- Same-file overlapping line ranges are merged/deduped. +- Budget enforces max chunks, max estimated tokens, and max chunks per file deterministically. +- Top relevant result is preserved when within budget. + +**Fail if** +- Duplicate vector+lexical hits consume multiple slots. +- Budget fields are ignored. + +### AC-M4.3 Context packs + +**Pass requires** +- Context packs are generated without invoking an LLM. +- Pack includes query, repo id, snippets, citation candidates, score/ranking reasons, token estimate, budget, and capability metadata. +- MCP, REST, and CLI expose the pack as structured JSON; CLI also exposes markdown. + +**Fail if** +- Context pack generation requires cloud or local generation. +- Pack lacks provenance for why files/snippets were selected. + +### AC-M4.4 Compatibility + +**Pass requires** +- Existing `query_code` behavior still returns answer/citations/evidence/grounded. +- Existing `search_code` compatibility is preserved or a migration test proves the shape remains usable. +- API/MCP/CLI tests cover new outputs. + +**Fail if** +- Existing M0-M3 tests are weakened or removed. +- MCP/CLI/API disagree on context-pack result shape. + +## Required Verification + +Run before closing the slice: + +```bash +bun run /Users/sebastian/.omp/omp-pantheon/skills/specsafe/bin/specsafe.ts status +PYTHONPATH=src .venv/bin/python -m pytest tests/test_m4_hybrid_retrieval.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_m3_citations.py +.venv/bin/python -m ruff check src/coderag/retrieval src/coderag/models src/coderag/services src/coderag/indexing/vectorstore.py src/coderag/api/routes.py src/coderag/api/schemas.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/cli.py tests/test_m4_hybrid_retrieval.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_m3_citations.py scripts/evalfly_smoke.py +PYTHONPATH=src .venv/bin/mypy src/coderag/retrieval src/coderag/models src/coderag/services src/coderag/indexing/vectorstore.py src/coderag/api/routes.py src/coderag/api/schemas.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py tests/test_m4_hybrid_retrieval.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_m3_citations.py scripts/evalfly_smoke.py +.venv/bin/python scripts/evalfly_smoke.py --suite smoke +.venv/bin/python -m py_compile src/coderag/retrieval/*.py src/coderag/models/*.py src/coderag/services/*.py src/coderag/indexing/vectorstore.py src/coderag/api/routes.py src/coderag/api/schemas.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/cli.py tests/test_m4_hybrid_retrieval.py +.venv/bin/python -m build +``` + +Then run the local CI suite with M0-M4 tests. + +## Review Gate + +A reviewer must approve that: +- hybrid retrieval is not vector-only; +- context packs do not call generation; +- score/budget/dedupe tests are behavior-level; +- public adapters remain JSON-safe; +- M4 does not implement graph/autonomous behavior. diff --git a/docs/superpowers/plans/2026-06-30-m4-m7-execution-plan.md b/docs/superpowers/plans/2026-06-30-m4-m7-execution-plan.md new file mode 100644 index 0000000..a273b9c --- /dev/null +++ b/docs/superpowers/plans/2026-06-30-m4-m7-execution-plan.md @@ -0,0 +1,166 @@ +# CodeRAG M4-M7 Execution Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development or superpowers:executing-plans task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Complete CodeRAG Milestones 4 through 7 without changing the product into an autonomous coding agent. + +**Architecture:** Keep API, MCP, CLI, and UI as adapters over shared services. Add hybrid retrieval and context packs first, then graph metadata, then stabilize public interfaces/docs, then prove release readiness with deterministic evidence. + +**Tech Stack:** Python 3.11+, FastAPI, Click, MCP/FastMCP, SQLite registry, ChromaDB behind the existing vector-store boundary, Tree-sitter/Python AST where available, pytest, ruff, mypy, EvalFly, GitHub Actions. + +## Global Constraints + +- CodeRAG remains a RAG/code-intelligence product; it must not edit files, create commits for users, open PRs for users, or run autonomous issue-solving loops. +- Default quickstart must not require a local generative LLM. +- Retrieval-only flows must not require cloud or local LLM credentials. +- Any cloud generation path must remain profile-gated and documented; no blanket local-first privacy claim. +- No production-ready claim is allowed without fresh tests, EvalFly evidence, branch E2E/run records where feasible, GitHub CI, and reviewer validation. +- Existing public query behavior remains compatible unless a migration is documented and tested. +- Each milestone uses its own branch, SpecSafe slice, contract file, TDD implementation, multiple atomic commits, guarded push per commit, PR, CI watch, review, merge, and local main sync. + +--- + +## Milestone 4: Hybrid Retrieval and Context Packs + +**Branch:** `feat/m4-hybrid-retrieval-context-packs` + +**Slice:** `CRAG-20260630-M4-HYBRID-CONTEXT-PACKS` + +**Goal:** Replace vector-only retrieval with explainable hybrid retrieval and add no-generation context packs. + +**Files:** +- Create: `src/coderag/retrieval/hybrid.py` +- Create: `src/coderag/models/context.py` +- Modify: `src/coderag/retrieval/retriever.py` +- Modify: `src/coderag/services/retrieval.py` +- Modify: `src/coderag/indexing/vectorstore.py` +- Modify: `src/coderag/api/schemas.py` +- Modify: `src/coderag/api/routes.py` +- Modify: `src/coderag/mcp/handlers.py` +- Modify: `src/coderag/mcp/tools.py` +- Modify: `src/coderag/cli.py` +- Test: `tests/test_m4_hybrid_retrieval.py` +- Test: `tests/test_api_routes.py` +- Test: `tests/test_mcp_handlers.py` +- Test: `tests/test_mcp_tools.py` +- Test: `tests/test_m3_citations.py` only if query result shapes need CLI coverage reuse. +- Modify: `evals/config.json` +- Modify: `.github/workflows/ci.yml` + +**Interfaces:** +- `VectorStore.get_repo_chunks(repo_id: str) -> list[Chunk]`: returns indexed chunks with content from Chroma documents/metadatas. +- `HybridRetriever.retrieve(query: str, repo_id: str, top_k: int, max_tokens: int | None = None) -> list[RetrievedChunk]`: returns deduped, scored, explained chunks. +- `RetrievalService.get_context_pack(repo_id: str, query: str, top_k: int = 8, max_tokens: int = 4000) -> ContextPack`. +- MCP tool: `get_context_pack(repo_id, query, top_k=8, max_tokens=4000)`. +- CLI command: `coderag context-pack --format json|markdown`. + +**Acceptance:** +- Hybrid retrieval combines vector candidates and local lexical candidates. +- Each retrieved chunk exposes score components and `retrieval_sources` through metadata-compatible fields. +- Dedupe merges overlapping same-file ranges. +- Context pack generation invokes no LLM. +- Context pack includes query, repository id, files/snippets, citation candidates, ranking reasons, token estimate, budget, and profile/capability metadata. +- EvalFly includes retrieval/context-pack smoke assertions. + +--- + +## Milestone 5: Code Intelligence Graph MVP + +**Branch:** `feat/m5-code-intelligence-graph` + +**Slice:** `CRAG-20260630-M5-CODE-GRAPH` + +**Goal:** Add deterministic Python and TS/JS symbol graph with blast-radius APIs. + +**Files:** +- Create: `src/coderag/graph/models.py` +- Create: `src/coderag/graph/extractor.py` +- Create: `src/coderag/services/graph.py` +- Modify: `src/coderag/services/registry.py` for graph tables or cached graph records. +- Modify: `src/coderag/api/schemas.py` +- Modify: `src/coderag/api/routes.py` +- Modify: `src/coderag/mcp/handlers.py` +- Modify: `src/coderag/mcp/tools.py` +- Modify: `src/coderag/cli.py` +- Test: `tests/test_m5_graph.py` +- Test: API/MCP/CLI tests for `find_symbol`, `find_references`, `get_blast_radius`. +- Modify: `evals/config.json` +- Modify: `.github/workflows/ci.yml` + +**Interfaces:** +- `GraphService.build_graph(repo_id: str) -> CodeGraph`. +- `GraphService.find_symbol(repo_id: str, name: str) -> list[SymbolRecord]`. +- `GraphService.find_references(repo_id: str, name: str) -> list[ReferenceRecord]`. +- `GraphService.get_blast_radius(repo_id: str, path: str | None = None, symbol: str | None = None) -> BlastRadius`. + +**Acceptance:** +- Python fixture extracts modules, classes, functions, methods, imports, and test associations. +- TS/JS fixture extracts imports, exported functions/classes, and references with deterministic regex/Tree-sitter-safe fallback. +- Unsupported languages return explicit capability fallback, not fake graph confidence. +- Incremental cache uses file content hash. +- CLI/API/MCP expose graph operations with structured JSON. + +--- + +## Milestone 6: Production Interfaces and Docs + +**Branch:** `feat/m6-production-interfaces-docs` + +**Slice:** `CRAG-20260630-M6-PRODUCTION-INTERFACES` + +**Goal:** Stabilize user-facing contracts and align docs with tested behavior. + +**Files:** +- Modify: `src/coderag/api/schemas.py` +- Modify: `src/coderag/api/routes.py` +- Modify: `src/coderag/mcp/tools.py` +- Modify: `src/coderag/mcp/handlers.py` +- Modify: `src/coderag/cli.py` +- Modify: `README.md` +- Create/modify docs under existing docs path only if needed: privacy/profile matrix, migration notes, troubleshooting. +- Test: contract snapshot tests for CLI/API/MCP representative outputs. +- Modify: `evals/config.json` +- Modify: `.github/workflows/ci.yml` + +**Acceptance:** +- MCP tools include index/update/list/query/search_hybrid/get_context_pack/find_symbol/find_references/get_blast_radius/verify_citations where implemented. +- CLI exposes equivalent useful commands. +- REST schemas cover outputs and error shapes. +- README positions CodeRAG as verified codebase context engine, not autonomous agent. +- Docs do not claim untested production readiness or blanket privacy. + +--- + +## Milestone 7: Production Release Candidate + +**Branch:** `feat/m7-production-release-candidate` + +**Slice:** `CRAG-20260630-M7-PRODUCTION-RC` + +**Goal:** Prove the product works end to end and only then mark release-candidate readiness. + +**Files:** +- Modify: `evals/config.json` +- Create: fresh `evals/runs/.json` and `evals/reports/.md` if the existing EvalFly harness writes them. +- Create: `docs/superpowers/plans/2026-06-30-m7-release-run-record.md`. +- Modify: `README.md` only for evidence badges/claims that are actually supported. +- Modify CI if a release gate is missing. + +**Acceptance:** +- Clean install path is exercised from fresh venv or equivalent isolated environment. +- CI-equivalent local transcript passes. +- EvalFly smoke, retrieval, citations, graph, interfaces/security smoke cases pass or unsupported cases are explicitly excluded with evidence and user-visible wording. +- Branch E2E/run record covers CLI/API/MCP flows; UI flow is included only if Gradio dependencies are installable in the environment. +- Final reviewer validates AC-0 through AC-10. +- Release checklist in roadmap is all PASS or explicitly out of release scope. + +--- + +## Execution Rules + +- Use TDD for behavior changes: red test, expected failure, green implementation, full affected gate. +- Do not weaken existing M0-M3 tests or CI gates. +- Do not add autonomous editing behavior. +- Commit messages use English semantic style observed in repo history. +- Every commit includes `Spec-Slice: ` trailer and is pushed immediately with the guarded push wrapper. +- Remote PR/merge operations use the guarded GitHub wrapper with preview before `--i-approve`. diff --git a/evals/config.json b/evals/config.json index 1df9645..0b92946 100644 --- a/evals/config.json +++ b/evals/config.json @@ -506,6 +506,138 @@ } ] } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m4-contract-exists", + "title": "Milestone 4 hybrid retrieval contract exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "delivery_governance", + "source": { + "kind": "acceptance_criteria", + "criteria": "M4-AC1 M4-AC2 M4-AC3" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "The Milestone 4 contract is committed for hybrid retrieval and context packs." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "docs/superpowers/plans/2026-06-30-m4-hybrid-retrieval-context-packs-contract.md" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m4-hybrid-retriever-exists", + "title": "Hybrid retriever implementation exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "retrieval_quality", + "source": { + "kind": "acceptance_criteria", + "criteria": "M4-AC1 M4-AC2" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "HybridRetriever is implemented as the retrieval merge seam." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_contains", + "path": "src/coderag/retrieval/hybrid.py", + "text": "class HybridRetriever" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m4-context-pack-api-exists", + "title": "Context pack adapters exist", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "adapter_contract", + "source": { + "kind": "acceptance_criteria", + "criteria": "M4-AC3" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "REST and MCP expose a context pack path without generation." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_contains", + "path": "src/coderag/api/routes.py", + "text": "@router.post(\"/context-pack\"" + }, + { + "type": "file_contains", + "path": "src/coderag/mcp/tools.py", + "text": "async def get_context_pack" + }, + { + "type": "file_contains", + "path": "src/coderag/cli.py", + "text": "@cli.command(\"context-pack\")" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m4-targeted-tests-exist", + "title": "Milestone 4 targeted tests exist", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "test_coverage", + "source": { + "kind": "acceptance_criteria", + "criteria": "M4-AC1 M4-AC2 M4-AC3" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "Targeted hybrid retrieval and context pack tests are committed." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "tests/test_m4_hybrid_retrieval.py" + } + ] + } } ] } diff --git a/src/coderag/api/routes.py b/src/coderag/api/routes.py index bf7cf0c..9709789 100644 --- a/src/coderag/api/routes.py +++ b/src/coderag/api/routes.py @@ -7,6 +7,9 @@ from coderag.api.schemas import ( CitationResponse, CitationVerificationResponse, + ContextPackRequest, + ContextPackResponse, + ContextSnippetResponse, IndexRepositoryRequest, IndexRepositoryResponse, ListRepositoriesResponse, @@ -154,6 +157,10 @@ async def query_repository(request: QueryRequest) -> QueryResponse: chunk_type=c.chunk_type, name=c.name, content=c.content, + score_breakdown=c.score_breakdown, + retrieval_sources=c.retrieval_sources, + token_estimate=c.token_estimate, + ranking_reason=c.ranking_reason, ) for c in response.retrieved_chunks ], @@ -162,6 +169,47 @@ async def query_repository(request: QueryRequest) -> QueryResponse: ) +@router.post("/context-pack", response_model=ContextPackResponse) +async def context_pack(request: ContextPackRequest) -> ContextPackResponse: + """Build a hybrid retrieval context pack without LLM generation.""" + repo = get_repo_or_404(request.repo_id) + try: + pack = retrieval_service.get_context_pack( + repo.id, + request.query, + top_k=request.top_k, + max_tokens=request.max_tokens, + max_chunks_per_file=request.max_chunks_per_file, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return ContextPackResponse( + repo_id=pack.repo_id, + query=pack.query, + snippets=[ + ContextSnippetResponse( + chunk_id=snippet.chunk_id, + file_path=snippet.file_path, + start_line=snippet.start_line, + end_line=snippet.end_line, + citation=snippet.citation, + chunk_type=snippet.chunk_type, + name=snippet.name, + content=snippet.content, + relevance_score=snippet.relevance_score, + score_breakdown=snippet.score_breakdown, + retrieval_sources=snippet.retrieval_sources, + token_estimate=snippet.token_estimate, + ranking_reason=snippet.ranking_reason, + ) + for snippet in pack.snippets + ], + token_estimate=pack.token_estimate, + budget=pack.budget, + capabilities=pack.capabilities, + ) + + @router.get("/repos", response_model=ListRepositoriesResponse) async def list_repositories() -> ListRepositoriesResponse: """List all repositories.""" diff --git a/src/coderag/api/schemas.py b/src/coderag/api/schemas.py index b920ff2..a9dbe21 100644 --- a/src/coderag/api/schemas.py +++ b/src/coderag/api/schemas.py @@ -63,6 +63,10 @@ class RetrievedChunkResponse(BaseModel): chunk_type: str name: str | None = None content: str + score_breakdown: dict[str, float] = Field(default_factory=dict) + retrieval_sources: list[str] = Field(default_factory=list) + token_estimate: int = 0 + ranking_reason: str | None = None model_config = ConfigDict(from_attributes=True) @@ -79,6 +83,45 @@ class QueryResponse(BaseModel): grounded: bool = Field(..., description="Whether every parsed citation is verified against retrieved evidence") query_id: str = Field(..., description="Query ID") + +class ContextPackRequest(BaseModel): + """Request to build a retrieval-only context pack.""" + + query: str = Field(..., description="Search query for the context pack") + repo_id: str = Field(..., description="Repository ID to search") + top_k: int = Field(10, ge=1, le=20, description="Maximum snippets to include") + max_tokens: int = Field(4000, ge=1, description="Maximum context token estimate") + max_chunks_per_file: int = Field(3, ge=1, description="Maximum snippets per file") + + +class ContextSnippetResponse(BaseModel): + """A retrieval-only context snippet.""" + + chunk_id: str + file_path: str + start_line: int + end_line: int + citation: str + chunk_type: str + name: str | None = None + content: str + relevance_score: float = 0.0 + score_breakdown: dict[str, float] = Field(default_factory=dict) + retrieval_sources: list[str] = Field(default_factory=list) + token_estimate: int = 0 + ranking_reason: str | None = None + + +class ContextPackResponse(BaseModel): + """Retrieval-only context pack response.""" + + repo_id: str + query: str + snippets: list[ContextSnippetResponse] + token_estimate: int + budget: dict[str, int] + capabilities: dict[str, object] + class RepositoryInfo(BaseModel): """Repository information.""" diff --git a/src/coderag/cli.py b/src/coderag/cli.py index bb4d81a..84069af 100644 --- a/src/coderag/cli.py +++ b/src/coderag/cli.py @@ -6,11 +6,9 @@ import shutil import sys from pathlib import Path -from typing import Optional import click - # Config directory and file CONFIG_DIR = Path.home() / ".config" / "coderag" CONFIG_FILE = CONFIG_DIR / "config.json" @@ -32,7 +30,7 @@ def save_config(config: dict) -> None: CONFIG_FILE.write_text(json.dumps(config, indent=2)) -def get_claude_config_path() -> Optional[Path]: +def get_claude_config_path() -> Path | None: """Get Claude Desktop config path based on OS.""" system = platform.system() @@ -62,7 +60,7 @@ def cli(): @click.option("--provider", type=click.Choice(["groq", "openai", "anthropic", "openrouter", "together", "local"]), default=None, help="LLM provider to use") @click.option("--api-key", default=None, help="API key for the provider") -def setup(provider: Optional[str], api_key: Optional[str]): +def setup(provider: str | None, api_key: str | None): """Interactive setup wizard for CodeRAG. Configures the LLM provider and API key. Configuration is saved to @@ -161,8 +159,9 @@ def serve(host: str, port: int, reload: bool): _apply_config_to_env() import uvicorn - from coderag.main import create_app + from coderag.config import get_settings + from coderag.main import create_app settings = get_settings() app = create_app() @@ -264,7 +263,7 @@ def mcp_install(dry_run: bool): # Show diff click.echo("\nšŸ“ Changes to be made:") if existing: - click.echo(f" Update: mcpServers.coderag") + click.echo(" Update: mcpServers.coderag") click.echo(f" From: {json.dumps(existing)}") click.echo(f" To: {json.dumps(new_mcp_config)}") else: @@ -303,6 +302,7 @@ def index(url: str, branch: str): _apply_config_to_env() import asyncio + from coderag.mcp.handlers import get_mcp_handlers click.echo(f"\nšŸ“¦ Indexing repository: {url}") @@ -318,7 +318,7 @@ async def run_index(): result = asyncio.run(run_index()) if result.get("success"): - click.echo(f"\nāœ… Repository indexed successfully!") + click.echo("\nāœ… Repository indexed successfully!") click.echo(f" Repo ID: {result['repo_id']}") click.echo(f" Name: {result['name']}") click.echo(f" Files processed: {result['files_processed']}") @@ -346,6 +346,7 @@ def query(repo_id: str, question: str, top_k: int, output_format: str): _apply_config_to_env() import asyncio + from coderag.mcp.handlers import get_mcp_handlers handlers = get_mcp_handlers() @@ -388,6 +389,65 @@ async def run_query(): click.echo(f" - {chunk['file']}:{chunk['start_line']}-{chunk['end_line']} (relevance: {chunk['relevance']})") + +@cli.command("context-pack") +@click.argument("repo_id") +@click.argument("query_text") +@click.option("--top-k", default=10, type=int, help="Maximum snippets to include") +@click.option("--max-tokens", default=4000, type=int, help="Maximum context token estimate") +@click.option("--max-chunks-per-file", default=3, type=int, help="Maximum snippets per file") +@click.option("--format", "output_format", type=click.Choice(["markdown", "json"]), default="markdown", help="Output format") +def context_pack(repo_id: str, query_text: str, top_k: int, max_tokens: int, max_chunks_per_file: int, output_format: str): + """Build a retrieval-only context pack for agents. + + REPO_ID: Repository ID (full or first 8 characters) + QUERY_TEXT: Retrieval query for the context pack + """ + _apply_config_to_env() + + import asyncio + + from coderag.mcp.handlers import get_mcp_handlers + + handlers = get_mcp_handlers() + + async def run_context_pack(): + return await handlers.get_context_pack( + repo_id=repo_id, + query=query_text, + top_k=top_k, + max_tokens=max_tokens, + max_chunks_per_file=max_chunks_per_file, + ) + + result = asyncio.run(run_context_pack()) + if result.get("error"): + click.echo(f"āŒ Error: {result['error']}") + sys.exit(1) + + if output_format == "json": + click.echo(json.dumps(result, indent=2)) + return + + click.echo(f"# Context Pack: {result.get('repo_id', repo_id)}") + click.echo("") + click.echo(f"Query: {result.get('query', query_text)}") + click.echo(f"Token estimate: {result.get('token_estimate', 0)}") + capabilities = result.get("capabilities", {}) + click.echo(f"Generation required: {capabilities.get('generation_required', False)}") + click.echo("") + click.echo("## Snippets") + for index, snippet in enumerate(result.get("snippets", []), 1): + click.echo("") + click.echo(f"### {index}. {snippet.get('citation', 'unknown')}") + sources = ", ".join(snippet.get("retrieval_sources", [])) or "none" + click.echo(f"Sources: {sources}") + if snippet.get("ranking_reason"): + click.echo(f"Ranking: {snippet['ranking_reason']}") + click.echo("```") + click.echo(snippet.get("content", "")) + click.echo("```") + @cli.command("repos") @click.option("--format", "output_format", type=click.Choice(["text", "json"]), default="text", help="Output format") def repos(output_format: str): @@ -396,6 +456,7 @@ def repos(output_format: str): _apply_config_to_env() import asyncio + from coderag.mcp.handlers import get_mcp_handlers handlers = get_mcp_handlers() @@ -438,6 +499,7 @@ def update(repo_id: str): _apply_config_to_env() import asyncio + from coderag.mcp.handlers import get_mcp_handlers click.echo(f"\nšŸ”„ Updating repository: {repo_id}\n") @@ -484,6 +546,7 @@ def delete(repo_id: str, force: bool): _apply_config_to_env() import asyncio + from coderag.mcp.handlers import get_mcp_handlers handlers = get_mcp_handlers() @@ -537,6 +600,7 @@ def clean(force: bool): _apply_config_to_env() import asyncio + from coderag.mcp.handlers import get_mcp_handlers handlers = get_mcp_handlers() @@ -560,16 +624,17 @@ async def get_repos(): status_icon = "āŒ" if repo["status"] == "error" else "ā³" click.echo(f" {status_icon} {repo['id'][:8]} {repo['name']} ({repo['status']})") - if not force: - if not click.confirm(f"\nDelete these {len(to_clean)} repositories?"): - click.echo("Cancelled.") - return + if not force and not click.confirm(f"\nDelete these {len(to_clean)} repositories?"): + click.echo("Cancelled.") + return # Delete each repo deleted = 0 for repo in to_clean: - async def run_delete(): - return await handlers.delete_repository(repo_id=repo["id"]) + repo_id = repo["id"] + + async def run_delete(repo_id: str = repo_id): + return await handlers.delete_repository(repo_id=repo_id) try: result = asyncio.run(run_delete()) @@ -608,7 +673,7 @@ def doctor(): if config.get("llm_provider"): click.echo(f" Provider: {config['llm_provider']}") else: - click.echo(f"āš ļø No config file. Run 'coderag setup' to configure.") + click.echo("āš ļø No config file. Run 'coderag setup' to configure.") # Check API key api_key = config.get("llm_api_key") or os.environ.get("MODEL_LLM_API_KEY") diff --git a/src/coderag/indexing/vectorstore.py b/src/coderag/indexing/vectorstore.py index 0c699cf..337d4f1 100644 --- a/src/coderag/indexing/vectorstore.py +++ b/src/coderag/indexing/vectorstore.py @@ -1,7 +1,6 @@ """ChromaDB vector store operations.""" from pathlib import Path -from typing import Optional import chromadb from chromadb.config import Settings @@ -18,14 +17,14 @@ class VectorStore: def __init__( self, - persist_directory: Optional[Path] = None, - collection_name: Optional[str] = None, + persist_directory: Path | None = None, + collection_name: str | None = None, ) -> None: settings = get_settings() self.persist_directory = persist_directory or settings.vectorstore.persist_directory self.collection_name = collection_name or settings.vectorstore.collection_name - self._client: Optional[chromadb.PersistentClient] = None - self._collection: Optional[chromadb.Collection] = None + self._client: chromadb.PersistentClient | None = None + self._collection: chromadb.Collection | None = None @property def client(self) -> chromadb.PersistentClient: @@ -112,6 +111,22 @@ def query( return chunks_with_scores + def get_repo_chunks(self, repo_id: str) -> list[Chunk]: + """Return all stored chunks for a repository without similarity ranking.""" + results = self.collection.get( + where={"repo_id": repo_id}, + include=["documents", "metadatas"], + ) + + chunks: list[Chunk] = [] + if results["ids"]: + for index, chunk_id in enumerate(results["ids"]): + metadata = results["metadatas"][index] + metadata["id"] = chunk_id + metadata["content"] = results["documents"][index] + chunks.append(Chunk.from_dict(metadata)) + return chunks + def delete_repo_chunks(self, repo_id: str) -> int: # Get all chunks for this repo results = self.collection.get(where={"repo_id": repo_id}, include=[]) diff --git a/src/coderag/mcp/handlers.py b/src/coderag/mcp/handlers.py index 550dd82..33d8a34 100644 --- a/src/coderag/mcp/handlers.py +++ b/src/coderag/mcp/handlers.py @@ -35,6 +35,22 @@ def _citation_verification_dict(item: Any) -> dict[str, Any]: "chunk_id": _read_field(item, "chunk_id"), } +def _retrieved_chunk_dict(chunk: Any) -> dict[str, Any]: + return { + "chunk_id": _read_field(chunk, "chunk_id"), + "file_path": _read_field(chunk, "file_path"), + "start_line": _read_field(chunk, "start_line"), + "end_line": _read_field(chunk, "end_line"), + "chunk_type": _read_field(chunk, "chunk_type"), + "name": _read_field(chunk, "name"), + "content": _read_field(chunk, "content"), + "relevance_score": round(_read_field(chunk, "relevance_score", 0.0) or 0.0, 3), + "score_breakdown": _read_field(chunk, "score_breakdown", {}), + "retrieval_sources": _read_field(chunk, "retrieval_sources", []), + "token_estimate": _read_field(chunk, "token_estimate", 0), + "ranking_reason": _read_field(chunk, "ranking_reason"), + } + class MCPHandlers: """Handlers for MCP tools backed by shared application services.""" @@ -241,6 +257,41 @@ async def search_code( logger.error("MCP: Search failed", error=str(exc)) return {"results": [], "error": str(exc)} + async def search_hybrid( + self, + repo_id: str, + query: str, + top_k: int = 10, + max_tokens: int = 4000, + max_chunks_per_file: int = 3, + ) -> dict[str, Any]: + """Hybrid code search without LLM generation.""" + try: + results = self.retrieval_service.search_hybrid(repo_id, query, top_k, max_tokens, max_chunks_per_file) + return { + "results": [_retrieved_chunk_dict(chunk) for chunk in results], + "count": len(results), + } + except Exception as exc: + logger.error("MCP: Hybrid search failed", error=str(exc)) + return {"results": [], "error": str(exc)} + + async def get_context_pack( + self, + repo_id: str, + query: str, + top_k: int = 10, + max_tokens: int = 4000, + max_chunks_per_file: int = 3, + ) -> dict[str, Any]: + """Build a hybrid retrieval context pack without LLM generation.""" + try: + pack = self.retrieval_service.get_context_pack(repo_id, query, top_k, max_tokens, max_chunks_per_file) + return pack.to_dict() + except Exception as exc: + logger.error("MCP: Context pack failed", error=str(exc)) + return {"snippets": [], "error": str(exc)} + _mcp_handlers: MCPHandlers | None = None diff --git a/src/coderag/mcp/tools.py b/src/coderag/mcp/tools.py index 1824454..5c6e882 100644 --- a/src/coderag/mcp/tools.py +++ b/src/coderag/mcp/tools.py @@ -137,3 +137,64 @@ async def search_code( file_filter=file_filter, chunk_type=chunk_type, ) + + + +@mcp.tool() +async def search_hybrid( + repo_id: str, + query: str, + top_k: int = 10, + max_tokens: int = 4000, + max_chunks_per_file: int = 3, +) -> dict: + """Hybrid code search combining vector and lexical signals without LLM generation. + + Args: + repo_id: Repository ID (full or first 8 characters) + query: Search query + top_k: Maximum number of results (default: 10) + max_tokens: Maximum context token estimate + max_chunks_per_file: Maximum results from a single file + + Returns: + dict with ranked results, score breakdowns, sources, and token estimates + """ + handlers = get_mcp_handlers() + return await handlers.search_hybrid( + repo_id=repo_id, + query=query, + top_k=top_k, + max_tokens=max_tokens, + max_chunks_per_file=max_chunks_per_file, + ) + + +@mcp.tool() +async def get_context_pack( + repo_id: str, + query: str, + top_k: int = 10, + max_tokens: int = 4000, + max_chunks_per_file: int = 3, +) -> dict: + """Build a retrieval-only context pack for agents and API clients. + + Args: + repo_id: Repository ID (full or first 8 characters) + query: Context query + top_k: Maximum number of snippets + max_tokens: Maximum context token estimate + max_chunks_per_file: Maximum snippets from a single file + + Returns: + dict with snippets, citations, score breakdowns, budget, and capabilities + """ + handlers = get_mcp_handlers() + return await handlers.get_context_pack( + repo_id=repo_id, + query=query, + top_k=top_k, + max_tokens=max_tokens, + max_chunks_per_file=max_chunks_per_file, + ) diff --git a/src/coderag/models/__init__.py b/src/coderag/models/__init__.py index d51e33a..12eff77 100644 --- a/src/coderag/models/__init__.py +++ b/src/coderag/models/__init__.py @@ -1,11 +1,14 @@ """Models module: Core dataclasses and entities.""" from coderag.models.chunk import Chunk, ChunkMetadata, ChunkType +from coderag.models.context import ContextPack, ContextSnippet from coderag.models.document import Document, DocumentMetadata from coderag.models.repository import Repository, RepositoryStatus from coderag.models.response import Citation, Query, Response, RetrievedChunk __all__ = [ + "ContextPack", + "ContextSnippet", "Document", "DocumentMetadata", "Chunk", diff --git a/src/coderag/models/context.py b/src/coderag/models/context.py new file mode 100644 index 0000000..f29716a --- /dev/null +++ b/src/coderag/models/context.py @@ -0,0 +1,119 @@ +"""Context pack models for retrieval-only agent handoff.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from coderag.models.response import RetrievedChunk + + +@dataclass +class ContextSnippet: + """A retrieved code snippet included in a context pack.""" + + chunk_id: str + content: str + file_path: str + start_line: int + end_line: int + chunk_type: str + name: str | None = None + relevance_score: float = 0.0 + score_breakdown: dict[str, float] = field(default_factory=dict) + retrieval_sources: list[str] = field(default_factory=list) + token_estimate: int = 0 + ranking_reason: str | None = None + + @property + def citation(self) -> str: + """Return stable source citation.""" + return f"[{self.file_path}:{self.start_line}-{self.end_line}]" + + @classmethod + def from_retrieved_chunk(cls, chunk: RetrievedChunk) -> ContextSnippet: + """Create a context snippet from a retrieved chunk.""" + return cls( + chunk_id=chunk.chunk_id, + content=chunk.content, + file_path=chunk.file_path, + start_line=chunk.start_line, + end_line=chunk.end_line, + chunk_type=chunk.chunk_type, + name=chunk.name, + relevance_score=chunk.relevance_score, + score_breakdown=dict(chunk.score_breakdown), + retrieval_sources=list(chunk.retrieval_sources), + token_estimate=chunk.token_estimate, + ranking_reason=chunk.ranking_reason, + ) + + def to_dict(self) -> dict[str, Any]: + return { + "chunk_id": self.chunk_id, + "content": self.content, + "file_path": self.file_path, + "start_line": self.start_line, + "end_line": self.end_line, + "citation": self.citation, + "chunk_type": self.chunk_type, + "name": self.name, + "relevance_score": self.relevance_score, + "score_breakdown": self.score_breakdown, + "retrieval_sources": self.retrieval_sources, + "token_estimate": self.token_estimate, + "ranking_reason": self.ranking_reason, + } + + +@dataclass +class ContextPack: + """Retrieval-only context bundle for agents and adapters.""" + + repo_id: str + query: str + snippets: list[ContextSnippet] + token_estimate: int + budget: dict[str, int] + capabilities: dict[str, Any] = field(default_factory=dict) + + def to_dict(self) -> dict[str, Any]: + return { + "repo_id": self.repo_id, + "query": self.query, + "snippets": [snippet.to_dict() for snippet in self.snippets], + "token_estimate": self.token_estimate, + "budget": self.budget, + "capabilities": self.capabilities, + } + + def to_markdown(self) -> str: + lines = [ + f"# Context Pack: {self.repo_id}", + "", + f"Query: {self.query}", + f"Token estimate: {self.token_estimate}", + f"Generation required: {self.capabilities.get('generation_required', False)}", + "", + "## Snippets", + ] + if not self.snippets: + lines.append("No snippets retrieved.") + return "\n".join(lines) + + for index, snippet in enumerate(self.snippets, 1): + title = f"{index}. {snippet.citation}" + if snippet.name: + title += f" — {snippet.name}" + lines.extend( + [ + "", + f"### {title}", + f"Sources: {', '.join(snippet.retrieval_sources) or 'none'}", + f"Ranking: {snippet.ranking_reason or 'n/a'}", + "```", + snippet.content, + "```", + ] + ) + return "\n".join(lines) diff --git a/src/coderag/models/response.py b/src/coderag/models/response.py index 72fdfad..b57d8aa 100644 --- a/src/coderag/models/response.py +++ b/src/coderag/models/response.py @@ -52,6 +52,10 @@ class RetrievedChunk: relevance_score: float chunk_type: str name: str | None = None + score_breakdown: dict[str, float] = field(default_factory=dict) + retrieval_sources: list[str] = field(default_factory=list) + token_estimate: int = 0 + ranking_reason: str | None = None @property def citation(self) -> str: diff --git a/src/coderag/retrieval/__init__.py b/src/coderag/retrieval/__init__.py index 523edbe..bef4c07 100644 --- a/src/coderag/retrieval/__init__.py +++ b/src/coderag/retrieval/__init__.py @@ -1,5 +1,13 @@ """Retrieval module: Query processing and similarity search.""" -from coderag.retrieval.retriever import Retriever +from typing import Any __all__ = ["Retriever"] + + +def __getattr__(name: str) -> Any: + if name == "Retriever": + from coderag.retrieval.retriever import Retriever + + return Retriever + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/coderag/retrieval/hybrid.py b/src/coderag/retrieval/hybrid.py new file mode 100644 index 0000000..0729496 --- /dev/null +++ b/src/coderag/retrieval/hybrid.py @@ -0,0 +1,243 @@ +"""Hybrid retrieval over vector and lexical indexed chunks.""" + +from __future__ import annotations + +import math +import re +from collections import Counter, defaultdict +from dataclasses import dataclass +from typing import Any + +from coderag.models.chunk import Chunk +from coderag.models.response import RetrievedChunk + +_TOKEN_RE = re.compile(r"[A-Za-z0-9_]+") + + +@dataclass(frozen=True) +class RetrievalBudget: + """Deterministic caps for context-producing retrieval.""" + + max_chunks: int + max_tokens: int + max_chunks_per_file: int = 3 + + +@dataclass +class _Candidate: + chunk: Chunk + vector_score: float = 0.0 + lexical_score: float = 0.0 + path_boost: float = 0.0 + name_boost: float = 0.0 + sources: set[str] | None = None + + def __post_init__(self) -> None: + if self.sources is None: + self.sources = set() + + @property + def combined_score(self) -> float: + return (0.35 * self.vector_score) + (0.55 * self.lexical_score) + self.path_boost + self.name_boost + + +class HybridRetriever: + """Merge semantic vector hits with deterministic lexical matches.""" + + def __init__(self, vectorstore: Any, embedder: Any | None = None, similarity_threshold: float = 0.0) -> None: + self.vectorstore = vectorstore + self.embedder = embedder + self.similarity_threshold = similarity_threshold + + def retrieve( + self, + query: str, + repo_id: str, + top_k: int = 5, + budget: RetrievalBudget | None = None, + ) -> list[RetrievedChunk]: + """Return ranked, deduplicated chunks using vector and lexical evidence.""" + top_k = max(int(top_k), 1) + budget = budget or RetrievalBudget(max_chunks=top_k, max_tokens=4000, max_chunks_per_file=3) + query_tokens = _tokenize(query) + candidates: list[_Candidate] = [] + + for chunk, score in self._semantic_candidates(query, repo_id, top_k): + candidate = _Candidate(chunk=chunk, vector_score=float(score), sources={"vector"}) + candidates.append(candidate) + + candidates.extend(self._lexical_candidates(query_tokens, repo_id)) + merged = self._merge_candidates(candidates) + ranked = sorted( + merged, + key=lambda candidate: ( + -candidate.combined_score, + candidate.chunk.file_path, + candidate.chunk.start_line, + candidate.chunk.end_line, + candidate.chunk.id, + ), + ) + return self._apply_budget(ranked, budget) + + def _semantic_candidates(self, query: str, repo_id: str, top_k: int) -> list[tuple[Chunk, float]]: + if self.embedder is None or not hasattr(self.vectorstore, "query"): + return [] + query_embedding = self.embedder.generate_embedding(query, is_query=True) + return list( + self.vectorstore.query( + query_embedding=query_embedding, + repo_id=repo_id, + top_k=top_k, + similarity_threshold=self.similarity_threshold, + ) + ) + + def _lexical_candidates(self, query_tokens: list[str], repo_id: str) -> list[_Candidate]: + if not query_tokens or not hasattr(self.vectorstore, "get_repo_chunks"): + return [] + chunks = list(self.vectorstore.get_repo_chunks(repo_id)) + if not chunks: + return [] + + query_counts = Counter(query_tokens) + chunk_tokens = [_tokenize(_lexical_text(chunk)) for chunk in chunks] + document_frequency: Counter[str] = Counter() + for tokens in chunk_tokens: + document_frequency.update(set(tokens)) + + raw_scores: list[float] = [] + for tokens in chunk_tokens: + counts = Counter(tokens) + length_norm = max(len(tokens), 1) + score = 0.0 + for token, query_weight in query_counts.items(): + term_frequency = counts[token] + if term_frequency == 0: + continue + idf = math.log((1 + len(chunks)) / (1 + document_frequency[token])) + 1.0 + score += query_weight * idf * (term_frequency / length_norm) + raw_scores.append(score) + + max_score = max(raw_scores, default=0.0) + candidates: list[_Candidate] = [] + for chunk, raw_score in zip(chunks, raw_scores, strict=True): + if raw_score <= 0.0: + continue + lexical_score = raw_score / max_score if max_score > 0.0 else 0.0 + path_boost = 0.08 if _contains_any(chunk.file_path, query_tokens) else 0.0 + name_boost = 0.12 if chunk.name and _contains_any(chunk.name, query_tokens) else 0.0 + candidates.append( + _Candidate( + chunk=chunk, + lexical_score=lexical_score, + path_boost=path_boost, + name_boost=name_boost, + sources={"lexical"}, + ) + ) + return candidates + + def _merge_candidates(self, candidates: list[_Candidate]) -> list[_Candidate]: + merged: list[_Candidate] = [] + for candidate in candidates: + existing = next((item for item in merged if _overlaps(item.chunk, candidate.chunk)), None) + if existing is None: + merged.append(candidate) + continue + if candidate.vector_score > existing.vector_score: + existing.vector_score = candidate.vector_score + if candidate.lexical_score > existing.lexical_score: + existing.lexical_score = candidate.lexical_score + existing.path_boost = max(existing.path_boost, candidate.path_boost) + existing.name_boost = max(existing.name_boost, candidate.name_boost) + if candidate.sources: + existing.sources = (existing.sources or set()) | candidate.sources + return merged + + def _apply_budget(self, candidates: list[_Candidate], budget: RetrievalBudget) -> list[RetrievedChunk]: + selected: list[RetrievedChunk] = [] + tokens_used = 0 + per_file: defaultdict[str, int] = defaultdict(int) + max_chunks = max(int(budget.max_chunks), 1) + max_tokens = max(int(budget.max_tokens), 1) + max_chunks_per_file = max(int(budget.max_chunks_per_file), 1) + + for candidate in candidates: + if len(selected) >= max_chunks: + break + chunk = candidate.chunk + token_estimate = _estimate_tokens(chunk.content) + if per_file[chunk.file_path] >= max_chunks_per_file: + continue + if selected and tokens_used + token_estimate > max_tokens: + continue + if not selected and token_estimate > max_tokens: + token_estimate = max_tokens + + selected.append( + RetrievedChunk( + chunk_id=chunk.id, + content=chunk.content, + file_path=chunk.file_path, + start_line=chunk.start_line, + end_line=chunk.end_line, + relevance_score=candidate.combined_score, + chunk_type=chunk.chunk_type.value, + name=chunk.name, + score_breakdown={ + "vector": candidate.vector_score, + "lexical": candidate.lexical_score, + "path": candidate.path_boost, + "name": candidate.name_boost, + "combined": candidate.combined_score, + }, + retrieval_sources=_ordered_sources(candidate.sources or set()), + token_estimate=token_estimate, + ranking_reason=_ranking_reason(candidate), + ) + ) + tokens_used += token_estimate + per_file[chunk.file_path] += 1 + return selected + + +def _tokenize(text: str) -> list[str]: + return [token.lower() for token in _TOKEN_RE.findall(text)] + + +def _lexical_text(chunk: Chunk) -> str: + return " ".join(part for part in [chunk.file_path, chunk.name or "", chunk.content] if part) + + +def _contains_any(text: str, tokens: list[str]) -> bool: + haystack = text.lower() + return any(token in haystack for token in tokens) + + +def _overlaps(left: Chunk, right: Chunk) -> bool: + return left.file_path == right.file_path and left.start_line <= right.end_line and right.start_line <= left.end_line + + +def _estimate_tokens(content: str) -> int: + words = content.split() + if words: + return len(words) + return max(1, math.ceil(len(content) / 4)) + + +def _ordered_sources(sources: set[str]) -> list[str]: + return [source for source in ["vector", "lexical"] if source in sources] + + +def _ranking_reason(candidate: _Candidate) -> str: + parts: list[str] = [] + if candidate.vector_score > 0: + parts.append(f"vector={candidate.vector_score:.3f}") + if candidate.lexical_score > 0: + parts.append(f"lexical={candidate.lexical_score:.3f}") + if candidate.path_boost > 0: + parts.append(f"path_boost={candidate.path_boost:.3f}") + if candidate.name_boost > 0: + parts.append(f"name_boost={candidate.name_boost:.3f}") + return ", ".join(parts) or "ranked by stable fallback order" diff --git a/src/coderag/retrieval/retriever.py b/src/coderag/retrieval/retriever.py index c3555ce..efcc6c3 100644 --- a/src/coderag/retrieval/retriever.py +++ b/src/coderag/retrieval/retriever.py @@ -1,13 +1,11 @@ """Retrieval module for semantic search.""" -from typing import Optional +from typing import Any from coderag.config import get_settings -from coderag.indexing.embeddings import EmbeddingGenerator -from coderag.indexing.vectorstore import VectorStore from coderag.logging import get_logger -from coderag.models.chunk import Chunk from coderag.models.response import RetrievedChunk +from coderag.retrieval.hybrid import HybridRetriever, RetrievalBudget logger = get_logger(__name__) @@ -17,12 +15,12 @@ class Retriever: def __init__( self, - vectorstore: Optional[VectorStore] = None, - embedder: Optional[EmbeddingGenerator] = None, + vectorstore: Any | None = None, + embedder: Any | None = None, ) -> None: settings = get_settings() - self.vectorstore = vectorstore or VectorStore() - self.embedder = embedder or EmbeddingGenerator() + self.vectorstore = vectorstore or self._default_vectorstore() + self.embedder = embedder or self._default_embedder() self.default_top_k = settings.retrieval.default_top_k self.max_top_k = settings.retrieval.max_top_k self.similarity_threshold = settings.retrieval.similarity_threshold @@ -31,48 +29,43 @@ def retrieve( self, query: str, repo_id: str, - top_k: Optional[int] = None, - similarity_threshold: Optional[float] = None, + top_k: int | None = None, + similarity_threshold: float | None = None, ) -> list[RetrievedChunk]: top_k = min(top_k or self.default_top_k, self.max_top_k) threshold = similarity_threshold if similarity_threshold is not None else self.similarity_threshold logger.info("Retrieving chunks", query=query[:100], repo_id=repo_id, top_k=top_k) - # Generate query embedding - query_embedding = self.embedder.generate_embedding(query, is_query=True) - - # Search vector store - results = self.vectorstore.query( - query_embedding=query_embedding, + retriever = HybridRetriever( + vectorstore=self.vectorstore, + embedder=self.embedder, + similarity_threshold=threshold, + ) + retrieved_chunks = retriever.retrieve( + query=query, repo_id=repo_id, top_k=top_k, - similarity_threshold=threshold, + budget=RetrievalBudget(max_chunks=top_k, max_tokens=4000, max_chunks_per_file=3), ) - - # Convert to RetrievedChunk - retrieved_chunks = [] - for chunk, score in results: - retrieved_chunk = RetrievedChunk( - chunk_id=chunk.id, - content=chunk.content, - file_path=chunk.file_path, - start_line=chunk.start_line, - end_line=chunk.end_line, - relevance_score=score, - chunk_type=chunk.chunk_type.value, - name=chunk.name, - ) - retrieved_chunks.append(retrieved_chunk) - logger.info("Chunks retrieved", count=len(retrieved_chunks)) return retrieved_chunks + def _default_vectorstore(self) -> Any: + from coderag.indexing.vectorstore import VectorStore + + return VectorStore() + + def _default_embedder(self) -> Any: + from coderag.indexing.embeddings import EmbeddingGenerator + + return EmbeddingGenerator() + def retrieve_with_context( self, query: str, repo_id: str, - top_k: Optional[int] = None, + top_k: int | None = None, ) -> tuple[list[RetrievedChunk], str]: chunks = self.retrieve(query, repo_id, top_k) diff --git a/src/coderag/services/retrieval.py b/src/coderag/services/retrieval.py index 0c38101..3d93d71 100644 --- a/src/coderag/services/retrieval.py +++ b/src/coderag/services/retrieval.py @@ -8,9 +8,11 @@ from coderag.config import get_settings from coderag.logging import get_logger from coderag.models.chunk import Chunk +from coderag.models.context import ContextPack, ContextSnippet from coderag.models.query import Query from coderag.models.repository import Repository, RepositoryStatus -from coderag.models.response import Response +from coderag.models.response import Response, RetrievedChunk +from coderag.retrieval.hybrid import HybridRetriever, RetrievalBudget from coderag.services.providers import ProviderConfigService from coderag.services.registry import RepositoryRegistry @@ -105,6 +107,64 @@ def search_code( return cast(list[tuple[Chunk, float]], results[:top_k]) + def search_hybrid( + self, + repo_id: str, + query: str, + top_k: int = 10, + max_tokens: int = 4000, + max_chunks_per_file: int = 3, + ) -> list[RetrievedChunk]: + """Hybrid code search without LLM generation.""" + repo, error = self.require_ready_repository(repo_id) + if error is not None: + raise ValueError(error) + assert repo is not None + + top_k = self._clamp_top_k(top_k) + budget = RetrievalBudget( + max_chunks=top_k, + max_tokens=max(int(max_tokens), 1), + max_chunks_per_file=max(int(max_chunks_per_file), 1), + ) + return self._get_hybrid_retriever().retrieve(query=query, repo_id=repo.id, top_k=top_k, budget=budget) + + def get_context_pack( + self, + repo_id: str, + query: str, + top_k: int = 10, + max_tokens: int = 4000, + max_chunks_per_file: int = 3, + ) -> ContextPack: + """Build a retrieval-only context pack for agents and API clients.""" + chunks = self.search_hybrid( + repo_id=repo_id, + query=query, + top_k=top_k, + max_tokens=max_tokens, + max_chunks_per_file=max_chunks_per_file, + ) + budget = { + "max_chunks": self._clamp_top_k(top_k), + "max_tokens": max(int(max_tokens), 1), + "max_chunks_per_file": max(int(max_chunks_per_file), 1), + } + snippets = [ContextSnippet.from_retrieved_chunk(chunk) for chunk in chunks] + return ContextPack( + repo_id=repo_id, + query=query, + snippets=snippets, + token_estimate=sum(snippet.token_estimate for snippet in snippets), + budget=budget, + capabilities={ + "generation_required": False, + "retrieval": "hybrid", + "semantic": True, + "lexical": True, + }, + ) + def _clamp_top_k(self, top_k: int) -> int: return min(max(int(top_k), 1), self.retrieval_settings.max_top_k) @@ -131,3 +191,10 @@ def _get_generator(self) -> Any: if self._generator is None: self._generator = self.generator_factory() return self._generator + + def _get_hybrid_retriever(self) -> HybridRetriever: + return HybridRetriever( + vectorstore=self._get_vectorstore(), + embedder=self._get_embedder(), + similarity_threshold=self.retrieval_settings.similarity_threshold, + ) diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index 544c81a..10a9272 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -6,7 +6,7 @@ import pytest from coderag.api import routes -from coderag.api.schemas import IndexRepositoryRequest, QueryRequest +from coderag.api.schemas import ContextPackRequest, IndexRepositoryRequest, QueryRequest from coderag.models.repository import Repository, RepositoryStatus from coderag.services.indexing import RepositoryDeleteResult from coderag.services.retrieval import QueryServiceResult @@ -111,6 +111,32 @@ def query_code(self, repo_id: str, question: str, top_k: int): return QueryServiceResult(repo=self.repo, error=f"Repository not ready: status is {self.repo.status.value}") return QueryServiceResult(repo=self.repo, response=FakeGeneratedResponse()) + def get_context_pack(self, repo_id: str, query: str, top_k: int = 10, max_tokens: int = 4000, max_chunks_per_file: int = 3): + from coderag.models.context import ContextPack, ContextSnippet + + self.calls.append((repo_id, query, top_k, max_tokens, max_chunks_per_file)) + return ContextPack( + repo_id=repo_id, + query=query, + snippets=[ + ContextSnippet( + chunk_id="chunk-1", + file_path="src/app.py", + start_line=1, + end_line=2, + chunk_type="function", + name="app", + content="def app(): pass", + retrieval_sources=["lexical"], + token_estimate=3, + score_breakdown={"lexical": 1.0, "combined": 0.55}, + ) + ], + token_estimate=3, + budget={"max_chunks": top_k, "max_tokens": max_tokens, "max_chunks_per_file": max_chunks_per_file}, + capabilities={"generation_required": False, "retrieval": "hybrid"}, + ) + @pytest.mark.asyncio async def test_index_repository_route_creates_record_through_indexing_service(monkeypatch): @@ -147,6 +173,25 @@ async def test_query_route_uses_retrieval_service_and_preserves_response(monkeyp assert response.citation_verifications[0].reason == "verified" +@pytest.mark.asyncio +async def test_context_pack_route_uses_retrieval_service_without_generation(monkeypatch): + repo = Repository(id="repo-ready", url="https://github.com/acme/demo", status=RepositoryStatus.READY) + retrieval = FakeRetrievalService(repo) + monkeypatch.setattr(routes, "registry", FakeRegistry(repo)) + monkeypatch.setattr(routes, "retrieval_service", retrieval) + + response = await routes.context_pack( + ContextPackRequest(repo_id="repo-ready", query="app function", top_k=3, max_tokens=1000) + ) + + assert response.repo_id == "repo-ready" + assert response.query == "app function" + assert response.capabilities["generation_required"] is False + assert response.snippets[0].citation == "[src/app.py:1-2]" + assert response.snippets[0].score_breakdown["lexical"] == 1.0 + assert retrieval.calls[-1] == ("repo-ready", "app function", 3, 1000, 3) + + @pytest.mark.asyncio async def test_query_route_maps_missing_and_not_ready_errors(monkeypatch): repo = Repository(id="repo-pending", url="https://github.com/acme/demo", status=RepositoryStatus.INDEXING) diff --git a/tests/test_m4_hybrid_retrieval.py b/tests/test_m4_hybrid_retrieval.py new file mode 100644 index 0000000..38db20f --- /dev/null +++ b/tests/test_m4_hybrid_retrieval.py @@ -0,0 +1,230 @@ +"""Milestone 4 hybrid retrieval and context pack tests.""" +# mypy: disable-error-code="no-untyped-def,no-untyped-call" + +from coderag.models.chunk import Chunk, ChunkMetadata, ChunkType +from coderag.models.repository import Repository, RepositoryStatus +from coderag.services.registry import RepositoryRegistry + + +def chunk( + *, + content: str, + file_path: str, + start_line: int, + end_line: int, + name: str | None = None, + chunk_id: str | None = None, +) -> Chunk: + return Chunk( + id=chunk_id or f"{file_path}:{start_line}-{end_line}", + repo_id="repo-1", + content=content, + metadata=ChunkMetadata( + file_path=file_path, + start_line=start_line, + end_line=end_line, + chunk_type=ChunkType.FUNCTION, + language="python", + name=name, + ), + ) + + +class FakeVectorStore: + def __init__(self, vector_results: list[tuple[Chunk, float]], repo_chunks: list[Chunk]) -> None: + self.vector_results = vector_results + self.repo_chunks = repo_chunks + + def query(self, query_embedding, repo_id: str, top_k: int, similarity_threshold: float = 0.0): + _ = (query_embedding, repo_id, top_k, similarity_threshold) + return self.vector_results[:top_k] + + def get_repo_chunks(self, repo_id: str) -> list[Chunk]: + _ = repo_id + return self.repo_chunks + + +class FakeEmbedder: + def generate_embedding(self, text: str, is_query: bool = False) -> list[float]: + _ = (text, is_query) + return [0.0] + + +def test_hybrid_retrieval_surfaces_lexical_exact_match_ahead_of_unrelated_vector_hit(): + from coderag.retrieval.hybrid import HybridRetriever, RetrievalBudget + + auth = chunk( + content="def validate_token(token):\n return token.startswith('auth_')", + file_path="src/auth.py", + start_line=10, + end_line=11, + name="validate_token", + ) + billing = chunk( + content="def charge_card(card):\n return gateway.charge(card)", + file_path="src/billing.py", + start_line=3, + end_line=4, + name="charge_card", + ) + retriever = HybridRetriever( + vectorstore=FakeVectorStore(vector_results=[(billing, 0.95)], repo_chunks=[auth, billing]), + embedder=FakeEmbedder(), + ) + + results = retriever.retrieve("auth token validation", "repo-1", top_k=2, budget=RetrievalBudget(max_chunks=2, max_tokens=1000, max_chunks_per_file=2)) + + assert [result.file_path for result in results] == ["src/auth.py", "src/billing.py"] + assert results[0].retrieval_sources == ["lexical"] + assert results[0].score_breakdown["lexical"] > 0 + assert results[1].score_breakdown["vector"] == 0.95 + + +def test_hybrid_retrieval_dedupes_overlapping_vector_and_lexical_candidates(): + from coderag.retrieval.hybrid import HybridRetriever, RetrievalBudget + + lexical = chunk( + content="class AuthService:\n def validate_token(self, token):\n return True", + file_path="src/auth.py", + start_line=1, + end_line=3, + name="AuthService", + chunk_id="auth-lexical", + ) + vector = chunk( + content=lexical.content, + file_path="src/auth.py", + start_line=2, + end_line=4, + name="validate_token", + chunk_id="auth-vector", + ) + retriever = HybridRetriever( + vectorstore=FakeVectorStore(vector_results=[(vector, 0.8)], repo_chunks=[lexical]), + embedder=FakeEmbedder(), + ) + + results = retriever.retrieve("validate token", "repo-1", top_k=5, budget=RetrievalBudget(max_chunks=5, max_tokens=1000, max_chunks_per_file=5)) + + assert len(results) == 1 + assert results[0].file_path == "src/auth.py" + assert results[0].retrieval_sources == ["vector", "lexical"] + assert results[0].score_breakdown["vector"] == 0.8 + assert results[0].score_breakdown["lexical"] > 0 + + +def test_hybrid_retrieval_applies_deterministic_budget_caps(): + from coderag.retrieval.hybrid import HybridRetriever, RetrievalBudget + + chunks = [ + chunk(content="auth token " * 20, file_path="src/auth.py", start_line=1, end_line=2, name="first"), + chunk(content="auth token " * 20, file_path="src/auth.py", start_line=5, end_line=6, name="second"), + chunk(content="auth token " * 20, file_path="src/users.py", start_line=1, end_line=2, name="third"), + ] + retriever = HybridRetriever(vectorstore=FakeVectorStore(vector_results=[], repo_chunks=chunks), embedder=FakeEmbedder()) + + results = retriever.retrieve("auth token", "repo-1", top_k=10, budget=RetrievalBudget(max_chunks=10, max_tokens=80, max_chunks_per_file=1)) + + assert len(results) == 2 + assert [result.file_path for result in results] == ["src/auth.py", "src/users.py"] + assert sum(result.token_estimate for result in results) <= 80 + + +def test_retrieval_service_context_pack_uses_hybrid_results_without_generation(tmp_path): + from coderag.models.context import ContextPack + from coderag.services.retrieval import RetrievalService + + repo = Repository( + id="repo-1", + url="https://github.com/example/repo", + branch="main", + status=RepositoryStatus.READY, + ) + registry = RepositoryRegistry(db_path=tmp_path / "registry.sqlite3") + registry.add(repo) + auth = chunk( + content="def validate_token(token):\n return token.startswith('auth_')", + file_path="src/auth.py", + start_line=10, + end_line=11, + name="validate_token", + ) + service = RetrievalService( + registry=registry, + vectorstore=FakeVectorStore(vector_results=[], repo_chunks=[auth]), + embedder=FakeEmbedder(), + ) + + pack = service.get_context_pack("repo-1", "auth token validation", top_k=5, max_tokens=1000) + + assert isinstance(pack, ContextPack) + assert pack.repo_id == "repo-1" + assert pack.query == "auth token validation" + assert pack.snippets[0].file_path == "src/auth.py" + assert pack.snippets[0].citation == "[src/auth.py:10-11]" + assert pack.capabilities["generation_required"] is False + assert "auth token validation" in pack.to_markdown() + + +def test_legacy_retriever_uses_hybrid_ranking_when_dependencies_are_injected(): + from coderag.retrieval.retriever import Retriever + + auth = chunk( + content="def validate_token(token):\n return token.startswith('auth_')", + file_path="src/auth.py", + start_line=10, + end_line=11, + name="validate_token", + ) + billing = chunk( + content="def charge_card(card):\n return gateway.charge(card)", + file_path="src/billing.py", + start_line=3, + end_line=4, + name="charge_card", + ) + retriever = Retriever( + vectorstore=FakeVectorStore(vector_results=[(billing, 0.95)], repo_chunks=[auth, billing]), + embedder=FakeEmbedder(), + ) + + results = retriever.retrieve("auth token validation", "repo-1", top_k=2) + + assert [result.file_path for result in results] == ["src/auth.py", "src/billing.py"] + assert results[0].retrieval_sources == ["lexical"] + +def test_cli_context_pack_renders_markdown(monkeypatch): + import sys + from types import SimpleNamespace + + from click.testing import CliRunner + + from coderag.cli import cli + + async def get_context_pack(**kwargs): + return { + "repo_id": kwargs["repo_id"], + "query": kwargs["query"], + "snippets": [ + { + "citation": "[src/auth.py:10-11]", + "content": "def validate_token(token): pass", + "retrieval_sources": ["lexical"], + "ranking_reason": "lexical=1.000", + } + ], + "token_estimate": 4, + "capabilities": {"generation_required": False}, + } + + monkeypatch.setitem( + sys.modules, + "coderag.mcp.handlers", + SimpleNamespace(get_mcp_handlers=lambda: SimpleNamespace(get_context_pack=get_context_pack)), + ) + + result = CliRunner().invoke(cli, ["context-pack", "repo-1", "auth token", "--format", "markdown"]) + + assert result.exit_code == 0 + assert "Context Pack: repo-1" in result.output + assert "[src/auth.py:10-11]" in result.output diff --git a/tests/test_mcp_handlers.py b/tests/test_mcp_handlers.py index f4855be..5d7b517 100644 --- a/tests/test_mcp_handlers.py +++ b/tests/test_mcp_handlers.py @@ -90,6 +90,60 @@ def search_code(self, repo_id: str, query: str, top_k: int = 10, file_filter=Non ) ] + def search_hybrid(self, repo_id: str, query: str, top_k: int = 10, max_tokens: int = 4000, max_chunks_per_file: int = 3): + _ = (query, top_k, max_tokens, max_chunks_per_file) + repo = self.registry.get(repo_id) + if repo is None: + raise ValueError(f"Repository not found: {repo_id}") + return [ + SimpleNamespace( + chunk_id="chunk-1", + file_path="src/app.py", + start_line=1, + end_line=3, + chunk_type="function", + name="app", + content="def app(): pass", + relevance_score=0.9, + score_breakdown={"vector": 0.2, "lexical": 1.0, "combined": 0.62}, + retrieval_sources=["lexical"], + token_estimate=3, + ranking_reason="lexical=1.000", + ) + ] + + def get_context_pack(self, repo_id: str, query: str, top_k: int = 10, max_tokens: int = 4000, max_chunks_per_file: int = 3): + _ = (top_k, max_tokens, max_chunks_per_file) + repo = self.registry.get(repo_id) + if repo is None: + raise ValueError(f"Repository not found: {repo_id}") + return SimpleNamespace( + repo_id=repo_id, + query=query, + token_estimate=3, + budget={"max_chunks": 10, "max_tokens": max_tokens, "max_chunks_per_file": max_chunks_per_file}, + capabilities={"generation_required": False, "retrieval": "hybrid"}, + to_dict=lambda: { + "repo_id": repo_id, + "query": query, + "token_estimate": 3, + "budget": {"max_chunks": 10, "max_tokens": max_tokens, "max_chunks_per_file": max_chunks_per_file}, + "capabilities": {"generation_required": False, "retrieval": "hybrid"}, + "snippets": [ + { + "chunk_id": "chunk-1", + "file_path": "src/app.py", + "start_line": 1, + "end_line": 3, + "citation": "[src/app.py:1-3]", + "content": "def app(): pass", + "retrieval_sources": ["lexical"], + "token_estimate": 3, + } + ], + }, + ) + @pytest.fixture def registry(tmp_path): @@ -278,6 +332,33 @@ async def test_search_code_success_shape(self, mcp_handlers, registry, sample_re } +class TestHybridSearchAndContextPack: + @pytest.mark.asyncio + async def test_search_hybrid_success_shape(self, mcp_handlers, registry, sample_repository): + registry.add(sample_repository) + + result = await mcp_handlers.search_hybrid(sample_repository.id, "app function", top_k=1) + + assert result["count"] == 1 + assert result["results"][0]["file_path"] == "src/app.py" + assert result["results"][0]["score_breakdown"]["lexical"] == 1.0 + assert result["results"][0]["retrieval_sources"] == ["lexical"] + assert result["results"][0]["ranking_reason"] == "lexical=1.000" + + @pytest.mark.asyncio + async def test_context_pack_success_shape_and_json_serializable(self, mcp_handlers, registry, sample_repository): + registry.add(sample_repository) + + result = await mcp_handlers.get_context_pack(sample_repository.id, "app function", top_k=1, max_tokens=1000) + + encoded = json.dumps(result) + decoded = json.loads(encoded) + assert decoded["repo_id"] == sample_repository.id + assert decoded["query"] == "app function" + assert decoded["capabilities"]["generation_required"] is False + assert decoded["snippets"][0]["citation"] == "[src/app.py:1-3]" + + class TestUpdateRepository: @pytest.mark.asyncio diff --git a/tests/test_mcp_tools.py b/tests/test_mcp_tools.py index e8e9d62..1f9f768 100644 --- a/tests/test_mcp_tools.py +++ b/tests/test_mcp_tools.py @@ -36,6 +36,8 @@ def test_tools_are_registered(self): assert hasattr(tools, "delete_repository") assert hasattr(tools, "update_repository") assert hasattr(tools, "search_code") + assert hasattr(tools, "search_hybrid") + assert hasattr(tools, "get_context_pack") class TestToolDelegation: @@ -176,3 +178,57 @@ async def test_search_code_delegates_to_handler(self): chunk_type="function", ) assert result["count"] == 0 + + @pytest.mark.asyncio + async def test_search_hybrid_delegates_to_handler(self): + """Test that search_hybrid tool delegates to handler.""" + with patch("coderag.mcp.tools.get_mcp_handlers") as mock_get_handlers: + mock_handler = MagicMock() + mock_handler.search_hybrid = AsyncMock(return_value={"results": [], "count": 0}) + mock_get_handlers.return_value = mock_handler + + from coderag.mcp.tools import search_hybrid + + result = await search_hybrid( + repo_id="test-id", + query="function", + top_k=10, + max_tokens=1000, + max_chunks_per_file=2, + ) + + mock_handler.search_hybrid.assert_called_once_with( + repo_id="test-id", + query="function", + top_k=10, + max_tokens=1000, + max_chunks_per_file=2, + ) + assert result["count"] == 0 + + @pytest.mark.asyncio + async def test_get_context_pack_delegates_to_handler(self): + """Test that get_context_pack tool delegates to handler.""" + with patch("coderag.mcp.tools.get_mcp_handlers") as mock_get_handlers: + mock_handler = MagicMock() + mock_handler.get_context_pack = AsyncMock(return_value={"snippets": [], "token_estimate": 0}) + mock_get_handlers.return_value = mock_handler + + from coderag.mcp.tools import get_context_pack + + result = await get_context_pack( + repo_id="test-id", + query="function", + top_k=10, + max_tokens=1000, + max_chunks_per_file=2, + ) + + mock_handler.get_context_pack.assert_called_once_with( + repo_id="test-id", + query="function", + top_k=10, + max_tokens=1000, + max_chunks_per_file=2, + ) + assert result["token_estimate"] == 0