Milestone 3: verified citation grounding

.github/workflows/ci.yml

@@ -25,17 +25,17 @@ jobs:
           python -m pip install -e . --no-deps
           python -m pip install pytest pytest-asyncio pydantic pydantic-settings click PyYAML fastapi structlog ruff mypy mcp httpx
 
-      - name: Run Milestone 0-2 tests
-        run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py
+      - name: Run Milestone 0-3 tests
+        run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py
 
-      - name: Run Milestone 0-2 lint baseline
-        run: ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
+      - name: Run Milestone 0-3 lint baseline
+        run: ruff check src/coderag/generation src/coderag/models src/coderag/services src/coderag/config.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
 
-      - name: Run Milestone 0-2 type baseline
-        run: PYTHONPATH=src mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
+      - name: Run Milestone 0-3 type baseline
+        run: PYTHONPATH=src mypy src/coderag/generation src/coderag/models src/coderag/services src/coderag/config.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
 
       - name: Compile changed runtime modules
-        run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/generator.py src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
+        run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/*.py src/coderag/models/*.py src/coderag/services/*.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
 
       - name: Build package
         run: python -m build

.pi/.specsafe-state.json

@@ -100,6 +100,26 @@
           "turns": 0
         }
       }
+    },
+    {
+      "sliceId": "CRAG-20260628-M3-VERIFIED-CITATIONS",
+      "workspaceId": "CodeRAG",
+      "sessionId": "omp-session",
+      "beganAt": "2026-06-28T03:53:31.850Z",
+      "endedAt": "2026-06-28T04:36:32.656Z",
+      "outcome": "PASS",
+      "costSummary": {
+        "externalMemoryCalls": 0,
+        "externalMemoryCost": 0,
+        "subagentTokens": {
+          "input": 0,
+          "output": 0,
+          "cacheRead": 0,
+          "cacheWrite": 0,
+          "cost": 0,
+          "turns": 0
+        }
+      }
     }
   ]
 }

docs/superpowers/plans/2026-06-28-m3-verified-citations-contract.md

@@ -0,0 +1,107 @@
+# Milestone 3 Verified Citation Engine Contract
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use `subagent-driven-development` or inline TDD execution. Do not implement runtime code before observing failing M3 citation tests.
+
+**Slice:** `CRAG-20260628-M3-VERIFIED-CITATIONS`  
+**Branch:** `feat/m3-verified-citations`  
+**Source of truth:** `docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md`, Milestone 3 lines 505-529, plus current `main` after M2 merge.
+
+## Goal
+
+Make CodeRAG's `grounded` claim depend on deterministic citation verification instead of citation string presence.
+
+## Scope
+
+M3 may add or modify:
+
+- citation verification models/helpers under `src/coderag/generation/` or `src/coderag/services/`;
+- response model fields needed to expose verification status while preserving existing `answer`, `citations`, `retrieved_chunks`, `grounded`, and `query_id` fields;
+- `ResponseGenerator` citation handling;
+- `RetrievalService`, API, MCP, CLI, and UI adapters only as needed to expose verification status;
+- registry/file/chunk metadata reads only as needed for citation verification;
+- EvalFly deterministic smoke coverage and citation-focused tests.
+
+M3 must not add:
+
+- hybrid/BM25 retrieval;
+- context pack APIs;
+- code graph extraction;
+- autonomous editing behavior;
+- broad provider or model changes;
+- production-ready claims.
+
+## Definitions
+
+A citation is **verified** only if all are true:
+
+1. the parsed citation path matches a retrieved chunk path;
+2. the citation line range is valid (`start_line >= 1`, `end_line >= start_line`);
+3. the citation line range overlaps at least one retrieved chunk from the same file;
+4. the cited chunk belongs to the current query's retrieved evidence set.
+
+A citation is **unverified** if parsing succeeds but any verification condition fails.
+
+A response is `grounded=True` only when:
+
+- at least one citation is present;
+- at least one citation is verified;
+- no parsed citation is unverified.
+
+No-citation responses remain `grounded=False`.
+
+## Acceptance Criteria
+
+### M3-AC1: Citation verifier
+
+- Add a deterministic verifier that accepts parsed `Citation` objects and current `RetrievedChunk` evidence.
+- It returns per-citation verification status and reason.
+- It detects hallucinated paths, non-overlapping line ranges, invalid ranges, and stale/non-retrieved citations.
+
+### M3-AC2: Grounded semantics
+
+- `ResponseGenerator.generate()` must compute `grounded` from verifier output, not from `len(citations) > 0 and len(chunks) > 0`.
+- A response with any unverified parsed citation is not grounded.
+- A response with verified citations is grounded.
+- A response with no citations remains not grounded.
+
+### M3-AC3: Adapter exposure
+
+- REST `QueryResponse` exposes citation verification status without removing existing fields.
+- MCP query result exposes citation verification status without removing existing keys.
+- CLI/UI may display a basic verified/unverified status, but must not invent new behavior beyond M3.
+
+### M3-AC4: EvalFly and tests
+
+- Add citation verifier unit tests with negative cases for:
+  - hallucinated path;
+  - out-of-range/non-overlapping lines;
+  - invalid line range;
+  - mixed verified and unverified citations.
+- Add generator/service tests proving `grounded` follows verifier results.
+- Extend committed EvalFly smoke with deterministic citation-engine checks.
+
+### M3-AC5: Governance
+
+- Keep all commits on `feat/m3-verified-citations` with `Spec-Slice: CRAG-20260628-M3-VERIFIED-CITATIONS` trailers.
+- Push after every commit using the guarded push wrapper.
+- Open PR with guarded GitHub wrapper only after local verification and review.
+
+## Required Evidence
+
+Before PR:
+
+```bash
+bun run /Users/sebastian/.omp/omp-pantheon/skills/specsafe/bin/specsafe.ts status
+PYTHONPATH=src .venv/bin/python -m pytest tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py
+.venv/bin/python -m ruff check src/coderag/generation src/coderag/models src/coderag/services src/coderag/api/routes.py src/coderag/api/schemas.py src/coderag/mcp/handlers.py src/coderag/ui/handlers.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
+PYTHONPATH=src .venv/bin/python -m mypy src/coderag/generation src/coderag/models src/coderag/services src/coderag/api/routes.py src/coderag/api/schemas.py src/coderag/mcp/handlers.py src/coderag/ui/handlers.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
+.venv/bin/python scripts/evalfly_smoke.py --suite smoke
+.venv/bin/python -m py_compile src/coderag/generation/*.py src/coderag/models/*.py src/coderag/services/*.py src/coderag/api/routes.py src/coderag/api/schemas.py src/coderag/mcp/handlers.py src/coderag/ui/handlers.py tests/test_citations.py tests/test_m3_citations.py
+.venv/bin/python -m build
+```
+
+## Non-goals
+
+- Do not require reading the live filesystem during query verification. M3 verifies citations against retrieved evidence, not against mutable working tree state.
+- Do not claim stale working-tree detection unless file snapshot hashes are actually checked in the query path.
+- Do not change retrieval ranking or chunking behavior.

evals/config.json

@@ -418,6 +418,94 @@
           }
         ]
       }
+    },
+    {
+      "schema_version": "evalfly.case.v1",
+      "case_id": "m3-citation-verifier-exists",
+      "title": "Milestone 3 citation verifier implementation exists",
+      "suite": "smoke",
+      "risk_tier": "critical",
+      "task_type": "grounding_verification",
+      "source": {
+        "kind": "specsafe_slice",
+        "slice_id": "CRAG-20260628-M3-VERIFIED-CITATIONS"
+      },
+      "privacy": {
+        "contains_user_data": false,
+        "contains_code": true,
+        "data_classification": "public_repo"
+      },
+      "expected": {
+        "artifact": "src/coderag/generation/citations.py"
+      },
+      "judge": {
+        "type": "deterministic",
+        "assertions": [
+          {
+            "type": "file_exists",
+            "path": "src/coderag/generation/citations.py"
+          },
+          {
+            "type": "file_contains",
+            "path": "src/coderag/generation/citations.py",
+            "text": "class CitationVerifier"
+          },
+          {
+            "type": "file_contains",
+            "path": "src/coderag/generation/citations.py",
+            "text": "path_not_retrieved"
+          },
+          {
+            "type": "file_contains",
+            "path": "src/coderag/generation/citations.py",
+            "text": "line_range_not_retrieved"
+          },
+          {
+            "type": "file_contains",
+            "path": "src/coderag/generation/citations.py",
+            "text": "invalid_line_range"
+          }
+        ]
+      }
+    },
+    {
+      "schema_version": "evalfly.case.v1",
+      "case_id": "m3-citation-tests-exist",
+      "title": "Milestone 3 citation verification tests exist",
+      "suite": "smoke",
+      "risk_tier": "critical",
+      "task_type": "test_coverage",
+      "source": {
+        "kind": "repo_file",
+        "path": "tests/test_m3_citations.py"
+      },
+      "privacy": {
+        "contains_user_data": false,
+        "contains_code": true,
+        "data_classification": "public_repo"
+      },
+      "expected": {
+        "artifact": "tests/test_m3_citations.py"
+      },
+      "judge": {
+        "type": "deterministic",
+        "assertions": [
+          {
+            "type": "file_exists",
+            "path": "tests/test_m3_citations.py"
+          },
+          {
+            "type": "file_contains",
+            "path": "tests/test_m3_citations.py",
+            "text": "test_citation_verifier_requires_all_citations_to_be_verified_for_grounding"
+          },
+          {
+            "type": "file_contains",
+            "path": "tests/test_m3_citations.py",
+            "text": "test_response_generator_grounds_only_when_all_citations_verify"
+          }
+        ]
+      }
     }
   ]
 }

src/coderag/api/routes.py

@@ -1,9 +1,12 @@
 """REST API routes."""
 
+from typing import Any
+
 from fastapi import APIRouter, BackgroundTasks, HTTPException
 
 from coderag.api.schemas import (
     CitationResponse,
+    CitationVerificationResponse,
     IndexRepositoryRequest,
     IndexRepositoryResponse,
     ListRepositoriesResponse,
@@ -13,6 +16,7 @@
     RetrievedChunkResponse,
 )
 from coderag.logging import get_logger
+from coderag.models.repository import Repository
 from coderag.services.indexing import IndexingOptions, IndexingService
 from coderag.services.registry import RepositoryRegistry
 from coderag.services.retrieval import RetrievalService
@@ -25,14 +29,31 @@
 retrieval_service = RetrievalService(registry=registry)
 
 
-def get_repo_or_404(repo_id: str):
+def get_repo_or_404(repo_id: str) -> Repository:
     """Get a repository by full ID or unambiguous prefix, raising 404 if not found."""
     repo = registry.get_unique(repo_id)
     if repo is None:
         raise HTTPException(status_code=404, detail="Repository not found")
     return repo
 
 
+def _read_field(item: Any, field: str, default: Any = None) -> Any:
+    if isinstance(item, dict):
+        return item.get(field, default)
+    return getattr(item, field, default)
+
+
+def _citation_verification_response(item: Any) -> CitationVerificationResponse:
+    return CitationVerificationResponse(
+        file_path=_read_field(item, "file_path"),
+        start_line=_read_field(item, "start_line"),
+        end_line=_read_field(item, "end_line"),
+        verified=_read_field(item, "verified"),
+        reason=_read_field(item, "reason"),
+        chunk_id=_read_field(item, "chunk_id"),
+    )
+
+
 async def index_repository_task(
     url: str,
     repo_id: str,
@@ -119,6 +140,10 @@ async def query_repository(request: QueryRequest) -> QueryResponse:
             )
             for c in response.citations
         ],
+        citation_verifications=[
+            _citation_verification_response(verification)
+            for verification in getattr(response, "citation_verifications", [])
+        ],
         retrieved_chunks=[
             RetrievedChunkResponse(
                 chunk_id=c.chunk_id,
@@ -172,7 +197,7 @@ async def get_repository(repo_id: str) -> RepositoryInfo:
 
 
 @router.delete("/repos/{repo_id}")
-async def delete_repository(repo_id: str) -> dict:
+async def delete_repository(repo_id: str) -> dict[str, str]:
     """Delete a repository by full ID or prefix."""
     repo = get_repo_or_404(repo_id)
     try: