From 3648d7c5239c81ad3891ac7c6dc2b634f608d4ea Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 22:38:03 -0600 Subject: [PATCH 1/7] docs: freeze Milestone 3 verified citations contract Spec-Slice: CRAG-20260628-M3-VERIFIED-CITATIONS --- .pi/.specsafe-state.json | 20 ++++ ...26-06-28-m3-verified-citations-contract.md | 107 ++++++++++++++++++ 2 files changed, 127 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-28-m3-verified-citations-contract.md diff --git a/.pi/.specsafe-state.json b/.pi/.specsafe-state.json index 2630bcd..a945447 100644 --- a/.pi/.specsafe-state.json +++ b/.pi/.specsafe-state.json @@ -100,6 +100,26 @@ "turns": 0 } } + }, + { + "sliceId": "CRAG-20260628-M3-VERIFIED-CITATIONS", + "workspaceId": "CodeRAG", + "sessionId": "omp-session", + "beganAt": "2026-06-28T03:53:31.850Z", + "endedAt": "2026-06-28T04:36:32.656Z", + "outcome": "PASS", + "costSummary": { + "externalMemoryCalls": 0, + "externalMemoryCost": 0, + "subagentTokens": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "cost": 0, + "turns": 0 + } + } } ] } diff --git a/docs/superpowers/plans/2026-06-28-m3-verified-citations-contract.md b/docs/superpowers/plans/2026-06-28-m3-verified-citations-contract.md new file mode 100644 index 0000000..1f19150 --- /dev/null +++ b/docs/superpowers/plans/2026-06-28-m3-verified-citations-contract.md @@ -0,0 +1,107 @@ +# Milestone 3 Verified Citation Engine Contract + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `subagent-driven-development` or inline TDD execution. Do not implement runtime code before observing failing M3 citation tests. + +**Slice:** `CRAG-20260628-M3-VERIFIED-CITATIONS` +**Branch:** `feat/m3-verified-citations` +**Source of truth:** `docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md`, Milestone 3 lines 505-529, plus current `main` after M2 merge. + +## Goal + +Make CodeRAG's `grounded` claim depend on deterministic citation verification instead of citation string presence. + +## Scope + +M3 may add or modify: + +- citation verification models/helpers under `src/coderag/generation/` or `src/coderag/services/`; +- response model fields needed to expose verification status while preserving existing `answer`, `citations`, `retrieved_chunks`, `grounded`, and `query_id` fields; +- `ResponseGenerator` citation handling; +- `RetrievalService`, API, MCP, CLI, and UI adapters only as needed to expose verification status; +- registry/file/chunk metadata reads only as needed for citation verification; +- EvalFly deterministic smoke coverage and citation-focused tests. + +M3 must not add: + +- hybrid/BM25 retrieval; +- context pack APIs; +- code graph extraction; +- autonomous editing behavior; +- broad provider or model changes; +- production-ready claims. + +## Definitions + +A citation is **verified** only if all are true: + +1. the parsed citation path matches a retrieved chunk path; +2. the citation line range is valid (`start_line >= 1`, `end_line >= start_line`); +3. the citation line range overlaps at least one retrieved chunk from the same file; +4. the cited chunk belongs to the current query's retrieved evidence set. + +A citation is **unverified** if parsing succeeds but any verification condition fails. + +A response is `grounded=True` only when: + +- at least one citation is present; +- at least one citation is verified; +- no parsed citation is unverified. + +No-citation responses remain `grounded=False`. + +## Acceptance Criteria + +### M3-AC1: Citation verifier + +- Add a deterministic verifier that accepts parsed `Citation` objects and current `RetrievedChunk` evidence. +- It returns per-citation verification status and reason. +- It detects hallucinated paths, non-overlapping line ranges, invalid ranges, and stale/non-retrieved citations. + +### M3-AC2: Grounded semantics + +- `ResponseGenerator.generate()` must compute `grounded` from verifier output, not from `len(citations) > 0 and len(chunks) > 0`. +- A response with any unverified parsed citation is not grounded. +- A response with verified citations is grounded. +- A response with no citations remains not grounded. + +### M3-AC3: Adapter exposure + +- REST `QueryResponse` exposes citation verification status without removing existing fields. +- MCP query result exposes citation verification status without removing existing keys. +- CLI/UI may display a basic verified/unverified status, but must not invent new behavior beyond M3. + +### M3-AC4: EvalFly and tests + +- Add citation verifier unit tests with negative cases for: + - hallucinated path; + - out-of-range/non-overlapping lines; + - invalid line range; + - mixed verified and unverified citations. +- Add generator/service tests proving `grounded` follows verifier results. +- Extend committed EvalFly smoke with deterministic citation-engine checks. + +### M3-AC5: Governance + +- Keep all commits on `feat/m3-verified-citations` with `Spec-Slice: CRAG-20260628-M3-VERIFIED-CITATIONS` trailers. +- Push after every commit using the guarded push wrapper. +- Open PR with guarded GitHub wrapper only after local verification and review. + +## Required Evidence + +Before PR: + +```bash +bun run /Users/sebastian/.omp/omp-pantheon/skills/specsafe/bin/specsafe.ts status +PYTHONPATH=src .venv/bin/python -m pytest tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py +.venv/bin/python -m ruff check src/coderag/generation src/coderag/models src/coderag/services src/coderag/api/routes.py src/coderag/api/schemas.py src/coderag/mcp/handlers.py src/coderag/ui/handlers.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py scripts/evalfly_smoke.py +PYTHONPATH=src .venv/bin/python -m mypy src/coderag/generation src/coderag/models src/coderag/services src/coderag/api/routes.py src/coderag/api/schemas.py src/coderag/mcp/handlers.py src/coderag/ui/handlers.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py scripts/evalfly_smoke.py +.venv/bin/python scripts/evalfly_smoke.py --suite smoke +.venv/bin/python -m py_compile src/coderag/generation/*.py src/coderag/models/*.py src/coderag/services/*.py src/coderag/api/routes.py src/coderag/api/schemas.py src/coderag/mcp/handlers.py src/coderag/ui/handlers.py tests/test_citations.py tests/test_m3_citations.py +.venv/bin/python -m build +``` + +## Non-goals + +- Do not require reading the live filesystem during query verification. M3 verifies citations against retrieved evidence, not against mutable working tree state. +- Do not claim stale working-tree detection unless file snapshot hashes are actually checked in the query path. +- Do not change retrieval ranking or chunking behavior. From 88954b6af8d7629a5948c5c07d186a0bd54a0d6b Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 22:38:46 -0600 Subject: [PATCH 2/7] feat: add verified citation response model Spec-Slice: CRAG-20260628-M3-VERIFIED-CITATIONS --- src/coderag/models/__init__.py | 4 ++-- src/coderag/models/chunk.py | 22 +++++++++---------- src/coderag/models/document.py | 7 +++---- src/coderag/models/repository.py | 16 +++++++------- src/coderag/models/response.py | 36 +++++++++++++++++++++++++++++--- 5 files changed, 57 insertions(+), 28 deletions(-) diff --git a/src/coderag/models/__init__.py b/src/coderag/models/__init__.py index dedc597..d51e33a 100644 --- a/src/coderag/models/__init__.py +++ b/src/coderag/models/__init__.py @@ -1,9 +1,9 @@ """Models module: Core dataclasses and entities.""" -from coderag.models.document import Document, DocumentMetadata from coderag.models.chunk import Chunk, ChunkMetadata, ChunkType -from coderag.models.response import Query, Response, Citation, RetrievedChunk +from coderag.models.document import Document, DocumentMetadata from coderag.models.repository import Repository, RepositoryStatus +from coderag.models.response import Citation, Query, Response, RetrievedChunk __all__ = [ "Document", diff --git a/src/coderag/models/chunk.py b/src/coderag/models/chunk.py index 8937037..6ad73f0 100644 --- a/src/coderag/models/chunk.py +++ b/src/coderag/models/chunk.py @@ -2,11 +2,11 @@ from dataclasses import dataclass, field from enum import Enum -from typing import Optional +from typing import Any from uuid import uuid4 -class ChunkType(str, Enum): +class ChunkType(str, Enum): # noqa: UP042 """Type of code chunk.""" FUNCTION = "function" @@ -26,11 +26,11 @@ class ChunkMetadata: start_line: int end_line: int chunk_type: ChunkType - language: Optional[str] = None - name: Optional[str] = None - signature: Optional[str] = None - docstring: Optional[str] = None - parent_name: Optional[str] = None + language: str | None = None + name: str | None = None + signature: str | None = None + docstring: str | None = None + parent_name: str | None = None @property def line_range(self) -> str: @@ -51,7 +51,7 @@ class Chunk: metadata: ChunkMetadata repo_id: str id: str = field(default_factory=lambda: str(uuid4())) - embedding: Optional[list[float]] = None + embedding: list[float] | None = None @property def file_path(self) -> str: @@ -74,7 +74,7 @@ def chunk_type(self) -> ChunkType: return self.metadata.chunk_type @property - def name(self) -> Optional[str]: + def name(self) -> str | None: """Convenience accessor for name.""" return self.metadata.name @@ -83,7 +83,7 @@ def citation(self) -> str: """Get citation format.""" return self.metadata.citation - def to_dict(self) -> dict: + def to_dict(self) -> dict[str, Any]: """Convert to dictionary for storage.""" return { "id": self.id, @@ -101,7 +101,7 @@ def to_dict(self) -> dict: } @classmethod - def from_dict(cls, data: dict, embedding: Optional[list[float]] = None) -> "Chunk": + def from_dict(cls, data: dict[str, Any], embedding: list[float] | None = None) -> "Chunk": """Create from dictionary.""" metadata = ChunkMetadata( file_path=data["file_path"], diff --git a/src/coderag/models/document.py b/src/coderag/models/document.py index 3bb7707..24de63f 100644 --- a/src/coderag/models/document.py +++ b/src/coderag/models/document.py @@ -2,7 +2,6 @@ from dataclasses import dataclass from pathlib import Path -from typing import Optional @dataclass @@ -10,7 +9,7 @@ class DocumentMetadata: """Metadata for a source document.""" file_path: str - language: Optional[str] = None + language: str | None = None size_bytes: int = 0 line_count: int = 0 encoding: str = "utf-8" @@ -35,7 +34,7 @@ def file_path(self) -> str: return self.metadata.file_path @property - def language(self) -> Optional[str]: + def language(self) -> str | None: """Convenience accessor for language.""" return self.metadata.language @@ -58,7 +57,7 @@ def from_file(cls, file_path: Path, repo_root: Path, repo_id: str = "") -> "Docu return cls(content=content, metadata=metadata, repo_id=repo_id) -def _detect_language(extension: str) -> Optional[str]: +def _detect_language(extension: str) -> str | None: """Detect programming language from file extension.""" extension_map = { ".py": "python", diff --git a/src/coderag/models/repository.py b/src/coderag/models/repository.py index 09042e3..d0577b7 100644 --- a/src/coderag/models/repository.py +++ b/src/coderag/models/repository.py @@ -4,11 +4,11 @@ from datetime import datetime from enum import Enum from pathlib import Path -from typing import Optional +from typing import Any from uuid import uuid4 -class RepositoryStatus(str, Enum): +class RepositoryStatus(str, Enum): # noqa: UP042 """Repository indexing status.""" PENDING = "pending" @@ -25,12 +25,12 @@ class Repository: url: str branch: str = "main" id: str = field(default_factory=lambda: str(uuid4())) - clone_path: Optional[Path] = None - indexed_at: Optional[datetime] = None + clone_path: Path | None = None + indexed_at: datetime | None = None chunk_count: int = 0 status: RepositoryStatus = RepositoryStatus.PENDING - error_message: Optional[str] = None - last_commit: Optional[str] = None # SHA of last indexed commit (for incremental updates) + error_message: str | None = None + last_commit: str | None = None # SHA of last indexed commit (for incremental updates) @property def name(self) -> str: @@ -48,7 +48,7 @@ def full_name(self) -> str: """Get owner/repo format.""" return f"{self.owner}/{self.name}" - def to_dict(self) -> dict: + def to_dict(self) -> dict[str, Any]: """Convert to dictionary for serialization.""" return { "id": self.id, @@ -63,7 +63,7 @@ def to_dict(self) -> dict: } @classmethod - def from_dict(cls, data: dict) -> "Repository": + def from_dict(cls, data: dict[str, Any]) -> "Repository": """Create from dictionary.""" return cls( id=data["id"], diff --git a/src/coderag/models/response.py b/src/coderag/models/response.py index 8b02c69..2efbc5e 100644 --- a/src/coderag/models/response.py +++ b/src/coderag/models/response.py @@ -2,7 +2,6 @@ from dataclasses import dataclass, field from datetime import datetime -from typing import Optional from uuid import uuid4 @@ -19,7 +18,7 @@ def __str__(self) -> str: return f"[{self.file_path}:{self.start_line}-{self.end_line}]" @classmethod - def parse(cls, citation_str: str) -> Optional["Citation"]: + def parse(cls, citation_str: str) -> "Citation" | None: """Parse citation from string format [file:start-end].""" try: citation_str = citation_str.strip("[]") @@ -51,7 +50,7 @@ class RetrievedChunk: end_line: int relevance_score: float chunk_type: str - name: Optional[str] = None + name: str | None = None @property def citation(self) -> str: @@ -59,6 +58,36 @@ def citation(self) -> str: return f"[{self.file_path}:{self.start_line}-{self.end_line}]" +@dataclass +class CitationVerification: + """Verification status for a parsed citation.""" + + file_path: str + start_line: int + end_line: int + verified: bool + reason: str + chunk_id: str | None = None + + @classmethod + def from_citation( + cls, + citation: Citation, + *, + verified: bool, + reason: str, + chunk_id: str | None = None, + ) -> "CitationVerification": + return cls( + file_path=citation.file_path, + start_line=citation.start_line, + end_line=citation.end_line, + verified=verified, + reason=reason, + chunk_id=chunk_id, + ) + + @dataclass class Query: """A user's question about a repository.""" @@ -80,6 +109,7 @@ class Response: grounded: bool query_id: str = "" confidence_score: float = 0.0 + citation_verifications: list[CitationVerification] = field(default_factory=list) @property def has_evidence(self) -> bool: From 5e7a248e384f4394a528771999028759bd6e4a1c Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 22:39:26 -0600 Subject: [PATCH 3/7] feat: verify citations against retrieved chunks Spec-Slice: CRAG-20260628-M3-VERIFIED-CITATIONS --- src/coderag/generation/__init__.py | 13 +- src/coderag/generation/citations.py | 48 ++++++- src/coderag/generation/generator.py | 37 ++--- tests/test_citations.py | 4 +- tests/test_m3_citations.py | 215 ++++++++++++++++++++++++++++ 5 files changed, 291 insertions(+), 26 deletions(-) create mode 100644 tests/test_m3_citations.py diff --git a/src/coderag/generation/__init__.py b/src/coderag/generation/__init__.py index 9a1fea6..f77ad53 100644 --- a/src/coderag/generation/__init__.py +++ b/src/coderag/generation/__init__.py @@ -1,7 +1,14 @@ """Generation module: LLM inference and response generation with citations.""" -from coderag.generation.generator import ResponseGenerator +from coderag.generation.citations import CitationParser, CitationVerifier from coderag.generation.prompts import SYSTEM_PROMPT, build_prompt -from coderag.generation.citations import CitationParser -__all__ = ["ResponseGenerator", "SYSTEM_PROMPT", "build_prompt", "CitationParser"] +__all__ = ["CitationParser", "CitationVerifier", "ResponseGenerator", "SYSTEM_PROMPT", "build_prompt"] + + +def __getattr__(name: str) -> object: + if name == "ResponseGenerator": + from coderag.generation.generator import ResponseGenerator + + return ResponseGenerator + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/coderag/generation/citations.py b/src/coderag/generation/citations.py index 6097184..b0c4e18 100644 --- a/src/coderag/generation/citations.py +++ b/src/coderag/generation/citations.py @@ -1,9 +1,53 @@ """Citation parsing and formatting.""" import re -from typing import Optional +from dataclasses import dataclass -from coderag.models.response import Citation +from coderag.models.response import Citation, CitationVerification, RetrievedChunk + + +@dataclass(frozen=True) +class CitationVerificationResult: + """Verification summary for a response's citations.""" + + verifications: list[CitationVerification] + grounded: bool + + +class CitationVerifier: + """Verifies citations against retrieved evidence chunks.""" + + def verify( + self, + citations: list[Citation], + retrieved_chunks: list[RetrievedChunk], + ) -> CitationVerificationResult: + verifications = [self.verify_one(citation, retrieved_chunks) for citation in citations] + grounded = bool(verifications) and all(verification.verified for verification in verifications) + return CitationVerificationResult(verifications=verifications, grounded=grounded) + + def verify_one(self, citation: Citation, retrieved_chunks: list[RetrievedChunk]) -> CitationVerification: + if citation.start_line < 1 or citation.end_line < citation.start_line: + return CitationVerification.from_citation(citation, verified=False, reason="invalid_line_range") + + chunks_for_path = [chunk for chunk in retrieved_chunks if chunk.file_path == citation.file_path] + if not chunks_for_path: + return CitationVerification.from_citation(citation, verified=False, reason="path_not_retrieved") + + for chunk in chunks_for_path: + if self._ranges_overlap(citation.start_line, citation.end_line, chunk.start_line, chunk.end_line): + return CitationVerification.from_citation( + citation, + verified=True, + reason="verified", + chunk_id=chunk.chunk_id, + ) + + return CitationVerification.from_citation(citation, verified=False, reason="line_range_not_retrieved") + + @staticmethod + def _ranges_overlap(start_a: int, end_a: int, start_b: int, end_b: int) -> bool: + return start_a <= end_b and start_b <= end_a class CitationParser: diff --git a/src/coderag/generation/generator.py b/src/coderag/generation/generator.py index b37218e..d72df9f 100644 --- a/src/coderag/generation/generator.py +++ b/src/coderag/generation/generator.py @@ -1,13 +1,13 @@ """Response generation using local or remote LLMs.""" -from typing import Optional +from typing import Any from coderag.config import get_settings -from coderag.generation.citations import CitationParser -from coderag.generation.prompts import SYSTEM_PROMPT, build_prompt, build_no_context_response +from coderag.generation.citations import CitationParser, CitationVerifier +from coderag.generation.prompts import SYSTEM_PROMPT, build_no_context_response, build_prompt from coderag.logging import get_logger -from coderag.models.response import Response from coderag.models.query import Query +from coderag.models.response import Response from coderag.retrieval.retriever import Retriever logger = get_logger(__name__) @@ -18,21 +18,21 @@ class ResponseGenerator: def __init__( self, - retriever: Optional[Retriever] = None, + retriever: Retriever | None = None, ) -> None: self.settings = get_settings() self.retriever = retriever or Retriever() self.citation_parser = CitationParser() + self.citation_verifier = CitationVerifier() self.provider = self.settings.models.llm_provider.lower() - self._client = None - self._local_model = None - self._local_tokenizer = None + self._client: Any | None = None + self._local_model: Any | None = None + self._local_tokenizer: Any | None = None logger.info("ResponseGenerator initialized", provider=self.provider) - def _get_api_client(self): - """Get or create API client for remote providers.""" + def _get_api_client(self) -> Any: if self._client is not None: return self._client @@ -88,7 +88,7 @@ def _get_api_client(self): logger.info("API client created", provider=self.provider, model=self.model_name) return self._client - def _load_local_model(self): + def _load_local_model(self) -> None: """Load local model with transformers.""" if self._local_model is not None: return @@ -158,18 +158,17 @@ def generate(self, query: Query) -> Response: else: answer = self._generate_api(prompt) - # Parse citations from answer + # Parse and verify citations from answer citations = self.citation_parser.parse_citations(answer) - - # Determine if response is grounded - grounded = len(citations) > 0 and len(chunks) > 0 + verification_result = self.citation_verifier.verify(citations, chunks) return Response( answer=answer, citations=citations, retrieved_chunks=chunks, - grounded=grounded, + grounded=verification_result.grounded, query_id=query.id, + citation_verifications=verification_result.verifications, ) def _generate_api(self, prompt: str) -> str: @@ -189,7 +188,7 @@ def _generate_api(self, prompt: str) -> str: top_p=self.settings.models.llm_top_p, ) - return response.choices[0].message.content.strip() + return str(response.choices[0].message.content).strip() def _generate_local(self, prompt: str) -> str: """Generate using local model.""" @@ -197,6 +196,8 @@ def _generate_local(self, prompt: str) -> str: self._load_local_model() + assert self._local_tokenizer is not None + assert self._local_model is not None messages = [ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": prompt}, @@ -223,7 +224,7 @@ def _generate_local(self, prompt: str) -> str: generated = outputs[0][inputs["input_ids"].shape[1]:] response = self._local_tokenizer.decode(generated, skip_special_tokens=True) - return response.strip() + return str(response).strip() def unload(self) -> None: """Unload models from memory.""" diff --git a/tests/test_citations.py b/tests/test_citations.py index 3f54280..551f622 100644 --- a/tests/test_citations.py +++ b/tests/test_citations.py @@ -1,9 +1,7 @@ """Tests for citation parsing.""" - -import pytest +# mypy: disable-error-code="no-untyped-def" from coderag.generation.citations import CitationParser -from coderag.models.response import Citation def test_parse_single_citation(): diff --git a/tests/test_m3_citations.py b/tests/test_m3_citations.py new file mode 100644 index 0000000..9f6b0f6 --- /dev/null +++ b/tests/test_m3_citations.py @@ -0,0 +1,215 @@ +"""Milestone 3 verified citation engine tests.""" +# mypy: disable-error-code="no-untyped-def,no-untyped-call,method-assign,attr-defined" +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType + +from click.testing import CliRunner + +from coderag.generation.citations import CitationParser, CitationVerifier +from coderag.models.query import Query +from coderag.models.response import Citation, RetrievedChunk + +REPO_ROOT = Path(__file__).resolve().parents[1] +CITATIONS_PATH = REPO_ROOT / "src" / "coderag" / "generation" / "citations.py" + + +def _citation_verifier_class(): + spec = importlib.util.spec_from_file_location("coderag_generation_citations_under_test", CITATIONS_PATH) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module.CitationVerifier + + +def chunk( + file_path: str = "src/auth.py", + start_line: int = 10, + end_line: int = 20, + chunk_id: str = "chunk-1", +) -> RetrievedChunk: + return RetrievedChunk( + chunk_id=chunk_id, + content="def authenticate():\n return True", + file_path=file_path, + start_line=start_line, + end_line=end_line, + relevance_score=0.95, + chunk_type="function", + name="authenticate", + ) + + +def test_citation_verifier_marks_overlapping_retrieved_chunk_verified(): + verifier = _citation_verifier_class()() + + result = verifier.verify( + [Citation(file_path="src/auth.py", start_line=12, end_line=15)], + [chunk()], + ) + + assert result.grounded is True + assert result.verifications[0].verified is True + assert result.verifications[0].reason == "verified" + assert result.verifications[0].chunk_id == "chunk-1" + + +def test_citation_verifier_rejects_hallucinated_path(): + verifier = _citation_verifier_class()() + + result = verifier.verify( + [Citation(file_path="src/missing.py", start_line=12, end_line=15)], + [chunk()], + ) + + assert result.grounded is False + assert result.verifications[0].verified is False + assert result.verifications[0].reason == "path_not_retrieved" + + +def test_citation_verifier_rejects_non_overlapping_lines(): + verifier = _citation_verifier_class()() + + result = verifier.verify( + [Citation(file_path="src/auth.py", start_line=30, end_line=40)], + [chunk()], + ) + + assert result.grounded is False + assert result.verifications[0].verified is False + assert result.verifications[0].reason == "line_range_not_retrieved" + + +def test_citation_verifier_rejects_invalid_line_range(): + verifier = _citation_verifier_class()() + + result = verifier.verify( + [Citation(file_path="src/auth.py", start_line=20, end_line=10)], + [chunk()], + ) + + assert result.grounded is False + assert result.verifications[0].verified is False + assert result.verifications[0].reason == "invalid_line_range" + + +def test_citation_verifier_requires_all_citations_to_be_verified_for_grounding(): + verifier = _citation_verifier_class()() + + result = verifier.verify( + [ + Citation(file_path="src/auth.py", start_line=12, end_line=15), + Citation(file_path="src/missing.py", start_line=1, end_line=2), + ], + [chunk()], + ) + + assert result.grounded is False + assert [verification.verified for verification in result.verifications] == [True, False] + + +def test_citation_verifier_returns_not_grounded_for_no_citations(): + verifier = _citation_verifier_class()() + + result = verifier.verify([], [chunk()]) + + assert result.grounded is False + assert result.verifications == [] + + +class FakeRetriever: + def __init__(self, retrieved_chunks: list[RetrievedChunk]) -> None: + self.retrieved_chunks = retrieved_chunks + + def retrieve_with_context(self, question: str, repo_id: str, top_k: int): + _ = (question, repo_id, top_k) + return self.retrieved_chunks, "retrieved context" + + +def _response_generator_class(monkeypatch): + retriever_module = ModuleType("coderag.retrieval.retriever") + retriever_module.Retriever = object + monkeypatch.setitem(sys.modules, "coderag.retrieval.retriever", retriever_module) + + generator_path = REPO_ROOT / "src" / "coderag" / "generation" / "generator.py" + spec = importlib.util.spec_from_file_location("coderag_generation_generator_under_test", generator_path) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module.ResponseGenerator + + +def test_response_generator_grounds_only_when_all_citations_verify(monkeypatch): + ResponseGenerator = _response_generator_class(monkeypatch) + generator = ResponseGenerator.__new__(ResponseGenerator) + generator.retriever = FakeRetriever([chunk()]) + generator.citation_parser = CitationParser() + generator.citation_verifier = CitationVerifier() + generator.provider = "stub" + generator._generate_api = lambda _prompt: "Uses auth [src/auth.py:12-15] and missing [src/missing.py:1-2]." + + result = generator.generate(Query(question="How?", repo_id="repo-1")) + + assert result.grounded is False + assert [verification.verified for verification in result.citation_verifications] == [True, False] + + +def test_cli_query_surfaces_citation_verification_status(): + cli_source = (REPO_ROOT / "src" / "coderag" / "cli.py").read_text() + + assert "citation_verifications" in cli_source + assert "verified" in cli_source + assert "unverified" in cli_source + + + +class FakeCliHandlers: + async def query_code(self, repo_id: str, question: str, top_k: int = 5): + _ = (repo_id, question, top_k) + return { + "answer": "Answer", + "citations": [{"file_path": "src/app.py", "start_line": 1, "end_line": 2}], + "citation_verifications": [ + {"file_path": "src/app.py", "start_line": 1, "end_line": 2, "verified": True, "reason": "verified"}, + { + "file_path": "src/missing.py", + "start_line": 1, + "end_line": 2, + "verified": False, + "reason": "path_not_retrieved", + }, + ], + "evidence": [], + "grounded": False, + } + + +def test_cli_query_json_output_includes_serialized_citation_verifications(monkeypatch): + import coderag.mcp.handlers as mcp_handlers + from coderag.cli import cli + + monkeypatch.setattr(mcp_handlers, "get_mcp_handlers", lambda: FakeCliHandlers()) + + result = CliRunner().invoke(cli, ["query", "repo-1", "How?", "--format", "json"]) + + assert result.exit_code == 0 + payload_text = result.output.split("\n", 2)[2] + payload = json.loads(payload_text) + assert payload["citation_verifications"][1]["reason"] == "path_not_retrieved" + + +def test_cli_query_text_output_marks_verified_and_unverified_citations(monkeypatch): + import coderag.mcp.handlers as mcp_handlers + from coderag.cli import cli + + monkeypatch.setattr(mcp_handlers, "get_mcp_handlers", lambda: FakeCliHandlers()) + + result = CliRunner().invoke(cli, ["query", "repo-1", "How?"]) + + assert result.exit_code == 0 + assert "src/app.py:1-2 [verified: verified]" in result.output + assert "src/missing.py:1-2 [unverified: path_not_retrieved]" in result.output From 3f678dcced4e9f72876eec3037dd4029c5f2b29b Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 22:40:02 -0600 Subject: [PATCH 4/7] feat: expose citation verification in APIs Spec-Slice: CRAG-20260628-M3-VERIFIED-CITATIONS --- src/coderag/api/routes.py | 29 +++++++++++++++++++++++-- src/coderag/api/schemas.py | 43 +++++++++++++++++++++++-------------- src/coderag/mcp/handlers.py | 32 ++++++++++++++++++++++++++- tests/test_api_routes.py | 9 ++++++++ tests/test_mcp_handlers.py | 32 +++++++++++++++++++++++---- 5 files changed, 122 insertions(+), 23 deletions(-) diff --git a/src/coderag/api/routes.py b/src/coderag/api/routes.py index ff9c243..bf7cf0c 100644 --- a/src/coderag/api/routes.py +++ b/src/coderag/api/routes.py @@ -1,9 +1,12 @@ """REST API routes.""" +from typing import Any + from fastapi import APIRouter, BackgroundTasks, HTTPException from coderag.api.schemas import ( CitationResponse, + CitationVerificationResponse, IndexRepositoryRequest, IndexRepositoryResponse, ListRepositoriesResponse, @@ -13,6 +16,7 @@ RetrievedChunkResponse, ) from coderag.logging import get_logger +from coderag.models.repository import Repository from coderag.services.indexing import IndexingOptions, IndexingService from coderag.services.registry import RepositoryRegistry from coderag.services.retrieval import RetrievalService @@ -25,7 +29,7 @@ retrieval_service = RetrievalService(registry=registry) -def get_repo_or_404(repo_id: str): +def get_repo_or_404(repo_id: str) -> Repository: """Get a repository by full ID or unambiguous prefix, raising 404 if not found.""" repo = registry.get_unique(repo_id) if repo is None: @@ -33,6 +37,23 @@ def get_repo_or_404(repo_id: str): return repo +def _read_field(item: Any, field: str, default: Any = None) -> Any: + if isinstance(item, dict): + return item.get(field, default) + return getattr(item, field, default) + + +def _citation_verification_response(item: Any) -> CitationVerificationResponse: + return CitationVerificationResponse( + file_path=_read_field(item, "file_path"), + start_line=_read_field(item, "start_line"), + end_line=_read_field(item, "end_line"), + verified=_read_field(item, "verified"), + reason=_read_field(item, "reason"), + chunk_id=_read_field(item, "chunk_id"), + ) + + async def index_repository_task( url: str, repo_id: str, @@ -119,6 +140,10 @@ async def query_repository(request: QueryRequest) -> QueryResponse: ) for c in response.citations ], + citation_verifications=[ + _citation_verification_response(verification) + for verification in getattr(response, "citation_verifications", []) + ], retrieved_chunks=[ RetrievedChunkResponse( chunk_id=c.chunk_id, @@ -172,7 +197,7 @@ async def get_repository(repo_id: str) -> RepositoryInfo: @router.delete("/repos/{repo_id}") -async def delete_repository(repo_id: str) -> dict: +async def delete_repository(repo_id: str) -> dict[str, str]: """Delete a repository by full ID or prefix.""" repo = get_repo_or_404(repo_id) try: diff --git a/src/coderag/api/schemas.py b/src/coderag/api/schemas.py index b3834e0..b920ff2 100644 --- a/src/coderag/api/schemas.py +++ b/src/coderag/api/schemas.py @@ -1,18 +1,17 @@ """Pydantic schemas for REST API.""" from datetime import datetime -from typing import Optional -from pydantic import BaseModel, Field, HttpUrl +from pydantic import BaseModel, ConfigDict, Field class IndexRepositoryRequest(BaseModel): """Request to index a repository.""" url: str = Field(..., description="GitHub repository URL") - branch: Optional[str] = Field(None, description="Branch name (default: main)") - include_patterns: Optional[list[str]] = Field(None, description="File patterns to include") - exclude_patterns: Optional[list[str]] = Field(None, description="File patterns to exclude") + branch: str | None = Field(None, description="Branch name (default: main)") + include_patterns: list[str] | None = Field(None, description="File patterns to include") + exclude_patterns: list[str] | None = Field(None, description="File patterns to exclude") class IndexRepositoryResponse(BaseModel): @@ -38,8 +37,19 @@ class CitationResponse(BaseModel): start_line: int end_line: int - class Config: - from_attributes = True + model_config = ConfigDict(from_attributes=True) + +class CitationVerificationResponse(BaseModel): + """Verification status for a parsed citation.""" + + file_path: str + start_line: int + end_line: int + verified: bool + reason: str + chunk_id: str | None = None + + model_config = ConfigDict(from_attributes=True) class RetrievedChunkResponse(BaseModel): @@ -51,11 +61,9 @@ class RetrievedChunkResponse(BaseModel): end_line: int relevance_score: float chunk_type: str - name: Optional[str] = None + name: str | None = None content: str - - class Config: - from_attributes = True + model_config = ConfigDict(from_attributes=True) class QueryResponse(BaseModel): @@ -63,11 +71,14 @@ class QueryResponse(BaseModel): answer: str = Field(..., description="Generated answer") citations: list[CitationResponse] = Field(..., description="Citations in the answer") + citation_verifications: list[CitationVerificationResponse] = Field( + default_factory=list, + description="Verification status for each parsed citation", + ) retrieved_chunks: list[RetrievedChunkResponse] = Field(..., description="Evidence chunks") - grounded: bool = Field(..., description="Whether response is grounded in evidence") + grounded: bool = Field(..., description="Whether every parsed citation is verified against retrieved evidence") query_id: str = Field(..., description="Query ID") - class RepositoryInfo(BaseModel): """Repository information.""" @@ -76,8 +87,8 @@ class RepositoryInfo(BaseModel): branch: str chunk_count: int status: str - indexed_at: Optional[datetime] = None - error_message: Optional[str] = None + indexed_at: datetime | None = None + error_message: str | None = None class ListRepositoriesResponse(BaseModel): @@ -98,4 +109,4 @@ class ErrorResponse(BaseModel): """Error response.""" error: str - detail: Optional[str] = None + detail: str | None = None diff --git a/src/coderag/mcp/handlers.py b/src/coderag/mcp/handlers.py index 7f1f966..550dd82 100644 --- a/src/coderag/mcp/handlers.py +++ b/src/coderag/mcp/handlers.py @@ -11,6 +11,31 @@ logger = get_logger(__name__) +def _read_field(item: Any, field: str, default: Any = None) -> Any: + if isinstance(item, dict): + return item.get(field, default) + return getattr(item, field, default) + + +def _citation_dict(item: Any) -> dict[str, Any]: + return { + "file_path": _read_field(item, "file_path"), + "start_line": _read_field(item, "start_line"), + "end_line": _read_field(item, "end_line"), + } + + +def _citation_verification_dict(item: Any) -> dict[str, Any]: + return { + "file_path": _read_field(item, "file_path"), + "start_line": _read_field(item, "start_line"), + "end_line": _read_field(item, "end_line"), + "verified": _read_field(item, "verified"), + "reason": _read_field(item, "reason"), + "chunk_id": _read_field(item, "chunk_id"), + } + + class MCPHandlers: """Handlers for MCP tools backed by shared application services.""" @@ -82,6 +107,7 @@ async def query_code( "answer": "", "citations": [], "evidence": [], + "citation_verifications": [], "grounded": False, "error": result.error or "Query failed", } @@ -89,7 +115,7 @@ async def query_code( response = result.response return { "answer": response.answer, - "citations": response.citations, + "citations": [_citation_dict(citation) for citation in response.citations], "evidence": [ { "file": chunk.file_path, @@ -101,6 +127,10 @@ async def query_code( for chunk in response.retrieved_chunks ], "grounded": response.grounded, + "citation_verifications": [ + _citation_verification_dict(verification) + for verification in getattr(response, "citation_verifications", []) + ], } async def list_repositories(self) -> dict[str, Any]: diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index d22ea05..544c81a 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -86,10 +86,16 @@ class FakeGeneratedResponse: retrieved_chunks: list = None grounded: bool = False query_id: str = "query-1" + citation_verifications: list = None def __post_init__(self) -> None: self.citations = [] if self.citations is None else self.citations self.retrieved_chunks = [] if self.retrieved_chunks is None else self.retrieved_chunks + self.citation_verifications = ( + [{"file_path": "src/app.py", "start_line": 1, "end_line": 2, "verified": True, "reason": "verified"}] + if self.citation_verifications is None + else self.citation_verifications + ) class FakeRetrievalService: @@ -136,6 +142,9 @@ async def test_query_route_uses_retrieval_service_and_preserves_response(monkeyp assert response.answer == "shared answer" assert response.query_id == "query-1" assert retrieval.calls == [("repo-ready", "How?", 3)] + assert response.citation_verifications[0].file_path == "src/app.py" + assert response.citation_verifications[0].verified is True + assert response.citation_verifications[0].reason == "verified" @pytest.mark.asyncio diff --git a/tests/test_mcp_handlers.py b/tests/test_mcp_handlers.py index fdad52b..f4855be 100644 --- a/tests/test_mcp_handlers.py +++ b/tests/test_mcp_handlers.py @@ -1,5 +1,6 @@ """Unit tests for service-backed MCP handlers.""" - +# mypy: disable-error-code="no-untyped-def,arg-type,no-untyped-call,index,union-attr" +import json from datetime import datetime from types import SimpleNamespace @@ -7,6 +8,7 @@ from coderag.mcp.handlers import MCPHandlers, get_mcp_handlers from coderag.models.repository import Repository, RepositoryStatus +from coderag.models.response import Citation, CitationVerification from coderag.services.registry import RepositoryRegistry @@ -59,7 +61,15 @@ def query_code(self, repo_id: str, question: str, top_k: int = 5): return SimpleNamespace(response=None, repo=None, error=f"Repository not found: {repo_id}") if repo.status is not RepositoryStatus.READY: return SimpleNamespace(response=None, repo=repo, error=f"Repository not ready: status is {repo.status.value}") - response = SimpleNamespace(answer="Test answer", citations=[], retrieved_chunks=[], grounded=False) + response = SimpleNamespace( + answer="Test answer", + citations=[Citation(file_path="src/app.py", start_line=1, end_line=2)], + retrieved_chunks=[], + grounded=True, + citation_verifications=[ + CitationVerification(file_path="src/app.py", start_line=1, end_line=2, verified=True, reason="verified") + ], + ) return SimpleNamespace(response=response, repo=repo, error=None) def search_code(self, repo_id: str, query: str, top_k: int = 10, file_filter=None, chunk_type=None): @@ -223,9 +233,23 @@ async def test_query_code_success_shape(self, mcp_handlers, registry, sample_rep result = await mcp_handlers.query_code(sample_repository.id, "What does this do?", top_k=3) assert result["answer"] == "Test answer" - assert result["citations"] == [] + assert result["citations"] == [{"file_path": "src/app.py", "start_line": 1, "end_line": 2}] assert result["evidence"] == [] - assert result["grounded"] is False + assert result["grounded"] is True + assert result["citation_verifications"][0]["file_path"] == "src/app.py" + assert result["citation_verifications"][0]["verified"] is True + + @pytest.mark.asyncio + async def test_query_code_result_is_json_serializable_with_citations(self, mcp_handlers, registry, sample_repository): + registry.add(sample_repository) + + result = await mcp_handlers.query_code(sample_repository.id, "How does app work?") + + encoded = json.dumps(result) + decoded = json.loads(encoded) + assert decoded["citations"] == [{"file_path": "src/app.py", "start_line": 1, "end_line": 2}] + assert decoded["citation_verifications"][0]["reason"] == "verified" + assert result["citation_verifications"][0]["reason"] == "verified" From a4a7021b3deb728bafaefacedcee49780e28cb3f Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 22:40:36 -0600 Subject: [PATCH 5/7] feat: surface verified citation status in clients Spec-Slice: CRAG-20260628-M3-VERIFIED-CITATIONS --- src/coderag/cli.py | 10 ++++++++++ src/coderag/ui/grounding.py | 15 +++++++++++++++ src/coderag/ui/handlers.py | 7 ++++++- tests/test_ui_handlers.py | 19 +++++++++++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 src/coderag/ui/grounding.py diff --git a/src/coderag/cli.py b/src/coderag/cli.py index 68634ff..bb4d81a 100644 --- a/src/coderag/cli.py +++ b/src/coderag/cli.py @@ -372,6 +372,16 @@ async def run_query(): for citation in result["citations"]: click.echo(f" {citation}") + if result.get("citation_verifications"): + click.echo("\n🧾 Citation verification:") + for verification in result["citation_verifications"]: + status = "verified" if verification.get("verified") else "unverified" + path = verification.get("file_path") + start_line = verification.get("start_line") + end_line = verification.get("end_line") + reason = verification.get("reason") + click.echo(f" - {path}:{start_line}-{end_line} [{status}: {reason}]") + if result.get("evidence"): click.echo("\n📂 Evidence:") for chunk in result["evidence"][:3]: # Show top 3 diff --git a/src/coderag/ui/grounding.py b/src/coderag/ui/grounding.py new file mode 100644 index 0000000..0dcdbff --- /dev/null +++ b/src/coderag/ui/grounding.py @@ -0,0 +1,15 @@ +"""Grounding status formatting for UI surfaces.""" + + +def format_grounding_status( + *, + grounded: bool, + has_citations: bool, + has_citation_verifications: bool, +) -> str: + """Return user-facing grounding status from verified citation state.""" + if grounded: + return "Grounded (verified citations)" + if has_citations and has_citation_verifications: + return "Not grounded (unverified citations)" + return "Not grounded (no verified citations)" diff --git a/src/coderag/ui/handlers.py b/src/coderag/ui/handlers.py index 3f5d81f..f8289e4 100644 --- a/src/coderag/ui/handlers.py +++ b/src/coderag/ui/handlers.py @@ -24,6 +24,7 @@ from coderag.models.repository import Repository, RepositoryStatus from coderag.services.providers import ProviderConfigService from coderag.services.registry import RepositoryRegistry +from coderag.ui.grounding import format_grounding_status logger = get_logger(__name__) @@ -470,7 +471,11 @@ def ask_question( # Format evidence evidence_md = response.format_evidence() - status = "Grounded" if response.grounded else "Not grounded (no citations)" + status = format_grounding_status( + grounded=response.grounded, + has_citations=bool(response.citations), + has_citation_verifications=bool(response.citation_verifications), + ) return answer_md, evidence_md, status diff --git a/tests/test_ui_handlers.py b/tests/test_ui_handlers.py index b359812..a0ef316 100644 --- a/tests/test_ui_handlers.py +++ b/tests/test_ui_handlers.py @@ -9,6 +9,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] HANDLERS_PATH = REPO_ROOT / "src" / "coderag" / "ui" / "handlers.py" METADATA_PATH = REPO_ROOT / "src" / "coderag" / "ui" / "repository_metadata.py" +GROUNDING_PATH = REPO_ROOT / "src" / "coderag" / "ui" / "grounding.py" def _load_metadata_module(): @@ -50,6 +51,24 @@ def test_ui_ask_question_checks_provider_profile_before_generation_setup(): assert call_names.index("generation_block_reason") < call_names.index("generate") +def test_ui_grounding_status_uses_verified_citation_state(): + spec = importlib.util.spec_from_file_location("coderag_ui_grounding_under_test", GROUNDING_PATH) + assert spec is not None + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + format_grounding_status = module.format_grounding_status + + assert format_grounding_status(grounded=True, has_citations=True, has_citation_verifications=True) == "Grounded (verified citations)" + assert ( + format_grounding_status(grounded=False, has_citations=True, has_citation_verifications=True) + == "Not grounded (unverified citations)" + ) + assert ( + format_grounding_status(grounded=False, has_citations=False, has_citation_verifications=False) + == "Not grounded (no verified citations)" + ) + def test_ui_repository_state_uses_shared_registry_not_json_helpers(): module = ast.parse(HANDLERS_PATH.read_text()) imported_names = { From 31f4b0bd8fdb8ec0e639ac693910cc7199b8e9a8 Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 22:41:10 -0600 Subject: [PATCH 6/7] ci: extend Milestone 3 validation gates Spec-Slice: CRAG-20260628-M3-VERIFIED-CITATIONS --- .github/workflows/ci.yml | 14 +++---- evals/config.json | 88 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3bf711..3176a73 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,17 +25,17 @@ jobs: python -m pip install -e . --no-deps python -m pip install pytest pytest-asyncio pydantic pydantic-settings click PyYAML fastapi structlog ruff mypy mcp httpx - - name: Run Milestone 0-2 tests - run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py + - name: Run Milestone 0-3 tests + run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py - - name: Run Milestone 0-2 lint baseline - run: ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py + - name: Run Milestone 0-3 lint baseline + run: ruff check src/coderag/generation src/coderag/models src/coderag/services src/coderag/config.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py - - name: Run Milestone 0-2 type baseline - run: PYTHONPATH=src mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py + - name: Run Milestone 0-3 type baseline + run: PYTHONPATH=src mypy src/coderag/generation src/coderag/models src/coderag/services src/coderag/config.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_ui_handlers.py scripts/evalfly_smoke.py - name: Compile changed runtime modules - run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/generator.py src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py + run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/*.py src/coderag/models/*.py src/coderag/services/*.py src/coderag/api/schemas.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/grounding.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py - name: Build package run: python -m build diff --git a/evals/config.json b/evals/config.json index d27e761..1df9645 100644 --- a/evals/config.json +++ b/evals/config.json @@ -418,6 +418,94 @@ } ] } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m3-citation-verifier-exists", + "title": "Milestone 3 citation verifier implementation exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "grounding_verification", + "source": { + "kind": "specsafe_slice", + "slice_id": "CRAG-20260628-M3-VERIFIED-CITATIONS" + }, + "privacy": { + "contains_user_data": false, + "contains_code": true, + "data_classification": "public_repo" + }, + "expected": { + "artifact": "src/coderag/generation/citations.py" + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "src/coderag/generation/citations.py" + }, + { + "type": "file_contains", + "path": "src/coderag/generation/citations.py", + "text": "class CitationVerifier" + }, + { + "type": "file_contains", + "path": "src/coderag/generation/citations.py", + "text": "path_not_retrieved" + }, + { + "type": "file_contains", + "path": "src/coderag/generation/citations.py", + "text": "line_range_not_retrieved" + }, + { + "type": "file_contains", + "path": "src/coderag/generation/citations.py", + "text": "invalid_line_range" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m3-citation-tests-exist", + "title": "Milestone 3 citation verification tests exist", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "test_coverage", + "source": { + "kind": "repo_file", + "path": "tests/test_m3_citations.py" + }, + "privacy": { + "contains_user_data": false, + "contains_code": true, + "data_classification": "public_repo" + }, + "expected": { + "artifact": "tests/test_m3_citations.py" + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "tests/test_m3_citations.py" + }, + { + "type": "file_contains", + "path": "tests/test_m3_citations.py", + "text": "test_citation_verifier_requires_all_citations_to_be_verified_for_grounding" + }, + { + "type": "file_contains", + "path": "tests/test_m3_citations.py", + "text": "test_response_generator_grounds_only_when_all_citations_verify" + } + ] + } } ] } From f79f6fb4fe52bc4168b3ba9541313e18dc9027b7 Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 22:45:36 -0600 Subject: [PATCH 7/7] fix: preserve Python 3.11 citation annotations Spec-Slice: CRAG-20260628-M3-VERIFIED-CITATIONS --- src/coderag/models/response.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/coderag/models/response.py b/src/coderag/models/response.py index 2efbc5e..72fdfad 100644 --- a/src/coderag/models/response.py +++ b/src/coderag/models/response.py @@ -1,4 +1,5 @@ """Response entity models for Q&A results.""" +from __future__ import annotations from dataclasses import dataclass, field from datetime import datetime @@ -18,7 +19,7 @@ def __str__(self) -> str: return f"[{self.file_path}:{self.start_line}-{self.end_line}]" @classmethod - def parse(cls, citation_str: str) -> "Citation" | None: + def parse(cls, citation_str: str) -> Citation | None: """Parse citation from string format [file:start-end].""" try: citation_str = citation_str.strip("[]") @@ -77,7 +78,7 @@ def from_citation( verified: bool, reason: str, chunk_id: str | None = None, - ) -> "CitationVerification": + ) -> CitationVerification: return cls( file_path=citation.file_path, start_line=citation.start_line,