Milestone 2: SQLite registry and durable job state

.github/workflows/ci.yml

@@ -25,17 +25,17 @@ jobs:
           python -m pip install -e . --no-deps
           python -m pip install pytest pytest-asyncio pydantic pydantic-settings click PyYAML fastapi structlog ruff mypy mcp httpx
 
-      - name: Run Milestone 0 and 1 tests
-        run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py
+      - name: Run Milestone 0-2 tests
+        run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py
 
-      - name: Run Milestone 0 and 1 lint baseline
-        run: ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
+      - name: Run Milestone 0-2 lint baseline
+        run: ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
 
-      - name: Run Milestone 0 and 1 type baseline
-        run: PYTHONPATH=src mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py scripts/evalfly_smoke.py
+      - name: Run Milestone 0-2 type baseline
+        run: PYTHONPATH=src mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
 
       - name: Compile changed runtime modules
-        run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/generator.py src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
+        run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/generator.py src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
 
       - name: Build package
         run: python -m build

.pi/.specsafe-state.json

@@ -80,6 +80,26 @@
           "turns": 0
         }
       }
+    },
+    {
+      "sliceId": "CRAG-20260627-M2-SQLITE-REGISTRY",
+      "workspaceId": "CodeRAG",
+      "sessionId": "omp-session",
+      "beganAt": "2026-06-27T23:00:19.118Z",
+      "endedAt": "2026-06-28T03:35:40.222Z",
+      "outcome": "PASS",
+      "costSummary": {
+        "externalMemoryCalls": 0,
+        "externalMemoryCost": 0,
+        "subagentTokens": {
+          "input": 0,
+          "output": 0,
+          "cacheRead": 0,
+          "cacheWrite": 0,
+          "cost": 0,
+          "turns": 0
+        }
+      }
     }
   ]
 }

docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md

@@ -0,0 +1,125 @@
+# CodeRAG Milestone 2 SQLite Registry and Durable Job State Contract
+
+Slice: `CRAG-20260627-M2-SQLITE-REGISTRY`
+Parent plan: `docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md`
+Branch: `feat/m2-sqlite-registry-state`
+
+## Scope
+
+Milestone 2 replaces fragile JSON repository metadata with a SQLite-backed registry and durable job records while keeping the Milestone 1 service seam. This milestone is a persistence/state milestone: it must not introduce hybrid retrieval, verified citation semantics, graph extraction, context packs, or autonomous editing behavior.
+
+## Source of truth
+
+1. Roadmap AC-3 and Milestone 2 (`docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md`).
+2. Current `main` after PR #2 merge.
+3. Existing public API/MCP/CLI/UI repository state behavior unless this contract explicitly narrows it.
+4. Pre-1.0 alpha users may either receive a deterministic `repositories.json` migration or an explicit reset-path behavior with tests; this milestone chooses migration because the existing Repository model already serializes the current alpha metadata shape.
+
+## Frozen acceptance criteria
+
+### M2-AC1: SQLite registry schema and migration
+
+Pass requires:
+- `RepositoryRegistry` uses SQLite as the canonical store for repository metadata.
+- Schema creation is explicit and versioned enough for current migrations, not implicit ad-hoc table creation at each call site.
+- The registry tracks repo id, URL/path, branch, head SHA, indexed_at, status, error, file count, chunk count, and index/provider metadata fields.
+- Existing `repositories.json` entries are migrated into SQLite on first registry load when a SQLite DB does not already contain repositories.
+- Corrupt `repositories.json` still fails closed and is not overwritten or silently ignored during migration.
+
+Fail if:
+- Repository metadata remains canonical in a global dict or `repositories.json` after M2.
+- A valid existing `repositories.json` user loses repositories without an explicit tested reset path.
+- Migration hides corrupt JSON by creating an empty SQLite registry.
+
+Required evidence:
+- Unit tests using a temp SQLite DB for schema creation, add/update/list/get/remove, partial-id behavior, and reopen/restart persistence.
+- Migration test from a sample `repositories.json` into SQLite.
+- Corrupt JSON migration test proving fail-closed behavior.
+
+### M2-AC2: Durable indexing job state
+
+Pass requires:
+- Registry has durable job records for index/update/delete operations.
+- Job records include id, repo id, operation, status, error, started_at, finished_at, and counters for files/chunks when available.
+- `IndexingService` records job start/success/failure for full indexing and deletion.
+- Failed indexing records both repository error status and durable job error.
+- Restarting a process can read the last job state from SQLite.
+
+Fail if:
+- Job state only exists in memory or only in logs.
+- Indexing failure persists repository error but no job error.
+- Delete removes all evidence of the delete operation before callers can inspect its job state.
+
+Required evidence:
+- Unit tests proving successful index job, failed index job, and delete job state are persisted in temp SQLite.
+- Restart simulation test reopening the registry and reading the recorded jobs.
+
+### M2-AC3: Shared registry across API/MCP/CLI/UI
+
+Pass requires:
+- API, MCP, indexing/retrieval services, and UI repository listing/status paths use the same `RepositoryRegistry` abstraction.
+- Public response shapes remain compatible unless covered by a migration note and regression test.
+- REST destructive operations continue to reject ambiguous partial ids before delete/query/get service calls.
+- MCP first-match prefix compatibility from M1 remains unless a test documents an intentional compatibility change.
+
+Fail if:
+- UI still owns independent `repositories.json` load/save state for repository metadata.
+- API and MCP can report conflicting repository state after a process restart.
+- M1 partial-id ambiguity safety regresses.
+
+Required evidence:
+- API/MCP shared-state integration test using the same temp SQLite DB.
+- UI repository table/status test using the shared registry path.
+- Regression test for ambiguous REST delete/query/get behavior or shared helper coverage plus destructive delete test.
+
+### M2-AC4: Governance and evidence
+
+Pass requires:
+- SpecSafe slice `CRAG-20260627-M2-SQLITE-REGISTRY` is open before implementation edits and closed only after verification.
+- EvalFly smoke/config is extended with M2 deterministic registry/job-state checks, or an explicit committed reason documents why deeper state evals wait for a later fixture layer.
+- CI targeted tests include M2 registry/job-state tests.
+- Work is committed on `feat/m2-sqlite-registry-state`, pushed after every commit, and opened as a PR after local verification.
+
+Fail if:
+- Tests/evals are weakened, deleted, bypassed, or relabeled to pass.
+- M2 is declared production-ready or claims hybrid/citation/graph/context-pack capabilities.
+- The SQLite schema is only exercised through mocks that bypass the real registry persistence layer.
+
+Required evidence:
+- SpecSafe status showing the active slice.
+- Targeted pytest output for M0/M1/M2 tests and affected API/MCP/UI tests.
+- Ruff and mypy output for touched modules and tests.
+- `scripts/evalfly_smoke.py --suite smoke` output.
+- Package build output.
+- Post-implementation review against this contract.
+
+## Implementation boundaries
+
+Allowed:
+- Replace `RepositoryRegistry` internals with SQLite while preserving its public methods where practical.
+- Add registry/job dataclasses or enums inside `src/coderag/services/registry.py` or a focused sibling module.
+- Add SQLite schema/migration helpers.
+- Add tests under `tests/test_m2_registry.py` and update M1/API/MCP/UI tests to use temp SQLite DBs.
+- Route UI repository metadata listing/status through `RepositoryRegistry`.
+- Extend EvalFly smoke with deterministic static checks for SQLite registry/job-state artifacts.
+
+Not allowed in M2:
+- Hybrid/BM25 retrieval.
+- Verified citation semantics or `grounded=True` behavior changes.
+- Code graph extraction.
+- Context pack APIs.
+- Autonomous editing-agent behavior.
+
+## Evidence commands
+
+Run before PR:
+
+```bash
+bun run /Users/sebastian/.omp/omp-pantheon/skills/specsafe/bin/specsafe.ts status
+PYTHONPATH=src .venv/bin/python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py
+.venv/bin/python -m ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
+PYTHONPATH=src .venv/bin/python -m mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py
+.venv/bin/python scripts/evalfly_smoke.py --suite smoke
+.venv/bin/python -m py_compile src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py
+.venv/bin/python -m build
+```

evals/config.json

@@ -301,6 +301,123 @@
           }
         ]
       }
+    },
+    {
+      "schema_version": "evalfly.case.v1",
+      "case_id": "m2-sqlite-contract-exists",
+      "title": "Milestone 2 SQLite registry contract exists",
+      "suite": "smoke",
+      "risk_tier": "critical",
+      "task_type": "state_persistence",
+      "source": {
+        "kind": "repo_file",
+        "path": "docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md"
+      },
+      "privacy": {
+        "contains_user_data": false,
+        "contains_code": true,
+        "data_classification": "public_repo"
+      },
+      "expected": {
+        "artifact": "docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md"
+      },
+      "judge": {
+        "type": "deterministic",
+        "assertions": [
+          {
+            "type": "file_exists",
+            "path": "docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md"
+          }
+        ]
+      }
+    },
+    {
+      "schema_version": "evalfly.case.v1",
+      "case_id": "m2-registry-implementation-exists",
+      "title": "Milestone 2 SQLite registry implementation exists",
+      "suite": "smoke",
+      "risk_tier": "critical",
+      "task_type": "state_persistence",
+      "source": {
+        "kind": "repo_file",
+        "path": "src/coderag/services/registry.py"
+      },
+      "privacy": {
+        "contains_user_data": false,
+        "contains_code": true,
+        "data_classification": "public_repo"
+      },
+      "expected": {
+        "artifact": "src/coderag/services/registry.py"
+      },
+      "judge": {
+        "type": "deterministic",
+        "assertions": [
+          {
+            "type": "file_exists",
+            "path": "src/coderag/services/registry.py"
+          },
+          {
+            "type": "file_contains",
+            "path": "src/coderag/services/registry.py",
+            "text": "CREATE TABLE IF NOT EXISTS registry_metadata"
+          },
+          {
+            "type": "file_contains",
+            "path": "src/coderag/services/registry.py",
+            "text": "CREATE TABLE IF NOT EXISTS index_jobs"
+          },
+          {
+            "type": "file_contains",
+            "path": "src/coderag/services/registry.py",
+            "text": "repositories_json_migrated_at"
+          }
+        ]
+      }
+    },
+    {
+      "schema_version": "evalfly.case.v1",
+      "case_id": "m2-registry-tests-exist",
+      "title": "Milestone 2 registry and job-state tests exist",
+      "suite": "smoke",
+      "risk_tier": "critical",
+      "task_type": "state_persistence",
+      "source": {
+        "kind": "repo_file",
+        "path": "tests/test_m2_registry.py"
+      },
+      "privacy": {
+        "contains_user_data": false,
+        "contains_code": true,
+        "data_classification": "public_repo"
+      },
+      "expected": {
+        "artifact": "tests/test_m2_registry.py"
+      },
+      "judge": {
+        "type": "deterministic",
+        "assertions": [
+          {
+            "type": "file_exists",
+            "path": "tests/test_m2_registry.py"
+          },
+          {
+            "type": "file_contains",
+            "path": "tests/test_m2_registry.py",
+            "text": "test_sqlite_registry_does_not_resurrect_json_after_delete"
+          },
+          {
+            "type": "file_contains",
+            "path": "tests/test_m2_registry.py",
+            "text": "test_sqlite_registry_save_preserves_file_and_chunk_metadata"
+          },
+          {
+            "type": "file_contains",
+            "path": "tests/test_m2_registry.py",
+            "text": "test_update_repository_persists_failed_update_job"
+          }
+        ]
+      }
     }
   ]
 }

scripts/evalfly_smoke.py

@@ -1,7 +1,7 @@
 """Run the committed deterministic subset of the EvalFly smoke suite.
 
 This CI helper is intentionally narrow: it validates the repository-local EvalFly
-configuration enough to execute deterministic `file_exists` assertions in GitHub
+configuration enough to execute deterministic repository-local assertions in GitHub
 Actions, while the full OMP EvalFly CLI remains the authoritative local tool.
 """
 
@@ -56,13 +56,28 @@ def _run_case(case: dict[str, Any]) -> None:
     for assertion in assertions:
         if not isinstance(assertion, dict):
             raise SmokeFailure(f"{case_id}: assertion must be an object")
-        if assertion.get("type") != "file_exists":
-            raise SmokeFailure(f"{case_id}: unsupported assertion type {assertion.get('type')!r}")
+        assertion_type = assertion.get("type")
         raw_path = assertion.get("path")
         if not isinstance(raw_path, str) or not raw_path:
-            raise SmokeFailure(f"{case_id}: file_exists assertion requires a path")
-        if not (REPO_ROOT / raw_path).is_file():
-            raise SmokeFailure(f"{case_id}: expected file does not exist: {raw_path}")
+            raise SmokeFailure(f"{case_id}: assertion requires a path")
+        path = REPO_ROOT / raw_path
+
+        if assertion_type == "file_exists":
+            if not path.is_file():
+                raise SmokeFailure(f"{case_id}: expected file does not exist: {raw_path}")
+            continue
+
+        if assertion_type == "file_contains":
+            expected_text = assertion.get("text")
+            if not isinstance(expected_text, str) or not expected_text:
+                raise SmokeFailure(f"{case_id}: file_contains assertion requires text")
+            if not path.is_file():
+                raise SmokeFailure(f"{case_id}: expected file does not exist: {raw_path}")
+            if expected_text not in path.read_text():
+                raise SmokeFailure(f"{case_id}: expected {raw_path} to contain {expected_text!r}")
+            continue
+
+        raise SmokeFailure(f"{case_id}: unsupported assertion type {assertion_type!r}")
 
 
 def run_smoke() -> int:

src/coderag/services/__init__.py

@@ -7,7 +7,13 @@
     IndexingService,
 )
 from coderag.services.providers import ProviderConfigService
-from coderag.services.registry import RepositoryRegistry, RepositoryRegistryError
+from coderag.services.registry import (
+    RegistryJob,
+    RepositoryChunkRecord,
+    RepositoryFileRecord,
+    RepositoryRegistry,
+    RepositoryRegistryError,
+)
 from coderag.services.retrieval import QueryServiceResult, RetrievalService
 
 __all__ = [
@@ -17,6 +23,9 @@
     "IndexingService",
     "ProviderConfigService",
     "QueryServiceResult",
+    "RegistryJob",
+    "RepositoryChunkRecord",
+    "RepositoryFileRecord",
     "RepositoryRegistry",
     "RepositoryRegistryError",
     "RetrievalService",