diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cee8cc2..d3bf711 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,17 +25,17 @@ jobs: python -m pip install -e . --no-deps python -m pip install pytest pytest-asyncio pydantic pydantic-settings click PyYAML fastapi structlog ruff mypy mcp httpx - - name: Run Milestone 0 and 1 tests - run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py + - name: Run Milestone 0-2 tests + run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py - - name: Run Milestone 0 and 1 lint baseline - run: ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py + - name: Run Milestone 0-2 lint baseline + run: ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py - - name: Run Milestone 0 and 1 type baseline - run: PYTHONPATH=src mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py scripts/evalfly_smoke.py + - name: Run Milestone 0-2 type baseline + run: PYTHONPATH=src mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py - name: Compile changed runtime modules - run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/generator.py src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py + run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/generator.py src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py - name: Build package run: python -m build diff --git a/.pi/.specsafe-state.json b/.pi/.specsafe-state.json index e4bda32..2630bcd 100644 --- a/.pi/.specsafe-state.json +++ b/.pi/.specsafe-state.json @@ -80,6 +80,26 @@ "turns": 0 } } + }, + { + "sliceId": "CRAG-20260627-M2-SQLITE-REGISTRY", + "workspaceId": "CodeRAG", + "sessionId": "omp-session", + "beganAt": "2026-06-27T23:00:19.118Z", + "endedAt": "2026-06-28T03:35:40.222Z", + "outcome": "PASS", + "costSummary": { + "externalMemoryCalls": 0, + "externalMemoryCost": 0, + "subagentTokens": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "cost": 0, + "turns": 0 + } + } } ] } diff --git a/docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md b/docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md new file mode 100644 index 0000000..f5d1abc --- /dev/null +++ b/docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md @@ -0,0 +1,125 @@ +# CodeRAG Milestone 2 SQLite Registry and Durable Job State Contract + +Slice: `CRAG-20260627-M2-SQLITE-REGISTRY` +Parent plan: `docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md` +Branch: `feat/m2-sqlite-registry-state` + +## Scope + +Milestone 2 replaces fragile JSON repository metadata with a SQLite-backed registry and durable job records while keeping the Milestone 1 service seam. This milestone is a persistence/state milestone: it must not introduce hybrid retrieval, verified citation semantics, graph extraction, context packs, or autonomous editing behavior. + +## Source of truth + +1. Roadmap AC-3 and Milestone 2 (`docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md`). +2. Current `main` after PR #2 merge. +3. Existing public API/MCP/CLI/UI repository state behavior unless this contract explicitly narrows it. +4. Pre-1.0 alpha users may either receive a deterministic `repositories.json` migration or an explicit reset-path behavior with tests; this milestone chooses migration because the existing Repository model already serializes the current alpha metadata shape. + +## Frozen acceptance criteria + +### M2-AC1: SQLite registry schema and migration + +Pass requires: +- `RepositoryRegistry` uses SQLite as the canonical store for repository metadata. +- Schema creation is explicit and versioned enough for current migrations, not implicit ad-hoc table creation at each call site. +- The registry tracks repo id, URL/path, branch, head SHA, indexed_at, status, error, file count, chunk count, and index/provider metadata fields. +- Existing `repositories.json` entries are migrated into SQLite on first registry load when a SQLite DB does not already contain repositories. +- Corrupt `repositories.json` still fails closed and is not overwritten or silently ignored during migration. + +Fail if: +- Repository metadata remains canonical in a global dict or `repositories.json` after M2. +- A valid existing `repositories.json` user loses repositories without an explicit tested reset path. +- Migration hides corrupt JSON by creating an empty SQLite registry. + +Required evidence: +- Unit tests using a temp SQLite DB for schema creation, add/update/list/get/remove, partial-id behavior, and reopen/restart persistence. +- Migration test from a sample `repositories.json` into SQLite. +- Corrupt JSON migration test proving fail-closed behavior. + +### M2-AC2: Durable indexing job state + +Pass requires: +- Registry has durable job records for index/update/delete operations. +- Job records include id, repo id, operation, status, error, started_at, finished_at, and counters for files/chunks when available. +- `IndexingService` records job start/success/failure for full indexing and deletion. +- Failed indexing records both repository error status and durable job error. +- Restarting a process can read the last job state from SQLite. + +Fail if: +- Job state only exists in memory or only in logs. +- Indexing failure persists repository error but no job error. +- Delete removes all evidence of the delete operation before callers can inspect its job state. + +Required evidence: +- Unit tests proving successful index job, failed index job, and delete job state are persisted in temp SQLite. +- Restart simulation test reopening the registry and reading the recorded jobs. + +### M2-AC3: Shared registry across API/MCP/CLI/UI + +Pass requires: +- API, MCP, indexing/retrieval services, and UI repository listing/status paths use the same `RepositoryRegistry` abstraction. +- Public response shapes remain compatible unless covered by a migration note and regression test. +- REST destructive operations continue to reject ambiguous partial ids before delete/query/get service calls. +- MCP first-match prefix compatibility from M1 remains unless a test documents an intentional compatibility change. + +Fail if: +- UI still owns independent `repositories.json` load/save state for repository metadata. +- API and MCP can report conflicting repository state after a process restart. +- M1 partial-id ambiguity safety regresses. + +Required evidence: +- API/MCP shared-state integration test using the same temp SQLite DB. +- UI repository table/status test using the shared registry path. +- Regression test for ambiguous REST delete/query/get behavior or shared helper coverage plus destructive delete test. + +### M2-AC4: Governance and evidence + +Pass requires: +- SpecSafe slice `CRAG-20260627-M2-SQLITE-REGISTRY` is open before implementation edits and closed only after verification. +- EvalFly smoke/config is extended with M2 deterministic registry/job-state checks, or an explicit committed reason documents why deeper state evals wait for a later fixture layer. +- CI targeted tests include M2 registry/job-state tests. +- Work is committed on `feat/m2-sqlite-registry-state`, pushed after every commit, and opened as a PR after local verification. + +Fail if: +- Tests/evals are weakened, deleted, bypassed, or relabeled to pass. +- M2 is declared production-ready or claims hybrid/citation/graph/context-pack capabilities. +- The SQLite schema is only exercised through mocks that bypass the real registry persistence layer. + +Required evidence: +- SpecSafe status showing the active slice. +- Targeted pytest output for M0/M1/M2 tests and affected API/MCP/UI tests. +- Ruff and mypy output for touched modules and tests. +- `scripts/evalfly_smoke.py --suite smoke` output. +- Package build output. +- Post-implementation review against this contract. + +## Implementation boundaries + +Allowed: +- Replace `RepositoryRegistry` internals with SQLite while preserving its public methods where practical. +- Add registry/job dataclasses or enums inside `src/coderag/services/registry.py` or a focused sibling module. +- Add SQLite schema/migration helpers. +- Add tests under `tests/test_m2_registry.py` and update M1/API/MCP/UI tests to use temp SQLite DBs. +- Route UI repository metadata listing/status through `RepositoryRegistry`. +- Extend EvalFly smoke with deterministic static checks for SQLite registry/job-state artifacts. + +Not allowed in M2: +- Hybrid/BM25 retrieval. +- Verified citation semantics or `grounded=True` behavior changes. +- Code graph extraction. +- Context pack APIs. +- Autonomous editing-agent behavior. + +## Evidence commands + +Run before PR: + +```bash +bun run /Users/sebastian/.omp/omp-pantheon/skills/specsafe/bin/specsafe.ts status +PYTHONPATH=src .venv/bin/python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py +.venv/bin/python -m ruff check src/coderag/services src/coderag/config.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/mcp/tools.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py scripts/evalfly_smoke.py +PYTHONPATH=src .venv/bin/python -m mypy src/coderag/services src/coderag/config.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py scripts/evalfly_smoke.py +.venv/bin/python scripts/evalfly_smoke.py --suite smoke +.venv/bin/python -m py_compile src/coderag/services/*.py src/coderag/api/routes.py src/coderag/mcp/handlers.py src/coderag/ui/repository_metadata.py src/coderag/ui/handlers.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_api_routes.py tests/test_ui_handlers.py +.venv/bin/python -m build +``` diff --git a/evals/config.json b/evals/config.json index 9d78fa9..d27e761 100644 --- a/evals/config.json +++ b/evals/config.json @@ -301,6 +301,123 @@ } ] } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m2-sqlite-contract-exists", + "title": "Milestone 2 SQLite registry contract exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "state_persistence", + "source": { + "kind": "repo_file", + "path": "docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md" + }, + "privacy": { + "contains_user_data": false, + "contains_code": true, + "data_classification": "public_repo" + }, + "expected": { + "artifact": "docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md" + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "docs/superpowers/plans/2026-06-27-m2-sqlite-registry-contract.md" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m2-registry-implementation-exists", + "title": "Milestone 2 SQLite registry implementation exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "state_persistence", + "source": { + "kind": "repo_file", + "path": "src/coderag/services/registry.py" + }, + "privacy": { + "contains_user_data": false, + "contains_code": true, + "data_classification": "public_repo" + }, + "expected": { + "artifact": "src/coderag/services/registry.py" + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "src/coderag/services/registry.py" + }, + { + "type": "file_contains", + "path": "src/coderag/services/registry.py", + "text": "CREATE TABLE IF NOT EXISTS registry_metadata" + }, + { + "type": "file_contains", + "path": "src/coderag/services/registry.py", + "text": "CREATE TABLE IF NOT EXISTS index_jobs" + }, + { + "type": "file_contains", + "path": "src/coderag/services/registry.py", + "text": "repositories_json_migrated_at" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m2-registry-tests-exist", + "title": "Milestone 2 registry and job-state tests exist", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "state_persistence", + "source": { + "kind": "repo_file", + "path": "tests/test_m2_registry.py" + }, + "privacy": { + "contains_user_data": false, + "contains_code": true, + "data_classification": "public_repo" + }, + "expected": { + "artifact": "tests/test_m2_registry.py" + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "tests/test_m2_registry.py" + }, + { + "type": "file_contains", + "path": "tests/test_m2_registry.py", + "text": "test_sqlite_registry_does_not_resurrect_json_after_delete" + }, + { + "type": "file_contains", + "path": "tests/test_m2_registry.py", + "text": "test_sqlite_registry_save_preserves_file_and_chunk_metadata" + }, + { + "type": "file_contains", + "path": "tests/test_m2_registry.py", + "text": "test_update_repository_persists_failed_update_job" + } + ] + } } ] } diff --git a/scripts/evalfly_smoke.py b/scripts/evalfly_smoke.py index 6ea6a03..38cd335 100644 --- a/scripts/evalfly_smoke.py +++ b/scripts/evalfly_smoke.py @@ -1,7 +1,7 @@ """Run the committed deterministic subset of the EvalFly smoke suite. This CI helper is intentionally narrow: it validates the repository-local EvalFly -configuration enough to execute deterministic `file_exists` assertions in GitHub +configuration enough to execute deterministic repository-local assertions in GitHub Actions, while the full OMP EvalFly CLI remains the authoritative local tool. """ @@ -56,13 +56,28 @@ def _run_case(case: dict[str, Any]) -> None: for assertion in assertions: if not isinstance(assertion, dict): raise SmokeFailure(f"{case_id}: assertion must be an object") - if assertion.get("type") != "file_exists": - raise SmokeFailure(f"{case_id}: unsupported assertion type {assertion.get('type')!r}") + assertion_type = assertion.get("type") raw_path = assertion.get("path") if not isinstance(raw_path, str) or not raw_path: - raise SmokeFailure(f"{case_id}: file_exists assertion requires a path") - if not (REPO_ROOT / raw_path).is_file(): - raise SmokeFailure(f"{case_id}: expected file does not exist: {raw_path}") + raise SmokeFailure(f"{case_id}: assertion requires a path") + path = REPO_ROOT / raw_path + + if assertion_type == "file_exists": + if not path.is_file(): + raise SmokeFailure(f"{case_id}: expected file does not exist: {raw_path}") + continue + + if assertion_type == "file_contains": + expected_text = assertion.get("text") + if not isinstance(expected_text, str) or not expected_text: + raise SmokeFailure(f"{case_id}: file_contains assertion requires text") + if not path.is_file(): + raise SmokeFailure(f"{case_id}: expected file does not exist: {raw_path}") + if expected_text not in path.read_text(): + raise SmokeFailure(f"{case_id}: expected {raw_path} to contain {expected_text!r}") + continue + + raise SmokeFailure(f"{case_id}: unsupported assertion type {assertion_type!r}") def run_smoke() -> int: diff --git a/src/coderag/services/__init__.py b/src/coderag/services/__init__.py index 0cb32e5..e7f18ac 100644 --- a/src/coderag/services/__init__.py +++ b/src/coderag/services/__init__.py @@ -7,7 +7,13 @@ IndexingService, ) from coderag.services.providers import ProviderConfigService -from coderag.services.registry import RepositoryRegistry, RepositoryRegistryError +from coderag.services.registry import ( + RegistryJob, + RepositoryChunkRecord, + RepositoryFileRecord, + RepositoryRegistry, + RepositoryRegistryError, +) from coderag.services.retrieval import QueryServiceResult, RetrievalService __all__ = [ @@ -17,6 +23,9 @@ "IndexingService", "ProviderConfigService", "QueryServiceResult", + "RegistryJob", + "RepositoryChunkRecord", + "RepositoryFileRecord", "RepositoryRegistry", "RepositoryRegistryError", "RetrievalService", diff --git a/src/coderag/services/indexing.py b/src/coderag/services/indexing.py index 37948f0..054a729 100644 --- a/src/coderag/services/indexing.py +++ b/src/coderag/services/indexing.py @@ -133,6 +133,7 @@ def index_existing_repository( ) -> IndexingResult: """Index into an existing repository record created by an adapter.""" options = options or IndexingOptions() + job = self.registry.begin_job(repo.id, "index") try: logger.info("Indexing service: cloning repository", url=repo.url, branch=branch) repo.status = RepositoryStatus.CLONING @@ -150,6 +151,11 @@ def index_existing_repository( files = list(file_filter.filter_files(repo_path)) self._get_vectorstore().delete_repo_chunks(repo.id) + self.registry.clear_file_metadata(repo.id) + self.registry.clear_chunk_metadata(repo.id) + for file_path in files: + relative_path = str(file_path.relative_to(repo_path)) + self.registry.record_file_metadata(repo.id, relative_path, size_bytes=file_path.stat().st_size) chunks_indexed = self.index_files(files, repo_path, repo.id) last_commit = self._resolve_commit(repo_path) @@ -158,11 +164,13 @@ def index_existing_repository( repo.status = RepositoryStatus.READY repo.last_commit = last_commit self.registry.update(repo) + self.registry.finish_job(job.id, "succeeded", files_processed=len(files), chunks_indexed=chunks_indexed) return IndexingResult(repo=repo, files_processed=len(files), chunks_indexed=chunks_indexed, last_commit=last_commit) except Exception as exc: repo.status = RepositoryStatus.ERROR repo.error_message = str(exc) self.registry.update(repo) + self.registry.finish_job(job.id, "failed", error=str(exc)) logger.error("Indexing service: indexing failed", repo_id=repo.id, error=str(exc)) raise finally: @@ -199,75 +207,98 @@ def delete_repository(self, repo_id: str) -> RepositoryDeleteResult | None: if repo is None: return None - vectorstore = self._get_vectorstore() - chunks_deleted = vectorstore.get_repo_chunk_count(repo.id) - vectorstore.delete_repo_chunks(repo.id) - + job = self.registry.begin_job(repo.id, "delete") try: - self._get_loader().delete_cache(type("RepoInfo", (), {"owner": repo.owner, "name": repo.name})()) - except Exception: - logger.debug("Repository cache delete skipped", repo_id=repo.id) + vectorstore = self._get_vectorstore() + chunks_deleted = vectorstore.get_repo_chunk_count(repo.id) + vectorstore.delete_repo_chunks(repo.id) - removed = self.registry.remove(repo.id) - if removed is None: - return None - return RepositoryDeleteResult(repo=removed, chunks_deleted=chunks_deleted) + try: + self._get_loader().delete_cache(type("RepoInfo", (), {"owner": repo.owner, "name": repo.name})()) + except Exception: + logger.debug("Repository cache delete skipped", repo_id=repo.id) + + removed = self.registry.remove(repo.id) + if removed is None: + self.registry.finish_job(job.id, "failed", error="Repository not found") + return None + self.registry.finish_job(job.id, "succeeded", chunks_indexed=chunks_deleted) + return RepositoryDeleteResult(repo=removed, chunks_deleted=chunks_deleted) + except Exception as exc: + self.registry.finish_job(job.id, "failed", error=str(exc)) + raise def update_repository(self, repo_id: str) -> IncrementalIndexingResult: """Incrementally update an indexed repository from its cached clone.""" repo = self.registry.get(repo_id) if repo is None: raise ValueError(f"Repository not found: {repo_id}") - if not repo.last_commit: - raise ValueError("No previous indexing found. Please re-index the full repository.") - if not repo.clone_path or not Path(repo.clone_path).exists(): - raise ValueError("Repository cache not found. Please re-index.") - - repo_path = Path(repo.clone_path) - self._get_loader()._update_repository(repo_path, repo.branch, None) - current_commit = self._resolve_commit(repo_path) - if current_commit == repo.last_commit: + job = self.registry.begin_job(repo.id, "update") + try: + if not repo.last_commit: + raise ValueError("No previous indexing found. Please re-index the full repository.") + if not repo.clone_path or not Path(repo.clone_path).exists(): + raise ValueError("Repository cache not found. Please re-index.") + + repo_path = Path(repo.clone_path) + self._get_loader()._update_repository(repo_path, repo.branch, None) + current_commit = self._resolve_commit(repo_path) + if current_commit == repo.last_commit: + self.registry.finish_job(job.id, "succeeded", files_processed=0, chunks_indexed=0) + return IncrementalIndexingResult( + repo=repo, + files_changed=0, + files_added=0, + files_modified=0, + files_deleted=0, + chunks_added=0, + chunks_deleted=0, + total_chunks=repo.chunk_count, + already_up_to_date=True, + ) + + added, modified, deleted = self._get_changed_files(repo_path, repo.last_commit, current_commit or repo.last_commit) + chunks_deleted = 0 + for file_path in deleted | modified: + count = self._get_vectorstore().delete_file_chunks(repo.id, file_path) + chunks_deleted += count or 0 + self.registry.remove_file_metadata(repo.id, file_path) + + file_filter = self._get_file_filter_factory()() + files_to_index = [] + for file_path in added | modified: + full_path = repo_path / file_path + if full_path.exists() and file_filter.should_include(full_path, repo_path): + files_to_index.append(full_path) + for indexed_path in files_to_index: + relative_path = str(indexed_path.relative_to(repo_path)) + self.registry.record_file_metadata(repo.id, relative_path, size_bytes=indexed_path.stat().st_size) + + chunks_added = self.index_files(files_to_index, repo_path, repo.id) if files_to_index else 0 + repo.last_commit = current_commit + repo.indexed_at = datetime.now() + repo.chunk_count = self._get_vectorstore().get_repo_chunk_count(repo.id) + self.registry.update(repo) + self.registry.finish_job( + job.id, + "succeeded", + files_processed=len(added | modified | deleted), + chunks_indexed=chunks_added, + ) + return IncrementalIndexingResult( repo=repo, - files_changed=0, - files_added=0, - files_modified=0, - files_deleted=0, - chunks_added=0, - chunks_deleted=0, + files_changed=len(added | modified | deleted), + files_added=len(added), + files_modified=len(modified), + files_deleted=len(deleted), + chunks_added=chunks_added, + chunks_deleted=chunks_deleted, total_chunks=repo.chunk_count, - already_up_to_date=True, ) - - added, modified, deleted = self._get_changed_files(repo_path, repo.last_commit, current_commit or repo.last_commit) - chunks_deleted = 0 - for file_path in deleted | modified: - count = self._get_vectorstore().delete_file_chunks(repo.id, file_path) - chunks_deleted += count or 0 - - file_filter = self._get_file_filter_factory()() - files_to_index = [] - for file_path in added | modified: - full_path = repo_path / file_path - if full_path.exists() and file_filter.should_include(full_path, repo_path): - files_to_index.append(full_path) - - chunks_added = self.index_files(files_to_index, repo_path, repo.id) if files_to_index else 0 - repo.last_commit = current_commit - repo.indexed_at = datetime.now() - repo.chunk_count = self._get_vectorstore().get_repo_chunk_count(repo.id) - self.registry.update(repo) - - return IncrementalIndexingResult( - repo=repo, - files_changed=len(added | modified | deleted), - files_added=len(added), - files_modified=len(modified), - files_deleted=len(deleted), - chunks_added=chunks_added, - chunks_deleted=chunks_deleted, - total_chunks=repo.chunk_count, - ) + except Exception as exc: + self.registry.finish_job(job.id, "failed", error=str(exc)) + raise def get_indexed_files(self, repo_id: str) -> set[str]: return cast(set[str], self._get_vectorstore().get_indexed_files(repo_id)) @@ -281,6 +312,8 @@ def _process_batch(self, chunks: list[Chunk]) -> int: try: embedded = self._get_embedder().embed_chunks(chunks, show_progress=False) self._get_vectorstore().add_chunks(embedded) + for chunk in embedded: + self.registry.record_chunk_metadata(chunk) return len(chunks) finally: try: diff --git a/src/coderag/services/registry.py b/src/coderag/services/registry.py index c6876c6..d3f782f 100644 --- a/src/coderag/services/registry.py +++ b/src/coderag/services/registry.py @@ -1,49 +1,109 @@ -"""Repository metadata registry service.""" +"""SQLite-backed repository metadata registry service.""" + +from __future__ import annotations import json +import sqlite3 +from builtins import list as list_type +from dataclasses import dataclass +from datetime import datetime from pathlib import Path +from uuid import uuid4 from coderag.config import get_settings from coderag.logging import get_logger +from coderag.models.chunk import Chunk from coderag.models.repository import Repository logger = get_logger(__name__) +SCHEMA_VERSION = 1 class RepositoryRegistryError(RuntimeError): """Raised when existing repository metadata cannot be safely read.""" +@dataclass(frozen=True) +class RegistryJob: + """Durable indexing/update/delete job state.""" + + id: str + repo_id: str | None + operation: str + status: str + started_at: datetime + finished_at: datetime | None = None + error: str | None = None + files_processed: int | None = None + chunks_indexed: int | None = None + + +@dataclass(frozen=True) +class RepositoryFileRecord: + """Indexed file metadata stored by repository.""" + + repo_id: str + path: str + content_hash: str | None = None + size_bytes: int | None = None + language: str | None = None + indexed_at: datetime | None = None + + +@dataclass(frozen=True) +class RepositoryChunkRecord: + """Indexed chunk metadata stored by repository.""" + + id: str + repo_id: str + file_path: str + start_line: int + end_line: int + chunk_type: str + name: str | None = None + content_hash: str | None = None + metadata_json: str | None = None + + class RepositoryRegistry: - """JSON-backed repository metadata registry. + """SQLite-backed repository metadata registry. - This preserves the pre-Milestone-1 `repositories.json` file shape while - centralizing load/save/partial-id semantics for API, MCP, CLI, and UI - adapters. Milestone 2 can replace this backing store with SQLite behind the - same interface. + SQLite is the canonical Milestone 2 store. A pre-1.0 alpha + ``repositories.json`` file is read once as a migration source when the + SQLite repository table is empty; corrupt JSON fails closed and is never + overwritten. """ - def __init__(self, repos_file: Path | None = None) -> None: + def __init__(self, repos_file: Path | None = None, db_path: Path | None = None) -> None: settings = get_settings() + if db_path is None and repos_file is not None and repos_file.suffix in {".db", ".sqlite", ".sqlite3"}: + db_path = repos_file + repos_file = repos_file.with_name("repositories.json") + self.repos_file = repos_file or settings.data_dir / "repositories.json" + self.db_path = db_path or self.repos_file.with_name("registry.sqlite3") + self._ensure_schema() def load(self) -> dict[str, Repository]: """Load all repositories keyed by id.""" - if not self.repos_file.exists(): - return {} - - try: - data = json.loads(self.repos_file.read_text()) - return {item["id"]: Repository.from_dict(item) for item in data} - except Exception as exc: - logger.error("Failed to load repositories", path=str(self.repos_file), error=str(exc)) - raise RepositoryRegistryError(f"Failed to load repositories from {self.repos_file}") from exc + self._migrate_json_if_needed() + with self._connect() as conn: + rows = conn.execute("SELECT * FROM repositories ORDER BY rowid").fetchall() + return {row["id"]: self._repo_from_row(row) for row in rows} def save(self, repositories: dict[str, Repository]) -> None: - """Persist repositories using the existing JSON list shape.""" - self.repos_file.parent.mkdir(parents=True, exist_ok=True) - data = [repo.to_dict() for repo in repositories.values()] - self.repos_file.write_text(json.dumps(data, indent=2)) + """Persist repositories into SQLite without deleting metadata for retained repos.""" + self._migrate_json_if_needed() + with self._connect() as conn: + incoming_ids = set(repositories) + existing_ids = { + row["id"] + for row in conn.execute("SELECT id FROM repositories").fetchall() + } + for repo in repositories.values(): + self._upsert_repo(conn, repo) + for repo_id in existing_ids - incoming_ids: + conn.execute("DELETE FROM repositories WHERE id = ?", (repo_id,)) def list(self) -> list[Repository]: """Return repositories from the latest registry snapshot.""" @@ -89,16 +149,16 @@ def get_unique(self, repo_id: str) -> Repository | None: def add(self, repo: Repository) -> Repository: """Add a repository and persist it.""" - repositories = self.load() - repositories[repo.id] = repo - self.save(repositories) + self._migrate_json_if_needed() + with self._connect() as conn: + self._upsert_repo(conn, repo) return repo def update(self, repo: Repository) -> Repository: """Update a repository and persist it.""" - repositories = self.load() - repositories[repo.id] = repo - self.save(repositories) + self._migrate_json_if_needed() + with self._connect() as conn: + self._upsert_repo(conn, repo) return repo def remove(self, repo_id: str) -> Repository | None: @@ -108,6 +168,407 @@ def remove(self, repo_id: str) -> Repository | None: if resolved_id is None: return None - repo = repositories.pop(resolved_id) - self.save(repositories) + repo = repositories[resolved_id] + with self._connect() as conn: + conn.execute("DELETE FROM repository_chunks WHERE repo_id = ?", (resolved_id,)) + conn.execute("DELETE FROM repository_files WHERE repo_id = ?", (resolved_id,)) + conn.execute("DELETE FROM repositories WHERE id = ?", (resolved_id,)) return repo + + def begin_job(self, repo_id: str | None, operation: str) -> RegistryJob: + """Create a durable running job record.""" + job = RegistryJob( + id=str(uuid4()), + repo_id=repo_id, + operation=operation, + status="running", + started_at=datetime.now(), + ) + with self._connect() as conn: + conn.execute( + """ + INSERT INTO index_jobs ( + id, repo_id, operation, status, started_at, finished_at, + error, files_processed, chunks_indexed + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + self._job_values(job), + ) + return job + + def finish_job( + self, + job_id: str, + status: str, + *, + error: str | None = None, + files_processed: int | None = None, + chunks_indexed: int | None = None, + ) -> RegistryJob: + """Mark a durable job complete or failed.""" + finished_at = datetime.now() + with self._connect() as conn: + conn.execute( + """ + UPDATE index_jobs + SET status = ?, finished_at = ?, error = ?, files_processed = ?, chunks_indexed = ? + WHERE id = ? + """, + ( + status, + finished_at.isoformat(), + error, + files_processed, + chunks_indexed, + job_id, + ), + ) + job = self.get_job(job_id) + if job is None: + raise RepositoryRegistryError(f"Job not found: {job_id}") + return job + + def get_job(self, job_id: str) -> RegistryJob | None: + """Return one durable job record.""" + with self._connect() as conn: + row = conn.execute("SELECT * FROM index_jobs WHERE id = ?", (job_id,)).fetchone() + return self._job_from_row(row) if row else None + + def list_jobs(self, repo_id: str | None = None) -> list_type[RegistryJob]: + """Return durable job records, newest first.""" + with self._connect() as conn: + if repo_id is None: + rows = conn.execute("SELECT * FROM index_jobs ORDER BY started_at DESC, rowid DESC").fetchall() + else: + rows = conn.execute( + "SELECT * FROM index_jobs WHERE repo_id = ? ORDER BY started_at DESC, rowid DESC", + (repo_id,), + ).fetchall() + return [self._job_from_row(row) for row in rows] + + def clear_file_metadata(self, repo_id: str) -> None: + """Delete indexed file metadata for a repository.""" + with self._connect() as conn: + conn.execute("DELETE FROM repository_files WHERE repo_id = ?", (repo_id,)) + + def remove_file_metadata(self, repo_id: str, path: str) -> None: + """Delete one file's durable metadata and chunk metadata.""" + with self._connect() as conn: + conn.execute("DELETE FROM repository_chunks WHERE repo_id = ? AND file_path = ?", (repo_id, path)) + conn.execute("DELETE FROM repository_files WHERE repo_id = ? AND path = ?", (repo_id, path)) + conn.execute( + "UPDATE repositories SET file_count = (SELECT COUNT(*) FROM repository_files WHERE repo_id = ?) WHERE id = ?", + (repo_id, repo_id), + ) + + def record_file_metadata( + self, + repo_id: str, + path: str, + *, + content_hash: str | None = None, + size_bytes: int | None = None, + language: str | None = None, + ) -> RepositoryFileRecord: + """Persist indexed file metadata.""" + record = RepositoryFileRecord( + repo_id=repo_id, + path=path, + content_hash=content_hash, + size_bytes=size_bytes, + language=language, + indexed_at=datetime.now(), + ) + with self._connect() as conn: + conn.execute( + """ + INSERT INTO repository_files (repo_id, path, content_hash, size_bytes, language, indexed_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT(repo_id, path) DO UPDATE SET + content_hash = excluded.content_hash, + size_bytes = excluded.size_bytes, + language = excluded.language, + indexed_at = excluded.indexed_at + """, + ( + record.repo_id, + record.path, + record.content_hash, + record.size_bytes, + record.language, + record.indexed_at.isoformat() if record.indexed_at else None, + ), + ) + conn.execute( + "UPDATE repositories SET file_count = (SELECT COUNT(*) FROM repository_files WHERE repo_id = ?) WHERE id = ?", + (repo_id, repo_id), + ) + return record + + def list_files(self, repo_id: str) -> list_type[RepositoryFileRecord]: + """Return indexed file metadata for a repository.""" + with self._connect() as conn: + rows = conn.execute("SELECT * FROM repository_files WHERE repo_id = ? ORDER BY path", (repo_id,)).fetchall() + return [self._file_from_row(row) for row in rows] + + def clear_chunk_metadata(self, repo_id: str) -> None: + """Delete indexed chunk metadata for a repository.""" + with self._connect() as conn: + conn.execute("DELETE FROM repository_chunks WHERE repo_id = ?", (repo_id,)) + + def record_chunk_metadata(self, chunk: Chunk) -> RepositoryChunkRecord: + """Persist indexed chunk metadata.""" + record = RepositoryChunkRecord( + id=chunk.id, + repo_id=chunk.repo_id, + file_path=chunk.file_path, + start_line=chunk.start_line, + end_line=chunk.end_line, + chunk_type=chunk.chunk_type.value, + name=chunk.name, + metadata_json=json.dumps(chunk.metadata.__dict__, default=str, sort_keys=True), + ) + with self._connect() as conn: + conn.execute( + """ + INSERT INTO repository_chunks ( + id, repo_id, file_path, start_line, end_line, chunk_type, + name, content_hash, metadata_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + repo_id = excluded.repo_id, + file_path = excluded.file_path, + start_line = excluded.start_line, + end_line = excluded.end_line, + chunk_type = excluded.chunk_type, + name = excluded.name, + content_hash = excluded.content_hash, + metadata_json = excluded.metadata_json + """, + ( + record.id, + record.repo_id, + record.file_path, + record.start_line, + record.end_line, + record.chunk_type, + record.name, + record.content_hash, + record.metadata_json, + ), + ) + return record + + def list_chunks(self, repo_id: str) -> list_type[RepositoryChunkRecord]: + """Return indexed chunk metadata for a repository.""" + with self._connect() as conn: + rows = conn.execute( + "SELECT * FROM repository_chunks WHERE repo_id = ? ORDER BY file_path, start_line, end_line", + (repo_id,), + ).fetchall() + return [self._chunk_from_row(row) for row in rows] + + def _connect(self) -> sqlite3.Connection: + self.db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + return conn + + def _ensure_schema(self) -> None: + with self._connect() as conn: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS registry_metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS repositories ( + id TEXT PRIMARY KEY, + url TEXT NOT NULL, + branch TEXT NOT NULL, + clone_path TEXT, + indexed_at TEXT, + status TEXT NOT NULL, + error_message TEXT, + last_commit TEXT, + file_count INTEGER NOT NULL DEFAULT 0, + chunk_count INTEGER NOT NULL DEFAULT 0, + index_metadata TEXT NOT NULL DEFAULT '{}', + provider_metadata TEXT NOT NULL DEFAULT '{}' + ); + + CREATE TABLE IF NOT EXISTS repository_files ( + repo_id TEXT NOT NULL, + path TEXT NOT NULL, + content_hash TEXT, + size_bytes INTEGER, + language TEXT, + indexed_at TEXT, + PRIMARY KEY (repo_id, path), + FOREIGN KEY (repo_id) REFERENCES repositories(id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS repository_chunks ( + id TEXT PRIMARY KEY, + repo_id TEXT NOT NULL, + file_path TEXT NOT NULL, + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + chunk_type TEXT NOT NULL, + name TEXT, + content_hash TEXT, + metadata_json TEXT, + FOREIGN KEY (repo_id) REFERENCES repositories(id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS index_jobs ( + id TEXT PRIMARY KEY, + repo_id TEXT, + operation TEXT NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT, + error TEXT, + files_processed INTEGER, + chunks_indexed INTEGER + ); + """ + ) + conn.execute( + "INSERT OR IGNORE INTO schema_migrations (version, applied_at) VALUES (?, ?)", + (SCHEMA_VERSION, datetime.now().isoformat()), + ) + + def _migrate_json_if_needed(self) -> None: + if not self.repos_file.exists(): + return + with self._connect() as conn: + migrated = conn.execute( + "SELECT value FROM registry_metadata WHERE key = 'repositories_json_migrated_at'" + ).fetchone() + if migrated: + return + + existing = conn.execute("SELECT COUNT(*) FROM repositories").fetchone()[0] + if existing: + self._mark_json_migrated(conn) + return + + try: + data = json.loads(self.repos_file.read_text()) + repos = [Repository.from_dict(item) for item in data] + except Exception as exc: + logger.error("Failed to migrate repositories", path=str(self.repos_file), error=str(exc)) + raise RepositoryRegistryError(f"Failed to migrate repositories from {self.repos_file}") from exc + for repo in repos: + self._upsert_repo(conn, repo) + self._mark_json_migrated(conn) + + def _mark_json_migrated(self, conn: sqlite3.Connection) -> None: + conn.execute( + "INSERT OR REPLACE INTO registry_metadata (key, value) VALUES ('repositories_json_migrated_at', ?)", + (datetime.now().isoformat(),), + ) + def _upsert_repo(self, conn: sqlite3.Connection, repo: Repository) -> None: + conn.execute( + """ + INSERT INTO repositories ( + id, url, branch, clone_path, indexed_at, status, error_message, + last_commit, file_count, chunk_count, index_metadata, provider_metadata + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, COALESCE((SELECT file_count FROM repositories WHERE id = ?), 0), ?, COALESCE((SELECT index_metadata FROM repositories WHERE id = ?), '{}'), COALESCE((SELECT provider_metadata FROM repositories WHERE id = ?), '{}')) + ON CONFLICT(id) DO UPDATE SET + url = excluded.url, + branch = excluded.branch, + clone_path = excluded.clone_path, + indexed_at = excluded.indexed_at, + status = excluded.status, + error_message = excluded.error_message, + last_commit = excluded.last_commit, + chunk_count = excluded.chunk_count + """, + ( + repo.id, + repo.url, + repo.branch, + str(repo.clone_path) if repo.clone_path else None, + repo.indexed_at.isoformat() if repo.indexed_at else None, + repo.status.value, + repo.error_message, + repo.last_commit, + repo.id, + repo.chunk_count, + repo.id, + repo.id, + ), + ) + + def _repo_from_row(self, row: sqlite3.Row) -> Repository: + return Repository.from_dict( + { + "id": row["id"], + "url": row["url"], + "branch": row["branch"], + "clone_path": row["clone_path"], + "indexed_at": row["indexed_at"], + "chunk_count": row["chunk_count"], + "status": row["status"], + "error_message": row["error_message"], + "last_commit": row["last_commit"], + } + ) + + def _job_values(self, job: RegistryJob) -> tuple[object, ...]: + return ( + job.id, + job.repo_id, + job.operation, + job.status, + job.started_at.isoformat(), + job.finished_at.isoformat() if job.finished_at else None, + job.error, + job.files_processed, + job.chunks_indexed, + ) + + def _job_from_row(self, row: sqlite3.Row) -> RegistryJob: + return RegistryJob( + id=row["id"], + repo_id=row["repo_id"], + operation=row["operation"], + status=row["status"], + started_at=datetime.fromisoformat(row["started_at"]), + finished_at=datetime.fromisoformat(row["finished_at"]) if row["finished_at"] else None, + error=row["error"], + files_processed=row["files_processed"], + chunks_indexed=row["chunks_indexed"], + ) + + def _file_from_row(self, row: sqlite3.Row) -> RepositoryFileRecord: + return RepositoryFileRecord( + repo_id=row["repo_id"], + path=row["path"], + content_hash=row["content_hash"], + size_bytes=row["size_bytes"], + language=row["language"], + indexed_at=datetime.fromisoformat(row["indexed_at"]) if row["indexed_at"] else None, + ) + + def _chunk_from_row(self, row: sqlite3.Row) -> RepositoryChunkRecord: + return RepositoryChunkRecord( + id=row["id"], + repo_id=row["repo_id"], + file_path=row["file_path"], + start_line=row["start_line"], + end_line=row["end_line"], + chunk_type=row["chunk_type"], + name=row["name"], + content_hash=row["content_hash"], + metadata_json=row["metadata_json"], + ) diff --git a/src/coderag/ui/handlers.py b/src/coderag/ui/handlers.py index 41d7cea..3f5d81f 100644 --- a/src/coderag/ui/handlers.py +++ b/src/coderag/ui/handlers.py @@ -1,8 +1,11 @@ """UI event handlers for Gradio interface.""" +from __future__ import annotations + +from collections.abc import Callable, Iterator from datetime import datetime from pathlib import Path -from typing import Iterator, Optional +from typing import cast import torch @@ -20,7 +23,7 @@ from coderag.models.query import Query from coderag.models.repository import Repository, RepositoryStatus from coderag.services.providers import ProviderConfigService -from coderag.ui.repository_metadata import load_repositories, save_repositories +from coderag.services.registry import RepositoryRegistry logger = get_logger(__name__) @@ -36,22 +39,22 @@ def __init__(self) -> None: self.chunker = CodeChunker() self.embedder = EmbeddingGenerator() self.vectorstore = VectorStore() - self.generator: Optional[ResponseGenerator] = None + self.generator: ResponseGenerator | None = None self.provider_config = ProviderConfigService(self.settings.models) # Repository metadata storage - self.repos_file = self.settings.data_dir / "repositories.json" + self.registry = RepositoryRegistry() self.repositories: dict[str, Repository] = self._load_repositories() def _load_repositories(self) -> dict[str, Repository]: try: - return load_repositories(self.repos_file) + return self.registry.load() except Exception as e: logger.error("Failed to load repositories", error=str(e)) raise def _save_repositories(self) -> None: - save_repositories(self.repos_file, self.repositories) + self.registry.save(self.repositories) # ========================================================================= # Streaming Methods (Nivel 1) @@ -78,6 +81,8 @@ def _process_batch(self, chunks: list[Chunk]) -> int: try: embedded = self.embedder.embed_chunks(chunks, show_progress=False) self.vectorstore.add_chunks(embedded) + for chunk in embedded: + self.registry.record_chunk_metadata(chunk) finally: # Always release memory, even on partial failure if torch.cuda.is_available(): @@ -90,7 +95,7 @@ def _stream_index_repository( documents: Iterator[Document], repo_id: str, batch_size: int = 100, - progress_callback: Optional[callable] = None, + progress_callback: Callable[[int, int], None] | None = None, ) -> int: """Index using streaming with batches.""" total_chunks = 0 @@ -122,9 +127,9 @@ def _stream_index_repository( # Validation Methods (Nivel 2) # ========================================================================= - def _estimate_repo_size(self, files: list[Path]) -> dict: + def _estimate_repo_size(self, files: list[Path]) -> dict[str, int | float | bool]: """Estimate repository size before indexing.""" - total_size_kb = 0 + total_size_kb = 0.0 estimated_chunks = 0 chunk_size = self.settings.ingestion.chunk_size @@ -146,7 +151,7 @@ def _estimate_repo_size(self, files: list[Path]) -> dict: "warn_large_repo": len(files) > self.settings.ingestion.warn_files_threshold, } - def _validate_repo_size(self, estimate: dict) -> tuple[bool, str]: + def _validate_repo_size(self, estimate: dict[str, int | float | bool]) -> tuple[bool, str]: """Validate if the repository can be indexed.""" if estimate["exceeds_file_limit"]: return False, f"Repository exceeds file limit ({estimate['file_count']} > {self.settings.ingestion.max_files_per_repo})" @@ -167,7 +172,7 @@ def _get_current_commit(self, repo_path: Path) -> str: """Get the SHA of the current commit.""" from git import Repo git_repo = Repo(repo_path) - return git_repo.head.commit.hexsha + return cast(str, git_repo.head.commit.hexsha) def _get_changed_files( self, @@ -211,12 +216,19 @@ def index_repository_incremental(self, repo_id: str) -> str: return "Repository not found" repo = found_repo + job = self.registry.begin_job(repo.id, "update") + if not repo.last_commit: - return "No previous indexing found. Please re-index the full repository." + message = "No previous indexing found. Please re-index the full repository." + self.registry.finish_job(job.id, "failed", error=message) + return message if not repo.clone_path or not Path(repo.clone_path).exists(): - return "Repository cache not found. Please re-index." + message = "Repository cache not found. Please re-index." + self.registry.finish_job(job.id, "failed", error=message) + return message + try: repo_path = Path(repo.clone_path) @@ -228,6 +240,7 @@ def index_repository_incremental(self, repo_id: str) -> str: current_commit = self._get_current_commit(repo_path) if current_commit == repo.last_commit: + self.registry.finish_job(job.id, "succeeded", files_processed=0, chunks_indexed=0) return "Repository is already up to date." added, modified, deleted = self._get_changed_files( @@ -244,6 +257,7 @@ def index_repository_incremental(self, repo_id: str) -> str: # Delete chunks for deleted/modified files for file_path in deleted | modified: self.vectorstore.delete_file_chunks(repo.id, file_path) + self.registry.remove_file_metadata(repo.id, file_path) # Index new/modified files files_to_index = [] @@ -252,6 +266,9 @@ def index_repository_incremental(self, repo_id: str) -> str: full_path = repo_path / file_path if full_path.exists() and file_filter.should_include(full_path, repo_path): files_to_index.append(full_path) + for indexed_path in files_to_index: + relative_path = str(indexed_path.relative_to(repo_path)) + self.registry.record_file_metadata(repo.id, relative_path, size_bytes=indexed_path.stat().st_size) new_chunks = 0 if files_to_index: @@ -263,6 +280,12 @@ def index_repository_incremental(self, repo_id: str) -> str: repo.last_commit = current_commit repo.indexed_at = datetime.now() repo.chunk_count = self.vectorstore.get_repo_chunk_count(repo.id) + self.registry.finish_job( + job.id, + "succeeded", + files_processed=len(added | modified | deleted), + chunks_indexed=new_chunks, + ) self._save_repositories() return ( @@ -274,6 +297,7 @@ def index_repository_incremental(self, repo_id: str) -> str: ) except Exception as e: + self.registry.finish_job(job.id, "failed", error=str(e)) logger.error("Incremental indexing failed", error=str(e), exc_info=True) return f"Error: {str(e)}" @@ -299,6 +323,8 @@ def index_repository( status=RepositoryStatus.CLONING, ) self.repositories[repo.id] = repo + self.registry.add(repo) + job = self.registry.begin_job(repo.id, "index") # Clone repository yield f"Cloning {repo_info.full_name} (branch: {branch})..." @@ -306,7 +332,7 @@ def index_repository( repo_path = self.loader.clone_repository(repo_info, branch) repo.clone_path = repo_path repo.status = RepositoryStatus.INDEXING - + self.registry.update(repo) # Setup filter with custom patterns include = [p.strip() for p in include_patterns.split(",") if p.strip()] or None exclude = [p.strip() for p in exclude_patterns.split(",") if p.strip()] or None @@ -326,7 +352,8 @@ def index_repository( if not can_proceed: repo.status = RepositoryStatus.ERROR repo.error_message = message - self._save_repositories() + self.registry.update(repo) + self.registry.finish_job(job.id, "failed", error=message) yield f"Error: {message}" return @@ -339,6 +366,11 @@ def index_repository( # Delete existing chunks for this repo (re-indexing) logger.info("Deleting previous chunks for repo", repo_id=repo.id) self.vectorstore.delete_repo_chunks(repo.id) + self.registry.clear_file_metadata(repo.id) + self.registry.clear_chunk_metadata(repo.id) + for file_path in files: + relative_path = str(file_path.relative_to(repo_path)) + self.registry.record_file_metadata(repo.id, relative_path, size_bytes=file_path.stat().st_size) # Stream indexing with batches and progress updates yield f"Indexing... (0/{file_count} files, 0 chunks)" @@ -349,10 +381,8 @@ def index_repository( # Process with progress updates total_chunks = 0 batch: list[Chunk] = [] - doc_count = 0 - for doc in doc_generator: - doc_count += 1 + for doc_count, doc in enumerate(doc_generator, start=1): for chunk in self.chunker.chunk_document(doc): chunk.repo_id = repo.id batch.append(chunk) @@ -379,7 +409,9 @@ def index_repository( repo.chunk_count = total_chunks repo.indexed_at = datetime.now() repo.status = RepositoryStatus.READY - self._save_repositories() + self.registry.update(repo) + self.registry.finish_job(job.id, "succeeded", files_processed=file_count, chunks_indexed=total_chunks) + self.repositories = self._load_repositories() result = f"Successfully indexed {repo_info.full_name}\n{file_count} files processed\n{total_chunks} chunks indexed" logger.info("Indexing complete", result=result) @@ -393,7 +425,9 @@ def index_repository( if "repo" in locals(): repo.status = RepositoryStatus.ERROR repo.error_message = str(e) - self._save_repositories() + self.registry.update(repo) + if "job" in locals(): + self.registry.finish_job(job.id, "failed", error=str(e)) yield f"Error: {str(e)}" def ask_question( @@ -444,9 +478,10 @@ def ask_question( logger.error("Question failed", error=str(e)) return "", "", f"Error: {str(e)}" - def get_repositories(self): + def get_repositories(self) -> object: """Get list of repositories for dropdown.""" import gradio as gr + self.repositories = self._load_repositories() choices = [] for repo in self.repositories.values(): if repo.status == RepositoryStatus.READY: @@ -454,9 +489,10 @@ def get_repositories(self): choices.append((label, repo.id)) return gr.update(choices=choices) - def get_repositories_table(self) -> list[list]: + def get_repositories_table(self) -> list[list[object]]: """Get repositories as table data.""" rows = [] + self.repositories = self._load_repositories() for repo in self.repositories.values(): rows.append([ repo.id[:8], @@ -468,21 +504,17 @@ def get_repositories_table(self) -> list[list]: ]) return rows - def delete_repository(self, repo_id: str) -> tuple[str, list[list]]: + def delete_repository(self, repo_id: str) -> tuple[str, list[list[object]]]: """Delete a repository.""" repo_id = repo_id.strip() - # Find by full or partial ID - found_repo = None - for rid, repo in self.repositories.items(): - if rid == repo_id or rid.startswith(repo_id): - found_repo = repo - break + found_repo = self.registry.get(repo_id) if not found_repo: return "Repository not found", self.get_repositories_table() try: + job = self.registry.begin_job(found_repo.id, "delete") # Delete from vector store self.vectorstore.delete_repo_chunks(found_repo.id) @@ -492,11 +524,14 @@ def delete_repository(self, repo_id: str) -> tuple[str, list[list]]: ) # Remove from records - del self.repositories[found_repo.id] - self._save_repositories() + self.registry.remove(found_repo.id) + self.registry.finish_job(job.id, "succeeded") + self.repositories = self._load_repositories() return f"Deleted {found_repo.full_name}", self.get_repositories_table() except Exception as e: + if "job" in locals(): + self.registry.finish_job(job.id, "failed", error=str(e)) logger.error("Delete failed", error=str(e)) return f"Error: {str(e)}", self.get_repositories_table() diff --git a/tests/test_m1_services.py b/tests/test_m1_services.py index 505faef..2579c02 100644 --- a/tests/test_m1_services.py +++ b/tests/test_m1_services.py @@ -138,7 +138,7 @@ def generate(self, query): return FakeResponse() -def test_repository_registry_preserves_json_shape_and_partial_ids(tmp_path): +def test_repository_registry_preserves_partial_ids_and_persists_metadata(tmp_path): registry = RepositoryRegistry(tmp_path / "repositories.json") repo = Repository( id="abcdef123456", @@ -152,9 +152,6 @@ def test_repository_registry_preserves_json_shape_and_partial_ids(tmp_path): assert registry.resolve_id("abcdef12") == repo.id assert registry.get("abcdef12") == repo - raw = (tmp_path / "repositories.json").read_text() - assert raw.startswith("[") - assert '"url": "https://github.com/acme/demo"' in raw reloaded = RepositoryRegistry(tmp_path / "repositories.json") assert reloaded.list()[0].id == repo.id diff --git a/tests/test_m2_registry.py b/tests/test_m2_registry.py new file mode 100644 index 0000000..3e2f94b --- /dev/null +++ b/tests/test_m2_registry.py @@ -0,0 +1,265 @@ +"""Milestone 2 SQLite registry and durable job-state tests.""" +# mypy: disable-error-code="no-untyped-def,type-arg,var-annotated,attr-defined,assignment,union-attr,arg-type" + +import json +import sqlite3 +from datetime import datetime + +import pytest + +from coderag.models.chunk import Chunk, ChunkMetadata, ChunkType +from coderag.models.repository import Repository, RepositoryStatus +from coderag.services.indexing import IndexingOptions, IndexingService +from coderag.services.registry import RepositoryRegistry, RepositoryRegistryError +from tests.test_m1_services import ( + FakeChunker, + FakeEmbedder, + FakeFilter, + FakeLoader, + FakeValidator, + FakeVectorStore, +) + + +def test_sqlite_registry_persists_repositories_across_restart(tmp_path): + db_path = tmp_path / "registry.sqlite3" + registry = RepositoryRegistry(db_path=db_path, repos_file=tmp_path / "repositories.json") + repo = registry.add( + Repository( + id="abcdef123456", + url="https://github.com/acme/demo", + branch="main", + status=RepositoryStatus.READY, + chunk_count=7, + indexed_at=datetime(2026, 6, 27, 12, 0, 0), + last_commit="abc123", + ) + ) + + reopened = RepositoryRegistry(db_path=db_path, repos_file=tmp_path / "repositories.json") + + assert reopened.get("abcdef12") == repo + assert reopened.get_unique("abcdef12") == repo + assert reopened.list()[0].last_commit == "abc123" + + removed = reopened.remove("abcdef12") + assert removed == repo + assert RepositoryRegistry(db_path=db_path, repos_file=tmp_path / "repositories.json").list() == [] + + +def test_sqlite_registry_migrates_existing_json_once(tmp_path): + repos_file = tmp_path / "repositories.json" + db_path = tmp_path / "registry.sqlite3" + repos_file.write_text( + json.dumps( + [ + Repository( + id="json-repo", + url="https://github.com/acme/json-demo", + branch="main", + status=RepositoryStatus.READY, + chunk_count=2, + ).to_dict() + ] + ) + ) + + registry = RepositoryRegistry(repos_file=repos_file, db_path=db_path) + + assert registry.get("json") is not None + assert registry.get("json").chunk_count == 2 + repos_file.write_text("[]") + assert RepositoryRegistry(repos_file=repos_file, db_path=db_path).get("json-repo") is not None + + +def test_sqlite_registry_does_not_resurrect_json_after_delete(tmp_path): + repos_file = tmp_path / "repositories.json" + db_path = tmp_path / "registry.sqlite3" + repos_file.write_text( + json.dumps( + [ + Repository( + id="json-repo", + url="https://github.com/acme/json-demo", + branch="main", + status=RepositoryStatus.READY, + ).to_dict() + ] + ) + ) + + registry = RepositoryRegistry(repos_file=repos_file, db_path=db_path) + assert registry.get("json-repo") is not None + + assert registry.remove("json-repo") is not None + + reopened = RepositoryRegistry(repos_file=repos_file, db_path=db_path) + assert reopened.list() == [] + + +def test_sqlite_registry_save_preserves_file_and_chunk_metadata(tmp_path): + db_path = tmp_path / "registry.sqlite3" + registry = RepositoryRegistry(db_path=db_path, repos_file=tmp_path / "repositories.json") + repo = registry.add(Repository(id="repo-meta", url="https://github.com/acme/demo", status=RepositoryStatus.READY)) + registry.record_file_metadata(repo.id, "app.py", content_hash="sha256:abc", size_bytes=12, language="python") + registry.record_chunk_metadata( + Chunk( + id="chunk-1", + repo_id=repo.id, + content="def app(): pass", + metadata=ChunkMetadata( + file_path="app.py", + start_line=1, + end_line=1, + chunk_type=ChunkType.FUNCTION, + language="python", + name="app", + ), + ) + ) + + repo.status = RepositoryStatus.ERROR + registry.save({repo.id: repo}) + + reopened = RepositoryRegistry(db_path=db_path, repos_file=tmp_path / "repositories.json") + assert reopened.get(repo.id).status is RepositoryStatus.ERROR + assert reopened.list_files(repo.id)[0].content_hash == "sha256:abc" + assert reopened.list_chunks(repo.id)[0].name == "app" + +def test_sqlite_registry_removes_single_file_and_chunk_metadata(tmp_path): + registry = RepositoryRegistry(db_path=tmp_path / "registry.sqlite3", repos_file=tmp_path / "repositories.json") + repo = registry.add(Repository(id="repo-incremental", url="https://github.com/acme/demo", status=RepositoryStatus.READY)) + for path in ("deleted.py", "kept.py"): + registry.record_file_metadata(repo.id, path, size_bytes=12, language="python") + registry.record_chunk_metadata( + Chunk( + id=f"chunk-{path}", + repo_id=repo.id, + content="x = 1", + metadata=ChunkMetadata(file_path=path, start_line=1, end_line=1, chunk_type=ChunkType.TEXT), + ) + ) + + registry.remove_file_metadata(repo.id, "deleted.py") + + reopened = RepositoryRegistry(db_path=tmp_path / "registry.sqlite3", repos_file=tmp_path / "repositories.json") + assert [record.path for record in reopened.list_files(repo.id)] == ["kept.py"] + assert [record.file_path for record in reopened.list_chunks(repo.id)] == ["kept.py"] + + +def test_sqlite_registry_corrupt_json_migration_fails_closed(tmp_path): + repos_file = tmp_path / "repositories.json" + repos_file.write_text("{not-json") + registry = RepositoryRegistry(repos_file=repos_file, db_path=tmp_path / "registry.sqlite3") + + with pytest.raises(RepositoryRegistryError): + registry.list() + + assert repos_file.read_text() == "{not-json" + with sqlite3.connect(tmp_path / "registry.sqlite3") as conn: + assert conn.execute("SELECT COUNT(*) FROM repositories").fetchone()[0] == 0 + + +def test_sqlite_registry_tracks_jobs_files_chunks_and_metadata(tmp_path): + registry = RepositoryRegistry(db_path=tmp_path / "registry.sqlite3", repos_file=tmp_path / "repositories.json") + repo = registry.add(Repository(id="repo-ready", url="https://github.com/acme/demo", status=RepositoryStatus.READY)) + + job = registry.begin_job(repo.id, "index") + registry.record_file_metadata(repo.id, "app.py", content_hash="sha256:abc", size_bytes=12, language="python") + registry.finish_job(job.id, "succeeded", files_processed=1, chunks_indexed=1) + + reopened = RepositoryRegistry(db_path=tmp_path / "registry.sqlite3", repos_file=tmp_path / "repositories.json") + [saved_job] = reopened.list_jobs(repo.id) + [saved_file] = reopened.list_files(repo.id) + + assert saved_job.operation == "index" + assert saved_job.status == "succeeded" + assert saved_job.files_processed == 1 + assert saved_file.path == "app.py" + assert saved_file.content_hash == "sha256:abc" + + +def test_indexing_service_persists_success_and_failure_jobs(tmp_path): + repo_path = tmp_path / "repo" + repo_path.mkdir() + (repo_path / "app.py").write_text("print('hello')\n") + registry = RepositoryRegistry(db_path=tmp_path / "registry.sqlite3", repos_file=tmp_path / "repositories.json") + service = IndexingService( + registry=registry, + validator=FakeValidator(), + loader=FakeLoader(repo_path), + file_filter_factory=FakeFilter, + chunker=FakeChunker(), + embedder=FakeEmbedder(), + vectorstore=FakeVectorStore(), + commit_resolver=lambda _: "abc123", + ) + + result = service.index_repository_sync("https://github.com/acme/demo", IndexingOptions(branch="main")) + + reopened = RepositoryRegistry(db_path=tmp_path / "registry.sqlite3", repos_file=tmp_path / "repositories.json") + [success_job] = reopened.list_jobs(result.repo.id) + assert success_job.status == "succeeded" + assert success_job.files_processed == 1 + assert success_job.chunks_indexed == 1 + assert reopened.list_files(result.repo.id)[0].path == "app.py" + assert reopened.list_chunks(result.repo.id)[0].file_path == "app.py" + + class ExplodingLoader(FakeLoader): + def clone_repository(self, _repo_info, _branch): + raise RuntimeError("clone failed") + + failing_registry = RepositoryRegistry(db_path=tmp_path / "failing.sqlite3", repos_file=tmp_path / "failing.json") + failing_service = IndexingService( + registry=failing_registry, + validator=FakeValidator(), + loader=ExplodingLoader(tmp_path / "missing"), + file_filter_factory=FakeFilter, + chunker=FakeChunker(), + embedder=FakeEmbedder(), + vectorstore=FakeVectorStore(), + ) + + with pytest.raises(RuntimeError, match="clone failed"): + failing_service.index_repository_sync("https://github.com/acme/demo") + + [failed_repo] = failing_registry.list() + [failed_job] = failing_registry.list_jobs(failed_repo.id) + assert failed_repo.status is RepositoryStatus.ERROR + assert failed_job.status == "failed" + assert failed_job.error == "clone failed" + + +def test_delete_repository_persists_delete_job_after_repo_removal(tmp_path): + registry = RepositoryRegistry(db_path=tmp_path / "registry.sqlite3", repos_file=tmp_path / "repositories.json") + repo_path = tmp_path / "repo" + repo_path.mkdir() + repo = registry.add(Repository(id="repo-delete", url="https://github.com/acme/demo", status=RepositoryStatus.READY)) + vectorstore = FakeVectorStore() + service = IndexingService( + registry=registry, + loader=FakeLoader(repo_path), + vectorstore=vectorstore, + ) + + result = service.delete_repository(repo.id) + + assert result is not None + assert registry.get(repo.id) is None + [delete_job] = registry.list_jobs(repo.id) + assert delete_job.operation == "delete" + assert delete_job.status == "succeeded" + + +def test_update_repository_persists_failed_update_job(tmp_path): + registry = RepositoryRegistry(db_path=tmp_path / "registry.sqlite3", repos_file=tmp_path / "repositories.json") + repo = registry.add(Repository(id="repo-update", url="https://github.com/acme/demo", status=RepositoryStatus.READY)) + service = IndexingService(registry=registry) + + with pytest.raises(ValueError, match="No previous indexing"): + service.update_repository(repo.id) + + [update_job] = registry.list_jobs(repo.id) + assert update_job.operation == "update" + assert update_job.status == "failed" + assert update_job.error == "No previous indexing found. Please re-index the full repository." diff --git a/tests/test_ui_handlers.py b/tests/test_ui_handlers.py index 0074069..b359812 100644 --- a/tests/test_ui_handlers.py +++ b/tests/test_ui_handlers.py @@ -1,4 +1,5 @@ """UI privacy/profile guard tests.""" +# mypy: disable-error-code="no-untyped-def,no-untyped-call" import ast import importlib.util @@ -49,6 +50,49 @@ def test_ui_ask_question_checks_provider_profile_before_generation_setup(): assert call_names.index("generation_block_reason") < call_names.index("generate") +def test_ui_repository_state_uses_shared_registry_not_json_helpers(): + module = ast.parse(HANDLERS_PATH.read_text()) + imported_names = { + alias.name + for node in module.body + if isinstance(node, ast.ImportFrom) + for alias in node.names + } + init_method = _function(module, "__init__") + + init_call_names = [name for node in ast.walk(init_method) if (name := _call_name(node)) is not None] + + assert "RepositoryRegistry" in imported_names + assert "load_repositories" not in imported_names + assert "save_repositories" not in imported_names + assert "RepositoryRegistry" in init_call_names + + +def test_ui_repository_operations_record_durable_jobs(): + module = ast.parse(HANDLERS_PATH.read_text()) + index_method = _function(module, "index_repository") + update_method = _function(module, "index_repository_incremental") + delete_method = _function(module, "delete_repository") + + index_calls = [name for node in ast.walk(index_method) if (name := _call_name(node)) is not None] + update_calls = [name for node in ast.walk(update_method) if (name := _call_name(node)) is not None] + delete_calls = [name for node in ast.walk(delete_method) if (name := _call_name(node)) is not None] + + assert "begin_job" in index_calls + assert "finish_job" in index_calls + assert "record_file_metadata" in index_calls + assert "begin_job" in update_calls + assert "finish_job" in update_calls + update_source = ast.get_source_segment(HANDLERS_PATH.read_text(), update_method) + assert update_source is not None + assert update_source.index("begin_job") < update_source.index("last_commit") + assert update_source.index("begin_job") < update_source.index("clone_path") + assert "remove_file_metadata" in update_calls + assert "record_file_metadata" in update_calls + assert "begin_job" in delete_calls + assert "finish_job" in delete_calls + + def test_ui_repository_loader_fails_closed_on_corrupt_metadata(tmp_path): metadata = _load_metadata_module() repos_file = tmp_path / "repositories.json"