From a773adf8305f92116bbe63f67234d57b9e07e4df Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 14:42:11 -0600 Subject: [PATCH 1/8] docs: add CodeRAG 2.0 planning contracts --- ...2026-06-27-coderag-2-production-roadmap.md | 778 ++++++++++++++++++ .../plans/2026-06-27-m0-baseline-contract.md | 92 +++ 2 files changed, 870 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md create mode 100644 docs/superpowers/plans/2026-06-27-m0-baseline-contract.md diff --git a/docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md b/docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md new file mode 100644 index 0000000..54203bf --- /dev/null +++ b/docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md @@ -0,0 +1,778 @@ +# CodeRAG 2.0 Production Roadmap and Acceptance Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use `superpowers:subagent-driven-development` or `superpowers:executing-plans` to implement this plan task-by-task. Do not implement any task until the acceptance criteria and evidence requirements in this document are copied into the active SpecSafe slice and EvalFly contract. + +**Goal:** Evolve CodeRAG from a repository Q&A RAG demo into a production-ready, provider-flexible codebase context engine with hybrid retrieval, verified citations, code intelligence, MCP/API/CLI interfaces, reproducible evals, and safe deployment defaults. + +**Architecture:** CodeRAG remains a RAG/code-intelligence tool, not an autonomous coding agent. The core product indexes repositories, builds searchable code context, verifies file/line evidence, and exposes structured context to humans, IDEs, CI, and optional external coding agents. Cloud inference is the low-friction default; local/private execution is an optional profile, not a hard requirement. + +**Tech Stack:** Python 3.11+, FastAPI, Click CLI, MCP/FastMCP, SQLite registry, ChromaDB initially behind a vector-store interface, Tree-sitter, SentenceTransformers or provider embeddings behind adapters, pytest, ruff, mypy, EvalFly, GitHub Actions. + +## Global Constraints + +- CodeRAG must remain a RAG/code-intelligence product; it must not become an autonomous editing agent in this roadmap. +- Default user path must work on ordinary laptops without requiring a local generative LLM. +- Cloud/model providers must be configurable; local/private profiles must be optional and documented. +- Repository indexing, metadata, citation validation, graph metadata, and retrieval orchestration must run locally by default. +- Any code snippet sent to a cloud provider must be explicit in the selected profile and documented in the privacy model. +- No production-ready claim is allowed without fresh tests, EvalFly evidence, and reviewer validation against this contract. +- Existing public interfaces must either remain compatible or have migration notes and regression tests. +- Security defaults must not expose a credentialed local API to arbitrary origins. +- Evidence must be deterministic first; LLM-as-judge is advisory only unless explicitly approved later. + +--- + +## Source of Truth + +1. User request in this session: complete and refine the adjusted roadmap as a disciplined engineering/PM plan, with acceptance criteria and evidence defined before implementation. +2. Current repository state observed in `/Users/sebastian/Documents/CodeRAG`. +3. Prior analysis of current CodeRAG architecture and gaps. +4. Current-market reference points gathered from public repositories/docs: ChunkHound, Claude Context, code-review-graph, Mantic.sh, Aider, mini-SWE-agent, OpenHands Agent Canvas, Repomix, and MCP docs. + +## Current Repository Baseline + +Observed facts from the current clone: + +- Python package `code-rag-me` is version `0.1.2` in `pyproject.toml`; `src/coderag/__init__.py` and `configs/default.yaml` still report `0.1.0`. +- Stack: FastAPI REST API, Gradio UI, Click CLI, MCP server, ChromaDB, SentenceTransformers, GitPython, Tree-sitter Python. +- Structural chunking exists only for Python; all other included languages fall back to text chunks. +- Retrieval is vector-only: query embedding -> Chroma top-k -> prompt. +- Citations are parsed from LLM output but not verified against retrieved chunks, file snapshots, or line content. +- API and MCP share repository metadata through global in-memory state plus `repositories.json`. +- Server config allows `0.0.0.0`; FastAPI CORS is `allow_origins=["*"]` with `allow_credentials=True`. +- CI only builds/publishes package on release/workflow dispatch; it does not run tests, lint, typecheck, or evals. +- Verification observed: `python3 -m compileall src tests scripts` passed; `python3 -m pytest` did not run because global Python lacks pytest. +- Current repo became dirty from generated `__pycache__/` artifacts during compile verification; implementation tasks must clean generated artifacts before commits. + +## Product Positioning + +### Corrected Positioning + +**CodeRAG 2.0: verified codebase context engine.** + +One-liner: + +> CodeRAG retrieves minimal, source-cited, verifiable context from codebases for humans, IDEs, CI, and AI coding tools. + +### Non-Goals + +- CodeRAG will not edit files, create commits, open PRs, run autonomous issue-solving loops, or compete directly with Claude Code/Codex/Cursor/OpenHands/Aider. +- CodeRAG will not require users to run a local generative LLM. +- CodeRAG will not claim full privacy when cloud providers are selected. +- CodeRAG will not mark citation-formatted answers as grounded unless citations are verified. + +### Primary Personas + +1. **Developer asking questions locally:** wants trustworthy file/line-cited answers and context. +2. **AI coding tool user:** wants Claude Code/Codex/Cursor/etc. to read the right files before acting. +3. **Reviewer/maintainer:** wants changed-file context, blast radius, and suggested tests. +4. **PM/engineering lead:** wants demonstrable quality through eval reports, CI, and acceptance gates. + +--- + +## Frozen Acceptance Criteria + +### AC-0: Delivery Governance + +**Pass requires** + +- A SpecSafe slice is open before implementation starts and references this plan. +- Acceptance criteria and required evidence are frozen before each implementation milestone. +- EvalFly is initialized before behavior-changing implementation begins. +- Every milestone has deterministic tests and EvalFly evidence or an explicit documented reason why EvalFly does not apply. +- No eval/test is weakened, deleted, bypassed, or relabeled to make a failing implementation pass. + +**Fail if** + +- Implementation starts without a slice or without criteria/evidence captured. +- A milestone is declared complete based only on code review or successful import. +- An eval/test is removed instead of fixing the underlying behavior. + +**Required evidence** + +- SpecSafe status showing the active slice. +- `evalfly validate` report once EvalFly exists. +- Milestone-specific verification logs. + +### AC-1: Production Baseline and Maintainability + +**Pass requires** + +- Project installs reproducibly with a documented command using `uv` or a committed equivalent lock strategy. +- Version is single-sourced or synchronized across package, runtime, and config surfaces. +- CI runs tests, lint, typecheck or documented baseline, package build, and EvalFly smoke suite. +- Generated artifacts are ignored and absent from commits. +- API, CLI, MCP, and UI use shared services instead of duplicated indexing/query logic. + +**Fail if** + +- CI only publishes package but does not validate behavior. +- Handlers continue to own duplicated core indexing/retrieval behavior. +- Version mismatch remains visible to users or package consumers. + +**Required evidence** + +- CI workflow file review. +- Local `uv run pytest`, `uv run ruff check .`, `uv run mypy src` or typed-baseline report, and `python -m build`. +- Unit tests for extracted services. + +### AC-2: Safe Defaults and Privacy Profiles + +**Pass requires** + +- Default server binds to `127.0.0.1` unless explicitly configured otherwise. +- CORS defaults to local trusted origins; wildcard plus credentials is not the default. +- Profiles are documented and enforced: `cloud`, `hybrid`, `private`, and `retrieval-only`. +- Cloud profile is the default for low-friction use and clearly states what code context may leave the machine. +- Private profile uses local embeddings and local generation adapters when configured. +- Retrieval-only mode requires no generative model. +- API keys are not logged, emitted in diagnostics, or stored in repo-local files. + +**Fail if** + +- A fresh `coderag serve` exposes credentialed API access broadly. +- Local generative model is required for the default quickstart. +- Privacy claims imply no data egress while cloud generation is configured. + +**Required evidence** + +- Config tests for default host/CORS/profile behavior. +- CLI doctor tests proving secrets are redacted. +- README/privacy model review. +- Negative tests for disallowed CORS defaults. + +### AC-3: Repository Registry and Job State + +**Pass requires** + +- Repository metadata moves from global dict plus `repositories.json` to SQLite-backed registry with explicit schema and migrations. +- Registry tracks repo id, URL/path, branch, head SHA, indexed_at, status, error, files, chunks, and model/index metadata. +- Index jobs have durable status and error reporting. +- API/MCP/UI access the same registry abstraction. +- Existing `repositories.json` users have a migration path or a documented reset path for pre-1.0 alpha. + +**Fail if** + +- Concurrent API/MCP operations can corrupt repository metadata. +- Index state is lost on process restart except for vector DB contents. +- Different interfaces report conflicting repo state. + +**Required evidence** + +- Registry unit tests using temp SQLite DB. +- Migration tests from sample `repositories.json` or explicit alpha reset behavior test. +- API/MCP integration tests proving shared state. + +### AC-4: Hybrid Retrieval Foundation + +**Pass requires** + +- Retrieval supports semantic vector search, lexical/BM25 or equivalent local text search, path/filename/symbol boosting, score normalization, dedupe, and deterministic context budgeting. +- Vector store is behind an interface so ChromaDB can remain initial backend without coupling the architecture. +- Query results include source, score components, token estimates, and citation candidates. +- Existing `query_code` behavior remains available or has migration tests. + +**Fail if** + +- Retrieval remains vector-only after this milestone. +- Search result order cannot be explained or debugged. +- Duplicated chunks consume context budget without dedupe. + +**Required evidence** + +- Retrieval fixture repo with known expected files/symbols. +- EvalFly retrieval suite comparing vector-only baseline vs hybrid. +- Unit tests for score merging, dedupe, and budget caps. + +### AC-5: Verified Citations and Grounded Answers + +**Pass requires** + +- Citation verifier validates file exists in indexed snapshot, line range exists, range intersects retrieved evidence, and cited content hash matches indexed snapshot. +- `grounded=True` is emitted only when citations are verified. +- Invalid, unsupported, or hallucinated citations are reported separately and degrade answer confidence. +- Answer schema separates answer, retrieved evidence, verified citations, invalid citations, and unsupported claims where implemented. +- Existing citation parser tests are preserved and expanded. + +**Fail if** + +- Any answer is called grounded solely because the LLM emitted `[file:start-end]`. +- Citations to non-retrieved files are silently accepted. +- Stale line numbers pass verification after file changes. + +**Required evidence** + +- Deterministic tests for valid citations, nonexistent file, out-of-range line, stale hash, non-retrieved citation, and malformed citation. +- EvalFly citation-validity suite. +- Negative tests with hallucinated citations. + +### AC-6: Code Intelligence Graph + +**Pass requires** + +- CodeRAG extracts symbols and edges for at least Python and TypeScript/JavaScript fixtures. +- Stored graph supports definitions, references or imports, basic call/import edges where feasible, file-to-symbol mapping, and test-file association heuristics. +- CLI/MCP/API expose `find_symbol`, `find_references`, and `get_blast_radius` or equivalent names. +- Graph extraction is incremental by file hash. +- Unsupported languages degrade to text search with explicit capability flags. + +**Fail if** + +- Tree-sitter remains Python-only while docs claim multi-language code intelligence. +- Blast radius is produced without explaining graph evidence. +- Graph rebuild requires full reindex for every small file change. + +**Required evidence** + +- Fixture repos for Python and TS/JS with expected symbol and edge snapshots. +- Unit tests for parser outputs. +- EvalFly graph suite for symbol lookup and blast-radius expected sets. +- Negative tests for unsupported language fallback. + +### AC-7: Context Packs for Humans and AI Tools + +**Pass requires** + +- CodeRAG can return structured context packs without invoking a generative LLM. +- Context pack includes task/query, files, symbols, snippets, verified citation candidates, score/ranking reasons, token count estimate, and capability/profile metadata. +- MCP exposes context-pack retrieval for external tools. +- CLI can emit JSON and human-readable markdown. +- Context packs enforce max token/character budgets deterministically. + +**Fail if** + +- Context pack generation requires cloud or local LLM. +- Output lacks enough provenance for a reviewer to trust why files were selected. +- Budget settings are ignored. + +**Required evidence** + +- Snapshot tests for JSON schema and markdown rendering. +- MCP tool tests. +- EvalFly context-pack suite measuring expected relevant files and token budget compliance. + +### AC-8: MCP/API/CLI Production Interfaces + +**Pass requires** + +- MCP tools are stable, documented, and covered by tests: index, update, list, query, search_hybrid, get_context_pack, find_symbol, find_references, get_blast_radius, verify_citations. +- CLI exposes equivalent commands where useful. +- REST API has schemas for core operations and error responses. +- All interfaces use shared core services. +- Error responses distinguish validation, missing repo, not indexed, provider failure, and internal failure. + +**Fail if** + +- MCP output is unstructured text where structured data is required. +- CLI/API/MCP disagree on result shape or state. +- Provider errors expose secrets or raw stack traces to normal users. + +**Required evidence** + +- MCP handler/tool tests. +- API route tests through FastAPI test client. +- CLI tests through Click runner. +- Contract snapshots for representative outputs. + +### AC-9: EvalFly and Quality Gates + +**Pass requires** + +- `evals/` exists with config, cases, reports/runs directories, and deterministic smoke suite. +- EvalFly validates and runs locally. +- CI invokes EvalFly check at least for smoke suite before release or merge-readiness claims. +- Reports are cited in milestone completion notes. +- Eval cases cover retrieval, citation verification, context pack budget, graph lookup, and MCP contract smoke. + +**Fail if** + +- Evals exist but are not connected to acceptance criteria. +- Evals require private credentials or unsanitized traces. +- Production-ready claim is made without a passing EvalFly report. + +**Required evidence** + +- `bun run ~/.omp/agent/skills/evalfly/bin/evalfly.ts validate`. +- `bun run ~/.omp/agent/skills/evalfly/bin/evalfly.ts check --suite smoke --commit-range `. +- Saved `evals/reports/.md` and `evals/runs/.json`. + +### AC-10: Production Release Readiness + +**Pass requires** + +- Fresh install path works from clean environment. +- Default quickstart indexes a small public repo or local fixture, answers/retrieves context, and verifies citations. +- Cloud profile works without local LLM hardware. +- Retrieval-only mode works without LLM key. +- Private/local profile is documented as optional and passes smoke on supported hardware/runtime where available. +- Docs match actual commands and behavior. +- Security/privacy defaults pass tests. +- Reviewer validates against original contract. + +**Fail if** + +- Production release requires undocumented credentials, hardware, or manual DB repair. +- README claims features not covered by tests/evals. +- Branch E2E is skipped for user-visible CLI/API/UI behavior. + +**Required evidence** + +- Clean-environment install log. +- Branch E2E run record for CLI/API/UI/MCP user flows. +- Passing CI. +- Passing EvalFly report. +- Reviewer/validator output. + +--- + +## Required Evidence Plan + +### Targeted Tests + +- Unit tests for config profiles, CORS/host defaults, registry schema, migration/reset behavior, vector-store interface, hybrid ranking, dedupe, context budgeting, citation verifier, parser graph extraction, and output schemas. +- Integration tests for API, MCP, CLI using fixture repos and temp storage. +- Contract snapshot tests for context pack JSON, MCP outputs, and human-readable CLI output. +- Negative tests for invalid repo URL, unsupported language fallback, provider failure, missing API key, invalid citation, stale citation, and secret redaction. + +### EvalFly Suites + +Create these deterministic suites: + +1. `smoke` + - validates one fixture repo can index; + - retrieves expected context for one query; + - verifies at least one valid citation; + - checks MCP tool schema for context pack; + - confirms retrieval-only mode does not require LLM key. + +2. `retrieval` + - known query -> expected relevant files/symbols in top-k; + - vector-only baseline recorded; + - hybrid must improve or match recall while staying within budget. + +3. `citations` + - valid file/range/hash passes; + - nonexistent file fails; + - out-of-range line fails; + - non-retrieved citation fails; + - stale snapshot hash fails. + +4. `graph` + - Python fixture symbol extraction; + - TS/JS fixture symbol extraction; + - imports/references/calls where implemented; + - blast radius includes expected dependent file and suggested test file. + +5. `interfaces` + - CLI output JSON schema; + - MCP tool output schema; + - FastAPI response schema; + - errors are typed and secret-redacted. + +6. `security` + - default host is loopback; + - wildcard credentialed CORS is not default; + - `.env` and secret-like files are excluded; + - diagnostics redact configured API keys. + +### Branch E2E Evidence + +Use `agentic-branch-e2e` after implementation for user-visible behavior. Required run records: + +1. **CLI flow** + - create clean temp environment; + - install package; + - run setup with safe test profile or retrieval-only profile; + - index fixture repo; + - run `coderag context` and `coderag query` where applicable; + - verify JSON/markdown output and citation validity; + - negative case: missing provider key in cloud generation path fails closed with actionable error. + +2. **API flow** + - start real FastAPI server with local test config; + - call health; + - index fixture repo; + - poll job; + - retrieve context; + - verify citations; + - negative case: hostile origin/CORS behavior and invalid repo id. + +3. **MCP flow** + - run MCP server through stdio or supported test client; + - call list/index/search/context tools; + - validate structured output; + - negative case: unknown repo and invalid citation verification request. + +4. **UI flow** if Gradio remains supported in release scope + - start real app; + - drive browser-visible index/query path; + - capture screenshot/logs/network; + - negative case: query before repo ready. + +### Logs, Reports, and Reviewer Evidence + +- Startup logs for API/MCP/CLI E2E runs. +- EvalFly report paths and run ids. +- CI run URL or local CI-equivalent transcript. +- Validator/reviewer output against this plan, not just diff. +- Sanitized traces only if trace evidence is used; raw traces must remain under `.pi/evalfly/raw/` and not be committed. + +--- + +## Milestone Roadmap + +### Milestone 0: Governance and Baseline Stabilization + +**Purpose:** Make the project safe to change. + +**Scope** + +- Open SpecSafe slice for implementation. +- Initialize EvalFly with deterministic smoke skeleton. +- Adopt reproducible dev commands using `uv` or equivalent. +- Add CI for tests/lint/typecheck/build/eval smoke. +- Remove generated `__pycache__/` artifacts and ensure they stay ignored. +- Synchronize version fields. +- Document current alpha migration/reset policy. + +**PM go/no-go gate** + +- Go only if a new contributor can clone, install, run tests, and see smoke evidence without manual package guessing. + +**Acceptance criteria covered** + +- AC-0, AC-1, AC-9 foundation. + +**Required verification** + +- `uv run pytest`. +- `uv run ruff check .`. +- `uv run mypy src` or a committed typed-baseline decision with issue list. +- `python -m build`. +- `evalfly validate` and smoke placeholder validation. + +### Milestone 1: Core Service Extraction and Safe Config + +**Purpose:** Eliminate duplicated logic and fix unsafe defaults before new features. + +**Scope** + +- Create shared services: `RepositoryRegistry`, `IndexingService`, `RetrievalService`, `CitationService`, `ProviderConfigService`. +- API/MCP/CLI/UI call services instead of owning pipelines. +- Default host loopback and safe CORS. +- Add privacy/profile model: `cloud`, `hybrid`, `private`, `retrieval-only`. +- Ensure default quickstart does not require local LLM. + +**PM go/no-go gate** + +- Go only if old user flows still work through the refactored services and unsafe defaults are corrected. + +**Acceptance criteria covered** + +- AC-1, AC-2, AC-8 partial. + +**Required verification** + +- Unit tests for profiles and safe config. +- API/MCP/CLI regression tests. +- Branch E2E smoke for CLI/API if behavior changes are visible. + +### Milestone 2: SQLite Registry and Durable Job State + +**Purpose:** Replace fragile global state and JSON metadata. + +**Scope** + +- SQLite registry schema for repos, files, chunks metadata, jobs, provider/index metadata. +- Migration or alpha reset from `repositories.json`. +- Job state for indexing/update/delete. +- Shared registry across API/MCP/CLI/UI. + +**PM go/no-go gate** + +- Go only if process restart preserves accurate repo/job state and interfaces agree. + +**Acceptance criteria covered** + +- AC-3. + +**Required verification** + +- Registry unit/integration tests. +- Restart simulation test. +- API/MCP state consistency test. + +### Milestone 3: Verified Citation Engine + +**Purpose:** Make “verifiable citations” true. + +**Scope** + +- Store file snapshot hashes and line metadata. +- Implement citation verifier. +- Change grounded semantics to require verified citations. +- Expose citation verification through CLI/API/MCP. +- Expand citation tests and EvalFly citation suite. + +**PM go/no-go gate** + +- Go only if hallucinated or stale citations fail deterministically. + +**Acceptance criteria covered** + +- AC-5. + +**Required verification** + +- Citation unit tests. +- EvalFly `citations` suite. +- Negative cases for hallucinated/stale/non-retrieved citations. + +### Milestone 4: Hybrid Retrieval and Context Packs + +**Purpose:** Modernize beyond vector-only RAG and enable no-generation workflows. + +**Scope** + +- Add lexical retrieval. +- Add path/name/symbol metadata boosting where available. +- Add vector-store abstraction. +- Add result merge/dedupe/ranking explanation. +- Add context budgeter. +- Add `ContextPack` schema and CLI/API/MCP outputs. +- Support retrieval-only mode. + +**PM go/no-go gate** + +- Go only if context packs are useful without any generative LLM and beat or match vector-only baseline on eval fixtures. + +**Acceptance criteria covered** + +- AC-4, AC-7. + +**Required verification** + +- Hybrid retrieval unit tests. +- Context pack snapshot tests. +- EvalFly `retrieval` and `context-pack` cases. +- Branch E2E CLI/API context-pack flow. + +### Milestone 5: Code Intelligence Graph MVP + +**Purpose:** Add code-structure awareness. + +**Scope** + +- Multi-language parser MVP: Python and TypeScript/JavaScript. +- Symbol table and file-to-symbol mapping. +- Import/reference/call edges where feasible. +- Test-file association heuristics. +- Incremental graph update by file hash. +- CLI/API/MCP: `find_symbol`, `find_references`, `get_blast_radius`. + +**PM go/no-go gate** + +- Go only if graph outputs are deterministic on fixtures and unsupported languages explicitly degrade. + +**Acceptance criteria covered** + +- AC-6. + +**Required verification** + +- Parser fixture tests. +- Graph snapshot tests. +- EvalFly `graph` suite. +- Negative fallback tests. + +### Milestone 6: Production Interfaces and Docs + +**Purpose:** Make the product usable and supportable. + +**Scope** + +- Stabilize CLI commands and MCP tool contracts. +- Add OpenAPI schemas and examples. +- Rewrite README around corrected positioning. +- Add privacy/profile docs. +- Add migration docs. +- Add troubleshooting docs for provider keys, local/private profiles, and retrieval-only mode. + +**PM go/no-go gate** + +- Go only if docs match commands proven by E2E and do not claim unsupported features. + +**Acceptance criteria covered** + +- AC-8, AC-10 docs portion. + +**Required verification** + +- CLI/API/MCP contract tests. +- README command smoke tests where feasible. +- Reviewer docs audit. + +### Milestone 7: Production Release Candidate + +**Purpose:** Prove the whole product works end-to-end. + +**Scope** + +- Clean install from source and package build. +- Full targeted tests. +- EvalFly smoke/retrieval/citations/graph/interfaces/security. +- Branch E2E CLI/API/MCP/UI where supported. +- Reviewer validation against this contract. +- Release checklist and rollback notes. + +**PM go/no-go gate** + +- Go only if every AC is PASS or explicitly removed from release scope by user approval before implementation; no silent scope shrink. + +**Acceptance criteria covered** + +- AC-0 through AC-10. + +**Required verification** + +- Passing CI-equivalent transcript. +- EvalFly passing report paths. +- Branch E2E run record(s). +- Validator/reviewer output. + +--- + +## PM Gap Register + +| ID | Gap | Severity | Risk if ignored | Resolution before development | +|---|---|---:|---|---| +| PM-1 | Release persona priority not locked | High | Features optimize for wrong user | Prioritize developer + external tool context workflows over autonomous agents | +| PM-2 | Privacy wording can overpromise | High | Trust/reputation risk | Use profile-specific privacy matrix; no blanket local-first claim | +| PM-3 | Local model expectation unclear | High | Adoption blocked by hardware assumptions | Cloud profile default; local/private optional | +| PM-4 | Existing PyPI users migration unclear | Medium | Upgrade confusion | Decide alpha reset vs JSON migration in Milestone 2 | +| PM-5 | UI importance unclear | Medium | Gradio absorbs time with low strategic value | Treat UI as secondary unless explicitly required in release scope | +| PM-6 | Supported languages could sprawl | High | Graph MVP never finishes | Commit to Python + TS/JS first; fallback for others | +| PM-7 | Eval success metrics not public | High | CV/project claims look ungrounded | Publish retrieval/citation/token-budget metrics in README after evals pass | +| PM-8 | Provider costs and data egress unpriced | Medium | User surprise | Add provider matrix with cost/privacy tradeoffs | +| PM-9 | MCP client support scope can balloon | Medium | Installation complexity | Support generic MCP stdio first; document Claude/Codex/Cursor examples after tests | +| PM-10 | Production readiness conflated with feature completeness | Critical | Premature release | Use AC-10 gate and reviewer validation before release claim | + +## Technical Decision Log Required Before Milestone 1 + +1. Dev environment manager: use `uv` unless a conflicting project constraint appears. +2. SQLite migration policy: support JSON migration or declare pre-1.0 reset. +3. Vector abstraction: keep ChromaDB backend initially; define interface before adding alternatives. +4. Lexical backend: choose BM25 implementation or SQLite FTS/ripgrep-backed index. +5. Parser package strategy: choose explicit Tree-sitter grammars vs `tree-sitter-language-pack`. +6. Provider abstraction: choose direct adapters vs LiteLLM. Default recommendation: provider adapter boundary now, LiteLLM evaluation before broad provider expansion. +7. Public API stability level: mark 2.0 pre-release contracts as beta until contract tests stabilize. + +## Out-of-Scope for First Production Release + +- Autonomous code editing. +- Pull request creation. +- Multi-agent orchestration. +- Hosted SaaS deployment. +- Full 30+ language graph support. +- LLM judge as a required gate. +- Enterprise auth/multi-user RBAC. +- Performance guarantees for very large monorepos beyond documented fixture benchmarks. + +## Release Readiness Checklist + +- [ ] SpecSafe slice references this plan. +- [ ] EvalFly initialized and validated. +- [ ] CI runs tests/lint/typecheck/build/eval smoke. +- [ ] Default quickstart requires no local generative LLM. +- [ ] Retrieval-only flow works without LLM credentials. +- [ ] Cloud profile documents code-context egress. +- [ ] Citation verifier blocks invalid/stale/non-retrieved citations. +- [ ] Hybrid retrieval eval passes against vector-only baseline. +- [ ] Graph MVP works for Python + TS/JS fixtures. +- [ ] MCP/API/CLI output contracts are tested. +- [ ] Branch E2E run records exist for visible flows. +- [ ] Reviewer validates implementation against AC-0 through AC-10. +- [ ] README claims match tested behavior. + +## Final Planning Verdict + +This plan is ready to use as a PM/engineering contract for implementation planning, but the product itself is not production-ready yet. The correct next action is to begin Milestone 0 in a fresh implementation slice, initialize EvalFly, and implement only against the frozen acceptance criteria above. + +--- + +## Parallel Readiness Review Integration + +Three specialist reviews were run against this plan before final delivery: + +- `PMReadinessReview`: product/delivery readiness. +- `ArchitectureReadinessReview`: production architecture readiness. +- `EvidencePlanReview`: EvalFly/evidence readiness. + +Their findings are incorporated into the gates above. The non-negotiable deltas from those reviews are: + +1. **Fase 0 is not optional.** Before feature-heavy work, freeze the product contract: ICP, top workflows, no-goals, North Star metric, and beta scope. +2. **Verified citations are the product trust core.** Citation verification must land before graph-heavy or UI-heavy work. +3. **Service seam precedes feature work.** API, MCP, CLI, and UI must become adapters over shared services before hybrid retrieval/graph work. +4. **SQLite registry must model snapshots, jobs, chunks, and schema versions.** It cannot be a shallow replacement for `repositories.json`. +5. **Source snapshots and stable chunk identity are prerequisites.** Citation verification, hybrid retrieval, graph, and incremental indexing depend on them. +6. **Hybrid retrieval must be measured against vector-only baseline.** Use retrieval traces and EvalFly reports; do not claim improvement from architecture alone. +7. **Graph MVP must be scoped.** Python + TS/JS is the maximum first production scope unless fixtures/evals prove more languages. +8. **Security/cloud readiness is a first-class milestone.** CORS, auth, quotas, clone restrictions, secret redaction, and storage state cannot be left to final cleanup. +9. **MCP-first does not mean agentic autonomy.** CodeRAG exposes context/evidence tools; it does not edit, execute, branch, commit, or open PRs. +10. **Production-ready means evidence-complete.** The final release gate requires CI, EvalFly reports, branch E2E run records, security checks, and reviewer validation. + +## PM Approval Packet Required Before Development + +A project manager can approve Milestone 0 only after this packet exists: + +- ICP: primary user is explicitly chosen. +- Top 3 workflows are ranked. +- Non-goals are signed off: no autonomous editing, no mandatory local LLM, no unsupported privacy claims. +- North Star metric is chosen. +- Beta scope is chosen: recommended default is public/local repos first, private/cloud multi-tenant only after security gates. +- Language support matrix is chosen. +- Production target is chosen: package/CLI, self-hosted API, cloud service, or staged combination. +- Cost/privacy matrix for provider profiles is published. +- Owner is assigned for each go/no-go gate. + +Recommended default PM contract: + +- **Primary user:** developer or AI-engineering practitioner who needs trustworthy codebase context for human use and external coding tools. +- **Top workflows:** `index repo -> get context pack`, `ask question -> verified citations`, `changed files -> blast radius/test suggestions`. +- **North Star:** percentage of real queries that return useful context with 100% verified citations within the configured token budget. +- **Beta:** CLI/MCP/API over local/public repos first; Gradio UI secondary; cloud/private repos only after security/storage hardening. + +## Additional Technical Gates from Architecture Review + +Before each milestone can close, the implementation must prove: + +- `RepositoryRegistry` is the single source of truth for repo state. +- `SourceSnapshot` can reconstruct cited lines by snapshot id without trusting mutable working-tree `HEAD`. +- Chunk IDs are stable and include file path/range/content hash/chunker version or equivalent deterministic identity. +- Retrieval returns traceable scores per stage: vector, lexical, graph expansion, fusion, rerank if present. +- Provider adapters return usage, latency, provider/model, finish reason, and retryability; Anthropic native is not treated as OpenAI-compatible unless routed through an OpenAI-compatible gateway. +- Cloud/default Docker path does not require NVIDIA GPU. +- API/MCP/CLI parity is tested with one deterministic provider or generation stub. + +## Additional EvalFly Gates from Evidence Review + +The minimum blocking `smoke` suite must include: + +- exact-symbol retrieval fixture; +- behavioral multi-file retrieval fixture; +- no-evidence/abstention fixture; +- valid citation fixture; +- hallucinated citation rejection fixture; +- context-pack schema and budget fixture; +- Python graph extraction fixture; +- stale graph edge removal fixture once graph exists; +- repository ingestion security boundary fixture; +- cloud config/CORS fixture; +- MCP query contract fixture; +- CLI/API/MCP parity fixture. + +Global production verdict rules: + +- **PASS** requires deterministic smoke + regression suites, 100% valid citations for grounded fixture answers, context-pack schema validity, graph claims backed by fixtures, security gates, MCP E2E, branch E2E, CI, and human readiness review. +- **FAIL** if a hallucinated/stale citation can be marked grounded, security fails open, or CI/EvalFly gates are absent. +- **INCONCLUSIVE** if any claimed capability lacks fixture/golden labels, relies only on mocks that bypass real ingestion/retrieval interfaces, or is judged only by LLM/human review when deterministic evidence is possible. diff --git a/docs/superpowers/plans/2026-06-27-m0-baseline-contract.md b/docs/superpowers/plans/2026-06-27-m0-baseline-contract.md new file mode 100644 index 0000000..da24399 --- /dev/null +++ b/docs/superpowers/plans/2026-06-27-m0-baseline-contract.md @@ -0,0 +1,92 @@ +# CodeRAG Milestone 0 Baseline Contract + +Slice: `CRAG-20260627-M0-BASELINE` +Parent plan: `docs/superpowers/plans/2026-06-27-coderag-2-production-roadmap.md` + +## Scope + +Milestone 0 establishes the delivery harness and fixes the highest-risk baseline contradictions before feature work begins. It does not implement hybrid retrieval, verified citations, graph extraction, SQLite registry, or autonomous agent behavior. + +## Frozen acceptance criteria for this slice + +### M0-AC1: Governance + +Pass requires: +- SpecSafe slice `CRAG-20260627-M0-BASELINE` is open before edits. +- This contract exists before implementation edits. +- EvalFly project shape exists and validates. + +Fail if: +- Product code changes before the slice/contract/eval baseline exist. +- Missing evals are relabeled as not applicable. + +Required evidence: +- SpecSafe status. +- `evalfly validate` output. + +### M0-AC2: Version baseline + +Pass requires: +- Package runtime version, default settings version, and default YAML version agree with `pyproject.toml`. + +Fail if: +- Any user-visible surface still reports `0.1.0` while package metadata reports `0.1.2`. + +Required evidence: +- Targeted pytest for version synchronization. + +### M0-AC3: Safe default server exposure + +Pass requires: +- Default server bind host is `127.0.0.1`. +- CLI `serve` default host is `127.0.0.1`. +- CORS defaults do not combine wildcard origins with credentials. +- Default YAML mirrors safe server defaults. + +Fail if: +- A fresh default serve path binds to `0.0.0.0`. +- Default CORS allows `*` with credentials. + +Required evidence: +- Targeted pytest for settings, CLI defaults, and YAML defaults. + +### M0-AC4: Secret redaction smoke + +Pass requires: +- `coderag doctor` does not print configured API key values. + +Fail if: +- Any API key value appears in doctor output. + +Required evidence: +- Targeted pytest using isolated CLI config. + +### M0-AC5: CI baseline + +Pass requires: +- A GitHub Actions validation workflow exists separately from publish. +- It includes test, lint, typecheck/baseline, package build, and eval configuration validation steps. + +Fail if: +- Publishing remains the only workflow. + +Required evidence: +- Workflow file exists. +- EvalFly smoke case checks the workflow file exists. + +## Evidence plan + +Run after implementation: + +```bash +PYTHONPATH=src .venv/bin/python -m pytest tests/test_m0_baseline.py +bun run ~/.omp/agent/skills/evalfly/bin/evalfly.ts validate +bun run ~/.omp/agent/skills/evalfly/bin/evalfly.ts run --suite smoke --commit-range main..HEAD +bun run ~/.omp/agent/skills/evalfly/bin/evalfly.ts check --suite smoke --commit-range main..HEAD +``` + +Optional if dependency installation completes: + +```bash +.venv/bin/python -m build +``` From a3b73cb58cf636fa451394f719a51c6afc31f53d Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 14:42:19 -0600 Subject: [PATCH 2/8] fix: align runtime version and safe server defaults --- configs/default.yaml | 8 ++++++-- src/coderag/__init__.py | 7 ++++++- src/coderag/cli.py | 2 +- src/coderag/config.py | 10 ++++++++-- src/coderag/main.py | 4 ++-- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/configs/default.yaml b/configs/default.yaml index 1d37d28..7b730c9 100644 --- a/configs/default.yaml +++ b/configs/default.yaml @@ -3,16 +3,20 @@ app: name: CodeRAG - version: 0.1.0 + version: 0.1.2 debug: false server: - host: 0.0.0.0 + host: 127.0.0.1 port: 8000 reload: false workers: 1 log_level: info + cors_allowed_origins: + - http://127.0.0.1:8000 + - http://localhost:8000 + cors_allow_credentials: true data: base_dir: ./data repos_cache: ./data/repos diff --git a/src/coderag/__init__.py b/src/coderag/__init__.py index e55695a..db48044 100644 --- a/src/coderag/__init__.py +++ b/src/coderag/__init__.py @@ -1,3 +1,8 @@ """CodeRAG: RAG-based Q&A system for code repositories with verifiable citations.""" -__version__ = "0.1.0" +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version("code-rag-me") +except PackageNotFoundError: + __version__ = "0.1.2" diff --git a/src/coderag/cli.py b/src/coderag/cli.py index e4c327b..68634ff 100644 --- a/src/coderag/cli.py +++ b/src/coderag/cli.py @@ -149,7 +149,7 @@ def _validate_api_key(provider: str, api_key: str) -> bool: @cli.command() -@click.option("--host", default="0.0.0.0", help="Host to bind to") +@click.option("--host", default="127.0.0.1", help="Host to bind to") @click.option("--port", default=8000, type=int, help="Port to bind to") @click.option("--reload", is_flag=True, help="Enable auto-reload for development") def serve(host: str, port: int, reload: bool): diff --git a/src/coderag/config.py b/src/coderag/config.py index eb430d4..0362242 100644 --- a/src/coderag/config.py +++ b/src/coderag/config.py @@ -5,6 +5,7 @@ from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict +from coderag import __version__ class ModelSettings(BaseSettings): @@ -29,6 +30,7 @@ class ModelSettings(BaseSettings): # Local model settings llm_use_4bit: bool = True llm_device_map: str = "auto" + allow_remote_code: bool = False embedding_name: str = "nomic-ai/nomic-embed-text-v1.5" embedding_dimension: int = 768 @@ -108,11 +110,15 @@ class ServerSettings(BaseSettings): model_config = SettingsConfigDict(env_prefix="SERVER_") - host: str = "0.0.0.0" + host: str = "127.0.0.1" port: int = 8000 reload: bool = False workers: int = 1 log_level: str = "info" + cors_allowed_origins: list[str] = Field( + default_factory=lambda: ["http://127.0.0.1:8000", "http://localhost:8000"] + ) + cors_allow_credentials: bool = True class Settings(BaseSettings): @@ -125,7 +131,7 @@ class Settings(BaseSettings): ) app_name: str = "CodeRAG" - app_version: str = "0.1.0" + app_version: str = __version__ debug: bool = False data_dir: Path = Path("./data") diff --git a/src/coderag/main.py b/src/coderag/main.py index 7c473b8..e0b3c99 100644 --- a/src/coderag/main.py +++ b/src/coderag/main.py @@ -42,8 +42,8 @@ def create_app() -> FastAPI: # CORS middleware app.add_middleware( CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, + allow_origins=settings.server.cors_allowed_origins, + allow_credentials=settings.server.cors_allow_credentials, allow_methods=["*"], allow_headers=["*"], ) From 3afeb83d8cbdf0534c3ce7caa23f04a4e1aa41d2 Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 14:42:27 -0600 Subject: [PATCH 3/8] fix: disable remote model code by default --- .env.example | 11 +++++++++-- configs/models.yaml | 2 +- src/coderag/generation/generator.py | 4 ++-- src/coderag/indexing/embeddings.py | 4 ++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index e027955..278c447 100644 --- a/.env.example +++ b/.env.example @@ -14,16 +14,20 @@ # ============================================================================= DEBUG=false APP_NAME=CodeRAG -APP_VERSION=0.1.0 +APP_VERSION=0.1.2 # ============================================================================= # Server Settings # ============================================================================= -SERVER_HOST=0.0.0.0 +SERVER_HOST=127.0.0.1 SERVER_PORT=8000 SERVER_LOG_LEVEL=info SERVER_WORKERS=1 SERVER_RELOAD=false +# CORS defaults trust only loopback browser origins. +# Use JSON list syntax when overriding through environment variables. +SERVER_CORS_ALLOWED_ORIGINS='["http://127.0.0.1:8000","http://localhost:8000"]' +SERVER_CORS_ALLOW_CREDENTIALS=true # ============================================================================= # Model Settings @@ -51,6 +55,9 @@ MODEL_LLM_API_BASE= # For local: Qwen/Qwen2.5-Coder-3B-Instruct (requires GPU with 8GB VRAM) MODEL_LLM_NAME=llama-3.3-70b-versatile MODEL_LLM_MAX_NEW_TOKENS=1024 +# Remote model repository code execution is disabled by default. +# Set true only after reviewing and pinning the model source. +MODEL_ALLOW_REMOTE_CODE=false MODEL_LLM_TEMPERATURE=0.1 MODEL_LLM_TOP_P=0.95 MODEL_LLM_USE_4BIT=true diff --git a/configs/models.yaml b/configs/models.yaml index 4071173..c00e8d7 100644 --- a/configs/models.yaml +++ b/configs/models.yaml @@ -18,7 +18,7 @@ llm: # Device settings device_map: auto - trust_remote_code: true + trust_remote_code: false # Fallback model for constrained environments fallback: diff --git a/src/coderag/generation/generator.py b/src/coderag/generation/generator.py index 8055e3a..b37218e 100644 --- a/src/coderag/generation/generator.py +++ b/src/coderag/generation/generator.py @@ -118,14 +118,14 @@ def _load_local_model(self): self._local_tokenizer = AutoTokenizer.from_pretrained( self.settings.models.llm_name, - trust_remote_code=True, + trust_remote_code=self.settings.models.allow_remote_code, ) self._local_model = AutoModelForCausalLM.from_pretrained( self.settings.models.llm_name, quantization_config=bnb_config, device_map=self.settings.models.llm_device_map, - trust_remote_code=True, + trust_remote_code=self.settings.models.allow_remote_code, torch_dtype=torch.float16, ) diff --git a/src/coderag/indexing/embeddings.py b/src/coderag/indexing/embeddings.py index ae123c5..2aae924 100644 --- a/src/coderag/indexing/embeddings.py +++ b/src/coderag/indexing/embeddings.py @@ -76,7 +76,7 @@ def _load_model(self) -> None: self._model = SentenceTransformer( self.model_name, device=self.device, - trust_remote_code=True, + trust_remote_code=settings.models.allow_remote_code, ) logger.info("Embedding model loaded", device=self.device) except (torch.cuda.OutOfMemoryError, RuntimeError) as e: @@ -87,7 +87,7 @@ def _load_model(self) -> None: self._model = SentenceTransformer( self.model_name, device="cpu", - trust_remote_code=True, + trust_remote_code=settings.models.allow_remote_code, ) logger.info("Embedding model loaded on CPU (fallback)") else: From 598f1182646a2594608253da52809175a5842f61 Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 14:42:33 -0600 Subject: [PATCH 4/8] docs: document loopback deployment defaults --- README.md | 7 +++++-- docker-compose.yaml | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a8bacc5..afc4ebc 100644 --- a/README.md +++ b/README.md @@ -278,9 +278,12 @@ MODEL_LLM_API_KEY=your-api-key # Embeddings (runs locally on CPU by default) MODEL_EMBEDDING_DEVICE=auto # auto, cuda, or cpu -# Server -SERVER_HOST=0.0.0.0 +# Server defaults to loopback for local safety. +SERVER_HOST=127.0.0.1 SERVER_PORT=8000 + +# Bind to 0.0.0.0 only as an explicit deployment override +# after configuring trusted CORS origins and network access controls. ``` ### Config File diff --git a/docker-compose.yaml b/docker-compose.yaml index d1c9718..1b65531 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -19,7 +19,8 @@ services: capabilities: [gpu] ports: - - "8000:8000" + - "127.0.0.1:8000:8000" + # The container still binds internally; host exposure stays loopback by default. environment: - DEBUG=false From 8bf134b93657d9576f5033dc35a09470e5df69c7 Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 14:42:39 -0600 Subject: [PATCH 5/8] test: add Milestone 0 baseline coverage --- tests/test_m0_baseline.py | 130 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 tests/test_m0_baseline.py diff --git a/tests/test_m0_baseline.py b/tests/test_m0_baseline.py new file mode 100644 index 0000000..289d9ca --- /dev/null +++ b/tests/test_m0_baseline.py @@ -0,0 +1,130 @@ +"""Milestone 0 baseline tests. + +These tests pin the governance-critical defaults before feature work continues. +""" + +import json +import tomllib +from pathlib import Path +from typing import Any + +import pytest +import yaml +from click.testing import CliRunner + +import coderag +import coderag.cli as cli_module +from coderag.config import ModelSettings, ServerSettings, Settings + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def test_runtime_and_default_config_versions_match_pyproject() -> None: + """User-visible version surfaces must agree with package metadata.""" + pyproject = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) + expected_version = pyproject["project"]["version"] + default_yaml: dict[str, Any] = yaml.safe_load( + (REPO_ROOT / "configs" / "default.yaml").read_text() + ) + + assert coderag.__version__ == expected_version + assert Settings().app_version == expected_version + assert default_yaml["app"]["version"] == expected_version + + +def test_env_example_does_not_reintroduce_stale_version_or_broad_bind() -> None: + """Copying .env.example must preserve the safe Milestone 0 defaults.""" + entries = {} + for line in (REPO_ROOT / ".env.example").read_text().splitlines(): + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + entries[key] = value + + pyproject = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text()) + assert entries["APP_VERSION"] == pyproject["project"]["version"] + assert entries["SERVER_HOST"] == "127.0.0.1" + + +def test_readme_configuration_documents_loopback_server_default() -> None: + """README quick configuration must not tell local users to bind broadly by default.""" + readme = (REPO_ROOT / "README.md").read_text() + + assert "SERVER_HOST=127.0.0.1" in readme + + +def test_docker_compose_exposes_service_on_host_loopback() -> None: + """The compose profile can bind inside the container while exposing only host loopback.""" + compose = (REPO_ROOT / "docker-compose.yaml").read_text() + + assert '"127.0.0.1:8000:8000"' in compose + + +def test_ci_runs_committed_evalfly_smoke_runner() -> None: + """CI must execute a semantic smoke check for EvalFly cases, not only JSON syntax.""" + workflow = (REPO_ROOT / ".github" / "workflows" / "ci.yml").read_text() + + assert "scripts/evalfly_smoke.py" in workflow + assert (REPO_ROOT / "scripts" / "evalfly_smoke.py").exists() + + +def test_default_model_loading_does_not_trust_remote_code() -> None: + """Remote model code execution must be an explicit opt-in, never a default path.""" + assert ModelSettings().allow_remote_code is False + + embedding_source = (REPO_ROOT / "src" / "coderag" / "indexing" / "embeddings.py").read_text() + generator_source = (REPO_ROOT / "src" / "coderag" / "generation" / "generator.py").read_text() + models_yaml = yaml.safe_load((REPO_ROOT / "configs" / "models.yaml").read_text()) + + assert "trust_remote_code=True" not in embedding_source + assert "trust_remote_code=True" not in generator_source + assert models_yaml["llm"]["trust_remote_code"] is False + +def test_server_defaults_bind_to_loopback_and_disallow_wildcard_credentialed_cors() -> None: + """Fresh server defaults must not expose a credentialed API to arbitrary origins.""" + settings = ServerSettings() + + assert settings.host == "127.0.0.1" + assert "*" not in settings.cors_allowed_origins + assert not (settings.cors_allow_credentials and settings.cors_allowed_origins == ["*"]) + + +def test_default_yaml_mirrors_safe_server_defaults() -> None: + """The shipped YAML config must mirror the safe runtime defaults.""" + default_yaml: dict[str, Any] = yaml.safe_load( + (REPO_ROOT / "configs" / "default.yaml").read_text() + ) + server = default_yaml["server"] + + assert server["host"] == "127.0.0.1" + assert "*" not in server["cors_allowed_origins"] + assert not (server["cors_allow_credentials"] and server["cors_allowed_origins"] == ["*"]) + + +def test_cli_serve_default_host_is_loopback() -> None: + """The CLI serve command must inherit the safe loopback default.""" + serve_command = cli_module.cli.commands["serve"] + host_param = next(param for param in serve_command.params if param.name == "host") + + assert host_param.default == "127.0.0.1" + + +def test_doctor_does_not_print_configured_api_key( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Doctor may report that a key exists, but must never print the secret value.""" + secret = "sk-test-secret-value-that-must-not-leak" + config_dir = tmp_path / "config" + config_file = config_dir / "config.json" + config_dir.mkdir() + config_file.write_text(json.dumps({"llm_provider": "groq", "llm_api_key": secret})) + + monkeypatch.setattr(cli_module, "CONFIG_DIR", config_dir) + monkeypatch.setattr(cli_module, "CONFIG_FILE", config_file) + monkeypatch.setenv("CODERAG_DATA_DIR", str(tmp_path / "data")) + + result = CliRunner().invoke(cli_module.cli, ["doctor"]) + + assert result.exit_code == 0 + assert secret not in result.output + assert "API key configured" in result.output From c79205537fcda61fbc82f99225f28a04fe9f51db Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 14:42:45 -0600 Subject: [PATCH 6/8] ci: add Milestone 0 validation gate --- .github/workflows/ci.yml | 44 +++++++++++++++++++ scripts/evalfly_smoke.py | 94 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 scripts/evalfly_smoke.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8e59139 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + validate: + name: Validate package + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install baseline validation tools + run: | + python -m pip install --upgrade pip + python -m pip install build + python -m pip install -e . --no-deps + python -m pip install pytest pydantic pydantic-settings click PyYAML fastapi structlog ruff mypy + + - name: Run Milestone 0 tests + run: PYTHONPATH=src python -m pytest tests/test_m0_baseline.py + + - name: Run Milestone 0 lint baseline + run: ruff check tests/test_m0_baseline.py scripts/evalfly_smoke.py + + - name: Run Milestone 0 type baseline + run: PYTHONPATH=src mypy tests/test_m0_baseline.py scripts/evalfly_smoke.py + + - name: Compile changed runtime modules + run: python -m py_compile src/coderag/__init__.py src/coderag/config.py src/coderag/main.py src/coderag/cli.py src/coderag/indexing/embeddings.py src/coderag/generation/generator.py tests/test_m0_baseline.py scripts/evalfly_smoke.py + + - name: Build package + run: python -m build + + - name: Run committed EvalFly smoke baseline + run: python scripts/evalfly_smoke.py --suite smoke diff --git a/scripts/evalfly_smoke.py b/scripts/evalfly_smoke.py new file mode 100644 index 0000000..6ea6a03 --- /dev/null +++ b/scripts/evalfly_smoke.py @@ -0,0 +1,94 @@ +"""Run the committed deterministic subset of the EvalFly smoke suite. + +This CI helper is intentionally narrow: it validates the repository-local EvalFly +configuration enough to execute deterministic `file_exists` assertions in GitHub +Actions, while the full OMP EvalFly CLI remains the authoritative local tool. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any, cast + +REPO_ROOT = Path(__file__).resolve().parents[1] +CONFIG_PATH = REPO_ROOT / "evals" / "config.json" + + +class SmokeFailure(Exception): + """Raised when the committed smoke suite fails.""" + + +def _load_config() -> dict[str, Any]: + try: + config = cast(dict[str, Any], json.loads(CONFIG_PATH.read_text())) + except FileNotFoundError as exc: + raise SmokeFailure(f"missing EvalFly config: {CONFIG_PATH}") from exc + except json.JSONDecodeError as exc: + raise SmokeFailure(f"invalid EvalFly config JSON: {exc}") from exc + + if config.get("schema_version") != "evalfly.config.v1": + raise SmokeFailure("evals/config.json must use schema_version evalfly.config.v1") + if not isinstance(config.get("name"), str) or not config["name"]: + raise SmokeFailure("evals/config.json must define a non-empty name") + if not isinstance(config.get("cases"), list) or not config["cases"]: + raise SmokeFailure("evals/config.json must define at least one case") + return config + + +def _run_case(case: dict[str, Any]) -> None: + case_id = case.get("case_id", "") + if case.get("schema_version") != "evalfly.case.v1": + raise SmokeFailure(f"{case_id}: expected schema_version evalfly.case.v1") + if case.get("suite") != "smoke": + raise SmokeFailure(f"{case_id}: committed runner only supports smoke cases") + + judge = case.get("judge") + if not isinstance(judge, dict) or judge.get("type") != "deterministic": + raise SmokeFailure(f"{case_id}: smoke cases must use deterministic judges") + + assertions = judge.get("assertions") + if not isinstance(assertions, list) or not assertions: + raise SmokeFailure(f"{case_id}: deterministic judge must include assertions") + + for assertion in assertions: + if not isinstance(assertion, dict): + raise SmokeFailure(f"{case_id}: assertion must be an object") + if assertion.get("type") != "file_exists": + raise SmokeFailure(f"{case_id}: unsupported assertion type {assertion.get('type')!r}") + raw_path = assertion.get("path") + if not isinstance(raw_path, str) or not raw_path: + raise SmokeFailure(f"{case_id}: file_exists assertion requires a path") + if not (REPO_ROOT / raw_path).is_file(): + raise SmokeFailure(f"{case_id}: expected file does not exist: {raw_path}") + + +def run_smoke() -> int: + """Run all committed deterministic smoke cases.""" + config = _load_config() + cases = config["cases"] + for case in cases: + if not isinstance(case, dict): + raise SmokeFailure("each EvalFly case must be an object") + _run_case(case) + print(f"evalfly committed smoke passed: {len(cases)} cases") + return len(cases) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--suite", default="smoke", choices=["smoke"]) + parser.parse_args(argv) + + try: + run_smoke() + except SmokeFailure as exc: + print(f"evalfly committed smoke failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From f3d083dc5e66cc67a683ed205101cbd0860374d7 Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 14:42:52 -0600 Subject: [PATCH 7/8] chore: record Milestone 0 evaluation evidence Spec-Slice: CRAG-20260627-M0-REVIEW-FIXES --- .pi/.specsafe-state.json | 65 +++++++++++ evals/config.json | 126 ++++++++++++++++++++++ evals/reports/.gitkeep | 0 evals/reports/run-smoke-20260627202411.md | 19 ++++ evals/reports/run-smoke-20260627202414.md | 19 ++++ evals/reports/run-smoke-20260627203416.md | 20 ++++ evals/reports/run-smoke-20260627203451.md | 20 ++++ evals/rubrics/README.md | 7 ++ evals/runs/.gitkeep | 0 evals/runs/run-smoke-20260627202411.json | 58 ++++++++++ evals/runs/run-smoke-20260627202414.json | 58 ++++++++++ evals/runs/run-smoke-20260627203416.json | 68 ++++++++++++ evals/runs/run-smoke-20260627203451.json | 70 ++++++++++++ evals/traces/sanitized/.gitkeep | 0 14 files changed, 530 insertions(+) create mode 100644 .pi/.specsafe-state.json create mode 100644 evals/config.json create mode 100644 evals/reports/.gitkeep create mode 100644 evals/reports/run-smoke-20260627202411.md create mode 100644 evals/reports/run-smoke-20260627202414.md create mode 100644 evals/reports/run-smoke-20260627203416.md create mode 100644 evals/reports/run-smoke-20260627203451.md create mode 100644 evals/rubrics/README.md create mode 100644 evals/runs/.gitkeep create mode 100644 evals/runs/run-smoke-20260627202411.json create mode 100644 evals/runs/run-smoke-20260627202414.json create mode 100644 evals/runs/run-smoke-20260627203416.json create mode 100644 evals/runs/run-smoke-20260627203451.json create mode 100644 evals/traces/sanitized/.gitkeep diff --git a/.pi/.specsafe-state.json b/.pi/.specsafe-state.json new file mode 100644 index 0000000..c57a86e --- /dev/null +++ b/.pi/.specsafe-state.json @@ -0,0 +1,65 @@ +{ + "currentSlice": null, + "history": [ + { + "sliceId": "CRAG-20260627-ROADMAP", + "workspaceId": "CodeRAG", + "sessionId": "omp-session", + "beganAt": "2026-06-27T19:59:14.649Z", + "endedAt": "2026-06-27T20:08:58.201Z", + "outcome": "PASS", + "costSummary": { + "externalMemoryCalls": 0, + "externalMemoryCost": 0, + "subagentTokens": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "cost": 0, + "turns": 0 + } + } + }, + { + "sliceId": "CRAG-20260627-M0-BASELINE", + "workspaceId": "CodeRAG", + "sessionId": "omp-session", + "beganAt": "2026-06-27T20:18:54.230Z", + "endedAt": "2026-06-27T20:24:37.068Z", + "outcome": "PASS", + "costSummary": { + "externalMemoryCalls": 0, + "externalMemoryCost": 0, + "subagentTokens": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "cost": 0, + "turns": 0 + } + } + }, + { + "sliceId": "CRAG-20260627-M0-REVIEW-FIXES", + "workspaceId": "CodeRAG", + "sessionId": "omp-session", + "beganAt": "2026-06-27T20:34:36.861Z", + "endedAt": "2026-06-27T20:39:34.398Z", + "outcome": "PASS", + "costSummary": { + "externalMemoryCalls": 0, + "externalMemoryCost": 0, + "subagentTokens": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "cost": 0, + "turns": 0 + } + } + } + ] +} diff --git a/evals/config.json b/evals/config.json new file mode 100644 index 0000000..922a725 --- /dev/null +++ b/evals/config.json @@ -0,0 +1,126 @@ +{ + "schema_version": "evalfly.config.v1", + "name": "CodeRAG Milestone 0 baseline smoke suite", + "cases": [ + { + "schema_version": "evalfly.case.v1", + "case_id": "m0-roadmap-contract-exists", + "title": "Milestone 0 acceptance contract exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "delivery_governance", + "source": { + "kind": "specsafe_slice", + "slice_id": "CRAG-20260627-M0-BASELINE" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "The Milestone 0 acceptance and evidence contract is present in the repository." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "docs/superpowers/plans/2026-06-27-m0-baseline-contract.md" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m0-targeted-tests-exist", + "title": "Milestone 0 targeted tests exist", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "test_coverage", + "source": { + "kind": "acceptance_criteria", + "criteria": "M0-AC2 M0-AC3 M0-AC4" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "The targeted baseline tests for version, safe defaults, and secret redaction exist." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "tests/test_m0_baseline.py" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m0-ci-workflow-exists", + "title": "CI validation workflow exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "ci_baseline", + "source": { + "kind": "acceptance_criteria", + "criteria": "M0-AC5" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "A non-publish GitHub Actions workflow exists for validation gates." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": ".github/workflows/ci.yml" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m0-committed-evalfly-runner-exists", + "title": "Committed EvalFly smoke runner exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "ci_baseline", + "source": { + "kind": "review_finding", + "finding": "CI must run a semantic smoke check, not only JSON syntax" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "A committed deterministic runner exists for CI to execute EvalFly smoke file_exists assertions." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "scripts/evalfly_smoke.py" + } + ] + } + } + ] +} diff --git a/evals/reports/.gitkeep b/evals/reports/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/evals/reports/run-smoke-20260627202411.md b/evals/reports/run-smoke-20260627202411.md new file mode 100644 index 0000000..4d6df76 --- /dev/null +++ b/evals/reports/run-smoke-20260627202411.md @@ -0,0 +1,19 @@ +# EvalFly Report run-smoke-20260627202411 + +Suite: smoke +Verdict: pass +Passed: 3 +Failed: 0 +critical_regressions: 0 +Privacy: sanitized + +## Context +Spec-Slice: CRAG-20260627-M0-BASELINE +Session: omp-session +Commit range: main..HEAD +evalReportPath: evals/reports/run-smoke-20260627202411.md + +## Results +- PASS m0-roadmap-contract-exists (critical) +- PASS m0-targeted-tests-exist (critical) +- PASS m0-ci-workflow-exists (critical) diff --git a/evals/reports/run-smoke-20260627202414.md b/evals/reports/run-smoke-20260627202414.md new file mode 100644 index 0000000..8aada11 --- /dev/null +++ b/evals/reports/run-smoke-20260627202414.md @@ -0,0 +1,19 @@ +# EvalFly Report run-smoke-20260627202414 + +Suite: smoke +Verdict: pass +Passed: 3 +Failed: 0 +critical_regressions: 0 +Privacy: sanitized + +## Context +Spec-Slice: CRAG-20260627-M0-BASELINE +Session: omp-session +Commit range: main..HEAD +evalReportPath: evals/reports/run-smoke-20260627202414.md + +## Results +- PASS m0-roadmap-contract-exists (critical) +- PASS m0-targeted-tests-exist (critical) +- PASS m0-ci-workflow-exists (critical) diff --git a/evals/reports/run-smoke-20260627203416.md b/evals/reports/run-smoke-20260627203416.md new file mode 100644 index 0000000..0576982 --- /dev/null +++ b/evals/reports/run-smoke-20260627203416.md @@ -0,0 +1,20 @@ +# EvalFly Report run-smoke-20260627203416 + +Suite: smoke +Verdict: pass +Passed: 4 +Failed: 0 +critical_regressions: 0 +Privacy: sanitized + +## Context +Spec-Slice: not linked +Session: not linked +Commit range: main..HEAD +evalReportPath: evals/reports/run-smoke-20260627203416.md + +## Results +- PASS m0-roadmap-contract-exists (critical) +- PASS m0-targeted-tests-exist (critical) +- PASS m0-ci-workflow-exists (critical) +- PASS m0-committed-evalfly-runner-exists (critical) diff --git a/evals/reports/run-smoke-20260627203451.md b/evals/reports/run-smoke-20260627203451.md new file mode 100644 index 0000000..c5b914b --- /dev/null +++ b/evals/reports/run-smoke-20260627203451.md @@ -0,0 +1,20 @@ +# EvalFly Report run-smoke-20260627203451 + +Suite: smoke +Verdict: pass +Passed: 4 +Failed: 0 +critical_regressions: 0 +Privacy: sanitized + +## Context +Spec-Slice: CRAG-20260627-M0-REVIEW-FIXES +Session: omp-session +Commit range: main..HEAD +evalReportPath: evals/reports/run-smoke-20260627203451.md + +## Results +- PASS m0-roadmap-contract-exists (critical) +- PASS m0-targeted-tests-exist (critical) +- PASS m0-ci-workflow-exists (critical) +- PASS m0-committed-evalfly-runner-exists (critical) diff --git a/evals/rubrics/README.md b/evals/rubrics/README.md new file mode 100644 index 0000000..4f0eea9 --- /dev/null +++ b/evals/rubrics/README.md @@ -0,0 +1,7 @@ +# Eval rubrics + +Milestone 0 uses deterministic EvalFly assertions only. + +Human or LLM judge rubrics are intentionally absent from the blocking baseline. Add them only after a deterministic check cannot express the risk and the acceptance contract names the reviewer, evidence path, pass conditions, fail conditions, and privacy classification. + +Raw traces do not belong in this tree. Keep raw local material under `.pi/evalfly/raw/` and commit only sanitized fixtures under `evals/traces/sanitized/` when they protect an active eval case. diff --git a/evals/runs/.gitkeep b/evals/runs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/evals/runs/run-smoke-20260627202411.json b/evals/runs/run-smoke-20260627202411.json new file mode 100644 index 0000000..201b0a9 --- /dev/null +++ b/evals/runs/run-smoke-20260627202411.json @@ -0,0 +1,58 @@ +{ + "schema_version": "evalfly.run.v1", + "run_id": "run-smoke-20260627202411", + "suite": "smoke", + "config_name": "CodeRAG Milestone 0 baseline smoke suite", + "created_at": "2026-06-27T20:24:11.466Z", + "context": { + "spec_slice": "CRAG-20260627-M0-BASELINE", + "session_id": "omp-session", + "commit_range": "main..HEAD", + "eval_report_path": "evals/reports/run-smoke-20260627202411.md" + }, + "results": [ + { + "case_id": "m0-roadmap-contract-exists", + "title": "Milestone 0 acceptance contract exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-targeted-tests-exist", + "title": "Milestone 0 targeted tests exist", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-ci-workflow-exists", + "title": "CI validation workflow exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + } + ], + "summary": { + "total": 3, + "passed": 3, + "failed": 0, + "critical_regressions": 0 + }, + "verdict": "pass" +} diff --git a/evals/runs/run-smoke-20260627202414.json b/evals/runs/run-smoke-20260627202414.json new file mode 100644 index 0000000..e9875bb --- /dev/null +++ b/evals/runs/run-smoke-20260627202414.json @@ -0,0 +1,58 @@ +{ + "schema_version": "evalfly.run.v1", + "run_id": "run-smoke-20260627202414", + "suite": "smoke", + "config_name": "CodeRAG Milestone 0 baseline smoke suite", + "created_at": "2026-06-27T20:24:14.746Z", + "context": { + "spec_slice": "CRAG-20260627-M0-BASELINE", + "session_id": "omp-session", + "commit_range": "main..HEAD", + "eval_report_path": "evals/reports/run-smoke-20260627202414.md" + }, + "results": [ + { + "case_id": "m0-roadmap-contract-exists", + "title": "Milestone 0 acceptance contract exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-targeted-tests-exist", + "title": "Milestone 0 targeted tests exist", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-ci-workflow-exists", + "title": "CI validation workflow exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + } + ], + "summary": { + "total": 3, + "passed": 3, + "failed": 0, + "critical_regressions": 0 + }, + "verdict": "pass" +} diff --git a/evals/runs/run-smoke-20260627203416.json b/evals/runs/run-smoke-20260627203416.json new file mode 100644 index 0000000..2050844 --- /dev/null +++ b/evals/runs/run-smoke-20260627203416.json @@ -0,0 +1,68 @@ +{ + "schema_version": "evalfly.run.v1", + "run_id": "run-smoke-20260627203416", + "suite": "smoke", + "config_name": "CodeRAG Milestone 0 baseline smoke suite", + "created_at": "2026-06-27T20:34:16.072Z", + "context": { + "commit_range": "main..HEAD", + "eval_report_path": "evals/reports/run-smoke-20260627203416.md" + }, + "results": [ + { + "case_id": "m0-roadmap-contract-exists", + "title": "Milestone 0 acceptance contract exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-targeted-tests-exist", + "title": "Milestone 0 targeted tests exist", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-ci-workflow-exists", + "title": "CI validation workflow exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-committed-evalfly-runner-exists", + "title": "Committed EvalFly smoke runner exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + } + ], + "summary": { + "total": 4, + "passed": 4, + "failed": 0, + "critical_regressions": 0 + }, + "verdict": "pass" +} diff --git a/evals/runs/run-smoke-20260627203451.json b/evals/runs/run-smoke-20260627203451.json new file mode 100644 index 0000000..9c15aa4 --- /dev/null +++ b/evals/runs/run-smoke-20260627203451.json @@ -0,0 +1,70 @@ +{ + "schema_version": "evalfly.run.v1", + "run_id": "run-smoke-20260627203451", + "suite": "smoke", + "config_name": "CodeRAG Milestone 0 baseline smoke suite", + "created_at": "2026-06-27T20:34:51.248Z", + "context": { + "spec_slice": "CRAG-20260627-M0-REVIEW-FIXES", + "session_id": "omp-session", + "commit_range": "main..HEAD", + "eval_report_path": "evals/reports/run-smoke-20260627203451.md" + }, + "results": [ + { + "case_id": "m0-roadmap-contract-exists", + "title": "Milestone 0 acceptance contract exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-targeted-tests-exist", + "title": "Milestone 0 targeted tests exist", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-ci-workflow-exists", + "title": "CI validation workflow exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + }, + { + "case_id": "m0-committed-evalfly-runner-exists", + "title": "Committed EvalFly smoke runner exists", + "risk_tier": "critical", + "critical": true, + "passed": true, + "privacy": { + "classification": "public", + "sanitized": true + }, + "errors": [] + } + ], + "summary": { + "total": 4, + "passed": 4, + "failed": 0, + "critical_regressions": 0 + }, + "verdict": "pass" +} diff --git a/evals/traces/sanitized/.gitkeep b/evals/traces/sanitized/.gitkeep new file mode 100644 index 0000000..e69de29 From 12b026748fc2b6868f162fba29b4c4e2ec2c425b Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Sat, 27 Jun 2026 14:58:02 -0600 Subject: [PATCH 8/8] chore: ignore local agent audit logs Spec-Slice: CRAG-20260627-M0-REVIEW-FIXES --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index 7a98a4f..f1d3b05 100644 --- a/.gitignore +++ b/.gitignore @@ -165,3 +165,10 @@ specs/ AGENTS.md CLAUDE.md QWEN.md + + +# ============================================================================= +# Local AI Agent Audit Logs +# ============================================================================= +.pi/.push-log.jsonl +.pi/.github-log.jsonl \ No newline at end of file