From 3e432b9d2e0878e6416869ae2b2b390f68cd41ce Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Tue, 30 Jun 2026 16:01:51 -0600 Subject: [PATCH 1/2] docs: add Milestone 7 release candidate evidence Spec-Slice: CRAG-20260630-M7-PRODUCTION-RC --- README.md | 2 + .../2026-06-30-m7-production-rc-contract.md | 20 +++++++ .../plans/2026-06-30-m7-release-run-record.md | 60 +++++++++++++++++++ evals/reports/20260630-m7-production-rc.md | 21 +++++++ evals/runs/20260630-m7-production-rc.json | 23 +++++++ 5 files changed, 126 insertions(+) create mode 100644 docs/superpowers/plans/2026-06-30-m7-production-rc-contract.md create mode 100644 docs/superpowers/plans/2026-06-30-m7-release-run-record.md create mode 100644 evals/reports/20260630-m7-production-rc.md create mode 100644 evals/runs/20260630-m7-production-rc.json diff --git a/README.md b/README.md index 17e366e..abf8db2 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,8 @@ Current release-candidate work is evidence-gated. The supported default is local - Unsupported graph languages degrade explicitly to text/chunk retrieval; they do not report fake graph confidence. - CodeRAG verifies citations against retrieved chunks, but it does not prove repository correctness, execute arbitrary project tests, or make autonomous code changes. +Release-candidate evidence is committed in `docs/superpowers/plans/2026-06-30-m7-release-run-record.md`, `evals/runs/20260630-m7-production-rc.json`, and `evals/reports/20260630-m7-production-rc.md`. + ## 🏗️ Architecture ``` diff --git a/docs/superpowers/plans/2026-06-30-m7-production-rc-contract.md b/docs/superpowers/plans/2026-06-30-m7-production-rc-contract.md new file mode 100644 index 0000000..ac37f86 --- /dev/null +++ b/docs/superpowers/plans/2026-06-30-m7-production-rc-contract.md @@ -0,0 +1,20 @@ +# Milestone 7 Contract: Production Release Candidate + +## Scope + +Produce release-candidate evidence for CodeRAG after M0-M6. This milestone does not add product features unless a release blocker is found; it records and gates the evidence needed to call the branch a release candidate. + +## Acceptance Criteria + +- M7-AC1: A clean install path is exercised from an isolated virtual environment using the built wheel. +- M7-AC2: CI-equivalent local gates pass for M0-M6 tests, lint, typecheck, EvalFly smoke, py_compile, and build. +- M7-AC3: A run record documents CLI, REST, MCP, retrieval/context, citation, graph, and interface evidence; unsupported UI/E2E scope is explicitly named if not exercised. +- M7-AC4: EvalFly run and report artifacts are committed for the release-candidate pass. +- M7-AC5: README release-candidate wording is limited to verified evidence and does not claim unrestricted production readiness. +- M7-AC6: CI includes a release-candidate artifact gate so future changes cannot drop the M7 evidence files silently. + +## Non-goals + +- No new retrieval, graph, generation, or UI features unless required to fix a release blocker. +- No claim of hosted/SaaS hardening. +- No autonomous code-editing behavior. diff --git a/docs/superpowers/plans/2026-06-30-m7-release-run-record.md b/docs/superpowers/plans/2026-06-30-m7-release-run-record.md new file mode 100644 index 0000000..45ab468 --- /dev/null +++ b/docs/superpowers/plans/2026-06-30-m7-release-run-record.md @@ -0,0 +1,60 @@ +# M7 Release Candidate Run Record + +## Verdict + +PASS for release-candidate evidence. This is not a hosted/SaaS production-readiness claim. + +## Scope + +Branch: `feat/m7-production-release-candidate` + +Slice: `CRAG-20260630-M7-PRODUCTION-RC` + +Goal: prove the CodeRAG M0-M6 product surfaces work as a release candidate and preserve evidence in committed artifacts. + +## Evidence transcript + +| Gate | Command / scenario | Result | +| --- | --- | --- | +| Build wheel | `.venv/bin/python -m build` | Built `code_rag_me-0.1.2.tar.gz` and `code_rag_me-0.1.2-py3-none-any.whl`. | +| Clean install | `python3 -m venv /tmp/coderag-m7-venv && /tmp/coderag-m7-venv/bin/python -m pip install dist/code_rag_me-0.1.2-py3-none-any.whl` | Installed wheel and dependencies in isolated venv. | +| CLI smoke | `/tmp/coderag-m7-venv/bin/coderag --help` | Listed `query`, `search-hybrid`, `context-pack`, `find-symbol`, `find-references`, `blast-radius`, and repository commands. | +| Package import | `/tmp/coderag-m7-venv/bin/python -c "import coderag; print(coderag.__version__)"` | Printed `0.1.2`. | +| Targeted tests | `PYTHONPATH=src .venv/bin/python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py tests/test_m4_hybrid_retrieval.py tests/test_m5_code_graph.py tests/test_m6_interfaces_docs.py` | `103 passed`. | +| Lint | `.venv/bin/ruff check ` | OK. | +| Typecheck | `PYTHONPATH=src .venv/bin/mypy ` | OK. | +| EvalFly smoke | `.venv/bin/python scripts/evalfly_smoke.py --suite smoke` | `evalfly committed smoke passed: 33 cases`. | +| Installed API/MCP smoke | `/tmp/coderag-m7-venv/bin/python -c "from fastapi.testclient import TestClient; from coderag.main import create_app; ..."` | `/health` returned 200, OpenAPI contained `/api/v1/search-hybrid` and `/api/v1/graph/blast-radius`, MCP mount was present. | +| Installed UI smoke | `/tmp/coderag-m7-venv/bin/python -c "from fastapi.testclient import TestClient; from coderag.main import create_app; ... client.get('/') ..."` | `/` returned 200 and response text contained Gradio/CodeRAG UI content. | +| Compile | `.venv/bin/python -m py_compile ` | OK. | + +## Surface coverage + +- CLI: clean-install `coderag --help` proves entrypoint availability; M6 tests cover structured `search-hybrid` JSON; M5 tests cover `blast-radius` JSON; M4 tests cover `context-pack` output. +- REST: installed-app OpenAPI smoke proves user-visible routes are mounted; API route tests cover index/query/context-pack/search-hybrid/graph contracts with explicit schema assertions. +- MCP: installed-app smoke proves MCP mount is present; MCP handler/tool tests cover indexing, query, search, context pack, and graph tool availability/shape. +- UI: installed-app TestClient smoke proves the Gradio root route renders an HTTP 200 page containing Gradio/CodeRAG content. +- Retrieval/context: M4 tests cover hybrid ranking, dedupe, budgets, and context-pack snippets. +- Citations: M3 tests cover parsed citation verification, grounded flag semantics, and adapter exposure. +- Graph: M5 tests cover Python and TypeScript/JavaScript extraction, persisted replacement, symbol lookup, references, and blast radius/test association. +- Docs: M6 tests assert README wording: verified codebase context engine, not an autonomous coding agent, explicit production-readiness limits, and unsupported graph language fallback. + +## Explicit exclusions + +- Hosted deployment hardening is not claimed. Public binding still requires explicit host, CORS, authentication, and network controls. +- The UI smoke is HTTP-level through FastAPI TestClient, not a browser-visible Playwright journey. +- CodeRAG does not autonomously edit code or run target repository tests. + +## Release checklist against roadmap AC-0..AC-10 + +- AC-0 governance: PASS — SpecSafe slice and committed contracts exist. +- Baseline/CI: PASS — CI-equivalent local gates pass and CI workflow includes M0-M6 gates plus EvalFly M7 artifact checks. +- Safe defaults/privacy profiles: PASS — loopback defaults and provider profile guardrails covered by M0/M1 tests and README wording. +- SQLite registry/job state: PASS — M2 registry tests pass. +- Hybrid retrieval/context packs: PASS — M4 tests pass. +- Verified citations: PASS — M3 tests pass. +- Code graph/context intelligence: PASS — M5 tests pass for supported languages with explicit fallback. +- MCP/API/CLI/UI interfaces: PASS — installed-app smoke plus M4-M6 tests cover mounted routes and structured outputs. +- EvalFly gates: PASS — committed smoke suite has 33 passing cases, including M7 artifact and behavior coverage-pointer gates. +- Release evidence: PASS — this run record plus EvalFly run/report artifacts committed. +- Production claim discipline: PASS — docs avoid unrestricted production-ready claims. diff --git a/evals/reports/20260630-m7-production-rc.md b/evals/reports/20260630-m7-production-rc.md new file mode 100644 index 0000000..632f58a --- /dev/null +++ b/evals/reports/20260630-m7-production-rc.md @@ -0,0 +1,21 @@ +# EvalFly Report: 20260630-m7-production-rc + +## Verdict + +PASS. + +## Evidence + +- EvalFly smoke: `.venv/bin/python scripts/evalfly_smoke.py --suite smoke` → `evalfly committed smoke passed: 33 cases`. +- CI-equivalent pytest gate: M0-M6 targeted tests → `103 passed`. +- Clean install gate: built `dist/code_rag_me-0.1.2-py3-none-any.whl`, installed it into `/tmp/coderag-m7-venv`, ran `coderag --help`, and imported `coderag.__version__` → `0.1.2`. + +## Covered surfaces + +- Governance contracts M0-M7. +- Safe defaults, provider profiles, citation verification, hybrid retrieval, context packs, graph extraction/blast radius, REST/MCP/CLI interface contracts, README readiness wording. + +## Explicit exclusions + +- Hosted/SaaS production hardening is not claimed. +- Browser-visible Gradio E2E is not claimed in this report; local UI launch remains covered only by package/build/import and existing unit-level handlers. diff --git a/evals/runs/20260630-m7-production-rc.json b/evals/runs/20260630-m7-production-rc.json new file mode 100644 index 0000000..7dae5ed --- /dev/null +++ b/evals/runs/20260630-m7-production-rc.json @@ -0,0 +1,23 @@ +{ + "schema_version": "evalfly.run.v1", + "run_id": "20260630-m7-production-rc", + "suite": "smoke", + "milestone": "M7", + "status": "PASS", + "cases_passed": 33, + "cases_failed": 0, + "evidence": [ + { + "command": ".venv/bin/python scripts/evalfly_smoke.py --suite smoke", + "observed": "evalfly committed smoke passed: 33 cases" + }, + { + "command": "PYTHONPATH=src .venv/bin/python -m pytest tests/test_m0_baseline.py tests/test_m1_services.py tests/test_m2_registry.py tests/test_citations.py tests/test_m3_citations.py tests/test_api_routes.py tests/test_mcp_handlers.py tests/test_mcp_tools.py tests/test_ui_handlers.py tests/test_m4_hybrid_retrieval.py tests/test_m5_code_graph.py tests/test_m6_interfaces_docs.py", + "observed": "103 passed" + }, + { + "command": "clean wheel install in /tmp/coderag-m7-venv; coderag --help; import coderag", + "observed": "wheel installed, CLI listed production interface commands, coderag.__version__ printed 0.1.2" + } + ] +} From a5c7ff4b79906485ed722b469ae1e3d9617fbac5 Mon Sep 17 00:00:00 2001 From: Sebastiangmz Date: Tue, 30 Jun 2026 16:01:55 -0600 Subject: [PATCH 2/2] ci: add Milestone 7 release artifact gates Spec-Slice: CRAG-20260630-M7-PRODUCTION-RC --- .github/workflows/ci.yml | 8 ++ .pi/.specsafe-state.json | 20 ++++ evals/config.json | 224 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 252 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 715c531..f72056e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,3 +42,11 @@ jobs: - name: Run committed EvalFly smoke baseline run: python scripts/evalfly_smoke.py --suite smoke + + - name: Check Milestone 7 release artifacts + run: | + test -f docs/superpowers/plans/2026-06-30-m7-production-rc-contract.md + test -f docs/superpowers/plans/2026-06-30-m7-release-run-record.md + test -f evals/runs/20260630-m7-production-rc.json + test -f evals/reports/20260630-m7-production-rc.md + python -m json.tool evals/runs/20260630-m7-production-rc.json >/dev/null diff --git a/.pi/.specsafe-state.json b/.pi/.specsafe-state.json index d050397..aad5455 100644 --- a/.pi/.specsafe-state.json +++ b/.pi/.specsafe-state.json @@ -179,6 +179,26 @@ "turns": 0 } } + }, + { + "sliceId": "CRAG-20260630-M7-PRODUCTION-RC", + "workspaceId": "CodeRAG", + "sessionId": "omp-session", + "beganAt": "2026-06-30T21:42:53.203Z", + "endedAt": "2026-06-30T22:01:40.052Z", + "outcome": "PASS", + "costSummary": { + "externalMemoryCalls": 0, + "externalMemoryCost": 0, + "subagentTokens": { + "input": 0, + "output": 0, + "cacheRead": 0, + "cacheWrite": 0, + "cost": 0, + "turns": 0 + } + } } ] } diff --git a/evals/config.json b/evals/config.json index cbee5f1..44b8ebd 100644 --- a/evals/config.json +++ b/evals/config.json @@ -866,6 +866,230 @@ } ] } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m7-production-rc-contract-exists", + "title": "Milestone 7 release candidate contract exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "delivery_governance", + "source": { + "kind": "acceptance_criteria", + "criteria": "M7-AC1 M7-AC2 M7-AC3 M7-AC4" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "The Milestone 7 release-candidate contract is committed." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "docs/superpowers/plans/2026-06-30-m7-production-rc-contract.md" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m7-release-run-record-exists", + "title": "Release-candidate run record exists", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "release_evidence", + "source": { + "kind": "acceptance_criteria", + "criteria": "M7-AC1 M7-AC2 M7-AC3" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "The release run record documents clean install, local gates, covered surfaces, and exclusions." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_contains", + "path": "docs/superpowers/plans/2026-06-30-m7-release-run-record.md", + "text": "Clean install" + }, + { + "type": "file_contains", + "path": "docs/superpowers/plans/2026-06-30-m7-release-run-record.md", + "text": "Explicit exclusions" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m7-evalfly-artifacts-exist", + "title": "Release-candidate EvalFly artifacts exist", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "release_evidence", + "source": { + "kind": "acceptance_criteria", + "criteria": "M7-AC4 M7-AC6" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "EvalFly run and report artifacts are committed for the release candidate." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_exists", + "path": "evals/runs/20260630-m7-production-rc.json" + }, + { + "type": "file_exists", + "path": "evals/reports/20260630-m7-production-rc.md" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m7-evalfly-hybrid-behavior-covered", + "title": "Hybrid retrieval behavior coverage is present", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "behavioral_coverage_pointer", + "source": { + "kind": "acceptance_criteria", + "criteria": "M7-AC2 M7-AC3 M7-AC4" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "The committed test suite contains a deterministic behavior-level coverage pointer for this release-candidate surface." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_contains", + "path": "tests/test_m4_hybrid_retrieval.py", + "text": "test_hybrid_retrieval_surfaces_lexical_exact_match_ahead_of_unrelated_vector_hit" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m7-evalfly-citation-behavior-covered", + "title": "Citation verification behavior coverage is present", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "behavioral_coverage_pointer", + "source": { + "kind": "acceptance_criteria", + "criteria": "M7-AC2 M7-AC3 M7-AC4" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "The committed test suite contains a deterministic behavior-level coverage pointer for this release-candidate surface." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_contains", + "path": "tests/test_m3_citations.py", + "text": "test_response_generator_grounds_only_when_all_citations_verify" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m7-evalfly-graph-behavior-covered", + "title": "Code graph behavior coverage is present", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "behavioral_coverage_pointer", + "source": { + "kind": "acceptance_criteria", + "criteria": "M7-AC2 M7-AC3 M7-AC4" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "The committed test suite contains a deterministic behavior-level coverage pointer for this release-candidate surface." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_contains", + "path": "tests/test_m5_code_graph.py", + "text": "test_graph_service_finds_symbols_references_and_blast_radius" + } + ] + } + }, + { + "schema_version": "evalfly.case.v1", + "case_id": "m7-evalfly-interface-behavior-covered", + "title": "Interface behavior coverage is present", + "suite": "smoke", + "risk_tier": "critical", + "task_type": "behavioral_coverage_pointer", + "source": { + "kind": "acceptance_criteria", + "criteria": "M7-AC2 M7-AC3 M7-AC4" + }, + "privacy": { + "classification": "public", + "sanitized": true + }, + "expected": { + "success_criteria": [ + "The committed test suite contains a deterministic behavior-level coverage pointer for this release-candidate surface." + ] + }, + "judge": { + "type": "deterministic", + "assertions": [ + { + "type": "file_contains", + "path": "tests/test_m6_interfaces_docs.py", + "text": "test_cli_hybrid_search_outputs_structured_json" + } + ] + } } ] }