From 430acf97ee9f6129d182c88382c15278927f9302 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:48:03 +0800 Subject: [PATCH 1/2] ci: tier compatibility tests and require nightly wheel acceptance --- .github/AGENTS.md | 1 + .github/scripts/run_dataset_acceptance.py | 120 +++++++++++++++++ .github/workflows/AGENTS.md | 35 ++++- .github/workflows/test.yml | 70 +++++++++- tests/core/test_dataset_id_pagination.py | 1 + tests/test_ci_tiers.py | 110 +++++++++++++++ tests/test_dataset_acceptance_runner.py | 157 ++++++++++++++++++++++ tests/test_wheel_audit.py | 6 + 8 files changed, 494 insertions(+), 6 deletions(-) create mode 100644 .github/scripts/run_dataset_acceptance.py create mode 100644 tests/test_ci_tiers.py create mode 100644 tests/test_dataset_acceptance_runner.py diff --git a/.github/AGENTS.md b/.github/AGENTS.md index c4b73b2..e4569fe 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -9,6 +9,7 @@ Parent scope: [repository](../AGENTS.md). | `pull_request_template.md` | Default PR template and machine-parsed snapshot decision labels | | `scripts/check_snapshot_provenance.py` | Base/head Git-blob comparison and PR declaration validation | | `scripts/clean_wheel_agent_acceptance.py` | Supplied wheel acceptance in temporary home/CWD/venv | +| `scripts/run_dataset_acceptance.py` | Required installed-wheel dataset test with fail-closed JUnit and artifact evidence | ## Provenance Guard diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py new file mode 100644 index 0000000..89fc223 --- /dev/null +++ b/.github/scripts/run_dataset_acceptance.py @@ -0,0 +1,120 @@ +"""Run the installed-wheel gate and reject empty, skipped or stale evidence. + +This deliberately selects one existing public CLI/API acceptance test. It does +not claim clean dependency isolation or complete default-scale coverage; those +are contracts of the selected test, not of this CI runner. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TEST_NAME = "test_installed_trusted_producer_default_export_cli_and_api" +TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" + + +class AcceptanceError(RuntimeError): + pass + + +def read_evidence(report: Path, wheel_sha256: str) -> dict: + """A green pytest process alone does not prove the requested test ran.""" + if not report.is_file(): + raise AcceptanceError("pytest did not write its JUnit report") + cases = ET.parse(report).findall(".//testcase") + if len(cases) != 1 or cases[0].get("name") != TEST_NAME: + raise AcceptanceError(f"Expected exactly one {TEST_NAME} result, got {len(cases)}") + case = cases[0] + if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): + raise AcceptanceError("Installed-wheel acceptance must pass, without skips") + properties = case.findall("./properties/property[@name='dataset_acceptance']") + if len(properties) != 1: + raise AcceptanceError("Expected exactly one dataset_acceptance evidence property") + evidence = json.loads(properties[0].get("value", "")) + if not isinstance(evidence, dict) or evidence.get("wheel_sha256") != wheel_sha256: + raise AcceptanceError("Acceptance evidence does not match the supplied wheel SHA-256") + return evidence + + +def run_acceptance(wheel: Path, output_dir: Path) -> None: + wheel = wheel.resolve() + output_dir = output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + report = output_dir / "junit.xml" + # Reusing an output directory must never let a previous green run mask this one. + for name in ("junit.xml", "summary.json", "pytest.log"): + (output_dir / name).unlink(missing_ok=True) + command = [ + sys.executable, + "-m", + "pytest", + TEST_NODE, + "-o", + "junit_family=xunit1", + f"--junitxml={report}", + ] + env = os.environ.copy() + env["PT_SNAP_ACCEPTANCE_WHEEL"] = str(wheel) + env["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] = str(output_dir) + summary = { + "wheel": str(wheel), + "test_node": TEST_NODE, + "command": command, + "status": "failed", + } + try: + if not wheel.is_file() or wheel.suffix != ".whl": + raise AcceptanceError(f"A built wheel is required: {wheel}") + wheel_sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + summary["wheel_sha256"] = wheel_sha256 + source_sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout.strip() + summary["source_sha"] = source_sha + print(f"Running {TEST_NODE} at {source_sha}", flush=True) + with (output_dir / "pytest.log").open("w") as log: + result = subprocess.run( + command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT + ) + print((output_dir / "pytest.log").read_text(), end="", flush=True) + summary["pytest_exit_code"] = result.returncode + if result.returncode != 0: + raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log") + summary["acceptance"] = read_evidence(report, wheel_sha256) + summary["status"] = "passed" + except Exception as exc: + summary["error"] = str(exc) + raise + finally: + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + try: + run_acceptance(args.wheel, args.output_dir) + except ( + AcceptanceError, + OSError, + ValueError, + ET.ParseError, + subprocess.CalledProcessError, + ) as exc: + print(f"Dataset acceptance failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 2959c6d..91e9211 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -5,23 +5,49 @@ Parent scope: [automation](../AGENTS.md). ## Test Workflow `test.yml` runs for main pushes, main-targeted PR opened/synchronize/reopened/edited -events, and `workflow_call`. Provenance, lint and Python test-matrix jobs are -independent. Concurrency is keyed by GitHub ref and superseded runs are cancelled. +events, `workflow_call`, nightly at 03:17 UTC, and `workflow_dispatch`. Provenance, +lint, browser and Python test-matrix jobs are independent. Checkouts use the PR +head SHA or the triggering SHA; manual dispatch checks the selected ref's SHA. +Concurrency is keyed by GitHub ref and superseded runs are cancelled. | Job/step | Gate | | --- | --- | | Snapshot provenance | Full Git history; base/head comparison; PR decision required only in PR mode | | Lint & Format | Python 3.13, editable dev install, Ruff, Black, basedpyright with the selected interpreter | -| Test matrix | Python 3.10–3.13; fixture provenance first, then pytest with branch coverage | +| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow"` | +| Full test tier | Python 3.13; fixture provenance first, then all tests with branch coverage, including slow tests | | Overall coverage | `pyproject.toml` fail-under 90 with branches enabled | | Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently | | Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance | +| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then required installed-wheel CLI/API test | + +Million-event deep-offset pagination tests are marked `slow`; their small +pagination contracts still run across the compatibility matrix. Coverage reporting writes `.coverage`/`coverage.xml`; building writes `dist/` and intermediates. Dependency installation/build isolation may access the network. The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses temporary paths but imports a reviewed executable fixture. These are more than YAML checks. +The nightly/manual job invokes `run_dataset_acceptance.py` with the single built +wheel. Missing wheels, missing/empty JUnit, skipped/failed tests, extra test cases, +or evidence for a different wheel fail the gate. It uploads the pytest log, JUnit +(including the test's evidence), and a summary with actual checkout SHA and wheel +SHA-256, even after failure. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` is available for +additional test-owned artifacts. Local invocation: + +```bash +python .github/scripts/run_dataset_acceptance.py \ + --wheel dist/.whl --output-dir .tmp/dataset-acceptance +``` + +This tier initially exercises the existing 500,002-event `pip --target` acceptance +test. It does not claim a clean venv, two full 500,000-event shards, complete CLI +template coverage, live agents or upstream GUI interoperability. The ordinary +full suite may still skip this opt-in wheel test; the required nightly/manual +runner must never do so. Manually dispatch the exact published branch/commit and +verify the run's head SHA and summary; a PR run alone is not nightly evidence. + ## Release Workflow `release.yml` triggers on `v*` tag pushes and enforces: @@ -44,6 +70,7 @@ validation. For non-PR provenance checks, tag runs compare against the prior tag Review `pyproject.toml`, both workflow YAMLs, the provenance/acceptance scripts and `tests/test_release_workflow.py` together when changing execution order. Validate -YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py` from +YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py +tests/test_ci_tiers.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from the repository root. Full CI tests the matrix and installed wheel; static tests do not prove a release environment is configured or a publication occurred. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1000c8d..e71defe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: Test on: workflow_call: + workflow_dispatch: + schedule: + - cron: "17 3 * * *" push: branches: [main] pull_request: @@ -23,6 +26,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Require provenance evaluation for snapshot runtime changes if: github.event_name == 'pull_request' @@ -56,6 +60,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: @@ -78,6 +84,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: python-version: "3.13" @@ -99,6 +107,8 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: @@ -114,10 +124,19 @@ jobs: - name: Install dependencies run: pip install -e ".[dev]" - - name: Run tests with coverage - run: pytest tests/test_fixture_provenance.py && pytest --cov --cov-report=xml --cov-report=term-missing + - name: Verify executable fixture provenance + run: pytest tests/test_fixture_provenance.py + + - name: Run non-slow compatibility tests + if: matrix.python-version != '3.13' + run: pytest -m "not slow" + + - name: Run all tests with coverage + if: matrix.python-version == '3.13' + run: pytest --cov --cov-report=xml --cov-report=term-missing - name: Guard critical runtime coverage + if: matrix.python-version == '3.13' # Separate floors prevent strong modules from masking a weak boundary. run: | python -m coverage report --include='*/core/snapshot_import_backend.py' --fail-under=90 @@ -153,3 +172,50 @@ jobs: python .github/scripts/clean_wheel_agent_acceptance.py \ --wheel "$WHEEL" \ --fixture tests/fixtures/snapshots/snapshot_with_empty_cache.pkl + + dataset-acceptance: + name: Installed-wheel dataset acceptance + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.sha }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Verify executable fixture provenance + run: pytest tests/test_fixture_provenance.py + + - name: Build wheel + run: python -m build --wheel + + - name: Audit wheel contents against source + run: | + set -euo pipefail + WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') + python .github/scripts/audit_wheel.py --wheel "$WHEEL" + + - name: Run required installed-wheel dataset acceptance + run: | + set -euo pipefail + WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') + python .github/scripts/run_dataset_acceptance.py \ + --wheel "$WHEEL" --output-dir .tmp/dataset-acceptance + + - name: Upload dataset acceptance evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: dataset-acceptance-${{ github.sha }} + path: .tmp/dataset-acceptance/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 diff --git a/tests/core/test_dataset_id_pagination.py b/tests/core/test_dataset_id_pagination.py index df4756c..56e4254 100644 --- a/tests/core/test_dataset_id_pagination.py +++ b/tests/core/test_dataset_id_pagination.py @@ -269,6 +269,7 @@ def million_event_dataset(tmp_path_factory): return root +@pytest.mark.slow @pytest.mark.parametrize("template", ["event", "allocation"]) @pytest.mark.parametrize("direction", ["ASC", "DESC"]) def test_million_offset_keeps_fetched_work_constant(million_event_dataset, template, direction): diff --git a/tests/test_ci_tiers.py b/tests/test_ci_tiers.py new file mode 100644 index 0000000..9a6e10f --- /dev/null +++ b/tests/test_ci_tiers.py @@ -0,0 +1,110 @@ +"""Protect fast/full/nightly boundaries and the existing quality gates.""" + +from pathlib import Path + +import yaml + + +def workflow(): + return yaml.load(Path(".github/workflows/test.yml").read_text(), Loader=yaml.BaseLoader) + + +def named_steps(job): + return {step["name"]: step for step in job["steps"] if "name" in step} + + +def test_compatibility_matrix_leaves_slow_and_coverage_on_python_313(): + job = workflow()["jobs"]["test"] + assert job["strategy"]["matrix"]["python-version"] == ["3.10", "3.11", "3.12", "3.13"] + steps = named_steps(job) + fast = steps["Run non-slow compatibility tests"] + assert fast["if"] == "matrix.python-version != '3.13'" + assert fast["run"] == 'pytest -m "not slow"' + full = steps["Run all tests with coverage"] + assert full["if"] == "matrix.python-version == '3.13'" + assert full["run"] == "pytest --cov --cov-report=xml --cov-report=term-missing" + assert "if" not in steps["Verify executable fixture provenance"] + names = list(steps) + assert names.index("Verify executable fixture provenance") < names.index(fast["name"]) + assert names.index("Verify executable fixture provenance") < names.index(full["name"]) + + +def test_full_tier_preserves_independent_coverage_floors_and_packaging(): + steps = named_steps(workflow()["jobs"]["test"]) + guard = steps["Guard critical runtime coverage"] + assert guard["if"] == "matrix.python-version == '3.13'" + for file, floor in ( + ("core/snapshot_import_backend.py", 90), + ("simulate/simulated_caching_allocator.py", 88), + ("simulate/snapshot_mutator.py", 84), + ): + assert f"--include='*/{file}' --fail-under={floor}" in guard["run"] + config = Path("pyproject.toml").read_text() + assert "branch = true" in config and "fail_under = 90" in config + for name in ( + "Build wheel", + "Audit wheel contents against source", + "Clean-wheel Agent acceptance", + ): + assert steps[name]["if"] == "matrix.python-version == '3.13'" + assert steps["Upload coverage report"]["if"] == "matrix.python-version == '3.13' && always()" + + +def test_ci_preserves_least_privilege_and_checks_the_published_commit(): + config = workflow() + assert config["permissions"] == {"contents": "read"} + assert "pull_request_target" not in config["on"] + assert config["on"]["pull_request"]["types"] == ["opened", "synchronize", "reopened", "edited"] + assert "workflow_call" in config["on"] + jobs = config["jobs"] + for name in ("snapshot-provenance", "lint", "browser", "test"): + checkout = jobs[name]["steps"][0] + assert checkout["uses"] == "actions/checkout@v4" + assert checkout["with"]["ref"] == "${{ github.event.pull_request.head.sha || github.sha }}" + assert jobs["snapshot-provenance"]["steps"][0]["with"]["fetch-depth"] == "0" + assert ( + "--require-decision" + in named_steps(jobs["snapshot-provenance"])[ + "Require provenance evaluation for snapshot runtime changes" + ]["run"] + ) + browser = named_steps(jobs["browser"])["Run required Chromium interaction tests"] + assert browser["env"]["PT_SNAP_REQUIRE_BROWSER"] == "1" + assert "tests/test_memory_tree_browser.py" in browser["run"] + lint = named_steps(jobs["lint"]) + for name in ("Ruff lint check", "Black format check", "basedPyright check"): + assert name in lint + + +def test_nightly_and_manual_gate_build_and_audit_before_required_acceptance(): + config = workflow() + assert "workflow_dispatch" in config["on"] + assert config["on"]["schedule"] == [{"cron": "17 3 * * *"}] + job = config["jobs"]["dataset-acceptance"] + assert ( + job["if"] == "github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'" + ) + assert job["timeout-minutes"] == "30" + assert job["steps"][0]["with"] == {"fetch-depth": "0", "ref": "${{ github.sha }}"} + steps = named_steps(job) + names = list(steps) + order = [ + "Verify executable fixture provenance", + "Build wheel", + "Audit wheel contents against source", + "Run required installed-wheel dataset acceptance", + "Upload dataset acceptance evidence", + ] + assert [names.index(name) for name in order] == sorted(names.index(name) for name in order) + acceptance = steps["Run required installed-wheel dataset acceptance"]["run"] + assert "assert len(wheels) == 1" in acceptance + assert "python .github/scripts/run_dataset_acceptance.py" in acceptance + assert '--wheel "$WHEEL" --output-dir .tmp/dataset-acceptance' in acceptance + for step in job["steps"]: + assert "continue-on-error" not in step + upload = steps["Upload dataset acceptance evidence"] + assert upload["if"] == "always()" + assert upload["with"]["if-no-files-found"] == "error" + assert upload["with"]["include-hidden-files"] == "true" + assert upload["with"]["path"] == ".tmp/dataset-acceptance/" + assert "${{ github.sha }}" in upload["with"]["name"] diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py new file mode 100644 index 0000000..36ff22e --- /dev/null +++ b/tests/test_dataset_acceptance_runner.py @@ -0,0 +1,157 @@ +"""Fail-closed CI wrapper contracts without repeating the expensive scale test.""" + +import hashlib +import importlib.util +import json +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +RUNNER = Path(".github/scripts/run_dataset_acceptance.py") + + +@pytest.fixture +def runner(): + spec = importlib.util.spec_from_file_location("dataset_acceptance_runner", RUNNER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def write_report(path, name, evidence, outcome=None): + suite = ET.Element("testsuite") + case = ET.SubElement(suite, "testcase", name=name) + if outcome: + ET.SubElement(case, outcome) + if evidence is not None: + properties = ET.SubElement(case, "properties") + ET.SubElement(properties, "property", name="dataset_acceptance", value=json.dumps(evidence)) + ET.ElementTree(suite).write(path) + + +def test_runner_accepts_matching_wheel_evidence(runner, tmp_path): + report = tmp_path / "junit.xml" + evidence = {"wheel_sha256": "abc", "events": 500_002} + write_report(report, runner.TEST_NAME, evidence) + assert runner.read_evidence(report, "abc") == evidence + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_nonpassing_acceptance(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}, outcome) + with pytest.raises(runner.AcceptanceError, match="without skips"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("evidence", [None, {}, {"wheel_sha256": "other"}, []]) +def test_runner_rejects_missing_or_wrong_artifact_evidence(runner, tmp_path, evidence): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, evidence) + with pytest.raises(runner.AcceptanceError): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("count", [0, 2]) +def test_runner_rejects_empty_or_extra_collection(runner, tmp_path, count): + report = tmp_path / "junit.xml" + suite = ET.Element("testsuite") + for _ in range(count): + ET.SubElement(suite, "testcase", name=runner.TEST_NAME) + ET.ElementTree(suite).write(report) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +def test_runner_rejects_wrong_test_even_if_it_passes(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, "unrelated_test", {"wheel_sha256": "abc"}) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +def test_runner_rejects_duplicate_evidence(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + tree = ET.parse(report) + properties = tree.find(".//properties") + assert properties is not None + ET.SubElement(properties, "property", name="dataset_acceptance", value="{}") + tree.write(report) + with pytest.raises(runner.AcceptanceError, match="exactly one dataset_acceptance"): + runner.read_evidence(report, "abc") + + +def test_missing_wheel_fails_before_pytest(tmp_path): + output = tmp_path / "results" + output.mkdir() + (output / "summary.json").write_text('{"status": "passed"}') + (output / "junit.xml").write_text("old passing report") + (output / "pytest.log").write_text("old passing output") + result = subprocess.run( + [ + sys.executable, + str(RUNNER), + "--wheel", + str(tmp_path / "missing.whl"), + "--output-dir", + str(tmp_path / "results"), + ], + capture_output=True, + text=True, + ) + assert result.returncode == 1 + assert "A built wheel is required" in result.stderr + assert not (output / "junit.xml").exists() + assert not (output / "pytest.log").exists() + summary = json.loads((output / "summary.json").read_text()) + assert summary["status"] == "failed" + assert "A built wheel is required" in summary["error"] + + +@pytest.mark.parametrize("exit_code,write_junit", [(0, True), (0, False), (1, True), (5, False)]) +def test_runner_records_checkout_and_artifact_with_fail_closed_results( + runner, tmp_path, monkeypatch, exit_code, write_junit +): + wheel = tmp_path / "built.whl" + wheel.write_bytes(b"test artifact") + wheel_hash = hashlib.sha256(wheel.read_bytes()).hexdigest() + output = tmp_path / "results" + output.mkdir() + report = output / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": wheel_hash}) + calls = [] + + def run(command, **kwargs): + calls.append(command) + assert kwargs["cwd"] == runner.ROOT + if command[0] == "git": + return subprocess.CompletedProcess(command, 0, stdout="checkout-sha\n") + assert command[:3] == [sys.executable, "-m", "pytest"] + assert runner.TEST_NODE in command + assert "junit_family=xunit1" in command + assert kwargs["env"]["PT_SNAP_ACCEPTANCE_WHEEL"] == str(wheel) + assert kwargs["env"]["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] == str(output) + assert not report.exists(), "stale JUnit must be removed before this run" + kwargs["stdout"].write("captured pytest output\n") + if write_junit: + write_report(report, runner.TEST_NAME, {"wheel_sha256": wheel_hash}) + return subprocess.CompletedProcess(command, exit_code) + + monkeypatch.setattr(runner.subprocess, "run", run) + if exit_code or not write_junit: + with pytest.raises(runner.AcceptanceError): + runner.run_acceptance(wheel, output) + else: + runner.run_acceptance(wheel, output) + assert len(calls) == 2 + summary = json.loads((output / "summary.json").read_text()) + assert summary["source_sha"] == "checkout-sha" + assert summary["wheel_sha256"] == wheel_hash + assert summary["pytest_exit_code"] == exit_code + assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") + assert (output / "pytest.log").read_text() == "captured pytest output\n" diff --git a/tests/test_wheel_audit.py b/tests/test_wheel_audit.py index 15bb04c..4bc413a 100644 --- a/tests/test_wheel_audit.py +++ b/tests/test_wheel_audit.py @@ -177,6 +177,12 @@ def build(directory): ("workflow", "job", "build", "next_step"), [ ("test.yml", "test", "Build wheel", "Clean-wheel Agent acceptance"), + ( + "test.yml", + "dataset-acceptance", + "Build wheel", + "Run required installed-wheel dataset acceptance", + ), ("release.yml", "build", "Build sdist and wheel", "Upload build artifacts"), ], ) From 68d89670b188c93e41b1a5d8f41903057e5ada06 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:59:11 +0800 Subject: [PATCH 2/2] ci: enforce explicit nightly selection and required test evidence --- .github/scripts/run_dataset_acceptance.py | 66 ++++++++--- .github/workflows/AGENTS.md | 20 ++-- .github/workflows/test.yml | 6 +- pyproject.toml | 1 + tests/core/test_dataset_scale_acceptance.py | 1 + tests/test_ci_tiers.py | 10 +- tests/test_dataset_acceptance_runner.py | 120 +++++++++++++++++++- 7 files changed, 194 insertions(+), 30 deletions(-) diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py index 89fc223..d1d8512 100644 --- a/.github/scripts/run_dataset_acceptance.py +++ b/.github/scripts/run_dataset_acceptance.py @@ -1,6 +1,6 @@ """Run the installed-wheel gate and reject empty, skipped or stale evidence. -This deliberately selects one existing public CLI/API acceptance test. It does +This selects nightly tests and requires the public CLI/API wheel case. It does not claim clean dependency isolation or complete default-scale coverage; those are contracts of the selected test, not of this CI runner. """ @@ -14,28 +14,60 @@ import subprocess import sys import xml.etree.ElementTree as ET -from pathlib import Path +from pathlib import Path, PurePosixPath ROOT = Path(__file__).resolve().parents[2] TEST_NAME = "test_installed_trusted_producer_default_export_cli_and_api" TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" +# Extend this tuple alongside new mandatory nightly gates. Never list future +# files that have not landed: the runner rejects any missing configured node. +REQUIRED_TEST_NODES = (TEST_NODE,) class AcceptanceError(RuntimeError): pass -def read_evidence(report: Path, wheel_sha256: str) -> dict: - """A green pytest process alone does not prove the requested test ran.""" +def test_node_id(case: ET.Element) -> str: + """Recover a complete pytest identity from the required xunit1 fields.""" + file = case.get("file", "") + name = case.get("name", "") + classname = case.get("classname", "") + path = PurePosixPath(file) + if not name or path.is_absolute() or ".." in path.parts or path.suffix != ".py": + raise AcceptanceError(f"JUnit testcase has no valid source identity: {case.attrib}") + module = file.removesuffix(".py").replace("/", ".") + if classname == module: + classes = "" + elif classname.startswith(module + "."): + classes = classname[len(module) + 1 :].replace(".", "::") + "::" + else: + raise AcceptanceError(f"JUnit testcase has an inconsistent class identity: {case.attrib}") + return f"{file}::{classes}{name}" + + +def read_cases(report: Path) -> list[ET.Element]: if not report.is_file(): raise AcceptanceError("pytest did not write its JUnit report") - cases = ET.parse(report).findall(".//testcase") - if len(cases) != 1 or cases[0].get("name") != TEST_NAME: - raise AcceptanceError(f"Expected exactly one {TEST_NAME} result, got {len(cases)}") - case = cases[0] - if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): - raise AcceptanceError("Installed-wheel acceptance must pass, without skips") - properties = case.findall("./properties/property[@name='dataset_acceptance']") + return ET.parse(report).findall(".//testcase") + + +def read_evidence(report: Path, wheel_sha256: str) -> dict: + """Require the wheel case plus every registered gate, with no silent skips.""" + cases = read_cases(report) + nodes = [test_node_id(case) for case in cases] + if nodes.count(TEST_NODE) != 1: + raise AcceptanceError(f"Expected exactly one {TEST_NODE} result") + if len(nodes) != len(set(nodes)): + raise AcceptanceError("Duplicate nightly testcase identities") + missing = set(REQUIRED_TEST_NODES) - set(nodes) + if missing: + raise AcceptanceError(f"Missing required nightly test nodes: {sorted(missing)}") + for node, case in zip(nodes, cases, strict=True): + if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): + raise AcceptanceError(f"Nightly acceptance must pass, without skips: {node}") + wheel_case = cases[nodes.index(TEST_NODE)] + properties = wheel_case.findall("./properties/property[@name='dataset_acceptance']") if len(properties) != 1: raise AcceptanceError("Expected exactly one dataset_acceptance evidence property") evidence = json.loads(properties[0].get("value", "")) @@ -56,7 +88,8 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: sys.executable, "-m", "pytest", - TEST_NODE, + "-m", + "nightly", "-o", "junit_family=xunit1", f"--junitxml={report}", @@ -67,6 +100,8 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: summary = { "wheel": str(wheel), "test_node": TEST_NODE, + "required_test_nodes": list(REQUIRED_TEST_NODES), + "test_nodes": [], "command": command, "status": "failed", } @@ -79,13 +114,18 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True ).stdout.strip() summary["source_sha"] = source_sha - print(f"Running {TEST_NODE} at {source_sha}", flush=True) + print( + f"Running nightly acceptance at {source_sha}; required: {REQUIRED_TEST_NODES}", + flush=True, + ) with (output_dir / "pytest.log").open("w") as log: result = subprocess.run( command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT ) print((output_dir / "pytest.log").read_text(), end="", flush=True) summary["pytest_exit_code"] = result.returncode + if report.is_file(): + summary["test_nodes"] = [test_node_id(case) for case in read_cases(report)] if result.returncode != 0: raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log") summary["acceptance"] = read_evidence(report, wheel_sha256) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 91e9211..c2a3c57 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -14,12 +14,12 @@ Concurrency is keyed by GitHub ref and superseded runs are cancelled. | --- | --- | | Snapshot provenance | Full Git history; base/head comparison; PR decision required only in PR mode | | Lint & Format | Python 3.13, editable dev install, Ruff, Black, basedpyright with the selected interpreter | -| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow"` | -| Full test tier | Python 3.13; fixture provenance first, then all tests with branch coverage, including slow tests | +| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow and not nightly"` | +| Full PR test tier | Python 3.13; `pytest -m "not nightly"` with branch coverage, preserving the existing slow tests | | Overall coverage | `pyproject.toml` fail-under 90 with branches enabled | | Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently | | Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance | -| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then required installed-wheel CLI/API test | +| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then `nightly` tests including the required installed-wheel CLI/API test | Million-event deep-offset pagination tests are marked `slow`; their small pagination contracts still run across the compatibility matrix. @@ -30,9 +30,13 @@ The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses tempora paths but imports a reviewed executable fixture. These are more than YAML checks. The nightly/manual job invokes `run_dataset_acceptance.py` with the single built -wheel. Missing wheels, missing/empty JUnit, skipped/failed tests, extra test cases, -or evidence for a different wheel fail the gate. It uploads the pytest log, JUnit -(including the test's evidence), and a summary with actual checkout SHA and wheel +wheel. All `nightly` tests are selected automatically; the installed-wheel case +must appear exactly once. New mandatory gates also add their complete pytest +node IDs to `REQUIRED_TEST_NODES` in that runner. Do not register absent future +files. Missing wheels, missing/empty JUnit, missing required nodes, duplicate +identities, any skipped/failed/error test, or evidence for a different wheel fail +the gate. It uploads the pytest log, JUnit (including the test's evidence), and a +summary with complete executed/required test node IDs, actual checkout SHA and wheel SHA-256, even after failure. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` is available for additional test-owned artifacts. Local invocation: @@ -44,8 +48,8 @@ python .github/scripts/run_dataset_acceptance.py \ This tier initially exercises the existing 500,002-event `pip --target` acceptance test. It does not claim a clean venv, two full 500,000-event shards, complete CLI template coverage, live agents or upstream GUI interoperability. The ordinary -full suite may still skip this opt-in wheel test; the required nightly/manual -runner must never do so. Manually dispatch the exact published branch/commit and +PR tier excludes `nightly` tests; the required nightly/manual runner must never +skip them. Manually dispatch the exact published branch/commit and verify the run's head SHA and summary; a PR run alone is not nightly evidence. ## Release Workflow diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e71defe..0465fca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -129,11 +129,11 @@ jobs: - name: Run non-slow compatibility tests if: matrix.python-version != '3.13' - run: pytest -m "not slow" + run: pytest -m "not slow and not nightly" - - name: Run all tests with coverage + - name: Run full PR tests with coverage if: matrix.python-version == '3.13' - run: pytest --cov --cov-report=xml --cov-report=term-missing + run: pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing - name: Guard critical runtime coverage if: matrix.python-version == '3.13' diff --git a/pyproject.toml b/pyproject.toml index f97f49b..f65fa0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ select = ["E", "F", "W", "I", "N", "UP", "B", "C4"] ignore = ["E501"] [tool.pytest.ini_options] +markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"] testpaths = ["tests"] python_files = ["test_*.py"] python_functions = ["test_*"] diff --git a/tests/core/test_dataset_scale_acceptance.py b/tests/core/test_dataset_scale_acceptance.py index e2d1b0d..8fb8a05 100644 --- a/tests/core/test_dataset_scale_acceptance.py +++ b/tests/core/test_dataset_scale_acceptance.py @@ -226,6 +226,7 @@ def installed_runtime(tmp_path): return cli, env, module, hashlib.sha256(wheel.read_bytes()).hexdigest() +@pytest.mark.nightly def test_installed_trusted_producer_default_export_cli_and_api( tmp_path, installed_runtime, record_property ): diff --git a/tests/test_ci_tiers.py b/tests/test_ci_tiers.py index 9a6e10f..f011586 100644 --- a/tests/test_ci_tiers.py +++ b/tests/test_ci_tiers.py @@ -19,10 +19,14 @@ def test_compatibility_matrix_leaves_slow_and_coverage_on_python_313(): steps = named_steps(job) fast = steps["Run non-slow compatibility tests"] assert fast["if"] == "matrix.python-version != '3.13'" - assert fast["run"] == 'pytest -m "not slow"' - full = steps["Run all tests with coverage"] + assert fast["run"] == 'pytest -m "not slow and not nightly"' + full = steps["Run full PR tests with coverage"] assert full["if"] == "matrix.python-version == '3.13'" - assert full["run"] == "pytest --cov --cov-report=xml --cov-report=term-missing" + assert full["run"] == 'pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing' + assert ( + 'markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"]' + in Path("pyproject.toml").read_text() + ) assert "if" not in steps["Verify executable fixture provenance"] names = list(steps) assert names.index("Verify executable fixture provenance") < names.index(fast["name"]) diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py index 36ff22e..037e488 100644 --- a/tests/test_dataset_acceptance_runner.py +++ b/tests/test_dataset_acceptance_runner.py @@ -24,7 +24,13 @@ def runner(): def write_report(path, name, evidence, outcome=None): suite = ET.Element("testsuite") - case = ET.SubElement(suite, "testcase", name=name) + case = ET.SubElement( + suite, + "testcase", + name=name, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) if outcome: ET.SubElement(case, outcome) if evidence is not None: @@ -61,7 +67,13 @@ def test_runner_rejects_empty_or_extra_collection(runner, tmp_path, count): report = tmp_path / "junit.xml" suite = ET.Element("testsuite") for _ in range(count): - ET.SubElement(suite, "testcase", name=runner.TEST_NAME) + ET.SubElement( + suite, + "testcase", + name=runner.TEST_NAME, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) ET.ElementTree(suite).write(report) with pytest.raises(runner.AcceptanceError, match="exactly one"): runner.read_evidence(report, "abc") @@ -74,6 +86,106 @@ def test_runner_rejects_wrong_test_even_if_it_passes(runner, tmp_path): runner.read_evidence(report, "abc") +def add_case( + report, + *, + name="test_extra", + file="tests/test_extra.py", + classname="tests.test_extra", + outcome=None, +): + tree = ET.parse(report) + case = ET.SubElement(tree.getroot(), "testcase", name=name, file=file, classname=classname) + if outcome: + ET.SubElement(case, outcome) + tree.write(report) + + +def test_runner_accepts_additional_nightly_cases_and_records_complete_identities(runner, tmp_path): + report = tmp_path / "junit.xml" + evidence = {"wheel_sha256": "abc"} + write_report(report, runner.TEST_NAME, evidence) + add_case(report, name="test_extra[v2]", classname="tests.test_extra.TestRoutes") + assert runner.read_evidence(report, "abc") == evidence + assert [runner.test_node_id(case) for case in runner.read_cases(report)] == [ + runner.TEST_NODE, + "tests/test_extra.py::TestRoutes::test_extra[v2]", + ] + + +def test_runner_rejects_missing_configured_gate(runner, tmp_path, monkeypatch): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + required = "tests/test_extra.py::test_extra" + monkeypatch.setattr(runner, "REQUIRED_TEST_NODES", (runner.TEST_NODE, required)) + with pytest.raises(runner.AcceptanceError, match="Missing required nightly test nodes"): + runner.read_evidence(report, "abc") + add_case(report) + assert runner.read_evidence(report, "abc") == {"wheel_sha256": "abc"} + + +def test_runner_rejects_duplicate_additional_cases(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + add_case(report) + add_case(report) + with pytest.raises(runner.AcceptanceError, match="Duplicate nightly"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_any_nonpassing_additional_case(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + add_case(report, outcome=outcome) + with pytest.raises( + runner.AcceptanceError, match="without skips: tests/test_extra.py::test_extra" + ): + runner.read_evidence(report, "abc") + + +def test_same_function_name_in_another_file_cannot_replace_required_wheel_case(runner, tmp_path): + report = tmp_path / "junit.xml" + ET.ElementTree(ET.Element("testsuite")).write(report) + add_case(report, name=runner.TEST_NAME) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize( + "attributes", [{"file": ""}, {"file": "../tests/test_extra.py"}, {"classname": "wrong"}] +) +def test_runner_rejects_incomplete_or_inconsistent_junit_identity(runner, attributes): + case = ET.Element( + "testcase", file="tests/test_extra.py", classname="tests.test_extra", name="test_extra" + ) + case.attrib.update(attributes) + with pytest.raises(runner.AcceptanceError, match="identity"): + runner.test_node_id(case) + + +def test_nightly_marker_selection_collects_every_configured_gate(runner): + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "--strict-markers", + "-q", + "-o", + "addopts=", + "-m", + "nightly", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + collected = {line for line in result.stdout.splitlines() if "::" in line} + assert set(runner.REQUIRED_TEST_NODES) <= collected + + def test_runner_rejects_duplicate_evidence(runner, tmp_path): report = tmp_path / "junit.xml" write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) @@ -132,7 +244,7 @@ def run(command, **kwargs): if command[0] == "git": return subprocess.CompletedProcess(command, 0, stdout="checkout-sha\n") assert command[:3] == [sys.executable, "-m", "pytest"] - assert runner.TEST_NODE in command + assert command[3:5] == ["-m", "nightly"] assert "junit_family=xunit1" in command assert kwargs["env"]["PT_SNAP_ACCEPTANCE_WHEEL"] == str(wheel) assert kwargs["env"]["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] == str(output) @@ -153,5 +265,7 @@ def run(command, **kwargs): assert summary["source_sha"] == "checkout-sha" assert summary["wheel_sha256"] == wheel_hash assert summary["pytest_exit_code"] == exit_code + assert summary["required_test_nodes"] == [runner.TEST_NODE] + assert summary["test_nodes"] == ([runner.TEST_NODE] if write_junit else []) assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") assert (output / "pytest.log").read_text() == "captured pytest output\n"