diff --git a/README.md b/README.md index 798bafc..15b23e6 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,9 @@ The package gate executes: source before running any archived verifier or tests; release-manifest hashes are checked when present, and the input binding is checked again at the end; - all repository and subatomic tests against separate clean wheel and source installs; +- the current collected test inventory, with every test completing successfully, + no skips, and matching JUnit counts; the historical release's 209-test count + is not imposed on newer source revisions; - complete installed-distribution byte maps, import origins, exact UCNS source maps, and complete source snapshots before and after each replay; - the preregistered molecular comparison, requiring all 14 current standings (including the original four) to remain `FALSIFIED`; @@ -73,6 +76,10 @@ output directories. The replay gate records the exact source and artifact bindin in `candidate-source.json`; a stale or incomplete sdist is rejected before its code can run. Source distributions include every tracked repository file, including CI definitions and the complete local operational skill snapshot. +When adding a root `epac_*.py` module, add it to `tool.setuptools.py-modules`; +the archive-coverage gate rejects omitted modules. Current candidates also +package `epac_atomic_derivation` and `epac_b_derivation`; the immutable v0.1.0 +release retains its original bytes. ```bash python -m pip install uv==0.11.18 diff --git a/pyproject.toml b/pyproject.toml index 507f77c..e08ee5b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ test = ["pytest==9.1.1"] build = ["build==1.5.0", "twine==7.0.0", "setuptools==84.0.0", "wheel==0.48.0"] [tool.setuptools] -py-modules = ["epac_evidence_cache", "epac_atomic", "epac_comparison", "epac_dimensional_arity", "epac_molecular", "epac_periodic", "epac_public_gonol", "epac_ucns_provenance", "epac_boundary_minimal_refinement", "epac_boundary_nondegeneracy", "epac_boundary_probe_completeness", "epac_boundary_quotient", "epac_cross_scale_closure"] +py-modules = ["epac_evidence_cache", "epac_atomic", "epac_atomic_derivation", "epac_b_derivation", "epac_comparison", "epac_dimensional_arity", "epac_molecular", "epac_periodic", "epac_public_gonol", "epac_ucns_provenance", "epac_boundary_minimal_refinement", "epac_boundary_nondegeneracy", "epac_boundary_probe_completeness", "epac_boundary_quotient", "epac_cross_scale_closure"] packages = ["epac_data", "epac_subatomic", "epac_viz"] package-dir = {epac_data = "data", epac_subatomic = "subatomic", epac_viz = "viz"} diff --git a/tests/test_distribution_replay.py b/tests/test_distribution_replay.py index d13d02a..5c6c580 100644 --- a/tests/test_distribution_replay.py +++ b/tests/test_distribution_replay.py @@ -6,6 +6,13 @@ # requires: python3, bash # mutates: temporary fixture archives, virtual environment, and installer trace # cleanup: temporary directory context removes all fixture state +# +# id: check_epac_complete_test_evidence +# proves: epac_installation_replays_independently +# call: self::test_installed_replay_requires_complete_test_evidence +# requires: python3 +# mutates: filesystem +# cleanup: temporary directory context removes fixture state # === END CHECKS === from pathlib import Path import io @@ -25,6 +32,44 @@ from unittest.mock import patch +def test_installed_replay_requires_complete_test_evidence(): + """Usage: pytest tests/test_distribution_replay.py; exercise false-green receipts.""" + path = Path(__file__).resolve().parents[1] / "tools/verify_installed.py" + spec = importlib.util.spec_from_file_location("epac_installed_fixture", path) + verifier = importlib.util.module_from_spec(spec) + spec.loader.exec_module(verifier) + import pytest + from _pytest.subtests import SubtestReport, SubtestContext + inventory = verifier.TestInventory() + parent = pytest.TestReport("a", ("test.py", 0, "a"), {}, "passed", None, "call") + subtest = SubtestReport("a", ("test.py", 0, "a"), {}, "passed", None, "call") + subtest.context = SubtestContext(msg=None, kwargs={"case": 1}) + inventory.pytest_runtest_logreport(subtest) + inventory.pytest_runtest_logreport(parent) + assert inventory.passed == ["a"] + with TemporaryDirectory() as directory: + xml = Path(directory) / "results.xml" + good = '' + xml.write_text(good, encoding="utf-8") + assert verifier.verify_test_evidence(xml, ["a", "b"], ["a", "b"]) == 2 + for collected, passed, report in ( + ([], [], good), + (["a", "b"], ["a"], good), + (["a", "a"], ["a", "a"], good), + (["a", "b"], ["a", "b"], good.replace('tests="2"', 'tests="3"')), + (["a", "b"], ["a", "b"], good.replace('failures="0"', 'failures="1"')), + (["a", "b"], ["a", "b"], good.replace('', '')), + (["a", "b"], ["a", "b"], good.replace('', '')), + ): + xml.write_text(report, encoding="utf-8") + try: + verifier.verify_test_evidence(xml, collected, passed) + except AssertionError: + pass + else: + raise AssertionError("incomplete test evidence accepted") + + def _write_fixture_dist(dist, contents, wheel_payload): info = "interdependency_epac-0.1.0.dist-info/" payload = dict(wheel_payload) diff --git a/tests/test_verify_installed_junit.py b/tests/test_verify_installed_junit.py new file mode 100644 index 0000000..feb096d --- /dev/null +++ b/tests/test_verify_installed_junit.py @@ -0,0 +1,39 @@ +"""Regression coverage for pytest subtest accounting in installed replay evidence.""" +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path + +import pytest + + +VERIFIER_PATH = Path(__file__).resolve().parents[1] / "tools" / "verify_installed.py" +SPEC = spec_from_file_location("epac_installed_junit_fixture", VERIFIER_PATH) +VERIFIER = module_from_spec(SPEC) +SPEC.loader.exec_module(VERIFIER) + + +def _write_xml(path, tests): + path.write_text( + f'' + '', + encoding="utf-8", + ) + + +def test_junit_total_accepts_explicit_successful_subtest_count(tmp_path): + xml = tmp_path / "results.xml" + _write_xml(xml, 3) + assert VERIFIER.verify_test_evidence(xml, ["a", "b"], ["a", "b"], passed_subtests=1) == 2 + + +def test_junit_total_rejects_unexplained_extra_results(tmp_path): + xml = tmp_path / "results.xml" + _write_xml(xml, 3) + with pytest.raises(AssertionError, match="aggregate"): + VERIFIER.verify_test_evidence(xml, ["a", "b"], ["a", "b"], passed_subtests=0) + + +def test_junit_total_rejects_inflated_subtest_claim(tmp_path): + xml = tmp_path / "results.xml" + _write_xml(xml, 3) + with pytest.raises(AssertionError, match="aggregate"): + VERIFIER.verify_test_evidence(xml, ["a", "b"], ["a", "b"], passed_subtests=2) diff --git a/tools/verify_installed.py b/tools/verify_installed.py index 094e530..d8d7eb1 100644 --- a/tools/verify_installed.py +++ b/tools/verify_installed.py @@ -38,6 +38,44 @@ import zipfile +class TestInventory: + """Bind successful calls and successful subtests to this pytest run.""" + + def __init__(self): + self.collected = [] + self.passed = [] + self.passed_subtests = 0 + + def pytest_collection_finish(self, session): + self.collected = [item.nodeid for item in session.items] + + def pytest_runtest_logreport(self, report): + import pytest + if report.when != "call" or not report.passed: + return + if type(report) is pytest.TestReport: + self.passed.append(report.nodeid) + elif type(report).__name__ == "SubtestReport": + self.passed_subtests += 1 + + +def verify_test_evidence(xml_path, collected, passed, passed_subtests=0): + """Reject empty, partial, duplicate, skipped or contradictory test evidence.""" + assert collected and len(set(collected)) == len(collected), "empty or duplicate test inventory" + assert passed == collected, "successful calls differ from complete test inventory" + root = ET.parse(xml_path).getroot() + cases = list(root.iter("testcase")) + assert len(cases) == len(collected), "JUnit inventory differs from collected tests" + assert not any(c.find(tag) is not None for c in cases for tag in ("skipped", "failure", "error")) + suites = list(root.iter("testsuite")) + leaf_suites = [suite for suite in suites if not any(child.tag == "testsuite" for child in suite)] + assert leaf_suites, "JUnit evidence contains no test suite" + reported_tests = sum(int(suite.attrib["tests"]) for suite in leaf_suites) + assert reported_tests == len(cases) + passed_subtests, "JUnit aggregate differs from parent tests plus successful subtests" + for suite in suites: + assert all(int(suite.attrib[tag]) == 0 for tag in ("failures", "errors", "skipped")) + return len(cases) + EXPECTED_STANDINGS = { "atomic_shells_as_sealed_shape_prediction": "FALSIFIED", @@ -123,10 +161,10 @@ def payload(): dependencies=(ucns.public_gonol_function, ucns.native_mobius_state)) assert identity == epac_public_gonol.PINNED_UCNS_COMMIT xml_path = receipt_path.with_suffix(".xml") - result = pytest.main([str(source / "tests"), "--junitxml=" + str(xml_path), "-q", "-x", "-p", "no:cacheprovider", "-o", "xfail_strict=true"]) + inventory = TestInventory() + result = pytest.main([str(source / "tests"), "--junitxml=" + str(xml_path), "-q", "-x", "-p", "no:cacheprovider", "-o", "xfail_strict=true"], plugins=[inventory]) assert result == 0, result - cases = list(ET.parse(xml_path).getroot().iter("testcase")) - assert len(cases) == 209 and not any(c.find(tag) is not None for c in cases for tag in ("skipped", "failure", "error")) + test_count = verify_test_evidence(xml_path, inventory.collected, inventory.passed, inventory.passed_subtests) assert payload() == before origins = {name: str(Path(module.__file__).resolve()) for name, module in sys.modules.items() if name.startswith("epac_") and getattr(module, "__file__", None)} @@ -139,8 +177,8 @@ def payload(): receipt = {"artifact_kind": artifact_kind, "artifact_sha256": artifact_digest, "artifact_name": artifact.name, "schema": "epac.installed-replay", "version": 1, "status": "passed", "python": sys.version, "verifier_sha256": hashlib.sha256(verifier_bytes).hexdigest(), "source_files_sha256": archived_inputs, "outcomes_xml_sha256": hashlib.sha256(xml_path.read_bytes()).hexdigest(), "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), "ucns_source_commit": identity, - "installed_payload_sha256": immutable, "installed_distribution_sha256": before, "imported_origins": origins, "tests": len(cases), - "skips": 0, "comparison_standings": standings, "empirical_status_transfer": False} + "installed_payload_sha256": immutable, "installed_distribution_sha256": before, "imported_origins": origins, "tests": test_count, "test_inventory": inventory.collected, + "successful_subtests": inventory.passed_subtests, "skips": 0, "comparison_standings": standings, "empirical_status_transfer": False} receipt_path.write_text(json.dumps(receipt, indent=2, sort_keys=True) + "\n")