diff --git a/devtools/pytest_invocation.py b/devtools/pytest_invocation.py index 5f26e5083..d60422798 100644 --- a/devtools/pytest_invocation.py +++ b/devtools/pytest_invocation.py @@ -50,9 +50,9 @@ def managed_plugin_args(*, testmon: bool) -> tuple[str, ...]: """Return the explicit plugin profile for a managed pytest mode. - Complete-corpus runs already execute every collected test. Loading - testmon there retains a dependency tracer and graph in every xdist worker - without changing coverage, so only selecting and bootstrap runs load it. + A run that omits testmon writes no fingerprints, so it leaves the datafile + exactly as it found it. Only a mode whose collection is not a corpus opts + out of tracing. """ if testmon: return MANAGED_PLUGIN_ARGS diff --git a/devtools/verify.py b/devtools/verify.py index ee6cc894c..4e660a957 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -167,8 +167,16 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: def _pytest_steps(*, selection: str, worker_args: Sequence[str]) -> list[tuple[str, list[str]]]: - """Build one complete collection, or an affected collection with tracing.""" - testmon = selection not in {"all", "descriptor"} + """Build one complete collection, or an affected collection, both tracing. + + Both tiers load testmon so every managed corpus or affected run advances the + one datafile. ``all`` deselects nothing -- it executes the whole collection + and records what it traced, which is what makes the next affected run + selectable. Only ``descriptor`` opts out: it collects a contract slice, not + a corpus, so its fingerprints would describe a collection no later run has. + """ + testmon = selection != "descriptor" + select_flag = "--testmon-noselect" if selection == "all" else "--testmon-forceselect" collection_args = CLOSED_WORLD_COLLECTION_ARGS[:-1] if selection == "descriptor" else CLOSED_WORLD_COLLECTION_ARGS command = [ venv_python(root=ROOT), @@ -186,7 +194,7 @@ def _pytest_steps(*, selection: str, worker_args: Sequence[str]) -> list[tuple[s PROGRESS_PLUGIN_NAME, *managed_plugin_args(testmon=testmon), *collection_args, - *(["--testmon", f"--testmon-env={TESTMON_ENVIRONMENT}", "--testmon-forceselect"] if testmon else []), + *(["--testmon", f"--testmon-env={TESTMON_ENVIRONMENT}", select_flag] if testmon else []), "-p", "no:randomly", *worker_args, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 192f3f15f..9dff70e2e 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -4,7 +4,9 @@ import io import json +import os import signal +import sqlite3 import subprocess import sys from pathlib import Path @@ -22,7 +24,7 @@ verify_runs, why, ) -from devtools.testmon_provision import TestmonGraphStatus +from devtools.testmon_provision import TESTMON_ENVIRONMENT, TestmonGraphStatus from devtools.verification_result import declared_verification_result from devtools.verify_runs import ( CURRENT_RUN_PATH, @@ -926,3 +928,68 @@ def test_rerun_selector_strips_the_xdist_group_suffix() -> None: assert _report_nodeid_to_selector("tests/a.py::T::test_x[p]@grp") == "tests/a.py::T::test_x[p]" assert _report_nodeid_to_selector("tests/a.py::test_x[a@b]") == "tests/a.py::test_x[a@b]" assert _report_nodeid_to_selector("tests/a.py::test_x") == "tests/a.py::test_x" + + +def test_complete_corpus_tier_traces_and_deselects_nothing() -> None: + """The ``all`` tier must load testmon and select every collected test.""" + command = verify.build_verify_steps(quick=False, selection="all")[-1][1] + + assert "pytest-testmon" in command + assert "--testmon" in command + assert f"--testmon-env={TESTMON_ENVIRONMENT}" in command + assert "--testmon-noselect" in command + assert "--testmon-forceselect" not in command + + +def _verify_shaped_argv(*, selection: str, target: Path, tmp_path: Path) -> list[str]: + """The real verifier argv, redirected at one temporary test file.""" + command = list(verify.build_verify_steps(quick=False, selection=selection)[-1][1]) + command[command.index("tests")] = str(target) + command[command.index("-n") + 1] = "2" + return [ + argument for argument in command if not argument.startswith(("--junitxml=", "--json-report-file=", "--ignore=")) + ] + [f"--json-report-file={tmp_path / 'report.json'}"] + + +@pytest.mark.slow +def test_complete_corpus_run_records_a_usable_testmon_graph(tmp_path: Path) -> None: + """A verify-shaped ``all`` session leaves fingerprints behind. + + Anti-vacuity: dropping ``pytest-testmon`` from the tier's plugin profile, or + replacing ``--testmon-noselect`` with no testmon flags at all -- the state + this checkout shipped, under which a 58-minute corpus run wrote nothing -- + leaves ``test_execution`` empty and makes this red. + """ + target = tmp_path / "test_traced.py" + target.write_text("def test_one():\n assert True\n\n\ndef test_two():\n assert True\n") + datafile = tmp_path / "testmondata" + checkout_root = Path(__file__).resolve().parents[3] + + env = dict(os.environ) + env.update( + { + "TESTMON_DATAFILE": str(datafile), + "POLYLOGUE_PYTEST_RUN_ID": "testmon-graph-regression", + "POLYLOGUE_PYTEST_EVENTS_DIR": str(tmp_path / "events"), + "POLYLOGUE_PYTEST_SELECTION_PATH": str(tmp_path / "selection.json"), + "POLYLOGUE_PYTEST_SUMMARY_PATH": str(tmp_path / "summary.json"), + } + ) + env.pop("PYTEST_DISABLE_PLUGIN_AUTOLOAD", None) + + result = subprocess.run( + _verify_shaped_argv(selection="all", target=target, tmp_path=tmp_path), + cwd=checkout_root, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert datafile.exists(), result.stdout + result.stderr + with sqlite3.connect(datafile) as connection: + recorded = connection.execute("SELECT count(*) FROM test_execution").fetchone()[0] + environments = [row[0] for row in connection.execute("SELECT environment_name FROM environment")] + assert recorded == 2 + assert environments == [TESTMON_ENVIRONMENT]