From e9d823ca85e35472547025dd350cdaf6b5efbcc3 Mon Sep 17 00:00:00 2001 From: Sinity Date: Sat, 5 Sep 2026 20:35:36 +0200 Subject: [PATCH] fix(devtools): let the corpus run write the testmon graph again The complete-corpus tier ran with `-p no:pytest-testmon` and no testmon flags, so it wrote no fingerprints and left the checkout datafile exactly as it found it. Combined with the provision step discarding an unusable seed at run start, a 58-minute corpus run ended with an empty .cache/testmon directory and every later affected run forced to reseed. The `all` tier now loads testmon and passes `--testmon-noselect`: it executes the whole collection and records what it traced. `descriptor` still opts out; its collection is a contract slice, not a corpus. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01DNGJ3awfNrsLaMdHgQZvid --- devtools/pytest_invocation.py | 6 +-- devtools/verify.py | 14 ++++-- tests/unit/devtools/test_verify.py | 69 +++++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/devtools/pytest_invocation.py b/devtools/pytest_invocation.py index 5f26e5083a..d60422798c 100644 --- a/devtools/pytest_invocation.py +++ b/devtools/pytest_invocation.py @@ -50,9 +50,9 @@ def managed_plugin_args(*, testmon: bool) -> tuple[str, ...]: """Return the explicit plugin profile for a managed pytest mode. - Complete-corpus runs already execute every collected test. Loading - testmon there retains a dependency tracer and graph in every xdist worker - without changing coverage, so only selecting and bootstrap runs load it. + A run that omits testmon writes no fingerprints, so it leaves the datafile + exactly as it found it. Only a mode whose collection is not a corpus opts + out of tracing. """ if testmon: return MANAGED_PLUGIN_ARGS diff --git a/devtools/verify.py b/devtools/verify.py index ee6cc894cd..4e660a9571 100644 --- a/devtools/verify.py +++ b/devtools/verify.py @@ -167,8 +167,16 @@ def _pytest_worker_args(*, maximum: int | None = None) -> list[str]: def _pytest_steps(*, selection: str, worker_args: Sequence[str]) -> list[tuple[str, list[str]]]: - """Build one complete collection, or an affected collection with tracing.""" - testmon = selection not in {"all", "descriptor"} + """Build one complete collection, or an affected collection, both tracing. + + Both tiers load testmon so every managed corpus or affected run advances the + one datafile. ``all`` deselects nothing -- it executes the whole collection + and records what it traced, which is what makes the next affected run + selectable. Only ``descriptor`` opts out: it collects a contract slice, not + a corpus, so its fingerprints would describe a collection no later run has. + """ + testmon = selection != "descriptor" + select_flag = "--testmon-noselect" if selection == "all" else "--testmon-forceselect" collection_args = CLOSED_WORLD_COLLECTION_ARGS[:-1] if selection == "descriptor" else CLOSED_WORLD_COLLECTION_ARGS command = [ venv_python(root=ROOT), @@ -186,7 +194,7 @@ def _pytest_steps(*, selection: str, worker_args: Sequence[str]) -> list[tuple[s PROGRESS_PLUGIN_NAME, *managed_plugin_args(testmon=testmon), *collection_args, - *(["--testmon", f"--testmon-env={TESTMON_ENVIRONMENT}", "--testmon-forceselect"] if testmon else []), + *(["--testmon", f"--testmon-env={TESTMON_ENVIRONMENT}", select_flag] if testmon else []), "-p", "no:randomly", *worker_args, diff --git a/tests/unit/devtools/test_verify.py b/tests/unit/devtools/test_verify.py index 192f3f15f9..9dff70e2ec 100644 --- a/tests/unit/devtools/test_verify.py +++ b/tests/unit/devtools/test_verify.py @@ -4,7 +4,9 @@ import io import json +import os import signal +import sqlite3 import subprocess import sys from pathlib import Path @@ -22,7 +24,7 @@ verify_runs, why, ) -from devtools.testmon_provision import TestmonGraphStatus +from devtools.testmon_provision import TESTMON_ENVIRONMENT, TestmonGraphStatus from devtools.verification_result import declared_verification_result from devtools.verify_runs import ( CURRENT_RUN_PATH, @@ -926,3 +928,68 @@ def test_rerun_selector_strips_the_xdist_group_suffix() -> None: assert _report_nodeid_to_selector("tests/a.py::T::test_x[p]@grp") == "tests/a.py::T::test_x[p]" assert _report_nodeid_to_selector("tests/a.py::test_x[a@b]") == "tests/a.py::test_x[a@b]" assert _report_nodeid_to_selector("tests/a.py::test_x") == "tests/a.py::test_x" + + +def test_complete_corpus_tier_traces_and_deselects_nothing() -> None: + """The ``all`` tier must load testmon and select every collected test.""" + command = verify.build_verify_steps(quick=False, selection="all")[-1][1] + + assert "pytest-testmon" in command + assert "--testmon" in command + assert f"--testmon-env={TESTMON_ENVIRONMENT}" in command + assert "--testmon-noselect" in command + assert "--testmon-forceselect" not in command + + +def _verify_shaped_argv(*, selection: str, target: Path, tmp_path: Path) -> list[str]: + """The real verifier argv, redirected at one temporary test file.""" + command = list(verify.build_verify_steps(quick=False, selection=selection)[-1][1]) + command[command.index("tests")] = str(target) + command[command.index("-n") + 1] = "2" + return [ + argument for argument in command if not argument.startswith(("--junitxml=", "--json-report-file=", "--ignore=")) + ] + [f"--json-report-file={tmp_path / 'report.json'}"] + + +@pytest.mark.slow +def test_complete_corpus_run_records_a_usable_testmon_graph(tmp_path: Path) -> None: + """A verify-shaped ``all`` session leaves fingerprints behind. + + Anti-vacuity: dropping ``pytest-testmon`` from the tier's plugin profile, or + replacing ``--testmon-noselect`` with no testmon flags at all -- the state + this checkout shipped, under which a 58-minute corpus run wrote nothing -- + leaves ``test_execution`` empty and makes this red. + """ + target = tmp_path / "test_traced.py" + target.write_text("def test_one():\n assert True\n\n\ndef test_two():\n assert True\n") + datafile = tmp_path / "testmondata" + checkout_root = Path(__file__).resolve().parents[3] + + env = dict(os.environ) + env.update( + { + "TESTMON_DATAFILE": str(datafile), + "POLYLOGUE_PYTEST_RUN_ID": "testmon-graph-regression", + "POLYLOGUE_PYTEST_EVENTS_DIR": str(tmp_path / "events"), + "POLYLOGUE_PYTEST_SELECTION_PATH": str(tmp_path / "selection.json"), + "POLYLOGUE_PYTEST_SUMMARY_PATH": str(tmp_path / "summary.json"), + } + ) + env.pop("PYTEST_DISABLE_PLUGIN_AUTOLOAD", None) + + result = subprocess.run( + _verify_shaped_argv(selection="all", target=target, tmp_path=tmp_path), + cwd=checkout_root, + env=env, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert datafile.exists(), result.stdout + result.stderr + with sqlite3.connect(datafile) as connection: + recorded = connection.execute("SELECT count(*) FROM test_execution").fetchone()[0] + environments = [row[0] for row in connection.execute("SELECT environment_name FROM environment")] + assert recorded == 2 + assert environments == [TESTMON_ENVIRONMENT]