-
Notifications
You must be signed in to change notification settings - Fork 2
fix(devtools): run the corpus and the required check at the pool width #4682
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
d1d36dd
5862467
09782f8
3f1249b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -319,7 +319,7 @@ def _write_launch( | |
| "argv": list(argv), | ||
| "working_directory": cwd, | ||
| "environment": dict(env), | ||
| "timeout_seconds": 3600, | ||
| "timeout_seconds": 7200, | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When the required check needs close to the new two-hour allowance, it can never receive that full allowance: AGENTS.md reference: AGENTS.md:L151-L154 Useful? React with 👍 / 👎. |
||
| "result_kind": "exit", | ||
| "log_path": str(log_path), | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -27,10 +27,11 @@ | |
| import subprocess | ||
| import sys | ||
| import time | ||
| from collections.abc import Mapping | ||
| from pathlib import Path | ||
| from typing import Any, cast | ||
|
|
||
| from devtools.agent_env import agent_worker_cap, inside_agent_job | ||
| from devtools.agent_env import HARNESS_RUN_ENV, agent_worker_cap, inside_agent_job | ||
| from devtools.checkout_guard import ( | ||
| CheckoutImportMismatchError, | ||
| assert_polylogue_matches_checkout, | ||
|
|
@@ -370,16 +371,23 @@ def build_pytest_cmd(selection: list[str]) -> list[str]: | |
| *collection_args, | ||
| # A focused run traces into the one checkout datafile and writes back, | ||
| # so the graph is advanced by every managed run. It never selects: the | ||
| # caller already named what to run. | ||
| "--testmon", | ||
| f"--testmon-env={TESTMON_ENVIRONMENT}", | ||
| "--testmon-noselect", | ||
| # caller already named what to run. A run spawned from inside another | ||
| # managed run (a test exercising the harness) must not touch that | ||
| # datafile: its session would reset the outer run's pending graph. | ||
| *_testmon_args(os.environ), | ||
| *selection, | ||
| *worker_args, | ||
| *_xdist_distribution_args(selection, worker_args), | ||
| ] | ||
|
|
||
|
|
||
| def _testmon_args(env: Mapping[str, str]) -> tuple[str, ...]: | ||
| """testmon flags for a focused run; none when nested in a managed run.""" | ||
| if env.get(HARNESS_RUN_ENV): | ||
| return ("-p", "no:testmon") | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a test starts a nested AGENTS.md reference: AGENTS.md:L163-L168 Useful? React with 👍 / 👎. |
||
| return ("--testmon", f"--testmon-env={TESTMON_ENVIRONMENT}", "--testmon-noselect") | ||
|
|
||
|
|
||
| def _selection_targets_benchmarks(selection: list[str]) -> bool: | ||
| """Keep benchmark collection available only when the caller asks for it.""" | ||
| return any("tests/benchmarks" in argument for argument in selection) | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -76,9 +76,12 @@ | |
| PYTEST_SUMMARY_PATH = PYTEST_REPORT_DIR / "current-pytest-summary.json" | ||
| PYTEST_OUTPUT_PATH = PYTEST_REPORT_DIR / "current-pytest-output.log" | ||
| PYTEST_JUNIT_REPORT_DIR = PYTEST_REPORT_DIR / "junit" | ||
| #: SQLite archive construction makes the corpus IO-bound. Two workers provide | ||
| #: overlap without multiplying cache churn or exhausting the pytest cgroup. | ||
| CORPUS_MAX_WORKERS = 2 | ||
| #: One fixed width for the corpus and the runner's affected tier, sized to the | ||
| #: pytest pool's 12 GiB cgroup ceiling (eight workers peak near 10 GB) rather | ||
| #: than host cores or free RAM. Measured 2026-09-03 uncontended: 47 minutes for | ||
| #: 20,860 tests at eight workers; at two the same run takes about seven hours | ||
| #: and the required check cannot finish inside its slot timeout. | ||
| CORPUS_MAX_WORKERS = 8 | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When the scheduled Useful? React with 👍 / 👎. |
||
| _AGENTCTL_OPERATION_ARGV = {"verify_affected": (), "verify_quick": ("--quick",), "verify_all": ("--all",)} | ||
| _PROJECT_DESCRIPTOR = ".agentctl/project.toml" | ||
| # These tests read the AgentCTL descriptor directly. They are the bounded | ||
|
|
@@ -293,6 +296,24 @@ def _read_json(path: Path) -> dict[str, Any] | None: | |
| MAX_RERUN_NODEIDS = 300 | ||
|
|
||
|
|
||
| def _report_nodeid_to_selector(nodeid: str) -> str: | ||
| """Strip xdist's ``@<group>`` suffix so a report node id selects again. | ||
|
|
||
| ``--dist=loadgroup`` reports ``path::test[param]@group``; pytest cannot | ||
| collect that literal, so a rerun built from it errors before running. | ||
| A parametrization id may itself contain ``@``, so only a suffix after the | ||
| closing bracket (or after the bare test name) is removed. | ||
| """ | ||
| head, sep, tail = nodeid.rpartition("@") | ||
| if not sep or "::" not in head: | ||
| return nodeid | ||
| if "[" in tail or "]" in tail or "/" in tail or "::" in tail: | ||
| return nodeid | ||
| if head.endswith("]") or "[" not in head.rsplit("::", 1)[-1]: | ||
| return head | ||
| return nodeid | ||
|
|
||
|
|
||
| def _rerun_failed_once(command: Sequence[str], *, env: Mapping[str, str], artifacts: Any) -> dict[str, Any] | None: | ||
| """Rerun exactly the failed tests once, alone and unselected. | ||
|
|
||
|
|
@@ -306,7 +327,7 @@ def _rerun_failed_once(command: Sequence[str], *, env: Mapping[str, str], artifa | |
| if not isinstance(report, Mapping): | ||
| return None | ||
| failed = [ | ||
| str(test["nodeid"]) | ||
| _report_nodeid_to_selector(str(test["nodeid"])) | ||
| for test in report.get("tests", []) | ||
| if isinstance(test, Mapping) and test.get("outcome") in {"failed", "error"} and test.get("nodeid") | ||
|
Comment on lines
329
to
332
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When an Useful? React with 👍 / 👎. |
||
| ] | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,7 +10,7 @@ | |
|
|
||
| from devtools.pytest_invocation import MANAGED_PLUGIN_ARGS | ||
| from devtools.toolchain import venv_python | ||
| from devtools.verify import _pytest_worker_args | ||
| from devtools.verify import CORPUS_MAX_WORKERS, _pytest_worker_args | ||
|
|
||
|
|
||
| def main(_argv: list[str] | None = None) -> int: | ||
|
|
@@ -27,8 +27,8 @@ def main(_argv: list[str] | None = None) -> int: | |
| finally: | ||
| if configured_workers is not None: | ||
| os.environ["POLYLOGUE_PYTEST_WORKERS"] = configured_workers | ||
| if default_worker_args != ["--dist=loadgroup", "-n", "2"]: | ||
| print("testmon-selection: managed verification does not default to two workers") | ||
| if default_worker_args != ["--dist=loadgroup", "-n", str(CORPUS_MAX_WORKERS)]: | ||
| print(f"testmon-selection: managed verification does not default to {CORPUS_MAX_WORKERS} workers") | ||
|
Comment on lines
+30
to
+31
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
The anti-vacuity mutation named by this gate, setting AGENTS.md reference: AGENTS.md:L192-L193 Useful? React with 👍 / 👎. |
||
| return 1 | ||
| with tempfile.TemporaryDirectory(prefix="polylogue-testmon-gate-") as temporary: | ||
| root = Path(temporary) | ||
|
|
@@ -93,7 +93,7 @@ def main(_argv: list[str] | None = None) -> int: | |
| if not selected or not total or selected * 100 >= total * 5: | ||
| print(f"testmon-selection: selected {selected} of {total}, expected under 5%\n{output}") | ||
| return 1 | ||
| print(f"testmon-selection: selected {selected} of {total}; workers=2") | ||
| print(f"testmon-selection: selected {selected} of {total}; workers={CORPUS_MAX_WORKERS}") | ||
| return 0 | ||
|
|
||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
At this reviewed head, running
devtools test tests/unit/devtools/test_verify.py::test_verify_quick_descriptor_accepts_the_declared_json_projectionfails because line 305 still requires the oldenv POLYLOGUE_PYTEST_WORKERS=2command. Update that contract expectation alongside this intentional descriptor change; otherwise any complete corpus run—and any affected run selecting this test—reports a verification failure.AGENTS.md reference: AGENTS.md:L172-L173
Useful? React with 👍 / 👎.