Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .agentctl/project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ verification-graph = ["devtools/verify.py", "pyproject.toml"]

[operations.verify_affected]
description = "Run Polylogue's affected-test verification plan"
exec = ["env", "POLYLOGUE_PYTEST_WORKERS=2", "devtools", "verify"]
exec = ["devtools", "verify"]

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Update the descriptor test for the new command

At this reviewed head, running devtools test tests/unit/devtools/test_verify.py::test_verify_quick_descriptor_accepts_the_declared_json_projection fails because line 305 still requires the old env POLYLOGUE_PYTEST_WORKERS=2 command. Update that contract expectation alongside this intentional descriptor change; otherwise any complete corpus run—and any affected run selecting this test—reports a verification failure.

AGENTS.md reference: AGENTS.md:L172-L173

Useful? React with 👍 / 👎.

pool = "pytest"
result = "pytest"
cache = "tree+environment"
timeout_seconds = 3600
timeout_seconds = 7200

[operations.verify_quick]
description = "Run Polylogue's static fast verification gates"
Expand Down
2 changes: 1 addition & 1 deletion devtools/pytest_slot.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,7 @@ def _write_launch(
"argv": list(argv),
"working_directory": cwd,
"environment": dict(env),
"timeout_seconds": 3600,
"timeout_seconds": 7200,

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Leave queueing headroom below the workflow timeout

When the required check needs close to the new two-hour allowance, it can never receive that full allowance: .github/workflows/verify.yml still gives the entire job 120 minutes, including checkout, graph seeding, and the wait behind other tasks in the host's serialized pytest queue, while this task's own 7,200-second timer starts only after it reaches the front. GitHub can therefore kill a valid queued run before this deadline or its terminal receipt; either lower this inner timeout or raise the workflow timeout enough to cover queue wait and setup.

AGENTS.md reference: AGENTS.md:L151-L154

Useful? React with 👍 / 👎.

"result_kind": "exit",
"log_path": str(log_path),
}
Expand Down
18 changes: 13 additions & 5 deletions devtools/run_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@
import subprocess
import sys
import time
from collections.abc import Mapping
from pathlib import Path
from typing import Any, cast

from devtools.agent_env import agent_worker_cap, inside_agent_job
from devtools.agent_env import HARNESS_RUN_ENV, agent_worker_cap, inside_agent_job
from devtools.checkout_guard import (
CheckoutImportMismatchError,
assert_polylogue_matches_checkout,
Expand Down Expand Up @@ -370,16 +371,23 @@ def build_pytest_cmd(selection: list[str]) -> list[str]:
*collection_args,
# A focused run traces into the one checkout datafile and writes back,
# so the graph is advanced by every managed run. It never selects: the
# caller already named what to run.
"--testmon",
f"--testmon-env={TESTMON_ENVIRONMENT}",
"--testmon-noselect",
# caller already named what to run. A run spawned from inside another
# managed run (a test exercising the harness) must not touch that
# datafile: its session would reset the outer run's pending graph.
*_testmon_args(os.environ),
*selection,
*worker_args,
*_xdist_distribution_args(selection, worker_args),
]


def _testmon_args(env: Mapping[str, str]) -> tuple[str, ...]:
"""testmon flags for a focused run; none when nested in a managed run."""
if env.get(HARNESS_RUN_ENV):
return ("-p", "no:testmon")

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Disable the plugin by its registered entry-point name

When a test starts a nested devtools test, this branch does not actually disable testmon: MANAGED_PLUGIN_ARGS has already loaded the entry point as pytest-testmon, while -p no:testmon blocks only the differently named testmon/pytest_testmon plugins. As python -m pytest --help specifies, -p name loads the given module or entry-point name and no: avoids loading that named plugin; using no:pytest-testmon (or omitting it from the managed plugin list) is required. Otherwise the nested run continues writing the checkout's shared testmon datafile and can again replace the outer corpus graph with its focused scope.

AGENTS.md reference: AGENTS.md:L163-L168

Useful? React with 👍 / 👎.

return ("--testmon", f"--testmon-env={TESTMON_ENVIRONMENT}", "--testmon-noselect")


def _selection_targets_benchmarks(selection: list[str]) -> bool:
"""Keep benchmark collection available only when the caller asks for it."""
return any("tests/benchmarks" in argument for argument in selection)
Expand Down
29 changes: 25 additions & 4 deletions devtools/verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,12 @@
PYTEST_SUMMARY_PATH = PYTEST_REPORT_DIR / "current-pytest-summary.json"
PYTEST_OUTPUT_PATH = PYTEST_REPORT_DIR / "current-pytest-output.log"
PYTEST_JUNIT_REPORT_DIR = PYTEST_REPORT_DIR / "junit"
#: SQLite archive construction makes the corpus IO-bound. Two workers provide
#: overlap without multiplying cache churn or exhausting the pytest cgroup.
CORPUS_MAX_WORKERS = 2
#: One fixed width for the corpus and the runner's affected tier, sized to the
#: pytest pool's 12 GiB cgroup ceiling (eight workers peak near 10 GB) rather
#: than host cores or free RAM. Measured 2026-09-03 uncontended: 47 minutes for
#: 20,860 tests at eight workers; at two the same run takes about seven hours
#: and the required check cannot finish inside its slot timeout.
CORPUS_MAX_WORKERS = 8

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Remove the stale two-worker override from verify_all

When the scheduled operations.verify_all route runs, .agentctl/project.toml still exports POLYLOGUE_PYTEST_WORKERS=2 and gives the operation only 14,400 seconds. _pytest_worker_args(maximum=CORPUS_MAX_WORKERS) preserves that lower explicit value, so this new eight-worker corpus default never applies to the nightly full run; given the change's documented seven-hour runtime at two workers, the operation will time out after four hours instead of publishing complete-corpus evidence.

Useful? React with 👍 / 👎.

_AGENTCTL_OPERATION_ARGV = {"verify_affected": (), "verify_quick": ("--quick",), "verify_all": ("--all",)}
_PROJECT_DESCRIPTOR = ".agentctl/project.toml"
# These tests read the AgentCTL descriptor directly. They are the bounded
Expand Down Expand Up @@ -293,6 +296,24 @@ def _read_json(path: Path) -> dict[str, Any] | None:
MAX_RERUN_NODEIDS = 300


def _report_nodeid_to_selector(nodeid: str) -> str:
"""Strip xdist's ``@<group>`` suffix so a report node id selects again.

``--dist=loadgroup`` reports ``path::test[param]@group``; pytest cannot
collect that literal, so a rerun built from it errors before running.
A parametrization id may itself contain ``@``, so only a suffix after the
closing bracket (or after the bare test name) is removed.
"""
head, sep, tail = nodeid.rpartition("@")
if not sep or "::" not in head:
return nodeid
if "[" in tail or "]" in tail or "/" in tail or "::" in tail:
return nodeid
if head.endswith("]") or "[" not in head.rsplit("::", 1)[-1]:
return head
return nodeid


def _rerun_failed_once(command: Sequence[str], *, env: Mapping[str, str], artifacts: Any) -> dict[str, Any] | None:
"""Rerun exactly the failed tests once, alone and unselected.

Expand All @@ -306,7 +327,7 @@ def _rerun_failed_once(command: Sequence[str], *, env: Mapping[str, str], artifa
if not isinstance(report, Mapping):
return None
failed = [
str(test["nodeid"])
_report_nodeid_to_selector(str(test["nodeid"]))
for test in report.get("tests", [])
if isinstance(test, Mapping) and test.get("outcome") in {"failed", "error"} and test.get("nodeid")
Comment on lines 329 to 332

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve grouped IDs when patching accepted flakes

When an xdist_group test fails initially and passes alone, this stores only its stripped selector in failed/flaky, but lines 396 and 409 later compare that selector with the original report node ID ending in @group. Consequently the verifier may exit green while the canonical report leaves the test outcome and failed count unchanged (and also increments the passed count). Preserve the original-to-selector mapping or normalize those later comparisons as well.

Useful? React with 👍 / 👎.

]
Expand Down
8 changes: 4 additions & 4 deletions devtools/verify_testmon_selection.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

from devtools.pytest_invocation import MANAGED_PLUGIN_ARGS
from devtools.toolchain import venv_python
from devtools.verify import _pytest_worker_args
from devtools.verify import CORPUS_MAX_WORKERS, _pytest_worker_args


def main(_argv: list[str] | None = None) -> int:
Expand All @@ -27,8 +27,8 @@ def main(_argv: list[str] | None = None) -> int:
finally:
if configured_workers is not None:
os.environ["POLYLOGUE_PYTEST_WORKERS"] = configured_workers
if default_worker_args != ["--dist=loadgroup", "-n", "2"]:
print("testmon-selection: managed verification does not default to two workers")
if default_worker_args != ["--dist=loadgroup", "-n", str(CORPUS_MAX_WORKERS)]:
print(f"testmon-selection: managed verification does not default to {CORPUS_MAX_WORKERS} workers")
Comment on lines +30 to +31

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Keep the worker-width assertion independent

The anti-vacuity mutation named by this gate, setting CORPUS_MAX_WORKERS = 0, no longer makes it fail: _pytest_worker_args() and the expected value now both read the same constant, so they both produce -n 0 and execution continues. The gate therefore cannot detect the zero-worker default it explicitly claims to prevent; assert an independent nonzero or intended-width value here.

AGENTS.md reference: AGENTS.md:L192-L193

Useful? React with 👍 / 👎.

return 1
with tempfile.TemporaryDirectory(prefix="polylogue-testmon-gate-") as temporary:
root = Path(temporary)
Expand Down Expand Up @@ -93,7 +93,7 @@ def main(_argv: list[str] | None = None) -> int:
if not selected or not total or selected * 100 >= total * 5:
print(f"testmon-selection: selected {selected} of {total}, expected under 5%\n{output}")
return 1
print(f"testmon-selection: selected {selected} of {total}; workers=2")
print(f"testmon-selection: selected {selected} of {total}; workers={CORPUS_MAX_WORKERS}")
return 0


Expand Down
2 changes: 1 addition & 1 deletion tests/unit/devtools/test_pytest_slot.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ def test_outside_a_task_the_run_is_queued(tmp_path: Path, monkeypatch: pytest.Mo
assert launch["operation"] == "test"
assert launch["pool"] == "pytest"
assert launch["result_kind"] == "exit"
assert launch["timeout_seconds"] == 3600
assert launch["timeout_seconds"] == 7200
assert [call["argv"][0] for call in _calls(record)] == ["add", "wait", "status"]


Expand Down
11 changes: 11 additions & 0 deletions tests/unit/devtools/test_run_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -625,3 +625,14 @@ def test_absent_paths_are_resolved_against_the_checkout_not_the_caller_cwd(
]

assert run_tests.absent_selection_paths(selection, root=checkout) == ["tests/unit/test_deleted.py"]


def test_nested_managed_run_never_traces_into_the_checkout_datafile(monkeypatch: pytest.MonkeyPatch) -> None:
"""Anti-vacuity: tracing unconditionally lets a test that spawns
`devtools test` reset the outer corpus run's graph (2026-09-05: a full
corpus left a one-test datafile)."""
from devtools.agent_env import HARNESS_RUN_ENV
from devtools.run_tests import _testmon_args

assert "--testmon" in _testmon_args({})
assert tuple(_testmon_args({HARNESS_RUN_ENV: "run-1"})) == ("-p", "no:testmon")
12 changes: 12 additions & 0 deletions tests/unit/devtools/test_verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -914,3 +914,15 @@ def fake_run(command: list[str], **kwargs: Any) -> subprocess.CompletedProcess[s

assert exit_code == 0
assert recorded["capture_output"] is True


def test_rerun_selector_strips_the_xdist_group_suffix() -> None:
"""Anti-vacuity: passing the report node id through unchanged makes the
rerun error with "not found" for every grouped test, which is what wiped
the 2026-09-05 corpus rerun."""
from devtools.verify import _report_nodeid_to_selector

assert _report_nodeid_to_selector("tests/a.py::test_x@web-reader") == "tests/a.py::test_x"
assert _report_nodeid_to_selector("tests/a.py::T::test_x[p]@grp") == "tests/a.py::T::test_x[p]"
assert _report_nodeid_to_selector("tests/a.py::test_x[a@b]") == "tests/a.py::test_x[a@b]"
assert _report_nodeid_to_selector("tests/a.py::test_x") == "tests/a.py::test_x"