diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index ab7d111..91fb4c0 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -24,6 +24,26 @@ Concurrency is keyed by GitHub ref and superseded runs are cancelled. Million-event deep-offset pagination tests are marked `slow`; their small pagination contracts still run across the compatibility matrix. +`tests/test_ci_tiers.py` checks parsed commands, matrix rows, marker truth tables, +conditions, independent file floors and gate ordering, not step display names or +complete shell text. `tests/_ci_contract.py` intentionally supports only a bounded +CI language: simple failure-propagating commands, shell quoting/continuations, +sequential steps or a terminal `&&` chain, and explicit boolean conditions. +Unknown shell control flow, OR/pipes, failure suppression, selector overrides and +unrecognized conditions fail closed. Extend it with positive and negative +mutation tests when introducing new syntax; do not weaken it into substring +checks. Action version bumps, cache configuration, renaming, equivalent selectors +and step splits/merges do not change a gate. + +The Linux browser job installs Chromium and sets `PT_SNAP_REQUIRE_BROWSER=1`. +Browser cases are always collected even if Playwright is absent. Optional matrix +and platform runs report each missing-dependency case as skipped plus an explicit +**not executed** summary; that is not browser-pass evidence. Required mode fails +for missing package/binary, any browser skip, empty/deselected collection or +incomplete execution. `--collect-only` remains a collection diagnostic and is +explicitly reported as not run. Broken installations and launch failures fail in +both modes. Browser results are not added to non-browser platform requirements. + Coverage reporting writes `.coverage`/`coverage.xml`; building writes `dist/` and intermediates. Dependency installation/build isolation may access the network. The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses temporary @@ -93,7 +113,8 @@ validation. For non-PR provenance checks, tag runs compare against the prior tag Review `pyproject.toml`, both workflow YAMLs, the provenance/acceptance scripts and `tests/test_release_workflow.py` together when changing execution order. Validate YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py -tests/test_ci_tiers.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from +tests/test_ci_tiers.py tests/test_ci_contract_mutations.py +tests/test_browser_execution_contract.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from the repository root. Full CI tests the matrix and installed wheel; static tests do not prove a release environment is configured or a publication occurred. diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index 61d62c7..01fe2b1 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -56,7 +56,8 @@ jobs: run: python -m pytest tests/test_fixture_provenance.py - name: Run exhaustive non-slow partition # No coverage denominator or scale matrix here. Resource-warning gates - # remain enabled; required Chromium interaction runs in the Linux job. + # remain enabled; browser cases stay collected and explicitly report + # optional skips. Real Chromium execution is required in the Linux job. run: ${{ matrix.command }} - name: Upload platform test results if: always() diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6b7b2be..e65bbe7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -96,7 +96,8 @@ jobs: - name: Run required Chromium interaction tests env: PT_SNAP_REQUIRE_BROWSER: "1" - run: pytest tests/test_fixture_provenance.py tests/test_memory_tree_browser.py + # Required mode rejects missing dependencies, skips and incomplete execution. + run: pytest -rs tests/test_fixture_provenance.py tests/test_memory_tree_browser.py test: name: Test (Python ${{ matrix.python-version }}) diff --git a/tests/_browser_support.py b/tests/_browser_support.py new file mode 100644 index 0000000..3ff3063 --- /dev/null +++ b/tests/_browser_support.py @@ -0,0 +1,129 @@ +"""Optional-browser fixtures and visible per-test execution accounting. + +Import Playwright only during fixture setup: absent optional dependencies must +never remove browser regression identities from pytest's collection. +""" + +from __future__ import annotations + +import importlib +import os +from pathlib import Path + +import pytest + + +def _unavailable(reason): + message = f"Browser regressions NOT EXECUTED: {reason}" + if os.environ.get("PT_SNAP_REQUIRE_BROWSER") == "1": + pytest.fail(message, pytrace=False) + pytest.skip(message) + + +@pytest.fixture(scope="session") +def playwright_api(): + try: + return importlib.import_module("playwright.sync_api") + except ModuleNotFoundError as exc: + # A broken installation (for example missing greenlet) is not optional. + if exc.name not in {"playwright", "playwright.sync_api"}: + raise + _unavailable("Playwright is not installed; install the .[dev] extra") + + +@pytest.fixture(scope="module") +def chromium_browser(playwright_api): + with playwright_api.sync_playwright() as runtime: + executable = os.environ.get("PT_SNAP_TEST_CHROMIUM") + binary = executable or runtime.chromium.executable_path + if not Path(binary).is_file(): + _unavailable( + "Chromium executable is missing; run python -m playwright install chromium " + "or set PT_SNAP_TEST_CHROMIUM to an existing executable" + ) + options = {"headless": True} + if executable: + options["executable_path"] = executable + try: + browser = runtime.chromium.launch(**options) + except playwright_api.Error as exc: + # Only the explicit preflight above can skip. Crashes, permission + # errors, bad shared libraries and other launch failures always fail. + pytest.fail(f"Cannot launch Chromium for frame navigation tests: {exc}") + try: + yield browser + finally: + browser.close() + + +def browser_expect(locator): + """Assertions run only after browser setup has succeeded.""" + return importlib.import_module("playwright.sync_api").expect(locator) + + +class BrowserSummary: + def __init__(self): + self.nodes = set() + self.outcomes = {} + self.skip_reasons = set() + + @pytest.hookimpl(hookwrapper=True, tryfirst=True) + def pytest_collection_modifyitems(self, items): + self.nodes = {item.nodeid for item in items if item.get_closest_marker("browser")} + yield + + def pytest_runtest_logreport(self, report): + if report.nodeid not in self.nodes: + return + if report.failed: + self.outcomes[report.nodeid] = "failed" + elif self.outcomes.get(report.nodeid) != "failed": + if report.skipped: + self.outcomes[report.nodeid] = "skipped" + reason = ( + report.longrepr[2] if isinstance(report.longrepr, tuple) else report.longrepr + ) + self.skip_reasons.add(str(reason)) + elif report.when == "call": + self.outcomes[report.nodeid] = "passed" + + def pytest_sessionfinish(self, session): + if os.environ.get("PT_SNAP_REQUIRE_BROWSER") != "1" or session.config.option.collectonly: + return + incomplete = not self.nodes or len(self.outcomes) < len(self.nodes) + if (incomplete or "skipped" in self.outcomes.values()) and session.exitstatus in { + pytest.ExitCode.OK, + pytest.ExitCode.NO_TESTS_COLLECTED, + }: + session.exitstatus = pytest.ExitCode.TESTS_FAILED + + def pytest_terminal_summary(self, terminalreporter): + if not self.nodes and os.environ.get("PT_SNAP_REQUIRE_BROWSER") != "1": + return + counts = { + outcome: list(self.outcomes.values()).count(outcome) + for outcome in ("passed", "skipped", "failed") + } + mode = "required" if os.environ.get("PT_SNAP_REQUIRE_BROWSER") == "1" else "optional" + terminalreporter.section(f"Browser regression execution ({mode})") + terminalreporter.write_line( + f"collected={len(self.nodes)} passed={counts['passed']} skipped={counts['skipped']} " + f"failed={counts['failed']} not_run={len(self.nodes) - len(self.outcomes)}" + ) + if not self.nodes or len(self.outcomes) < len(self.nodes): + terminalreporter.write_line( + "Browser regressions NOT EXECUTED: collection/deselection or early termination is not browser execution." + ) + if counts["skipped"]: + terminalreporter.write_line( + "Browser regressions NOT EXECUTED for skipped cases; this is not a browser pass." + ) + for reason in sorted(self.skip_reasons): + terminalreporter.write_line(reason) + + +def pytest_configure(config): + config.addinivalue_line( + "markers", "browser: real Chromium interaction regressions (required in Linux browser CI)" + ) + config.pluginmanager.register(BrowserSummary(), "browser-execution-summary") diff --git a/tests/_ci_contract.py b/tests/_ci_contract.py new file mode 100644 index 0000000..00db830 --- /dev/null +++ b/tests/_ci_contract.py @@ -0,0 +1,408 @@ +"""Small, fail-closed semantic reader for this repository's CI gate language. + +This is deliberately not a general shell or GitHub Actions interpreter. Harmless +formatting/quoting, action versions, boolean rearrangements and sequential step +splits/merges are supported. New shell control flow or expression features need +an explicit contract extension and negative tests, not substring acceptance. +""" + +from __future__ import annotations + +import ast +import itertools +import re +import shlex +from dataclasses import dataclass + +EVENTS = ("pull_request", "push", "workflow_call", "schedule", "workflow_dispatch") +PYTHONS = ("3.10", "3.11", "3.12", "3.13") + + +def require(condition, message): + if not condition: + raise AssertionError(message) + + +def expression(source, values, status="success"): + """Evaluate the supported GHA grammar, including its quoting and precedence. + + GitHub string equality ignores case; quotes escape by doubling apostrophes, + not by Python backslash escapes. Mixed-type loose coercion is unsupported. + Reference: https://docs.github.com/en/actions/reference/workflows-and-actions/expressions + """ + if source is None: + return status == "success" + source = str(source).strip() + if source.startswith("${{") and source.endswith("}}"): + source = source[3:-2].strip() + pattern = re.compile(r"\s*(?:('(?:[^']|'')*')|([\w.-]+)|(&&|\|\||==|!=|!|\(|\)))") + position = 0 + tokens = [] + uses_status = False + while position < len(source): + match = pattern.match(source, position) + require(match is not None, f"Unsupported CI condition: {source}") + literal, name, operator = match.groups() + position = match.end() + if literal is not None: + value = literal[1:-1].replace("''", "'") + require(value.isascii(), "Non-ASCII CI comparison literals need explicit support") + tokens.append(("value", value)) + elif name is not None: + if name in {"always", "success", "failure", "cancelled"}: + call = re.match(r"\s*\(\s*\)", source[position:]) + require(call is not None, "Status predicate must be called") + position += call.end() + uses_status = True + tokens.append(("value", name == "always" or name == status)) + elif name in {"true", "false"}: + tokens.append(("value", name == "true")) + else: + require(name in values, f"Unsupported CI context: {name}") + tokens.append(("value", values[name])) + else: + tokens.append((operator, None)) + index = 0 + + def take(kind): + nonlocal index + if index < len(tokens) and tokens[index][0] == kind: + index += 1 + return True + return False + + def unary(): + nonlocal index + if take("!"): + return not unary() + if take("("): + value = disjunction() + require(take(")"), f"Unclosed CI condition: {source}") + return value + require(index < len(tokens) and tokens[index][0] == "value", f"Expected CI value: {source}") + value = tokens[index][1] + index += 1 + return value + + def comparison(): + left = unary() + operator = tokens[index][0] if index < len(tokens) else None + if operator in {"==", "!="}: + take(operator) + right = unary() + require(type(left) is type(right), "Mixed-type CI coercion is unsupported") + if isinstance(left, str): + require( + left.isascii() and right.isascii(), + "Non-ASCII CI comparison needs explicit support", + ) + left, right = left.lower(), right.lower() + else: + require(isinstance(left, bool), "Only string/bool CI comparisons are supported") + return (left == right) if operator == "==" else (left != right) + return left + + def conjunction(): + value = comparison() + while take("&&"): + right = comparison() + value = right if value else value + return value + + def disjunction(): + value = conjunction() + while take("||"): + right = conjunction() + value = value if value else right + return value + + value = disjunction() + require(index == len(tokens), f"Unsupported CI condition: {source}") + return bool(value) and (uses_status or status == "success") + + +def matrix_rows(matrix): + axes = {key: value for key, value in matrix.items() if key not in {"include", "exclude"}} + require( + all(isinstance(value, list) and value for value in axes.values()), + "Matrix axes must be explicit nonempty lists", + ) + originals = ( + [dict(zip(axes, values, strict=True)) for values in itertools.product(*axes.values())] + if axes + else [] + ) + # GitHub applies excludes before includes, which may add excluded rows back. + originals = [ + row + for row in originals + if not any( + all(row.get(key) == value for key, value in exclusion.items()) + for exclusion in matrix.get("exclude", []) + ) + ] + rows = [dict(row) for row in originals] + for addition in matrix.get("include", []): + matched = False + for index, original in enumerate(originals): + row = rows[index] + if all( + key not in original or original[key] == value for key, value in addition.items() + ): + row.update(addition) + matched = True + if not matched: + rows.append(dict(addition)) + # Includes do not form new Cartesian base combinations. + return rows + + +WHEEL_PROGRAM = """from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])""" + + +def _canonical_program(program): + try: + return ast.dump(ast.parse(program)) + except SyntaxError as exc: + raise AssertionError("Invalid wheel-selection program") from exc + + +def command_tokens(script): + """Read failure-propagating simple commands, never search inside run text.""" + script = script.replace("\\\n", " ") + + def wheel(match): + name, invocation = match.groups() + words = shlex.split(invocation) + require( + len(words) == 3 and words[0] in {"python", "python3"} and words[1] == "-c", + "Unsupported wheel selection", + ) + require( + _canonical_program(words[2]) == _canonical_program(WHEEL_PROGRAM), + "Wheel selection must require exactly one built wheel", + ) + return f"__verified_wheel__ {name}" + + script = re.sub(r"(?m)^\s*([A-Z_][A-Z_0-9]*)=\$\((.+)\)\s*$", wheel, script) + # The interpreter path probe is one audited, side-effect-free substitution. + script = script.replace( + "\"$(python -c 'import sys; print(sys.executable)')\"", '"__selected_python__"' + ) + tokens = [] + for line in script.splitlines(): + lexer = shlex.shlex(line, posix=True, punctuation_chars=";&|") + lexer.whitespace_split = True + lexer.commenters = "#" + tokens.extend(lexer) + tokens.append("\n") + commands, current, separators = [], [], [] + for token in tokens + ["\n"]: + if token and all(char in ";&|\n" for char in token): + require( + "|" not in token and token.replace("\n", "") in {"", ";", "&&"}, + "Shell OR/pipes/background/control flow can hide a failed gate", + ) + if current: + commands.append(current) + current = [] + separators.append(token.replace("\n", "") or ";") + else: + current.append(token) + # `a && b; c` suppresses errexit for a. Only a single terminal AND chain is safe. + if "&&" in separators: + require( + all(separator == "&&" for separator in separators[:-1]), + "Mixed AND/sequential shell lists can ignore failures", + ) + result = [] + for words in commands: + if words[0] == "set": + require( + words[1:] in [["-euo", "pipefail"], ["-eu"], ["-e"]], + "Shell failure handling must not be disabled", + ) + continue + if words[0] == "__verified_wheel__": + result.append(tuple(words)) + continue + require( + not any( + any(char in word for char in ("`", ">", "<")) or "$(" in word for word in words + ), + "Unsupported shell substitution/redirection", + ) + if words[0] in {"python", "python3", "python3.13"}: + words = words[2:] if len(words) > 2 and words[1] == "-m" else words[1:] + require( + words + and words[0] + in { + "pytest", + "pip", + "ruff", + "black", + "basedpyright", + "coverage", + "build", + "playwright", + ".github/scripts/audit_wheel.py", + ".github/scripts/clean_wheel_agent_acceptance.py", + ".github/scripts/run_dataset_acceptance.py", + ".github/scripts/run_differential_acceptance.py", + ".github/scripts/check_snapshot_provenance.py", + ".github/scripts/check_critical_coverage.py", + }, + f"Unsupported gate command: {words}", + ) + result.append(tuple(words)) + return result + + +def options(words, valued=(), flags=()): + result, positional = {}, [] + index = 0 + while index < len(words): + word = words[index] + if word.startswith("-"): + key, separator, value = word.partition("=") + require(key in valued or key in flags, f"Unsupported gate option: {word}") + if key in valued and not separator: + index += 1 + require(index < len(words), f"Missing value: {key}") + value = words[index] + if key in flags: + require(not separator, f"Flag does not take a value: {key}") + value = True + result.setdefault(key, []).append(value) + else: + positional.append(word) + index += 1 + return result, positional + + +@dataclass +class Command: + words: tuple[str, ...] + step: dict + index: int + env: dict + + +def commands(job, workflow=None): + require(job.get("continue-on-error", "false") == "false", "Jobs must propagate gate failures") + require(not job.get("needs"), "Independent gates cannot depend on skippable jobs") + defaults = (workflow or {}).get("defaults", {}).get("run", {}) | job.get("defaults", {}).get( + "run", {} + ) + require(not defaults.get("working-directory"), "Gate working directory must be repository root") + result = [] + for step in job["steps"]: + require( + step.get("continue-on-error", "false") == "false", "Steps must propagate gate failures" + ) + if "run" not in step: + continue + require( + step.get("shell", defaults.get("shell", "bash")) == "bash", + "Gate shell must retain bash errexit", + ) + require(not step.get("working-directory"), "Gate working directory must be repository root") + env = (workflow or {}).get("env", {}) | job.get("env", {}) | step.get("env", {}) + require( + not any( + key.startswith(("PYTEST_", "COVERAGE_")) + or key + in { + "BASH_ENV", + "ENV", + "PATH", + "PYTHONPATH", + "PYTHONHOME", + "VIRTUAL_ENV", + "PYTHONOPTIMIZE", + "PYTHONWARNINGS", + } + for key in env + ), + "Environment must not override gate execution", + ) + for words in command_tokens(step["run"]): + result.append(Command(words, step, len(result), env)) + return result + + +def select(commands_, program): + return [command for command in commands_ if command.words[0] == program] + + +def one(commands_, program): + matching = select(commands_, program) + require(len(matching) == 1, f"Expected exactly one {program} gate, got {len(matching)}") + return matching[0] + + +def condition_matches( + job, step, *, versions=PYTHONS, events=EVENTS, predicate, statuses=("success",) +): + for version, event, status in itertools.product(versions, events, statuses): + values = {"matrix.python-version": version, "github.event_name": event} + actual = expression(job.get("if"), values, status) and expression( + step.get("if"), values, status + ) + # Job-level default success does not suppress always() artifact steps in + # an already-running job after a step failure. + if status != "success": + actual = expression(job.get("if"), values, "success") and expression( + step.get("if"), values, status + ) + require( + actual == predicate(version, event, status), + f"Wrong gate condition for Python {version}, {event}, {status}: {step.get('if')}", + ) + + +def marker_truth(source, slow, nightly): + tree = ast.parse(source, mode="eval") + + def evaluate(node): + if isinstance(node, ast.Name): + require(node.id in {"slow", "nightly"}, f"Unexpected marker selector: {node.id}") + return {"slow": slow, "nightly": nightly}[node.id] + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not evaluate(node.operand) + if isinstance(node, ast.BoolOp): + values = [evaluate(value) for value in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + raise AssertionError(f"Unsupported pytest marker selector: {source}") + + return evaluate(tree.body) + + +def pytest_options(command, expected_marker=None, paths=(), coverage=False): + opts, positional = options( + command.words[1:], + valued=("-m", "--cov-report"), + flags=("--cov", "-q", "-v", "-vv", "-rs", "-ra"), + ) + require( + set(positional) == set(paths) and len(positional) == len(paths), + "Pytest paths must cover the complete intended gate", + ) + require( + opts.get("--cov", []) == ([True] if coverage else []), + "Coverage must run only on the full tier", + ) + if expected_marker is None: + require("-m" not in opts, "Required test paths cannot be marker-filtered") + else: + require(len(opts.get("-m", [])) == 1, "Exactly one pytest marker expression is required") + for slow, nightly in itertools.product((False, True), repeat=2): + require( + marker_truth(opts["-m"][0], slow, nightly) == expected_marker(slow, nightly), + "Pytest marker selector changes tier membership", + ) + require( + set(opts.get("--cov-report", [])) == ({"xml", "term-missing"} if coverage else set()), + "Coverage outputs changed", + ) diff --git a/tests/conftest.py b/tests/conftest.py index 587bc27..11cd944 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,6 +7,8 @@ from tests._fixture_provenance import FixtureProvenanceError, verify_snapshot_fixtures +pytest_plugins = ("tests._browser_support",) + def pytest_sessionstart(session: pytest.Session) -> None: """Stop before collection can execute a changed or unreviewed pickle.""" diff --git a/tests/test_browser_execution_contract.py b/tests/test_browser_execution_contract.py new file mode 100644 index 0000000..c9c3cde --- /dev/null +++ b/tests/test_browser_execution_contract.py @@ -0,0 +1,169 @@ +"""Collection and exit-code contracts without requiring a downloaded browser.""" + +from __future__ import annotations + +import ast +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +BROWSER_TEST = ROOT / "tests/test_memory_tree_browser.py" + + +def _browser_nodes(): + return { + node.name + for node in ast.parse(BROWSER_TEST.read_text(encoding="utf-8")).body + if isinstance(node, ast.FunctionDef) and node.name.startswith("test_") + } + + +def _run_unavailable(tmp_path, mode, *, required=False, collect_only=False, extra_args=()): + # Copy the actual regression module, not a stand-in with fewer tests. No + # fixture bytes, browser downloads, or production behavior are exercised. + (tmp_path / "test_browser_copy.py").write_text( + BROWSER_TEST.read_text(encoding="utf-8"), encoding="utf-8" + ) + (tmp_path / "pytest.ini").write_text( + "[pytest]\nmarkers = browser: browser regression\n", encoding="utf-8" + ) + (tmp_path / "conftest.py").write_text( + """ +import importlib +import importlib.abc +import sys +from contextlib import nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytest_plugins = ("tests._browser_support",) + +original = importlib.import_module + +def pytest_configure(config): + binary = Path(__file__).parent / "fake-chromium" + if MODE == "broken_browser": + binary.touch() + class BrowserError(Exception): + pass + def launch(**kwargs): + raise BrowserError("synthetic browser launch failure") + runtime = SimpleNamespace(chromium=SimpleNamespace(executable_path=str(binary), launch=launch)) + api = SimpleNamespace(sync_playwright=lambda: nullcontext(runtime), Error=BrowserError) + def load(name, *args, **kwargs): + if name != "playwright.sync_api": + return original(name, *args, **kwargs) + if MODE in {"missing_package", "broken_package"}: + dependency = "playwright" if MODE == "missing_package" else "greenlet" + raise ModuleNotFoundError("No module named " + dependency, name=dependency) + return api + importlib.import_module = load + if MODE == "missing_package": + class MissingPlaywright(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "playwright" or fullname.startswith("playwright."): + raise ModuleNotFoundError("No module named playwright", name="playwright") + sys.meta_path.insert(0, MissingPlaywright()) + +def pytest_unconfigure(config): + importlib.import_module = original +""".replace("MODE", repr(mode)), + encoding="utf-8", + ) + env = os.environ.copy() + for key in ( + "PT_SNAP_REQUIRE_BROWSER", + "PT_SNAP_TEST_CHROMIUM", + "PYTEST_ADDOPTS", + "PYTEST_PLUGINS", + ): + env.pop(key, None) + env["PYTHONPATH"] = os.pathsep.join((str(ROOT), str(ROOT / "src"))) + env["PYTEST_DISABLE_PLUGIN_AUTOLOAD"] = "1" + if required: + env["PT_SNAP_REQUIRE_BROWSER"] = "1" + args = [sys.executable, "-m", "pytest", "test_browser_copy.py", "-v", "-rs", "--tb=short"] + args.extend(extra_args) + if collect_only: + args.append("--collect-only") + return subprocess.run(args, cwd=tmp_path, env=env, capture_output=True, text=True, timeout=60) + + +@pytest.mark.parametrize("required", [False, True], ids=["optional", "required"]) +def test_browser_collection_keeps_every_real_regression_without_playwright(tmp_path, required): + result = _run_unavailable(tmp_path, "missing_package", required=required, collect_only=True) + nodes = _browser_nodes() + assert len(nodes) >= 12, "Existing browser regressions must not disappear" + assert result.returncode == 0, result.stdout + result.stderr + for node in nodes: + assert node in result.stdout + assert ( + f"collected={len(nodes)} passed=0 skipped=0 failed=0 not_run={len(nodes)}" in result.stdout + ) + + +@pytest.mark.parametrize("mode", ["missing_package", "missing_binary"]) +@pytest.mark.parametrize("required", [False, True], ids=["optional", "required"]) +def test_missing_browser_dependencies_are_visible_and_required_mode_fails(tmp_path, mode, required): + result = _run_unavailable(tmp_path, mode, required=required) + count = len(_browser_nodes()) + assert result.returncode == (1 if required else 0), result.stdout + result.stderr + for node in _browser_nodes(): + assert node in result.stdout + if required: + assert f"collected={count} passed=0 skipped=0 failed={count} not_run=0" in result.stdout + else: + assert f"collected={count} passed=0 skipped={count} failed=0 not_run=0" in result.stdout + assert "Browser regressions NOT EXECUTED" in result.stdout + assert ( + "Playwright is not installed" + if mode == "missing_package" + else "Chromium executable is missing" + ) in result.stdout + + +@pytest.mark.parametrize("mode", ["broken_package", "broken_browser"]) +def test_broken_browser_installations_never_become_optional_skips(tmp_path, mode): + result = _run_unavailable(tmp_path, mode) + count = len(_browser_nodes()) + assert result.returncode == 1, result.stdout + result.stderr + assert f"collected={count} passed=0 skipped=0 failed={count} not_run=0" in result.stdout + assert ( + "greenlet" if mode == "broken_package" else "synthetic browser launch failure" + ) in result.stdout + + +def test_required_mode_rejects_deselected_browser_execution(tmp_path): + result = _run_unavailable( + tmp_path, + "missing_package", + required=True, + extra_args=("-k", "nothing_matches_this_selector"), + ) + count = len(_browser_nodes()) + assert result.returncode == 1, result.stdout + result.stderr + assert f"collected={count} passed=0 skipped=0 failed=0 not_run={count}" in result.stdout + + +def test_required_summary_fails_for_skips_and_incomplete_execution(monkeypatch): + from types import SimpleNamespace + + from tests._browser_support import BrowserSummary + + monkeypatch.setenv("PT_SNAP_REQUIRE_BROWSER", "1") + for nodes, outcomes in ((set(), {}), ({"one"}, {}), ({"one"}, {"one": "skipped"})): + summary = BrowserSummary() + summary.nodes = nodes + summary.outcomes = outcomes + session = SimpleNamespace( + config=SimpleNamespace(option=SimpleNamespace(collectonly=False)), + exitstatus=pytest.ExitCode.OK, + ) + summary.pytest_sessionfinish(session) + assert session.exitstatus == pytest.ExitCode.TESTS_FAILED diff --git a/tests/test_ci_contract_mutations.py b/tests/test_ci_contract_mutations.py new file mode 100644 index 0000000..a579e2a --- /dev/null +++ b/tests/test_ci_contract_mutations.py @@ -0,0 +1,414 @@ +"""Acceptance and adversarial mutations for the bounded CI semantic reader.""" + +from copy import deepcopy + +import pytest + +from tests._ci_contract import command_tokens, commands, expression, matrix_rows +from tests.test_ci_tiers import project, validate_workflow, workflow + + +def _step(config, job, program, contains=None): + matching = [ + command.step + for command in commands(config["jobs"][job], config) + if command.words[0] == program and (contains is None or contains in command.words) + ] + assert matching + return matching[0] + + +def _rewrite(config, job, program, before, after, contains=None): + step = _step(config, job, program, contains) + assert before in step["run"] + step["run"] = step["run"].replace(before, after) + + +def _renamed(config): + for job in config["jobs"].values(): + job["name"] = "A display label, not a contract" + for step in job["steps"]: + step.pop("name", None) + + +def _actions_and_cache(config): + for job in config["jobs"].values(): + for step in job["steps"]: + if "uses" in step: + step["uses"] = step["uses"].split("@")[0] + "@v99" + if step.get("uses", "").startswith("actions/setup-python@"): + step.setdefault("with", {})["cache"] = "pip" + + +def _quotes_and_commands(config): + _rewrite( + config, + "test", + "pytest", + 'pytest -m "not slow and not nightly"', + "python -m pytest -m 'not (nightly or slow)'", + "not slow and not nightly", + ) + _rewrite( + config, + "test", + "pytest", + 'pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing', + "python3 -m pytest --cov-report term-missing --cov -m 'not nightly' --cov-report xml", + "--cov", + ) + guard = _step(config, "test", "coverage") + guard["run"] = ( + guard["run"] + .replace("--include='", '--include "') + .replace(".py'", '.py"') + .replace("--fail-under=", "--fail-under ") + ) + + +def _conditions(config): + _step(config, "test", "pytest", "--cov")["if"] = "${{ !(matrix.python-version != '3.13') }}" + _step(config, "test", "pytest", "not slow and not nightly")[ + "if" + ] = "('3.13' != matrix.python-version) && success()" + config["jobs"]["dataset-acceptance"][ + "if" + ] = "${{ github.event_name == 'workflow_dispatch' || ('schedule' == github.event_name) }}" + + +def _uppercase_events(config): + config["jobs"]["dataset-acceptance"][ + "if" + ] = "github.event_name == 'SCHEDULE' || github.event_name == 'WORKFLOW_DISPATCH'" + + +def _excluded_then_restored_matrix(config): + matrix = config["jobs"]["test"]["strategy"]["matrix"] + matrix["exclude"] = [{"python-version": "3.13"}] + matrix["include"] = [{"python-version": "3.13"}] + + +def _split_and_merge(config): + job = config["jobs"]["test"] + guard = _step(config, "test", "coverage") + index = job["steps"].index(guard) + parts = [dict(guard, run=line) for line in guard["run"].splitlines() if line.strip()] + job["steps"][index : index + 1] = parts + # Merge full pytest with the first independent floor; labels are irrelevant. + full = _step(config, "test", "pytest", "--cov") + full["run"] += "\n" + parts[0]["run"] + job["steps"].remove(parts[0]) + + +def _and_chain(config): + guard = _step(config, "test", "coverage") + guard["run"] = " && ".join(guard["run"].splitlines()) + + +def _include_matrix(config): + matrix = config["jobs"]["test"]["strategy"]["matrix"] + versions = matrix.pop("python-version") + matrix["include"] = [{"python-version": version} for version in versions] + + +@pytest.mark.parametrize( + "mutation", + [ + _renamed, + _actions_and_cache, + _quotes_and_commands, + _conditions, + _uppercase_events, + _excluded_then_restored_matrix, + _split_and_merge, + _and_chain, + _include_matrix, + ], + ids=lambda mutation: mutation.__name__.removeprefix("_"), +) +def test_semantically_equivalent_workflow_refactors_are_accepted(mutation): + config = workflow() + mutation(config) + validate_workflow(config, project()) + + +def _weaken(config, case): + job = config["jobs"]["test"] + full = _step(config, "test", "pytest", "--cov") + fast = _step(config, "test", "pytest", "not slow and not nightly") + guard = _step(config, "test", "coverage") + browser = _step(config, "browser", "pytest") + if case == "missing-python": + job["strategy"]["matrix"]["python-version"].remove("3.12") + elif case == "excluded-python": + job["strategy"]["matrix"]["exclude"] = [{"python-version": "3.12"}] + elif case == "extra-python": + job["strategy"]["matrix"]["include"] = [{"python-version": "3.14"}] + elif case == "wrong-matrix-interpreter": + next( + step for step in job["steps"] if step.get("uses", "").startswith("actions/setup-python") + )["with"]["python-version"] = "3.13" + elif case == "filtered-pr-trigger": + config["on"]["pull_request"]["paths"] = ["never-tests/**"] + elif case == "wrong-full-condition": + full["if"] = "matrix.python-version == '3.12'" + elif case == "uppercase-pr-exclusion": + full["if"] += " && github.event_name != 'PULL_REQUEST'" + elif case == "python-escape-condition": + full["if"] += r" && '\x61' == 'a'" + elif case == "wrong-unary-precedence": + fast["if"] = "!matrix.python-version == '3.13'" + elif case == "mixed-type-coercion": + full["if"] += " && (true && github.event_name) == true" + elif case == "wrong-provenance-base": + step = next( + step + for step in config["jobs"]["snapshot-provenance"]["steps"] + if "PR_BODY" in step.get("env", {}) + ) + step["env"]["BASE_SHA"] = "${{ github.event.pull_request.head.sha }}" + elif case == "full-or-bypass": + full["if"] += " || true" + elif case == "skip-full-job": + job["if"] = "github.event_name != 'pull_request'" + elif case == "skippable-dependency": + job["needs"] = "dataset-acceptance" + elif case == "ignored-job-failure": + job["continue-on-error"] = "true" + elif case == "ignored-step-failure": + full["continue-on-error"] = "true" + elif case == "disabled-errexit": + full["run"] = "set +e\n" + full["run"] + elif case == "custom-shell": + full["shell"] = "bash {0}" + elif case == "default-shell": + job["defaults"] = {"run": {"shell": "bash {0}"}} + elif case == "wrong-directory": + full["working-directory"] = "tests/core" + elif case == "hidden-pytest-selector": + full["env"] = {"PYTEST_ADDOPTS": "-k passing_only"} + elif case == "optimized-away-wheel-assert": + job["env"] = {"PYTHONOPTIMIZE": "1"} + elif case == "suppressed-resource-warnings": + full["env"] = {"PYTHONWARNINGS": "ignore"} + elif case == "redirected-pythonpath": + job["env"] = {"PYTHONPATH": "other-source"} + elif case == "redirected-path": + full["env"] = {"PATH": "fake-gate-binaries"} + elif case == "shell-or-true": + full["run"] += " || true" + elif case == "shell-pipe": + full["run"] += " | tee ignored.log" + elif case == "and-list-swallow": + full["run"] += " && pip --version\npip --version" + elif case == "wrong-fast-selector": + fast["run"] = fast["run"].replace("not slow and not nightly", "not slow or not nightly") + elif case == "full-excludes-slow": + full["run"] = full["run"].replace("not nightly", "not slow and not nightly") + elif case == "full-includes-nightly": + full["run"] = full["run"].replace("not nightly", "not nightly or slow") + elif case == "pytest-k-filter": + full["run"] += " -k passing_only" + elif case == "pytest-path-filter": + full["run"] += " tests/core" + elif case == "coverage-threshold": + guard["run"] = guard["run"].replace("--fail-under=90", "--fail-under=89") + elif case == "coverage-combined-selector": + guard["run"] = guard["run"].replace("*/core/snapshot_import_backend.py", "*/core/*.py") + elif case == "coverage-wrong-file": + guard["run"] = guard["run"].replace( + "*/core/snapshot_import_backend.py", "*/core/import_metadata.py" + ) + elif case == "coverage-before-measurement": + job["steps"].remove(guard) + job["steps"].insert(job["steps"].index(full), guard) + elif case == "fixture-after-tests": + fixture = _step(config, "test", "pytest", "tests/test_fixture_provenance.py") + job["steps"].remove(fixture) + job["steps"].append(fixture) + elif case == "optional-browser": + browser["env"]["PT_SNAP_REQUIRE_BROWSER"] = "0" + elif case == "filtered-browser": + browser["run"] += " -k zero_total" + elif case == "missing-browser-install": + install = _step(config, "browser", "playwright") + install["run"] = "pip install -e '.[dev]'" + elif case == "non-linux-browser": + config["jobs"]["browser"]["runs-on"] = "macos-latest" + elif case == "conditional-browser": + browser["if"] = "github.event_name == 'schedule'" + elif case == "nightly-runs-on-pr": + config["jobs"]["dataset-acceptance"].pop("if") + elif case == "nightly-not-manual": + config["jobs"]["dataset-acceptance"]["if"] = "github.event_name == 'schedule'" + elif case == "ambiguous-wheel": + audit = _step(config, "test", ".github/scripts/audit_wheel.py") + audit["run"] = audit["run"].replace( + "assert len(wheels) == 1, wheels", "assert len(wheels) >= 1, wheels" + ) + elif case == "cross-step-wheel-substitution": + acceptance = _step(config, "test", ".github/scripts/clean_wheel_agent_acceptance.py") + acceptance["run"] = "\n".join( + line for line in acceptance["run"].splitlines() if not line.startswith("WHEEL=") + ) + acceptance["env"] = {"WHEEL": "unrelated.whl"} + elif case == "wrong-wheel": + acceptance = _step(config, "test", ".github/scripts/clean_wheel_agent_acceptance.py") + acceptance["run"] = acceptance["run"].replace( + '--wheel "$WHEEL"', '--wheel "dist/unrelated.whl"' + ) + elif case == "missing-sharded-acceptance": + acceptance = _step(config, "test", ".github/scripts/clean_wheel_agent_acceptance.py") + acceptance["run"] = acceptance["run"].replace( + "--events-per-slice 4096", "--events-per-slice 8192" + ) + elif case == "missing-differential-seeds": + acceptance = _step( + config, "dataset-acceptance", ".github/scripts/run_differential_acceptance.py" + ) + acceptance["run"] = acceptance["run"].replace("--seed-count 200", "--seed-count 20") + elif case == "hidden-failed-evidence": + for step in config["jobs"]["dataset-acceptance"]["steps"]: + if step.get("uses", "").startswith("actions/upload-artifact"): + step.pop("if") + elif case == "checkout-main-not-pr": + job["steps"][0]["with"]["ref"] = "main" + else: + raise AssertionError(case) + + +@pytest.mark.parametrize( + "case", + [ + "missing-python", + "excluded-python", + "extra-python", + "wrong-matrix-interpreter", + "filtered-pr-trigger", + "wrong-full-condition", + "full-or-bypass", + "uppercase-pr-exclusion", + "python-escape-condition", + "wrong-unary-precedence", + "mixed-type-coercion", + "wrong-provenance-base", + "skip-full-job", + "skippable-dependency", + "ignored-job-failure", + "ignored-step-failure", + "disabled-errexit", + "custom-shell", + "default-shell", + "wrong-directory", + "hidden-pytest-selector", + "redirected-pythonpath", + "optimized-away-wheel-assert", + "suppressed-resource-warnings", + "redirected-path", + "shell-or-true", + "shell-pipe", + "and-list-swallow", + "wrong-fast-selector", + "full-excludes-slow", + "full-includes-nightly", + "pytest-k-filter", + "pytest-path-filter", + "coverage-threshold", + "coverage-combined-selector", + "coverage-wrong-file", + "coverage-before-measurement", + "fixture-after-tests", + "optional-browser", + "filtered-browser", + "missing-browser-install", + "non-linux-browser", + "conditional-browser", + "nightly-runs-on-pr", + "nightly-not-manual", + "ambiguous-wheel", + "wrong-wheel", + "cross-step-wheel-substitution", + "missing-sharded-acceptance", + "missing-differential-seeds", + "hidden-failed-evidence", + "checkout-main-not-pr", + ], +) +def test_workflow_gate_regressions_are_rejected(case): + config = workflow() + _weaken(config, case) + with pytest.raises(AssertionError): + validate_workflow(config, project()) + + +@pytest.mark.parametrize("field,value", [("branch", False), ("fail_under", 89)]) +def test_coverage_configuration_cannot_weaken(field, value): + settings = deepcopy(project()) + settings["tool"]["coverage"]["run" if field == "branch" else "report"][field] = value + with pytest.raises(AssertionError): + validate_workflow(workflow(), settings) + + +def test_include_matrix_rows_never_become_new_cartesian_originals(): + assert matrix_rows( + { + "python-version": ["3.10"], + "include": [{"python-version": "3.13"}, {"python-version": "3.13", "extra": "second"}], + } + ) == [ + {"python-version": "3.10"}, + {"python-version": "3.13"}, + {"python-version": "3.13", "extra": "second"}, + ] + assert matrix_rows( + { + "python-version": ["3.10", "3.13"], + "include": [{"cache": "pip"}], + "exclude": [{"python-version": "3.10"}], + } + ) == [{"python-version": "3.13", "cache": "pip"}] + + +@pytest.mark.parametrize( + "source", + [ + "contains(github.event_name, 'push')", + "unknown.context == 'yes'", + "success || true", + "github.event_name =~ 'push'", + ], +) +def test_unknown_condition_syntax_is_not_guessed(source): + with pytest.raises(AssertionError): + expression(source, {"github.event_name": "push"}) + + +@pytest.mark.parametrize( + "script", + [ + "pytest || true", + "pytest && pip --version; pip --version", + "if true; then pytest; fi", + "! pytest", + "pytest &", + "pytest > result.txt", + "set +e\npytest", + "exit 0\npytest", + "pytest() { return 0; }; pytest", + ], +) +def test_shell_failure_suppression_and_uninterpreted_control_flow_are_rejected(script): + with pytest.raises(AssertionError): + command_tokens(script) + + +def test_expression_semantics_match_github_not_python_literals_or_precedence(): + assert expression("'PUSH' == 'push'", {}) + assert expression("!'nonempty' == false", {}) + assert expression("'p''ush' != 'push'", {}) + assert not expression(r"'\x61' == 'a'", {}) + assert expression("'p''ush' == 'P''USH'", {}) + with pytest.raises(AssertionError, match="Mixed-type"): + expression("(true && 'push') == true", {}) + with pytest.raises(AssertionError): + expression("\"push\" == 'push'", {}) diff --git a/tests/test_ci_tiers.py b/tests/test_ci_tiers.py index f011586..0be3274 100644 --- a/tests/test_ci_tiers.py +++ b/tests/test_ci_tiers.py @@ -1,114 +1,381 @@ -"""Protect fast/full/nightly boundaries and the existing quality gates.""" +"""Protect CI behavior without coupling gates to human labels or shell quoting.""" from pathlib import Path import yaml +try: + import tomllib +except ModuleNotFoundError: # Python 3.10: also a pytest dependency. + import tomli as tomllib + +from tests._ci_contract import ( + EVENTS, + PYTHONS, + commands, + condition_matches, + matrix_rows, + one, + options, + pytest_options, + require, + select, +) + def workflow(): - return yaml.load(Path(".github/workflows/test.yml").read_text(), Loader=yaml.BaseLoader) - - -def named_steps(job): - return {step["name"]: step for step in job["steps"] if "name" in step} - - -def test_compatibility_matrix_leaves_slow_and_coverage_on_python_313(): - job = workflow()["jobs"]["test"] - assert job["strategy"]["matrix"]["python-version"] == ["3.10", "3.11", "3.12", "3.13"] - steps = named_steps(job) - fast = steps["Run non-slow compatibility tests"] - assert fast["if"] == "matrix.python-version != '3.13'" - assert fast["run"] == 'pytest -m "not slow and not nightly"' - full = steps["Run full PR tests with coverage"] - assert full["if"] == "matrix.python-version == '3.13'" - assert full["run"] == 'pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing' - assert ( - 'markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"]' - in Path("pyproject.toml").read_text() - ) - assert "if" not in steps["Verify executable fixture provenance"] - names = list(steps) - assert names.index("Verify executable fixture provenance") < names.index(fast["name"]) - assert names.index("Verify executable fixture provenance") < names.index(full["name"]) - - -def test_full_tier_preserves_independent_coverage_floors_and_packaging(): - steps = named_steps(workflow()["jobs"]["test"]) - guard = steps["Guard critical runtime coverage"] - assert guard["if"] == "matrix.python-version == '3.13'" - for file, floor in ( - ("core/snapshot_import_backend.py", 90), - ("simulate/simulated_caching_allocator.py", 88), - ("simulate/snapshot_mutator.py", 84), - ): - assert f"--include='*/{file}' --fail-under={floor}" in guard["run"] - config = Path("pyproject.toml").read_text() - assert "branch = true" in config and "fail_under = 90" in config - for name in ( - "Build wheel", - "Audit wheel contents against source", - "Clean-wheel Agent acceptance", - ): - assert steps[name]["if"] == "matrix.python-version == '3.13'" - assert steps["Upload coverage report"]["if"] == "matrix.python-version == '3.13' && always()" + return yaml.load( + Path(".github/workflows/test.yml").read_text(encoding="utf-8"), Loader=yaml.BaseLoader + ) -def test_ci_preserves_least_privilege_and_checks_the_published_commit(): - config = workflow() - assert config["permissions"] == {"contents": "read"} - assert "pull_request_target" not in config["on"] - assert config["on"]["pull_request"]["types"] == ["opened", "synchronize", "reopened", "edited"] - assert "workflow_call" in config["on"] - jobs = config["jobs"] - for name in ("snapshot-provenance", "lint", "browser", "test"): - checkout = jobs[name]["steps"][0] - assert checkout["uses"] == "actions/checkout@v4" - assert checkout["with"]["ref"] == "${{ github.event.pull_request.head.sha || github.sha }}" - assert jobs["snapshot-provenance"]["steps"][0]["with"]["fetch-depth"] == "0" - assert ( - "--require-decision" - in named_steps(jobs["snapshot-provenance"])[ - "Require provenance evaluation for snapshot runtime changes" - ]["run"] - ) - browser = named_steps(jobs["browser"])["Run required Chromium interaction tests"] - assert browser["env"]["PT_SNAP_REQUIRE_BROWSER"] == "1" - assert "tests/test_memory_tree_browser.py" in browser["run"] - lint = named_steps(jobs["lint"]) - for name in ("Ruff lint check", "Black format check", "basedPyright check"): - assert name in lint - - -def test_nightly_and_manual_gate_build_and_audit_before_required_acceptance(): - config = workflow() - assert "workflow_dispatch" in config["on"] - assert config["on"]["schedule"] == [{"cron": "17 3 * * *"}] - job = config["jobs"]["dataset-acceptance"] - assert ( - job["if"] == "github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'" - ) - assert job["timeout-minutes"] == "30" - assert job["steps"][0]["with"] == {"fetch-depth": "0", "ref": "${{ github.sha }}"} - steps = named_steps(job) - names = list(steps) - order = [ - "Verify executable fixture provenance", - "Build wheel", - "Audit wheel contents against source", - "Run required installed-wheel dataset acceptance", - "Upload dataset acceptance evidence", +def project(): + return tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + + +def _action_steps(job, action): + return [step for step in job["steps"] if step.get("uses", "").partition("@")[0] == action] + + +def _full(job, step): + condition_matches(job, step, predicate=lambda version, event, status: version == "3.13") + + +def _all(job, step): + condition_matches(job, step, predicate=lambda version, event, status: True) + + +def _check_wheel_gate(job, all_commands, command, *, nightly=False): + opts, positional = options( + command.words[1:], valued=("--wheel", "--fixture", "--events-per-slice", "--output-dir") + ) + require( + not positional and len(opts.get("--wheel", [])) == 1, + "Wheel gate must consume one verified wheel", + ) + variable = opts["--wheel"][0] + require( + variable.startswith("$") and len(variable) > 1, "Wheel gate must consume the built wheel" + ) + variable = variable[1:].removeprefix("{").removesuffix("}") + selectors = [ + item + for item in all_commands + if item.words == ("__verified_wheel__", variable) + and item.index < command.index + and item.step is command.step ] - assert [names.index(name) for name in order] == sorted(names.index(name) for name in order) - acceptance = steps["Run required installed-wheel dataset acceptance"]["run"] - assert "assert len(wheels) == 1" in acceptance - assert "python .github/scripts/run_dataset_acceptance.py" in acceptance - assert '--wheel "$WHEEL" --output-dir .tmp/dataset-acceptance' in acceptance - for step in job["steps"]: - assert "continue-on-error" not in step - upload = steps["Upload dataset acceptance evidence"] - assert upload["if"] == "always()" - assert upload["with"]["if-no-files-found"] == "error" - assert upload["with"]["include-hidden-files"] == "true" - assert upload["with"]["path"] == ".tmp/dataset-acceptance/" - assert "${{ github.sha }}" in upload["with"]["name"] + require(selectors, "Wheel selection must validate exactly one artifact before use") + if nightly: + _nightly(job, selectors[-1].step) + _nightly(job, command.step) + else: + _full(job, selectors[-1].step) + _full(job, command.step) + return opts + + +def validate_tiers(config, settings): + job = config["jobs"]["test"] + rows = matrix_rows(job["strategy"]["matrix"]) + require( + sorted(row["python-version"] for row in rows) == list(PYTHONS), + "Compatibility matrix must execute each supported Python exactly once", + ) + all_commands = commands(job, config) + pytest = select(all_commands, "pytest") + require(len(pytest) == 3, "Fixture, compatibility and full pytest gates are mandatory") + fixture = [command for command in pytest if "tests/test_fixture_provenance.py" in command.words] + require(len(fixture) == 1, "Missing fixture provenance gate") + fixture = fixture[0] + pytest_options(fixture, paths=("tests/test_fixture_provenance.py",)) + _all(job, fixture.step) + full = [command for command in pytest if "--cov" in command.words] + require(len(full) == 1, "Exactly one full coverage gate is mandatory") + full = full[0] + fast = next(command for command in pytest if command is not full and command is not fixture) + pytest_options(fast, expected_marker=lambda slow, nightly: not slow and not nightly) + pytest_options(full, expected_marker=lambda slow, nightly: not nightly, coverage=True) + condition_matches(job, fast.step, predicate=lambda version, event, status: version != "3.13") + _full(job, full.step) + require( + fixture.index < fast.index and fixture.index < full.index, + "Fixture provenance must precede execution", + ) + require( + any( + marker.partition(":")[0] == "nightly" + for marker in settings["tool"]["pytest"]["ini_options"]["markers"] + ), + "Nightly marker must be registered", + ) + + coverage = settings["tool"]["coverage"] + require(coverage["run"]["branch"] is True, "Branch coverage is mandatory") + require(coverage["report"]["fail_under"] >= 90, "Overall coverage floor must not decrease") + expected = { + "*/core/snapshot_import_backend.py": 90, + "*/simulate/simulated_caching_allocator.py": 88, + "*/simulate/snapshot_mutator.py": 84, + } + reports = select(all_commands, "coverage") + require(len(reports) == len(expected), "Each critical file needs its own coverage floor") + seen = set() + for report in reports: + require(report.words[1] == "report", "Critical coverage must report measured results") + opts, positional = options(report.words[2:], valued=("--include", "--fail-under")) + require( + not positional and set(opts) == {"--include", "--fail-under"}, + "Coverage report must have an independent selector and floor", + ) + require( + all(len(values) == 1 for values in opts.values()), + "Duplicate coverage options are ambiguous", + ) + path = opts["--include"][0] + require( + path in expected and path not in seen, "Wrong/combined critical-file coverage selector" + ) + require( + float(opts["--fail-under"][0]) >= expected[path], + "Critical coverage floor must not decrease", + ) + seen.add(path) + require(full.index < report.index, "Coverage must be measured before its floors") + _full(job, report.step) + + build = one(all_commands, "build") + require(build.words[1:] == ("--wheel",), "Build must produce the wheel") + _full(job, build.step) + audit = one(all_commands, ".github/scripts/audit_wheel.py") + require( + set(_check_wheel_gate(job, all_commands, audit)) == {"--wheel"}, + "Wheel audit arguments changed", + ) + acceptance = one(all_commands, ".github/scripts/clean_wheel_agent_acceptance.py") + opts = _check_wheel_gate(job, all_commands, acceptance) + require( + opts.get("--fixture") == ["tests/fixtures/snapshots/snapshot_with_empty_cache.pkl"] + and opts.get("--events-per-slice") == ["4096"] + and set(opts) == {"--wheel", "--fixture", "--events-per-slice"}, + "Clean-wheel fixture/sharded acceptance is mandatory", + ) + require( + full.index < build.index < audit.index < acceptance.index, + "Wheel acceptance must follow tests, build and audit", + ) + upload = [ + step + for step in _action_steps(job, "actions/upload-artifact") + if step.get("with", {}).get("path") == "coverage.xml" + ] + require(len(upload) == 1, "Coverage evidence must be uploaded") + condition_matches( + job, + upload[0], + predicate=lambda version, event, status: version == "3.13", + statuses=("success", "failure", "cancelled"), + ) + + +def _nightly(job, step, *, always=False): + condition_matches( + job, + step, + predicate=lambda version, event, status: event in {"schedule", "workflow_dispatch"}, + statuses=("success", "failure", "cancelled") if always else ("success",), + ) + + +def validate_nightly(config): + require("workflow_dispatch" in config["on"], "Manual acceptance dispatch is mandatory") + require( + config["on"].get("schedule") == [{"cron": "17 3 * * *"}], + "Nightly acceptance schedule changed", + ) + job = config["jobs"]["dataset-acceptance"] + require(job["timeout-minutes"] == "30", "Dataset acceptance timeout changed") + checkout = _action_steps(job, "actions/checkout") + require( + len(checkout) == 1 and checkout[0]["with"].get("fetch-depth") == "0", + "Dataset acceptance needs full checkout history", + ) + require( + "".join(checkout[0]["with"].get("ref", "").split()) == "${{github.sha}}", + "Dataset acceptance must use the dispatched commit", + ) + setup = _action_steps(job, "actions/setup-python") + require( + len(setup) == 1 and setup[0].get("with", {}).get("python-version") == "3.13", + "Nightly acceptance requires Python 3.13", + ) + _nightly(job, setup[0]) + all_commands = commands(job, config) + for command in all_commands: + _nightly(job, command.step) + fixture = one(all_commands, "pytest") + pytest_options(fixture, paths=("tests/test_fixture_provenance.py",)) + build = one(all_commands, "build") + require(build.words[1:] == ("--wheel",), "Nightly acceptance requires a built wheel") + audit = one(all_commands, ".github/scripts/audit_wheel.py") + require( + set(_check_wheel_gate(job, all_commands, audit, nightly=True)) == {"--wheel"}, + "Nightly wheel audit arguments changed", + ) + acceptance = one(all_commands, ".github/scripts/run_dataset_acceptance.py") + opts = _check_wheel_gate(job, all_commands, acceptance, nightly=True) + require( + opts.get("--output-dir") == [".tmp/dataset-acceptance"] + and set(opts) == {"--wheel", "--output-dir"}, + "Nightly evidence destination changed", + ) + differential = one(all_commands, ".github/scripts/run_differential_acceptance.py") + opts, positional = options(differential.words[1:], valued=("--seed-count", "--output-dir")) + require( + not positional + and opts == {"--seed-count": ["200"], "--output-dir": [".tmp/differential-acceptance"]}, + "Required 200-seed paired acceptance must not weaken", + ) + require( + fixture.index < build.index < audit.index < acceptance.index < differential.index, + "Nightly acceptance order must preserve provenance/build/audit", + ) + uploads = _action_steps(job, "actions/upload-artifact") + for path in (".tmp/dataset-acceptance/", ".tmp/differential-acceptance/*.log"): + matching = [ + step for step in uploads if path in step.get("with", {}).get("path", "").splitlines() + ] + require(len(matching) == 1, f"Missing acceptance evidence upload: {path}") + upload = matching[0] + _nightly(job, upload, always=True) + require( + upload["with"].get("if-no-files-found") == "error" + and upload["with"].get("include-hidden-files") == "true", + "Missing acceptance evidence must fail", + ) + require( + "${{github.sha}}" in "".join(upload["with"].get("name", "").split()), + "Evidence must identify the checkout commit", + ) + require( + job["steps"].index(upload) > job["steps"].index(differential.step), + "Evidence must be uploaded after acceptance", + ) + + +def validate_other_gates(config): + require(config["permissions"] == {"contents": "read"}, "CI permissions must stay read-only") + require( + "pull_request_target" not in config["on"], "Untrusted PRs cannot use privileged trigger" + ) + require( + set(config["on"]["pull_request"]["types"]) + == {"opened", "synchronize", "reopened", "edited"}, + "PR gate triggers changed", + ) + require(set(EVENTS) <= set(config["on"]), "Required CI triggers are missing") + for event in ("push", "pull_request"): + require(config["on"][event].get("branches") == ["main"], "Main push/PR gates are required") + require( + not set(config["on"][event]) & {"paths", "paths-ignore", "branches-ignore"}, + "Gate triggers cannot silently filter changes", + ) + for name in ("snapshot-provenance", "lint", "browser", "test"): + job = config["jobs"][name] + require( + job.get("continue-on-error", "false") == "false", "Job failures must not be ignored" + ) + require(not job.get("needs"), "Independent CI gates cannot depend on skippable jobs") + require( + job.get("permissions", {"contents": "read"}) == {"contents": "read"}, + "Job permissions must stay read-only", + ) + checkout = _action_steps(job, "actions/checkout") + require(len(checkout) == 1, "Exactly one gate checkout is required") + require( + "".join(checkout[0].get("with", {}).get("ref", "").split()) + == "${{github.event.pull_request.head.sha||github.sha}}", + "CI must check the published PR/triggering commit", + ) + _all(job, checkout[0]) + require(job["steps"].index(checkout[0]) == 0, "Gate checkout must precede commands") + if name != "snapshot-provenance": + setup = _action_steps(job, "actions/setup-python") + require(len(setup) == 1, "Every execution tier must select its Python") + version = "".join(setup[0].get("with", {}).get("python-version", "").split()) + require( + version == ("${{matrix.python-version}}" if name == "test" else "3.13"), + "Python setup must match its execution tier", + ) + _all(job, setup[0]) + provenance = config["jobs"]["snapshot-provenance"] + require( + _action_steps(provenance, "actions/checkout")[0]["with"].get("fetch-depth") == "0", + "Provenance requires full history", + ) + # This PR gate is a simple command. The separate non-PR base-resolution + # script is deliberately not interpreted as part of the tier contract. + pr_steps = [step for step in provenance["steps"] if "PR_BODY" in step.get("env", {})] + require(len(pr_steps) == 1, "PR provenance decision gate is required") + pr = one(commands({"steps": pr_steps}, config), ".github/scripts/check_snapshot_provenance.py") + for key, context in (("BASE_SHA", "base.sha"), ("HEAD_SHA", "head.sha"), ("PR_BODY", "body")): + require( + "".join(pr.env.get(key, "").split()) + == "${{github.event.pull_request." + context + "}}", + "PR provenance must compare the actual base/head and decision body", + ) + opts, positional = options( + pr.words[1:], valued=("--base", "--head"), flags=("--require-decision",) + ) + require( + not positional + and opts + == {"--base": ["$BASE_SHA"], "--head": ["$HEAD_SHA"], "--require-decision": [True]}, + "PR provenance decision cannot be bypassed", + ) + condition_matches( + provenance, pr.step, predicate=lambda version, event, status: event == "pull_request" + ) + lint = config["jobs"]["lint"] + lint_commands = commands(lint, config) + for program, arguments in ( + ("ruff", ("check", ".")), + ("black", ("--check", ".")), + ("basedpyright", ("--pythonpath", "__selected_python__")), + ): + command = one(lint_commands, program) + require( + command.words[1:] == arguments, f"{program} must check the intended source/interpreter" + ) + _all(lint, command.step) + browser = config["jobs"]["browser"] + require( + browser["runs-on"].startswith("ubuntu-"), "Real Chromium regression must remain on Linux" + ) + browser_commands = commands(browser, config) + install = one(browser_commands, "playwright") + opts, positional = options(install.words[1:], flags=("--with-deps",)) + require( + positional == ["install", "chromium"] and opts == {"--with-deps": [True]}, + "CI must install real Chromium with OS dependencies", + ) + test = one(browser_commands, "pytest") + pytest_options( + test, paths=("tests/test_fixture_provenance.py", "tests/test_memory_tree_browser.py") + ) + require( + test.env.get("PT_SNAP_REQUIRE_BROWSER") == "1", "Browser absence must fail the required job" + ) + require(install.index < test.index, "Chromium must be installed before required regression") + _all(browser, install.step) + _all(browser, test.step) + + +def validate_workflow(config, settings): + validate_tiers(config, settings) + validate_nightly(config) + validate_other_gates(config) + + +def test_ci_semantic_contract(): + validate_workflow(workflow(), project()) diff --git a/tests/test_memory_tree_browser.py b/tests/test_memory_tree_browser.py index baef121..f9c70bc 100644 --- a/tests/test_memory_tree_browser.py +++ b/tests/test_memory_tree_browser.py @@ -6,8 +6,6 @@ from __future__ import annotations -import importlib -import os import re from collections.abc import Iterator @@ -15,12 +13,9 @@ from pt_snap_cli.core.models import QueryResult from pt_snap_cli.memory_tree_render import render_memory_tree +from tests._browser_support import browser_expect as expect -playwright = ( - importlib.import_module("playwright.sync_api") - if os.environ.get("PT_SNAP_REQUIRE_BROWSER") == "1" - else pytest.importorskip("playwright.sync_api") -) +pytestmark = pytest.mark.browser ROOT_BYTES = 471_498_368 GIANT_BYTES = 283_057_288 @@ -82,27 +77,6 @@ def frame_rows(): return rows -@pytest.fixture(scope="module") -def chromium_browser(): - with playwright.sync_playwright() as runtime: - options = {"headless": True} - executable = os.environ.get("PT_SNAP_TEST_CHROMIUM") - if executable: - options["executable_path"] = executable - try: - browser = runtime.chromium.launch(**options) - except playwright.Error as exc: - # A broken running browser is a test failure, not an optional dependency. - missing = "executable doesn't exist" in str(exc).lower() - if missing and os.environ.get("PT_SNAP_REQUIRE_BROWSER") != "1": - pytest.skip("Chromium is not installed; run python -m playwright install chromium") - pytest.fail(f"Cannot launch Chromium for frame navigation tests: {exc}") - try: - yield browser - finally: - browser.close() - - @pytest.fixture def browser_page(chromium_browser) -> Iterator: context = chromium_browser.new_context(viewport={"width": 1440, "height": 1000}, locale="en-US") @@ -126,7 +100,7 @@ def _render(page, rows): total_is_exact=True, ) page.set_content(render_memory_tree(result, event_id=17, html=True), wait_until="load") - playwright.expect(page.locator("#chart .node").first).to_be_attached() + expect(page.locator("#chart .node").first).to_be_attached() def _button(page, container, node_id): @@ -152,8 +126,8 @@ def _assert_path(locator, *labels): def _assert_selected(page, node_id, *labels): - playwright.expect(page.locator("#chart .node").first).to_have_attribute("data-node-id", node_id) - playwright.expect(page.locator("#detail")).to_contain_text(node_id) + expect(page.locator("#chart .node").first).to_have_attribute("data-node-id", node_id) + expect(page.locator("#detail")).to_contain_text(node_id) _assert_path(page.locator("#focus-path"), *labels) @@ -170,9 +144,9 @@ def _tab_to(page, target): def test_search_keeps_true_proportional_geometry_until_navigation(browser_page, frame_rows): page = browser_page _render(page, frame_rows) - playwright.expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() - playwright.expect(page.locator("#search-status")).not_to_be_empty() - playwright.expect(page.locator("#search-results button")).to_have_count(0) + expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() + expect(page.locator("#search-status")).not_to_be_empty() + expect(page.locator("#search-results button")).to_have_count(0) _assert_selected(page, "root", "All active memory") giant_width = float(_rect(page, "frame:giant").get_attribute("width")) @@ -182,7 +156,7 @@ def test_search_keeps_true_proportional_geometry_until_navigation(browser_page, original_geometry = _geometry(page) page.locator("#search").fill("SHARED_FUNCTION") - playwright.expect(page.locator("#search-results button")).to_have_count(2) + expect(page.locator("#search-results button")).to_have_count(2) assert _geometry(page) == original_geometry _assert_selected(page, "root", "All active memory") @@ -201,26 +175,26 @@ def test_search_disambiguates_duplicate_paths_and_supports_native_keyboard( page = browser_page _render(page, frame_rows) page.locator("#search").fill("sHaReD_fUnCtIoN") - playwright.expect(page.locator("#search-results button")).to_have_count(2) - playwright.expect(page.locator("#search-status")).to_contain_text(re.compile(r"\b2\b")) + expect(page.locator("#search-results button")).to_have_count(2) + expect(page.locator("#search-status")).to_contain_text(re.compile(r"\b2\b")) for node_id, branch in ( ("frame:giant/tiny", "giant_branch"), ("frame:other/tiny", "other_branch"), ): button = _button(page, "search-results", node_id) _assert_path(button, "All active memory", branch, "shared_function") - playwright.expect(button).to_contain_text(node_id) - playwright.expect(button).to_contain_text(re.compile(r"40\s+bytes", re.IGNORECASE)) + expect(button).to_contain_text(node_id) + expect(button).to_contain_text(re.compile(r"40\s+bytes", re.IGNORECASE)) target = _button(page, "search-results", "frame:giant/tiny") _tab_to(page, target) page.keyboard.press("Enter") _assert_selected( page, "frame:giant/tiny", "All active memory", "giant_branch", "shared_function" ) - playwright.expect(page.locator("#detail")).to_contain_text("Inclusive: 40 bytes") + expect(page.locator("#detail")).to_contain_text("Inclusive: 40 bytes") # The other same-named frame remains available outside the focused subtree. - playwright.expect(page.locator("#search-results button")).to_have_count(2) + expect(page.locator("#search-results button")).to_have_count(2) target = _button(page, "search-results", "frame:other/tiny") _tab_to(page, target) page.keyboard.press("Space") @@ -234,13 +208,13 @@ def test_node_id_search_is_case_insensitive_and_global_after_focus(browser_page, _render(page, frame_rows) _button(page, "children-list", "frame:giant").click() page.locator("#search").fill("FRAME:OTHER/TINY") - playwright.expect(page.locator("#search-results button")).to_have_count(1) + expect(page.locator("#search-results button")).to_have_count(1) _button(page, "search-results", "frame:other/tiny").click() _assert_selected( page, "frame:other/tiny", "All active memory", "other_branch", "shared_function" ) page.locator("#search").fill("ZERO_LEAF") - playwright.expect(page.locator("#search-results button")).to_have_count(1) + expect(page.locator("#search-results button")).to_have_count(1) _button(page, "search-results", "frame:giant/zero/leaf").click() _assert_selected( page, @@ -250,34 +224,34 @@ def test_node_id_search_is_case_insensitive_and_global_after_focus(browser_page, "zero_parent", "zero_leaf", ) - playwright.expect(page.locator("#detail")).to_contain_text("Inclusive: 0 bytes") + expect(page.locator("#detail")).to_contain_text("Inclusive: 0 bytes") def test_children_reach_tiny_and_zero_frames_and_parent_retraces_path(browser_page, frame_rows): page = browser_page _render(page, frame_rows) - playwright.expect(page.locator("#children-list button")).to_have_count(3) + expect(page.locator("#children-list button")).to_have_count(3) giant = _button(page, "children-list", "frame:giant") _assert_path(giant, "All active memory", "giant_branch") - playwright.expect(giant).to_contain_text("frame:giant") - playwright.expect(giant).to_contain_text("283,057,288 bytes") + expect(giant).to_contain_text("frame:giant") + expect(giant).to_contain_text("283,057,288 bytes") giant.click() - playwright.expect(page.locator("#children-list button")).to_have_count(2) + expect(page.locator("#children-list button")).to_have_count(2) tiny = _button(page, "children-list", "frame:giant/tiny") _assert_path(tiny, "All active memory", "giant_branch", "shared_function") - playwright.expect(tiny).to_contain_text("40 bytes") + expect(tiny).to_contain_text("40 bytes") _tab_to(page, tiny) page.keyboard.press("Enter") _assert_selected( page, "frame:giant/tiny", "All active memory", "giant_branch", "shared_function" ) - playwright.expect(page.locator("#children-list button")).to_have_count(0) - playwright.expect(page.locator("#children-status")).not_to_be_empty() + expect(page.locator("#children-list button")).to_have_count(0) + expect(page.locator("#children-status")).not_to_be_empty() page.get_by_role("button", name="Parent frame", exact=True).click() zero = _button(page, "children-list", "frame:giant/zero") - playwright.expect(zero).to_contain_text("0 bytes") + expect(zero).to_contain_text("0 bytes") _tab_to(page, zero) page.keyboard.press("Space") _assert_selected(page, "frame:giant/zero", "All active memory", "giant_branch", "zero_parent") @@ -290,8 +264,8 @@ def test_children_reach_tiny_and_zero_frames_and_parent_retraces_path(browser_pa "zero_parent", "zero_leaf", ) - playwright.expect(page.locator("#children-list button")).to_have_count(0) - playwright.expect(page.locator("#children-status")).not_to_be_empty() + expect(page.locator("#children-list button")).to_have_count(0) + expect(page.locator("#children-status")).not_to_be_empty() for node_id, labels in ( ("frame:giant/zero", ("All active memory", "giant_branch", "zero_parent")), ("frame:giant", ("All active memory", "giant_branch")), @@ -299,7 +273,7 @@ def test_children_reach_tiny_and_zero_frames_and_parent_retraces_path(browser_pa ): page.get_by_role("button", name="Parent frame", exact=True).click() _assert_selected(page, node_id, *labels) - playwright.expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() + expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() def test_reset_is_repeatable_and_preserves_global_search(browser_page, frame_rows): @@ -311,12 +285,10 @@ def test_reset_is_repeatable_and_preserves_global_search(browser_page, frame_row _button(page, "search-results", node_id).click() page.locator("#reset").click() _assert_selected(page, "root", "All active memory") - playwright.expect(page.locator("#search")).to_have_value("shared_function") - playwright.expect(page.locator("#search-results button")).to_have_count(2) - playwright.expect(page.locator("#children-list button")).to_have_count(3) - playwright.expect( - page.get_by_role("button", name="Parent frame", exact=True) - ).to_be_disabled() + expect(page.locator("#search")).to_have_value("shared_function") + expect(page.locator("#search-results button")).to_have_count(2) + expect(page.locator("#children-list button")).to_have_count(3) + expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() assert _geometry(page) == original_geometry page.locator("#reset").click() _assert_selected(page, "root", "All active memory") @@ -329,14 +301,14 @@ def test_no_results_and_clearing_search_leave_navigation_usable(browser_page, fr _button(page, "children-list", "frame:giant").click() geometry = _geometry(page) page.locator("#search").fill("there is no such frame") - playwright.expect(page.locator("#search-results button")).to_have_count(0) - playwright.expect(page.locator("#search-status")).to_have_text("No matching frames.") + expect(page.locator("#search-results button")).to_have_count(0) + expect(page.locator("#search-status")).to_have_text("No matching frames.") assert _geometry(page) == geometry _assert_selected(page, "frame:giant", "All active memory", "giant_branch") - playwright.expect(page.locator("#children-list button")).to_have_count(2) + expect(page.locator("#children-list button")).to_have_count(2) page.locator("#search").fill("") - playwright.expect(page.locator("#search-status")).to_have_text(instructions) - playwright.expect(page.locator("#search-results button")).to_have_count(0) + expect(page.locator("#search-status")).to_have_text(instructions) + expect(page.locator("#search-results button")).to_have_count(0) _button(page, "children-list", "frame:giant/tiny").click() _assert_selected( page, "frame:giant/tiny", "All active memory", "giant_branch", "shared_function" @@ -347,22 +319,20 @@ def test_unicode_and_html_like_labels_remain_plain_text_everywhere(browser_page, page = browser_page _render(page, frame_rows) root_child = _button(page, "children-list", "frame:malicious") - playwright.expect(root_child).to_contain_text(MALICIOUS_LABEL) + expect(root_child).to_contain_text(MALICIOUS_LABEL) page.locator("#search").fill("中文") - playwright.expect(page.locator("#search-results button")).to_have_count(1) + expect(page.locator("#search-results button")).to_have_count(1) result = _button(page, "search-results", "frame:malicious") - playwright.expect(result).to_contain_text(MALICIOUS_LABEL) + expect(result).to_contain_text(MALICIOUS_LABEL) result.click() _assert_selected(page, "frame:malicious", "All active memory", MALICIOUS_LABEL) - playwright.expect(page.locator("#detail")).to_contain_text(MALICIOUS_LABEL) - playwright.expect(page.locator("#chart text")).to_have_text(MALICIOUS_LABEL) - playwright.expect(page.locator("img")).to_have_count(0) - playwright.expect(page.locator("script")).to_have_count(2) + expect(page.locator("#detail")).to_contain_text(MALICIOUS_LABEL) + expect(page.locator("#chart text")).to_have_text(MALICIOUS_LABEL) + expect(page.locator("img")).to_have_count(0) + expect(page.locator("script")).to_have_count(2) assert page.evaluate("window.__frameInjected === undefined") page.locator("#reset").click() - playwright.expect(_button(page, "children-list", "frame:malicious")).to_contain_text( - MALICIOUS_LABEL - ) + expect(_button(page, "children-list", "frame:malicious")).to_contain_text(MALICIOUS_LABEL) def test_search_does_not_silently_truncate_more_than_100_matches(browser_page): @@ -373,13 +343,13 @@ def test_search_does_not_silently_truncate_more_than_100_matches(browser_page): ) _render(page, rows) page.locator("#search").fill("BATCH FRAME") - playwright.expect(page.locator("#search-results button")).to_have_count(151) - playwright.expect(page.locator("#search-status")).to_contain_text(re.compile(r"\b151\b")) + expect(page.locator("#search-results button")).to_have_count(151) + expect(page.locator("#search-status")).to_contain_text(re.compile(r"\b151\b")) _button(page, "search-results", "batch:150").click() _assert_selected(page, "batch:150", "All active memory", "batch frame 150") - playwright.expect(page.locator("#search-results button")).to_have_count(151) + expect(page.locator("#search-results button")).to_have_count(151) page.locator("#reset").click() - playwright.expect(page.locator("#children-list button")).to_have_count(151) + expect(page.locator("#children-list button")).to_have_count(151) def test_zero_total_tree_can_be_explored_and_reset(browser_page): @@ -395,15 +365,15 @@ def test_zero_total_tree_can_be_explored_and_reset(browser_page): _render(page, rows) original_geometry = _geometry(page) assert float(_rect(page, "empty:branch").get_attribute("width")) == 0 - playwright.expect(page.locator("#children-list button")).to_have_count(2) + expect(page.locator("#children-list button")).to_have_count(2) _button(page, "children-list", "empty:branch").click() leaf = _button(page, "children-list", "empty:leaf") _tab_to(page, leaf) page.keyboard.press("Enter") _assert_selected(page, "empty:leaf", "Empty active memory", "empty branch", "empty leaf") - playwright.expect(page.locator("#detail")).to_contain_text("Inclusive: 0 bytes") - playwright.expect(page.locator("#detail")).to_contain_text("n/a") - playwright.expect(page.locator("#children-status")).not_to_be_empty() + expect(page.locator("#detail")).to_contain_text("Inclusive: 0 bytes") + expect(page.locator("#detail")).to_contain_text("n/a") + expect(page.locator("#children-status")).not_to_be_empty() page.locator("#search").fill("empty sibling") _button(page, "search-results", "empty:sibling").click() _assert_selected(page, "empty:sibling", "Empty active memory", "empty sibling") @@ -418,10 +388,10 @@ def test_existing_svg_click_and_hover_keep_working(browser_page, frame_rows): page = browser_page _render(page, frame_rows) _rect(page, "frame:giant").hover() - playwright.expect(page.locator("#detail")).to_contain_text("283,057,288 bytes") + expect(page.locator("#detail")).to_contain_text("283,057,288 bytes") _rect(page, "frame:giant").click() _assert_selected(page, "frame:giant", "All active memory", "giant_branch") - playwright.expect(page.locator("#children-list button")).to_have_count(2) + expect(page.locator("#children-list button")).to_have_count(2) page.locator("#reset").click() _assert_selected(page, "root", "All active memory") @@ -431,17 +401,17 @@ def test_keyboard_focus_survives_continuous_descent_parent_and_reset(browser_pag _render(page, frame_rows) path = page.locator("#focus-path") _button(page, "children-list", "frame:giant").click() - playwright.expect(path).to_be_focused() + expect(path).to_be_focused() # No focus() helper between navigations: the next Tab reaches the new children. page.keyboard.press("Tab") - playwright.expect(_button(page, "children-list", "frame:giant/tiny")).to_be_focused() + expect(_button(page, "children-list", "frame:giant/tiny")).to_be_focused() page.keyboard.press("Tab") - playwright.expect(_button(page, "children-list", "frame:giant/zero")).to_be_focused() + expect(_button(page, "children-list", "frame:giant/zero")).to_be_focused() page.keyboard.press("Space") - playwright.expect(path).to_be_focused() + expect(path).to_be_focused() page.keyboard.press("Tab") - playwright.expect(_button(page, "children-list", "frame:giant/zero/leaf")).to_be_focused() + expect(_button(page, "children-list", "frame:giant/zero/leaf")).to_be_focused() page.keyboard.press("Enter") _assert_selected( page, @@ -451,25 +421,25 @@ def test_keyboard_focus_survives_continuous_descent_parent_and_reset(browser_pag "zero_parent", "zero_leaf", ) - playwright.expect(path).to_be_focused() + expect(path).to_be_focused() page.keyboard.press("Shift+Tab") - playwright.expect(page.locator("#search")).to_be_focused() + expect(page.locator("#search")).to_be_focused() page.keyboard.press("Shift+Tab") - playwright.expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_focused() + expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_focused() page.keyboard.press("Enter") _assert_selected(page, "frame:giant/zero", "All active memory", "giant_branch", "zero_parent") - playwright.expect(path).to_be_focused() + expect(path).to_be_focused() for _ in range(3): page.keyboard.press("Shift+Tab") - playwright.expect(page.locator("#reset")).to_be_focused() + expect(page.locator("#reset")).to_be_focused() page.keyboard.press("Space") _assert_selected(page, "root", "All active memory") - playwright.expect(path).to_be_focused() - playwright.expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() + expect(path).to_be_focused() + expect(page.get_by_role("button", name="Parent frame", exact=True)).to_be_disabled() page.keyboard.press("Tab") - playwright.expect(_button(page, "children-list", "frame:giant")).to_be_focused() + expect(_button(page, "children-list", "frame:giant")).to_be_focused() def test_special_allocation_categories_remain_reachable(browser_page): @@ -492,10 +462,10 @@ def test_special_allocation_categories_remain_reachable(browser_page): assert float(_rect(page, node_id).get_attribute("width")) == 0 _button(page, "children-list", node_id).click() _assert_selected(page, node_id, "All active memory", label) - playwright.expect(page.locator("#children-status")).not_to_be_empty() + expect(page.locator("#children-status")).not_to_be_empty() page.locator("#reset").click() page.locator("#search").fill(node_id) - playwright.expect(page.locator("#search-results button")).to_have_count(1) + expect(page.locator("#search-results button")).to_have_count(1) _button(page, "search-results", node_id).click() _assert_selected(page, node_id, "All active memory", label) page.locator("#reset").click()