Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion .github/workflows/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,26 @@ Concurrency is keyed by GitHub ref and superseded runs are cancelled.
Million-event deep-offset pagination tests are marked `slow`; their small
pagination contracts still run across the compatibility matrix.

`tests/test_ci_tiers.py` checks parsed commands, matrix rows, marker truth tables,
conditions, independent file floors and gate ordering, not step display names or
complete shell text. `tests/_ci_contract.py` intentionally supports only a bounded
CI language: simple failure-propagating commands, shell quoting/continuations,
sequential steps or a terminal `&&` chain, and explicit boolean conditions.
Unknown shell control flow, OR/pipes, failure suppression, selector overrides and
unrecognized conditions fail closed. Extend it with positive and negative
mutation tests when introducing new syntax; do not weaken it into substring
checks. Action version bumps, cache configuration, renaming, equivalent selectors
and step splits/merges do not change a gate.

The Linux browser job installs Chromium and sets `PT_SNAP_REQUIRE_BROWSER=1`.
Browser cases are always collected even if Playwright is absent. Optional matrix
and platform runs report each missing-dependency case as skipped plus an explicit
**not executed** summary; that is not browser-pass evidence. Required mode fails
for missing package/binary, any browser skip, empty/deselected collection or
incomplete execution. `--collect-only` remains a collection diagnostic and is
explicitly reported as not run. Broken installations and launch failures fail in
both modes. Browser results are not added to non-browser platform requirements.

Coverage reporting writes `.coverage`/`coverage.xml`; building writes `dist/` and
intermediates. Dependency installation/build isolation may access the network.
The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses temporary
Expand Down Expand Up @@ -93,7 +113,8 @@ validation. For non-PR provenance checks, tag runs compare against the prior tag
Review `pyproject.toml`, both workflow YAMLs, the provenance/acceptance scripts and
`tests/test_release_workflow.py` together when changing execution order. Validate
YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py
tests/test_ci_tiers.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from
tests/test_ci_tiers.py tests/test_ci_contract_mutations.py
tests/test_browser_execution_contract.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from
the repository root. Full CI tests the matrix and installed wheel; static tests
do not prove a release environment is configured or a publication occurred.

Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/platform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ jobs:
run: python -m pytest tests/test_fixture_provenance.py
- name: Run exhaustive non-slow partition
# No coverage denominator or scale matrix here. Resource-warning gates
# remain enabled; required Chromium interaction runs in the Linux job.
# remain enabled; browser cases stay collected and explicitly report
# optional skips. Real Chromium execution is required in the Linux job.
run: ${{ matrix.command }}
- name: Upload platform test results
if: always()
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,8 @@ jobs:
- name: Run required Chromium interaction tests
env:
PT_SNAP_REQUIRE_BROWSER: "1"
run: pytest tests/test_fixture_provenance.py tests/test_memory_tree_browser.py
# Required mode rejects missing dependencies, skips and incomplete execution.
run: pytest -rs tests/test_fixture_provenance.py tests/test_memory_tree_browser.py

test:
name: Test (Python ${{ matrix.python-version }})
Expand Down
129 changes: 129 additions & 0 deletions tests/_browser_support.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
"""Optional-browser fixtures and visible per-test execution accounting.

Import Playwright only during fixture setup: absent optional dependencies must
never remove browser regression identities from pytest's collection.
"""

from __future__ import annotations

import importlib
import os
from pathlib import Path

import pytest


def _unavailable(reason):
message = f"Browser regressions NOT EXECUTED: {reason}"
if os.environ.get("PT_SNAP_REQUIRE_BROWSER") == "1":
pytest.fail(message, pytrace=False)
pytest.skip(message)


@pytest.fixture(scope="session")
def playwright_api():
try:
return importlib.import_module("playwright.sync_api")
except ModuleNotFoundError as exc:
# A broken installation (for example missing greenlet) is not optional.
if exc.name not in {"playwright", "playwright.sync_api"}:
raise
_unavailable("Playwright is not installed; install the .[dev] extra")


@pytest.fixture(scope="module")
def chromium_browser(playwright_api):
with playwright_api.sync_playwright() as runtime:
executable = os.environ.get("PT_SNAP_TEST_CHROMIUM")
binary = executable or runtime.chromium.executable_path
if not Path(binary).is_file():
_unavailable(
"Chromium executable is missing; run python -m playwright install chromium "
"or set PT_SNAP_TEST_CHROMIUM to an existing executable"
)
options = {"headless": True}
if executable:
options["executable_path"] = executable
try:
browser = runtime.chromium.launch(**options)
except playwright_api.Error as exc:
# Only the explicit preflight above can skip. Crashes, permission
# errors, bad shared libraries and other launch failures always fail.
pytest.fail(f"Cannot launch Chromium for frame navigation tests: {exc}")
try:
yield browser
finally:
browser.close()


def browser_expect(locator):
"""Assertions run only after browser setup has succeeded."""
return importlib.import_module("playwright.sync_api").expect(locator)


class BrowserSummary:
def __init__(self):
self.nodes = set()
self.outcomes = {}
self.skip_reasons = set()

@pytest.hookimpl(hookwrapper=True, tryfirst=True)
def pytest_collection_modifyitems(self, items):
self.nodes = {item.nodeid for item in items if item.get_closest_marker("browser")}
yield

def pytest_runtest_logreport(self, report):
if report.nodeid not in self.nodes:
return
if report.failed:
self.outcomes[report.nodeid] = "failed"
elif self.outcomes.get(report.nodeid) != "failed":
if report.skipped:
self.outcomes[report.nodeid] = "skipped"
reason = (
report.longrepr[2] if isinstance(report.longrepr, tuple) else report.longrepr
)
self.skip_reasons.add(str(reason))
elif report.when == "call":
self.outcomes[report.nodeid] = "passed"

def pytest_sessionfinish(self, session):
if os.environ.get("PT_SNAP_REQUIRE_BROWSER") != "1" or session.config.option.collectonly:
return
incomplete = not self.nodes or len(self.outcomes) < len(self.nodes)
if (incomplete or "skipped" in self.outcomes.values()) and session.exitstatus in {
pytest.ExitCode.OK,
pytest.ExitCode.NO_TESTS_COLLECTED,
}:
session.exitstatus = pytest.ExitCode.TESTS_FAILED

def pytest_terminal_summary(self, terminalreporter):
if not self.nodes and os.environ.get("PT_SNAP_REQUIRE_BROWSER") != "1":
return
counts = {
outcome: list(self.outcomes.values()).count(outcome)
for outcome in ("passed", "skipped", "failed")
}
mode = "required" if os.environ.get("PT_SNAP_REQUIRE_BROWSER") == "1" else "optional"
terminalreporter.section(f"Browser regression execution ({mode})")
terminalreporter.write_line(
f"collected={len(self.nodes)} passed={counts['passed']} skipped={counts['skipped']} "
f"failed={counts['failed']} not_run={len(self.nodes) - len(self.outcomes)}"
)
if not self.nodes or len(self.outcomes) < len(self.nodes):
terminalreporter.write_line(
"Browser regressions NOT EXECUTED: collection/deselection or early termination is not browser execution."
)
if counts["skipped"]:
terminalreporter.write_line(
"Browser regressions NOT EXECUTED for skipped cases; this is not a browser pass."
)
for reason in sorted(self.skip_reasons):
terminalreporter.write_line(reason)


def pytest_configure(config):
config.addinivalue_line(
"markers", "browser: real Chromium interaction regressions (required in Linux browser CI)"
)
config.pluginmanager.register(BrowserSummary(), "browser-execution-summary")
Loading
Loading