Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Parent scope: [repository](../AGENTS.md).
| `pull_request_template.md` | Default PR template and machine-parsed snapshot decision labels |
| `scripts/check_snapshot_provenance.py` | Base/head Git-blob comparison and PR declaration validation |
| `scripts/clean_wheel_agent_acceptance.py` | Supplied wheel acceptance in temporary home/CWD/venv |
| `scripts/run_dataset_acceptance.py` | Required installed-wheel dataset test with fail-closed JUnit and artifact evidence |

## Provenance Guard

Expand Down
160 changes: 160 additions & 0 deletions .github/scripts/run_dataset_acceptance.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
"""Run the installed-wheel gate and reject empty, skipped or stale evidence.

This selects nightly tests and requires the public CLI/API wheel case. It does
not claim clean dependency isolation or complete default-scale coverage; those
are contracts of the selected test, not of this CI runner.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import subprocess
import sys
import xml.etree.ElementTree as ET
from pathlib import Path, PurePosixPath

ROOT = Path(__file__).resolve().parents[2]
TEST_NAME = "test_installed_trusted_producer_default_export_cli_and_api"
TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}"
# Extend this tuple alongside new mandatory nightly gates. Never list future
# files that have not landed: the runner rejects any missing configured node.
REQUIRED_TEST_NODES = (TEST_NODE,)


class AcceptanceError(RuntimeError):
pass


def test_node_id(case: ET.Element) -> str:
"""Recover a complete pytest identity from the required xunit1 fields."""
file = case.get("file", "")
name = case.get("name", "")
classname = case.get("classname", "")
path = PurePosixPath(file)
if not name or path.is_absolute() or ".." in path.parts or path.suffix != ".py":
raise AcceptanceError(f"JUnit testcase has no valid source identity: {case.attrib}")
module = file.removesuffix(".py").replace("/", ".")
if classname == module:
classes = ""
elif classname.startswith(module + "."):
classes = classname[len(module) + 1 :].replace(".", "::") + "::"
else:
raise AcceptanceError(f"JUnit testcase has an inconsistent class identity: {case.attrib}")
return f"{file}::{classes}{name}"


def read_cases(report: Path) -> list[ET.Element]:
if not report.is_file():
raise AcceptanceError("pytest did not write its JUnit report")
return ET.parse(report).findall(".//testcase")


def read_evidence(report: Path, wheel_sha256: str) -> dict:
"""Require the wheel case plus every registered gate, with no silent skips."""
cases = read_cases(report)
nodes = [test_node_id(case) for case in cases]
if nodes.count(TEST_NODE) != 1:
raise AcceptanceError(f"Expected exactly one {TEST_NODE} result")
if len(nodes) != len(set(nodes)):
raise AcceptanceError("Duplicate nightly testcase identities")
missing = set(REQUIRED_TEST_NODES) - set(nodes)
if missing:
raise AcceptanceError(f"Missing required nightly test nodes: {sorted(missing)}")
for node, case in zip(nodes, cases, strict=True):
if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")):
raise AcceptanceError(f"Nightly acceptance must pass, without skips: {node}")
wheel_case = cases[nodes.index(TEST_NODE)]
properties = wheel_case.findall("./properties/property[@name='dataset_acceptance']")
if len(properties) != 1:
raise AcceptanceError("Expected exactly one dataset_acceptance evidence property")
evidence = json.loads(properties[0].get("value", ""))
if not isinstance(evidence, dict) or evidence.get("wheel_sha256") != wheel_sha256:
raise AcceptanceError("Acceptance evidence does not match the supplied wheel SHA-256")
return evidence


def run_acceptance(wheel: Path, output_dir: Path) -> None:
wheel = wheel.resolve()
output_dir = output_dir.resolve()
output_dir.mkdir(parents=True, exist_ok=True)
report = output_dir / "junit.xml"
# Reusing an output directory must never let a previous green run mask this one.
for name in ("junit.xml", "summary.json", "pytest.log"):
(output_dir / name).unlink(missing_ok=True)
command = [
sys.executable,
"-m",
"pytest",
"-m",
"nightly",
"-o",
"junit_family=xunit1",
f"--junitxml={report}",
]
env = os.environ.copy()
env["PT_SNAP_ACCEPTANCE_WHEEL"] = str(wheel)
env["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] = str(output_dir)
summary = {
"wheel": str(wheel),
"test_node": TEST_NODE,
"required_test_nodes": list(REQUIRED_TEST_NODES),
"test_nodes": [],
"command": command,
"status": "failed",
}
try:
if not wheel.is_file() or wheel.suffix != ".whl":
raise AcceptanceError(f"A built wheel is required: {wheel}")
wheel_sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest()
summary["wheel_sha256"] = wheel_sha256
source_sha = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True
).stdout.strip()
summary["source_sha"] = source_sha
print(
f"Running nightly acceptance at {source_sha}; required: {REQUIRED_TEST_NODES}",
flush=True,
)
with (output_dir / "pytest.log").open("w") as log:
result = subprocess.run(
command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT
)
print((output_dir / "pytest.log").read_text(), end="", flush=True)
summary["pytest_exit_code"] = result.returncode
if report.is_file():
summary["test_nodes"] = [test_node_id(case) for case in read_cases(report)]
if result.returncode != 0:
raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log")
summary["acceptance"] = read_evidence(report, wheel_sha256)
summary["status"] = "passed"
except Exception as exc:
summary["error"] = str(exc)
raise
finally:
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")


def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--wheel", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
args = parser.parse_args()
try:
run_acceptance(args.wheel, args.output_dir)
except (
AcceptanceError,
OSError,
ValueError,
ET.ParseError,
subprocess.CalledProcessError,
) as exc:
print(f"Dataset acceptance failed: {exc}", file=sys.stderr)
return 1
return 0


if __name__ == "__main__":
raise SystemExit(main())
39 changes: 35 additions & 4 deletions .github/workflows/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,53 @@ Parent scope: [automation](../AGENTS.md).
## Test Workflow

`test.yml` runs for main pushes, main-targeted PR opened/synchronize/reopened/edited
events, and `workflow_call`. Provenance, lint and Python test-matrix jobs are
independent. Concurrency is keyed by GitHub ref and superseded runs are cancelled.
events, `workflow_call`, nightly at 03:17 UTC, and `workflow_dispatch`. Provenance,
lint, browser and Python test-matrix jobs are independent. Checkouts use the PR
head SHA or the triggering SHA; manual dispatch checks the selected ref's SHA.
Concurrency is keyed by GitHub ref and superseded runs are cancelled.

| Job/step | Gate |
| --- | --- |
| Snapshot provenance | Full Git history; base/head comparison; PR decision required only in PR mode |
| Lint & Format | Python 3.13, editable dev install, Ruff, Black, basedpyright with the selected interpreter |
| Test matrix | Python 3.10–3.13; fixture provenance first, then pytest with branch coverage |
| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow and not nightly"` |
| Full PR test tier | Python 3.13; `pytest -m "not nightly"` with branch coverage, preserving the existing slow tests |
| Overall coverage | `pyproject.toml` fail-under 90 with branches enabled |
| Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently |
| Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance |
| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then `nightly` tests including the required installed-wheel CLI/API test |

Million-event deep-offset pagination tests are marked `slow`; their small
pagination contracts still run across the compatibility matrix.

Coverage reporting writes `.coverage`/`coverage.xml`; building writes `dist/` and
intermediates. Dependency installation/build isolation may access the network.
The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses temporary
paths but imports a reviewed executable fixture. These are more than YAML checks.

The nightly/manual job invokes `run_dataset_acceptance.py` with the single built
wheel. All `nightly` tests are selected automatically; the installed-wheel case
must appear exactly once. New mandatory gates also add their complete pytest
node IDs to `REQUIRED_TEST_NODES` in that runner. Do not register absent future
files. Missing wheels, missing/empty JUnit, missing required nodes, duplicate
identities, any skipped/failed/error test, or evidence for a different wheel fail
the gate. It uploads the pytest log, JUnit (including the test's evidence), and a
summary with complete executed/required test node IDs, actual checkout SHA and wheel
SHA-256, even after failure. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` is available for
additional test-owned artifacts. Local invocation:

```bash
python .github/scripts/run_dataset_acceptance.py \
--wheel dist/<built-wheel>.whl --output-dir .tmp/dataset-acceptance
```

This tier initially exercises the existing 500,002-event `pip --target` acceptance
test. It does not claim a clean venv, two full 500,000-event shards, complete CLI
template coverage, live agents or upstream GUI interoperability. The ordinary
PR tier excludes `nightly` tests; the required nightly/manual runner must never
skip them. Manually dispatch the exact published branch/commit and
verify the run's head SHA and summary; a PR run alone is not nightly evidence.

## Release Workflow

`release.yml` triggers on `v*` tag pushes and enforces:
Expand All @@ -44,6 +74,7 @@ validation. For non-PR provenance checks, tag runs compare against the prior tag

Review `pyproject.toml`, both workflow YAMLs, the provenance/acceptance scripts and
`tests/test_release_workflow.py` together when changing execution order. Validate
YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py` from
YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py
tests/test_ci_tiers.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from
the repository root. Full CI tests the matrix and installed wheel; static tests
do not prove a release environment is configured or a publication occurred.
70 changes: 68 additions & 2 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ name: Test

on:
workflow_call:
workflow_dispatch:
schedule:
- cron: "17 3 * * *"
push:
branches: [main]
pull_request:
Expand All @@ -23,6 +26,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha || github.sha }}

- name: Require provenance evaluation for snapshot runtime changes
if: github.event_name == 'pull_request'
Expand Down Expand Up @@ -56,6 +60,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}

- uses: actions/setup-python@v5
with:
Expand All @@ -78,6 +84,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}
- uses: actions/setup-python@v5
with:
python-version: "3.13"
Expand All @@ -99,6 +107,8 @@ jobs:
python-version: ["3.10", "3.11", "3.12", "3.13"]
steps:
- uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.head.sha || github.sha }}

- uses: actions/setup-python@v5
with:
Expand All @@ -114,10 +124,19 @@ jobs:
- name: Install dependencies
run: pip install -e ".[dev]"

- name: Run tests with coverage
run: pytest tests/test_fixture_provenance.py && pytest --cov --cov-report=xml --cov-report=term-missing
- name: Verify executable fixture provenance
run: pytest tests/test_fixture_provenance.py

- name: Run non-slow compatibility tests
if: matrix.python-version != '3.13'
run: pytest -m "not slow and not nightly"

- name: Run full PR tests with coverage
if: matrix.python-version == '3.13'
run: pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing

- name: Guard critical runtime coverage
if: matrix.python-version == '3.13'
# Separate floors prevent strong modules from masking a weak boundary.
run: |
python -m coverage report --include='*/core/snapshot_import_backend.py' --fail-under=90
Expand Down Expand Up @@ -153,3 +172,50 @@ jobs:
python .github/scripts/clean_wheel_agent_acceptance.py \
--wheel "$WHEEL" \
--fixture tests/fixtures/snapshots/snapshot_with_empty_cache.pkl

dataset-acceptance:
name: Installed-wheel dataset acceptance
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.sha }}

- uses: actions/setup-python@v5
with:
python-version: "3.13"

- name: Install dependencies
run: pip install -e ".[dev]"

- name: Verify executable fixture provenance
run: pytest tests/test_fixture_provenance.py

- name: Build wheel
run: python -m build --wheel

- name: Audit wheel contents against source
run: |
set -euo pipefail
WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])')
python .github/scripts/audit_wheel.py --wheel "$WHEEL"

- name: Run required installed-wheel dataset acceptance
run: |
set -euo pipefail
WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])')
python .github/scripts/run_dataset_acceptance.py \
--wheel "$WHEEL" --output-dir .tmp/dataset-acceptance

- name: Upload dataset acceptance evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: dataset-acceptance-${{ github.sha }}
path: .tmp/dataset-acceptance/
include-hidden-files: true
if-no-files-found: error
retention-days: 30
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ select = ["E", "F", "W", "I", "N", "UP", "B", "C4"]
ignore = ["E501"]

[tool.pytest.ini_options]
markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"]
testpaths = ["tests"]
python_files = ["test_*.py"]
python_functions = ["test_*"]
Expand Down
1 change: 1 addition & 0 deletions tests/core/test_dataset_id_pagination.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,7 @@ def million_event_dataset(tmp_path_factory):
return root


@pytest.mark.slow
@pytest.mark.parametrize("template", ["event", "allocation"])
@pytest.mark.parametrize("direction", ["ASC", "DESC"])
def test_million_offset_keeps_fetched_work_constant(million_event_dataset, template, direction):
Expand Down
1 change: 1 addition & 0 deletions tests/core/test_dataset_scale_acceptance.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,7 @@ def installed_runtime(tmp_path):
return cli, env, module, hashlib.sha256(wheel.read_bytes()).hexdigest()


@pytest.mark.nightly
def test_installed_trusted_producer_default_export_cli_and_api(
tmp_path, installed_runtime, record_property
):
Expand Down
Loading
Loading