Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ Parent scope: [repository](../AGENTS.md).
| `pull_request_template.md` | Default PR template and machine-parsed snapshot decision labels |
| `scripts/check_snapshot_provenance.py` | Base/head Git-blob comparison and PR declaration validation |
| `scripts/clean_wheel_agent_acceptance.py` | Supplied wheel acceptance in temporary home/CWD/venv |
| `scripts/run_differential_acceptance.py` | Source-query 200-seed/v1-v2 acceptance with exact executed identities and coverage evidence |
| `scripts/run_dataset_acceptance.py` | Required installed-wheel dataset test with fail-closed JUnit and artifact evidence |

## Provenance Guard
Expand Down
194 changes: 194 additions & 0 deletions .github/scripts/run_differential_acceptance.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
"""Run the required paired seed profile and reject skipped, partial or stale evidence."""

from __future__ import annotations

import argparse
import json
import os
import subprocess
import sys
import xml.etree.ElementTree as ET
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
TEST_FILE = "tests/core/test_dataset_realistic.py"
TEST_NAME = "test_seeded_lifecycle_windows_match_single_database_strictly"
TEST_NODE = f"{TEST_FILE}::{TEST_NAME}"


class AcceptanceError(RuntimeError):
pass


def expected_records(seed_count: int) -> dict[tuple[int, str], dict]:
# This runner is intentionally source-test acceptance, separate from the
# installed-wheel gate. Import the exact same deterministic profile contract.
sys.path[:0] = [str(ROOT), str(ROOT / "src")]
try:
from tests.core.dataset_differential_cases import (
SUPPORTED_TEMPLATES,
capacity_for_seed,
configured_seeds,
query_cases,
)
from tests.core.dataset_lifecycle_fixtures import EVENT_COUNT
finally:
del sys.path[:2]
seeds = configured_seeds({"PT_SNAP_DIFFERENTIAL_SEEDS": str(seed_count)})
return {
(seed, layout): {
"seed": seed,
"format": layout,
"capacity": capacity_for_seed(seed),
"shards": (EVENT_COUNT + capacity_for_seed(seed) - 1) // capacity_for_seed(seed),
"comparisons": len(query_cases(seed, capacity_for_seed(seed))),
"unexpected_errors": 0,
"templates": sorted(SUPPORTED_TEMPLATES),
}
for seed in seeds
for layout in ("v1", "v2")
}


def read_evidence(report: Path, temp_dir: Path, seed_count: int) -> dict:
expected = expected_records(seed_count)
if not report.is_file():
raise AcceptanceError("pytest did not write its JUnit report")
cases = ET.parse(report).findall(".//testcase")
names = [case.get("name") for case in cases]
expected_names = {f"{TEST_NAME}[{layout}-seed-{seed}]" for seed, layout in expected}
if len(names) != len(expected_names) or set(names) != expected_names:
raise AcceptanceError(f"Expected exactly {len(expected_names)} distinct seed/format tests")
case_by_key = {}
for case in cases:
if case.get("file") != TEST_FILE or case.get("classname") != TEST_FILE[:-3].replace(
"/", "."
):
raise AcceptanceError("JUnit seed case has the wrong source identity")
if any(case.find(tag) is not None for tag in ("failure", "error", "skipped")):
raise AcceptanceError(
f"Every seed/format test must pass without skips: {case.get('name')}"
)
properties = case.findall("./properties/property")
values = {p.get("name"): p.get("value") for p in properties}
if len(values) != len(properties):
raise AcceptanceError("Duplicate JUnit evidence properties")
try:
key = (int(values["seed"]), values["format"])
except (KeyError, TypeError, ValueError) as exc:
raise AcceptanceError("Missing or invalid seed/format properties") from exc
if key not in expected or case.get("name") != f"{TEST_NAME}[{key[1]}-seed-{key[0]}]":
raise AcceptanceError("JUnit seed properties do not match the executed test identity")
case_by_key[key] = values
paths = list(temp_dir.rglob("coverage.json"))
if len(paths) != len(expected):
raise AcceptanceError("Missing or extra per-seed coverage artifacts")
records = {}
for path in paths:
value = json.loads(path.read_text())
if not isinstance(value, dict):
raise AcceptanceError("Invalid per-seed coverage object")
if type(value.get("seed")) is not int or type(value.get("format")) is not str:
raise AcceptanceError("Invalid per-seed coverage identity types")
key = (value["seed"], value["format"])
if key not in expected or key in records:
raise AcceptanceError("Duplicate or unexpected per-seed coverage identity")
wanted = expected[key]
for field, required in wanted.items():
if type(value.get(field)) is not type(required) or value[field] != required:
raise AcceptanceError(f"Invalid seed coverage {key}: {field}")
builds = value.get("scratch_builds")
if type(builds) is not int or builds < 0:
raise AcceptanceError("Invalid observed scratch build count")
for field in (*wanted, "scratch_builds"):
if case_by_key[key].get(field) != str(value[field]):
raise AcceptanceError(f"JUnit and artifact evidence disagree for {key}: {field}")
if (path.parent / "differential-failure.json").exists():
raise AcceptanceError("A seed retained unexpected failure evidence")
records[key] = value
if not any(value["scratch_builds"] for value in records.values()):
raise AcceptanceError("Profile did not execute the scratch reducer")
return {
"seed_count": seed_count,
"seed_format_cases": len(records),
"seeds": sorted({seed for seed, _ in records}),
"formats": ["v1", "v2"],
"test_nodes": [f"{TEST_FILE}::{name}" for name in sorted(names)],
"comparisons": sum(value["comparisons"] for value in records.values()),
"unexpected_errors": 0,
"scratch_builds": sum(value["scratch_builds"] for value in records.values()),
"cases_with_scratch": sum(value["scratch_builds"] > 0 for value in records.values()),
"templates": next(iter(expected.values()))["templates"],
}


def run_acceptance(output_dir: Path, seed_count: int = 200) -> None:
output_dir = output_dir.resolve()
output_dir.mkdir(parents=True, exist_ok=True)
report, temp_dir = output_dir / "junit.xml", output_dir / "pytest-temp"
for name in ("junit.xml", "summary.json", "pytest.log"):
(output_dir / name).unlink(missing_ok=True)
command = [
sys.executable,
"-m",
"pytest",
TEST_NODE,
"-o",
"junit_family=xunit1",
f"--junitxml={report}",
f"--basetemp={temp_dir}",
]
env = os.environ.copy()
env.pop("PT_SNAP_DIFFERENTIAL_SEED", None)
env.pop("PYTEST_ADDOPTS", None)
env["PT_SNAP_DIFFERENTIAL_SEEDS"] = str(seed_count)
env["PYTHONPATH"] = str(ROOT / "src")
summary = {
"status": "failed",
"seed_count": seed_count,
"test_node": TEST_NODE,
"command": command,
}
try:
expected_records(seed_count) # Setup errors must replace stale green evidence too.
summary["source_sha"] = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, text=True, capture_output=True
).stdout.strip()
with (output_dir / "pytest.log").open("w") as log:
result = subprocess.run(
command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT
)
print((output_dir / "pytest.log").read_text(), end="", flush=True)
summary["pytest_exit_code"] = result.returncode
if result.returncode:
raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log")
summary["acceptance"] = read_evidence(report, temp_dir, seed_count)
summary["status"] = "passed"
except Exception as exc:
summary["error"] = str(exc)
raise
finally:
(output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n")


def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--seed-count", type=int, default=200)
args = parser.parse_args()
try:
run_acceptance(args.output_dir, args.seed_count)
except (
AcceptanceError,
OSError,
ValueError,
ET.ParseError,
subprocess.CalledProcessError,
) as exc:
print(f"Differential acceptance failed: {exc}", file=sys.stderr)
return 1
return 0


if __name__ == "__main__":
raise SystemExit(main())
22 changes: 12 additions & 10 deletions .github/scripts/run_platform_tests.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Run one of exactly two exhaustive, deterministic Windows non-slow shards."""
"""Run one of exactly four exhaustive, deterministic Windows non-slow shards."""

import argparse
import hashlib
Expand All @@ -8,37 +8,39 @@

import pytest

SHARD_COUNT = 4


def validate_partition(nodeids, partitions):
"""Fail before execution if any collected test is missing, repeated, or unknown."""
if len(nodeids) != len(set(nodeids)):
raise pytest.UsageError("platform collection contains duplicate node IDs")
if len(partitions) != 2 or any(not part for part in partitions):
raise pytest.UsageError("platform partition requires exactly two nonempty shards")
if len(partitions) != SHARD_COUNT or any(not part for part in partitions):
raise pytest.UsageError("platform partition requires exactly four nonempty shards")
if Counter(node for part in partitions for node in part) != Counter(nodeids):
raise pytest.UsageError("platform shards must cover every selected test exactly once")


def partition_nodeids(nodeids):
# Alternating individual cases, rather than whole directories/files, spreads
# expensive parameterized SQLite tests across both fixed-size CI jobs.
# expensive parameterized SQLite tests across the fixed-size CI jobs.
ordered = sorted(nodeids)
partitions = [ordered[0::2], ordered[1::2]]
partitions = [ordered[shard::SHARD_COUNT] for shard in range(SHARD_COUNT)]
validate_partition(nodeids, partitions)
return partitions


class PlatformPartition:
def __init__(self, shard, manifest=Path("platform-partition.json")):
if shard not in (0, 1):
raise pytest.UsageError("platform shard must be 0 or 1")
if shard not in range(SHARD_COUNT):
raise pytest.UsageError("platform shard must be between 0 and 3")
self.shard = shard
self.manifest = manifest

@pytest.hookimpl(hookwrapper=True, tryfirst=True)
def pytest_collection_modifyitems(self, config, items):
# Run after ordinary selection hooks, particularly pytest's -m filter.
# Both jobs collect the entire suite before assigning any non-slow case.
# All jobs collect the entire suite before assigning any non-slow case.
yield
nodeids = [item.nodeid for item in items]
partitions = partition_nodeids(nodeids)
Expand All @@ -56,7 +58,7 @@ def pytest_collection_modifyitems(self, config, items):
"schema_version": 1,
"selection": "not slow",
"shard": self.shard,
"shard_count": 2,
"shard_count": SHARD_COUNT,
"collection_sha256": hashlib.sha256(
"\n".join(ordered).encode("utf-8")
).hexdigest(),
Expand All @@ -74,7 +76,7 @@ def pytest_collection_modifyitems(self, config, items):

def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--shard", required=True, type=int, choices=(0, 1))
parser.add_argument("--shard", required=True, type=int, choices=range(SHARD_COUNT))
parser.add_argument("--collect-only", action="store_true")
args = parser.parse_args(argv)
pytest_args = [
Expand Down
14 changes: 14 additions & 0 deletions .github/workflows/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,3 +78,17 @@ YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py
tests/test_ci_tiers.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from
the repository root. Full CI tests the matrix and installed wheel; static tests
do not prove a release environment is configured or a publication occurred.

### Required paired differential profile

The scheduled/manual `dataset-acceptance` job also runs
`run_differential_acceptance.py --seed-count 200` after the installed-wheel
nightly gate. This separate source-query step is required, with no skip or
continue-on-error escape. Its JUnit report must contain exactly 400 distinct
seed/format test IDs (200 per v1/v2), matching all per-seed coverage artifacts,
expected query counts/templates, zero unexpected errors and actual scratch use.
A single-seed environment override is removed by the runner. Logs, source SHA,
summary, JUnit and compact timeline/coverage/failure JSON are always uploaded from
`.tmp/differential-acceptance/`; regenerated SQLite files stay out of stored artifacts. The ordinary PR matrix retains the default
20-seed profile. This does not make the differential step installed-wheel or
upstream-producer evidence.
6 changes: 6 additions & 0 deletions .github/workflows/platform.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ jobs:
- os: windows-latest
partition: shard-1
command: python .github/scripts/run_platform_tests.py --shard 1
- os: windows-latest
partition: shard-2
command: python .github/scripts/run_platform_tests.py --shard 2
- os: windows-latest
partition: shard-3
command: python .github/scripts/run_platform_tests.py --shard 3
- os: macos-latest
partition: all
command: python -m pytest -m "not slow" --junitxml=platform-tests.xml -o junit_family=legacy
Expand Down
21 changes: 21 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,11 @@ jobs:
python .github/scripts/run_dataset_acceptance.py \
--wheel "$WHEEL" --output-dir .tmp/dataset-acceptance

- name: Run required 200-seed paired differential acceptance
run: |
python .github/scripts/run_differential_acceptance.py \
--seed-count 200 --output-dir .tmp/differential-acceptance

- name: Upload dataset acceptance evidence
if: always()
uses: actions/upload-artifact@v4
Expand All @@ -219,3 +224,19 @@ jobs:
include-hidden-files: true
if-no-files-found: error
retention-days: 30

- name: Upload paired differential evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: differential-acceptance-${{ github.sha }}
path: |
.tmp/differential-acceptance/*.log
.tmp/differential-acceptance/*.xml
.tmp/differential-acceptance/summary.json
.tmp/differential-acceptance/pytest-temp/**/coverage.json
.tmp/differential-acceptance/pytest-temp/**/timeline.json
.tmp/differential-acceptance/pytest-temp/**/differential-failure.json
include-hidden-files: true
if-no-files-found: error
retention-days: 30
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -166,13 +166,13 @@ non-editable installed exporter tests are described in the
[dataset acceptance guide](docs/en/dataset-acceptance.md).

The supplemental platform workflow runs Python 3.13 non-slow tests on macOS and
Windows. Windows uses two deterministic, disjoint shards of the full non-slow
collection; macOS runs it in one job. All three jobs retain a 20-minute limit,
Windows. Windows uses four deterministic, disjoint shards of the full non-slow
collection; macOS runs it in one job. All five jobs retain a 20-minute limit,
with at most two running concurrently. Each Windows shard validates exhaustive
coverage before execution and uploads its full collection/partition manifest
alongside JUnit results. No tests are skipped to meet the time limit. To reproduce
one Windows partition locally, run
`python .github/scripts/run_platform_tests.py --shard 0` (or `--shard 1`);
`python .github/scripts/run_platform_tests.py --shard 0` (choose `--shard 0` through `--shard 3`);
add `--collect-only` to inspect the partition without executing tests.

### Building distributions
Expand Down
6 changes: 3 additions & 3 deletions README_zh.md
Original file line number Diff line number Diff line change
Expand Up @@ -148,11 +148,11 @@ black --check . && ruff check . # 检查格式和 lint
[数据集验收指南](docs/zh/dataset-acceptance.md)。

补充的平台工作流在 macOS 和 Windows 上运行 Python 3.13 非 slow 测试。
Windows 将完整的非 slow 集合确定性地划分为两个互不重叠的分片;macOS 使用一个任务。
三个任务均保持 20 分钟上限,最多同时运行两个。每个 Windows 分片在执行前验证
Windows 将完整的非 slow 集合确定性地划分为四个互不重叠的分片;macOS 使用一个任务。
五个任务均保持 20 分钟上限,最多同时运行两个。每个 Windows 分片在执行前验证
划分无遗漏、无重复,并随 JUnit 结果上传完整的收集及分片清单。不会为了满足时间上限
而跳过测试。本地复现一个 Windows 分片可运行
`python .github/scripts/run_platform_tests.py --shard 0`(或 `--shard 1`);
`python .github/scripts/run_platform_tests.py --shard 0`(可选 `--shard 0` 至 `--shard 3`);
增加 `--collect-only` 可仅检查划分,不执行测试。

### 构建分发包
Expand Down
Loading
Loading