From 430acf97ee9f6129d182c88382c15278927f9302 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:48:03 +0800 Subject: [PATCH 1/4] ci: tier compatibility tests and require nightly wheel acceptance --- .github/AGENTS.md | 1 + .github/scripts/run_dataset_acceptance.py | 120 +++++++++++++++++ .github/workflows/AGENTS.md | 35 ++++- .github/workflows/test.yml | 70 +++++++++- tests/core/test_dataset_id_pagination.py | 1 + tests/test_ci_tiers.py | 110 +++++++++++++++ tests/test_dataset_acceptance_runner.py | 157 ++++++++++++++++++++++ tests/test_wheel_audit.py | 6 + 8 files changed, 494 insertions(+), 6 deletions(-) create mode 100644 .github/scripts/run_dataset_acceptance.py create mode 100644 tests/test_ci_tiers.py create mode 100644 tests/test_dataset_acceptance_runner.py diff --git a/.github/AGENTS.md b/.github/AGENTS.md index c4b73b2..e4569fe 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -9,6 +9,7 @@ Parent scope: [repository](../AGENTS.md). | `pull_request_template.md` | Default PR template and machine-parsed snapshot decision labels | | `scripts/check_snapshot_provenance.py` | Base/head Git-blob comparison and PR declaration validation | | `scripts/clean_wheel_agent_acceptance.py` | Supplied wheel acceptance in temporary home/CWD/venv | +| `scripts/run_dataset_acceptance.py` | Required installed-wheel dataset test with fail-closed JUnit and artifact evidence | ## Provenance Guard diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py new file mode 100644 index 0000000..89fc223 --- /dev/null +++ b/.github/scripts/run_dataset_acceptance.py @@ -0,0 +1,120 @@ +"""Run the installed-wheel gate and reject empty, skipped or stale evidence. + +This deliberately selects one existing public CLI/API acceptance test. It does +not claim clean dependency isolation or complete default-scale coverage; those +are contracts of the selected test, not of this CI runner. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TEST_NAME = "test_installed_trusted_producer_default_export_cli_and_api" +TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" + + +class AcceptanceError(RuntimeError): + pass + + +def read_evidence(report: Path, wheel_sha256: str) -> dict: + """A green pytest process alone does not prove the requested test ran.""" + if not report.is_file(): + raise AcceptanceError("pytest did not write its JUnit report") + cases = ET.parse(report).findall(".//testcase") + if len(cases) != 1 or cases[0].get("name") != TEST_NAME: + raise AcceptanceError(f"Expected exactly one {TEST_NAME} result, got {len(cases)}") + case = cases[0] + if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): + raise AcceptanceError("Installed-wheel acceptance must pass, without skips") + properties = case.findall("./properties/property[@name='dataset_acceptance']") + if len(properties) != 1: + raise AcceptanceError("Expected exactly one dataset_acceptance evidence property") + evidence = json.loads(properties[0].get("value", "")) + if not isinstance(evidence, dict) or evidence.get("wheel_sha256") != wheel_sha256: + raise AcceptanceError("Acceptance evidence does not match the supplied wheel SHA-256") + return evidence + + +def run_acceptance(wheel: Path, output_dir: Path) -> None: + wheel = wheel.resolve() + output_dir = output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + report = output_dir / "junit.xml" + # Reusing an output directory must never let a previous green run mask this one. + for name in ("junit.xml", "summary.json", "pytest.log"): + (output_dir / name).unlink(missing_ok=True) + command = [ + sys.executable, + "-m", + "pytest", + TEST_NODE, + "-o", + "junit_family=xunit1", + f"--junitxml={report}", + ] + env = os.environ.copy() + env["PT_SNAP_ACCEPTANCE_WHEEL"] = str(wheel) + env["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] = str(output_dir) + summary = { + "wheel": str(wheel), + "test_node": TEST_NODE, + "command": command, + "status": "failed", + } + try: + if not wheel.is_file() or wheel.suffix != ".whl": + raise AcceptanceError(f"A built wheel is required: {wheel}") + wheel_sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + summary["wheel_sha256"] = wheel_sha256 + source_sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout.strip() + summary["source_sha"] = source_sha + print(f"Running {TEST_NODE} at {source_sha}", flush=True) + with (output_dir / "pytest.log").open("w") as log: + result = subprocess.run( + command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT + ) + print((output_dir / "pytest.log").read_text(), end="", flush=True) + summary["pytest_exit_code"] = result.returncode + if result.returncode != 0: + raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log") + summary["acceptance"] = read_evidence(report, wheel_sha256) + summary["status"] = "passed" + except Exception as exc: + summary["error"] = str(exc) + raise + finally: + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + try: + run_acceptance(args.wheel, args.output_dir) + except ( + AcceptanceError, + OSError, + ValueError, + ET.ParseError, + subprocess.CalledProcessError, + ) as exc: + print(f"Dataset acceptance failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 2959c6d..91e9211 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -5,23 +5,49 @@ Parent scope: [automation](../AGENTS.md). ## Test Workflow `test.yml` runs for main pushes, main-targeted PR opened/synchronize/reopened/edited -events, and `workflow_call`. Provenance, lint and Python test-matrix jobs are -independent. Concurrency is keyed by GitHub ref and superseded runs are cancelled. +events, `workflow_call`, nightly at 03:17 UTC, and `workflow_dispatch`. Provenance, +lint, browser and Python test-matrix jobs are independent. Checkouts use the PR +head SHA or the triggering SHA; manual dispatch checks the selected ref's SHA. +Concurrency is keyed by GitHub ref and superseded runs are cancelled. | Job/step | Gate | | --- | --- | | Snapshot provenance | Full Git history; base/head comparison; PR decision required only in PR mode | | Lint & Format | Python 3.13, editable dev install, Ruff, Black, basedpyright with the selected interpreter | -| Test matrix | Python 3.10–3.13; fixture provenance first, then pytest with branch coverage | +| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow"` | +| Full test tier | Python 3.13; fixture provenance first, then all tests with branch coverage, including slow tests | | Overall coverage | `pyproject.toml` fail-under 90 with branches enabled | | Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently | | Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance | +| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then required installed-wheel CLI/API test | + +Million-event deep-offset pagination tests are marked `slow`; their small +pagination contracts still run across the compatibility matrix. Coverage reporting writes `.coverage`/`coverage.xml`; building writes `dist/` and intermediates. Dependency installation/build isolation may access the network. The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses temporary paths but imports a reviewed executable fixture. These are more than YAML checks. +The nightly/manual job invokes `run_dataset_acceptance.py` with the single built +wheel. Missing wheels, missing/empty JUnit, skipped/failed tests, extra test cases, +or evidence for a different wheel fail the gate. It uploads the pytest log, JUnit +(including the test's evidence), and a summary with actual checkout SHA and wheel +SHA-256, even after failure. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` is available for +additional test-owned artifacts. Local invocation: + +```bash +python .github/scripts/run_dataset_acceptance.py \ + --wheel dist/.whl --output-dir .tmp/dataset-acceptance +``` + +This tier initially exercises the existing 500,002-event `pip --target` acceptance +test. It does not claim a clean venv, two full 500,000-event shards, complete CLI +template coverage, live agents or upstream GUI interoperability. The ordinary +full suite may still skip this opt-in wheel test; the required nightly/manual +runner must never do so. Manually dispatch the exact published branch/commit and +verify the run's head SHA and summary; a PR run alone is not nightly evidence. + ## Release Workflow `release.yml` triggers on `v*` tag pushes and enforces: @@ -44,6 +70,7 @@ validation. For non-PR provenance checks, tag runs compare against the prior tag Review `pyproject.toml`, both workflow YAMLs, the provenance/acceptance scripts and `tests/test_release_workflow.py` together when changing execution order. Validate -YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py` from +YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py +tests/test_ci_tiers.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from the repository root. Full CI tests the matrix and installed wheel; static tests do not prove a release environment is configured or a publication occurred. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1000c8d..e71defe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: Test on: workflow_call: + workflow_dispatch: + schedule: + - cron: "17 3 * * *" push: branches: [main] pull_request: @@ -23,6 +26,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Require provenance evaluation for snapshot runtime changes if: github.event_name == 'pull_request' @@ -56,6 +60,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: @@ -78,6 +84,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: python-version: "3.13" @@ -99,6 +107,8 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: @@ -114,10 +124,19 @@ jobs: - name: Install dependencies run: pip install -e ".[dev]" - - name: Run tests with coverage - run: pytest tests/test_fixture_provenance.py && pytest --cov --cov-report=xml --cov-report=term-missing + - name: Verify executable fixture provenance + run: pytest tests/test_fixture_provenance.py + + - name: Run non-slow compatibility tests + if: matrix.python-version != '3.13' + run: pytest -m "not slow" + + - name: Run all tests with coverage + if: matrix.python-version == '3.13' + run: pytest --cov --cov-report=xml --cov-report=term-missing - name: Guard critical runtime coverage + if: matrix.python-version == '3.13' # Separate floors prevent strong modules from masking a weak boundary. run: | python -m coverage report --include='*/core/snapshot_import_backend.py' --fail-under=90 @@ -153,3 +172,50 @@ jobs: python .github/scripts/clean_wheel_agent_acceptance.py \ --wheel "$WHEEL" \ --fixture tests/fixtures/snapshots/snapshot_with_empty_cache.pkl + + dataset-acceptance: + name: Installed-wheel dataset acceptance + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.sha }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Verify executable fixture provenance + run: pytest tests/test_fixture_provenance.py + + - name: Build wheel + run: python -m build --wheel + + - name: Audit wheel contents against source + run: | + set -euo pipefail + WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') + python .github/scripts/audit_wheel.py --wheel "$WHEEL" + + - name: Run required installed-wheel dataset acceptance + run: | + set -euo pipefail + WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') + python .github/scripts/run_dataset_acceptance.py \ + --wheel "$WHEEL" --output-dir .tmp/dataset-acceptance + + - name: Upload dataset acceptance evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: dataset-acceptance-${{ github.sha }} + path: .tmp/dataset-acceptance/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 diff --git a/tests/core/test_dataset_id_pagination.py b/tests/core/test_dataset_id_pagination.py index df4756c..56e4254 100644 --- a/tests/core/test_dataset_id_pagination.py +++ b/tests/core/test_dataset_id_pagination.py @@ -269,6 +269,7 @@ def million_event_dataset(tmp_path_factory): return root +@pytest.mark.slow @pytest.mark.parametrize("template", ["event", "allocation"]) @pytest.mark.parametrize("direction", ["ASC", "DESC"]) def test_million_offset_keeps_fetched_work_constant(million_event_dataset, template, direction): diff --git a/tests/test_ci_tiers.py b/tests/test_ci_tiers.py new file mode 100644 index 0000000..9a6e10f --- /dev/null +++ b/tests/test_ci_tiers.py @@ -0,0 +1,110 @@ +"""Protect fast/full/nightly boundaries and the existing quality gates.""" + +from pathlib import Path + +import yaml + + +def workflow(): + return yaml.load(Path(".github/workflows/test.yml").read_text(), Loader=yaml.BaseLoader) + + +def named_steps(job): + return {step["name"]: step for step in job["steps"] if "name" in step} + + +def test_compatibility_matrix_leaves_slow_and_coverage_on_python_313(): + job = workflow()["jobs"]["test"] + assert job["strategy"]["matrix"]["python-version"] == ["3.10", "3.11", "3.12", "3.13"] + steps = named_steps(job) + fast = steps["Run non-slow compatibility tests"] + assert fast["if"] == "matrix.python-version != '3.13'" + assert fast["run"] == 'pytest -m "not slow"' + full = steps["Run all tests with coverage"] + assert full["if"] == "matrix.python-version == '3.13'" + assert full["run"] == "pytest --cov --cov-report=xml --cov-report=term-missing" + assert "if" not in steps["Verify executable fixture provenance"] + names = list(steps) + assert names.index("Verify executable fixture provenance") < names.index(fast["name"]) + assert names.index("Verify executable fixture provenance") < names.index(full["name"]) + + +def test_full_tier_preserves_independent_coverage_floors_and_packaging(): + steps = named_steps(workflow()["jobs"]["test"]) + guard = steps["Guard critical runtime coverage"] + assert guard["if"] == "matrix.python-version == '3.13'" + for file, floor in ( + ("core/snapshot_import_backend.py", 90), + ("simulate/simulated_caching_allocator.py", 88), + ("simulate/snapshot_mutator.py", 84), + ): + assert f"--include='*/{file}' --fail-under={floor}" in guard["run"] + config = Path("pyproject.toml").read_text() + assert "branch = true" in config and "fail_under = 90" in config + for name in ( + "Build wheel", + "Audit wheel contents against source", + "Clean-wheel Agent acceptance", + ): + assert steps[name]["if"] == "matrix.python-version == '3.13'" + assert steps["Upload coverage report"]["if"] == "matrix.python-version == '3.13' && always()" + + +def test_ci_preserves_least_privilege_and_checks_the_published_commit(): + config = workflow() + assert config["permissions"] == {"contents": "read"} + assert "pull_request_target" not in config["on"] + assert config["on"]["pull_request"]["types"] == ["opened", "synchronize", "reopened", "edited"] + assert "workflow_call" in config["on"] + jobs = config["jobs"] + for name in ("snapshot-provenance", "lint", "browser", "test"): + checkout = jobs[name]["steps"][0] + assert checkout["uses"] == "actions/checkout@v4" + assert checkout["with"]["ref"] == "${{ github.event.pull_request.head.sha || github.sha }}" + assert jobs["snapshot-provenance"]["steps"][0]["with"]["fetch-depth"] == "0" + assert ( + "--require-decision" + in named_steps(jobs["snapshot-provenance"])[ + "Require provenance evaluation for snapshot runtime changes" + ]["run"] + ) + browser = named_steps(jobs["browser"])["Run required Chromium interaction tests"] + assert browser["env"]["PT_SNAP_REQUIRE_BROWSER"] == "1" + assert "tests/test_memory_tree_browser.py" in browser["run"] + lint = named_steps(jobs["lint"]) + for name in ("Ruff lint check", "Black format check", "basedPyright check"): + assert name in lint + + +def test_nightly_and_manual_gate_build_and_audit_before_required_acceptance(): + config = workflow() + assert "workflow_dispatch" in config["on"] + assert config["on"]["schedule"] == [{"cron": "17 3 * * *"}] + job = config["jobs"]["dataset-acceptance"] + assert ( + job["if"] == "github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'" + ) + assert job["timeout-minutes"] == "30" + assert job["steps"][0]["with"] == {"fetch-depth": "0", "ref": "${{ github.sha }}"} + steps = named_steps(job) + names = list(steps) + order = [ + "Verify executable fixture provenance", + "Build wheel", + "Audit wheel contents against source", + "Run required installed-wheel dataset acceptance", + "Upload dataset acceptance evidence", + ] + assert [names.index(name) for name in order] == sorted(names.index(name) for name in order) + acceptance = steps["Run required installed-wheel dataset acceptance"]["run"] + assert "assert len(wheels) == 1" in acceptance + assert "python .github/scripts/run_dataset_acceptance.py" in acceptance + assert '--wheel "$WHEEL" --output-dir .tmp/dataset-acceptance' in acceptance + for step in job["steps"]: + assert "continue-on-error" not in step + upload = steps["Upload dataset acceptance evidence"] + assert upload["if"] == "always()" + assert upload["with"]["if-no-files-found"] == "error" + assert upload["with"]["include-hidden-files"] == "true" + assert upload["with"]["path"] == ".tmp/dataset-acceptance/" + assert "${{ github.sha }}" in upload["with"]["name"] diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py new file mode 100644 index 0000000..36ff22e --- /dev/null +++ b/tests/test_dataset_acceptance_runner.py @@ -0,0 +1,157 @@ +"""Fail-closed CI wrapper contracts without repeating the expensive scale test.""" + +import hashlib +import importlib.util +import json +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +RUNNER = Path(".github/scripts/run_dataset_acceptance.py") + + +@pytest.fixture +def runner(): + spec = importlib.util.spec_from_file_location("dataset_acceptance_runner", RUNNER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def write_report(path, name, evidence, outcome=None): + suite = ET.Element("testsuite") + case = ET.SubElement(suite, "testcase", name=name) + if outcome: + ET.SubElement(case, outcome) + if evidence is not None: + properties = ET.SubElement(case, "properties") + ET.SubElement(properties, "property", name="dataset_acceptance", value=json.dumps(evidence)) + ET.ElementTree(suite).write(path) + + +def test_runner_accepts_matching_wheel_evidence(runner, tmp_path): + report = tmp_path / "junit.xml" + evidence = {"wheel_sha256": "abc", "events": 500_002} + write_report(report, runner.TEST_NAME, evidence) + assert runner.read_evidence(report, "abc") == evidence + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_nonpassing_acceptance(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}, outcome) + with pytest.raises(runner.AcceptanceError, match="without skips"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("evidence", [None, {}, {"wheel_sha256": "other"}, []]) +def test_runner_rejects_missing_or_wrong_artifact_evidence(runner, tmp_path, evidence): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, evidence) + with pytest.raises(runner.AcceptanceError): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("count", [0, 2]) +def test_runner_rejects_empty_or_extra_collection(runner, tmp_path, count): + report = tmp_path / "junit.xml" + suite = ET.Element("testsuite") + for _ in range(count): + ET.SubElement(suite, "testcase", name=runner.TEST_NAME) + ET.ElementTree(suite).write(report) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +def test_runner_rejects_wrong_test_even_if_it_passes(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, "unrelated_test", {"wheel_sha256": "abc"}) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +def test_runner_rejects_duplicate_evidence(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + tree = ET.parse(report) + properties = tree.find(".//properties") + assert properties is not None + ET.SubElement(properties, "property", name="dataset_acceptance", value="{}") + tree.write(report) + with pytest.raises(runner.AcceptanceError, match="exactly one dataset_acceptance"): + runner.read_evidence(report, "abc") + + +def test_missing_wheel_fails_before_pytest(tmp_path): + output = tmp_path / "results" + output.mkdir() + (output / "summary.json").write_text('{"status": "passed"}') + (output / "junit.xml").write_text("old passing report") + (output / "pytest.log").write_text("old passing output") + result = subprocess.run( + [ + sys.executable, + str(RUNNER), + "--wheel", + str(tmp_path / "missing.whl"), + "--output-dir", + str(tmp_path / "results"), + ], + capture_output=True, + text=True, + ) + assert result.returncode == 1 + assert "A built wheel is required" in result.stderr + assert not (output / "junit.xml").exists() + assert not (output / "pytest.log").exists() + summary = json.loads((output / "summary.json").read_text()) + assert summary["status"] == "failed" + assert "A built wheel is required" in summary["error"] + + +@pytest.mark.parametrize("exit_code,write_junit", [(0, True), (0, False), (1, True), (5, False)]) +def test_runner_records_checkout_and_artifact_with_fail_closed_results( + runner, tmp_path, monkeypatch, exit_code, write_junit +): + wheel = tmp_path / "built.whl" + wheel.write_bytes(b"test artifact") + wheel_hash = hashlib.sha256(wheel.read_bytes()).hexdigest() + output = tmp_path / "results" + output.mkdir() + report = output / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": wheel_hash}) + calls = [] + + def run(command, **kwargs): + calls.append(command) + assert kwargs["cwd"] == runner.ROOT + if command[0] == "git": + return subprocess.CompletedProcess(command, 0, stdout="checkout-sha\n") + assert command[:3] == [sys.executable, "-m", "pytest"] + assert runner.TEST_NODE in command + assert "junit_family=xunit1" in command + assert kwargs["env"]["PT_SNAP_ACCEPTANCE_WHEEL"] == str(wheel) + assert kwargs["env"]["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] == str(output) + assert not report.exists(), "stale JUnit must be removed before this run" + kwargs["stdout"].write("captured pytest output\n") + if write_junit: + write_report(report, runner.TEST_NAME, {"wheel_sha256": wheel_hash}) + return subprocess.CompletedProcess(command, exit_code) + + monkeypatch.setattr(runner.subprocess, "run", run) + if exit_code or not write_junit: + with pytest.raises(runner.AcceptanceError): + runner.run_acceptance(wheel, output) + else: + runner.run_acceptance(wheel, output) + assert len(calls) == 2 + summary = json.loads((output / "summary.json").read_text()) + assert summary["source_sha"] == "checkout-sha" + assert summary["wheel_sha256"] == wheel_hash + assert summary["pytest_exit_code"] == exit_code + assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") + assert (output / "pytest.log").read_text() == "captured pytest output\n" diff --git a/tests/test_wheel_audit.py b/tests/test_wheel_audit.py index 15bb04c..4bc413a 100644 --- a/tests/test_wheel_audit.py +++ b/tests/test_wheel_audit.py @@ -177,6 +177,12 @@ def build(directory): ("workflow", "job", "build", "next_step"), [ ("test.yml", "test", "Build wheel", "Clean-wheel Agent acceptance"), + ( + "test.yml", + "dataset-acceptance", + "Build wheel", + "Run required installed-wheel dataset acceptance", + ), ("release.yml", "build", "Build sdist and wheel", "Upload build artifacts"), ], ) From 68d89670b188c93e41b1a5d8f41903057e5ada06 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:59:11 +0800 Subject: [PATCH 2/4] ci: enforce explicit nightly selection and required test evidence --- .github/scripts/run_dataset_acceptance.py | 66 ++++++++--- .github/workflows/AGENTS.md | 20 ++-- .github/workflows/test.yml | 6 +- pyproject.toml | 1 + tests/core/test_dataset_scale_acceptance.py | 1 + tests/test_ci_tiers.py | 10 +- tests/test_dataset_acceptance_runner.py | 120 +++++++++++++++++++- 7 files changed, 194 insertions(+), 30 deletions(-) diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py index 89fc223..d1d8512 100644 --- a/.github/scripts/run_dataset_acceptance.py +++ b/.github/scripts/run_dataset_acceptance.py @@ -1,6 +1,6 @@ """Run the installed-wheel gate and reject empty, skipped or stale evidence. -This deliberately selects one existing public CLI/API acceptance test. It does +This selects nightly tests and requires the public CLI/API wheel case. It does not claim clean dependency isolation or complete default-scale coverage; those are contracts of the selected test, not of this CI runner. """ @@ -14,28 +14,60 @@ import subprocess import sys import xml.etree.ElementTree as ET -from pathlib import Path +from pathlib import Path, PurePosixPath ROOT = Path(__file__).resolve().parents[2] TEST_NAME = "test_installed_trusted_producer_default_export_cli_and_api" TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" +# Extend this tuple alongside new mandatory nightly gates. Never list future +# files that have not landed: the runner rejects any missing configured node. +REQUIRED_TEST_NODES = (TEST_NODE,) class AcceptanceError(RuntimeError): pass -def read_evidence(report: Path, wheel_sha256: str) -> dict: - """A green pytest process alone does not prove the requested test ran.""" +def test_node_id(case: ET.Element) -> str: + """Recover a complete pytest identity from the required xunit1 fields.""" + file = case.get("file", "") + name = case.get("name", "") + classname = case.get("classname", "") + path = PurePosixPath(file) + if not name or path.is_absolute() or ".." in path.parts or path.suffix != ".py": + raise AcceptanceError(f"JUnit testcase has no valid source identity: {case.attrib}") + module = file.removesuffix(".py").replace("/", ".") + if classname == module: + classes = "" + elif classname.startswith(module + "."): + classes = classname[len(module) + 1 :].replace(".", "::") + "::" + else: + raise AcceptanceError(f"JUnit testcase has an inconsistent class identity: {case.attrib}") + return f"{file}::{classes}{name}" + + +def read_cases(report: Path) -> list[ET.Element]: if not report.is_file(): raise AcceptanceError("pytest did not write its JUnit report") - cases = ET.parse(report).findall(".//testcase") - if len(cases) != 1 or cases[0].get("name") != TEST_NAME: - raise AcceptanceError(f"Expected exactly one {TEST_NAME} result, got {len(cases)}") - case = cases[0] - if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): - raise AcceptanceError("Installed-wheel acceptance must pass, without skips") - properties = case.findall("./properties/property[@name='dataset_acceptance']") + return ET.parse(report).findall(".//testcase") + + +def read_evidence(report: Path, wheel_sha256: str) -> dict: + """Require the wheel case plus every registered gate, with no silent skips.""" + cases = read_cases(report) + nodes = [test_node_id(case) for case in cases] + if nodes.count(TEST_NODE) != 1: + raise AcceptanceError(f"Expected exactly one {TEST_NODE} result") + if len(nodes) != len(set(nodes)): + raise AcceptanceError("Duplicate nightly testcase identities") + missing = set(REQUIRED_TEST_NODES) - set(nodes) + if missing: + raise AcceptanceError(f"Missing required nightly test nodes: {sorted(missing)}") + for node, case in zip(nodes, cases, strict=True): + if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): + raise AcceptanceError(f"Nightly acceptance must pass, without skips: {node}") + wheel_case = cases[nodes.index(TEST_NODE)] + properties = wheel_case.findall("./properties/property[@name='dataset_acceptance']") if len(properties) != 1: raise AcceptanceError("Expected exactly one dataset_acceptance evidence property") evidence = json.loads(properties[0].get("value", "")) @@ -56,7 +88,8 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: sys.executable, "-m", "pytest", - TEST_NODE, + "-m", + "nightly", "-o", "junit_family=xunit1", f"--junitxml={report}", @@ -67,6 +100,8 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: summary = { "wheel": str(wheel), "test_node": TEST_NODE, + "required_test_nodes": list(REQUIRED_TEST_NODES), + "test_nodes": [], "command": command, "status": "failed", } @@ -79,13 +114,18 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True ).stdout.strip() summary["source_sha"] = source_sha - print(f"Running {TEST_NODE} at {source_sha}", flush=True) + print( + f"Running nightly acceptance at {source_sha}; required: {REQUIRED_TEST_NODES}", + flush=True, + ) with (output_dir / "pytest.log").open("w") as log: result = subprocess.run( command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT ) print((output_dir / "pytest.log").read_text(), end="", flush=True) summary["pytest_exit_code"] = result.returncode + if report.is_file(): + summary["test_nodes"] = [test_node_id(case) for case in read_cases(report)] if result.returncode != 0: raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log") summary["acceptance"] = read_evidence(report, wheel_sha256) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 91e9211..c2a3c57 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -14,12 +14,12 @@ Concurrency is keyed by GitHub ref and superseded runs are cancelled. | --- | --- | | Snapshot provenance | Full Git history; base/head comparison; PR decision required only in PR mode | | Lint & Format | Python 3.13, editable dev install, Ruff, Black, basedpyright with the selected interpreter | -| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow"` | -| Full test tier | Python 3.13; fixture provenance first, then all tests with branch coverage, including slow tests | +| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow and not nightly"` | +| Full PR test tier | Python 3.13; `pytest -m "not nightly"` with branch coverage, preserving the existing slow tests | | Overall coverage | `pyproject.toml` fail-under 90 with branches enabled | | Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently | | Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance | -| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then required installed-wheel CLI/API test | +| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then `nightly` tests including the required installed-wheel CLI/API test | Million-event deep-offset pagination tests are marked `slow`; their small pagination contracts still run across the compatibility matrix. @@ -30,9 +30,13 @@ The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses tempora paths but imports a reviewed executable fixture. These are more than YAML checks. The nightly/manual job invokes `run_dataset_acceptance.py` with the single built -wheel. Missing wheels, missing/empty JUnit, skipped/failed tests, extra test cases, -or evidence for a different wheel fail the gate. It uploads the pytest log, JUnit -(including the test's evidence), and a summary with actual checkout SHA and wheel +wheel. All `nightly` tests are selected automatically; the installed-wheel case +must appear exactly once. New mandatory gates also add their complete pytest +node IDs to `REQUIRED_TEST_NODES` in that runner. Do not register absent future +files. Missing wheels, missing/empty JUnit, missing required nodes, duplicate +identities, any skipped/failed/error test, or evidence for a different wheel fail +the gate. It uploads the pytest log, JUnit (including the test's evidence), and a +summary with complete executed/required test node IDs, actual checkout SHA and wheel SHA-256, even after failure. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` is available for additional test-owned artifacts. Local invocation: @@ -44,8 +48,8 @@ python .github/scripts/run_dataset_acceptance.py \ This tier initially exercises the existing 500,002-event `pip --target` acceptance test. It does not claim a clean venv, two full 500,000-event shards, complete CLI template coverage, live agents or upstream GUI interoperability. The ordinary -full suite may still skip this opt-in wheel test; the required nightly/manual -runner must never do so. Manually dispatch the exact published branch/commit and +PR tier excludes `nightly` tests; the required nightly/manual runner must never +skip them. Manually dispatch the exact published branch/commit and verify the run's head SHA and summary; a PR run alone is not nightly evidence. ## Release Workflow diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e71defe..0465fca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -129,11 +129,11 @@ jobs: - name: Run non-slow compatibility tests if: matrix.python-version != '3.13' - run: pytest -m "not slow" + run: pytest -m "not slow and not nightly" - - name: Run all tests with coverage + - name: Run full PR tests with coverage if: matrix.python-version == '3.13' - run: pytest --cov --cov-report=xml --cov-report=term-missing + run: pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing - name: Guard critical runtime coverage if: matrix.python-version == '3.13' diff --git a/pyproject.toml b/pyproject.toml index f97f49b..f65fa0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ select = ["E", "F", "W", "I", "N", "UP", "B", "C4"] ignore = ["E501"] [tool.pytest.ini_options] +markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"] testpaths = ["tests"] python_files = ["test_*.py"] python_functions = ["test_*"] diff --git a/tests/core/test_dataset_scale_acceptance.py b/tests/core/test_dataset_scale_acceptance.py index e2d1b0d..8fb8a05 100644 --- a/tests/core/test_dataset_scale_acceptance.py +++ b/tests/core/test_dataset_scale_acceptance.py @@ -226,6 +226,7 @@ def installed_runtime(tmp_path): return cli, env, module, hashlib.sha256(wheel.read_bytes()).hexdigest() +@pytest.mark.nightly def test_installed_trusted_producer_default_export_cli_and_api( tmp_path, installed_runtime, record_property ): diff --git a/tests/test_ci_tiers.py b/tests/test_ci_tiers.py index 9a6e10f..f011586 100644 --- a/tests/test_ci_tiers.py +++ b/tests/test_ci_tiers.py @@ -19,10 +19,14 @@ def test_compatibility_matrix_leaves_slow_and_coverage_on_python_313(): steps = named_steps(job) fast = steps["Run non-slow compatibility tests"] assert fast["if"] == "matrix.python-version != '3.13'" - assert fast["run"] == 'pytest -m "not slow"' - full = steps["Run all tests with coverage"] + assert fast["run"] == 'pytest -m "not slow and not nightly"' + full = steps["Run full PR tests with coverage"] assert full["if"] == "matrix.python-version == '3.13'" - assert full["run"] == "pytest --cov --cov-report=xml --cov-report=term-missing" + assert full["run"] == 'pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing' + assert ( + 'markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"]' + in Path("pyproject.toml").read_text() + ) assert "if" not in steps["Verify executable fixture provenance"] names = list(steps) assert names.index("Verify executable fixture provenance") < names.index(fast["name"]) diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py index 36ff22e..037e488 100644 --- a/tests/test_dataset_acceptance_runner.py +++ b/tests/test_dataset_acceptance_runner.py @@ -24,7 +24,13 @@ def runner(): def write_report(path, name, evidence, outcome=None): suite = ET.Element("testsuite") - case = ET.SubElement(suite, "testcase", name=name) + case = ET.SubElement( + suite, + "testcase", + name=name, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) if outcome: ET.SubElement(case, outcome) if evidence is not None: @@ -61,7 +67,13 @@ def test_runner_rejects_empty_or_extra_collection(runner, tmp_path, count): report = tmp_path / "junit.xml" suite = ET.Element("testsuite") for _ in range(count): - ET.SubElement(suite, "testcase", name=runner.TEST_NAME) + ET.SubElement( + suite, + "testcase", + name=runner.TEST_NAME, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) ET.ElementTree(suite).write(report) with pytest.raises(runner.AcceptanceError, match="exactly one"): runner.read_evidence(report, "abc") @@ -74,6 +86,106 @@ def test_runner_rejects_wrong_test_even_if_it_passes(runner, tmp_path): runner.read_evidence(report, "abc") +def add_case( + report, + *, + name="test_extra", + file="tests/test_extra.py", + classname="tests.test_extra", + outcome=None, +): + tree = ET.parse(report) + case = ET.SubElement(tree.getroot(), "testcase", name=name, file=file, classname=classname) + if outcome: + ET.SubElement(case, outcome) + tree.write(report) + + +def test_runner_accepts_additional_nightly_cases_and_records_complete_identities(runner, tmp_path): + report = tmp_path / "junit.xml" + evidence = {"wheel_sha256": "abc"} + write_report(report, runner.TEST_NAME, evidence) + add_case(report, name="test_extra[v2]", classname="tests.test_extra.TestRoutes") + assert runner.read_evidence(report, "abc") == evidence + assert [runner.test_node_id(case) for case in runner.read_cases(report)] == [ + runner.TEST_NODE, + "tests/test_extra.py::TestRoutes::test_extra[v2]", + ] + + +def test_runner_rejects_missing_configured_gate(runner, tmp_path, monkeypatch): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + required = "tests/test_extra.py::test_extra" + monkeypatch.setattr(runner, "REQUIRED_TEST_NODES", (runner.TEST_NODE, required)) + with pytest.raises(runner.AcceptanceError, match="Missing required nightly test nodes"): + runner.read_evidence(report, "abc") + add_case(report) + assert runner.read_evidence(report, "abc") == {"wheel_sha256": "abc"} + + +def test_runner_rejects_duplicate_additional_cases(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + add_case(report) + add_case(report) + with pytest.raises(runner.AcceptanceError, match="Duplicate nightly"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_any_nonpassing_additional_case(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + add_case(report, outcome=outcome) + with pytest.raises( + runner.AcceptanceError, match="without skips: tests/test_extra.py::test_extra" + ): + runner.read_evidence(report, "abc") + + +def test_same_function_name_in_another_file_cannot_replace_required_wheel_case(runner, tmp_path): + report = tmp_path / "junit.xml" + ET.ElementTree(ET.Element("testsuite")).write(report) + add_case(report, name=runner.TEST_NAME) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize( + "attributes", [{"file": ""}, {"file": "../tests/test_extra.py"}, {"classname": "wrong"}] +) +def test_runner_rejects_incomplete_or_inconsistent_junit_identity(runner, attributes): + case = ET.Element( + "testcase", file="tests/test_extra.py", classname="tests.test_extra", name="test_extra" + ) + case.attrib.update(attributes) + with pytest.raises(runner.AcceptanceError, match="identity"): + runner.test_node_id(case) + + +def test_nightly_marker_selection_collects_every_configured_gate(runner): + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "--strict-markers", + "-q", + "-o", + "addopts=", + "-m", + "nightly", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + collected = {line for line in result.stdout.splitlines() if "::" in line} + assert set(runner.REQUIRED_TEST_NODES) <= collected + + def test_runner_rejects_duplicate_evidence(runner, tmp_path): report = tmp_path / "junit.xml" write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) @@ -132,7 +244,7 @@ def run(command, **kwargs): if command[0] == "git": return subprocess.CompletedProcess(command, 0, stdout="checkout-sha\n") assert command[:3] == [sys.executable, "-m", "pytest"] - assert runner.TEST_NODE in command + assert command[3:5] == ["-m", "nightly"] assert "junit_family=xunit1" in command assert kwargs["env"]["PT_SNAP_ACCEPTANCE_WHEEL"] == str(wheel) assert kwargs["env"]["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] == str(output) @@ -153,5 +265,7 @@ def run(command, **kwargs): assert summary["source_sha"] == "checkout-sha" assert summary["wheel_sha256"] == wheel_hash assert summary["pytest_exit_code"] == exit_code + assert summary["required_test_nodes"] == [runner.TEST_NODE] + assert summary["test_nodes"] == ([runner.TEST_NODE] if write_junit else []) assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") assert (output / "pytest.log").read_text() == "captured pytest output\n" From 16e7031583f421e86685a8b10041b181b483ee05 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:17:25 +0800 Subject: [PATCH 3/4] test: run clean-wheel dataset CLI acceptance at default export scale --- .github/scripts/run_dataset_acceptance.py | 6 +- .github/workflows/AGENTS.md | 28 +- tests/core/AGENTS.md | 35 ++ tests/core/test_dataset_scale_acceptance.py | 477 ++++++++++++++------ tests/core/test_dataset_scale_workload.py | 66 +++ tests/core/wheel_scale_acceptance.py | 314 +++++++++++++ tests/test_dataset_acceptance_runner.py | 41 +- 7 files changed, 830 insertions(+), 137 deletions(-) create mode 100644 tests/core/test_dataset_scale_workload.py create mode 100644 tests/core/wheel_scale_acceptance.py diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py index d1d8512..397a34f 100644 --- a/.github/scripts/run_dataset_acceptance.py +++ b/.github/scripts/run_dataset_acceptance.py @@ -21,7 +21,11 @@ TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" # Extend this tuple alongside new mandatory nightly gates. Never list future # files that have not landed: the runner rejects any missing configured node. -REQUIRED_TEST_NODES = (TEST_NODE,) +BOUNDARY_TEST_NODE = ( + "tests/core/test_dataset_scale_acceptance.py::" + "test_installed_default_export_short_final_shard_boundary" +) +REQUIRED_TEST_NODES = (TEST_NODE, BOUNDARY_TEST_NODE) class AcceptanceError(RuntimeError): diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index c2a3c57..2e8b4c8 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -45,11 +45,29 @@ python .github/scripts/run_dataset_acceptance.py \ --wheel dist/.whl --output-dir .tmp/dataset-acceptance ``` -This tier initially exercises the existing 500,002-event `pip --target` acceptance -test. It does not claim a clean venv, two full 500,000-event shards, complete CLI -template coverage, live agents or upstream GUI interoperability. The ordinary -PR tier excludes `nightly` tests; the required nightly/manual runner must never -skip them. Manually dispatch the exact published branch/commit and +The required installed-wheel case creates a clean venv without system packages +and runs its actual `pt-snap` console script from isolated HOME/CWD. Its trusted, +test-owned input has exactly 1,000,000 events; the default exporter produces two +full 500,000-event shards. All installed catalog entries are classified: currently +11 dataset-supported templates succeed with installed CLI/API equivalence and +independent known-result checks, while `event_frames` and +`active_memory_frame_tree_at_event` return explicit unsupported-dataset errors. +Both report commands run: `peak-memory` succeeds and `memory-tree` preserves the +corresponding dataset rejection. The allocation/free-request/free-completion +boundary assertions remain covered, alongside static occupancy, offset pages, +terminal survivors and unchanged source/manifest/member hashes. A second required +nightly case separately imports 500,002 events and proves the original final +short shard has exactly two real events, with the crossing completed lifetime +and the final survivor. Its evidence is stored in its own subdirectory so the +two gates cannot overwrite each other's command logs. This adds one half-million +event import plus five query subprocesses to the nightly job. Existing +synthetic scale/default-budget regressions remain separate source-level tests. +`PT_SNAP_ACCEPTANCE_WHEELHOUSE` optionally supplies dependencies for an offline +clean-venv install. Timings are recorded without machine-speed thresholds; this +does not claim live agents, live hardware capture or upstream GUI interoperability. + +The ordinary PR tier excludes `nightly` tests; the required nightly/manual runner +must never skip them. Manually dispatch the exact published branch/commit and verify the run's head SHA and summary; a PR run alone is not nightly evidence. ## Release Workflow diff --git a/tests/core/AGENTS.md b/tests/core/AGENTS.md index 33d02dd..4855dce 100644 --- a/tests/core/AGENTS.md +++ b/tests/core/AGENTS.md @@ -37,3 +37,38 @@ Implementation: [core services](../../src/pt_snap_cli/core/AGENTS.md). Run `pytest tests/core` or the owning file from the repository root. Follow the parent's fixture gate before import/split cases and resource ownership rules for all cached services. Cross-runtime changes additionally use `tests/snapshot/`. + +## Installed Dataset Scale Acceptance + +`test_dataset_scale_workload.py` checks the test-owned allocator workload and +arithmetic oracle on 100 events. `test_dataset_scale_acceptance.py` retains the +synthetic default-budget gates and has one opt-in wheel acceptance node: +`test_installed_trusted_producer_default_export_cli_and_api`. + +Set `PT_SNAP_ACCEPTANCE_WHEEL` to an absolute wheel path. The node creates a fresh +venv without system packages, installs the wheel and its dependencies, clears +source/focus/query-budget overrides, and executes its actual `pt-snap` console +script in isolated HOME/CWD. Optional `PT_SNAP_ACCEPTANCE_WHEELHOUSE` enables +fully offline dependency installation with `--no-index --find-links`. +`PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` preserves subprocess commands, exit codes, +timings, stdout/stderr, versions, wheel/source/member hashes, and acceptance JSON. +The same acceptance is stored in the JUnit `dataset_acceptance` property (use +`-o junit_family=xunit1`). Timings are observations, not machine-speed thresholds. + +The real importer exports exactly 1,000,000 events using the unchanged 500,000 +default capacity. The complete installed catalog must be classified: currently +11 dataset-supported templates succeed with CLI/API result parity and independent +known-result assertions; `event_frames` and `active_memory_frame_tree_at_event` +are explicit dataset rejections. Both report commands run through the console: +`peak-memory` succeeds, while `memory-tree` preserves its documented unsupported +dataset contract. Source and every exported member remain unchanged. This is +trusted test-owned producer/replay acceptance, not live capture or upstream GUI +acceptance. Do not turn a missing wheel/skip into a successful nightly gate. + +The separate required nightly +`test_installed_default_export_short_final_shard_boundary` retains the original +500,002-event shape: 500,000 real events followed by exactly two. It checks exact +terminal member lifetimes and installed CLI/API completion/survival boundaries, +including unchanged hashes. Its `dataset_boundary_acceptance` JUnit property and +per-test evidence subdirectory preserve independent results without overwriting +the million-event gate's evidence. Both cases are mandatory in the runner. diff --git a/tests/core/test_dataset_scale_acceptance.py b/tests/core/test_dataset_scale_acceptance.py index 8fb8a05..415b8ed 100644 --- a/tests/core/test_dataset_scale_acceptance.py +++ b/tests/core/test_dataset_scale_acceptance.py @@ -6,12 +6,9 @@ run it; this is not live PyTorch/NPU capture or upstream msinsight GUI acceptance. """ -import hashlib import json import os import sqlite3 -import subprocess -import sys import time from contextlib import closing from pathlib import Path @@ -24,7 +21,13 @@ from pt_snap_cli.core.errors import QueryExecutionError from pt_snap_cli.core.msinsight_export import DEFAULT_CAPACITY from tests.core.dataset_lifecycle_fixtures import SCHEMA -from tests.snapshot.test_sharded_replay import block, event, segment, write_source +from tests.core.wheel_scale_acceptance import ( + API_QUERIES, + WheelRuntime, + query_cases, + sha256, + write_scale_source, +) from tests.test_dataset_focus import hashes pytestmark = pytest.mark.slow @@ -182,98 +185,174 @@ def test_long_stack_aggregate_and_small_page_succeed_but_full_output_is_byte_bou assert hashes(root) == before -def run(command, root, env): - result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, check=False) - assert result.returncode == 0, f"{command}\n{result.stdout}\n{result.stderr}" - return result - - @pytest.fixture -def installed_runtime(tmp_path): +def installed_runtime(tmp_path, request): wheel_value = os.environ.get("PT_SNAP_ACCEPTANCE_WHEEL") if not wheel_value: - pytest.skip("Set PT_SNAP_ACCEPTANCE_WHEEL to test a non-editable installed artifact") + pytest.skip("Set PT_SNAP_ACCEPTANCE_WHEEL to test a clean-venv installed artifact") wheel = Path(wheel_value).resolve() assert wheel.is_file() and wheel.suffix == ".whl" - target = tmp_path / "installed" - env = os.environ.copy() - env.pop("PYTHONHOME", None) - env.pop("PT_SNAP_DB_PATH", None) - env.pop("PT_SNAP_QUERY_TIMEOUT", None) - env["PYTHONPATH"] = str(target) - run( - [ - sys.executable, - "-m", - "pip", - "install", - "--no-deps", - "--no-index", - "--target", - str(target), - str(wheel), - ], - tmp_path, - env, + evidence_dir = Path(os.environ.get("PT_SNAP_ACCEPTANCE_EVIDENCE_DIR", tmp_path / "evidence")) + if request.node.name != "test_installed_trusted_producer_default_export_cli_and_api": + evidence_dir /= request.node.name + wheelhouse = os.environ.get("PT_SNAP_ACCEPTANCE_WHEELHOUSE") + return WheelRuntime( + tmp_path, wheel, evidence_dir.resolve(), Path(wheelhouse).resolve() if wheelhouse else None ) - probe = run( - [sys.executable, "-c", "import pt_snap_cli; print(pt_snap_cli.__file__)"], tmp_path, env - ) - module = Path(probe.stdout.strip()).resolve() - assert module.is_relative_to(target) - cli = target / "bin" / "pt-snap" - assert cli.is_file() - return cli, env, module, hashlib.sha256(wheel.read_bytes()).hexdigest() + + +def _assert_scale_oracle(queries, count, boundary): + """Independent arithmetic/geometry, not the same API used as its own oracle.""" + for name, value in queries.items(): + assert value["total_is_exact"], name + assert value["returned"] == len(value["rows"]), name + assert value["scope"]["boundary_events_included"] is False, name + scope = value["scope"] + if scope["kind"] == "event": + assert scope["source_coverage"]["range_complete"], name + else: + assert scope["range_complete"], name + peak = queries["memory_peak"]["rows"][0] + assert peak == { + "peak_allocated": 176, + "peak_allocated_event_id": 3, + "peak_active": 176, + "peak_active_event_id": 3, + "peak_reserved": 4096, + "peak_reserved_event_id": 0, + } + gap = queries["allocator_gap"]["rows"][0] + assert gap["reserved_active_gap_at_active_peak"] == 3920 + assert gap["active_at_reserved_peak"] == 112 + assert gap["all_peaks_same_event"] == 0 + for name in ("event", "allocation"): + result = queries[name] + assert [row["id"] for row in result["rows"]] == list(range(boundary - 2, boundary + 3)) + assert result["total"] == count + assert result["has_more"] and result["truncated"] + assert [row["active"] for row in result["rows"]] == [176, 176, 160, 176, 176] + blocks = queries["block"] + assert blocks["total"] == (count - 1) // 3 + 5 + assert [row["id"] for row in blocks["rows"]] == [0, 1, 2, 3, 6] + assert blocks["has_more"] and blocks["truncated"] + crossing = queries["crossing_block"]["rows"] + assert len(crossing) == 1 + assert crossing[0]["allocEventId"] == boundary - 2 + assert crossing[0]["freeEventId"] == boundary + assert crossing[0]["allocation_source"]["slice_index"] == 0 + assert crossing[0]["free_source"]["slice_index"] == 1 + assert crossing[0]["free_source"]["event"]["action"] == 6 + assert queries["freed_block_lifetime"]["rows"] == [ + { + "lifetime_events": "<1k", + "block_count": (count - 4) // 3, + "size_bytes": 16 * ((count - 4) // 3), + } + ] + leaks = queries["leak_detection"] + assert leaks["total"] == 4 and not leaks["has_more"] and not leaks["truncated"] + assert {row["allocEventId"] for row in leaks["rows"]} == {0, 1, 2, count - 1} + assert all(row["terminal_survivor"] for row in leaks["rows"]) + grouped = queries["callstack_analysis"]["rows"] + assert len(grouped) == 6 + assert sum(row["alloc_count"] for row in grouped) == count + assert sum(row["total_size"] for row in grouped) == 16 * count + 64 + expected_stacks = { + "retain_64": (1, 64), + "retain_32": (1, 32), + "retain_16": (1, 16), + "alloc": ((count - 1) // 3, 16 * ((count - 1) // 3)), + "free_requested": ((count - 4) // 3, 16 * ((count - 4) // 3)), + "free_completed": ((count - 4) // 3, 16 * ((count - 4) // 3)), + } + assert { + row["callstack"].rsplit(" ", 1)[1]: (row["alloc_count"], row["total_size"]) + for row in grouped + } == expected_stacks + for name in ("block", "crossing_block", "leak_detection", "freed_block_lifetime"): + coverage = queries[name]["scope"]["source_coverage"] + assert coverage["lifecycles"] == (count - 1) // 3 + 5 + assert coverage["allocation_events_resolved"] == (count - 1) // 3 + 3 + assert coverage["free_events_resolved"] == (count - 4) // 3 + assert coverage["unproved_identities"] == 0 and coverage["allocation_source_complete"] + assert queries["preexisting_live"]["rows"] == [{"block_count": 0, "size_bytes": 0}] + before = queries["active_before"]["rows"] + after = queries["active_after"]["rows"] + assert len(before) == 6 and sum(row["size"] for row in before) == 176 + assert len(after) == 5 and sum(row["size"] for row in after) == 160 + assert [row["allocEventId"] for row in before if row["allocEventId"] >= 0] == [ + 0, + 1, + 2, + boundary - 2, + ] + assert [row["allocEventId"] for row in after if row["allocEventId"] >= 0] == [0, 1, 2] + assert sum(row["size"] for row in before if row["category"] == "static") == 48 + page = queries["active_static_offset"] + assert page["rows"] == before[1:4] and page["total"] == 6 + assert page["has_more"] and page["truncated"] + dynamic = queries["active_dynamic_offset"] + assert [row["id"] for row in dynamic["rows"]] == [1, 2] and dynamic["total"] == 4 + assert dynamic["has_more"] and dynamic["truncated"] + groups = queries["active_groups"]["rows"] + assert sum(row["size_bytes"] for row in groups) == 176 + assert sum(row["block_count"] for row in groups) == 6 + assert next(row for row in groups if row["category"] == "static")["size_bytes"] == 48 @pytest.mark.nightly def test_installed_trusted_producer_default_export_cli_and_api( tmp_path, installed_runtime, record_property ): - cli, env, module, wheel_hash = installed_runtime - # Original, inspectable test-owned data. No committed or external pickle is loaded. - # 500000 ends after free_requested; free_completed is in the next shard. - count = DEFAULT_CAPACITY + 2 - traces = [event(("alloc", "free_requested", "free_completed")[i % 3]) for i in range(count)] - data = {"segments": [segment(0, 64, [block(1000, 16, 16)])], "device_traces": [traces]} - source = write_source(tmp_path, data) - # The installed importer owns its own full input load; do not retain a second - # 500002-event graph in the test process while it runs. - del traces, data - source_hash = hashlib.sha256(source.read_bytes()).hexdigest() - started = time.perf_counter() - imported = json.loads( - run( - [ - sys.executable, - str(cli), - "import", - str(source), - "--format", - "msinsight", - "--output-dir", - str(tmp_path / "exported"), - "--no-focus", - "--json", - ], - tmp_path, - env, - ).stdout + runtime = installed_runtime + assert DEFAULT_CAPACITY == 500_000 + count, boundary = 2 * DEFAULT_CAPACITY, DEFAULT_CAPACITY + source = write_scale_source(runtime.cwd / "input.pkl", count) + source_hash = sha256(source) + (runtime.evidence_dir / "inputs.json").write_text( + json.dumps( + { + "identity": runtime.identity, + "wheel_sha256": runtime.wheel_hash, + "source_sha256": source_hash, + "events": count, + "events_per_slice": boundary, + }, + indent=2, + ) + ) + imported = runtime.cli_json( + "import", + "import", + source, + "--format", + "msinsight", + "--output-dir", + runtime.cwd / "exported", + "--no-focus", + "--json", ) - import_seconds = time.perf_counter() - started root = Path(imported["db_path"]) manifest = json.loads((root / "manifest.json").read_text()) - assert manifest["eventsPerSlice"] == DEFAULT_CAPACITY + assert manifest["eventsPerSlice"] == boundary assert manifest["devices"]["0"]["eventCount"] == count slices = manifest["devices"]["0"]["slices"] assert [(s["startEventId"], s["endEventId"]) for s in slices] == [ (0, 499_999), - (500_000, 500_001), + (500_000, 999_999), ] before = hashes(root) - # Inspect actual producer membership before running any query. + (runtime.evidence_dir / "members-before.json").write_text(json.dumps(before, indent=2)) + # Preserve the old 500002-event boundary regression in the full-size workload: + # first-shard pending-free allocation completes only in the second shard. for item in slices: - with closing(sqlite3.connect(root / item["file"])) as conn: + with closing( + sqlite3.connect((root / item["file"]).as_uri() + "?mode=ro", uri=True) + ) as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM trace_entry_0 WHERE id>=0").fetchone()[0] + == boundary + ) assert ( conn.execute( "SELECT COUNT(*) FROM block_0 WHERE allocEventId>?", (item["endEventId"],) @@ -287,64 +366,208 @@ def test_installed_trusted_producer_default_export_cli_and_api( ).fetchone()[0] == 0 ) - with closing(sqlite3.connect(root / slices[1]["file"])) as conn: - assert conn.execute( - "SELECT allocEventId,freeEventId FROM block_0 ORDER BY id" - ).fetchall() == [(499_998, 500_000), (500_001, -1)] - peak = json.loads( - run( - [ - sys.executable, - str(cli), - "query", - str(root), - "--template-use", - "memory_peak", - "--json", - ], - tmp_path, - env, - ).stdout + assert conn.execute( + "SELECT allocEventId,freeEventId FROM block_0 WHERE id=499998" + ).fetchall() == [(499_998, 500_000)] + catalog = runtime.cli_json("capabilities", "capabilities", "--json") + cases = query_cases(boundary) + by_name = {entry["name"]: entry for entry in catalog["templates"]} + assert {case["template"] for case in cases} == set(by_name) + supported = {name for name, entry in by_name.items() if entry["dataset_support"]["supported"]} + unsupported = set(by_name) - supported + assert unsupported == {"event_frames", "active_memory_frame_tree_at_event"} + cases_file = runtime.cwd / "cases.json" + cases_file.write_text(json.dumps(cases)) + api = runtime.python_json("api", API_QUERIES, root, cases_file, boundary - 1) + assert catalog["ok"] is True and catalog["schema_version"] == 1 + assert {key: catalog[key] for key in api["catalog"]} == api["catalog"] + assert set(api["errors"]) == unsupported + cli_queries = {} + for case in cases: + success = case["template"] in supported + result = runtime.cli_json( + f"query-{case['name']}", + "query", + root, + "--template-use", + case["template"], + "--params", + json.dumps(case["params"]), + "--exact-total", + "--json", + expected=0 if success else 1, + ) + if success: + assert {key: result[key] for key in api["queries"][case["name"]]} == api["queries"][ + case["name"] + ] + cli_queries[case["name"]] = result + else: + assert result["error"]["code"] == api["errors"][case["name"]]["code"] == "QUERY_FAILED" + assert "Unsupported dataset-global" in result["error"]["message"] + _assert_scale_oracle(cli_queries, count, boundary) + peak = runtime.cli_json("report-peak", "report", "peak-memory", root, "--limit", "-1", "--json") + assert peak == api["peak_report"] + assert peak["event_id"] == 3 and peak["included_bytes"] == 176 + assert peak["active_bytes_at_event"] == 176 and peak["coverage_percent"] == 100 + assert peak["total_is_exact"] and not peak["truncated"] + tree = runtime.cli_json( + "report-tree", + "report", + "memory-tree", + root, + "--event-id", + boundary - 1, + "--json", + expected=1, ) - assert peak["rows"][0]["peak_active"] == 16 and peak["rows"][0]["peak_active_event_id"] == 0 - api_script = """ -import json, sys -from pt_snap_cli.api import SnapshotAnalyzer -from pt_snap_cli.core.report_service import ReportService -with SnapshotAnalyzer(sys.argv[1]) as analyzer: - rows = analyzer.execute_query("event", {"min_id": 0, "limit": 5}, exact_total=True) - assert rows["returned"] == 5 and rows["total"] == 500002 - peak = analyzer.execute_query("memory_peak")["rows"][0] - assert peak["peak_active"] == 16 and peak["peak_active_event_id"] == 0 - before = analyzer.execute_query("active_blocks_at_event", {"event_id": 499999}) - after = analyzer.execute_query("active_blocks_at_event", {"event_id": 500000}) - assert [r["allocEventId"] for r in before["rows"]] == [499998] - assert after["rows"] == [] - live = analyzer.execute_query("leak_detection", {"limit": 5}) - assert [r["allocEventId"] for r in live["rows"]] == [500001] -service = ReportService() -try: - report = service.peak_memory_report(sys.argv[1], limit=-1) - assert report.event_id == 0 and report.included_bytes == 16 - assert report.active_bytes_at_event == 16 and report.coverage_percent == 100 -finally: - service.close() -print(json.dumps({"query_total": rows["total"], "peak": peak, "report_bytes": report.included_bytes})) -""" - result = json.loads(run([sys.executable, "-c", api_script, str(root)], tmp_path, env).stdout) - assert result["query_total"] == count + assert tree["error"]["code"] == api["tree_report"]["code"] == "QUERY_FAILED" + assert "Unsupported dataset-global" in tree["error"]["message"] assert hashes(root) == before - assert hashlib.sha256(source.read_bytes()).hexdigest() == source_hash + assert sha256(source) == source_hash + assert not (runtime.cwd / ".pt-snap").exists() + assert not (runtime.home / ".config" / "pt-snap-cli").exists() evidence = { - "scope": "test-owned snapshot producer -> installed pt-snap replay/export -> installed CLI/API; no live collector or upstream GUI", - "module": str(module), - "wheel_sha256": wheel_hash, + "scope": "test-owned snapshot -> clean-venv installed console import/export, query/report and API; no live collector or upstream GUI", + "identity": runtime.identity, + "wheel_sha256": runtime.wheel_hash, "source_sha256": source_hash, + "members_sha256": before, "events": count, - "events_per_slice": DEFAULT_CAPACITY, + "events_per_slice": boundary, "slices": 2, - "import_seconds": import_seconds, - "result": result, + "templates_supported": sorted(supported), + "templates_expected_rejections": sorted(unsupported), + "reports": {"peak-memory": "passed", "memory-tree": "expected dataset rejection"}, + "commands": runtime.commands, + "read_only_verified": True, } record_property("dataset_acceptance", json.dumps(evidence)) - (tmp_path / "acceptance.json").write_text(json.dumps(evidence, indent=2)) + (runtime.evidence_dir / "acceptance.json").write_text(json.dumps(evidence, indent=2)) + + +@pytest.mark.nightly +def test_installed_default_export_short_final_shard_boundary(installed_runtime, record_property): + """Keep the original 500002-event final-two-row shard producer regression.""" + runtime = installed_runtime + assert DEFAULT_CAPACITY == 500_000 + count, boundary = DEFAULT_CAPACITY + 2, DEFAULT_CAPACITY + source = write_scale_source(runtime.cwd / "input.pkl", count) + source_hash = sha256(source) + imported = runtime.cli_json( + "import", + "import", + source, + "--format", + "msinsight", + "--output-dir", + runtime.cwd / "exported", + "--no-focus", + "--json", + ) + root = Path(imported["db_path"]) + manifest = json.loads((root / "manifest.json").read_text()) + assert manifest["eventsPerSlice"] == boundary + device = manifest["devices"]["0"] + assert device["eventCount"] == count and device["sliceCount"] == 2 + assert device["readySlices"] == [0, 1] + slices = device["slices"] + assert [(item["startEventId"], item["endEventId"]) for item in slices] == [ + (0, 499_999), + (500_000, 500_001), + ] + before = hashes(root) + for item, expected_count in zip(slices, (500_000, 2), strict=True): + with closing( + sqlite3.connect((root / item["file"]).as_uri() + "?mode=ro", uri=True) + ) as conn: + assert conn.execute( + "SELECT COUNT(*),MIN(id),MAX(id) FROM trace_entry_0 WHERE id>=0" + ).fetchone() == (expected_count, item["startEventId"], item["endEventId"]) + assert conn.execute( + "SELECT COUNT(*) FROM block_0 WHERE allocEventId>? OR (freeEventId>=0 AND freeEventId=0 ORDER BY id" + ).fetchall() == [(500_000, 6), (500_001, 4)] + assert conn.execute( + "SELECT allocEventId,freeEventId FROM block_0 ORDER BY id" + ).fetchall() == [ + (-1, -1), + (-1, -1), + (0, -1), + (1, -1), + (2, -1), + (499_998, 500_000), + (500_001, -1), + ] + names = {"event", "crossing_block", "active_before", "active_after", "leak_detection"} + cases = [case for case in query_cases(boundary) if case["name"] in names] + cases_file = runtime.cwd / "cases.json" + cases_file.write_text(json.dumps(cases)) + api = runtime.python_json("api", API_QUERIES, root, cases_file, boundary - 1) + assert api["errors"] == {} + results = {} + for case in cases: + result = runtime.cli_json( + f"query-{case['name']}", + "query", + root, + "--template-use", + case["template"], + "--params", + json.dumps(case["params"]), + "--exact-total", + "--json", + ) + assert {key: result[key] for key in api["queries"][case["name"]]} == api["queries"][ + case["name"] + ] + assert result["total_is_exact"] and result["returned"] == len(result["rows"]) + results[case["name"]] = result + tail = results["event"] + assert tail["total"] == count and tail["returned"] == 4 + assert not tail["has_more"] and tail["truncated"] + assert [(row["id"], row["action"], row["active"]) for row in tail["rows"]] == [ + (499_998, 4, 176), + (499_999, 5, 176), + (500_000, 6, 160), + (500_001, 4, 176), + ] + crossing = results["crossing_block"]["rows"] + assert len(crossing) == 1 + assert crossing[0]["allocEventId"] == 499_998 and crossing[0]["freeEventId"] == 500_000 + assert crossing[0]["allocation_source"]["slice_index"] == 0 + assert crossing[0]["free_source"]["slice_index"] == 1 + assert crossing[0]["free_source"]["event"]["id"] == 500_000 + assert crossing[0]["free_source"]["event"]["action"] == 6 + before_rows, after_rows = results["active_before"]["rows"], results["active_after"]["rows"] + assert [row["allocEventId"] for row in before_rows if row["allocEventId"] >= 0] == [ + 0, + 1, + 2, + 499_998, + ] + assert [row["allocEventId"] for row in after_rows if row["allocEventId"] >= 0] == [0, 1, 2] + assert sum(row["size"] for row in before_rows) == 176 + assert sum(row["size"] for row in after_rows) == 160 + leak = results["leak_detection"] + assert leak["total"] == 4 and not leak["has_more"] and not leak["truncated"] + assert {row["allocEventId"] for row in leak["rows"]} == {0, 1, 2, 500_001} + assert hashes(root) == before and sha256(source) == source_hash + evidence = { + "scope": "clean-wheel producer regression: one full default shard and a final two-event shard", + "wheel_sha256": runtime.wheel_hash, + "source_sha256": source_hash, + "members_sha256": before, + "events": count, + "events_per_slice": boundary, + "slice_event_counts": [500_000, 2], + "identity": runtime.identity, + "commands": runtime.commands, + "read_only_verified": True, + } + record_property("dataset_boundary_acceptance", json.dumps(evidence)) + (runtime.evidence_dir / "acceptance.json").write_text(json.dumps(evidence, indent=2)) diff --git a/tests/core/test_dataset_scale_workload.py b/tests/core/test_dataset_scale_workload.py new file mode 100644 index 0000000..dba0232 --- /dev/null +++ b/tests/core/test_dataset_scale_workload.py @@ -0,0 +1,66 @@ +"""Fast independent checks for the nightly clean-wheel scale workload/oracle.""" + +import json +from pathlib import Path + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.models import ImportOptions +from pt_snap_cli.query import registry +from pt_snap_cli.snapshot.representation import load_snapshot_representation +from pt_snap_cli.snapshot.simulate import SimulateDeviceSnapshot +from tests.core.test_dataset_scale_acceptance import _assert_scale_oracle +from tests.core.wheel_scale_acceptance import query_cases, sha256, write_scale_source +from tests.snapshot.helpers import assert_valid_snapshot + + +def test_scale_workload_is_replay_valid_at_every_event(tmp_path): + source = write_scale_source(tmp_path / "input.pkl", 100) + before = sha256(source) + simulator = SimulateDeviceSnapshot(load_snapshot_representation(source), 0) + for remaining in range(100, -1, -1): + assert simulator.replay_until(remaining) + assert_valid_snapshot(simulator.device_snapshot) + expected = ( + (112, 144, 160)[remaining - 1] + if 1 <= remaining <= 3 + else (48 if remaining == 0 else 160 + (16 if remaining % 3 != 0 else 0)) + ) + assert simulator.device_snapshot.total_activated == expected + assert sha256(source) == before + + +def test_small_scale_uses_same_independent_oracle_and_catalog(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + for name in ("PT_SNAP_DB_PATH", "PT_SNAP_QUERY_TIMEOUT"): + monkeypatch.delenv(name, raising=False) + # Other contract tests may register custom queries. Give this source-level + # smoke its own packaged catalog; the wheel subprocess always starts clean. + monkeypatch.setattr(registry._registry, "_queries", {}) + monkeypatch.setattr(registry._registry, "_factories", {}) + monkeypatch.setattr(registry, "_packaged", {}) + registry._load_all_templates() + source = write_scale_source(tmp_path / "input.pkl", 100) + imported = ImportService().import_snapshot( + ImportOptions( + source, + output_dir=tmp_path / "exported", + format="msinsight", + events_per_slice=50, + set_focus=False, + ) + ) + manifest = json.loads((imported.db_path / "manifest.json").read_text()) + assert manifest["devices"]["0"]["eventCount"] == 100 + with SnapshotAnalyzer(imported.db_path) as analyzer: + catalog = {entry["name"]: entry for entry in analyzer.list_capabilities()["templates"]} + cases = query_cases(50) + assert {case["template"] for case in cases} == set(catalog) + values = { + case["name"]: analyzer.execute_query(case["template"], case["params"], exact_total=True) + for case in cases + if catalog[case["template"]]["dataset_support"]["supported"] + } + _assert_scale_oracle(values, 100, 50) diff --git a/tests/core/wheel_scale_acceptance.py b/tests/core/wheel_scale_acceptance.py new file mode 100644 index 0000000..6cdd38c --- /dev/null +++ b/tests/core/wheel_scale_acceptance.py @@ -0,0 +1,314 @@ +"""Clean-wheel subprocess harness and an original, inspectable scale workload. + +Only the fixture generator runs in the checkout. Product imports in probes, +queries, and reports run with the installed venv interpreter and isolated CWD. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import pickle +import subprocess +import time +import venv +from pathlib import Path + +PROBE = """ +import importlib.metadata as metadata +import json, platform, site, sqlite3, sys +from dataclasses import asdict +from pathlib import Path +import pt_snap_cli, typer, yaml, jinja2 +from pt_snap_cli.core.query_limits import QueryLimits +print(json.dumps({ + "python": sys.version, "executable": sys.executable, + "prefix": sys.prefix, "base_prefix": sys.base_prefix, + "platform": platform.platform(), "sqlite": sqlite3.sqlite_version, + "user_site_enabled": site.ENABLE_USER_SITE, "sys_path": sys.path, + "modules": {m.__name__: str(Path(m.__file__).resolve()) + for m in (pt_snap_cli, typer, yaml, jinja2)}, + "versions": {name: metadata.version(name) + for name in ("pt-snap-cli", "typer", "pyyaml", "jinja2")}, + "entry_point": next(e.value for e in metadata.distribution("pt-snap-cli").entry_points + if e.group == "console_scripts" and e.name == "pt-snap"), + "query_limits": asdict(QueryLimits.resolve()), +})) +""" + +API_QUERIES = """ +import json, sys +from dataclasses import asdict +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.error_codes import classify_error +from pt_snap_cli.core.report_service import ReportService +root, cases_file, event = sys.argv[1], sys.argv[2], int(sys.argv[3]) +results, errors = {}, {} +with SnapshotAnalyzer(root) as analyzer: + catalog = analyzer.list_capabilities() + with open(cases_file) as stream: + cases = json.load(stream) + for case in cases: + try: + results[case["name"]] = analyzer.execute_query( + case["template"], case["params"], exact_total=True) + except Exception as exc: + errors[case["name"]] = {"type": type(exc).__name__, + "code": classify_error(exc)[0], "message": str(exc)} +service = ReportService() +try: + peak = asdict(service.peak_memory_report(root, limit=-1)) + try: + tree = {"event_id": event, **asdict(service.memory_tree_report(event, root))} + except Exception as exc: + tree = {"type": type(exc).__name__, "code": classify_error(exc)[0], + "message": str(exc)} +finally: + service.close() +print(json.dumps({"catalog": catalog, "queries": results, "errors": errors, + "peak_report": peak, "tree_report": tree})) +""" + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +class WheelRuntime: + """True installed console entry point, with failure-safe command evidence.""" + + def __init__(self, root, wheel, evidence_dir, wheelhouse=None): + self.root = root + self.home = root / "home" + self.cwd = root / "project" + self.venv = root / "venv" + self.evidence_dir = evidence_dir + self.evidence_dir.mkdir(parents=True, exist_ok=True) + self.home.mkdir() + self.cwd.mkdir() + self.commands = [] + venv.EnvBuilder(with_pip=True, system_site_packages=False).create(self.venv) + bin_dir = self.venv / ("Scripts" if os.name == "nt" else "bin") + self.python = bin_dir / ("python.exe" if os.name == "nt" else "python") + self.cli = bin_dir / ("pt-snap.exe" if os.name == "nt" else "pt-snap") + self.env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("PYTHON", "PT_SNAP_")) + and key not in {"CODEX_HOME", "CLAUDE_CONFIG_DIR", "VIRTUAL_ENV"} + } + self.env.update( + HOME=str(self.home), + USERPROFILE=str(self.home), + XDG_CONFIG_HOME=str(self.home / ".config"), + XDG_CACHE_HOME=str(self.home / ".cache"), + VIRTUAL_ENV=str(self.venv), + PATH=f"{bin_dir}{os.pathsep}{self.env.get('PATH', '')}", + PWD=str(self.cwd), + PYTHONNOUSERSITE="1", + PIP_DISABLE_PIP_VERSION_CHECK="1", + NO_COLOR="1", + ) + install = [str(self.python), "-m", "pip", "install"] + if wheelhouse is not None: + install += ["--no-index", "--find-links", str(wheelhouse)] + self.run("install", [*install, str(wheel)]) + assert self.cli.is_file() + self.identity = self.python_json("identity", PROBE) + assert self.identity["prefix"] != self.identity["base_prefix"] + assert Path(self.identity["prefix"]) == self.venv + assert Path(self.identity["executable"]) == self.python + assert self.identity["user_site_enabled"] is False + for module in self.identity["modules"].values(): + assert Path(module).is_relative_to(self.venv), module + source_root = Path(__file__).resolve().parents[2] + assert all( + not Path(path).resolve().is_relative_to(source_root) + for path in self.identity["sys_path"] + ) + assert self.identity["entry_point"] == "pt_snap_cli.cli:_safe_call" + assert self.identity["query_limits"] == { + "max_work_rows": 100_000, + "max_work_bytes": 64 * 1024 * 1024, + "max_scratch_db_bytes": 512 * 1024 * 1024, + } + self.wheel_hash = sha256(wheel) + + def run(self, label, command, *, expected=0): + started = time.perf_counter() + result = subprocess.run( + command, cwd=self.cwd, env=self.env, capture_output=True, text=True, check=False + ) + self.commands.append( + { + "label": label, + "command": [str(item) for item in command], + "cwd": str(self.cwd), + "home": str(self.home), + "returncode": result.returncode, + "seconds": time.perf_counter() - started, + } + ) + (self.evidence_dir / f"{label}.stdout").write_text(result.stdout, encoding="utf-8") + (self.evidence_dir / f"{label}.stderr").write_text(result.stderr, encoding="utf-8") + (self.evidence_dir / "commands.json").write_text( + json.dumps(self.commands, indent=2), encoding="utf-8" + ) + assert ( + result.returncode == expected + ), f"{command}\nexit={result.returncode}\n{result.stdout}\n{result.stderr}" + return result + + def cli_json(self, label, *arguments, expected=0): + result = self.run(label, [str(self.cli), *map(str, arguments)], expected=expected) + if expected == 0: + assert not result.stderr, result.stderr + return json.loads(result.stdout) + assert not result.stdout, result.stdout + return json.loads(result.stderr) + + def python_json(self, label, script, *arguments): + result = self.run(label, [str(self.python), "-I", "-c", script, *map(str, arguments)]) + assert not result.stderr, result.stderr + return json.loads(result.stdout) + + +def write_scale_source(path, count): + """Replay-valid allocator trace: static occupancy, survivors, repeated reuse. + + Three retained allocations precede 16-byte alloc/request/complete cycles. + count % 3 == 1 leaves the last cycle live. At the default 500000 boundary, + alloc 499998 requests free at 499999 and completes in shard two at 500000. + The final segment is contiguous, including the trailing inactive space. + No external or committed pickle is loaded to construct this test-owned file. + """ + assert count >= 10 and count % 3 == 1 + retained = [(4160, 64), (4224, 32), (4256, 16)] + + def trace(action, address, size, name): + return { + "action": action, + "addr": address, + "size": size, + "stream": 0, + "frames": [ + {"filename": "scale.py", "line": 10, "name": name}, + {"filename": "train.py", "line": 20, "name": "step"}, + ], + } + + events = [trace("alloc", addr, size, f"retain_{size}") for addr, size in retained] + cycle = [ + trace(action, 4096, 16, action) for action in ("alloc", "free_requested", "free_completed") + ] + events.extend(dict(cycle[index % 3]) for index in range(3, count)) + sizes = [16, 32, 16, 64, 32, 16, 4096 - 176] + blocks, address = [], 4096 + for index, size in enumerate(sizes): + blocks.append( + { + "address": address, + "size": size, + "requested_size": size if index < 6 else 0, + "state": "active_allocated" if index < 6 else "inactive", + "frames": [], + } + ) + address += size + data = { + "segments": [ + { + "address": 4096, + "total_size": 4096, + "stream": 0, + "segment_type": "small", + "allocated_size": 176, + "active_size": 176, + "device": 0, + "is_expandable": False, + "frames": [], + "blocks": blocks, + } + ], + "device_traces": [events], + } + with path.open("wb") as stream: + pickle.dump(data, stream, protocol=4) + return path + + +def query_cases(boundary): + """Catalog coverage plus cross-shard identity and real pagination cases.""" + return [ + {"name": name, "template": template, "params": params} + for name, template, params in [ + ("memory_peak", "memory_peak", {}), + ("allocator_gap", "allocator_gap", {}), + ( + "event", + "event", + {"min_id": 0, "limit": 5, "offset": boundary - 2, "order_dir": "ASC"}, + ), + ( + "allocation", + "allocation", + {"min_id": 0, "limit": 5, "offset": boundary - 2, "order_dir": "ASC"}, + ), + ("block", "block", {"limit": 5, "offset": 2, "order_by": "id", "order_dir": "ASC"}), + ("crossing_block", "block", {"id": boundary - 2}), + ("leak_detection", "leak_detection", {"limit": 5}), + ("freed_block_lifetime", "freed_block_lifetime", {}), + ("callstack_analysis", "callstack_analysis", {"min_count": 1}), + ("preexisting_live", "preexisting_live", {"event_id": boundary - 1}), + ( + "active_before", + "active_blocks_at_event", + {"event_id": boundary - 1, "order_by": "id", "order_dir": "ASC"}, + ), + ( + "active_after", + "active_blocks_at_event", + {"event_id": boundary, "order_by": "id", "order_dir": "ASC"}, + ), + ( + "active_static_offset", + "active_blocks_at_event", + { + "event_id": boundary - 1, + "include_static": True, + "order_by": "id", + "order_dir": "ASC", + "offset": 1, + "limit": 3, + }, + ), + ( + "active_dynamic_offset", + "active_blocks_at_event", + { + "event_id": boundary - 1, + "include_static": False, + "order_by": "id", + "order_dir": "ASC", + "offset": 1, + "limit": 2, + }, + ), + ( + "active_groups", + "active_memory_callstack_at_event", + {"event_id": boundary - 1, "top_n": -1}, + ), + ("event_frames", "event_frames", {"event_id": boundary - 1}), + ( + "active_memory_frame_tree_at_event", + "active_memory_frame_tree_at_event", + {"event_id": boundary - 1}, + ), + ] + ] diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py index 037e488..b409185 100644 --- a/tests/test_dataset_acceptance_runner.py +++ b/tests/test_dataset_acceptance_runner.py @@ -11,6 +11,7 @@ import pytest RUNNER = Path(".github/scripts/run_dataset_acceptance.py") +BOUNDARY_NAME = "test_installed_default_export_short_final_shard_boundary" @pytest.fixture @@ -22,7 +23,7 @@ def runner(): return module -def write_report(path, name, evidence, outcome=None): +def write_report(path, name, evidence, outcome=None, *, include_boundary=True): suite = ET.Element("testsuite") case = ET.SubElement( suite, @@ -36,12 +37,20 @@ def write_report(path, name, evidence, outcome=None): if evidence is not None: properties = ET.SubElement(case, "properties") ET.SubElement(properties, "property", name="dataset_acceptance", value=json.dumps(evidence)) + if include_boundary: + ET.SubElement( + suite, + "testcase", + name=BOUNDARY_NAME, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) ET.ElementTree(suite).write(path) def test_runner_accepts_matching_wheel_evidence(runner, tmp_path): report = tmp_path / "junit.xml" - evidence = {"wheel_sha256": "abc", "events": 500_002} + evidence = {"wheel_sha256": "abc", "events": 1_000_000} write_report(report, runner.TEST_NAME, evidence) assert runner.read_evidence(report, "abc") == evidence @@ -109,6 +118,7 @@ def test_runner_accepts_additional_nightly_cases_and_records_complete_identities assert runner.read_evidence(report, "abc") == evidence assert [runner.test_node_id(case) for case in runner.read_cases(report)] == [ runner.TEST_NODE, + runner.BOUNDARY_TEST_NODE, "tests/test_extra.py::TestRoutes::test_extra[v2]", ] @@ -265,7 +275,30 @@ def run(command, **kwargs): assert summary["source_sha"] == "checkout-sha" assert summary["wheel_sha256"] == wheel_hash assert summary["pytest_exit_code"] == exit_code - assert summary["required_test_nodes"] == [runner.TEST_NODE] - assert summary["test_nodes"] == ([runner.TEST_NODE] if write_junit else []) + assert summary["required_test_nodes"] == list(runner.REQUIRED_TEST_NODES) + assert summary["test_nodes"] == (list(runner.REQUIRED_TEST_NODES) if write_junit else []) assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") assert (output / "pytest.log").read_text() == "captured pytest output\n" + + +def test_runner_requires_original_short_final_shard_case(runner, tmp_path): + report = tmp_path / "junit.xml" + assert runner.BOUNDARY_TEST_NODE in runner.REQUIRED_TEST_NODES + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}, include_boundary=False) + with pytest.raises(runner.AcceptanceError, match="Missing required nightly test nodes"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_nonpassing_short_final_shard_case(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}, include_boundary=False) + add_case( + report, + name=BOUNDARY_NAME, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + outcome=outcome, + ) + with pytest.raises(runner.AcceptanceError, match="without skips"): + runner.read_evidence(report, "abc") From 5f95f1de51c68ff88eb235f634368bf2ccb19dcc Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:39:57 +0800 Subject: [PATCH 4/4] docs: align bilingual scale acceptance with isolated million and short-shard gates --- docs/en/dataset-acceptance.md | 46 +++++++++++++++++++++-------------- docs/zh/dataset-acceptance.md | 34 +++++++++++++++++--------- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/docs/en/dataset-acceptance.md b/docs/en/dataset-acceptance.md index d3bccba..05e94f7 100644 --- a/docs/en/dataset-acceptance.md +++ b/docs/en/dataset-acceptance.md @@ -52,12 +52,17 @@ pytest tests/core/test_dataset_realistic.py tests/core/test_dataset_attribution. gate does not claim all large ordered-frame or unlimited-output queries succeed. - A separate 10,000-event / 8-KiB-inline-stack dataset checks aggregate/small-page success and genuine 64-MiB full-output rejection without creating an 8-GB file. -- A trusted, test-owned snapshot with 500,002 alloc/free-request/free-completion - events goes through a non-editable installed `pt-snap import --format msinsight` - with **no capacity override**. The resulting 500,000 + 2 member layout and - cross-boundary lifecycle membership are checked before installed CLI/API peak, - page, terminal-candidate, point-event and report queries. Artifact hashes must - remain unchanged during queries. +- A trusted, test-owned snapshot with 1,000,000 alloc/free-request/free-completion + events goes through the clean-venv installed `pt-snap import --format msinsight` + with **no capacity override**, producing two full 500,000-event members. All + installed catalog templates are classified: 11 dataset-supported templates + have CLI/API parity and independent result checks; `event_frames` and + `active_memory_frame_tree_at_event` retain explicit dataset rejection. The + `peak-memory` report succeeds; `memory-tree` retains its dataset rejection. +- A second required installed-wheel case preserves the original 500,002-event + workload and its 500,000 + 2 member layout. It verifies the crossing completed + lifetime, final survivor, pages and point-event results. Both installed cases + require unchanged input, manifest and member hashes after queries. Build the wheel from the intended clean checkout using the [distribution instructions](../../README.md#building-distributions), then run: @@ -68,18 +73,23 @@ PT_SNAP_ACCEPTANCE_WHEEL="$(realpath dist/.whl)" \ pytest tests/core/test_dataset_scale_acceptance.py --junitxml=dataset-acceptance.xml ``` -Replace `` with the actual wheel name. The test installs that exact -wheel into its temporary directory using `--no-deps --no-index`; subprocesses -clear source `PYTHONPATH` and verify that `pt_snap_cli` resolves inside the -installation. Runtime dependencies are reused from the test interpreter: this -is not a fresh dependency-resolution test. Without `PT_SNAP_ACCEPTANCE_WHEEL`, -only the installed test is explicitly skipped; that skip is not E2E acceptance. -Use `pytest -m 'not slow'` for the regular suite. - -The installed test writes `acceptance.json` under its pytest temporary directory -and a `dataset_acceptance` JUnit property containing the wheel/source SHA-256, -package path, event counts, import duration and query assertions. Duration is -observational; there is no fixed timing assertion or RSS promise. +Replace `` with the actual wheel name. Each installed case creates +its own fresh venv without system packages, installs the exact wheel and runtime +dependencies, clears source `PYTHONPATH`, and checks that the package and +runtime dependencies resolve inside that venv. Dependency installation may use +the package index. Set `PT_SNAP_ACCEPTANCE_WHEELHOUSE` to a directory containing +compatible dependency wheels for an offline `--no-index --find-links` install. +Without `PT_SNAP_ACCEPTANCE_WHEEL`, both installed cases are explicitly skipped; +those skips are not E2E acceptance. The full PR tier uses `pytest -m 'not nightly'`; +the compatibility tier uses `pytest -m 'not slow and not nightly'`. + +The million-event installed case writes `acceptance.json` and a +`dataset_acceptance` JUnit property with wheel/source hashes, runtime identity, +event counts and command/query evidence. The short-final-shard case stores its +own evidence separately. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` selects a persistent +evidence directory; otherwise pytest temporary directories are used. The +required nightly runner rejects a missing or nonpassing case. Durations are +observational; these installed-wheel cases have no fixed timing or RSS promise. ## Evidence limits diff --git a/docs/zh/dataset-acceptance.md b/docs/zh/dataset-acceptance.md index 4a247a7..b718041 100644 --- a/docs/zh/dataset-acceptance.md +++ b/docs/zh/dataset-acceptance.md @@ -40,10 +40,15 @@ pytest tests/core/test_dataset_realistic.py tests/core/test_dataset_attribution. 大规模有序 frame 或无界输出查询均可成功。 - 独立 10,000-event / 8-KiB 内联栈数据集验证聚合/小页成功,以及完整输出触发真实 64-MiB 上限,无需创建 8-GB 数据文件。 -- 受信、测试自行构造的 500,002 个 alloc/free-request/free-completion 事件快照, - 经非 editable 安装的 `pt-snap import --format msinsight`,**不覆盖容量默认值**。 - 先核对 500,000 + 2 成员布局及跨界生命周期,再通过该安装包的 CLI/API 验证峰值、 - 分页、终片候选、定点事件及报告。查询前后产物 hash 必须不变。 +- 受信、测试自行构造的 1,000,000 个 alloc/free-request/free-completion 事件快照, + 经干净 venv 中安装的 `pt-snap import --format msinsight`,**不覆盖容量默认值**, + 生成两个完整的 500,000-event 成员。所有已安装目录模板均被分类:11 个支持数据集的 + 模板核对 CLI/API 一致性及独立结果;`event_frames` 和 + `active_memory_frame_tree_at_event` 保留明确的数据集拒绝。`peak-memory` 报告成功, + `memory-tree` 保留数据集拒绝。 +- 第二个必需的安装包用例保留原有 500,002-event 工作负载及 500,000 + 2 成员布局, + 验证跨界完成生命周期、最终存活分配、分页及定点事件结果。两个安装包用例均要求 + 查询前后输入、manifest 及成员 hash 不变。 按照[分发包构建说明](../../README_zh.md#构建分发包),从目标干净 checkout 构建 wheel: @@ -53,14 +58,19 @@ PT_SNAP_ACCEPTANCE_WHEEL="$(realpath dist/.whl)" \ pytest tests/core/test_dataset_scale_acceptance.py --junitxml=dataset-acceptance.xml ``` -将 `` 替换为实际名称。测试以 `--no-deps --no-index` 将该确切 wheel 安装到 -临时目录,子进程清除源码 `PYTHONPATH` 并核对 `pt_snap_cli` 来自该安装目录。运行依赖 -复用测试解释器环境,不代表全新依赖解析测试。未设置 `PT_SNAP_ACCEPTANCE_WHEEL` 时, -仅安装包用例明确 skip;skip 不算 E2E 通过。常规套件可用 `pytest -m 'not slow'`。 - -安装包用例在 pytest 临时目录写 `acceptance.json`,并记录 `dataset_acceptance` JUnit -属性,包括 wheel/source SHA-256、包路径、事件数量、导入耗时及查询断言。耗时只是 -单次观察,不设固定时限断言,也不承诺进程 RSS。 +将 `` 替换为实际名称。每个安装包用例创建独立且不含系统包的全新 venv, +安装该确切 wheel 及运行依赖,清除源码 `PYTHONPATH`,并核对产品包和运行依赖均来自 +该 venv。依赖安装可能访问包索引。可将 `PT_SNAP_ACCEPTANCE_WHEELHOUSE` 指向包含兼容 +依赖 wheel 的目录,以 `--no-index --find-links` 离线安装。 +未设置 `PT_SNAP_ACCEPTANCE_WHEEL` 时,两个安装包用例均明确 skip;skip 不算 E2E 通过。 +完整 PR 层使用 `pytest -m 'not nightly'`;兼容层使用 +`pytest -m 'not slow and not nightly'`。 + +百万事件安装包用例写入 `acceptance.json` 和 `dataset_acceptance` JUnit 属性,包含 +wheel/source hash、运行环境身份、事件数量及命令/查询证据。短终片用例单独保存其证据。 +`PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` 可指定持久证据目录,否则使用 pytest 临时目录。 +必需的 nightly runner 拒绝缺失或未通过的用例。耗时仅为观察记录;这些安装包用例 +不设固定时限,也不承诺进程 RSS。 ## 证据边界