From 430acf97ee9f6129d182c88382c15278927f9302 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:48:03 +0800 Subject: [PATCH 1/5] ci: tier compatibility tests and require nightly wheel acceptance --- .github/AGENTS.md | 1 + .github/scripts/run_dataset_acceptance.py | 120 +++++++++++++++++ .github/workflows/AGENTS.md | 35 ++++- .github/workflows/test.yml | 70 +++++++++- tests/core/test_dataset_id_pagination.py | 1 + tests/test_ci_tiers.py | 110 +++++++++++++++ tests/test_dataset_acceptance_runner.py | 157 ++++++++++++++++++++++ tests/test_wheel_audit.py | 6 + 8 files changed, 494 insertions(+), 6 deletions(-) create mode 100644 .github/scripts/run_dataset_acceptance.py create mode 100644 tests/test_ci_tiers.py create mode 100644 tests/test_dataset_acceptance_runner.py diff --git a/.github/AGENTS.md b/.github/AGENTS.md index c4b73b2..e4569fe 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -9,6 +9,7 @@ Parent scope: [repository](../AGENTS.md). | `pull_request_template.md` | Default PR template and machine-parsed snapshot decision labels | | `scripts/check_snapshot_provenance.py` | Base/head Git-blob comparison and PR declaration validation | | `scripts/clean_wheel_agent_acceptance.py` | Supplied wheel acceptance in temporary home/CWD/venv | +| `scripts/run_dataset_acceptance.py` | Required installed-wheel dataset test with fail-closed JUnit and artifact evidence | ## Provenance Guard diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py new file mode 100644 index 0000000..89fc223 --- /dev/null +++ b/.github/scripts/run_dataset_acceptance.py @@ -0,0 +1,120 @@ +"""Run the installed-wheel gate and reject empty, skipped or stale evidence. + +This deliberately selects one existing public CLI/API acceptance test. It does +not claim clean dependency isolation or complete default-scale coverage; those +are contracts of the selected test, not of this CI runner. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TEST_NAME = "test_installed_trusted_producer_default_export_cli_and_api" +TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" + + +class AcceptanceError(RuntimeError): + pass + + +def read_evidence(report: Path, wheel_sha256: str) -> dict: + """A green pytest process alone does not prove the requested test ran.""" + if not report.is_file(): + raise AcceptanceError("pytest did not write its JUnit report") + cases = ET.parse(report).findall(".//testcase") + if len(cases) != 1 or cases[0].get("name") != TEST_NAME: + raise AcceptanceError(f"Expected exactly one {TEST_NAME} result, got {len(cases)}") + case = cases[0] + if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): + raise AcceptanceError("Installed-wheel acceptance must pass, without skips") + properties = case.findall("./properties/property[@name='dataset_acceptance']") + if len(properties) != 1: + raise AcceptanceError("Expected exactly one dataset_acceptance evidence property") + evidence = json.loads(properties[0].get("value", "")) + if not isinstance(evidence, dict) or evidence.get("wheel_sha256") != wheel_sha256: + raise AcceptanceError("Acceptance evidence does not match the supplied wheel SHA-256") + return evidence + + +def run_acceptance(wheel: Path, output_dir: Path) -> None: + wheel = wheel.resolve() + output_dir = output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + report = output_dir / "junit.xml" + # Reusing an output directory must never let a previous green run mask this one. + for name in ("junit.xml", "summary.json", "pytest.log"): + (output_dir / name).unlink(missing_ok=True) + command = [ + sys.executable, + "-m", + "pytest", + TEST_NODE, + "-o", + "junit_family=xunit1", + f"--junitxml={report}", + ] + env = os.environ.copy() + env["PT_SNAP_ACCEPTANCE_WHEEL"] = str(wheel) + env["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] = str(output_dir) + summary = { + "wheel": str(wheel), + "test_node": TEST_NODE, + "command": command, + "status": "failed", + } + try: + if not wheel.is_file() or wheel.suffix != ".whl": + raise AcceptanceError(f"A built wheel is required: {wheel}") + wheel_sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + summary["wheel_sha256"] = wheel_sha256 + source_sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout.strip() + summary["source_sha"] = source_sha + print(f"Running {TEST_NODE} at {source_sha}", flush=True) + with (output_dir / "pytest.log").open("w") as log: + result = subprocess.run( + command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT + ) + print((output_dir / "pytest.log").read_text(), end="", flush=True) + summary["pytest_exit_code"] = result.returncode + if result.returncode != 0: + raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log") + summary["acceptance"] = read_evidence(report, wheel_sha256) + summary["status"] = "passed" + except Exception as exc: + summary["error"] = str(exc) + raise + finally: + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + try: + run_acceptance(args.wheel, args.output_dir) + except ( + AcceptanceError, + OSError, + ValueError, + ET.ParseError, + subprocess.CalledProcessError, + ) as exc: + print(f"Dataset acceptance failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 2959c6d..91e9211 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -5,23 +5,49 @@ Parent scope: [automation](../AGENTS.md). ## Test Workflow `test.yml` runs for main pushes, main-targeted PR opened/synchronize/reopened/edited -events, and `workflow_call`. Provenance, lint and Python test-matrix jobs are -independent. Concurrency is keyed by GitHub ref and superseded runs are cancelled. +events, `workflow_call`, nightly at 03:17 UTC, and `workflow_dispatch`. Provenance, +lint, browser and Python test-matrix jobs are independent. Checkouts use the PR +head SHA or the triggering SHA; manual dispatch checks the selected ref's SHA. +Concurrency is keyed by GitHub ref and superseded runs are cancelled. | Job/step | Gate | | --- | --- | | Snapshot provenance | Full Git history; base/head comparison; PR decision required only in PR mode | | Lint & Format | Python 3.13, editable dev install, Ruff, Black, basedpyright with the selected interpreter | -| Test matrix | Python 3.10–3.13; fixture provenance first, then pytest with branch coverage | +| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow"` | +| Full test tier | Python 3.13; fixture provenance first, then all tests with branch coverage, including slow tests | | Overall coverage | `pyproject.toml` fail-under 90 with branches enabled | | Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently | | Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance | +| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then required installed-wheel CLI/API test | + +Million-event deep-offset pagination tests are marked `slow`; their small +pagination contracts still run across the compatibility matrix. Coverage reporting writes `.coverage`/`coverage.xml`; building writes `dist/` and intermediates. Dependency installation/build isolation may access the network. The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses temporary paths but imports a reviewed executable fixture. These are more than YAML checks. +The nightly/manual job invokes `run_dataset_acceptance.py` with the single built +wheel. Missing wheels, missing/empty JUnit, skipped/failed tests, extra test cases, +or evidence for a different wheel fail the gate. It uploads the pytest log, JUnit +(including the test's evidence), and a summary with actual checkout SHA and wheel +SHA-256, even after failure. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` is available for +additional test-owned artifacts. Local invocation: + +```bash +python .github/scripts/run_dataset_acceptance.py \ + --wheel dist/.whl --output-dir .tmp/dataset-acceptance +``` + +This tier initially exercises the existing 500,002-event `pip --target` acceptance +test. It does not claim a clean venv, two full 500,000-event shards, complete CLI +template coverage, live agents or upstream GUI interoperability. The ordinary +full suite may still skip this opt-in wheel test; the required nightly/manual +runner must never do so. Manually dispatch the exact published branch/commit and +verify the run's head SHA and summary; a PR run alone is not nightly evidence. + ## Release Workflow `release.yml` triggers on `v*` tag pushes and enforces: @@ -44,6 +70,7 @@ validation. For non-PR provenance checks, tag runs compare against the prior tag Review `pyproject.toml`, both workflow YAMLs, the provenance/acceptance scripts and `tests/test_release_workflow.py` together when changing execution order. Validate -YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py` from +YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py +tests/test_ci_tiers.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from the repository root. Full CI tests the matrix and installed wheel; static tests do not prove a release environment is configured or a publication occurred. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1000c8d..e71defe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: Test on: workflow_call: + workflow_dispatch: + schedule: + - cron: "17 3 * * *" push: branches: [main] pull_request: @@ -23,6 +26,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Require provenance evaluation for snapshot runtime changes if: github.event_name == 'pull_request' @@ -56,6 +60,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: @@ -78,6 +84,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: python-version: "3.13" @@ -99,6 +107,8 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: @@ -114,10 +124,19 @@ jobs: - name: Install dependencies run: pip install -e ".[dev]" - - name: Run tests with coverage - run: pytest tests/test_fixture_provenance.py && pytest --cov --cov-report=xml --cov-report=term-missing + - name: Verify executable fixture provenance + run: pytest tests/test_fixture_provenance.py + + - name: Run non-slow compatibility tests + if: matrix.python-version != '3.13' + run: pytest -m "not slow" + + - name: Run all tests with coverage + if: matrix.python-version == '3.13' + run: pytest --cov --cov-report=xml --cov-report=term-missing - name: Guard critical runtime coverage + if: matrix.python-version == '3.13' # Separate floors prevent strong modules from masking a weak boundary. run: | python -m coverage report --include='*/core/snapshot_import_backend.py' --fail-under=90 @@ -153,3 +172,50 @@ jobs: python .github/scripts/clean_wheel_agent_acceptance.py \ --wheel "$WHEEL" \ --fixture tests/fixtures/snapshots/snapshot_with_empty_cache.pkl + + dataset-acceptance: + name: Installed-wheel dataset acceptance + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.sha }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Verify executable fixture provenance + run: pytest tests/test_fixture_provenance.py + + - name: Build wheel + run: python -m build --wheel + + - name: Audit wheel contents against source + run: | + set -euo pipefail + WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') + python .github/scripts/audit_wheel.py --wheel "$WHEEL" + + - name: Run required installed-wheel dataset acceptance + run: | + set -euo pipefail + WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') + python .github/scripts/run_dataset_acceptance.py \ + --wheel "$WHEEL" --output-dir .tmp/dataset-acceptance + + - name: Upload dataset acceptance evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: dataset-acceptance-${{ github.sha }} + path: .tmp/dataset-acceptance/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 diff --git a/tests/core/test_dataset_id_pagination.py b/tests/core/test_dataset_id_pagination.py index df4756c..56e4254 100644 --- a/tests/core/test_dataset_id_pagination.py +++ b/tests/core/test_dataset_id_pagination.py @@ -269,6 +269,7 @@ def million_event_dataset(tmp_path_factory): return root +@pytest.mark.slow @pytest.mark.parametrize("template", ["event", "allocation"]) @pytest.mark.parametrize("direction", ["ASC", "DESC"]) def test_million_offset_keeps_fetched_work_constant(million_event_dataset, template, direction): diff --git a/tests/test_ci_tiers.py b/tests/test_ci_tiers.py new file mode 100644 index 0000000..9a6e10f --- /dev/null +++ b/tests/test_ci_tiers.py @@ -0,0 +1,110 @@ +"""Protect fast/full/nightly boundaries and the existing quality gates.""" + +from pathlib import Path + +import yaml + + +def workflow(): + return yaml.load(Path(".github/workflows/test.yml").read_text(), Loader=yaml.BaseLoader) + + +def named_steps(job): + return {step["name"]: step for step in job["steps"] if "name" in step} + + +def test_compatibility_matrix_leaves_slow_and_coverage_on_python_313(): + job = workflow()["jobs"]["test"] + assert job["strategy"]["matrix"]["python-version"] == ["3.10", "3.11", "3.12", "3.13"] + steps = named_steps(job) + fast = steps["Run non-slow compatibility tests"] + assert fast["if"] == "matrix.python-version != '3.13'" + assert fast["run"] == 'pytest -m "not slow"' + full = steps["Run all tests with coverage"] + assert full["if"] == "matrix.python-version == '3.13'" + assert full["run"] == "pytest --cov --cov-report=xml --cov-report=term-missing" + assert "if" not in steps["Verify executable fixture provenance"] + names = list(steps) + assert names.index("Verify executable fixture provenance") < names.index(fast["name"]) + assert names.index("Verify executable fixture provenance") < names.index(full["name"]) + + +def test_full_tier_preserves_independent_coverage_floors_and_packaging(): + steps = named_steps(workflow()["jobs"]["test"]) + guard = steps["Guard critical runtime coverage"] + assert guard["if"] == "matrix.python-version == '3.13'" + for file, floor in ( + ("core/snapshot_import_backend.py", 90), + ("simulate/simulated_caching_allocator.py", 88), + ("simulate/snapshot_mutator.py", 84), + ): + assert f"--include='*/{file}' --fail-under={floor}" in guard["run"] + config = Path("pyproject.toml").read_text() + assert "branch = true" in config and "fail_under = 90" in config + for name in ( + "Build wheel", + "Audit wheel contents against source", + "Clean-wheel Agent acceptance", + ): + assert steps[name]["if"] == "matrix.python-version == '3.13'" + assert steps["Upload coverage report"]["if"] == "matrix.python-version == '3.13' && always()" + + +def test_ci_preserves_least_privilege_and_checks_the_published_commit(): + config = workflow() + assert config["permissions"] == {"contents": "read"} + assert "pull_request_target" not in config["on"] + assert config["on"]["pull_request"]["types"] == ["opened", "synchronize", "reopened", "edited"] + assert "workflow_call" in config["on"] + jobs = config["jobs"] + for name in ("snapshot-provenance", "lint", "browser", "test"): + checkout = jobs[name]["steps"][0] + assert checkout["uses"] == "actions/checkout@v4" + assert checkout["with"]["ref"] == "${{ github.event.pull_request.head.sha || github.sha }}" + assert jobs["snapshot-provenance"]["steps"][0]["with"]["fetch-depth"] == "0" + assert ( + "--require-decision" + in named_steps(jobs["snapshot-provenance"])[ + "Require provenance evaluation for snapshot runtime changes" + ]["run"] + ) + browser = named_steps(jobs["browser"])["Run required Chromium interaction tests"] + assert browser["env"]["PT_SNAP_REQUIRE_BROWSER"] == "1" + assert "tests/test_memory_tree_browser.py" in browser["run"] + lint = named_steps(jobs["lint"]) + for name in ("Ruff lint check", "Black format check", "basedPyright check"): + assert name in lint + + +def test_nightly_and_manual_gate_build_and_audit_before_required_acceptance(): + config = workflow() + assert "workflow_dispatch" in config["on"] + assert config["on"]["schedule"] == [{"cron": "17 3 * * *"}] + job = config["jobs"]["dataset-acceptance"] + assert ( + job["if"] == "github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'" + ) + assert job["timeout-minutes"] == "30" + assert job["steps"][0]["with"] == {"fetch-depth": "0", "ref": "${{ github.sha }}"} + steps = named_steps(job) + names = list(steps) + order = [ + "Verify executable fixture provenance", + "Build wheel", + "Audit wheel contents against source", + "Run required installed-wheel dataset acceptance", + "Upload dataset acceptance evidence", + ] + assert [names.index(name) for name in order] == sorted(names.index(name) for name in order) + acceptance = steps["Run required installed-wheel dataset acceptance"]["run"] + assert "assert len(wheels) == 1" in acceptance + assert "python .github/scripts/run_dataset_acceptance.py" in acceptance + assert '--wheel "$WHEEL" --output-dir .tmp/dataset-acceptance' in acceptance + for step in job["steps"]: + assert "continue-on-error" not in step + upload = steps["Upload dataset acceptance evidence"] + assert upload["if"] == "always()" + assert upload["with"]["if-no-files-found"] == "error" + assert upload["with"]["include-hidden-files"] == "true" + assert upload["with"]["path"] == ".tmp/dataset-acceptance/" + assert "${{ github.sha }}" in upload["with"]["name"] diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py new file mode 100644 index 0000000..36ff22e --- /dev/null +++ b/tests/test_dataset_acceptance_runner.py @@ -0,0 +1,157 @@ +"""Fail-closed CI wrapper contracts without repeating the expensive scale test.""" + +import hashlib +import importlib.util +import json +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +RUNNER = Path(".github/scripts/run_dataset_acceptance.py") + + +@pytest.fixture +def runner(): + spec = importlib.util.spec_from_file_location("dataset_acceptance_runner", RUNNER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def write_report(path, name, evidence, outcome=None): + suite = ET.Element("testsuite") + case = ET.SubElement(suite, "testcase", name=name) + if outcome: + ET.SubElement(case, outcome) + if evidence is not None: + properties = ET.SubElement(case, "properties") + ET.SubElement(properties, "property", name="dataset_acceptance", value=json.dumps(evidence)) + ET.ElementTree(suite).write(path) + + +def test_runner_accepts_matching_wheel_evidence(runner, tmp_path): + report = tmp_path / "junit.xml" + evidence = {"wheel_sha256": "abc", "events": 500_002} + write_report(report, runner.TEST_NAME, evidence) + assert runner.read_evidence(report, "abc") == evidence + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_nonpassing_acceptance(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}, outcome) + with pytest.raises(runner.AcceptanceError, match="without skips"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("evidence", [None, {}, {"wheel_sha256": "other"}, []]) +def test_runner_rejects_missing_or_wrong_artifact_evidence(runner, tmp_path, evidence): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, evidence) + with pytest.raises(runner.AcceptanceError): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("count", [0, 2]) +def test_runner_rejects_empty_or_extra_collection(runner, tmp_path, count): + report = tmp_path / "junit.xml" + suite = ET.Element("testsuite") + for _ in range(count): + ET.SubElement(suite, "testcase", name=runner.TEST_NAME) + ET.ElementTree(suite).write(report) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +def test_runner_rejects_wrong_test_even_if_it_passes(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, "unrelated_test", {"wheel_sha256": "abc"}) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +def test_runner_rejects_duplicate_evidence(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + tree = ET.parse(report) + properties = tree.find(".//properties") + assert properties is not None + ET.SubElement(properties, "property", name="dataset_acceptance", value="{}") + tree.write(report) + with pytest.raises(runner.AcceptanceError, match="exactly one dataset_acceptance"): + runner.read_evidence(report, "abc") + + +def test_missing_wheel_fails_before_pytest(tmp_path): + output = tmp_path / "results" + output.mkdir() + (output / "summary.json").write_text('{"status": "passed"}') + (output / "junit.xml").write_text("old passing report") + (output / "pytest.log").write_text("old passing output") + result = subprocess.run( + [ + sys.executable, + str(RUNNER), + "--wheel", + str(tmp_path / "missing.whl"), + "--output-dir", + str(tmp_path / "results"), + ], + capture_output=True, + text=True, + ) + assert result.returncode == 1 + assert "A built wheel is required" in result.stderr + assert not (output / "junit.xml").exists() + assert not (output / "pytest.log").exists() + summary = json.loads((output / "summary.json").read_text()) + assert summary["status"] == "failed" + assert "A built wheel is required" in summary["error"] + + +@pytest.mark.parametrize("exit_code,write_junit", [(0, True), (0, False), (1, True), (5, False)]) +def test_runner_records_checkout_and_artifact_with_fail_closed_results( + runner, tmp_path, monkeypatch, exit_code, write_junit +): + wheel = tmp_path / "built.whl" + wheel.write_bytes(b"test artifact") + wheel_hash = hashlib.sha256(wheel.read_bytes()).hexdigest() + output = tmp_path / "results" + output.mkdir() + report = output / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": wheel_hash}) + calls = [] + + def run(command, **kwargs): + calls.append(command) + assert kwargs["cwd"] == runner.ROOT + if command[0] == "git": + return subprocess.CompletedProcess(command, 0, stdout="checkout-sha\n") + assert command[:3] == [sys.executable, "-m", "pytest"] + assert runner.TEST_NODE in command + assert "junit_family=xunit1" in command + assert kwargs["env"]["PT_SNAP_ACCEPTANCE_WHEEL"] == str(wheel) + assert kwargs["env"]["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] == str(output) + assert not report.exists(), "stale JUnit must be removed before this run" + kwargs["stdout"].write("captured pytest output\n") + if write_junit: + write_report(report, runner.TEST_NAME, {"wheel_sha256": wheel_hash}) + return subprocess.CompletedProcess(command, exit_code) + + monkeypatch.setattr(runner.subprocess, "run", run) + if exit_code or not write_junit: + with pytest.raises(runner.AcceptanceError): + runner.run_acceptance(wheel, output) + else: + runner.run_acceptance(wheel, output) + assert len(calls) == 2 + summary = json.loads((output / "summary.json").read_text()) + assert summary["source_sha"] == "checkout-sha" + assert summary["wheel_sha256"] == wheel_hash + assert summary["pytest_exit_code"] == exit_code + assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") + assert (output / "pytest.log").read_text() == "captured pytest output\n" diff --git a/tests/test_wheel_audit.py b/tests/test_wheel_audit.py index 15bb04c..4bc413a 100644 --- a/tests/test_wheel_audit.py +++ b/tests/test_wheel_audit.py @@ -177,6 +177,12 @@ def build(directory): ("workflow", "job", "build", "next_step"), [ ("test.yml", "test", "Build wheel", "Clean-wheel Agent acceptance"), + ( + "test.yml", + "dataset-acceptance", + "Build wheel", + "Run required installed-wheel dataset acceptance", + ), ("release.yml", "build", "Build sdist and wheel", "Upload build artifacts"), ], ) From 68d89670b188c93e41b1a5d8f41903057e5ada06 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:59:11 +0800 Subject: [PATCH 2/5] ci: enforce explicit nightly selection and required test evidence --- .github/scripts/run_dataset_acceptance.py | 66 ++++++++--- .github/workflows/AGENTS.md | 20 ++-- .github/workflows/test.yml | 6 +- pyproject.toml | 1 + tests/core/test_dataset_scale_acceptance.py | 1 + tests/test_ci_tiers.py | 10 +- tests/test_dataset_acceptance_runner.py | 120 +++++++++++++++++++- 7 files changed, 194 insertions(+), 30 deletions(-) diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py index 89fc223..d1d8512 100644 --- a/.github/scripts/run_dataset_acceptance.py +++ b/.github/scripts/run_dataset_acceptance.py @@ -1,6 +1,6 @@ """Run the installed-wheel gate and reject empty, skipped or stale evidence. -This deliberately selects one existing public CLI/API acceptance test. It does +This selects nightly tests and requires the public CLI/API wheel case. It does not claim clean dependency isolation or complete default-scale coverage; those are contracts of the selected test, not of this CI runner. """ @@ -14,28 +14,60 @@ import subprocess import sys import xml.etree.ElementTree as ET -from pathlib import Path +from pathlib import Path, PurePosixPath ROOT = Path(__file__).resolve().parents[2] TEST_NAME = "test_installed_trusted_producer_default_export_cli_and_api" TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" +# Extend this tuple alongside new mandatory nightly gates. Never list future +# files that have not landed: the runner rejects any missing configured node. +REQUIRED_TEST_NODES = (TEST_NODE,) class AcceptanceError(RuntimeError): pass -def read_evidence(report: Path, wheel_sha256: str) -> dict: - """A green pytest process alone does not prove the requested test ran.""" +def test_node_id(case: ET.Element) -> str: + """Recover a complete pytest identity from the required xunit1 fields.""" + file = case.get("file", "") + name = case.get("name", "") + classname = case.get("classname", "") + path = PurePosixPath(file) + if not name or path.is_absolute() or ".." in path.parts or path.suffix != ".py": + raise AcceptanceError(f"JUnit testcase has no valid source identity: {case.attrib}") + module = file.removesuffix(".py").replace("/", ".") + if classname == module: + classes = "" + elif classname.startswith(module + "."): + classes = classname[len(module) + 1 :].replace(".", "::") + "::" + else: + raise AcceptanceError(f"JUnit testcase has an inconsistent class identity: {case.attrib}") + return f"{file}::{classes}{name}" + + +def read_cases(report: Path) -> list[ET.Element]: if not report.is_file(): raise AcceptanceError("pytest did not write its JUnit report") - cases = ET.parse(report).findall(".//testcase") - if len(cases) != 1 or cases[0].get("name") != TEST_NAME: - raise AcceptanceError(f"Expected exactly one {TEST_NAME} result, got {len(cases)}") - case = cases[0] - if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): - raise AcceptanceError("Installed-wheel acceptance must pass, without skips") - properties = case.findall("./properties/property[@name='dataset_acceptance']") + return ET.parse(report).findall(".//testcase") + + +def read_evidence(report: Path, wheel_sha256: str) -> dict: + """Require the wheel case plus every registered gate, with no silent skips.""" + cases = read_cases(report) + nodes = [test_node_id(case) for case in cases] + if nodes.count(TEST_NODE) != 1: + raise AcceptanceError(f"Expected exactly one {TEST_NODE} result") + if len(nodes) != len(set(nodes)): + raise AcceptanceError("Duplicate nightly testcase identities") + missing = set(REQUIRED_TEST_NODES) - set(nodes) + if missing: + raise AcceptanceError(f"Missing required nightly test nodes: {sorted(missing)}") + for node, case in zip(nodes, cases, strict=True): + if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): + raise AcceptanceError(f"Nightly acceptance must pass, without skips: {node}") + wheel_case = cases[nodes.index(TEST_NODE)] + properties = wheel_case.findall("./properties/property[@name='dataset_acceptance']") if len(properties) != 1: raise AcceptanceError("Expected exactly one dataset_acceptance evidence property") evidence = json.loads(properties[0].get("value", "")) @@ -56,7 +88,8 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: sys.executable, "-m", "pytest", - TEST_NODE, + "-m", + "nightly", "-o", "junit_family=xunit1", f"--junitxml={report}", @@ -67,6 +100,8 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: summary = { "wheel": str(wheel), "test_node": TEST_NODE, + "required_test_nodes": list(REQUIRED_TEST_NODES), + "test_nodes": [], "command": command, "status": "failed", } @@ -79,13 +114,18 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True ).stdout.strip() summary["source_sha"] = source_sha - print(f"Running {TEST_NODE} at {source_sha}", flush=True) + print( + f"Running nightly acceptance at {source_sha}; required: {REQUIRED_TEST_NODES}", + flush=True, + ) with (output_dir / "pytest.log").open("w") as log: result = subprocess.run( command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT ) print((output_dir / "pytest.log").read_text(), end="", flush=True) summary["pytest_exit_code"] = result.returncode + if report.is_file(): + summary["test_nodes"] = [test_node_id(case) for case in read_cases(report)] if result.returncode != 0: raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log") summary["acceptance"] = read_evidence(report, wheel_sha256) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 91e9211..c2a3c57 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -14,12 +14,12 @@ Concurrency is keyed by GitHub ref and superseded runs are cancelled. | --- | --- | | Snapshot provenance | Full Git history; base/head comparison; PR decision required only in PR mode | | Lint & Format | Python 3.13, editable dev install, Ruff, Black, basedpyright with the selected interpreter | -| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow"` | -| Full test tier | Python 3.13; fixture provenance first, then all tests with branch coverage, including slow tests | +| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow and not nightly"` | +| Full PR test tier | Python 3.13; `pytest -m "not nightly"` with branch coverage, preserving the existing slow tests | | Overall coverage | `pyproject.toml` fail-under 90 with branches enabled | | Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently | | Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance | -| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then required installed-wheel CLI/API test | +| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then `nightly` tests including the required installed-wheel CLI/API test | Million-event deep-offset pagination tests are marked `slow`; their small pagination contracts still run across the compatibility matrix. @@ -30,9 +30,13 @@ The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses tempora paths but imports a reviewed executable fixture. These are more than YAML checks. The nightly/manual job invokes `run_dataset_acceptance.py` with the single built -wheel. Missing wheels, missing/empty JUnit, skipped/failed tests, extra test cases, -or evidence for a different wheel fail the gate. It uploads the pytest log, JUnit -(including the test's evidence), and a summary with actual checkout SHA and wheel +wheel. All `nightly` tests are selected automatically; the installed-wheel case +must appear exactly once. New mandatory gates also add their complete pytest +node IDs to `REQUIRED_TEST_NODES` in that runner. Do not register absent future +files. Missing wheels, missing/empty JUnit, missing required nodes, duplicate +identities, any skipped/failed/error test, or evidence for a different wheel fail +the gate. It uploads the pytest log, JUnit (including the test's evidence), and a +summary with complete executed/required test node IDs, actual checkout SHA and wheel SHA-256, even after failure. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` is available for additional test-owned artifacts. Local invocation: @@ -44,8 +48,8 @@ python .github/scripts/run_dataset_acceptance.py \ This tier initially exercises the existing 500,002-event `pip --target` acceptance test. It does not claim a clean venv, two full 500,000-event shards, complete CLI template coverage, live agents or upstream GUI interoperability. The ordinary -full suite may still skip this opt-in wheel test; the required nightly/manual -runner must never do so. Manually dispatch the exact published branch/commit and +PR tier excludes `nightly` tests; the required nightly/manual runner must never +skip them. Manually dispatch the exact published branch/commit and verify the run's head SHA and summary; a PR run alone is not nightly evidence. ## Release Workflow diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e71defe..0465fca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -129,11 +129,11 @@ jobs: - name: Run non-slow compatibility tests if: matrix.python-version != '3.13' - run: pytest -m "not slow" + run: pytest -m "not slow and not nightly" - - name: Run all tests with coverage + - name: Run full PR tests with coverage if: matrix.python-version == '3.13' - run: pytest --cov --cov-report=xml --cov-report=term-missing + run: pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing - name: Guard critical runtime coverage if: matrix.python-version == '3.13' diff --git a/pyproject.toml b/pyproject.toml index f97f49b..f65fa0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ select = ["E", "F", "W", "I", "N", "UP", "B", "C4"] ignore = ["E501"] [tool.pytest.ini_options] +markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"] testpaths = ["tests"] python_files = ["test_*.py"] python_functions = ["test_*"] diff --git a/tests/core/test_dataset_scale_acceptance.py b/tests/core/test_dataset_scale_acceptance.py index e2d1b0d..8fb8a05 100644 --- a/tests/core/test_dataset_scale_acceptance.py +++ b/tests/core/test_dataset_scale_acceptance.py @@ -226,6 +226,7 @@ def installed_runtime(tmp_path): return cli, env, module, hashlib.sha256(wheel.read_bytes()).hexdigest() +@pytest.mark.nightly def test_installed_trusted_producer_default_export_cli_and_api( tmp_path, installed_runtime, record_property ): diff --git a/tests/test_ci_tiers.py b/tests/test_ci_tiers.py index 9a6e10f..f011586 100644 --- a/tests/test_ci_tiers.py +++ b/tests/test_ci_tiers.py @@ -19,10 +19,14 @@ def test_compatibility_matrix_leaves_slow_and_coverage_on_python_313(): steps = named_steps(job) fast = steps["Run non-slow compatibility tests"] assert fast["if"] == "matrix.python-version != '3.13'" - assert fast["run"] == 'pytest -m "not slow"' - full = steps["Run all tests with coverage"] + assert fast["run"] == 'pytest -m "not slow and not nightly"' + full = steps["Run full PR tests with coverage"] assert full["if"] == "matrix.python-version == '3.13'" - assert full["run"] == "pytest --cov --cov-report=xml --cov-report=term-missing" + assert full["run"] == 'pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing' + assert ( + 'markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"]' + in Path("pyproject.toml").read_text() + ) assert "if" not in steps["Verify executable fixture provenance"] names = list(steps) assert names.index("Verify executable fixture provenance") < names.index(fast["name"]) diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py index 36ff22e..037e488 100644 --- a/tests/test_dataset_acceptance_runner.py +++ b/tests/test_dataset_acceptance_runner.py @@ -24,7 +24,13 @@ def runner(): def write_report(path, name, evidence, outcome=None): suite = ET.Element("testsuite") - case = ET.SubElement(suite, "testcase", name=name) + case = ET.SubElement( + suite, + "testcase", + name=name, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) if outcome: ET.SubElement(case, outcome) if evidence is not None: @@ -61,7 +67,13 @@ def test_runner_rejects_empty_or_extra_collection(runner, tmp_path, count): report = tmp_path / "junit.xml" suite = ET.Element("testsuite") for _ in range(count): - ET.SubElement(suite, "testcase", name=runner.TEST_NAME) + ET.SubElement( + suite, + "testcase", + name=runner.TEST_NAME, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) ET.ElementTree(suite).write(report) with pytest.raises(runner.AcceptanceError, match="exactly one"): runner.read_evidence(report, "abc") @@ -74,6 +86,106 @@ def test_runner_rejects_wrong_test_even_if_it_passes(runner, tmp_path): runner.read_evidence(report, "abc") +def add_case( + report, + *, + name="test_extra", + file="tests/test_extra.py", + classname="tests.test_extra", + outcome=None, +): + tree = ET.parse(report) + case = ET.SubElement(tree.getroot(), "testcase", name=name, file=file, classname=classname) + if outcome: + ET.SubElement(case, outcome) + tree.write(report) + + +def test_runner_accepts_additional_nightly_cases_and_records_complete_identities(runner, tmp_path): + report = tmp_path / "junit.xml" + evidence = {"wheel_sha256": "abc"} + write_report(report, runner.TEST_NAME, evidence) + add_case(report, name="test_extra[v2]", classname="tests.test_extra.TestRoutes") + assert runner.read_evidence(report, "abc") == evidence + assert [runner.test_node_id(case) for case in runner.read_cases(report)] == [ + runner.TEST_NODE, + "tests/test_extra.py::TestRoutes::test_extra[v2]", + ] + + +def test_runner_rejects_missing_configured_gate(runner, tmp_path, monkeypatch): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + required = "tests/test_extra.py::test_extra" + monkeypatch.setattr(runner, "REQUIRED_TEST_NODES", (runner.TEST_NODE, required)) + with pytest.raises(runner.AcceptanceError, match="Missing required nightly test nodes"): + runner.read_evidence(report, "abc") + add_case(report) + assert runner.read_evidence(report, "abc") == {"wheel_sha256": "abc"} + + +def test_runner_rejects_duplicate_additional_cases(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + add_case(report) + add_case(report) + with pytest.raises(runner.AcceptanceError, match="Duplicate nightly"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_any_nonpassing_additional_case(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + add_case(report, outcome=outcome) + with pytest.raises( + runner.AcceptanceError, match="without skips: tests/test_extra.py::test_extra" + ): + runner.read_evidence(report, "abc") + + +def test_same_function_name_in_another_file_cannot_replace_required_wheel_case(runner, tmp_path): + report = tmp_path / "junit.xml" + ET.ElementTree(ET.Element("testsuite")).write(report) + add_case(report, name=runner.TEST_NAME) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize( + "attributes", [{"file": ""}, {"file": "../tests/test_extra.py"}, {"classname": "wrong"}] +) +def test_runner_rejects_incomplete_or_inconsistent_junit_identity(runner, attributes): + case = ET.Element( + "testcase", file="tests/test_extra.py", classname="tests.test_extra", name="test_extra" + ) + case.attrib.update(attributes) + with pytest.raises(runner.AcceptanceError, match="identity"): + runner.test_node_id(case) + + +def test_nightly_marker_selection_collects_every_configured_gate(runner): + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "--strict-markers", + "-q", + "-o", + "addopts=", + "-m", + "nightly", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + collected = {line for line in result.stdout.splitlines() if "::" in line} + assert set(runner.REQUIRED_TEST_NODES) <= collected + + def test_runner_rejects_duplicate_evidence(runner, tmp_path): report = tmp_path / "junit.xml" write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) @@ -132,7 +244,7 @@ def run(command, **kwargs): if command[0] == "git": return subprocess.CompletedProcess(command, 0, stdout="checkout-sha\n") assert command[:3] == [sys.executable, "-m", "pytest"] - assert runner.TEST_NODE in command + assert command[3:5] == ["-m", "nightly"] assert "junit_family=xunit1" in command assert kwargs["env"]["PT_SNAP_ACCEPTANCE_WHEEL"] == str(wheel) assert kwargs["env"]["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] == str(output) @@ -153,5 +265,7 @@ def run(command, **kwargs): assert summary["source_sha"] == "checkout-sha" assert summary["wheel_sha256"] == wheel_hash assert summary["pytest_exit_code"] == exit_code + assert summary["required_test_nodes"] == [runner.TEST_NODE] + assert summary["test_nodes"] == ([runner.TEST_NODE] if write_junit else []) assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") assert (output / "pytest.log").read_text() == "captured pytest output\n" From 8b28ae53257bfa5c3b9d7a9cb6e5e8806b0e1f7c Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:12:08 +0800 Subject: [PATCH 3/5] test: require paired dataset differential acceptance and fix zero-rank completeness --- .github/AGENTS.md | 1 + .../scripts/run_differential_acceptance.py | 194 ++++++++++++++++ .github/workflows/AGENTS.md | 14 ++ .github/workflows/test.yml | 21 ++ docs/en/dataset-acceptance.md | 88 +++++++- docs/en/querying.md | 6 + docs/zh/dataset-acceptance.md | 73 +++++- docs/zh/querying.md | 5 + src/pt_snap_cli/core/dataset_active_sql.py | 2 +- src/pt_snap_cli/core/dataset_attribution.py | 2 +- src/pt_snap_cli/core/query_service.py | 4 + tests/core/dataset_differential_assertions.py | 74 ++++++ tests/core/dataset_differential_cases.py | 213 ++++++++++++++++++ tests/core/dataset_lifecycle_fixtures.py | 145 ++++++++++-- tests/core/test_dataset_differential_cases.py | 151 +++++++++++++ tests/core/test_dataset_realistic.py | 84 +++++-- tests/core/test_dataset_zero_rank.py | 127 +++++++++++ tests/test_contract_cli_api.py | 34 +++ tests/test_differential_acceptance_runner.py | 206 +++++++++++++++++ tests/test_differential_ci.py | 46 ++++ 20 files changed, 1435 insertions(+), 55 deletions(-) create mode 100644 .github/scripts/run_differential_acceptance.py create mode 100644 tests/core/dataset_differential_assertions.py create mode 100644 tests/core/dataset_differential_cases.py create mode 100644 tests/core/test_dataset_differential_cases.py create mode 100644 tests/core/test_dataset_zero_rank.py create mode 100644 tests/test_differential_acceptance_runner.py create mode 100644 tests/test_differential_ci.py diff --git a/.github/AGENTS.md b/.github/AGENTS.md index e4569fe..e37b146 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -9,6 +9,7 @@ Parent scope: [repository](../AGENTS.md). | `pull_request_template.md` | Default PR template and machine-parsed snapshot decision labels | | `scripts/check_snapshot_provenance.py` | Base/head Git-blob comparison and PR declaration validation | | `scripts/clean_wheel_agent_acceptance.py` | Supplied wheel acceptance in temporary home/CWD/venv | +| `scripts/run_differential_acceptance.py` | Source-query 200-seed/v1-v2 acceptance with exact executed identities and coverage evidence | | `scripts/run_dataset_acceptance.py` | Required installed-wheel dataset test with fail-closed JUnit and artifact evidence | ## Provenance Guard diff --git a/.github/scripts/run_differential_acceptance.py b/.github/scripts/run_differential_acceptance.py new file mode 100644 index 0000000..50759b2 --- /dev/null +++ b/.github/scripts/run_differential_acceptance.py @@ -0,0 +1,194 @@ +"""Run the required paired seed profile and reject skipped, partial or stale evidence.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TEST_FILE = "tests/core/test_dataset_realistic.py" +TEST_NAME = "test_seeded_lifecycle_windows_match_single_database_strictly" +TEST_NODE = f"{TEST_FILE}::{TEST_NAME}" + + +class AcceptanceError(RuntimeError): + pass + + +def expected_records(seed_count: int) -> dict[tuple[int, str], dict]: + # This runner is intentionally source-test acceptance, separate from the + # installed-wheel gate. Import the exact same deterministic profile contract. + sys.path[:0] = [str(ROOT), str(ROOT / "src")] + try: + from tests.core.dataset_differential_cases import ( + SUPPORTED_TEMPLATES, + capacity_for_seed, + configured_seeds, + query_cases, + ) + from tests.core.dataset_lifecycle_fixtures import EVENT_COUNT + finally: + del sys.path[:2] + seeds = configured_seeds({"PT_SNAP_DIFFERENTIAL_SEEDS": str(seed_count)}) + return { + (seed, layout): { + "seed": seed, + "format": layout, + "capacity": capacity_for_seed(seed), + "shards": (EVENT_COUNT + capacity_for_seed(seed) - 1) // capacity_for_seed(seed), + "comparisons": len(query_cases(seed, capacity_for_seed(seed))), + "unexpected_errors": 0, + "templates": sorted(SUPPORTED_TEMPLATES), + } + for seed in seeds + for layout in ("v1", "v2") + } + + +def read_evidence(report: Path, temp_dir: Path, seed_count: int) -> dict: + expected = expected_records(seed_count) + if not report.is_file(): + raise AcceptanceError("pytest did not write its JUnit report") + cases = ET.parse(report).findall(".//testcase") + names = [case.get("name") for case in cases] + expected_names = {f"{TEST_NAME}[{layout}-seed-{seed}]" for seed, layout in expected} + if len(names) != len(expected_names) or set(names) != expected_names: + raise AcceptanceError(f"Expected exactly {len(expected_names)} distinct seed/format tests") + case_by_key = {} + for case in cases: + if case.get("file") != TEST_FILE or case.get("classname") != TEST_FILE[:-3].replace( + "/", "." + ): + raise AcceptanceError("JUnit seed case has the wrong source identity") + if any(case.find(tag) is not None for tag in ("failure", "error", "skipped")): + raise AcceptanceError( + f"Every seed/format test must pass without skips: {case.get('name')}" + ) + properties = case.findall("./properties/property") + values = {p.get("name"): p.get("value") for p in properties} + if len(values) != len(properties): + raise AcceptanceError("Duplicate JUnit evidence properties") + try: + key = (int(values["seed"]), values["format"]) + except (KeyError, TypeError, ValueError) as exc: + raise AcceptanceError("Missing or invalid seed/format properties") from exc + if key not in expected or case.get("name") != f"{TEST_NAME}[{key[1]}-seed-{key[0]}]": + raise AcceptanceError("JUnit seed properties do not match the executed test identity") + case_by_key[key] = values + paths = list(temp_dir.rglob("coverage.json")) + if len(paths) != len(expected): + raise AcceptanceError("Missing or extra per-seed coverage artifacts") + records = {} + for path in paths: + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise AcceptanceError("Invalid per-seed coverage object") + if type(value.get("seed")) is not int or type(value.get("format")) is not str: + raise AcceptanceError("Invalid per-seed coverage identity types") + key = (value["seed"], value["format"]) + if key not in expected or key in records: + raise AcceptanceError("Duplicate or unexpected per-seed coverage identity") + wanted = expected[key] + for field, required in wanted.items(): + if type(value.get(field)) is not type(required) or value[field] != required: + raise AcceptanceError(f"Invalid seed coverage {key}: {field}") + builds = value.get("scratch_builds") + if type(builds) is not int or builds < 0: + raise AcceptanceError("Invalid observed scratch build count") + for field in (*wanted, "scratch_builds"): + if case_by_key[key].get(field) != str(value[field]): + raise AcceptanceError(f"JUnit and artifact evidence disagree for {key}: {field}") + if (path.parent / "differential-failure.json").exists(): + raise AcceptanceError("A seed retained unexpected failure evidence") + records[key] = value + if not any(value["scratch_builds"] for value in records.values()): + raise AcceptanceError("Profile did not execute the scratch reducer") + return { + "seed_count": seed_count, + "seed_format_cases": len(records), + "seeds": sorted({seed for seed, _ in records}), + "formats": ["v1", "v2"], + "test_nodes": [f"{TEST_FILE}::{name}" for name in sorted(names)], + "comparisons": sum(value["comparisons"] for value in records.values()), + "unexpected_errors": 0, + "scratch_builds": sum(value["scratch_builds"] for value in records.values()), + "cases_with_scratch": sum(value["scratch_builds"] > 0 for value in records.values()), + "templates": next(iter(expected.values()))["templates"], + } + + +def run_acceptance(output_dir: Path, seed_count: int = 200) -> None: + output_dir = output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + report, temp_dir = output_dir / "junit.xml", output_dir / "pytest-temp" + for name in ("junit.xml", "summary.json", "pytest.log"): + (output_dir / name).unlink(missing_ok=True) + command = [ + sys.executable, + "-m", + "pytest", + TEST_NODE, + "-o", + "junit_family=xunit1", + f"--junitxml={report}", + f"--basetemp={temp_dir}", + ] + env = os.environ.copy() + env.pop("PT_SNAP_DIFFERENTIAL_SEED", None) + env.pop("PYTEST_ADDOPTS", None) + env["PT_SNAP_DIFFERENTIAL_SEEDS"] = str(seed_count) + env["PYTHONPATH"] = str(ROOT / "src") + summary = { + "status": "failed", + "seed_count": seed_count, + "test_node": TEST_NODE, + "command": command, + } + try: + expected_records(seed_count) # Setup errors must replace stale green evidence too. + summary["source_sha"] = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, text=True, capture_output=True + ).stdout.strip() + with (output_dir / "pytest.log").open("w") as log: + result = subprocess.run( + command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT + ) + print((output_dir / "pytest.log").read_text(), end="", flush=True) + summary["pytest_exit_code"] = result.returncode + if result.returncode: + raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log") + summary["acceptance"] = read_evidence(report, temp_dir, seed_count) + summary["status"] = "passed" + except Exception as exc: + summary["error"] = str(exc) + raise + finally: + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--seed-count", type=int, default=200) + args = parser.parse_args() + try: + run_acceptance(args.output_dir, args.seed_count) + except ( + AcceptanceError, + OSError, + ValueError, + ET.ParseError, + subprocess.CalledProcessError, + ) as exc: + print(f"Differential acceptance failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index c2a3c57..8088791 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -78,3 +78,17 @@ YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py tests/test_ci_tiers.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from the repository root. Full CI tests the matrix and installed wheel; static tests do not prove a release environment is configured or a publication occurred. + +### Required paired differential profile + +The scheduled/manual `dataset-acceptance` job also runs +`run_differential_acceptance.py --seed-count 200` after the installed-wheel +nightly gate. This separate source-query step is required, with no skip or +continue-on-error escape. Its JUnit report must contain exactly 400 distinct +seed/format test IDs (200 per v1/v2), matching all per-seed coverage artifacts, +expected query counts/templates, zero unexpected errors and actual scratch use. +A single-seed environment override is removed by the runner. Logs, source SHA, +summary, JUnit and compact timeline/coverage/failure JSON are always uploaded from +`.tmp/differential-acceptance/`; regenerated SQLite files stay out of stored artifacts. The ordinary PR matrix retains the default +20-seed profile. This does not make the differential step installed-wheel or +upstream-producer evidence. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0465fca..44b1b1b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -210,6 +210,11 @@ jobs: python .github/scripts/run_dataset_acceptance.py \ --wheel "$WHEEL" --output-dir .tmp/dataset-acceptance + - name: Run required 200-seed paired differential acceptance + run: | + python .github/scripts/run_differential_acceptance.py \ + --seed-count 200 --output-dir .tmp/differential-acceptance + - name: Upload dataset acceptance evidence if: always() uses: actions/upload-artifact@v4 @@ -219,3 +224,19 @@ jobs: include-hidden-files: true if-no-files-found: error retention-days: 30 + + - name: Upload paired differential evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: differential-acceptance-${{ github.sha }} + path: | + .tmp/differential-acceptance/*.log + .tmp/differential-acceptance/*.xml + .tmp/differential-acceptance/summary.json + .tmp/differential-acceptance/pytest-temp/**/coverage.json + .tmp/differential-acceptance/pytest-temp/**/timeline.json + .tmp/differential-acceptance/pytest-temp/**/differential-failure.json + include-hidden-files: true + if-no-files-found: error + retention-days: 30 diff --git a/docs/en/dataset-acceptance.md b/docs/en/dataset-acceptance.md index d3bccba..aba5a24 100644 --- a/docs/en/dataset-acceptance.md +++ b/docs/en/dataset-acceptance.md @@ -5,13 +5,35 @@ fixture is not evidence that a producer, collector, or exporter ran. ## Regular correctness gate -`tests/core/test_dataset_realistic.py` runs eight fixed seeds and twenty query -shapes per seed against a standalone reference. Each shard contains only blocks -allocated within its window and blocks still alive on entry. Completed lifetimes -do not appear in later windows; global allocation/free-completion IDs remain -unchanged. Cases include address reuse, a pending free crossing a window, +`tests/core/test_dataset_realistic.py` defaults to 20 deterministic seeds in both +compatibility-v1 and native-v2 layouts; the nightly profile extends the same bank +to 200 seeds. The original eight regression seeds and twenty query shapes remain. +Each seed now runs 69–73 query cases per layout (2,896 paired comparisons for the +20-seed bank; 28,864 for 200), covering all eleven dataset-supported templates. +The two standalone-only templates, `event_frames` and +`active_memory_frame_tree_at_event`, have separate explicit rejection tests and +never count as successful differential comparisons. + +Both layouts use the same independently generated 80-event history and a +layout-matched standalone SQL oracle. Capacities 2/3/4/5/10/16 produce 5–40 +members, including a partial final member. Native fixtures contain real local +stack references, finalized block-reference tables, metadata and member hashes; +they pass the actual native reader validation. Local stack IDs vary by member. +These are synthetic reader/query fixtures, not evidence of native replay/export. +Each shard contains only allocations within its window and carry-in lifetimes; +completed lifetimes do not appear in later windows. Global allocation and +free-completion IDs remain unchanged. Every seed includes address reuse, preexisting/static blocks, NULL/empty/whitespace stacks, a literal missing-stack -label, sorting, pagination, filters and dynamic `top_n` truncation. +label and separate free-request/completion events. Critical first/last, shard, +request/completion and reuse boundaries are queried under both static policies +and checked independently against generated active IDs, sizes and counters. + +Seeded parameters cover legal event ranges, filters, sorting, offset/limit, +`max_rows`, `top_n`, `include_static`, and both `exact_total` settings. Selected +first/last/random member ranges are checked for event/allocation/peak/gap queries. +Empty/full/truncated/beyond-end pages occur in every seed. Many-member cases +assert actual scratch reducer construction, rather than inferring it from shard +count. Input hashes and absence of SQLite write sidecars are checked afterward. Comparisons are exact, including four-decimal percentages. The minimal 1-byte / 127-byte case requires `0.7813`; an 8-MiB scale variant also checks six-decimal @@ -38,6 +60,60 @@ pytest tests/test_fixture_provenance.py pytest tests/core/test_dataset_realistic.py tests/core/test_dataset_attribution.py tests/core/test_dataset_global.py ``` +The scheduled/manual CI `dataset-acceptance` job runs the 200-seed profile as a +separate required step after its installed-wheel nightly gate. The fail-closed +`.github/scripts/run_differential_acceptance.py` validates exactly 400 distinct +executed test IDs, both formats for all 200 expected seeds, matching JUnit and +per-seed coverage, expected comparison/template counts, zero unexpected errors, +and observed scratch execution. Skips, missing/duplicate cases, stale reports, +or inconsistent evidence fail the job. Its source SHA, summary, log, XML and +compact timeline/coverage/failure JSON are always uploaded as +`differential-acceptance-`; generated SQLite files are reproducible from the seed +and are not retained in CI artifacts. +This step is source-query acceptance; installed-wheel evidence stays separate. + +### Seed profiles and replay + +Run these from the intended checkout/interpreter (`PYTHONPATH="$PWD/src"` selects +an isolated worktree without repointing another editable installation): + +```bash +# PR/default: 20 seeds, both formats, plus the generator/oracle guard tests +PYTHONPATH="$PWD/src" pytest tests/core/test_dataset_differential_cases.py tests/core/test_dataset_realistic.py + +# Nightly: 200 seeds, preserving per-seed evidence and JUnit properties +PYTHONPATH="$PWD/src" PT_SNAP_DIFFERENTIAL_SEEDS=200 pytest \ + tests/core/test_dataset_realistic.py -k seeded \ + --basetemp=test_reports/differential --junitxml=test_reports/differential.xml \ + -o junit_family=xunit1 + +# Replay one seed and layout (the single-seed override wins over the bank size) +PYTHONPATH="$PWD/src" PT_SNAP_DIFFERENTIAL_SEED=196 pytest \ + tests/core/test_dataset_realistic.py -k 'seeded and v2' +``` + +`PT_SNAP_DIFFERENTIAL_SEEDS` accepts 1–10,000; `PT_SNAP_DIFFERENTIAL_SEED` accepts +0–2^63−1. Invalid/blank values fail collection, rather than silently skipping work. +The first eight bank entries are the retained regressions, followed by ascending +nonduplicate integers; the 200-seed profile is a strict extension of the default. +The pytest `--basetemp` directory is disposable and is cleared on the next run. + +Every successful seed writes `timeline.json` and `coverage.json`, including the +seed, layout, capacity, shard count, comparison count, templates, observed scratch +builds and zero unexpected errors. JUnit carries the same coverage properties. +Failures print the seed/format/parameters and a replay command, and retain +`differential-failure.json` with both full result envelopes or exception details. +One-sided errors and both-sided errors always fail. Expected unsupported-template +rejections are tested separately. Preserve the XML and temporary directory as +CI artifacts when diagnosing a failure. + +Terminal-leak parity here applies only to these proved, window-faithful histories, +where every live allocation appears in the terminal shard. It does not erase the +intentional standalone/dataset distinctions for inactive, earlier-only or +unproved histories. `test_dataset_lifecycle_contract.py` retains those contracts +across attached SQL, scratch and bounded fallback. `top_n=0` completeness has +separate empty/static/dynamic/mixed regressions across both layouts and reducers. + ## Slow scale and installed export gates `tests/core/test_dataset_scale_acceptance.py` is marked `slow` and contains: diff --git a/docs/en/querying.md b/docs/en/querying.md index 62b0e37..d09cd04 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -544,6 +544,12 @@ This query: - groups by allocation callstack - emits static and preexisting memory as dedicated groups instead of inventing a callstack +A zero-sized inner rank window (`top_n=0`) is also conservatively incomplete: +without `--exact-total`, `has_more=true`, `truncated=true`, and +`total_is_exact=false`, even if no rows are returned. This applies to standalone +and dataset queries, with or without static groups. `--exact-total` counts the +matching groups before the rank window and resolves whether any were omitted. + `top_n` only bounds dynamic callstack groups; `static` and `preexisting_live_at_event` groups, when included and present, are exempt from that inner filter. The outer `-n` cap still applies to all output rows and can diff --git a/docs/zh/dataset-acceptance.md b/docs/zh/dataset-acceptance.md index 4a247a7..e1ae70a 100644 --- a/docs/zh/dataset-acceptance.md +++ b/docs/zh/dataset-acceptance.md @@ -4,11 +4,28 @@ ## 常规正确性门禁 -`tests/core/test_dataset_realistic.py` 对八个固定种子各运行二十种查询形态,并与单库 -参考结果比较。每片仅包含窗口内新分配及进入窗口时仍存活的 block;已完成释放的 -生命周期不进入后续窗口,全局分配/完成释放 ID 保持不变。覆盖地址复用、跨窗口 -pending free、preexisting/static、NULL/空/空白栈、字面量缺失栈标签、排序、分页、 -筛选及动态组 `top_n` 截断。 +`tests/core/test_dataset_realistic.py` 默认在兼容 v1 和原生 v2 两种布局上各执行 +20 个确定性种子;夜间配置将同一批种子扩展至 200 个。原有八个回归种子及二十种 +查询形态完整保留。每个种子每种布局运行 69–73 个查询用例(20 种子共 2,896 次 +成对比较,200 种子共 28,864 次),覆盖全部十一个 dataset 支持模板。 +仅单库支持的 `event_frames`、`active_memory_frame_tree_at_event` 另有明确拒绝测试, +不会计为差分成功。 + +两种布局使用相同的独立生成 80-event 历史,并分别与同布局单库 SQL 参考结果比较。 +容量 2/3/4/5/10/16 产生 5–40 片,包括非满尾片。原生夹具包含真实局部栈引用、 +最终 block-reference 表、元数据和成员 hash,并通过实际原生 reader 校验;栈 ID +在不同片间变化。这是合成 reader/query 证据,不代表执行了原生 replay/export。 +每片只包含窗口内新分配及进入窗口时仍存活的生命周期,已完成释放的生命周期 +不进入后续窗口,全局分配和完成释放 ID 不变。每个种子均含地址复用、 +preexisting/static、NULL/空/空白栈、字面量缺失栈标签,以及分离的请求释放与完成 +释放事件。首尾、分片、请求/完成释放、复用等关键边界在两种 static 策略下均查询, +并独立对照生成模型的 active ID、大小和计数器。 + +随机参数覆盖合法事件范围、筛选、排序、offset/limit、`max_rows`、`top_n`、 +`include_static` 和两种 `exact_total`。event/allocation/peak/gap 查询还覆盖 +首片、尾片及随机片范围。每个种子固定包含空页、完整页、截断页及越过结果尾部的 +分页。多片用例断言实际构建 scratch reducer,而非仅凭片数推断;查询后核对 +输入 hash 不变,且没有 SQLite 写 sidecar。 所有比较严格相等,包括四位小数百分比。1-byte / 127-byte 最小用例要求 `0.7813`, 8-MiB 倍率另检查六位小数 GiB 舍入。契约是与当前 SQLite 运行时的运算及 `ROUND` @@ -29,6 +46,52 @@ pytest tests/test_fixture_provenance.py pytest tests/core/test_dataset_realistic.py tests/core/test_dataset_attribution.py tests/core/test_dataset_global.py ``` +定时/手动 CI 的 `dataset-acceptance` job 在安装包 nightly 门禁之后,另行执行 +必需的 200 种子差分步骤。`.github/scripts/run_differential_acceptance.py` 严格 +验证 400 个不同的已执行测试 ID、全部 200 个预期种子的两种布局、JUnit 与逐种子 +覆盖记录一致、预期比较数/模板数、零个意外错误及实际 scratch 执行。skip、缺失/ +重复用例、旧报告或证据不一致均导致 job 失败。源码 SHA、摘要、日志、XML 和种子 +timeline/coverage/failure JSON 始终上传为 `differential-acceptance-`;生成的 +SQLite 可由种子复现,不存入 CI artifact。该步骤属于源码查询验收; +安装包证据仍由单独门禁提供。 + +### 种子配置与复现 + +在目标 checkout/解释器中执行;`PYTHONPATH="$PWD/src"` 可选择独立 worktree, +无需改变其他目录的 editable 安装: + +```bash +# PR/默认:20 种子、两种布局,以及生成器/参考比较器校验 +PYTHONPATH="$PWD/src" pytest tests/core/test_dataset_differential_cases.py tests/core/test_dataset_realistic.py + +# 夜间:200 种子,保留逐种子证据及 JUnit 属性 +PYTHONPATH="$PWD/src" PT_SNAP_DIFFERENTIAL_SEEDS=200 pytest \ + tests/core/test_dataset_realistic.py -k seeded \ + --basetemp=test_reports/differential --junitxml=test_reports/differential.xml \ + -o junit_family=xunit1 + +# 复现单个种子和布局;单种子配置优先于种子数量 +PYTHONPATH="$PWD/src" PT_SNAP_DIFFERENTIAL_SEED=196 pytest \ + tests/core/test_dataset_realistic.py -k 'seeded and v2' +``` + +`PT_SNAP_DIFFERENTIAL_SEEDS` 接受 1–10,000;`PT_SNAP_DIFFERENTIAL_SEED` 接受 +0–2^63−1。非法或空值导致 collection 失败,不会静默跳过。种子列表先保留原八个 +回归种子,再按升序补充不重复整数;200 种子严格包含默认集合。pytest 的 +`--basetemp` 是可丢弃目录,下一次运行时会清空。 + +每个成功种子写入 `timeline.json`、`coverage.json`,记录种子、布局、容量、片数、 +比较数、模板、实际 scratch 构建次数及零个意外错误;JUnit 包含同样的覆盖属性。 +失败时输出种子/格式/参数及复现命令,并在 `differential-failure.json` 保存两边 +完整结果或异常详情。单边错误及两边同时错误均失败。预期的不支持模板拒绝另行 +验证。诊断 CI 失败时应保留 XML 和临时目录产物。 + +这里的终片泄漏候选等价性只适用于已证明、成员符合窗口的历史,所有存活分配均 +出现在终片;不消除 inactive、仅早期片出现或 unproved 历史上单库/dataset 的 +有意区别。`test_dataset_lifecycle_contract.py` 保留 attached SQL、scratch、有界 +fallback 三条路径的这些契约。`top_n=0` 完整性另有空/static/dynamic/混合用例, +覆盖两种布局和 reducer。 + ## slow 规模及安装包导出门禁 `tests/core/test_dataset_scale_acceptance.py` 标记为 `slow`,包括: diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 8ee3a58..b791bec 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -457,6 +457,11 @@ pt-snap query --template-use active_memory_callstack_at_event --params '{"event_ - 按分配调用栈做聚合 - 对静态内存和 preexisting 内存单独分组,而不是伪造调用栈 +零大小的内部排名窗口(`top_n=0`)同样保守地视为不完整:未使用 +`--exact-total` 时,即使返回零行,也有 `has_more=true`、`truncated=true`、 +`total_is_exact=false`。此规则同时适用于单库和 dataset 查询,与是否包含静态组无关。 +`--exact-total` 会统计排名窗口之前的匹配分组,从而确定是否存在被省略的组。 + `top_n` 只限制动态调用栈分组的数量;启用包含且实际存在的 `static` 和 `preexisting_live_at_event` 分组不受这层内部筛选限制。外层 `-n` 仍限制所有 输出行,可能裁掉特殊组或动态 top 组。对于有限正整数 `top_n=N` 且 diff --git a/src/pt_snap_cli/core/dataset_active_sql.py b/src/pt_snap_cli/core/dataset_active_sql.py index dd655c4..10e035b 100644 --- a/src/pt_snap_cli/core/dataset_active_sql.py +++ b/src/pt_snap_cli/core/dataset_active_sql.py @@ -194,7 +194,7 @@ def reduce_active( ) total = cast(int, summary["total"]) dynamic = cast(int, summary["dynamic_groups"]) - rank_full = top_n >= 0 and dynamic > 0 and dynamic >= top_n + rank_full = top_n >= 0 and dynamic >= top_n offset = 0 coverage["source_queries"] = sources.query_count return rows, coverage, total, rank_full, offset diff --git a/src/pt_snap_cli/core/dataset_attribution.py b/src/pt_snap_cli/core/dataset_attribution.py index 7745e68..ec608c2 100644 --- a/src/pt_snap_cli/core/dataset_attribution.py +++ b/src/pt_snap_cli/core/dataset_attribution.py @@ -79,7 +79,7 @@ def event_attribution( all_rows = _groups(sources, blocks, references) top_n = cast(int, params["top_n"]) dynamic = [row for row in all_rows if row["category"] == "dynamic_live_at_event"] - rank_full = top_n >= 0 and bool(dynamic) and len(dynamic) >= top_n + rank_full = top_n >= 0 and len(dynamic) >= top_n selected = [row for row in all_rows if row["category"] != "dynamic_live_at_event"] + ( dynamic[:top_n] if top_n >= 0 else dynamic ) diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 2b5049e..bafec1b 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -491,6 +491,10 @@ def _inner_rank_window_full(rows: list[dict[str, Any]], rank_cap: int) -> bool: ``rank_cap`` is treated as possibly incomplete unless ``exact_total`` later proves ``total == returned``. """ + # A zero-cap window can hide every ranked group. Empty output cannot + # establish completeness until exact_total explicitly counts without it. + if rank_cap == 0: + return True if not rows: return False if any("category" in row for row in rows): diff --git a/tests/core/dataset_differential_assertions.py b/tests/core/dataset_differential_assertions.py new file mode 100644 index 0000000..68f6a07 --- /dev/null +++ b/tests/core/dataset_differential_assertions.py @@ -0,0 +1,74 @@ +"""Strict differential comparison with deterministic failure evidence.""" + +import dataclasses +import json + +# Other result metadata describes dataset-only source provenance. Every field +# returned by the standalone SQL oracle is compared, plus these envelope fields. +ENVELOPE_FIELDS = ( + "total", + "returned", + "total_is_exact", + "has_more", + "truncated", + "device_id", + "template", + "semantics_version", +) + + +def compare_case(actual, reference, query, root, seed, layout, capacity): + context = {"seed": seed, "format": layout, "capacity": capacity, **dataclasses.asdict(query)} + outcomes = {} + errors = {} + for label, analyzer in (("dataset", actual), ("standalone", reference)): + options = query.options() + if label == "dataset" and query.slice_index is not None: + options["slice_index"] = query.slice_index + try: + outcomes[label] = analyzer.execute_query(query.template, query.params, **options) + except Exception as exc: + # Capture both outcomes for diagnosis, but NEVER accept two failures. + errors[label] = {"type": type(exc).__name__, "message": str(exc)} + try: + assert not errors, f"unexpected query errors: {errors}" + got, want = outcomes["dataset"], outcomes["standalone"] + # Dataset adds provenance fields. Only state at a point is observational + # rather than comparable; the caller independently checks its owner-slice value. + omit = {"state"} if query.template == "active_blocks_at_event" else set() + fields = set(want["rows"][0]) - omit if want["rows"] else set() + actual_rows = [{key: row[key] for key in fields} for row in got["rows"]] + expected_rows = [{key: row[key] for key in fields} for row in want["rows"]] + assert actual_rows == expected_rows, "row values/order differ" + for key in ENVELOPE_FIELDS: + assert got[key] == want[key], f"{key}: {got[key]!r} != {want[key]!r}" + assert got["scope"]["boundary_events_included"] is False + if query.slice_index is not None: + assert got["scope"]["slice_indices"] == [query.slice_index] + if query.template == "leak_detection": + # #254: equality is valid here because this generated, proved history + # retains every live allocation at the terminal shard. It is NOT a + # universal equivalence for earlier-only/unproved/inactive histories. + for row in got["rows"]: + assert row["terminal_survivor"] and row["state"] in (0, 1) + assert row["identity_status"] == "proved_allocation_event" + assert row["freeEventId"] == -1 and row["allocEventId"] >= 0 + return got + except (AssertionError, KeyError) as exc: + evidence = root / "differential-failure.json" + evidence.write_text( + json.dumps( + {"case": context, "errors": errors, "outcomes": outcomes}, + ensure_ascii=False, + indent=2, + ) + ) + replay = ( + f"PT_SNAP_DIFFERENTIAL_SEED={seed} pytest " + "tests/core/test_dataset_realistic.py -k " + f"'seeded and {layout}'" + ) + raise AssertionError( + f"{exc}; case={json.dumps(context, sort_keys=True)}; " + f"evidence={evidence}; replay: {replay}" + ) from exc diff --git a/tests/core/dataset_differential_cases.py b/tests/core/dataset_differential_cases.py new file mode 100644 index 0000000..589d1c4 --- /dev/null +++ b/tests/core/dataset_differential_cases.py @@ -0,0 +1,213 @@ +"""Bounded, reproducible seed profiles and valid query cases for both layouts.""" + +import os +import random +import re +from dataclasses import dataclass, field + +from tests.core.dataset_lifecycle_fixtures import EVENT_COUNT, shapes + +REGRESSION_SEEDS = (0, 7, 31, 67, 136, 196, 2026, 8191) +DEFAULT_SEEDS = 20 +MAX_SEEDS = 10_000 +SUPPORTED_TEMPLATES = frozenset( + ( + "memory_peak", + "allocator_gap", + "allocation", + "event", + "block", + "leak_detection", + "freed_block_lifetime", + "callstack_analysis", + "preexisting_live", + "active_blocks_at_event", + "active_memory_callstack_at_event", + ) +) +UNSUPPORTED_TEMPLATES = frozenset(("event_frames", "active_memory_frame_tree_at_event")) + + +def configured_seeds(environ=None): + """The nightly bank strictly extends PR seeds; replay overrides the count.""" + environ = os.environ if environ is None else environ + + def integer(name, default, minimum, maximum): + raw = environ.get(name, str(default)).strip() + if re.fullmatch(r"[0-9]+", raw) is None or not minimum <= int(raw) <= maximum: + raise ValueError(f"{name} must be an integer in [{minimum}, {maximum}], got {raw!r}") + return int(raw) + + count = integer("PT_SNAP_DIFFERENTIAL_SEEDS", DEFAULT_SEEDS, 1, MAX_SEEDS) + if "PT_SNAP_DIFFERENTIAL_SEED" in environ: + return (integer("PT_SNAP_DIFFERENTIAL_SEED", 0, 0, 2**63 - 1),) + bank = list(REGRESSION_SEEDS) + candidate = 0 + while len(bank) < count: + if candidate not in REGRESSION_SEEDS: + bank.append(candidate) + candidate += 1 + return tuple(bank[:count]) + + +def capacity_for_seed(seed): + # 5 to 40 shards, including a partial final shard (capacity=3). + return (2, 3, 4, 5, 10, 16)[seed % 6] + + +@dataclass(frozen=True) +class QueryCase: + label: str + template: str + params: dict = field(default_factory=dict) + exact_total: bool = True + max_rows: int | None = None + slice_index: int | None = None + active_oracle: bool = False + + def options(self): + return {"exact_total": self.exact_total, "max_rows": self.max_rows} + + +def query_cases(seed, capacity): + rng = random.Random(seed + 1_000_003) + cases = [ + QueryCase(f"regression-{index}", template, params) + for index, (template, params) in enumerate(shapes(seed)) + ] + # Every seed exercises critical transitions and both static policies; random + # coverage cannot accidentally omit free_requested vs free_completed/reuse. + boundaries = sorted({0, capacity - 1, capacity, 17, 18, 22, 23, 24, 60, 61, EVENT_COUNT - 1}) + for at in boundaries: + for include_static in (True, False): + cases.append( + QueryCase( + f"boundary-{at}-static-{include_static}", + "active_blocks_at_event", + {"event_id": at, "include_static": include_static}, + exact_total=include_static, + active_oracle=True, + ) + ) + # Empty, full, truncated, and beyond-end pages with both exactness settings + # occur per seed, rather than waiting for the random parameter bank. + for exact in (False, True): + for offset, limit, max_rows in ((0, 0, None), (0, -1, None), (2, 7, 3), (999, 2, None)): + cases.append( + QueryCase( + f"page-{exact}-{offset}-{limit}", + "event", + { + "min_id": 0, + "offset": offset, + "limit": limit, + "order_by": "id", + "order_dir": rng.choice(("ASC", "DESC")), + }, + exact_total=exact, + max_rows=max_rows, + ) + ) + low, high = sorted(rng.sample(range(EVENT_COUNT), 2)) + at = rng.choice(boundaries) + + def paging(choices): + return { + "order_by": rng.choice(choices), + "order_dir": rng.choice(("ASC", "DESC")), + "offset": rng.choice((0, 1, 7, EVENT_COUNT + 1)), + "limit": rng.choice((-1, 0, 1, 5, 19)), + } + + params = { + "memory_peak": {"start_id": low, "end_id": high}, + "allocator_gap": {"start_id": low, "end_id": high}, + "allocation": { + "min_id": low, + "max_id": high, + "min_active": rng.choice((0, 256, 1024)), + **paging(("id", "allocated", "active", "reserved")), + }, + "event": { + "min_id": low, + "max_id": high, + "action": rng.choice((4, 5, 6, 7)), + "stream": rng.choice((0, 1)), + **paging( + ( + "id", + "action", + "address", + "size", + "stream", + "allocated", + "active", + "reserved", + "callstack", + ) + ), + }, + "block": { + "min_size": rng.choice((0, 16, 32, 128)), + "max_size": 256, + "min_allocEventId": rng.choice((-1, 0, low)), + **paging( + ("id", "address", "size", "requestedSize", "state", "allocEventId", "freeEventId") + ), + }, + "leak_detection": { + "min_size": rng.choice((0, 16, 64, 128)), + "limit": rng.choice((-1, 0, 1, 7)), + }, + "freed_block_lifetime": {}, + "callstack_analysis": { + "min_count": rng.choice((1, 2, 5)), + "min_size": rng.choice((0, 32, 128)), + "limit": rng.choice((-1, 0, 1, 7)), + }, + "preexisting_live": {"event_id": at}, + "active_blocks_at_event": { + "event_id": at, + "include_static": rng.choice((True, False)), + "min_size": rng.choice((0, 16, 128)), + **paging(("size", "requestedSize", "allocEventId", "freeEventId", "id")), + }, + "active_memory_callstack_at_event": { + "event_id": at, + "include_static": rng.choice((True, False)), + "min_size": rng.choice((0, 16, 128)), + "top_n": rng.choice((-1, 0, 1, 2, 20)), + }, + } + for template, parameters in params.items(): + cases.append( + QueryCase( + f"random-{template}", + template, + parameters, + exact_total=rng.choice((False, True)), + max_rows=rng.choice((None, 0, 1, 4)), + ) + ) + # Explicit shard selection has a matching standalone event predicate. Leak + # slice scope is intentionally unsupported and is not a differential pass. + for index in sorted( + {0, (EVENT_COUNT - 1) // capacity, rng.randrange((EVENT_COUNT - 1) // capacity + 1)} + ): + lo, hi = index * capacity, min((index + 1) * capacity, EVENT_COUNT) - 1 + for template in ("event", "allocation", "memory_peak", "allocator_gap"): + bounds = ( + {"min_id": lo, "max_id": hi} + if template in ("event", "allocation") + else {"start_id": lo, "end_id": hi} + ) + cases.append( + QueryCase( + f"slice-{index}-{template}", + template, + bounds, + exact_total=rng.choice((False, True)), + slice_index=index, + ) + ) + return cases diff --git a/tests/core/dataset_lifecycle_fixtures.py b/tests/core/dataset_lifecycle_fixtures.py index fd945f6..2e23c1e 100644 --- a/tests/core/dataset_lifecycle_fixtures.py +++ b/tests/core/dataset_lifecycle_fixtures.py @@ -1,16 +1,19 @@ -"""Deterministic synthetic v1 fixtures with replay-faithful window membership. +"""Deterministic synthetic v1/native-v2 fixtures with replay-faithful window membership. These fixtures exercise SQL/query contracts, not the snapshot producer/exporter. The independent model retains full lifecycle IDs across windows, includes only carry-in/new allocations, and drops lifetimes completed before a window starts. """ +import hashlib import json import random import sqlite3 from contextlib import closing from pt_snap_cli.core.dataset_contract import ACTION_NAMES, BLOCK_STATES, validate_dataset +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from pt_snap_cli.core.native_dataset_contract import NATIVE_FORMAT, cache_identity EVENT_COUNT = 80 CAPACITY = 10 @@ -44,13 +47,14 @@ def timeline(seed): }, } trace = [] - nextaddr = 2000 + # Slots exceed the largest block, so live address intervals cannot overlap. + nextaddr = 5000 fixed = { 0: ("a", 1000, "shared"), - 1: ("a", 1100, None), - 2: ("a", 1200, ""), - 3: ("a", 1300, "[missing callstack]"), - 4: ("a", 1400, " \n"), + 1: ("a", 1512, None), + 2: ("a", 2024, ""), + 3: ("a", 2536, "[missing callstack]"), + 4: ("a", 3048, " \n"), 17: ("r", -1), 18: ("f", -1), 22: ("r", 0), @@ -79,12 +83,12 @@ def timeline(seed): b["address"] for b in blocks.values() if b["free"] >= 0 - and b["address"] >= 2000 + and b["address"] >= 5000 and all(c["address"] != b["address"] or c["free"] >= 0 for c in blocks.values()) ] address = rng.choice(available) if available and rng.random() < 0.6 else nextaddr if address == nextaddr: - nextaddr += 100 + nextaddr += 512 op = ( "a", address, @@ -168,16 +172,20 @@ def db(path, trace, rows, boundary=True): c.executemany("INSERT INTO block_0 VALUES (?,?,?,?,?,?,?)", rows) -def build(out, seed): - root = out / f"seed-{seed:04d}" +def build(out, seed, *, layout="v1", capacity=CAPACITY): + if layout not in ("v1", "v2"): + raise ValueError(f"unsupported fixture layout: {layout}") + if not 1 <= capacity <= EVENT_COUNT: + raise ValueError(f"invalid fixture capacity: {capacity}") + root = out / f"seed-{seed:04d}-{layout}" root.mkdir() dataset = root / "dataset" (dataset / "device_0").mkdir(parents=True) trace, blocks = timeline(seed) slices = [] - layout = [] - for i, lo in enumerate(range(0, EVENT_COUNT, CAPACITY)): - hi = lo + CAPACITY - 1 + windows = [] + for i, lo in enumerate(range(0, EVENT_COUNT, capacity)): + hi = min(lo + capacity, EVENT_COUNT) - 1 rows = [row(b, state_at_left(b, lo)) for b in blocks if member(b, lo, hi)] assert all(b[5] == -1 or b[5] <= hi for b in rows) assert all(b[6] == -1 or b[6] >= lo for b in rows) @@ -186,13 +194,13 @@ def build(out, seed): slices.append( {"index": i, "startEventId": lo, "endEventId": hi, "file": file, "ready": True} ) - layout.append({"slice": i, "lo": lo, "hi": hi, "block_ids": [r[0] for r in rows]}) + windows.append({"slice": i, "lo": lo, "hi": hi, "block_ids": [r[0] for r in rows]}) manifest = { "schemaVersion": 1, "status": "complete", "sourceFile": "/synthetic/no-pickle", "cacheHash": "", - "eventsPerSlice": CAPACITY, + "eventsPerSlice": capacity, "devices": { "0": { "eventCount": EVENT_COUNT, @@ -211,11 +219,7 @@ def build(out, seed): b, state_at_left( b, - max( - i * CAPACITY - for i in range(EVENT_COUNT // CAPACITY) - if member(b, i * CAPACITY, (i + 1) * CAPACITY - 1) - ), + max(window["lo"] for window in windows if member(b, window["lo"], window["hi"])), ), ) for b in blocks @@ -223,12 +227,23 @@ def build(out, seed): db(single, trace, finalrows) (root / "timeline.json").write_text( json.dumps( - {"seed": seed, "trace": trace, "blocks": blocks, "layout": layout}, + { + "seed": seed, + "format": layout, + "capacity": capacity, + "trace": trace, + "blocks": blocks, + "layout": windows, + }, ensure_ascii=False, indent=2, ) ) - validate_dataset(dataset) + if layout == "v2": + native_dataset(dataset, single, trace, blocks, manifest, seed, capacity) + assert DatasetResolver().inspect(dataset).callstack_layout == "v2" + else: + validate_dataset(dataset) return root, dataset, single, trace, blocks @@ -274,3 +289,89 @@ def shapes(seed): {"event_id": event, "top_n": 1, "include_static": False, "min_size": 16}, ), ] + + +def native_dataset(dataset, single, trace, blocks, manifest, seed, capacity): + """Write independently modeled native tables, not a relabeled v1 manifest. + + All trace references, including NULL-text stacks, have real local stack IDs. + Their assignment rotates per shard so local IDs are never global identities. + This exercises the reader; it deliberately does not claim exporter coverage. + """ + source = json.dumps({"trace": trace, "blocks": blocks}, sort_keys=True).encode() + digest = hashlib.sha256(source).hexdigest() + metadata = { + "metadata_schema_version": 1, + "import_format_version": 2, + "source_sha256": digest, + "source_size": len(source), + "source_name": f"synthetic-seed-{seed}.json", + "requested_device": None, + "importer_name": "synthetic-differential-fixture", + "importer_version": "1", + "completed_at": "2026-01-01T00:00:00+00:00", + } + slices = manifest["devices"]["0"]["slices"] + for index, path in enumerate([*(dataset / s["file"] for s in slices), single]): + with closing(sqlite3.connect(path)) as connection, connection: + events = connection.execute("SELECT * FROM trace_entry_0 ORDER BY id").fetchall() + texts = list(dict.fromkeys(event[-1] for event in events)) + random.Random(seed + index).shuffle(texts) + stack_ids = {text: i for i, text in enumerate(texts)} + connection.executescript( + "ALTER TABLE trace_entry_0 RENAME TO inline_trace;" + "CREATE TABLE trace_entry_0 (id INTEGER PRIMARY KEY,action INTEGER," + "address INTEGER,size INTEGER,stream INTEGER,allocated INTEGER," + "active INTEGER,reserved INTEGER,callstackId INTEGER);" + "CREATE TABLE callstack (id INTEGER PRIMARY KEY,callstack TEXT);" + "CREATE TABLE pt_snap_block_reference (blockId INTEGER PRIMARY KEY," + "stream INTEGER,allocCallstack TEXT,freeCallstack TEXT);" + "CREATE TABLE pt_snap_metadata (id INTEGER PRIMARY KEY," + "metadata_schema_version INTEGER,import_format_version INTEGER," + "source_sha256 TEXT,source_size INTEGER,source_name TEXT," + "requested_device INTEGER,importer_name TEXT,importer_version TEXT," + "completed_at TEXT);" + ) + connection.executemany("INSERT INTO callstack VALUES (?,?)", enumerate(texts)) + connection.executemany( + "INSERT INTO trace_entry_0 VALUES (?,?,?,?,?,?,?,?,?)", + [(*event[:-1], stack_ids[event[-1]]) for event in events], + ) + connection.execute("DROP TABLE inline_trace") + connection.execute("INSERT INTO dictionary VALUES ('trace_entry_0','action','8','oom')") + contained = {row[0] for row in connection.execute("SELECT id FROM block_0")} + connection.executemany( + "INSERT INTO pt_snap_block_reference VALUES (?,?,?,?)", + [ + ( + b["id"], + trace[b["alloc"]][4] if b["alloc"] >= 0 else 0, + b["stack"], + trace[b["free"]][-1] if b["free"] >= 0 else None, + ) + for b in blocks + if b["id"] in contained + ], + ) + connection.execute( + "INSERT INTO pt_snap_metadata VALUES (1,?,?,?,?,?,?,?,?,?)", + list(metadata.values()), + ) + if index < len(slices): + item = slices[index] + item.update( + startPosition=item["startEventId"], + endPosition=item["endEventId"], + sha256=hashlib.sha256(path.read_bytes()).hexdigest(), + ) + native = { + "format": NATIVE_FORMAT, + "schemaVersion": 1, + "status": "complete", + "sourceFile": metadata["source_name"], + "identity": cache_identity(digest, None, capacity), + "metadata": metadata, + "omittedDevices": [], + "devices": manifest["devices"], + } + (dataset / "manifest.json").write_text(json.dumps(native)) diff --git a/tests/core/test_dataset_differential_cases.py b/tests/core/test_dataset_differential_cases.py new file mode 100644 index 0000000..1cf4f1f --- /dev/null +++ b/tests/core/test_dataset_differential_cases.py @@ -0,0 +1,151 @@ +"""Guard seed profiles, coverage classification, and the differential oracle itself.""" + +import json +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.dataset_support import DATASET_SUPPORT +from pt_snap_cli.core.errors import QueryExecutionError +from pt_snap_cli.query import registry +from tests.core.dataset_differential_assertions import compare_case +from tests.core.dataset_differential_cases import ( + REGRESSION_SEEDS, + SUPPORTED_TEMPLATES, + UNSUPPORTED_TEMPLATES, + QueryCase, + capacity_for_seed, + configured_seeds, + query_cases, +) +from tests.core.dataset_lifecycle_fixtures import EVENT_COUNT, build, shapes, timeline + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_QUERY_TIMEOUT", raising=False) + + +def test_default_and_nightly_banks_are_stable_unique_and_preserve_regressions(): + default = configured_seeds({}) + nightly = configured_seeds({"PT_SNAP_DIFFERENTIAL_SEEDS": "200"}) + assert len(default) == len(set(default)) == 20 + assert len(nightly) == len(set(nightly)) == 200 + assert default[:8] == REGRESSION_SEEDS + assert nightly[:20] == default + assert configured_seeds({"PT_SNAP_DIFFERENTIAL_SEEDS": "1"}) == (0,) + assert configured_seeds({"PT_SNAP_DIFFERENTIAL_SEED": "0"}) == (0,) + assert configured_seeds( + {"PT_SNAP_DIFFERENTIAL_SEED": "8191", "PT_SNAP_DIFFERENTIAL_SEEDS": "200"} + ) == (8191,) + assert configured_seeds({"PT_SNAP_DIFFERENTIAL_SEED": str(2**63 - 1)}) == (2**63 - 1,) + + +@pytest.mark.parametrize( + "name,values", + [ + ("PT_SNAP_DIFFERENTIAL_SEEDS", ("", "0", "-1", "200.0", "true", "10001", "bad")), + ("PT_SNAP_DIFFERENTIAL_SEED", ("", "-1", "1.5", "True", str(2**63), "1,2")), + ], +) +def test_invalid_seed_configuration_fails_instead_of_silently_skipping(name, values): + for value in values: + with pytest.raises(ValueError, match=name): + configured_seeds({name: value}) + + +def test_query_bank_preserves_original_shapes_and_all_supported_templates(monkeypatch): + assert set(DATASET_SUPPORT) == SUPPORTED_TEMPLATES + # Other suites legitimately register custom SQL. Check packaged discovery + # in a fresh registry, and restore both singleton and identity bookkeeping. + with monkeypatch.context() as patch: + patch.setattr(registry.QueryRegistry, "_instance", None) + patch.setattr(registry, "_registry", registry.QueryRegistry()) + patch.setattr(registry, "_packaged", {}) + registry._load_all_templates() + assert set(registry.list_queries()) == SUPPORTED_TEMPLATES | UNSUPPORTED_TEMPLATES + for seed in configured_seeds({"PT_SNAP_DIFFERENTIAL_SEEDS": "200"}): + capacity = capacity_for_seed(seed) + cases = query_cases(seed, capacity) + assert cases == query_cases(seed, capacity) + assert [(q.template, q.params) for q in cases[:20]] == shapes(seed) + assert {q.template for q in cases} == SUPPORTED_TEMPLATES + assert {q.exact_total for q in cases} == {False, True} + assert {q.params["include_static"] for q in cases if q.active_oracle} == {False, True} + for q in cases: + for key in ("event_id", "min_id", "max_id", "start_id", "end_id"): + if key in q.params: + assert 0 <= q.params[key] < EVENT_COUNT + if "min_id" in q.params and "max_id" in q.params: + assert q.params["min_id"] <= q.params["max_id"] + if "start_id" in q.params: + assert q.params["start_id"] <= q.params["end_id"] + if q.slice_index is not None: + assert q.template in {"event", "allocation", "memory_peak", "allocator_gap"} + assert 0 <= q.slice_index <= (EVENT_COUNT - 1) // capacity + traces, blocks = timeline(seed) + assert (traces, blocks) == timeline(seed) + for at in range(EVENT_COUNT): + alive = [b for b in blocks if b["alloc"] <= at and (b["free"] == -1 or b["free"] > at)] + spans = sorted((b["address"], b["address"] + b["size"]) for b in alive) + assert all( + left[1] <= right[0] for left, right in zip(spans, spans[1:], strict=False) + ), (seed, at) + assert sum(b["size"] for b in alive) == traces[at][6] + assert ( + sum(b["size"] for b in alive if b["request"] == -1 or b["request"] > at) + == traces[at][5] + ) + + +@pytest.mark.parametrize("layout", ("v1", "v2")) +@pytest.mark.parametrize("template", sorted(UNSUPPORTED_TEMPLATES)) +def test_unsupported_catalog_templates_reject_separately(tmp_path, layout, template): + _, dataset, _, _, _ = build(tmp_path, 0, layout=layout) + params = {"event_id": 0} + with SnapshotAnalyzer(dataset) as analyzer: + with pytest.raises(QueryExecutionError, match="Unsupported dataset-global template"): + analyzer.execute_query(template, params) + + +@pytest.mark.parametrize("errors", ((True, True), (True, False), (False, True))) +def test_any_unexpected_error_fails_and_records_both_outcomes(tmp_path, errors): + calls = [] + + def analyzer(index): + def execute(*args, **kwargs): + calls.append(index) + if errors[index]: + raise RuntimeError(f"injected failure {index}") + return {"rows": []} + + return SimpleNamespace(execute_query=execute) + + with pytest.raises(AssertionError, match="unexpected query errors.*seed"): + compare_case( + analyzer(0), analyzer(1), QueryCase("error-check", "event"), tmp_path, 31, "v2", 2 + ) + assert calls == [0, 1] + saved = json.loads((tmp_path / "differential-failure.json").read_text()) + assert len(saved["errors"]) == sum(errors) + assert len(saved["outcomes"]) == 2 - sum(errors) + assert saved["case"]["seed"] == 31 and saved["case"]["format"] == "v2" + + +def test_native_and_compat_fixtures_share_identical_generated_history(tmp_path): + left = build(tmp_path, 7, layout="v1", capacity=3) + right = build(tmp_path, 7, layout="v2", capacity=3) + assert left[3:] == right[3:] + first = json.loads((left[0] / "timeline.json").read_text()) + second = json.loads((right[0] / "timeline.json").read_text()) + assert {k: v for k, v in first.items() if k != "format"} == { + k: v for k, v in second.items() if k != "format" + } + assert len(first["layout"]) == 27 + assert first["layout"][-1]["hi"] == EVENT_COUNT - 1 diff --git a/tests/core/test_dataset_realistic.py b/tests/core/test_dataset_realistic.py index 0d61ad4..f2b295e 100644 --- a/tests/core/test_dataset_realistic.py +++ b/tests/core/test_dataset_realistic.py @@ -16,13 +16,17 @@ from pt_snap_cli.api import SnapshotAnalyzer from pt_snap_cli.cli import app from pt_snap_cli.core import dataset_attribution +from pt_snap_cli.core import dataset_lifecycle_scratch as scratch +from pt_snap_cli.core import dataset_lifecycle_sql as lifecycle_sql from pt_snap_cli.core.dataset_contract import validate_dataset from pt_snap_cli.core.dataset_sources import QueryBudget from pt_snap_cli.core.errors import QueryExecutionError, QueryTimeoutError from pt_snap_cli.core.import_service import ImportService from pt_snap_cli.core.models import ImportOptions from pt_snap_cli.core.query_service import QueryService -from tests.core.dataset_lifecycle_fixtures import CAPACITY, build, db, member, shapes +from tests.core.dataset_differential_assertions import compare_case +from tests.core.dataset_differential_cases import capacity_for_seed, configured_seeds, query_cases +from tests.core.dataset_lifecycle_fixtures import build, db, member from tests.core.test_dataset_attribution import case from tests.snapshot.test_sharded_replay import block, event, segment, write_source from tests.test_dataset_focus import hashes @@ -58,44 +62,84 @@ def test_existing_attribution_fixture_has_only_actual_window_members(tmp_path): assert -21 in set.intersection(*observed) -@pytest.mark.parametrize("seed", [0, 7, 31, 67, 136, 196, 2026, 8191]) -def test_seeded_lifecycle_windows_match_single_database_strictly(tmp_path, seed): - root, dataset, single, traces, blocks = build(tmp_path, seed) +@pytest.mark.parametrize("seed", configured_seeds(), ids=lambda seed: f"seed-{seed}") +@pytest.mark.parametrize("layout", ("v1", "v2")) +def test_seeded_lifecycle_windows_match_single_database_strictly( + tmp_path, monkeypatch, record_property, seed, layout +): + capacity = capacity_for_seed(seed) + root, dataset, single, traces, blocks = build(tmp_path, seed, layout=layout, capacity=capacity) before = hashes(root) # Every seed contains these cases, rather than relying on probability. assert {b["stack"] for b in blocks} >= {None, "", "[missing callstack]", " \n"} assert sum(b["address"] == 1000 for b in blocks) == 2 assert any(b["alloc"] == -1 and b["free"] == -1 for b in blocks) assert any(b["alloc"] == -1 and b["free"] >= 0 for b in blocks) + assert any(b["request"] >= 0 and b["free"] > b["request"] for b in blocks) assert any( - b["alloc"] >= 0 and b["free"] >= 0 and b["alloc"] // CAPACITY != b["free"] // CAPACITY + b["alloc"] >= 0 and b["free"] >= 0 and b["alloc"] // capacity != b["free"] // capacity for b in blocks ) - for entry in json.loads((root / "timeline.json").read_text())["layout"]: + windows = json.loads((root / "timeline.json").read_text())["layout"] + for entry in windows: expected = {b["id"] for b in blocks if member(b, entry["lo"], entry["hi"])} assert set(entry["block_ids"]) == expected + scratch_builds = [] + original_build = scratch._ScratchSources.build + + def observed_build(database, items, maximum): + result = original_build(database, items, maximum) + scratch_builds.append(len(items)) + return result + + monkeypatch.setattr(scratch._ScratchSources, "build", observed_build) + cases = query_cases(seed, capacity) with SnapshotAnalyzer(dataset) as actual, SnapshotAnalyzer(single) as reference: - for template, params in shapes(seed): - got = actual.execute_query(template, params, exact_total=True) - want = reference.execute_query(template, params, exact_total=True) - omitted = ("state",) if template == "active_blocks_at_event" else () - got_rows, want_rows = common_rows(got["rows"], want["rows"], omit=omitted) - assert got_rows == want_rows, (seed, template, params) - for key in ("total", "returned", "total_is_exact", "has_more", "truncated"): - assert got[key] == want[key], (seed, template, params, key) - if template == "active_blocks_at_event": - at = params["event_id"] + for query in cases: + got = compare_case(actual, reference, query, root, seed, layout, capacity) + if query.active_oracle: + at = query.params["event_id"] expected = { b["id"] for b in blocks if (b["alloc"] == -1 or b["alloc"] <= at) and (b["free"] == -1 or b["free"] > at) + and (query.params["include_static"] or b["alloc"] >= 0) } - assert {r["id"] for r in got["rows"]} == expected - assert sum(r["size"] for r in got["rows"]) == traces[at][6] - assert got["scope"]["source_coverage"]["allocation_source_complete"] + context = (seed, layout, query) + assert {r["id"] for r in got["rows"]} == expected, context + assert sum(r["size"] for r in got["rows"]) == sum( + b["size"] for b in blocks if b["id"] in expected + ), context + if query.params["include_static"]: + assert sum(r["size"] for r in got["rows"]) == traces[at][6], context + assert got["scope"]["source_coverage"]["allocation_source_complete"], context + for result in got["rows"]: + block = next(b for b in blocks if b["id"] == result["id"]) + assert result["state_scope"] == "slice_observation", context + assert result["state"] == ( + 0 if 0 <= block["request"] < (at // capacity) * capacity else 1 + ), context + # Native and compatibility inputs with more owners than SQLite can ATTACH + # must actually build the bounded on-disk reducer; shard count alone is not evidence. + if len(windows) > lifecycle_sql._attachment_limit() > 0: + assert scratch_builds, (seed, layout, capacity, len(windows)) + assert max(scratch_builds) == len(windows) assert hashes(root) == before - assert not list(root.rglob("*-wal")) + assert not any(root.rglob("*-wal")) and not any(root.rglob("*-journal")) + summary = { + "seed": seed, + "format": layout, + "capacity": capacity, + "shards": len(windows), + "comparisons": len(cases), + "unexpected_errors": 0, + "templates": sorted({q.template for q in cases}), + "scratch_builds": len(scratch_builds), + } + (root / "coverage.json").write_text(json.dumps(summary, indent=2)) + for key, value in summary.items(): + record_property(key, value) def rounding_case(tmp_path, sizes): diff --git a/tests/core/test_dataset_zero_rank.py b/tests/core/test_dataset_zero_rank.py new file mode 100644 index 0000000..f247249 --- /dev/null +++ b/tests/core/test_dataset_zero_rank.py @@ -0,0 +1,127 @@ +"""A full zero-sized inner rank window remains conservative until exact COUNT.""" + +import json +from pathlib import Path + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core import dataset_attribution +from pt_snap_cli.core.dataset_resolver import DatasetResolver +from tests.core.dataset_lifecycle_fixtures import db, native_dataset, row + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + monkeypatch.delenv("PT_SNAP_DB_PATH", raising=False) + monkeypatch.delenv("PT_SNAP_QUERY_TIMEOUT", raising=False) + + +def zero_rank_pair(tmp_path, layout, kind): + """Small explicit empty/static/dynamic/mixed histories, independent of RNG.""" + dataset = tmp_path / "dataset" + (dataset / "device_0").mkdir(parents=True) + static = { + "id": -1, + "address": 100, + "size": 16, + "requestedSize": 16, + "alloc": -1, + "free": -1, + "stack": None, + } + dynamic = { + "id": 0, + "address": 200, + "size": 32, + "requestedSize": 32, + "alloc": 0, + "free": -1, + "stack": "dynamic", + } + blocks = {"empty": [], "static": [static], "dynamic": [dynamic], "mixed": [static, dynamic]}[ + kind + ] + active = sum(b["size"] for b in blocks) + trace = [ + (0, 4, 200, 32, 0, active, active, 256, "dynamic"), + (1, 7, 0, 0, 0, active, active, 256, None), + ] + rows = [row(b, 1) for b in blocks] + slices = [] + for index in range(2): + file = f"device_0/slice_{index:05d}.db" + db(dataset / file, [trace[index]], rows) + slices.append( + { + "index": index, + "startEventId": index, + "endEventId": index, + "file": file, + "ready": True, + } + ) + manifest = { + "schemaVersion": 1, + "status": "complete", + "sourceFile": "/synthetic/no-pickle", + "cacheHash": "", + "eventsPerSlice": 1, + "devices": { + "0": {"eventCount": 2, "sliceCount": 2, "readySlices": [0, 1], "slices": slices} + }, + } + (dataset / "manifest.json").write_text(json.dumps(manifest)) + single = tmp_path / "single.db" + db(single, trace, rows) + if layout == "v2": + native_dataset(dataset, single, trace, blocks, manifest, 0, 1) + assert DatasetResolver().inspect(dataset).callstack_layout == layout + return dataset, single + + +@pytest.mark.parametrize("layout", ("v1", "v2")) +@pytest.mark.parametrize("kind", ("empty", "static", "dynamic", "mixed")) +@pytest.mark.parametrize("route", ("sql", "fallback")) +def test_zero_rank_is_conservative_until_exact_count(tmp_path, monkeypatch, layout, kind, route): + dataset, single = zero_rank_pair(tmp_path, layout, kind) + original_reduce = dataset_attribution.reduce_active + calls = [] + + def reduce(*args): + calls.append(route) + if route == "fallback": + return None + result = original_reduce(*args) + assert result is not None + return result + + monkeypatch.setattr(dataset_attribution, "reduce_active", reduce) + with SnapshotAnalyzer(dataset) as actual, SnapshotAnalyzer(single) as reference: + for include_static in (False, True): + params = {"event_id": 1, "top_n": 0, "include_static": include_static} + dynamic = int(kind in ("dynamic", "mixed")) + static = int(include_static and kind in ("static", "mixed")) + for exact in (False, True): + got = actual.execute_query( + "active_memory_callstack_at_event", params, exact_total=exact + ) + want = reference.execute_query( + "active_memory_callstack_at_event", params, exact_total=exact + ) + expected = { + "returned": static, + "total": static + dynamic if exact else static, + "has_more": bool(dynamic) if exact else True, + "truncated": bool(dynamic) if exact else True, + "total_is_exact": exact, + } + for result in (got, want): + assert {key: result[key] for key in expected} == expected + assert all(r["category"] == "static" for r in result["rows"]) + keys = set(want["rows"][0]) if want["rows"] else set() + assert [{k: r[k] for k in keys} for r in got["rows"]] == want["rows"] + assert calls == [route] * 4 diff --git a/tests/test_contract_cli_api.py b/tests/test_contract_cli_api.py index ee802f8..4662d47 100644 --- a/tests/test_contract_cli_api.py +++ b/tests/test_contract_cli_api.py @@ -1188,3 +1188,37 @@ def test_explicit_dataset_limits_and_failures_match_cli_api(tmp_path): ) assert failed.exit_code == 1 and failed.stdout == "" assert json.loads(failed.stderr)["error"]["message"] == str(error.value) + + +@pytest.mark.parametrize("layout", ("v1", "v2")) +@pytest.mark.parametrize("source", ("dataset", "standalone")) +def test_zero_rank_completeness_cli_api_contract(tmp_path, layout, source): + from tests.core.test_dataset_zero_rank import zero_rank_pair + + dataset, single = zero_rank_pair(tmp_path, layout, "mixed") + path = dataset if source == "dataset" else single + params = {"event_id": 1, "top_n": 0, "include_static": False} + with SnapshotAnalyzer(path) as analyzer: + for exact in (False, True): + api = analyzer.execute_query( + "active_memory_callstack_at_event", params, exact_total=exact + ) + cli = _json_stdout( + runner.invoke( + app, + [ + "query", + str(path), + "--template-use", + "active_memory_callstack_at_event", + "--params", + json.dumps(params), + "--json", + *(["--exact-total"] if exact else []), + ], + ) + ) + assert {key: cli[key] for key in api} == api + assert api["rows"] == [] and api["has_more"] and api["truncated"] + assert api["total_is_exact"] is exact + assert api["total"] == int(exact) diff --git a/tests/test_differential_acceptance_runner.py b/tests/test_differential_acceptance_runner.py new file mode 100644 index 0000000..09c446d --- /dev/null +++ b/tests/test_differential_acceptance_runner.py @@ -0,0 +1,206 @@ +"""Nightly differential evidence must prove every seed and both real layouts ran.""" + +import importlib.util +import json +import subprocess +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / ".github/scripts/run_differential_acceptance.py" + + +@pytest.fixture +def runner(): + spec = importlib.util.spec_from_file_location("differential_acceptance", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def evidence(runner, root, count=1): + report, temp = root / "junit.xml", root / "pytest-temp" + suite = ET.Element("testsuite") + for index, ((seed, layout), expected) in enumerate(runner.expected_records(count).items()): + record = {**expected, "scratch_builds": 8 if expected["shards"] > 10 else 0} + case = ET.SubElement( + suite, + "testcase", + { + "file": runner.TEST_FILE, + "classname": runner.TEST_FILE[:-3].replace("/", "."), + "name": f"{runner.TEST_NAME}[{layout}-seed-{seed}]", + }, + ) + props = ET.SubElement(case, "properties") + for name, value in record.items(): + ET.SubElement(props, "property", {"name": name, "value": str(value)}) + folder = temp / str(index) / f"seed-{seed}-{layout}" + folder.mkdir(parents=True) + (folder / "coverage.json").write_text(json.dumps(record)) + ET.ElementTree(suite).write(report) + return report, temp + + +def test_exact_200_profile_requires_all_400_identities_and_matching_artifacts(runner, tmp_path): + report, temp = evidence(runner, tmp_path, 200) + result = runner.read_evidence(report, temp, 200) + assert result["seed_format_cases"] == len(result["test_nodes"]) == 400 + assert result["seed_count"] == len(result["seeds"]) == 200 + assert result["comparisons"] == 28_864 + assert result["formats"] == ["v1", "v2"] and result["unexpected_errors"] == 0 + assert result["scratch_builds"] > 0 + + +@pytest.mark.parametrize( + "corruption", + [ + "missing_report", + "missing_case", + "extra_case", + "duplicate_case", + "wrong_seed", + "wrong_file", + "wrong_class", + "skipped", + "failure", + "error", + "missing_property", + "duplicate_property", + "wrong_property", + "missing_artifact", + "duplicate_artifact", + "wrong_artifact", + "error_artifact", + "failure_artifact", + "no_scratch", + "bad_identity", + ], +) +def test_empty_partial_skipped_duplicate_or_inconsistent_evidence_fails( + runner, tmp_path, corruption +): + report, temp = evidence(runner, tmp_path) + tree = ET.parse(report) + suite = tree.getroot() + case = suite.find("testcase") + props = case.find("properties") + paths = sorted(temp.rglob("coverage.json")) + if corruption == "missing_report": + report.unlink() + elif corruption == "missing_case": + suite.remove(case) + elif corruption in ("extra_case", "duplicate_case"): + ET.SubElement(suite, "testcase", dict(case.attrib)) + elif corruption == "wrong_seed": + case.set("name", f"{runner.TEST_NAME}[v1-seed-999]") + elif corruption == "wrong_file": + case.set("file", "tests/other.py") + elif corruption == "wrong_class": + case.set("classname", "tests.other") + elif corruption in ("skipped", "failure", "error"): + ET.SubElement(case, corruption) + elif corruption == "missing_property": + props.remove(props.find("property[@name='seed']")) + elif corruption == "duplicate_property": + ET.SubElement(props, "property", {"name": "seed", "value": "0"}) + elif corruption == "wrong_property": + props.find("property[@name='comparisons']").set("value", "1") + elif corruption == "missing_artifact": + paths[0].unlink() + elif corruption == "duplicate_artifact": + paths[1].write_bytes(paths[0].read_bytes()) + elif corruption in ("wrong_artifact", "error_artifact", "bad_identity"): + record = json.loads(paths[0].read_text()) + field, value = { + "wrong_artifact": ("comparisons", 1), + "error_artifact": ("unexpected_errors", 2), + "bad_identity": ("seed", []), + }[corruption] + record[field] = value + paths[0].write_text(json.dumps(record)) + elif corruption == "failure_artifact": + (paths[0].parent / "differential-failure.json").write_text("{}") + elif corruption == "no_scratch": + for path in paths: + record = json.loads(path.read_text()) + record["scratch_builds"] = 0 + path.write_text(json.dumps(record)) + for prop in suite.findall(".//property[@name='scratch_builds']"): + prop.set("value", "0") + if corruption != "missing_report": + tree.write(report) + with pytest.raises(runner.AcceptanceError): + runner.read_evidence(report, temp, 1) + + +def test_runner_requires_success_clears_stale_report_and_removes_seed_override( + runner, tmp_path, monkeypatch +): + output = tmp_path / "run" + output.mkdir() + (output / "junit.xml").write_text("old success") + (output / "summary.json").write_text('{"status":"passed"}') + monkeypatch.setenv("PT_SNAP_DIFFERENTIAL_SEED", "1234") + monkeypatch.setenv("PT_SNAP_DIFFERENTIAL_SEEDS", "1") + monkeypatch.setenv("PYTEST_ADDOPTS", "-k nothing") + calls = [] + + def run(command, **kwargs): + calls.append(command) + if command[0] == "git": + return subprocess.CompletedProcess(command, 0, "a" * 40 + "\n") + assert not (output / "junit.xml").exists() + env = kwargs["env"] + assert env["PT_SNAP_DIFFERENTIAL_SEEDS"] == "200" + assert "PT_SNAP_DIFFERENTIAL_SEED" not in env and "PYTEST_ADDOPTS" not in env + assert env["PYTHONPATH"] == str(runner.ROOT / "src") + assert command[3] == runner.TEST_NODE + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(runner.subprocess, "run", run) + with pytest.raises(runner.AcceptanceError, match="did not write"): + runner.run_acceptance(output) + saved = json.loads((output / "summary.json").read_text()) + assert saved["status"] == "failed" and saved["seed_count"] == 200 + assert saved["source_sha"] == "a" * 40 and len(calls) == 2 + + +@pytest.mark.parametrize("returncode", [1, 5]) +def test_failed_or_empty_pytest_never_passes(runner, tmp_path, monkeypatch, returncode): + def run(command, **kwargs): + if command[0] == "git": + return subprocess.CompletedProcess(command, 0, "a" * 40 + "\n") + return subprocess.CompletedProcess(command, returncode) + + monkeypatch.setattr(runner.subprocess, "run", run) + with pytest.raises(runner.AcceptanceError, match=f"pytest exited with {returncode}"): + runner.run_acceptance(tmp_path, 1) + assert json.loads((tmp_path / "summary.json").read_text())["status"] == "failed" + + +@pytest.mark.parametrize("count", (0, -1, 10001)) +def test_invalid_profile_replaces_old_passed_summary(runner, tmp_path, count): + (tmp_path / "summary.json").write_text('{"status":"passed"}') + (tmp_path / "junit.xml").write_text("old success") + with pytest.raises(ValueError, match="PT_SNAP_DIFFERENTIAL_SEEDS"): + runner.run_acceptance(tmp_path, count) + saved = json.loads((tmp_path / "summary.json").read_text()) + assert saved["status"] == "failed" and saved["seed_count"] == count + assert "PT_SNAP_DIFFERENTIAL_SEEDS" in saved["error"] + assert not (tmp_path / "junit.xml").exists() + + +def test_import_setup_failure_replaces_old_passed_summary(runner, tmp_path, monkeypatch): + (tmp_path / "summary.json").write_text('{"status":"passed"}') + + def broken_setup(count): + raise ImportError("synthetic setup failure") + + monkeypatch.setattr(runner, "expected_records", broken_setup) + with pytest.raises(ImportError, match="synthetic setup failure"): + runner.run_acceptance(tmp_path) + saved = json.loads((tmp_path / "summary.json").read_text()) + assert saved["status"] == "failed" and saved["error"] == "synthetic setup failure" diff --git a/tests/test_differential_ci.py b/tests/test_differential_ci.py new file mode 100644 index 0000000..8193698 --- /dev/null +++ b/tests/test_differential_ci.py @@ -0,0 +1,46 @@ +"""The 200-seed profile is a required scheduled/manual step, separate from PRs.""" + +from pathlib import Path + +import yaml + + +def test_nightly_job_executes_and_preserves_required_200_seed_profile(): + workflow = yaml.safe_load( + (Path(__file__).resolve().parents[1] / ".github/workflows/test.yml").read_text() + ) + job = workflow["jobs"]["dataset-acceptance"] + assert ( + job["if"] == "github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'" + ) + steps = job["steps"] + names = [step.get("name") for step in steps] + generic = names.index("Run required installed-wheel dataset acceptance") + index = names.index("Run required 200-seed paired differential acceptance") + assert generic < index + step = steps[index] + assert "if" not in step and not step.get("continue-on-error", False) + assert step["run"].split() == [ + "python", + ".github/scripts/run_differential_acceptance.py", + "\\", + "--seed-count", + "200", + "--output-dir", + ".tmp/differential-acceptance", + ] + upload = steps[names.index("Upload paired differential evidence")] + assert upload["if"] == "always()" and names.index(upload["name"]) > index + assert set(upload["with"]["path"].splitlines()) == { + ".tmp/differential-acceptance/*.log", + ".tmp/differential-acceptance/*.xml", + ".tmp/differential-acceptance/summary.json", + ".tmp/differential-acceptance/pytest-temp/**/coverage.json", + ".tmp/differential-acceptance/pytest-temp/**/timeline.json", + ".tmp/differential-acceptance/pytest-temp/**/differential-failure.json", + } + assert upload["with"]["include-hidden-files"] is True + assert upload["with"]["if-no-files-found"] == "error" + # The ordinary PR matrix keeps the default20 profile and never inherits200. + regular = workflow["jobs"]["test"] + assert "PT_SNAP_DIFFERENTIAL_SEEDS" not in str(regular) From 32bcd9e93e621961b9bad4342ae6fb32de89ade5 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:06:01 +0800 Subject: [PATCH 4/5] test: allow multiline differential failure diagnostics --- tests/core/test_dataset_differential_cases.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/test_dataset_differential_cases.py b/tests/core/test_dataset_differential_cases.py index 1cf4f1f..ae19278 100644 --- a/tests/core/test_dataset_differential_cases.py +++ b/tests/core/test_dataset_differential_cases.py @@ -127,7 +127,7 @@ def execute(*args, **kwargs): return SimpleNamespace(execute_query=execute) - with pytest.raises(AssertionError, match="unexpected query errors.*seed"): + with pytest.raises(AssertionError, match="(?s)unexpected query errors.*seed"): compare_case( analyzer(0), analyzer(1), QueryCase("error-check", "event"), tmp_path, 31, "v2", 2 ) From 6a7da43da80379407a8a8768a0b34dcb1d5f9978 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Mon, 12 Oct 2026 01:17:00 +0800 Subject: [PATCH 5/5] ci: retain exhaustive Windows coverage across four bounded partitions Keep the full non-slow union, twenty-minute job cap and max-parallel two. Update fail-closed partition guards and bilingual reproduction docs; production behavior and differential test cases are unchanged. --- .github/scripts/run_platform_tests.py | 22 ++++++----- .github/workflows/platform.yml | 6 +++ README.md | 6 +-- README_zh.md | 6 +-- tests/test_platform_workflow.py | 53 +++++++++++++++++---------- 5 files changed, 58 insertions(+), 35 deletions(-) diff --git a/.github/scripts/run_platform_tests.py b/.github/scripts/run_platform_tests.py index 46a7067..1c4a08f 100644 --- a/.github/scripts/run_platform_tests.py +++ b/.github/scripts/run_platform_tests.py @@ -1,4 +1,4 @@ -"""Run one of exactly two exhaustive, deterministic Windows non-slow shards.""" +"""Run one of exactly four exhaustive, deterministic Windows non-slow shards.""" import argparse import hashlib @@ -8,37 +8,39 @@ import pytest +SHARD_COUNT = 4 + def validate_partition(nodeids, partitions): """Fail before execution if any collected test is missing, repeated, or unknown.""" if len(nodeids) != len(set(nodeids)): raise pytest.UsageError("platform collection contains duplicate node IDs") - if len(partitions) != 2 or any(not part for part in partitions): - raise pytest.UsageError("platform partition requires exactly two nonempty shards") + if len(partitions) != SHARD_COUNT or any(not part for part in partitions): + raise pytest.UsageError("platform partition requires exactly four nonempty shards") if Counter(node for part in partitions for node in part) != Counter(nodeids): raise pytest.UsageError("platform shards must cover every selected test exactly once") def partition_nodeids(nodeids): # Alternating individual cases, rather than whole directories/files, spreads - # expensive parameterized SQLite tests across both fixed-size CI jobs. + # expensive parameterized SQLite tests across the fixed-size CI jobs. ordered = sorted(nodeids) - partitions = [ordered[0::2], ordered[1::2]] + partitions = [ordered[shard::SHARD_COUNT] for shard in range(SHARD_COUNT)] validate_partition(nodeids, partitions) return partitions class PlatformPartition: def __init__(self, shard, manifest=Path("platform-partition.json")): - if shard not in (0, 1): - raise pytest.UsageError("platform shard must be 0 or 1") + if shard not in range(SHARD_COUNT): + raise pytest.UsageError("platform shard must be between 0 and 3") self.shard = shard self.manifest = manifest @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_collection_modifyitems(self, config, items): # Run after ordinary selection hooks, particularly pytest's -m filter. - # Both jobs collect the entire suite before assigning any non-slow case. + # All jobs collect the entire suite before assigning any non-slow case. yield nodeids = [item.nodeid for item in items] partitions = partition_nodeids(nodeids) @@ -56,7 +58,7 @@ def pytest_collection_modifyitems(self, config, items): "schema_version": 1, "selection": "not slow", "shard": self.shard, - "shard_count": 2, + "shard_count": SHARD_COUNT, "collection_sha256": hashlib.sha256( "\n".join(ordered).encode("utf-8") ).hexdigest(), @@ -74,7 +76,7 @@ def pytest_collection_modifyitems(self, config, items): def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--shard", required=True, type=int, choices=(0, 1)) + parser.add_argument("--shard", required=True, type=int, choices=range(SHARD_COUNT)) parser.add_argument("--collect-only", action="store_true") args = parser.parse_args(argv) pytest_args = [ diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index b01fb44..61d62c7 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -31,6 +31,12 @@ jobs: - os: windows-latest partition: shard-1 command: python .github/scripts/run_platform_tests.py --shard 1 + - os: windows-latest + partition: shard-2 + command: python .github/scripts/run_platform_tests.py --shard 2 + - os: windows-latest + partition: shard-3 + command: python .github/scripts/run_platform_tests.py --shard 3 - os: macos-latest partition: all command: python -m pytest -m "not slow" --junitxml=platform-tests.xml -o junit_family=legacy diff --git a/README.md b/README.md index c9f040c..2c9b1b7 100644 --- a/README.md +++ b/README.md @@ -166,13 +166,13 @@ non-editable installed exporter tests are described in the [dataset acceptance guide](docs/en/dataset-acceptance.md). The supplemental platform workflow runs Python 3.13 non-slow tests on macOS and -Windows. Windows uses two deterministic, disjoint shards of the full non-slow -collection; macOS runs it in one job. All three jobs retain a 20-minute limit, +Windows. Windows uses four deterministic, disjoint shards of the full non-slow +collection; macOS runs it in one job. All five jobs retain a 20-minute limit, with at most two running concurrently. Each Windows shard validates exhaustive coverage before execution and uploads its full collection/partition manifest alongside JUnit results. No tests are skipped to meet the time limit. To reproduce one Windows partition locally, run -`python .github/scripts/run_platform_tests.py --shard 0` (or `--shard 1`); +`python .github/scripts/run_platform_tests.py --shard 0` (choose `--shard 0` through `--shard 3`); add `--collect-only` to inspect the partition without executing tests. ### Building distributions diff --git a/README_zh.md b/README_zh.md index 4c6a38d..b7b65c3 100644 --- a/README_zh.md +++ b/README_zh.md @@ -148,11 +148,11 @@ black --check . && ruff check . # 检查格式和 lint [数据集验收指南](docs/zh/dataset-acceptance.md)。 补充的平台工作流在 macOS 和 Windows 上运行 Python 3.13 非 slow 测试。 -Windows 将完整的非 slow 集合确定性地划分为两个互不重叠的分片;macOS 使用一个任务。 -三个任务均保持 20 分钟上限,最多同时运行两个。每个 Windows 分片在执行前验证 +Windows 将完整的非 slow 集合确定性地划分为四个互不重叠的分片;macOS 使用一个任务。 +五个任务均保持 20 分钟上限,最多同时运行两个。每个 Windows 分片在执行前验证 划分无遗漏、无重复,并随 JUnit 结果上传完整的收集及分片清单。不会为了满足时间上限 而跳过测试。本地复现一个 Windows 分片可运行 -`python .github/scripts/run_platform_tests.py --shard 0`(或 `--shard 1`); +`python .github/scripts/run_platform_tests.py --shard 0`(可选 `--shard 0` 至 `--shard 3`); 增加 `--collect-only` 可仅检查划分,不执行测试。 ### 构建分发包 diff --git a/tests/test_platform_workflow.py b/tests/test_platform_workflow.py index d4900d3..3aa3024 100644 --- a/tests/test_platform_workflow.py +++ b/tests/test_platform_workflow.py @@ -27,7 +27,7 @@ def workflow(): ) -def test_platform_ci_runs_exactly_three_bounded_non_linux_python_313_jobs(): +def test_platform_ci_runs_exactly_five_bounded_non_linux_python_313_jobs(): data = workflow() assert set(data["on"]) == {"push", "pull_request", "workflow_dispatch"} assert data["on"]["push"]["branches"] == ["main"] @@ -58,6 +58,16 @@ def test_platform_ci_runs_exactly_three_bounded_non_linux_python_313_jobs(): "partition": "shard-1", "command": "python .github/scripts/run_platform_tests.py --shard 1", }, + { + "os": "windows-latest", + "partition": "shard-2", + "command": "python .github/scripts/run_platform_tests.py --shard 2", + }, + { + "os": "windows-latest", + "partition": "shard-3", + "command": "python .github/scripts/run_platform_tests.py --shard 3", + }, { "os": "macos-latest", "partition": "all", @@ -103,20 +113,22 @@ def test_partition_is_deterministic_disjoint_exhaustive_and_balanced(): nodeids = [f"tests/test_sample.py::test_value[{n}]" for n in range(101)] parts = module.partition_nodeids(nodeids) assert parts == module.partition_nodeids(list(reversed(nodeids))) - assert len(parts[0]) == 51 and len(parts[1]) == 50 - assert not set(parts[0]) & set(parts[1]) - assert set(parts[0]) | set(parts[1]) == set(nodeids) + assert module.SHARD_COUNT == 4 + assert [len(part) for part in parts] == [26, 25, 25, 25] + flattened = [node for part in parts for node in part] + assert len(flattened) == len(set(flattened)) + assert set(flattened) == set(nodeids) @pytest.mark.parametrize( "nodeids,parts", [ - (["a", "b"], [["a"], []]), - (["a", "b", "c"], [["a"], ["b"]]), - (["a", "b"], [["a", "b"], ["a"]]), - (["a", "b"], [["a"], ["unknown"]]), - (["a", "b"], [["a", "b"]]), - (["a", "a"], [["a"], ["a"]]), + (["a", "b", "c"], [["a"], ["b"], ["c"], []]), + (["a", "b", "c", "d", "e"], [["a"], ["b"], ["c"], ["d"]]), + (["a", "b", "c", "d"], [["a", "b"], ["b"], ["c"], ["d"]]), + (["a", "b", "c", "d"], [["a"], ["b"], ["c"], ["unknown"]]), + (["a", "b", "c", "d"], [["a", "b"], ["c", "d"]]), + (["a", "a", "b", "c"], [["a"], ["a"], ["b"], ["c"]]), ], ) def test_partition_rejects_missing_duplicate_unknown_or_empty_coverage(nodeids, parts): @@ -124,7 +136,7 @@ def test_partition_rejects_missing_duplicate_unknown_or_empty_coverage(nodeids, runner().validate_partition(nodeids, parts) -@pytest.mark.parametrize("shard", [-1, 2]) +@pytest.mark.parametrize("shard", [-1, 4]) def test_partition_rejects_unknown_shard(shard): with pytest.raises(pytest.UsageError): runner().PlatformPartition(shard) @@ -148,7 +160,7 @@ def run_toy_suite(tmp_path, shard, *, corrupt=False): "module = importlib.util.module_from_spec(spec)\nspec.loader.exec_module(module)\n" ) if corrupt: - code += "module.partition_nodeids = lambda ids: [[ids[0]], [ids[1]]]\n" + code += "module.partition_nodeids = lambda ids: [[ids[n]] for n in range(4)]\n" code += ( f"raise SystemExit(pytest.main(['-m', 'not slow', '-v', '--junitxml=toy.xml'], " f"plugins=[module.PlatformPartition({shard})]))\n" @@ -160,23 +172,26 @@ def run_toy_suite(tmp_path, shard, *, corrupt=False): def test_real_pytest_shards_cover_exact_non_slow_collection(tmp_path): manifests = [] - for shard in (0, 1): + for shard in range(4): result = run_toy_suite(tmp_path, shard) assert result.returncode == 0, result.stdout + result.stderr manifest = json.loads((tmp_path / "platform-partition.json").read_text(encoding="utf-8")) assert manifest["shard"] == shard + assert manifest["shard_count"] == 4 + assert manifest["selected"] == manifest["partitions"][shard] assert manifest["selection"] == "not slow" assert len(manifest["full_collection"]) == 7 assert all("test_slow" not in node for node in manifest["full_collection"]) assert f"{len(manifest['selected'])} passed" in result.stdout manifests.append(manifest) - assert manifests[0]["collection_sha256"] == manifests[1]["collection_sha256"] - assert manifests[0]["full_collection"] == manifests[1]["full_collection"] - assert not set(manifests[0]["selected"]) & set(manifests[1]["selected"]) - assert ( - sorted(manifests[0]["selected"] + manifests[1]["selected"]) - == manifests[0]["full_collection"] + assert len({manifest["collection_sha256"] for manifest in manifests}) == 1 + assert all( + manifest["full_collection"] == manifests[0]["full_collection"] for manifest in manifests ) + assert all(manifest["partitions"] == manifests[0]["partitions"] for manifest in manifests) + flattened = [node for manifest in manifests for node in manifest["selected"]] + assert len(flattened) == len(set(flattened)) + assert sorted(flattened) == manifests[0]["full_collection"] def test_real_pytest_partition_corruption_fails_before_execution(tmp_path):