From 430acf97ee9f6129d182c88382c15278927f9302 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:48:03 +0800 Subject: [PATCH 1/3] ci: tier compatibility tests and require nightly wheel acceptance --- .github/AGENTS.md | 1 + .github/scripts/run_dataset_acceptance.py | 120 +++++++++++++++++ .github/workflows/AGENTS.md | 35 ++++- .github/workflows/test.yml | 70 +++++++++- tests/core/test_dataset_id_pagination.py | 1 + tests/test_ci_tiers.py | 110 +++++++++++++++ tests/test_dataset_acceptance_runner.py | 157 ++++++++++++++++++++++ tests/test_wheel_audit.py | 6 + 8 files changed, 494 insertions(+), 6 deletions(-) create mode 100644 .github/scripts/run_dataset_acceptance.py create mode 100644 tests/test_ci_tiers.py create mode 100644 tests/test_dataset_acceptance_runner.py diff --git a/.github/AGENTS.md b/.github/AGENTS.md index c4b73b2..e4569fe 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -9,6 +9,7 @@ Parent scope: [repository](../AGENTS.md). | `pull_request_template.md` | Default PR template and machine-parsed snapshot decision labels | | `scripts/check_snapshot_provenance.py` | Base/head Git-blob comparison and PR declaration validation | | `scripts/clean_wheel_agent_acceptance.py` | Supplied wheel acceptance in temporary home/CWD/venv | +| `scripts/run_dataset_acceptance.py` | Required installed-wheel dataset test with fail-closed JUnit and artifact evidence | ## Provenance Guard diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py new file mode 100644 index 0000000..89fc223 --- /dev/null +++ b/.github/scripts/run_dataset_acceptance.py @@ -0,0 +1,120 @@ +"""Run the installed-wheel gate and reject empty, skipped or stale evidence. + +This deliberately selects one existing public CLI/API acceptance test. It does +not claim clean dependency isolation or complete default-scale coverage; those +are contracts of the selected test, not of this CI runner. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TEST_NAME = "test_installed_trusted_producer_default_export_cli_and_api" +TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" + + +class AcceptanceError(RuntimeError): + pass + + +def read_evidence(report: Path, wheel_sha256: str) -> dict: + """A green pytest process alone does not prove the requested test ran.""" + if not report.is_file(): + raise AcceptanceError("pytest did not write its JUnit report") + cases = ET.parse(report).findall(".//testcase") + if len(cases) != 1 or cases[0].get("name") != TEST_NAME: + raise AcceptanceError(f"Expected exactly one {TEST_NAME} result, got {len(cases)}") + case = cases[0] + if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): + raise AcceptanceError("Installed-wheel acceptance must pass, without skips") + properties = case.findall("./properties/property[@name='dataset_acceptance']") + if len(properties) != 1: + raise AcceptanceError("Expected exactly one dataset_acceptance evidence property") + evidence = json.loads(properties[0].get("value", "")) + if not isinstance(evidence, dict) or evidence.get("wheel_sha256") != wheel_sha256: + raise AcceptanceError("Acceptance evidence does not match the supplied wheel SHA-256") + return evidence + + +def run_acceptance(wheel: Path, output_dir: Path) -> None: + wheel = wheel.resolve() + output_dir = output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + report = output_dir / "junit.xml" + # Reusing an output directory must never let a previous green run mask this one. + for name in ("junit.xml", "summary.json", "pytest.log"): + (output_dir / name).unlink(missing_ok=True) + command = [ + sys.executable, + "-m", + "pytest", + TEST_NODE, + "-o", + "junit_family=xunit1", + f"--junitxml={report}", + ] + env = os.environ.copy() + env["PT_SNAP_ACCEPTANCE_WHEEL"] = str(wheel) + env["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] = str(output_dir) + summary = { + "wheel": str(wheel), + "test_node": TEST_NODE, + "command": command, + "status": "failed", + } + try: + if not wheel.is_file() or wheel.suffix != ".whl": + raise AcceptanceError(f"A built wheel is required: {wheel}") + wheel_sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + summary["wheel_sha256"] = wheel_sha256 + source_sha = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout.strip() + summary["source_sha"] = source_sha + print(f"Running {TEST_NODE} at {source_sha}", flush=True) + with (output_dir / "pytest.log").open("w") as log: + result = subprocess.run( + command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT + ) + print((output_dir / "pytest.log").read_text(), end="", flush=True) + summary["pytest_exit_code"] = result.returncode + if result.returncode != 0: + raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log") + summary["acceptance"] = read_evidence(report, wheel_sha256) + summary["status"] = "passed" + except Exception as exc: + summary["error"] = str(exc) + raise + finally: + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + try: + run_acceptance(args.wheel, args.output_dir) + except ( + AcceptanceError, + OSError, + ValueError, + ET.ParseError, + subprocess.CalledProcessError, + ) as exc: + print(f"Dataset acceptance failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 2959c6d..91e9211 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -5,23 +5,49 @@ Parent scope: [automation](../AGENTS.md). ## Test Workflow `test.yml` runs for main pushes, main-targeted PR opened/synchronize/reopened/edited -events, and `workflow_call`. Provenance, lint and Python test-matrix jobs are -independent. Concurrency is keyed by GitHub ref and superseded runs are cancelled. +events, `workflow_call`, nightly at 03:17 UTC, and `workflow_dispatch`. Provenance, +lint, browser and Python test-matrix jobs are independent. Checkouts use the PR +head SHA or the triggering SHA; manual dispatch checks the selected ref's SHA. +Concurrency is keyed by GitHub ref and superseded runs are cancelled. | Job/step | Gate | | --- | --- | | Snapshot provenance | Full Git history; base/head comparison; PR decision required only in PR mode | | Lint & Format | Python 3.13, editable dev install, Ruff, Black, basedpyright with the selected interpreter | -| Test matrix | Python 3.10–3.13; fixture provenance first, then pytest with branch coverage | +| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow"` | +| Full test tier | Python 3.13; fixture provenance first, then all tests with branch coverage, including slow tests | | Overall coverage | `pyproject.toml` fail-under 90 with branches enabled | | Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently | | Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance | +| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then required installed-wheel CLI/API test | + +Million-event deep-offset pagination tests are marked `slow`; their small +pagination contracts still run across the compatibility matrix. Coverage reporting writes `.coverage`/`coverage.xml`; building writes `dist/` and intermediates. Dependency installation/build isolation may access the network. The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses temporary paths but imports a reviewed executable fixture. These are more than YAML checks. +The nightly/manual job invokes `run_dataset_acceptance.py` with the single built +wheel. Missing wheels, missing/empty JUnit, skipped/failed tests, extra test cases, +or evidence for a different wheel fail the gate. It uploads the pytest log, JUnit +(including the test's evidence), and a summary with actual checkout SHA and wheel +SHA-256, even after failure. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` is available for +additional test-owned artifacts. Local invocation: + +```bash +python .github/scripts/run_dataset_acceptance.py \ + --wheel dist/.whl --output-dir .tmp/dataset-acceptance +``` + +This tier initially exercises the existing 500,002-event `pip --target` acceptance +test. It does not claim a clean venv, two full 500,000-event shards, complete CLI +template coverage, live agents or upstream GUI interoperability. The ordinary +full suite may still skip this opt-in wheel test; the required nightly/manual +runner must never do so. Manually dispatch the exact published branch/commit and +verify the run's head SHA and summary; a PR run alone is not nightly evidence. + ## Release Workflow `release.yml` triggers on `v*` tag pushes and enforces: @@ -44,6 +70,7 @@ validation. For non-PR provenance checks, tag runs compare against the prior tag Review `pyproject.toml`, both workflow YAMLs, the provenance/acceptance scripts and `tests/test_release_workflow.py` together when changing execution order. Validate -YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py` from +YAML and run `pytest tests/test_governance.py tests/test_release_workflow.py +tests/test_ci_tiers.py tests/test_dataset_acceptance_runner.py tests/test_wheel_audit.py` from the repository root. Full CI tests the matrix and installed wheel; static tests do not prove a release environment is configured or a publication occurred. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1000c8d..e71defe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,6 +2,9 @@ name: Test on: workflow_call: + workflow_dispatch: + schedule: + - cron: "17 3 * * *" push: branches: [main] pull_request: @@ -23,6 +26,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: Require provenance evaluation for snapshot runtime changes if: github.event_name == 'pull_request' @@ -56,6 +60,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: @@ -78,6 +84,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: python-version: "3.13" @@ -99,6 +107,8 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} - uses: actions/setup-python@v5 with: @@ -114,10 +124,19 @@ jobs: - name: Install dependencies run: pip install -e ".[dev]" - - name: Run tests with coverage - run: pytest tests/test_fixture_provenance.py && pytest --cov --cov-report=xml --cov-report=term-missing + - name: Verify executable fixture provenance + run: pytest tests/test_fixture_provenance.py + + - name: Run non-slow compatibility tests + if: matrix.python-version != '3.13' + run: pytest -m "not slow" + + - name: Run all tests with coverage + if: matrix.python-version == '3.13' + run: pytest --cov --cov-report=xml --cov-report=term-missing - name: Guard critical runtime coverage + if: matrix.python-version == '3.13' # Separate floors prevent strong modules from masking a weak boundary. run: | python -m coverage report --include='*/core/snapshot_import_backend.py' --fail-under=90 @@ -153,3 +172,50 @@ jobs: python .github/scripts/clean_wheel_agent_acceptance.py \ --wheel "$WHEEL" \ --fixture tests/fixtures/snapshots/snapshot_with_empty_cache.pkl + + dataset-acceptance: + name: Installed-wheel dataset acceptance + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.sha }} + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install dependencies + run: pip install -e ".[dev]" + + - name: Verify executable fixture provenance + run: pytest tests/test_fixture_provenance.py + + - name: Build wheel + run: python -m build --wheel + + - name: Audit wheel contents against source + run: | + set -euo pipefail + WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') + python .github/scripts/audit_wheel.py --wheel "$WHEEL" + + - name: Run required installed-wheel dataset acceptance + run: | + set -euo pipefail + WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') + python .github/scripts/run_dataset_acceptance.py \ + --wheel "$WHEEL" --output-dir .tmp/dataset-acceptance + + - name: Upload dataset acceptance evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: dataset-acceptance-${{ github.sha }} + path: .tmp/dataset-acceptance/ + include-hidden-files: true + if-no-files-found: error + retention-days: 30 diff --git a/tests/core/test_dataset_id_pagination.py b/tests/core/test_dataset_id_pagination.py index df4756c..56e4254 100644 --- a/tests/core/test_dataset_id_pagination.py +++ b/tests/core/test_dataset_id_pagination.py @@ -269,6 +269,7 @@ def million_event_dataset(tmp_path_factory): return root +@pytest.mark.slow @pytest.mark.parametrize("template", ["event", "allocation"]) @pytest.mark.parametrize("direction", ["ASC", "DESC"]) def test_million_offset_keeps_fetched_work_constant(million_event_dataset, template, direction): diff --git a/tests/test_ci_tiers.py b/tests/test_ci_tiers.py new file mode 100644 index 0000000..9a6e10f --- /dev/null +++ b/tests/test_ci_tiers.py @@ -0,0 +1,110 @@ +"""Protect fast/full/nightly boundaries and the existing quality gates.""" + +from pathlib import Path + +import yaml + + +def workflow(): + return yaml.load(Path(".github/workflows/test.yml").read_text(), Loader=yaml.BaseLoader) + + +def named_steps(job): + return {step["name"]: step for step in job["steps"] if "name" in step} + + +def test_compatibility_matrix_leaves_slow_and_coverage_on_python_313(): + job = workflow()["jobs"]["test"] + assert job["strategy"]["matrix"]["python-version"] == ["3.10", "3.11", "3.12", "3.13"] + steps = named_steps(job) + fast = steps["Run non-slow compatibility tests"] + assert fast["if"] == "matrix.python-version != '3.13'" + assert fast["run"] == 'pytest -m "not slow"' + full = steps["Run all tests with coverage"] + assert full["if"] == "matrix.python-version == '3.13'" + assert full["run"] == "pytest --cov --cov-report=xml --cov-report=term-missing" + assert "if" not in steps["Verify executable fixture provenance"] + names = list(steps) + assert names.index("Verify executable fixture provenance") < names.index(fast["name"]) + assert names.index("Verify executable fixture provenance") < names.index(full["name"]) + + +def test_full_tier_preserves_independent_coverage_floors_and_packaging(): + steps = named_steps(workflow()["jobs"]["test"]) + guard = steps["Guard critical runtime coverage"] + assert guard["if"] == "matrix.python-version == '3.13'" + for file, floor in ( + ("core/snapshot_import_backend.py", 90), + ("simulate/simulated_caching_allocator.py", 88), + ("simulate/snapshot_mutator.py", 84), + ): + assert f"--include='*/{file}' --fail-under={floor}" in guard["run"] + config = Path("pyproject.toml").read_text() + assert "branch = true" in config and "fail_under = 90" in config + for name in ( + "Build wheel", + "Audit wheel contents against source", + "Clean-wheel Agent acceptance", + ): + assert steps[name]["if"] == "matrix.python-version == '3.13'" + assert steps["Upload coverage report"]["if"] == "matrix.python-version == '3.13' && always()" + + +def test_ci_preserves_least_privilege_and_checks_the_published_commit(): + config = workflow() + assert config["permissions"] == {"contents": "read"} + assert "pull_request_target" not in config["on"] + assert config["on"]["pull_request"]["types"] == ["opened", "synchronize", "reopened", "edited"] + assert "workflow_call" in config["on"] + jobs = config["jobs"] + for name in ("snapshot-provenance", "lint", "browser", "test"): + checkout = jobs[name]["steps"][0] + assert checkout["uses"] == "actions/checkout@v4" + assert checkout["with"]["ref"] == "${{ github.event.pull_request.head.sha || github.sha }}" + assert jobs["snapshot-provenance"]["steps"][0]["with"]["fetch-depth"] == "0" + assert ( + "--require-decision" + in named_steps(jobs["snapshot-provenance"])[ + "Require provenance evaluation for snapshot runtime changes" + ]["run"] + ) + browser = named_steps(jobs["browser"])["Run required Chromium interaction tests"] + assert browser["env"]["PT_SNAP_REQUIRE_BROWSER"] == "1" + assert "tests/test_memory_tree_browser.py" in browser["run"] + lint = named_steps(jobs["lint"]) + for name in ("Ruff lint check", "Black format check", "basedPyright check"): + assert name in lint + + +def test_nightly_and_manual_gate_build_and_audit_before_required_acceptance(): + config = workflow() + assert "workflow_dispatch" in config["on"] + assert config["on"]["schedule"] == [{"cron": "17 3 * * *"}] + job = config["jobs"]["dataset-acceptance"] + assert ( + job["if"] == "github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'" + ) + assert job["timeout-minutes"] == "30" + assert job["steps"][0]["with"] == {"fetch-depth": "0", "ref": "${{ github.sha }}"} + steps = named_steps(job) + names = list(steps) + order = [ + "Verify executable fixture provenance", + "Build wheel", + "Audit wheel contents against source", + "Run required installed-wheel dataset acceptance", + "Upload dataset acceptance evidence", + ] + assert [names.index(name) for name in order] == sorted(names.index(name) for name in order) + acceptance = steps["Run required installed-wheel dataset acceptance"]["run"] + assert "assert len(wheels) == 1" in acceptance + assert "python .github/scripts/run_dataset_acceptance.py" in acceptance + assert '--wheel "$WHEEL" --output-dir .tmp/dataset-acceptance' in acceptance + for step in job["steps"]: + assert "continue-on-error" not in step + upload = steps["Upload dataset acceptance evidence"] + assert upload["if"] == "always()" + assert upload["with"]["if-no-files-found"] == "error" + assert upload["with"]["include-hidden-files"] == "true" + assert upload["with"]["path"] == ".tmp/dataset-acceptance/" + assert "${{ github.sha }}" in upload["with"]["name"] diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py new file mode 100644 index 0000000..36ff22e --- /dev/null +++ b/tests/test_dataset_acceptance_runner.py @@ -0,0 +1,157 @@ +"""Fail-closed CI wrapper contracts without repeating the expensive scale test.""" + +import hashlib +import importlib.util +import json +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +RUNNER = Path(".github/scripts/run_dataset_acceptance.py") + + +@pytest.fixture +def runner(): + spec = importlib.util.spec_from_file_location("dataset_acceptance_runner", RUNNER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def write_report(path, name, evidence, outcome=None): + suite = ET.Element("testsuite") + case = ET.SubElement(suite, "testcase", name=name) + if outcome: + ET.SubElement(case, outcome) + if evidence is not None: + properties = ET.SubElement(case, "properties") + ET.SubElement(properties, "property", name="dataset_acceptance", value=json.dumps(evidence)) + ET.ElementTree(suite).write(path) + + +def test_runner_accepts_matching_wheel_evidence(runner, tmp_path): + report = tmp_path / "junit.xml" + evidence = {"wheel_sha256": "abc", "events": 500_002} + write_report(report, runner.TEST_NAME, evidence) + assert runner.read_evidence(report, "abc") == evidence + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_nonpassing_acceptance(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}, outcome) + with pytest.raises(runner.AcceptanceError, match="without skips"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("evidence", [None, {}, {"wheel_sha256": "other"}, []]) +def test_runner_rejects_missing_or_wrong_artifact_evidence(runner, tmp_path, evidence): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, evidence) + with pytest.raises(runner.AcceptanceError): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("count", [0, 2]) +def test_runner_rejects_empty_or_extra_collection(runner, tmp_path, count): + report = tmp_path / "junit.xml" + suite = ET.Element("testsuite") + for _ in range(count): + ET.SubElement(suite, "testcase", name=runner.TEST_NAME) + ET.ElementTree(suite).write(report) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +def test_runner_rejects_wrong_test_even_if_it_passes(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, "unrelated_test", {"wheel_sha256": "abc"}) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +def test_runner_rejects_duplicate_evidence(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + tree = ET.parse(report) + properties = tree.find(".//properties") + assert properties is not None + ET.SubElement(properties, "property", name="dataset_acceptance", value="{}") + tree.write(report) + with pytest.raises(runner.AcceptanceError, match="exactly one dataset_acceptance"): + runner.read_evidence(report, "abc") + + +def test_missing_wheel_fails_before_pytest(tmp_path): + output = tmp_path / "results" + output.mkdir() + (output / "summary.json").write_text('{"status": "passed"}') + (output / "junit.xml").write_text("old passing report") + (output / "pytest.log").write_text("old passing output") + result = subprocess.run( + [ + sys.executable, + str(RUNNER), + "--wheel", + str(tmp_path / "missing.whl"), + "--output-dir", + str(tmp_path / "results"), + ], + capture_output=True, + text=True, + ) + assert result.returncode == 1 + assert "A built wheel is required" in result.stderr + assert not (output / "junit.xml").exists() + assert not (output / "pytest.log").exists() + summary = json.loads((output / "summary.json").read_text()) + assert summary["status"] == "failed" + assert "A built wheel is required" in summary["error"] + + +@pytest.mark.parametrize("exit_code,write_junit", [(0, True), (0, False), (1, True), (5, False)]) +def test_runner_records_checkout_and_artifact_with_fail_closed_results( + runner, tmp_path, monkeypatch, exit_code, write_junit +): + wheel = tmp_path / "built.whl" + wheel.write_bytes(b"test artifact") + wheel_hash = hashlib.sha256(wheel.read_bytes()).hexdigest() + output = tmp_path / "results" + output.mkdir() + report = output / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": wheel_hash}) + calls = [] + + def run(command, **kwargs): + calls.append(command) + assert kwargs["cwd"] == runner.ROOT + if command[0] == "git": + return subprocess.CompletedProcess(command, 0, stdout="checkout-sha\n") + assert command[:3] == [sys.executable, "-m", "pytest"] + assert runner.TEST_NODE in command + assert "junit_family=xunit1" in command + assert kwargs["env"]["PT_SNAP_ACCEPTANCE_WHEEL"] == str(wheel) + assert kwargs["env"]["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] == str(output) + assert not report.exists(), "stale JUnit must be removed before this run" + kwargs["stdout"].write("captured pytest output\n") + if write_junit: + write_report(report, runner.TEST_NAME, {"wheel_sha256": wheel_hash}) + return subprocess.CompletedProcess(command, exit_code) + + monkeypatch.setattr(runner.subprocess, "run", run) + if exit_code or not write_junit: + with pytest.raises(runner.AcceptanceError): + runner.run_acceptance(wheel, output) + else: + runner.run_acceptance(wheel, output) + assert len(calls) == 2 + summary = json.loads((output / "summary.json").read_text()) + assert summary["source_sha"] == "checkout-sha" + assert summary["wheel_sha256"] == wheel_hash + assert summary["pytest_exit_code"] == exit_code + assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") + assert (output / "pytest.log").read_text() == "captured pytest output\n" diff --git a/tests/test_wheel_audit.py b/tests/test_wheel_audit.py index 15bb04c..4bc413a 100644 --- a/tests/test_wheel_audit.py +++ b/tests/test_wheel_audit.py @@ -177,6 +177,12 @@ def build(directory): ("workflow", "job", "build", "next_step"), [ ("test.yml", "test", "Build wheel", "Clean-wheel Agent acceptance"), + ( + "test.yml", + "dataset-acceptance", + "Build wheel", + "Run required installed-wheel dataset acceptance", + ), ("release.yml", "build", "Build sdist and wheel", "Upload build artifacts"), ], ) From 68d89670b188c93e41b1a5d8f41903057e5ada06 Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 21:59:11 +0800 Subject: [PATCH 2/3] ci: enforce explicit nightly selection and required test evidence --- .github/scripts/run_dataset_acceptance.py | 66 ++++++++--- .github/workflows/AGENTS.md | 20 ++-- .github/workflows/test.yml | 6 +- pyproject.toml | 1 + tests/core/test_dataset_scale_acceptance.py | 1 + tests/test_ci_tiers.py | 10 +- tests/test_dataset_acceptance_runner.py | 120 +++++++++++++++++++- 7 files changed, 194 insertions(+), 30 deletions(-) diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py index 89fc223..d1d8512 100644 --- a/.github/scripts/run_dataset_acceptance.py +++ b/.github/scripts/run_dataset_acceptance.py @@ -1,6 +1,6 @@ """Run the installed-wheel gate and reject empty, skipped or stale evidence. -This deliberately selects one existing public CLI/API acceptance test. It does +This selects nightly tests and requires the public CLI/API wheel case. It does not claim clean dependency isolation or complete default-scale coverage; those are contracts of the selected test, not of this CI runner. """ @@ -14,28 +14,60 @@ import subprocess import sys import xml.etree.ElementTree as ET -from pathlib import Path +from pathlib import Path, PurePosixPath ROOT = Path(__file__).resolve().parents[2] TEST_NAME = "test_installed_trusted_producer_default_export_cli_and_api" TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" +# Extend this tuple alongside new mandatory nightly gates. Never list future +# files that have not landed: the runner rejects any missing configured node. +REQUIRED_TEST_NODES = (TEST_NODE,) class AcceptanceError(RuntimeError): pass -def read_evidence(report: Path, wheel_sha256: str) -> dict: - """A green pytest process alone does not prove the requested test ran.""" +def test_node_id(case: ET.Element) -> str: + """Recover a complete pytest identity from the required xunit1 fields.""" + file = case.get("file", "") + name = case.get("name", "") + classname = case.get("classname", "") + path = PurePosixPath(file) + if not name or path.is_absolute() or ".." in path.parts or path.suffix != ".py": + raise AcceptanceError(f"JUnit testcase has no valid source identity: {case.attrib}") + module = file.removesuffix(".py").replace("/", ".") + if classname == module: + classes = "" + elif classname.startswith(module + "."): + classes = classname[len(module) + 1 :].replace(".", "::") + "::" + else: + raise AcceptanceError(f"JUnit testcase has an inconsistent class identity: {case.attrib}") + return f"{file}::{classes}{name}" + + +def read_cases(report: Path) -> list[ET.Element]: if not report.is_file(): raise AcceptanceError("pytest did not write its JUnit report") - cases = ET.parse(report).findall(".//testcase") - if len(cases) != 1 or cases[0].get("name") != TEST_NAME: - raise AcceptanceError(f"Expected exactly one {TEST_NAME} result, got {len(cases)}") - case = cases[0] - if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): - raise AcceptanceError("Installed-wheel acceptance must pass, without skips") - properties = case.findall("./properties/property[@name='dataset_acceptance']") + return ET.parse(report).findall(".//testcase") + + +def read_evidence(report: Path, wheel_sha256: str) -> dict: + """Require the wheel case plus every registered gate, with no silent skips.""" + cases = read_cases(report) + nodes = [test_node_id(case) for case in cases] + if nodes.count(TEST_NODE) != 1: + raise AcceptanceError(f"Expected exactly one {TEST_NODE} result") + if len(nodes) != len(set(nodes)): + raise AcceptanceError("Duplicate nightly testcase identities") + missing = set(REQUIRED_TEST_NODES) - set(nodes) + if missing: + raise AcceptanceError(f"Missing required nightly test nodes: {sorted(missing)}") + for node, case in zip(nodes, cases, strict=True): + if any(case.find(tag) is not None for tag in ("skipped", "failure", "error")): + raise AcceptanceError(f"Nightly acceptance must pass, without skips: {node}") + wheel_case = cases[nodes.index(TEST_NODE)] + properties = wheel_case.findall("./properties/property[@name='dataset_acceptance']") if len(properties) != 1: raise AcceptanceError("Expected exactly one dataset_acceptance evidence property") evidence = json.loads(properties[0].get("value", "")) @@ -56,7 +88,8 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: sys.executable, "-m", "pytest", - TEST_NODE, + "-m", + "nightly", "-o", "junit_family=xunit1", f"--junitxml={report}", @@ -67,6 +100,8 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: summary = { "wheel": str(wheel), "test_node": TEST_NODE, + "required_test_nodes": list(REQUIRED_TEST_NODES), + "test_nodes": [], "command": command, "status": "failed", } @@ -79,13 +114,18 @@ def run_acceptance(wheel: Path, output_dir: Path) -> None: ["git", "rev-parse", "HEAD"], cwd=ROOT, check=True, capture_output=True, text=True ).stdout.strip() summary["source_sha"] = source_sha - print(f"Running {TEST_NODE} at {source_sha}", flush=True) + print( + f"Running nightly acceptance at {source_sha}; required: {REQUIRED_TEST_NODES}", + flush=True, + ) with (output_dir / "pytest.log").open("w") as log: result = subprocess.run( command, cwd=ROOT, env=env, stdout=log, stderr=subprocess.STDOUT ) print((output_dir / "pytest.log").read_text(), end="", flush=True) summary["pytest_exit_code"] = result.returncode + if report.is_file(): + summary["test_nodes"] = [test_node_id(case) for case in read_cases(report)] if result.returncode != 0: raise AcceptanceError(f"pytest exited with {result.returncode}; see pytest.log") summary["acceptance"] = read_evidence(report, wheel_sha256) diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 91e9211..c2a3c57 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -14,12 +14,12 @@ Concurrency is keyed by GitHub ref and superseded runs are cancelled. | --- | --- | | Snapshot provenance | Full Git history; base/head comparison; PR decision required only in PR mode | | Lint & Format | Python 3.13, editable dev install, Ruff, Black, basedpyright with the selected interpreter | -| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow"` | -| Full test tier | Python 3.13; fixture provenance first, then all tests with branch coverage, including slow tests | +| Compatibility matrix | Python 3.10–3.12; fixture provenance first, then `pytest -m "not slow and not nightly"` | +| Full PR test tier | Python 3.13; `pytest -m "not nightly"` with branch coverage, preserving the existing slow tests | | Overall coverage | `pyproject.toml` fail-under 90 with branches enabled | | Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently | | Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance | -| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then required installed-wheel CLI/API test | +| Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then `nightly` tests including the required installed-wheel CLI/API test | Million-event deep-offset pagination tests are marked `slow`; their small pagination contracts still run across the compatibility matrix. @@ -30,9 +30,13 @@ The [acceptance script](../scripts/clean_wheel_agent_acceptance.py) uses tempora paths but imports a reviewed executable fixture. These are more than YAML checks. The nightly/manual job invokes `run_dataset_acceptance.py` with the single built -wheel. Missing wheels, missing/empty JUnit, skipped/failed tests, extra test cases, -or evidence for a different wheel fail the gate. It uploads the pytest log, JUnit -(including the test's evidence), and a summary with actual checkout SHA and wheel +wheel. All `nightly` tests are selected automatically; the installed-wheel case +must appear exactly once. New mandatory gates also add their complete pytest +node IDs to `REQUIRED_TEST_NODES` in that runner. Do not register absent future +files. Missing wheels, missing/empty JUnit, missing required nodes, duplicate +identities, any skipped/failed/error test, or evidence for a different wheel fail +the gate. It uploads the pytest log, JUnit (including the test's evidence), and a +summary with complete executed/required test node IDs, actual checkout SHA and wheel SHA-256, even after failure. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` is available for additional test-owned artifacts. Local invocation: @@ -44,8 +48,8 @@ python .github/scripts/run_dataset_acceptance.py \ This tier initially exercises the existing 500,002-event `pip --target` acceptance test. It does not claim a clean venv, two full 500,000-event shards, complete CLI template coverage, live agents or upstream GUI interoperability. The ordinary -full suite may still skip this opt-in wheel test; the required nightly/manual -runner must never do so. Manually dispatch the exact published branch/commit and +PR tier excludes `nightly` tests; the required nightly/manual runner must never +skip them. Manually dispatch the exact published branch/commit and verify the run's head SHA and summary; a PR run alone is not nightly evidence. ## Release Workflow diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e71defe..0465fca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -129,11 +129,11 @@ jobs: - name: Run non-slow compatibility tests if: matrix.python-version != '3.13' - run: pytest -m "not slow" + run: pytest -m "not slow and not nightly" - - name: Run all tests with coverage + - name: Run full PR tests with coverage if: matrix.python-version == '3.13' - run: pytest --cov --cov-report=xml --cov-report=term-missing + run: pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing - name: Guard critical runtime coverage if: matrix.python-version == '3.13' diff --git a/pyproject.toml b/pyproject.toml index f97f49b..f65fa0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -93,6 +93,7 @@ select = ["E", "F", "W", "I", "N", "UP", "B", "C4"] ignore = ["E501"] [tool.pytest.ini_options] +markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"] testpaths = ["tests"] python_files = ["test_*.py"] python_functions = ["test_*"] diff --git a/tests/core/test_dataset_scale_acceptance.py b/tests/core/test_dataset_scale_acceptance.py index e2d1b0d..8fb8a05 100644 --- a/tests/core/test_dataset_scale_acceptance.py +++ b/tests/core/test_dataset_scale_acceptance.py @@ -226,6 +226,7 @@ def installed_runtime(tmp_path): return cli, env, module, hashlib.sha256(wheel.read_bytes()).hexdigest() +@pytest.mark.nightly def test_installed_trusted_producer_default_export_cli_and_api( tmp_path, installed_runtime, record_property ): diff --git a/tests/test_ci_tiers.py b/tests/test_ci_tiers.py index 9a6e10f..f011586 100644 --- a/tests/test_ci_tiers.py +++ b/tests/test_ci_tiers.py @@ -19,10 +19,14 @@ def test_compatibility_matrix_leaves_slow_and_coverage_on_python_313(): steps = named_steps(job) fast = steps["Run non-slow compatibility tests"] assert fast["if"] == "matrix.python-version != '3.13'" - assert fast["run"] == 'pytest -m "not slow"' - full = steps["Run all tests with coverage"] + assert fast["run"] == 'pytest -m "not slow and not nightly"' + full = steps["Run full PR tests with coverage"] assert full["if"] == "matrix.python-version == '3.13'" - assert full["run"] == "pytest --cov --cov-report=xml --cov-report=term-missing" + assert full["run"] == 'pytest -m "not nightly" --cov --cov-report=xml --cov-report=term-missing' + assert ( + 'markers = ["nightly: million-scale acceptance gate (scheduled/manual only)"]' + in Path("pyproject.toml").read_text() + ) assert "if" not in steps["Verify executable fixture provenance"] names = list(steps) assert names.index("Verify executable fixture provenance") < names.index(fast["name"]) diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py index 36ff22e..037e488 100644 --- a/tests/test_dataset_acceptance_runner.py +++ b/tests/test_dataset_acceptance_runner.py @@ -24,7 +24,13 @@ def runner(): def write_report(path, name, evidence, outcome=None): suite = ET.Element("testsuite") - case = ET.SubElement(suite, "testcase", name=name) + case = ET.SubElement( + suite, + "testcase", + name=name, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) if outcome: ET.SubElement(case, outcome) if evidence is not None: @@ -61,7 +67,13 @@ def test_runner_rejects_empty_or_extra_collection(runner, tmp_path, count): report = tmp_path / "junit.xml" suite = ET.Element("testsuite") for _ in range(count): - ET.SubElement(suite, "testcase", name=runner.TEST_NAME) + ET.SubElement( + suite, + "testcase", + name=runner.TEST_NAME, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) ET.ElementTree(suite).write(report) with pytest.raises(runner.AcceptanceError, match="exactly one"): runner.read_evidence(report, "abc") @@ -74,6 +86,106 @@ def test_runner_rejects_wrong_test_even_if_it_passes(runner, tmp_path): runner.read_evidence(report, "abc") +def add_case( + report, + *, + name="test_extra", + file="tests/test_extra.py", + classname="tests.test_extra", + outcome=None, +): + tree = ET.parse(report) + case = ET.SubElement(tree.getroot(), "testcase", name=name, file=file, classname=classname) + if outcome: + ET.SubElement(case, outcome) + tree.write(report) + + +def test_runner_accepts_additional_nightly_cases_and_records_complete_identities(runner, tmp_path): + report = tmp_path / "junit.xml" + evidence = {"wheel_sha256": "abc"} + write_report(report, runner.TEST_NAME, evidence) + add_case(report, name="test_extra[v2]", classname="tests.test_extra.TestRoutes") + assert runner.read_evidence(report, "abc") == evidence + assert [runner.test_node_id(case) for case in runner.read_cases(report)] == [ + runner.TEST_NODE, + "tests/test_extra.py::TestRoutes::test_extra[v2]", + ] + + +def test_runner_rejects_missing_configured_gate(runner, tmp_path, monkeypatch): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + required = "tests/test_extra.py::test_extra" + monkeypatch.setattr(runner, "REQUIRED_TEST_NODES", (runner.TEST_NODE, required)) + with pytest.raises(runner.AcceptanceError, match="Missing required nightly test nodes"): + runner.read_evidence(report, "abc") + add_case(report) + assert runner.read_evidence(report, "abc") == {"wheel_sha256": "abc"} + + +def test_runner_rejects_duplicate_additional_cases(runner, tmp_path): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + add_case(report) + add_case(report) + with pytest.raises(runner.AcceptanceError, match="Duplicate nightly"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_any_nonpassing_additional_case(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) + add_case(report, outcome=outcome) + with pytest.raises( + runner.AcceptanceError, match="without skips: tests/test_extra.py::test_extra" + ): + runner.read_evidence(report, "abc") + + +def test_same_function_name_in_another_file_cannot_replace_required_wheel_case(runner, tmp_path): + report = tmp_path / "junit.xml" + ET.ElementTree(ET.Element("testsuite")).write(report) + add_case(report, name=runner.TEST_NAME) + with pytest.raises(runner.AcceptanceError, match="exactly one"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize( + "attributes", [{"file": ""}, {"file": "../tests/test_extra.py"}, {"classname": "wrong"}] +) +def test_runner_rejects_incomplete_or_inconsistent_junit_identity(runner, attributes): + case = ET.Element( + "testcase", file="tests/test_extra.py", classname="tests.test_extra", name="test_extra" + ) + case.attrib.update(attributes) + with pytest.raises(runner.AcceptanceError, match="identity"): + runner.test_node_id(case) + + +def test_nightly_marker_selection_collects_every_configured_gate(runner): + result = subprocess.run( + [ + sys.executable, + "-m", + "pytest", + "--collect-only", + "--strict-markers", + "-q", + "-o", + "addopts=", + "-m", + "nightly", + ], + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + collected = {line for line in result.stdout.splitlines() if "::" in line} + assert set(runner.REQUIRED_TEST_NODES) <= collected + + def test_runner_rejects_duplicate_evidence(runner, tmp_path): report = tmp_path / "junit.xml" write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}) @@ -132,7 +244,7 @@ def run(command, **kwargs): if command[0] == "git": return subprocess.CompletedProcess(command, 0, stdout="checkout-sha\n") assert command[:3] == [sys.executable, "-m", "pytest"] - assert runner.TEST_NODE in command + assert command[3:5] == ["-m", "nightly"] assert "junit_family=xunit1" in command assert kwargs["env"]["PT_SNAP_ACCEPTANCE_WHEEL"] == str(wheel) assert kwargs["env"]["PT_SNAP_ACCEPTANCE_EVIDENCE_DIR"] == str(output) @@ -153,5 +265,7 @@ def run(command, **kwargs): assert summary["source_sha"] == "checkout-sha" assert summary["wheel_sha256"] == wheel_hash assert summary["pytest_exit_code"] == exit_code + assert summary["required_test_nodes"] == [runner.TEST_NODE] + assert summary["test_nodes"] == ([runner.TEST_NODE] if write_junit else []) assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") assert (output / "pytest.log").read_text() == "captured pytest output\n" From 571fbb7e05349081b673db908bf0f216ab8d200c Mon Sep 17 00:00:00 2001 From: codeasier <48463012+codeasier@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:26:27 +0800 Subject: [PATCH 3/3] fix: verify dataset generation before publishing ordinary query results --- docs/en/dataset-acceptance.md | 42 ++++ docs/en/querying.md | 8 + docs/zh/dataset-acceptance.md | 33 +++ docs/zh/querying.md | 10 +- src/pt_snap_cli/core/query_service.py | 8 +- tests/core/test_dataset_concurrency.py | 105 +++++++++ tests/dataset_concurrency_support.py | 143 +++++++++++++ tests/test_dataset_mutation_acceptance.py | 248 ++++++++++++++++++++++ tests/test_dataset_signal_boundaries.py | 75 +++++++ 9 files changed, 666 insertions(+), 6 deletions(-) create mode 100644 tests/core/test_dataset_concurrency.py create mode 100644 tests/dataset_concurrency_support.py create mode 100644 tests/test_dataset_mutation_acceptance.py create mode 100644 tests/test_dataset_signal_boundaries.py diff --git a/docs/en/dataset-acceptance.md b/docs/en/dataset-acceptance.md index d3bccba..7ff796f 100644 --- a/docs/en/dataset-acceptance.md +++ b/docs/en/dataset-acceptance.md @@ -93,3 +93,45 @@ on inspectable test-owned input. It does not execute live PyTorch/NPU collection the upstream msinsight producer, or its GUI. Those acceptance steps remain separate. No new committed pickle is introduced. The provenance gate runs before collection; the test-owned temporary input is written during the test. + +## Concurrent readers and process interruption + +`tests/core/test_dataset_concurrency.py` runs two and four independent readers +in both threads and spawned processes, against the same synthetic dataset and +through both ATTACH and scratch routes. Barriers follow an actual SQLite read +or scratch insert; all readers hold their own query resources concurrently. +Their complete results must agree, scratch directories must be distinct and +removed on normal completion, and source hashes must stay unchanged. This does +not promise that one analyzer or its SQLite connections can be shared by threads. + +`tests/test_dataset_signal_boundaries.py` sends actual POSIX signals to the CLI +after a completed scratch insert. Normal completion and SIGINT unwind product +resources and remove scratch. **Current SIGTERM and SIGKILL terminate the process +without that cleanup: the private `pt-snap-lifecycle-*` directory and `derived.db` +remain.** The tests record this limitation, rather than treating test teardown as +product cleanup or claiming issue #252's no-residue kill goal is satisfied. Source +hashes remain unchanged and no partial query output is published. No signal +handler, startup reclamation, or automatic deletion policy is introduced. + +These small deterministic tests run in the regular suite (signal cases are POSIX +only). Timeout/error cleanup is separate from abrupt termination: Python-level +unwinding is not evidence that SIGTERM or SIGKILL runs a `finally` block. + +`tests/test_dataset_mutation_acceptance.py` pauses a spawned CLI/API query after +its first shard has actually supplied rows or a count, then changes test-owned +members in place. Both the default strong and explicit immutable cache modes must +reject the changed generation without publishing partial rows/counts. Unchanged +runs succeed with independently expected results. Hash-work assertions quantify +compatibility-v1 cold/warm queries: default queries scan each manifest/member +three/two times respectively (admission plus the new final check). Explicit POSIX +immutable queries scan members once/zero times and the manifest twice/zero times. +The final check itself adds one full scan in default mode, zero in metadata-only +immutable mode, and shares the query's deadline. Non-POSIX immutable requests do +not reuse admission based on creation times: both cold and warm queries scan each +member twice and the manifest three times, including a full final hash guard. +Default strong warm reuse still works on non-POSIX systems. The tests exercise +this conservative policy through a resolver-local simulated non-POSIX case too; +that simulation is not evidence of actual Windows execution. These are work-count assertions, +not fixed wall-time or RSS promises. The guard detects these controlled mutations; +it does not create an atomic filesystem snapshot or prove all change-and-restore +races impossible. diff --git a/docs/en/querying.md b/docs/en/querying.md index 62b0e37..07827e1 100644 --- a/docs/en/querying.md +++ b/docs/en/querying.md @@ -193,6 +193,14 @@ check cancellation. Individual filesystem calls are not OS-level hard-preempted. `PT_SNAP_DATASET_CACHE=immutable` explicitly opts into metadata-only validation reuse only when the caller guarantees immutable finalized files and reliable filesystem change times; see [dataset validation and cache contracts](sharded-snapshotdb.md). +Every dataset query also verifies that admitted generation before returning rows or +counts, including count-only queries. A detected concurrent change fails the whole +query with a database-integrity error; no partial result is published. By default, +this final guard adds one complete manifest/member hash pass under the same +operation deadline. The explicit immutable mode retains its metadata-only guard +when its existing filesystem requirements hold. This is a change-detection boundary, +not an atomic filesystem snapshot: it does not prove all change-and-restore races +impossible or support querying mutable live SQLite artifacts. Source artifacts are never repaired or written. A large lifecycle query may create a private temporary derived database as described below. diff --git a/docs/zh/dataset-acceptance.md b/docs/zh/dataset-acceptance.md index 4a247a7..1f2f962 100644 --- a/docs/zh/dataset-acceptance.md +++ b/docs/zh/dataset-acceptance.md @@ -71,3 +71,36 @@ PT_SNAP_ACCEPTANCE_WHEEL="$(realpath dist/.whl)" \ 新增导出门禁在可审查的测试输入上执行真实第一方 replay/export/query 链路;不执行 实时 PyTorch/NPU 采集、上游 msinsight 生产端或 GUI,这些验收仍独立。未增加任何提交 到仓库的 pickle。provenance 门禁在收集测试前运行;临时受信输入在测试期间自行写入。 + +## 并发读取与进程中断 + +`tests/core/test_dataset_concurrency.py` 对同一份合成数据集分别启动两个和四个 +独立读取者,覆盖线程、spawn 子进程,以及 ATTACH 和 scratch 两条路径。 +屏障位于实际 SQLite 读取或 scratch 插入完成之后,保证所有读取者同时持有各自的 +查询资源。完整结果必须一致,scratch 目录必须互不相同且在正常完成后移除,源文件 +哈希必须保持不变。这不代表同一个 analyzer 或其 SQLite 连接可以跨线程共享。 + +`tests/test_dataset_signal_boundaries.py` 在实际 scratch 插入完成后向 CLI 发送 +真实 POSIX 信号。正常完成和 SIGINT 会展开产品资源并移除 scratch。 +**目前 SIGTERM 和 SIGKILL 会直接终止进程,未执行上述清理,私有 +`pt-snap-lifecycle-*` 目录及 `derived.db` 会残留。** 测试明确记录这一限制, +不会把测试清理当作产品清理,也不宣称已经满足 issue #252 的 kill 后无残留目标。 +源文件哈希保持不变,不发布部分查询结果。本次未引入信号处理器、启动回收或自动删除策略。 + +这些小规模、确定性测试进入常规套件(信号用例仅限 POSIX)。超时及错误的清理与 +进程被直接终止是不同边界:Python 异常展开不能证明 SIGTERM 或 SIGKILL 会执行 +`finally`。 + +`tests/test_dataset_mutation_acceptance.py` 在 spawn 子进程中的 CLI/API 查询实际 +读取第一片的行或计数之后暂停,由父进程原地修改测试自有成员。默认强校验和显式 +immutable 缓存模式都必须拒绝已变更的 generation,不发布部分行或计数。 +未变更的运行必须成功且符合独立计算的结果。哈希工作量断言明确兼容 v1 的冷/热查询成本: +默认分别对每个 manifest/成员扫描三次/两次(准入加最终检查);POSIX immutable +模式对成员分别扫描一次/零次,对 manifest 分别扫描两次/零次。最终检查本身在默认模式 +增加一次完整扫描,在纯元数据 immutable 模式不增加哈希扫描,并共用查询 deadline。 +非 POSIX 上的 immutable 请求不会根据创建时间复用准入:冷/热查询都对每个成员扫描 +两次、对 manifest 扫描三次,包含完整的最终哈希检查。默认强校验在非 POSIX 上仍可复用 +经过内容验证的热准入。测试还通过仅替换 resolver 的平台对象,确定性覆盖这一保守策略; +此模拟不是实际 Windows 执行证据。 +这是工作量断言,不是固定耗时或 RSS 保证。检查会检测这些受控变更,但不会创建原子文件系统 +快照,也不证明所有先变更再恢复的竞态都能被检测。 diff --git a/docs/zh/querying.md b/docs/zh/querying.md index 8ee3a58..76e061e 100644 --- a/docs/zh/querying.md +++ b/docs/zh/querying.md @@ -163,8 +163,14 @@ SQLite、一个批次及单个 cell 有额外开销。默认仍执行强准入 分片数),新 CLI 进程仍冷启动。准入共用该 deadline:hash 分块、Python 行批次及 SQLite progress handler 都检查取消。 单个文件系统调用不提供 OS 级硬抢占。仅当调用方保证最终产物不可变且文件系统变更时间 可靠时,才可通过 `PT_SNAP_DATASET_CACHE=immutable` 显式启用纯元数据校验复用;参见 -[数据集校验与缓存合同](sharded-snapshotdb.md)。不修复或写入源产物;较大的生命周期查询 -可能生成下述私有临时派生数据库。 +[数据集校验与缓存合同](sharded-snapshotdb.md)。 +每次数据集查询在返回行或计数之前,都会再次检查已准入的 generation,纯计数查询也不例外。 +一旦检测到并发变更,整个查询以数据库完整性错误失败,不发布部分结果。默认情况下, +这一最终检查在同一操作 deadline 内额外扫描一次 manifest/全部成员的完整内容哈希。 +显式 immutable 模式在满足原有文件系统要求时,仍使用纯元数据的最终检查。 +这是变更检测边界,不是原子文件系统快照:不证明所有先变更再恢复的竞态都能被检测, +也不代表支持查询可变的在线 SQLite 产物。 +不修复或写入源产物;较大的生命周期查询可能生成下述私有临时派生数据库。 兼容导出器默认**每设备每片 500000 个真实事件**。该存储容量与 **100000 行 / 64 MiB 查询工作预算**独立:即使只有一个默认大小的分片,无足够筛选或分页的结果物化也可能 diff --git a/src/pt_snap_cli/core/query_service.py b/src/pt_snap_cli/core/query_service.py index 2b5049e..8fddf3d 100644 --- a/src/pt_snap_cli/core/query_service.py +++ b/src/pt_snap_cli/core/query_service.py @@ -302,10 +302,10 @@ def execute_query( if isinstance(stack_bytes, int) and stack_bytes >= 0: summarize_dataset_stacks(result.rows, stack_bytes) budget.remaining() - if _resolved is not None: - # Inspection establishes the baseline. Reject changed contents - # after each composed step, before any report can be returned. - dataset.require_unchanged(budget) + # Inspection establishes the baseline for standalone queries and + # composed reports alike. Reject mixed-generation rows or counts + # before either adapter can publish any part of the result. + dataset.require_unchanged(budget) if result.scope is not None: result.scope["work_budget"] = budget.describe() return result diff --git a/tests/core/test_dataset_concurrency.py b/tests/core/test_dataset_concurrency.py new file mode 100644 index 0000000..8d742bd --- /dev/null +++ b/tests/core/test_dataset_concurrency.py @@ -0,0 +1,105 @@ +"""Independent readers overlap real query stages without sharing analyzers.""" + +import multiprocessing +import queue +import threading + +import pytest + +from tests.core.dataset_lifecycle_fixtures import build +from tests.dataset_concurrency_support import ( + ISOLATED_ENV, + WAIT_SECONDS, + lifecycle_stage, + process_reader, + query_worker, + read_query, +) +from tests.test_dataset_focus import hashes + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + for name in ISOLATED_ENV: + monkeypatch.delenv(name, raising=False) + + +@pytest.mark.parametrize("readers", [2, 4]) +@pytest.mark.parametrize("route", ["attach", "scratch"]) +@pytest.mark.parametrize("kind", ["threads", "processes"]) +def test_simultaneous_readers_return_complete_identical_results(tmp_path, kind, route, readers): + _, root, _, _, blocks = build(tmp_path, 7) + before = hashes(root) + scratch_parent = tmp_path / "scratch" + scratch_parent.mkdir() + with lifecycle_stage(route, scratch_parent, lambda evidence: None): + expected = read_query(root, route) + assert expected["returned"] == expected["total"] == len(blocks) + assert expected["total_is_exact"] and not expected["has_more"] + assert {row["id"] for row in expected["rows"]} == {block["id"] for block in blocks} + + context = multiprocessing.get_context("spawn") if kind == "processes" else threading + ready = context.Barrier(readers + 1) + release = context.Event() + output = context.Queue() if kind == "processes" else queue.Queue() + workers = [] + + def reached(evidence): + output.put(("stage", evidence)) + ready.wait(WAIT_SECONDS) + if not release.wait(WAIT_SECONDS): + raise TimeoutError("Parent did not release the query-stage barrier") + + try: + with lifecycle_stage(route, scratch_parent, reached): + for _ in range(readers): + worker = ( + context.Process( + target=process_reader, + args=(root, route, scratch_parent, ready, release, output), + ) + if kind == "processes" + else threading.Thread(target=query_worker, args=(root, route, output)) + ) + workers.append(worker) + worker.start() + # Every reader has completed a real SQL operation and holds its own + # connection open. No scheduling speed or arbitrary sleep is evidence. + ready.wait(WAIT_SECONDS) + stages = [output.get(timeout=WAIT_SECONDS) for _ in range(readers)] + assert all(kind == "stage" for kind, _ in stages), stages + identities = {(item["pid"], item["thread"]) for _, item in stages} + assert len(identities) == readers + if route == "scratch": + paths = {item["database"] for _, item in stages} + assert len(paths) == readers + assert all(item["inserted_rows"] > 0 for _, item in stages) + assert len(list(scratch_parent.glob("*/derived.db"))) == readers + else: + assert not list(scratch_parent.iterdir()) + assert hashes(root) == before + release.set() + results = [output.get(timeout=WAIT_SECONDS) for _ in range(readers)] + assert all(kind == "result" for kind, _ in results), results + assert all(result == expected for _, result in results) + finally: + release.set() + ready.abort() + for worker in workers: + worker.join(WAIT_SECONDS) + if kind == "processes" and worker.is_alive(): + worker.terminate() + worker.join(WAIT_SECONDS) + assert not worker.is_alive() + if kind == "processes": + assert worker.exitcode == 0 + worker.close() + if kind == "processes": + output.close() + output.join_thread() + # Only after all product contexts have exited do we check cleanup. The test + # does not remove scratch directories to manufacture this observation. + assert not list(scratch_parent.iterdir()) + assert hashes(root) == before diff --git a/tests/dataset_concurrency_support.py b/tests/dataset_concurrency_support.py new file mode 100644 index 0000000..5cbf969 --- /dev/null +++ b/tests/dataset_concurrency_support.py @@ -0,0 +1,143 @@ +"""Real query-stage barriers shared by thread, spawned-process and signal tests. + +Hooks observe a completed SQLite read/insert, then block on explicit IPC. They +never replace query results, install signal handlers, or perform product cleanup. +""" + +import os +import sys +import threading +import traceback +from contextlib import ExitStack, contextmanager, redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +from pt_snap_cli.api import QueryLimits, SnapshotAnalyzer +from pt_snap_cli.core import dataset_lifecycle_scratch as scratch +from pt_snap_cli.core import dataset_lifecycle_sql as sql +from pt_snap_cli.core.dataset_lifecycle_connection import _AttachedSources + +WAIT_SECONDS = 30 +QUERY_PARAMS = {"order_by": "id", "order_dir": "ASC", "limit": -1} +ISOLATED_ENV = ( + "PT_SNAP_DB_PATH", + "PT_SNAP_DATASET_CACHE", + "PT_SNAP_QUERY_TIMEOUT", + "PT_SNAP_MAX_WORK_ROWS", + "PT_SNAP_MAX_WORK_BYTES", + "PT_SNAP_MAX_SCRATCH_DB_BYTES", +) + + +@contextmanager +def lifecycle_stage(route, scratch_parent, reached): + original_read = _AttachedSources.read + original_execute = scratch._ScratchSources.execute + original_directory = scratch.tempfile.TemporaryDirectory + + def observe(database): + if getattr(database, "_test_stage_reached", False): + return + database._test_stage_reached = True + evidence = {"pid": os.getpid(), "thread": threading.get_ident(), "route": route} + if route == "scratch": + # These are actual product-created bytes, not a synthetic ready file. + cursor = database.connection.execute("PRAGMA database_list") + try: + evidence["database"] = next(row[2] for row in cursor if row[1] == "main") + finally: + cursor.close() + cursor = database.connection.execute("SELECT COUNT(*) FROM lifecycle_state") + try: + evidence["inserted_rows"] = cursor.fetchone()[0] + finally: + cursor.close() + reached(evidence) + + def read(database, statement, values=None): + result = original_read(database, statement, values) + if route == "attach" and statement.startswith("WITH observations AS"): + observe(database) + return result + + def execute(database, statement, values=()): + result = original_execute(database, statement, values) + if route == "scratch" and statement.startswith("INSERT INTO lifecycle_state"): + observe(database) + return result + + def directory(*args, **kwargs): + return original_directory(*args, dir=scratch_parent, **kwargs) + + with ExitStack() as stack: + stack.enter_context(patch.object(_AttachedSources, "read", read)) + stack.enter_context(patch.object(scratch._ScratchSources, "execute", execute)) + stack.enter_context(patch.object(scratch.tempfile, "TemporaryDirectory", directory)) + if route == "scratch": + stack.enter_context(patch.object(sql, "_ATTACHED_OBSERVATION_LIMIT", 0)) + yield + + +def read_query(root, route): + with SnapshotAnalyzer(root) as analyzer: + return analyzer.execute_query( + "block", + QUERY_PARAMS, + exact_total=True, + query_limits=QueryLimits( + max_scratch_db_bytes=0 if route == "attach" else 512 * 1024**2 + ), + ) + + +def query_worker(root, route, output): + try: + output.put(("result", read_query(root, route))) + except BaseException: + output.put(("error", traceback.format_exc())) + + +def process_reader(root, route, scratch_parent, ready, release, output): + def reached(evidence): + output.put(("stage", evidence)) + ready.wait(WAIT_SECONDS) + if not release.wait(WAIT_SECONDS): + raise TimeoutError("Parent did not release the query-stage barrier") + + with lifecycle_stage(route, scratch_parent, reached): + query_worker(root, route, output) + + +def cli_at_scratch_stage(root, scratch_parent, channel, stdout_path, stderr_path): + """Call the real console entry point with Python's original signal behavior.""" + from pt_snap_cli.cli import _safe_call + + def reached(evidence): + channel.send(evidence) + # Parent either permits completion or sends an actual operating-system + # signal. No test-created handler turns SIGTERM into a Python exception. + if not channel.poll(WAIT_SECONDS): + raise TimeoutError("Parent did not release or signal the CLI") + assert channel.recv() == "continue" + + sys.argv = [ + "pt-snap", + "query", + str(root), + "--template-use", + "block", + "--params", + '{"limit":5}', + "--json", + ] + with ( + Path(stdout_path).open("w") as stdout, + Path(stderr_path).open("w") as stderr, + redirect_stdout(stdout), + redirect_stderr(stderr), + lifecycle_stage("scratch", scratch_parent, reached), + ): + try: + raise SystemExit(_safe_call()) + finally: + channel.close() diff --git a/tests/test_dataset_mutation_acceptance.py b/tests/test_dataset_mutation_acceptance.py new file mode 100644 index 0000000..40b0258 --- /dev/null +++ b/tests/test_dataset_mutation_acceptance.py @@ -0,0 +1,248 @@ +"""A query may not publish rows/counts assembled from different dataset generations.""" + +import io +import json +import multiprocessing +import os +import sqlite3 +import sys +from collections import Counter +from contextlib import closing, redirect_stderr, redirect_stdout +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core import dataset_resolver as resolver +from pt_snap_cli.core.dataset_sources import DatasetSourceResolver +from pt_snap_cli.core.errors import QueryTimeoutError +from tests.core.dataset_lifecycle_fixtures import build +from tests.dataset_concurrency_support import ISOLATED_ENV, WAIT_SECONDS +from tests.test_dataset_focus import hashes + + +def params_for(shape): + if shape == "count-only": + return {"action": 4, "limit": 0} + return {"order_by": "id", "order_dir": "ASC", "limit": -1} + + +def paused_query(root, adapter, shape, channel): + """Observe the first completed shard operation in an actual CLI/API query.""" + original = DatasetSourceResolver.read + observed = False + + def read(sources, path, statement, values=None): + nonlocal observed + result = original(sources, path, statement, values) + selected = ( + statement.startswith("SELECT MIN(id)") + if shape == "count-only" + else " WHERE t.id IN (" in statement + ) + if selected and not observed: + observed = True + channel.send({"stage": "first-shard-read", "member": path.name, "rows": len(result)}) + if not channel.poll(WAIT_SECONDS): + raise TimeoutError("Parent did not release the mutation barrier") + assert channel.recv() == "continue" + return result + + try: + with patch.object(DatasetSourceResolver, "read", read): + if adapter == "api": + try: + with SnapshotAnalyzer(root) as analyzer: + result = analyzer.execute_query( + "event", params_for(shape), exact_total=True + ) + outcome = {"status": "success", "payload": result} + except Exception as exc: + outcome = {"status": "error", "type": type(exc).__name__, "message": str(exc)} + else: + from pt_snap_cli.cli import _safe_call + + sys.argv = [ + "pt-snap", + "query", + str(root), + "--template-use", + "event", + "--params", + json.dumps(params_for(shape)), + "--exact-total", + "--json", + ] + stdout, stderr = io.StringIO(), io.StringIO() + with redirect_stdout(stdout), redirect_stderr(stderr): + code = _safe_call() + outcome = {"code": code, "stdout": stdout.getvalue(), "stderr": stderr.getvalue()} + channel.send(outcome) + finally: + channel.close() + + +@pytest.fixture(autouse=True) +def isolate(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + for name in ISOLATED_ENV: + monkeypatch.delenv(name, raising=False) + + +@pytest.mark.parametrize("cache", ["strong", "immutable"]) +@pytest.mark.parametrize("adapter", ["api", "cli-json"]) +@pytest.mark.parametrize("shape", ["rows", "count-only"]) +@pytest.mark.parametrize("mutate", [False, True], ids=["unchanged", "mutated"]) +def test_query_rejects_mid_read_mutation_without_partial_payload( + tmp_path, monkeypatch, cache, adapter, shape, mutate +): + if cache == "immutable": + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + _, root, _, trace, _ = build(tmp_path, 7) + before = hashes(root) + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe() + process = context.Process(target=paused_query, args=(root, adapter, shape, child)) + process.start() + child.close() + try: + assert parent.poll(WAIT_SECONDS), "Query did not reach its first completed shard read" + stage = parent.recv() + assert stage == { + "stage": "first-shard-read", + "member": "slice_00000.db", + "rows": 1 if shape == "count-only" else 10, + } + if mutate: + # Test-owned artifacts only. The query is already admitted and has + # consumed real data; update every shard before it can read another. + statement = ( + "UPDATE trace_entry_0 SET action=7 WHERE id>=0" + if shape == "count-only" + else "UPDATE trace_entry_0 SET allocated=allocated+10000 WHERE id>=0" + ) + for member in sorted(root.glob("device_0/*.db")): + with closing(sqlite3.connect(member)) as connection, connection: + connection.execute(statement) + expected_hashes = hashes(root) + assert (expected_hashes != before) == mutate + parent.send("continue") + assert parent.poll(WAIT_SECONDS), "Query did not finish after barrier release" + outcome = parent.recv() + process.join(WAIT_SECONDS) + assert process.exitcode == 0 + if mutate: + if adapter == "api": + assert outcome == { + "status": "error", + "type": "DatabaseSchemaError", + "message": "Dataset members changed during operation; retry.", + } + else: + assert outcome["code"] == 1 and outcome["stdout"] == "" + payload = json.loads(outcome["stderr"]) + assert payload["error"]["code"] == "DATABASE_SCHEMA_INVALID" + assert ( + payload["error"]["message"] + == "Dataset members changed during operation; retry." + ) + assert not {"rows", "total", "returned", "scope"} & payload.keys() + else: + if adapter == "api": + assert outcome["status"] == "success" + payload = outcome["payload"] + else: + assert outcome["code"] == 0 and outcome["stderr"] == "" + payload = json.loads(outcome["stdout"]) + expected_total = ( + sum(event[1] == 4 for event in trace) if shape == "count-only" else len(trace) + ) + assert payload["total"] == expected_total and payload["total_is_exact"] + assert payload["returned"] == (0 if shape == "count-only" else len(trace)) + if shape == "rows": + assert [(row["id"], row["allocated"]) for row in payload["rows"]] == [ + (event[0], event[5]) for event in trace + ] + # Readers must not modify either the pristine or deliberately changed bytes. + assert hashes(root) == expected_hashes + finally: + parent.close() + if process.is_alive(): + process.kill() + process.join(WAIT_SECONDS) + process.close() + + +@pytest.mark.parametrize("cache", ["strong", "immutable"]) +@pytest.mark.parametrize("temperature", ["cold", "warm"]) +@pytest.mark.parametrize("platform", ["native", "simulated-non-posix"]) +def test_final_query_guard_hash_work_is_explicit( + tmp_path, monkeypatch, cache, temperature, platform, record_property +): + if cache == "immutable": + monkeypatch.setenv("PT_SNAP_DATASET_CACHE", "immutable") + if platform == "simulated-non-posix": + # Exercise the actual conservative policy on every CI OS without + # changing pathlib's host platform or weakening any source guard. + monkeypatch.setattr(resolver, "os", SimpleNamespace(name="nt", environ=os.environ)) + metadata_only = cache == "immutable" and resolver.os.name == "posix" + _, root, _, _, _ = build(tmp_path, 7) + counts, byte_counts = Counter(), Counter() + original = resolver._hash_file + + def tracked(path, **kwargs): + counts[path] += 1 + byte_counts[path] += path.stat().st_size + return original(path, **kwargs) + + if temperature == "warm": + resolver.DatasetResolver().inspect(root) + with SnapshotAnalyzer(root) as analyzer: + monkeypatch.setattr(resolver, "_hash_file", tracked) + result = analyzer.execute_query("event", {"limit": 0}, exact_total=True) + assert result["total"] == 80 + files = [root / "manifest.json", *root.glob("device_0/*.db")] + # Default strong reuse is content-verified on every OS. Only explicit + # immutable metadata reuse requires POSIX change times. Otherwise immutable + # requests repeat cold admission AND the final guard hashes all bytes. + if cache == "strong": + member_scans = manifest_scans = 3 if temperature == "cold" else 2 + elif metadata_only: + member_scans = int(temperature == "cold") + manifest_scans = 2 * member_scans + else: + member_scans, manifest_scans = 2, 3 + assert root not in resolver._VALIDATED + expected_counts = dict.fromkeys(files, member_scans) if member_scans else {} + if manifest_scans: + expected_counts[root / "manifest.json"] = manifest_scans + assert dict(counts) == expected_counts + assert sum(byte_counts.values()) == sum( + path.stat().st_size * count for path, count in expected_counts.items() + ) + record_property("query_hash_calls", sum(counts.values())) + record_property("query_hash_bytes", sum(byte_counts.values())) + record_property("final_guard_extra_scans", int(not metadata_only)) + + +def test_final_query_guard_uses_same_deadline_and_can_time_out(tmp_path, monkeypatch): + _, root, _, _, _ = build(tmp_path, 7) + original = resolver.ResolvedDataset.require_unchanged + read = DatasetSourceResolver.read + budgets = [] + + def tracked(sources, *args, **kwargs): + budgets.append(sources.budget) + return read(sources, *args, **kwargs) + + def expire(dataset, budget=None): + assert budgets and all(item is budget for item in budgets) + budget.started -= budget.timeout_s + 1 + return original(dataset, budget) + + monkeypatch.setattr(DatasetSourceResolver, "read", tracked) + monkeypatch.setattr(resolver.ResolvedDataset, "require_unchanged", expire) + with SnapshotAnalyzer(root) as analyzer, pytest.raises(QueryTimeoutError): + analyzer.execute_query("event", {"limit": 0}, exact_total=True, timeout_s=30) diff --git a/tests/test_dataset_signal_boundaries.py b/tests/test_dataset_signal_boundaries.py new file mode 100644 index 0000000..35411b8 --- /dev/null +++ b/tests/test_dataset_signal_boundaries.py @@ -0,0 +1,75 @@ +"""Characterize current CLI signal boundaries; no SIGTERM/SIGKILL cleanup promise.""" + +import json +import multiprocessing +import os +import signal +from pathlib import Path + +import pytest + +from tests.core.dataset_lifecycle_fixtures import build +from tests.dataset_concurrency_support import ISOLATED_ENV, WAIT_SECONDS, cli_at_scratch_stage +from tests.test_dataset_focus import hashes + +pytestmark = pytest.mark.skipif(os.name != "posix", reason="Requires actual POSIX signal semantics") + + +@pytest.mark.parametrize("action", ["complete", "SIGINT", "SIGTERM", "SIGKILL"]) +def test_cli_scratch_signal_boundary(tmp_path, monkeypatch, action): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + for name in ISOLATED_ENV: + monkeypatch.delenv(name, raising=False) + _, root, _, _, _ = build(tmp_path, 7) + before = hashes(root) + scratch_parent = tmp_path / "scratch" + scratch_parent.mkdir() + stdout, stderr = tmp_path / "stdout", tmp_path / "stderr" + context = multiprocessing.get_context("spawn") + parent, child = context.Pipe() + process = context.Process( + target=cli_at_scratch_stage, + args=(root, scratch_parent, child, stdout, stderr), + ) + process.start() + child.close() + try: + assert parent.poll(WAIT_SECONDS), "CLI did not reach a completed scratch insert" + stage = parent.recv() + derived = Path(stage["database"]) + assert stage["pid"] == process.pid + assert stage["inserted_rows"] > 0 and derived.stat().st_size > 0 + assert derived.parent.parent == scratch_parent + assert process.is_alive() + assert hashes(root) == before + if action == "complete": + parent.send("continue") + else: + os.kill(process.pid, getattr(signal, action)) + process.join(WAIT_SECONDS) + assert not process.is_alive(), "CLI did not exit after release/signal" + if action == "complete": + assert process.exitcode == 0 + assert json.loads(stdout.read_text())["returned"] == 5 + assert stderr.read_text() == "" + assert not list(scratch_parent.iterdir()) + elif action == "SIGINT": + assert process.exitcode in (1, 130) + assert stdout.read_text() == "" + assert json.loads(stderr.read_text())["error"]["message"] == "Aborted!" + assert not list(scratch_parent.iterdir()) + else: + # Existing Python/OS behavior terminates before finally/TemporaryDirectory + # cleanup. Residue is deliberately observed, not deleted as evidence. + assert process.exitcode == -getattr(signal, action) + assert stdout.read_text() == stderr.read_text() == "" + assert derived.is_file() + assert list(scratch_parent.iterdir()) == [derived.parent] + assert hashes(root) == before + finally: + parent.close() + if process.is_alive(): + process.kill() + process.join(WAIT_SECONDS) + process.close()