diff --git a/.github/scripts/run_dataset_acceptance.py b/.github/scripts/run_dataset_acceptance.py index d1d8512..397a34f 100644 --- a/.github/scripts/run_dataset_acceptance.py +++ b/.github/scripts/run_dataset_acceptance.py @@ -21,7 +21,11 @@ TEST_NODE = f"tests/core/test_dataset_scale_acceptance.py::{TEST_NAME}" # Extend this tuple alongside new mandatory nightly gates. Never list future # files that have not landed: the runner rejects any missing configured node. -REQUIRED_TEST_NODES = (TEST_NODE,) +BOUNDARY_TEST_NODE = ( + "tests/core/test_dataset_scale_acceptance.py::" + "test_installed_default_export_short_final_shard_boundary" +) +REQUIRED_TEST_NODES = (TEST_NODE, BOUNDARY_TEST_NODE) class AcceptanceError(RuntimeError): diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 8088791..2bb0121 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -45,11 +45,29 @@ python .github/scripts/run_dataset_acceptance.py \ --wheel dist/.whl --output-dir .tmp/dataset-acceptance ``` -This tier initially exercises the existing 500,002-event `pip --target` acceptance -test. It does not claim a clean venv, two full 500,000-event shards, complete CLI -template coverage, live agents or upstream GUI interoperability. The ordinary -PR tier excludes `nightly` tests; the required nightly/manual runner must never -skip them. Manually dispatch the exact published branch/commit and +The required installed-wheel case creates a clean venv without system packages +and runs its actual `pt-snap` console script from isolated HOME/CWD. Its trusted, +test-owned input has exactly 1,000,000 events; the default exporter produces two +full 500,000-event shards. All installed catalog entries are classified: currently +11 dataset-supported templates succeed with installed CLI/API equivalence and +independent known-result checks, while `event_frames` and +`active_memory_frame_tree_at_event` return explicit unsupported-dataset errors. +Both report commands run: `peak-memory` succeeds and `memory-tree` preserves the +corresponding dataset rejection. The allocation/free-request/free-completion +boundary assertions remain covered, alongside static occupancy, offset pages, +terminal survivors and unchanged source/manifest/member hashes. A second required +nightly case separately imports 500,002 events and proves the original final +short shard has exactly two real events, with the crossing completed lifetime +and the final survivor. Its evidence is stored in its own subdirectory so the +two gates cannot overwrite each other's command logs. This adds one half-million +event import plus five query subprocesses to the nightly job. Existing +synthetic scale/default-budget regressions remain separate source-level tests. +`PT_SNAP_ACCEPTANCE_WHEELHOUSE` optionally supplies dependencies for an offline +clean-venv install. Timings are recorded without machine-speed thresholds; this +does not claim live agents, live hardware capture or upstream GUI interoperability. + +The ordinary PR tier excludes `nightly` tests; the required nightly/manual runner +must never skip them. Manually dispatch the exact published branch/commit and verify the run's head SHA and summary; a PR run alone is not nightly evidence. ## Release Workflow diff --git a/docs/en/dataset-acceptance.md b/docs/en/dataset-acceptance.md index 4ca8707..1ca5143 100644 --- a/docs/en/dataset-acceptance.md +++ b/docs/en/dataset-acceptance.md @@ -128,12 +128,17 @@ separate empty/static/dynamic/mixed regressions across both layouts and reducers gate does not claim all large ordered-frame or unlimited-output queries succeed. - A separate 10,000-event / 8-KiB-inline-stack dataset checks aggregate/small-page success and genuine 64-MiB full-output rejection without creating an 8-GB file. -- A trusted, test-owned snapshot with 500,002 alloc/free-request/free-completion - events goes through a non-editable installed `pt-snap import --format msinsight` - with **no capacity override**. The resulting 500,000 + 2 member layout and - cross-boundary lifecycle membership are checked before installed CLI/API peak, - page, terminal-candidate, point-event and report queries. Artifact hashes must - remain unchanged during queries. +- A trusted, test-owned snapshot with 1,000,000 alloc/free-request/free-completion + events goes through the clean-venv installed `pt-snap import --format msinsight` + with **no capacity override**, producing two full 500,000-event members. All + installed catalog templates are classified: 11 dataset-supported templates + have CLI/API parity and independent result checks; `event_frames` and + `active_memory_frame_tree_at_event` retain explicit dataset rejection. The + `peak-memory` report succeeds; `memory-tree` retains its dataset rejection. +- A second required installed-wheel case preserves the original 500,002-event + workload and its 500,000 + 2 member layout. It verifies the crossing completed + lifetime, final survivor, pages and point-event results. Both installed cases + require unchanged input, manifest and member hashes after queries. Build the wheel from the intended clean checkout using the [distribution instructions](../../README.md#building-distributions), then run: @@ -144,18 +149,23 @@ PT_SNAP_ACCEPTANCE_WHEEL="$(realpath dist/.whl)" \ pytest tests/core/test_dataset_scale_acceptance.py --junitxml=dataset-acceptance.xml ``` -Replace `` with the actual wheel name. The test installs that exact -wheel into its temporary directory using `--no-deps --no-index`; subprocesses -clear source `PYTHONPATH` and verify that `pt_snap_cli` resolves inside the -installation. Runtime dependencies are reused from the test interpreter: this -is not a fresh dependency-resolution test. Without `PT_SNAP_ACCEPTANCE_WHEEL`, -only the installed test is explicitly skipped; that skip is not E2E acceptance. -Use `pytest -m 'not slow'` for the regular suite. - -The installed test writes `acceptance.json` under its pytest temporary directory -and a `dataset_acceptance` JUnit property containing the wheel/source SHA-256, -package path, event counts, import duration and query assertions. Duration is -observational; there is no fixed timing assertion or RSS promise. +Replace `` with the actual wheel name. Each installed case creates +its own fresh venv without system packages, installs the exact wheel and runtime +dependencies, clears source `PYTHONPATH`, and checks that the package and +runtime dependencies resolve inside that venv. Dependency installation may use +the package index. Set `PT_SNAP_ACCEPTANCE_WHEELHOUSE` to a directory containing +compatible dependency wheels for an offline `--no-index --find-links` install. +Without `PT_SNAP_ACCEPTANCE_WHEEL`, both installed cases are explicitly skipped; +those skips are not E2E acceptance. The full PR tier uses `pytest -m 'not nightly'`; +the compatibility tier uses `pytest -m 'not slow and not nightly'`. + +The million-event installed case writes `acceptance.json` and a +`dataset_acceptance` JUnit property with wheel/source hashes, runtime identity, +event counts and command/query evidence. The short-final-shard case stores its +own evidence separately. `PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` selects a persistent +evidence directory; otherwise pytest temporary directories are used. The +required nightly runner rejects a missing or nonpassing case. Durations are +observational; these installed-wheel cases have no fixed timing or RSS promise. ## Evidence limits diff --git a/docs/zh/dataset-acceptance.md b/docs/zh/dataset-acceptance.md index 55b5a3a..d14041b 100644 --- a/docs/zh/dataset-acceptance.md +++ b/docs/zh/dataset-acceptance.md @@ -103,10 +103,15 @@ fallback 三条路径的这些契约。`top_n=0` 完整性另有空/static/dynam 大规模有序 frame 或无界输出查询均可成功。 - 独立 10,000-event / 8-KiB 内联栈数据集验证聚合/小页成功,以及完整输出触发真实 64-MiB 上限,无需创建 8-GB 数据文件。 -- 受信、测试自行构造的 500,002 个 alloc/free-request/free-completion 事件快照, - 经非 editable 安装的 `pt-snap import --format msinsight`,**不覆盖容量默认值**。 - 先核对 500,000 + 2 成员布局及跨界生命周期,再通过该安装包的 CLI/API 验证峰值、 - 分页、终片候选、定点事件及报告。查询前后产物 hash 必须不变。 +- 受信、测试自行构造的 1,000,000 个 alloc/free-request/free-completion 事件快照, + 经干净 venv 中安装的 `pt-snap import --format msinsight`,**不覆盖容量默认值**, + 生成两个完整的 500,000-event 成员。所有已安装目录模板均被分类:11 个支持数据集的 + 模板核对 CLI/API 一致性及独立结果;`event_frames` 和 + `active_memory_frame_tree_at_event` 保留明确的数据集拒绝。`peak-memory` 报告成功, + `memory-tree` 保留数据集拒绝。 +- 第二个必需的安装包用例保留原有 500,002-event 工作负载及 500,000 + 2 成员布局, + 验证跨界完成生命周期、最终存活分配、分页及定点事件结果。两个安装包用例均要求 + 查询前后输入、manifest 及成员 hash 不变。 按照[分发包构建说明](../../README_zh.md#构建分发包),从目标干净 checkout 构建 wheel: @@ -116,14 +121,19 @@ PT_SNAP_ACCEPTANCE_WHEEL="$(realpath dist/.whl)" \ pytest tests/core/test_dataset_scale_acceptance.py --junitxml=dataset-acceptance.xml ``` -将 `` 替换为实际名称。测试以 `--no-deps --no-index` 将该确切 wheel 安装到 -临时目录,子进程清除源码 `PYTHONPATH` 并核对 `pt_snap_cli` 来自该安装目录。运行依赖 -复用测试解释器环境,不代表全新依赖解析测试。未设置 `PT_SNAP_ACCEPTANCE_WHEEL` 时, -仅安装包用例明确 skip;skip 不算 E2E 通过。常规套件可用 `pytest -m 'not slow'`。 - -安装包用例在 pytest 临时目录写 `acceptance.json`,并记录 `dataset_acceptance` JUnit -属性,包括 wheel/source SHA-256、包路径、事件数量、导入耗时及查询断言。耗时只是 -单次观察,不设固定时限断言,也不承诺进程 RSS。 +将 `` 替换为实际名称。每个安装包用例创建独立且不含系统包的全新 venv, +安装该确切 wheel 及运行依赖,清除源码 `PYTHONPATH`,并核对产品包和运行依赖均来自 +该 venv。依赖安装可能访问包索引。可将 `PT_SNAP_ACCEPTANCE_WHEELHOUSE` 指向包含兼容 +依赖 wheel 的目录,以 `--no-index --find-links` 离线安装。 +未设置 `PT_SNAP_ACCEPTANCE_WHEEL` 时,两个安装包用例均明确 skip;skip 不算 E2E 通过。 +完整 PR 层使用 `pytest -m 'not nightly'`;兼容层使用 +`pytest -m 'not slow and not nightly'`。 + +百万事件安装包用例写入 `acceptance.json` 和 `dataset_acceptance` JUnit 属性,包含 +wheel/source hash、运行环境身份、事件数量及命令/查询证据。短终片用例单独保存其证据。 +`PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` 可指定持久证据目录,否则使用 pytest 临时目录。 +必需的 nightly runner 拒绝缺失或未通过的用例。耗时仅为观察记录;这些安装包用例 +不设固定时限,也不承诺进程 RSS。 ## 证据边界 diff --git a/tests/core/AGENTS.md b/tests/core/AGENTS.md index 33d02dd..4855dce 100644 --- a/tests/core/AGENTS.md +++ b/tests/core/AGENTS.md @@ -37,3 +37,38 @@ Implementation: [core services](../../src/pt_snap_cli/core/AGENTS.md). Run `pytest tests/core` or the owning file from the repository root. Follow the parent's fixture gate before import/split cases and resource ownership rules for all cached services. Cross-runtime changes additionally use `tests/snapshot/`. + +## Installed Dataset Scale Acceptance + +`test_dataset_scale_workload.py` checks the test-owned allocator workload and +arithmetic oracle on 100 events. `test_dataset_scale_acceptance.py` retains the +synthetic default-budget gates and has one opt-in wheel acceptance node: +`test_installed_trusted_producer_default_export_cli_and_api`. + +Set `PT_SNAP_ACCEPTANCE_WHEEL` to an absolute wheel path. The node creates a fresh +venv without system packages, installs the wheel and its dependencies, clears +source/focus/query-budget overrides, and executes its actual `pt-snap` console +script in isolated HOME/CWD. Optional `PT_SNAP_ACCEPTANCE_WHEELHOUSE` enables +fully offline dependency installation with `--no-index --find-links`. +`PT_SNAP_ACCEPTANCE_EVIDENCE_DIR` preserves subprocess commands, exit codes, +timings, stdout/stderr, versions, wheel/source/member hashes, and acceptance JSON. +The same acceptance is stored in the JUnit `dataset_acceptance` property (use +`-o junit_family=xunit1`). Timings are observations, not machine-speed thresholds. + +The real importer exports exactly 1,000,000 events using the unchanged 500,000 +default capacity. The complete installed catalog must be classified: currently +11 dataset-supported templates succeed with CLI/API result parity and independent +known-result assertions; `event_frames` and `active_memory_frame_tree_at_event` +are explicit dataset rejections. Both report commands run through the console: +`peak-memory` succeeds, while `memory-tree` preserves its documented unsupported +dataset contract. Source and every exported member remain unchanged. This is +trusted test-owned producer/replay acceptance, not live capture or upstream GUI +acceptance. Do not turn a missing wheel/skip into a successful nightly gate. + +The separate required nightly +`test_installed_default_export_short_final_shard_boundary` retains the original +500,002-event shape: 500,000 real events followed by exactly two. It checks exact +terminal member lifetimes and installed CLI/API completion/survival boundaries, +including unchanged hashes. Its `dataset_boundary_acceptance` JUnit property and +per-test evidence subdirectory preserve independent results without overwriting +the million-event gate's evidence. Both cases are mandatory in the runner. diff --git a/tests/core/test_dataset_scale_acceptance.py b/tests/core/test_dataset_scale_acceptance.py index 8fb8a05..415b8ed 100644 --- a/tests/core/test_dataset_scale_acceptance.py +++ b/tests/core/test_dataset_scale_acceptance.py @@ -6,12 +6,9 @@ run it; this is not live PyTorch/NPU capture or upstream msinsight GUI acceptance. """ -import hashlib import json import os import sqlite3 -import subprocess -import sys import time from contextlib import closing from pathlib import Path @@ -24,7 +21,13 @@ from pt_snap_cli.core.errors import QueryExecutionError from pt_snap_cli.core.msinsight_export import DEFAULT_CAPACITY from tests.core.dataset_lifecycle_fixtures import SCHEMA -from tests.snapshot.test_sharded_replay import block, event, segment, write_source +from tests.core.wheel_scale_acceptance import ( + API_QUERIES, + WheelRuntime, + query_cases, + sha256, + write_scale_source, +) from tests.test_dataset_focus import hashes pytestmark = pytest.mark.slow @@ -182,98 +185,174 @@ def test_long_stack_aggregate_and_small_page_succeed_but_full_output_is_byte_bou assert hashes(root) == before -def run(command, root, env): - result = subprocess.run(command, cwd=root, env=env, capture_output=True, text=True, check=False) - assert result.returncode == 0, f"{command}\n{result.stdout}\n{result.stderr}" - return result - - @pytest.fixture -def installed_runtime(tmp_path): +def installed_runtime(tmp_path, request): wheel_value = os.environ.get("PT_SNAP_ACCEPTANCE_WHEEL") if not wheel_value: - pytest.skip("Set PT_SNAP_ACCEPTANCE_WHEEL to test a non-editable installed artifact") + pytest.skip("Set PT_SNAP_ACCEPTANCE_WHEEL to test a clean-venv installed artifact") wheel = Path(wheel_value).resolve() assert wheel.is_file() and wheel.suffix == ".whl" - target = tmp_path / "installed" - env = os.environ.copy() - env.pop("PYTHONHOME", None) - env.pop("PT_SNAP_DB_PATH", None) - env.pop("PT_SNAP_QUERY_TIMEOUT", None) - env["PYTHONPATH"] = str(target) - run( - [ - sys.executable, - "-m", - "pip", - "install", - "--no-deps", - "--no-index", - "--target", - str(target), - str(wheel), - ], - tmp_path, - env, + evidence_dir = Path(os.environ.get("PT_SNAP_ACCEPTANCE_EVIDENCE_DIR", tmp_path / "evidence")) + if request.node.name != "test_installed_trusted_producer_default_export_cli_and_api": + evidence_dir /= request.node.name + wheelhouse = os.environ.get("PT_SNAP_ACCEPTANCE_WHEELHOUSE") + return WheelRuntime( + tmp_path, wheel, evidence_dir.resolve(), Path(wheelhouse).resolve() if wheelhouse else None ) - probe = run( - [sys.executable, "-c", "import pt_snap_cli; print(pt_snap_cli.__file__)"], tmp_path, env - ) - module = Path(probe.stdout.strip()).resolve() - assert module.is_relative_to(target) - cli = target / "bin" / "pt-snap" - assert cli.is_file() - return cli, env, module, hashlib.sha256(wheel.read_bytes()).hexdigest() + + +def _assert_scale_oracle(queries, count, boundary): + """Independent arithmetic/geometry, not the same API used as its own oracle.""" + for name, value in queries.items(): + assert value["total_is_exact"], name + assert value["returned"] == len(value["rows"]), name + assert value["scope"]["boundary_events_included"] is False, name + scope = value["scope"] + if scope["kind"] == "event": + assert scope["source_coverage"]["range_complete"], name + else: + assert scope["range_complete"], name + peak = queries["memory_peak"]["rows"][0] + assert peak == { + "peak_allocated": 176, + "peak_allocated_event_id": 3, + "peak_active": 176, + "peak_active_event_id": 3, + "peak_reserved": 4096, + "peak_reserved_event_id": 0, + } + gap = queries["allocator_gap"]["rows"][0] + assert gap["reserved_active_gap_at_active_peak"] == 3920 + assert gap["active_at_reserved_peak"] == 112 + assert gap["all_peaks_same_event"] == 0 + for name in ("event", "allocation"): + result = queries[name] + assert [row["id"] for row in result["rows"]] == list(range(boundary - 2, boundary + 3)) + assert result["total"] == count + assert result["has_more"] and result["truncated"] + assert [row["active"] for row in result["rows"]] == [176, 176, 160, 176, 176] + blocks = queries["block"] + assert blocks["total"] == (count - 1) // 3 + 5 + assert [row["id"] for row in blocks["rows"]] == [0, 1, 2, 3, 6] + assert blocks["has_more"] and blocks["truncated"] + crossing = queries["crossing_block"]["rows"] + assert len(crossing) == 1 + assert crossing[0]["allocEventId"] == boundary - 2 + assert crossing[0]["freeEventId"] == boundary + assert crossing[0]["allocation_source"]["slice_index"] == 0 + assert crossing[0]["free_source"]["slice_index"] == 1 + assert crossing[0]["free_source"]["event"]["action"] == 6 + assert queries["freed_block_lifetime"]["rows"] == [ + { + "lifetime_events": "<1k", + "block_count": (count - 4) // 3, + "size_bytes": 16 * ((count - 4) // 3), + } + ] + leaks = queries["leak_detection"] + assert leaks["total"] == 4 and not leaks["has_more"] and not leaks["truncated"] + assert {row["allocEventId"] for row in leaks["rows"]} == {0, 1, 2, count - 1} + assert all(row["terminal_survivor"] for row in leaks["rows"]) + grouped = queries["callstack_analysis"]["rows"] + assert len(grouped) == 6 + assert sum(row["alloc_count"] for row in grouped) == count + assert sum(row["total_size"] for row in grouped) == 16 * count + 64 + expected_stacks = { + "retain_64": (1, 64), + "retain_32": (1, 32), + "retain_16": (1, 16), + "alloc": ((count - 1) // 3, 16 * ((count - 1) // 3)), + "free_requested": ((count - 4) // 3, 16 * ((count - 4) // 3)), + "free_completed": ((count - 4) // 3, 16 * ((count - 4) // 3)), + } + assert { + row["callstack"].rsplit(" ", 1)[1]: (row["alloc_count"], row["total_size"]) + for row in grouped + } == expected_stacks + for name in ("block", "crossing_block", "leak_detection", "freed_block_lifetime"): + coverage = queries[name]["scope"]["source_coverage"] + assert coverage["lifecycles"] == (count - 1) // 3 + 5 + assert coverage["allocation_events_resolved"] == (count - 1) // 3 + 3 + assert coverage["free_events_resolved"] == (count - 4) // 3 + assert coverage["unproved_identities"] == 0 and coverage["allocation_source_complete"] + assert queries["preexisting_live"]["rows"] == [{"block_count": 0, "size_bytes": 0}] + before = queries["active_before"]["rows"] + after = queries["active_after"]["rows"] + assert len(before) == 6 and sum(row["size"] for row in before) == 176 + assert len(after) == 5 and sum(row["size"] for row in after) == 160 + assert [row["allocEventId"] for row in before if row["allocEventId"] >= 0] == [ + 0, + 1, + 2, + boundary - 2, + ] + assert [row["allocEventId"] for row in after if row["allocEventId"] >= 0] == [0, 1, 2] + assert sum(row["size"] for row in before if row["category"] == "static") == 48 + page = queries["active_static_offset"] + assert page["rows"] == before[1:4] and page["total"] == 6 + assert page["has_more"] and page["truncated"] + dynamic = queries["active_dynamic_offset"] + assert [row["id"] for row in dynamic["rows"]] == [1, 2] and dynamic["total"] == 4 + assert dynamic["has_more"] and dynamic["truncated"] + groups = queries["active_groups"]["rows"] + assert sum(row["size_bytes"] for row in groups) == 176 + assert sum(row["block_count"] for row in groups) == 6 + assert next(row for row in groups if row["category"] == "static")["size_bytes"] == 48 @pytest.mark.nightly def test_installed_trusted_producer_default_export_cli_and_api( tmp_path, installed_runtime, record_property ): - cli, env, module, wheel_hash = installed_runtime - # Original, inspectable test-owned data. No committed or external pickle is loaded. - # 500000 ends after free_requested; free_completed is in the next shard. - count = DEFAULT_CAPACITY + 2 - traces = [event(("alloc", "free_requested", "free_completed")[i % 3]) for i in range(count)] - data = {"segments": [segment(0, 64, [block(1000, 16, 16)])], "device_traces": [traces]} - source = write_source(tmp_path, data) - # The installed importer owns its own full input load; do not retain a second - # 500002-event graph in the test process while it runs. - del traces, data - source_hash = hashlib.sha256(source.read_bytes()).hexdigest() - started = time.perf_counter() - imported = json.loads( - run( - [ - sys.executable, - str(cli), - "import", - str(source), - "--format", - "msinsight", - "--output-dir", - str(tmp_path / "exported"), - "--no-focus", - "--json", - ], - tmp_path, - env, - ).stdout + runtime = installed_runtime + assert DEFAULT_CAPACITY == 500_000 + count, boundary = 2 * DEFAULT_CAPACITY, DEFAULT_CAPACITY + source = write_scale_source(runtime.cwd / "input.pkl", count) + source_hash = sha256(source) + (runtime.evidence_dir / "inputs.json").write_text( + json.dumps( + { + "identity": runtime.identity, + "wheel_sha256": runtime.wheel_hash, + "source_sha256": source_hash, + "events": count, + "events_per_slice": boundary, + }, + indent=2, + ) + ) + imported = runtime.cli_json( + "import", + "import", + source, + "--format", + "msinsight", + "--output-dir", + runtime.cwd / "exported", + "--no-focus", + "--json", ) - import_seconds = time.perf_counter() - started root = Path(imported["db_path"]) manifest = json.loads((root / "manifest.json").read_text()) - assert manifest["eventsPerSlice"] == DEFAULT_CAPACITY + assert manifest["eventsPerSlice"] == boundary assert manifest["devices"]["0"]["eventCount"] == count slices = manifest["devices"]["0"]["slices"] assert [(s["startEventId"], s["endEventId"]) for s in slices] == [ (0, 499_999), - (500_000, 500_001), + (500_000, 999_999), ] before = hashes(root) - # Inspect actual producer membership before running any query. + (runtime.evidence_dir / "members-before.json").write_text(json.dumps(before, indent=2)) + # Preserve the old 500002-event boundary regression in the full-size workload: + # first-shard pending-free allocation completes only in the second shard. for item in slices: - with closing(sqlite3.connect(root / item["file"])) as conn: + with closing( + sqlite3.connect((root / item["file"]).as_uri() + "?mode=ro", uri=True) + ) as conn: + assert ( + conn.execute("SELECT COUNT(*) FROM trace_entry_0 WHERE id>=0").fetchone()[0] + == boundary + ) assert ( conn.execute( "SELECT COUNT(*) FROM block_0 WHERE allocEventId>?", (item["endEventId"],) @@ -287,64 +366,208 @@ def test_installed_trusted_producer_default_export_cli_and_api( ).fetchone()[0] == 0 ) - with closing(sqlite3.connect(root / slices[1]["file"])) as conn: - assert conn.execute( - "SELECT allocEventId,freeEventId FROM block_0 ORDER BY id" - ).fetchall() == [(499_998, 500_000), (500_001, -1)] - peak = json.loads( - run( - [ - sys.executable, - str(cli), - "query", - str(root), - "--template-use", - "memory_peak", - "--json", - ], - tmp_path, - env, - ).stdout + assert conn.execute( + "SELECT allocEventId,freeEventId FROM block_0 WHERE id=499998" + ).fetchall() == [(499_998, 500_000)] + catalog = runtime.cli_json("capabilities", "capabilities", "--json") + cases = query_cases(boundary) + by_name = {entry["name"]: entry for entry in catalog["templates"]} + assert {case["template"] for case in cases} == set(by_name) + supported = {name for name, entry in by_name.items() if entry["dataset_support"]["supported"]} + unsupported = set(by_name) - supported + assert unsupported == {"event_frames", "active_memory_frame_tree_at_event"} + cases_file = runtime.cwd / "cases.json" + cases_file.write_text(json.dumps(cases)) + api = runtime.python_json("api", API_QUERIES, root, cases_file, boundary - 1) + assert catalog["ok"] is True and catalog["schema_version"] == 1 + assert {key: catalog[key] for key in api["catalog"]} == api["catalog"] + assert set(api["errors"]) == unsupported + cli_queries = {} + for case in cases: + success = case["template"] in supported + result = runtime.cli_json( + f"query-{case['name']}", + "query", + root, + "--template-use", + case["template"], + "--params", + json.dumps(case["params"]), + "--exact-total", + "--json", + expected=0 if success else 1, + ) + if success: + assert {key: result[key] for key in api["queries"][case["name"]]} == api["queries"][ + case["name"] + ] + cli_queries[case["name"]] = result + else: + assert result["error"]["code"] == api["errors"][case["name"]]["code"] == "QUERY_FAILED" + assert "Unsupported dataset-global" in result["error"]["message"] + _assert_scale_oracle(cli_queries, count, boundary) + peak = runtime.cli_json("report-peak", "report", "peak-memory", root, "--limit", "-1", "--json") + assert peak == api["peak_report"] + assert peak["event_id"] == 3 and peak["included_bytes"] == 176 + assert peak["active_bytes_at_event"] == 176 and peak["coverage_percent"] == 100 + assert peak["total_is_exact"] and not peak["truncated"] + tree = runtime.cli_json( + "report-tree", + "report", + "memory-tree", + root, + "--event-id", + boundary - 1, + "--json", + expected=1, ) - assert peak["rows"][0]["peak_active"] == 16 and peak["rows"][0]["peak_active_event_id"] == 0 - api_script = """ -import json, sys -from pt_snap_cli.api import SnapshotAnalyzer -from pt_snap_cli.core.report_service import ReportService -with SnapshotAnalyzer(sys.argv[1]) as analyzer: - rows = analyzer.execute_query("event", {"min_id": 0, "limit": 5}, exact_total=True) - assert rows["returned"] == 5 and rows["total"] == 500002 - peak = analyzer.execute_query("memory_peak")["rows"][0] - assert peak["peak_active"] == 16 and peak["peak_active_event_id"] == 0 - before = analyzer.execute_query("active_blocks_at_event", {"event_id": 499999}) - after = analyzer.execute_query("active_blocks_at_event", {"event_id": 500000}) - assert [r["allocEventId"] for r in before["rows"]] == [499998] - assert after["rows"] == [] - live = analyzer.execute_query("leak_detection", {"limit": 5}) - assert [r["allocEventId"] for r in live["rows"]] == [500001] -service = ReportService() -try: - report = service.peak_memory_report(sys.argv[1], limit=-1) - assert report.event_id == 0 and report.included_bytes == 16 - assert report.active_bytes_at_event == 16 and report.coverage_percent == 100 -finally: - service.close() -print(json.dumps({"query_total": rows["total"], "peak": peak, "report_bytes": report.included_bytes})) -""" - result = json.loads(run([sys.executable, "-c", api_script, str(root)], tmp_path, env).stdout) - assert result["query_total"] == count + assert tree["error"]["code"] == api["tree_report"]["code"] == "QUERY_FAILED" + assert "Unsupported dataset-global" in tree["error"]["message"] assert hashes(root) == before - assert hashlib.sha256(source.read_bytes()).hexdigest() == source_hash + assert sha256(source) == source_hash + assert not (runtime.cwd / ".pt-snap").exists() + assert not (runtime.home / ".config" / "pt-snap-cli").exists() evidence = { - "scope": "test-owned snapshot producer -> installed pt-snap replay/export -> installed CLI/API; no live collector or upstream GUI", - "module": str(module), - "wheel_sha256": wheel_hash, + "scope": "test-owned snapshot -> clean-venv installed console import/export, query/report and API; no live collector or upstream GUI", + "identity": runtime.identity, + "wheel_sha256": runtime.wheel_hash, "source_sha256": source_hash, + "members_sha256": before, "events": count, - "events_per_slice": DEFAULT_CAPACITY, + "events_per_slice": boundary, "slices": 2, - "import_seconds": import_seconds, - "result": result, + "templates_supported": sorted(supported), + "templates_expected_rejections": sorted(unsupported), + "reports": {"peak-memory": "passed", "memory-tree": "expected dataset rejection"}, + "commands": runtime.commands, + "read_only_verified": True, } record_property("dataset_acceptance", json.dumps(evidence)) - (tmp_path / "acceptance.json").write_text(json.dumps(evidence, indent=2)) + (runtime.evidence_dir / "acceptance.json").write_text(json.dumps(evidence, indent=2)) + + +@pytest.mark.nightly +def test_installed_default_export_short_final_shard_boundary(installed_runtime, record_property): + """Keep the original 500002-event final-two-row shard producer regression.""" + runtime = installed_runtime + assert DEFAULT_CAPACITY == 500_000 + count, boundary = DEFAULT_CAPACITY + 2, DEFAULT_CAPACITY + source = write_scale_source(runtime.cwd / "input.pkl", count) + source_hash = sha256(source) + imported = runtime.cli_json( + "import", + "import", + source, + "--format", + "msinsight", + "--output-dir", + runtime.cwd / "exported", + "--no-focus", + "--json", + ) + root = Path(imported["db_path"]) + manifest = json.loads((root / "manifest.json").read_text()) + assert manifest["eventsPerSlice"] == boundary + device = manifest["devices"]["0"] + assert device["eventCount"] == count and device["sliceCount"] == 2 + assert device["readySlices"] == [0, 1] + slices = device["slices"] + assert [(item["startEventId"], item["endEventId"]) for item in slices] == [ + (0, 499_999), + (500_000, 500_001), + ] + before = hashes(root) + for item, expected_count in zip(slices, (500_000, 2), strict=True): + with closing( + sqlite3.connect((root / item["file"]).as_uri() + "?mode=ro", uri=True) + ) as conn: + assert conn.execute( + "SELECT COUNT(*),MIN(id),MAX(id) FROM trace_entry_0 WHERE id>=0" + ).fetchone() == (expected_count, item["startEventId"], item["endEventId"]) + assert conn.execute( + "SELECT COUNT(*) FROM block_0 WHERE allocEventId>? OR (freeEventId>=0 AND freeEventId=0 ORDER BY id" + ).fetchall() == [(500_000, 6), (500_001, 4)] + assert conn.execute( + "SELECT allocEventId,freeEventId FROM block_0 ORDER BY id" + ).fetchall() == [ + (-1, -1), + (-1, -1), + (0, -1), + (1, -1), + (2, -1), + (499_998, 500_000), + (500_001, -1), + ] + names = {"event", "crossing_block", "active_before", "active_after", "leak_detection"} + cases = [case for case in query_cases(boundary) if case["name"] in names] + cases_file = runtime.cwd / "cases.json" + cases_file.write_text(json.dumps(cases)) + api = runtime.python_json("api", API_QUERIES, root, cases_file, boundary - 1) + assert api["errors"] == {} + results = {} + for case in cases: + result = runtime.cli_json( + f"query-{case['name']}", + "query", + root, + "--template-use", + case["template"], + "--params", + json.dumps(case["params"]), + "--exact-total", + "--json", + ) + assert {key: result[key] for key in api["queries"][case["name"]]} == api["queries"][ + case["name"] + ] + assert result["total_is_exact"] and result["returned"] == len(result["rows"]) + results[case["name"]] = result + tail = results["event"] + assert tail["total"] == count and tail["returned"] == 4 + assert not tail["has_more"] and tail["truncated"] + assert [(row["id"], row["action"], row["active"]) for row in tail["rows"]] == [ + (499_998, 4, 176), + (499_999, 5, 176), + (500_000, 6, 160), + (500_001, 4, 176), + ] + crossing = results["crossing_block"]["rows"] + assert len(crossing) == 1 + assert crossing[0]["allocEventId"] == 499_998 and crossing[0]["freeEventId"] == 500_000 + assert crossing[0]["allocation_source"]["slice_index"] == 0 + assert crossing[0]["free_source"]["slice_index"] == 1 + assert crossing[0]["free_source"]["event"]["id"] == 500_000 + assert crossing[0]["free_source"]["event"]["action"] == 6 + before_rows, after_rows = results["active_before"]["rows"], results["active_after"]["rows"] + assert [row["allocEventId"] for row in before_rows if row["allocEventId"] >= 0] == [ + 0, + 1, + 2, + 499_998, + ] + assert [row["allocEventId"] for row in after_rows if row["allocEventId"] >= 0] == [0, 1, 2] + assert sum(row["size"] for row in before_rows) == 176 + assert sum(row["size"] for row in after_rows) == 160 + leak = results["leak_detection"] + assert leak["total"] == 4 and not leak["has_more"] and not leak["truncated"] + assert {row["allocEventId"] for row in leak["rows"]} == {0, 1, 2, 500_001} + assert hashes(root) == before and sha256(source) == source_hash + evidence = { + "scope": "clean-wheel producer regression: one full default shard and a final two-event shard", + "wheel_sha256": runtime.wheel_hash, + "source_sha256": source_hash, + "members_sha256": before, + "events": count, + "events_per_slice": boundary, + "slice_event_counts": [500_000, 2], + "identity": runtime.identity, + "commands": runtime.commands, + "read_only_verified": True, + } + record_property("dataset_boundary_acceptance", json.dumps(evidence)) + (runtime.evidence_dir / "acceptance.json").write_text(json.dumps(evidence, indent=2)) diff --git a/tests/core/test_dataset_scale_workload.py b/tests/core/test_dataset_scale_workload.py new file mode 100644 index 0000000..dba0232 --- /dev/null +++ b/tests/core/test_dataset_scale_workload.py @@ -0,0 +1,66 @@ +"""Fast independent checks for the nightly clean-wheel scale workload/oracle.""" + +import json +from pathlib import Path + +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.import_service import ImportService +from pt_snap_cli.core.models import ImportOptions +from pt_snap_cli.query import registry +from pt_snap_cli.snapshot.representation import load_snapshot_representation +from pt_snap_cli.snapshot.simulate import SimulateDeviceSnapshot +from tests.core.test_dataset_scale_acceptance import _assert_scale_oracle +from tests.core.wheel_scale_acceptance import query_cases, sha256, write_scale_source +from tests.snapshot.helpers import assert_valid_snapshot + + +def test_scale_workload_is_replay_valid_at_every_event(tmp_path): + source = write_scale_source(tmp_path / "input.pkl", 100) + before = sha256(source) + simulator = SimulateDeviceSnapshot(load_snapshot_representation(source), 0) + for remaining in range(100, -1, -1): + assert simulator.replay_until(remaining) + assert_valid_snapshot(simulator.device_snapshot) + expected = ( + (112, 144, 160)[remaining - 1] + if 1 <= remaining <= 3 + else (48 if remaining == 0 else 160 + (16 if remaining % 3 != 0 else 0)) + ) + assert simulator.device_snapshot.total_activated == expected + assert sha256(source) == before + + +def test_small_scale_uses_same_independent_oracle_and_catalog(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(Path, "home", lambda: tmp_path) + for name in ("PT_SNAP_DB_PATH", "PT_SNAP_QUERY_TIMEOUT"): + monkeypatch.delenv(name, raising=False) + # Other contract tests may register custom queries. Give this source-level + # smoke its own packaged catalog; the wheel subprocess always starts clean. + monkeypatch.setattr(registry._registry, "_queries", {}) + monkeypatch.setattr(registry._registry, "_factories", {}) + monkeypatch.setattr(registry, "_packaged", {}) + registry._load_all_templates() + source = write_scale_source(tmp_path / "input.pkl", 100) + imported = ImportService().import_snapshot( + ImportOptions( + source, + output_dir=tmp_path / "exported", + format="msinsight", + events_per_slice=50, + set_focus=False, + ) + ) + manifest = json.loads((imported.db_path / "manifest.json").read_text()) + assert manifest["devices"]["0"]["eventCount"] == 100 + with SnapshotAnalyzer(imported.db_path) as analyzer: + catalog = {entry["name"]: entry for entry in analyzer.list_capabilities()["templates"]} + cases = query_cases(50) + assert {case["template"] for case in cases} == set(catalog) + values = { + case["name"]: analyzer.execute_query(case["template"], case["params"], exact_total=True) + for case in cases + if catalog[case["template"]]["dataset_support"]["supported"] + } + _assert_scale_oracle(values, 100, 50) diff --git a/tests/core/wheel_scale_acceptance.py b/tests/core/wheel_scale_acceptance.py new file mode 100644 index 0000000..6cdd38c --- /dev/null +++ b/tests/core/wheel_scale_acceptance.py @@ -0,0 +1,314 @@ +"""Clean-wheel subprocess harness and an original, inspectable scale workload. + +Only the fixture generator runs in the checkout. Product imports in probes, +queries, and reports run with the installed venv interpreter and isolated CWD. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import pickle +import subprocess +import time +import venv +from pathlib import Path + +PROBE = """ +import importlib.metadata as metadata +import json, platform, site, sqlite3, sys +from dataclasses import asdict +from pathlib import Path +import pt_snap_cli, typer, yaml, jinja2 +from pt_snap_cli.core.query_limits import QueryLimits +print(json.dumps({ + "python": sys.version, "executable": sys.executable, + "prefix": sys.prefix, "base_prefix": sys.base_prefix, + "platform": platform.platform(), "sqlite": sqlite3.sqlite_version, + "user_site_enabled": site.ENABLE_USER_SITE, "sys_path": sys.path, + "modules": {m.__name__: str(Path(m.__file__).resolve()) + for m in (pt_snap_cli, typer, yaml, jinja2)}, + "versions": {name: metadata.version(name) + for name in ("pt-snap-cli", "typer", "pyyaml", "jinja2")}, + "entry_point": next(e.value for e in metadata.distribution("pt-snap-cli").entry_points + if e.group == "console_scripts" and e.name == "pt-snap"), + "query_limits": asdict(QueryLimits.resolve()), +})) +""" + +API_QUERIES = """ +import json, sys +from dataclasses import asdict +from pt_snap_cli.api import SnapshotAnalyzer +from pt_snap_cli.core.error_codes import classify_error +from pt_snap_cli.core.report_service import ReportService +root, cases_file, event = sys.argv[1], sys.argv[2], int(sys.argv[3]) +results, errors = {}, {} +with SnapshotAnalyzer(root) as analyzer: + catalog = analyzer.list_capabilities() + with open(cases_file) as stream: + cases = json.load(stream) + for case in cases: + try: + results[case["name"]] = analyzer.execute_query( + case["template"], case["params"], exact_total=True) + except Exception as exc: + errors[case["name"]] = {"type": type(exc).__name__, + "code": classify_error(exc)[0], "message": str(exc)} +service = ReportService() +try: + peak = asdict(service.peak_memory_report(root, limit=-1)) + try: + tree = {"event_id": event, **asdict(service.memory_tree_report(event, root))} + except Exception as exc: + tree = {"type": type(exc).__name__, "code": classify_error(exc)[0], + "message": str(exc)} +finally: + service.close() +print(json.dumps({"catalog": catalog, "queries": results, "errors": errors, + "peak_report": peak, "tree_report": tree})) +""" + + +def sha256(path): + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +class WheelRuntime: + """True installed console entry point, with failure-safe command evidence.""" + + def __init__(self, root, wheel, evidence_dir, wheelhouse=None): + self.root = root + self.home = root / "home" + self.cwd = root / "project" + self.venv = root / "venv" + self.evidence_dir = evidence_dir + self.evidence_dir.mkdir(parents=True, exist_ok=True) + self.home.mkdir() + self.cwd.mkdir() + self.commands = [] + venv.EnvBuilder(with_pip=True, system_site_packages=False).create(self.venv) + bin_dir = self.venv / ("Scripts" if os.name == "nt" else "bin") + self.python = bin_dir / ("python.exe" if os.name == "nt" else "python") + self.cli = bin_dir / ("pt-snap.exe" if os.name == "nt" else "pt-snap") + self.env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("PYTHON", "PT_SNAP_")) + and key not in {"CODEX_HOME", "CLAUDE_CONFIG_DIR", "VIRTUAL_ENV"} + } + self.env.update( + HOME=str(self.home), + USERPROFILE=str(self.home), + XDG_CONFIG_HOME=str(self.home / ".config"), + XDG_CACHE_HOME=str(self.home / ".cache"), + VIRTUAL_ENV=str(self.venv), + PATH=f"{bin_dir}{os.pathsep}{self.env.get('PATH', '')}", + PWD=str(self.cwd), + PYTHONNOUSERSITE="1", + PIP_DISABLE_PIP_VERSION_CHECK="1", + NO_COLOR="1", + ) + install = [str(self.python), "-m", "pip", "install"] + if wheelhouse is not None: + install += ["--no-index", "--find-links", str(wheelhouse)] + self.run("install", [*install, str(wheel)]) + assert self.cli.is_file() + self.identity = self.python_json("identity", PROBE) + assert self.identity["prefix"] != self.identity["base_prefix"] + assert Path(self.identity["prefix"]) == self.venv + assert Path(self.identity["executable"]) == self.python + assert self.identity["user_site_enabled"] is False + for module in self.identity["modules"].values(): + assert Path(module).is_relative_to(self.venv), module + source_root = Path(__file__).resolve().parents[2] + assert all( + not Path(path).resolve().is_relative_to(source_root) + for path in self.identity["sys_path"] + ) + assert self.identity["entry_point"] == "pt_snap_cli.cli:_safe_call" + assert self.identity["query_limits"] == { + "max_work_rows": 100_000, + "max_work_bytes": 64 * 1024 * 1024, + "max_scratch_db_bytes": 512 * 1024 * 1024, + } + self.wheel_hash = sha256(wheel) + + def run(self, label, command, *, expected=0): + started = time.perf_counter() + result = subprocess.run( + command, cwd=self.cwd, env=self.env, capture_output=True, text=True, check=False + ) + self.commands.append( + { + "label": label, + "command": [str(item) for item in command], + "cwd": str(self.cwd), + "home": str(self.home), + "returncode": result.returncode, + "seconds": time.perf_counter() - started, + } + ) + (self.evidence_dir / f"{label}.stdout").write_text(result.stdout, encoding="utf-8") + (self.evidence_dir / f"{label}.stderr").write_text(result.stderr, encoding="utf-8") + (self.evidence_dir / "commands.json").write_text( + json.dumps(self.commands, indent=2), encoding="utf-8" + ) + assert ( + result.returncode == expected + ), f"{command}\nexit={result.returncode}\n{result.stdout}\n{result.stderr}" + return result + + def cli_json(self, label, *arguments, expected=0): + result = self.run(label, [str(self.cli), *map(str, arguments)], expected=expected) + if expected == 0: + assert not result.stderr, result.stderr + return json.loads(result.stdout) + assert not result.stdout, result.stdout + return json.loads(result.stderr) + + def python_json(self, label, script, *arguments): + result = self.run(label, [str(self.python), "-I", "-c", script, *map(str, arguments)]) + assert not result.stderr, result.stderr + return json.loads(result.stdout) + + +def write_scale_source(path, count): + """Replay-valid allocator trace: static occupancy, survivors, repeated reuse. + + Three retained allocations precede 16-byte alloc/request/complete cycles. + count % 3 == 1 leaves the last cycle live. At the default 500000 boundary, + alloc 499998 requests free at 499999 and completes in shard two at 500000. + The final segment is contiguous, including the trailing inactive space. + No external or committed pickle is loaded to construct this test-owned file. + """ + assert count >= 10 and count % 3 == 1 + retained = [(4160, 64), (4224, 32), (4256, 16)] + + def trace(action, address, size, name): + return { + "action": action, + "addr": address, + "size": size, + "stream": 0, + "frames": [ + {"filename": "scale.py", "line": 10, "name": name}, + {"filename": "train.py", "line": 20, "name": "step"}, + ], + } + + events = [trace("alloc", addr, size, f"retain_{size}") for addr, size in retained] + cycle = [ + trace(action, 4096, 16, action) for action in ("alloc", "free_requested", "free_completed") + ] + events.extend(dict(cycle[index % 3]) for index in range(3, count)) + sizes = [16, 32, 16, 64, 32, 16, 4096 - 176] + blocks, address = [], 4096 + for index, size in enumerate(sizes): + blocks.append( + { + "address": address, + "size": size, + "requested_size": size if index < 6 else 0, + "state": "active_allocated" if index < 6 else "inactive", + "frames": [], + } + ) + address += size + data = { + "segments": [ + { + "address": 4096, + "total_size": 4096, + "stream": 0, + "segment_type": "small", + "allocated_size": 176, + "active_size": 176, + "device": 0, + "is_expandable": False, + "frames": [], + "blocks": blocks, + } + ], + "device_traces": [events], + } + with path.open("wb") as stream: + pickle.dump(data, stream, protocol=4) + return path + + +def query_cases(boundary): + """Catalog coverage plus cross-shard identity and real pagination cases.""" + return [ + {"name": name, "template": template, "params": params} + for name, template, params in [ + ("memory_peak", "memory_peak", {}), + ("allocator_gap", "allocator_gap", {}), + ( + "event", + "event", + {"min_id": 0, "limit": 5, "offset": boundary - 2, "order_dir": "ASC"}, + ), + ( + "allocation", + "allocation", + {"min_id": 0, "limit": 5, "offset": boundary - 2, "order_dir": "ASC"}, + ), + ("block", "block", {"limit": 5, "offset": 2, "order_by": "id", "order_dir": "ASC"}), + ("crossing_block", "block", {"id": boundary - 2}), + ("leak_detection", "leak_detection", {"limit": 5}), + ("freed_block_lifetime", "freed_block_lifetime", {}), + ("callstack_analysis", "callstack_analysis", {"min_count": 1}), + ("preexisting_live", "preexisting_live", {"event_id": boundary - 1}), + ( + "active_before", + "active_blocks_at_event", + {"event_id": boundary - 1, "order_by": "id", "order_dir": "ASC"}, + ), + ( + "active_after", + "active_blocks_at_event", + {"event_id": boundary, "order_by": "id", "order_dir": "ASC"}, + ), + ( + "active_static_offset", + "active_blocks_at_event", + { + "event_id": boundary - 1, + "include_static": True, + "order_by": "id", + "order_dir": "ASC", + "offset": 1, + "limit": 3, + }, + ), + ( + "active_dynamic_offset", + "active_blocks_at_event", + { + "event_id": boundary - 1, + "include_static": False, + "order_by": "id", + "order_dir": "ASC", + "offset": 1, + "limit": 2, + }, + ), + ( + "active_groups", + "active_memory_callstack_at_event", + {"event_id": boundary - 1, "top_n": -1}, + ), + ("event_frames", "event_frames", {"event_id": boundary - 1}), + ( + "active_memory_frame_tree_at_event", + "active_memory_frame_tree_at_event", + {"event_id": boundary - 1}, + ), + ] + ] diff --git a/tests/test_dataset_acceptance_runner.py b/tests/test_dataset_acceptance_runner.py index 037e488..b409185 100644 --- a/tests/test_dataset_acceptance_runner.py +++ b/tests/test_dataset_acceptance_runner.py @@ -11,6 +11,7 @@ import pytest RUNNER = Path(".github/scripts/run_dataset_acceptance.py") +BOUNDARY_NAME = "test_installed_default_export_short_final_shard_boundary" @pytest.fixture @@ -22,7 +23,7 @@ def runner(): return module -def write_report(path, name, evidence, outcome=None): +def write_report(path, name, evidence, outcome=None, *, include_boundary=True): suite = ET.Element("testsuite") case = ET.SubElement( suite, @@ -36,12 +37,20 @@ def write_report(path, name, evidence, outcome=None): if evidence is not None: properties = ET.SubElement(case, "properties") ET.SubElement(properties, "property", name="dataset_acceptance", value=json.dumps(evidence)) + if include_boundary: + ET.SubElement( + suite, + "testcase", + name=BOUNDARY_NAME, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + ) ET.ElementTree(suite).write(path) def test_runner_accepts_matching_wheel_evidence(runner, tmp_path): report = tmp_path / "junit.xml" - evidence = {"wheel_sha256": "abc", "events": 500_002} + evidence = {"wheel_sha256": "abc", "events": 1_000_000} write_report(report, runner.TEST_NAME, evidence) assert runner.read_evidence(report, "abc") == evidence @@ -109,6 +118,7 @@ def test_runner_accepts_additional_nightly_cases_and_records_complete_identities assert runner.read_evidence(report, "abc") == evidence assert [runner.test_node_id(case) for case in runner.read_cases(report)] == [ runner.TEST_NODE, + runner.BOUNDARY_TEST_NODE, "tests/test_extra.py::TestRoutes::test_extra[v2]", ] @@ -265,7 +275,30 @@ def run(command, **kwargs): assert summary["source_sha"] == "checkout-sha" assert summary["wheel_sha256"] == wheel_hash assert summary["pytest_exit_code"] == exit_code - assert summary["required_test_nodes"] == [runner.TEST_NODE] - assert summary["test_nodes"] == ([runner.TEST_NODE] if write_junit else []) + assert summary["required_test_nodes"] == list(runner.REQUIRED_TEST_NODES) + assert summary["test_nodes"] == (list(runner.REQUIRED_TEST_NODES) if write_junit else []) assert summary["status"] == ("passed" if exit_code == 0 and write_junit else "failed") assert (output / "pytest.log").read_text() == "captured pytest output\n" + + +def test_runner_requires_original_short_final_shard_case(runner, tmp_path): + report = tmp_path / "junit.xml" + assert runner.BOUNDARY_TEST_NODE in runner.REQUIRED_TEST_NODES + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}, include_boundary=False) + with pytest.raises(runner.AcceptanceError, match="Missing required nightly test nodes"): + runner.read_evidence(report, "abc") + + +@pytest.mark.parametrize("outcome", ["skipped", "failure", "error"]) +def test_runner_rejects_nonpassing_short_final_shard_case(runner, tmp_path, outcome): + report = tmp_path / "junit.xml" + write_report(report, runner.TEST_NAME, {"wheel_sha256": "abc"}, include_boundary=False) + add_case( + report, + name=BOUNDARY_NAME, + file="tests/core/test_dataset_scale_acceptance.py", + classname="tests.core.test_dataset_scale_acceptance", + outcome=outcome, + ) + with pytest.raises(runner.AcceptanceError, match="without skips"): + runner.read_evidence(report, "abc")