diff --git a/.github/AGENTS.md b/.github/AGENTS.md index e37b146..1428d08 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -27,15 +27,29 @@ events so correcting a declaration can rerun CI. ## Installed-Wheel Acceptance -The acceptance script installs a supplied wheel offline into a temporary venv -with system dependencies available, and verifies that `pt_snap_cli` resolves -inside that venv. It clears source/catalog/focus environment overrides and uses -temporary HOME/CWD for helper installation and import-created focus. +The acceptance script audits a supplied wheel against source and validates the +reviewed fixture manifest before installing or importing. It installs offline +into a temporary venv with system dependencies available by default. Supplying +`--wheelhouse` instead installs dependencies offline into a venv without system +packages and verifies their installed paths too. It verifies that `pt_snap_cli` +resolves inside the venv, clears source/catalog/focus/resource-limit overrides, +disables user site packages, and uses temporary HOME/CWD for skill installation +and import-created focus. It then checks help, bundled catalog/install status, capabilities, trusted fixture import/reuse, overview and a peak query. This is executable CLI acceptance, not a -live model evaluation. The script accepts only the expected committed -`snapshot_with_empty_cache.pkl` path; run fixture provenance verification first. +live model evaluation. `--events-per-slice 4096` adds native-v2 and compatibility-v1 +sharded imports/reuse, directory/manifest discovery, installed helper/peak skill +guidance, and real peak/cross-shard event parity with the standalone CLI. It +requires at least two physical slices on the selected device and verifies that +explicit dataset operations do not change focus. The script accepts only the +expected committed `snapshot_with_empty_cache.pkl` path, rejects an unhydrated +pointer, and runs the provenance gate itself; also run fixture provenance tests +before acceptance. Its output explicitly distinguishes offline CLI execution, +stored-baseline grading (not run here), and live-agent execution (not run). + +Harness changes use `pytest tests/test_clean_wheel_agent_acceptance.py` plus an +actual audited-wheel invocation; mocked guard tests alone are not acceptance. ## Change Together diff --git a/.github/scripts/clean_wheel_agent_acceptance.py b/.github/scripts/clean_wheel_agent_acceptance.py index f245ac3..cd9b85d 100644 --- a/.github/scripts/clean_wheel_agent_acceptance.py +++ b/.github/scripts/clean_wheel_agent_acceptance.py @@ -1,3 +1,5 @@ +"""Offline installed-CLI/skill acceptance, never a live agent/model evaluation.""" + from __future__ import annotations import argparse @@ -74,6 +76,9 @@ def _run_json( raise AcceptanceError(f"{label} did not return complete JSON: {exc}") from exc if not isinstance(payload, dict): raise AcceptanceError(f"{label} returned a non-object JSON value") + # Skill/catalog payloads predate the common ok envelope. If one is present, + # never accept a structured failure just because the process exited zero. + _require("ok" not in payload or payload["ok"] is True, f"{label} did not report JSON success") return payload @@ -109,13 +114,21 @@ def _venv_paths(venv_dir: Path) -> tuple[Path, Path]: def _isolated_environment(home: Path, cwd: Path, venv_dir: Path, bin_dir: Path) -> dict[str, str]: - env = os.environ.copy() + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("PYTHON", "PT_SNAP_", "PIP_")) + } for name in CLEARED_ENV: env.pop(name, None) env.pop("PYTHONHOME", None) env["HOME"] = str(home) env["USERPROFILE"] = str(home) env["XDG_CONFIG_HOME"] = str(home / ".config") + env["XDG_CACHE_HOME"] = str(home / ".cache") + env["PYTHONNOUSERSITE"] = "1" + env["PIP_CONFIG_FILE"] = os.devnull + env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" env["VIRTUAL_ENV"] = str(venv_dir) env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}" env["PWD"] = str(cwd) @@ -124,7 +137,258 @@ def _isolated_environment(home: Path, cwd: Path, venv_dir: Path, bin_dir: Path) return env -def _run_acceptance(wheel: Path, fixture: Path) -> None: +def _check_dataset_skill(text: str, name: str) -> None: + """Check the installed instructions, not an authored checkout substitute.""" + for required in ( + "pt-snap-native-v2 dataset directory or `manifest.json`", + "overview.dataset.status", + "manifest_path", + "fingerprint", + "dataset_support.supported", + "scope.source_coverage.range_complete", + "scope.range_complete", + "has_more", + "truncated", + "pt-snap capabilities --json", + ): + _require(required in text, f"Installed {name} omits dataset guidance: {required}") + + +def _dataset_overview( + overview: dict[str, Any], target: Path, dataset: Path, expected_format: str +) -> dict[str, Any]: + _require( + _path_value(overview, "db_path", "dataset overview") == target, "Wrong overview target" + ) + _require( + overview.get("focus_source") == "explicit", "Dataset discovery did not use explicit target" + ) + metadata = overview.get("import_metadata", {}) + _require(metadata.get("status") == "available", "Dataset import provenance is not available") + details = overview.get("dataset", {}) + _require(details.get("status") == "complete", "Dataset is not complete") + _require(details.get("format") == expected_format, "Wrong dataset format") + _require( + _path_value(details, "manifest_path", "dataset overview") == dataset / "manifest.json", + "Wrong dataset manifest", + ) + fingerprint = details.get("fingerprint") + _require(isinstance(fingerprint, str) and bool(fingerprint), "Dataset has no fingerprint") + _require(details.get("cross_slice_queries") is True, "Cross-slice queries are unavailable") + return details + + +def _dataset_query( + query: dict[str, Any], target: Path, dataset: Path, fingerprint: str, device: int +) -> dict[str, Any]: + _require(_path_value(query, "db_path", "dataset query") == target, "Wrong query target") + _require(query.get("focus_source") == "explicit", "Dataset query did not use explicit target") + _require(query.get("device_id") == device, "Dataset query changed device") + rows = query.get("rows") + _require(isinstance(rows, list) and query.get("returned") == len(rows), "Invalid query rows") + _require( + query.get("has_more") is False and query.get("truncated") is False, "Incomplete query rows" + ) + scope = query.get("scope", {}) + _require(_path_value(scope, "dataset_path", "query scope") == dataset, "Wrong query dataset") + _require(scope.get("fingerprint") == fingerprint, "Query changed dataset generation") + _require(scope.get("range_complete") is True, "Incomplete query source range") + _require(scope.get("boundary_events_included") is False, "Query included synthetic boundaries") + return scope + + +def _sharded_acceptance( + pt_snap: Path, + fixture: Path, + output_dir: Path, + events_per_slice: int, + standalone: Path, + standalone_overview: dict[str, Any], + standalone_peak: dict[str, Any], + focus_file: Path, + *, + cwd: Path, + env: dict[str, str], +) -> list[dict[str, Any]]: + """Execute both complete-dataset forms through the installed public CLI.""" + + def cli(label: str, *args: object) -> dict[str, Any]: + return _run_json([str(pt_snap), *map(str, args), "--json"], cwd=cwd, env=env, label=label) + + device = standalone_peak["device_id"] + focus_before = focus_file.read_bytes() + evidence = [] + for format_name in ("pt-snap-native-v2", "compatibility-v1"): + import_args = ( + "import", + fixture, + "--output-dir", + output_dir / format_name, + "--events-per-slice", + events_per_slice, + "--format", + format_name, + "--no-focus", + ) + imported = cli(f"{format_name} import", *import_args) + dataset = _path_value(imported, "dataset_path", "sharded import") + _require( + dataset.is_dir() and dataset.is_relative_to(output_dir), + "Dataset was not published inside the isolated output", + ) + _require( + _path_value(imported, "db_path", "sharded import") == dataset, + "Import target is not the dataset", + ) + _require(imported.get("format") == format_name, "Import returned the wrong format") + _require( + imported.get("rebuilt") is True and imported.get("reused") is False, + "Initial dataset import was not built", + ) + _require(imported.get("focus_state") is None, "No-focus import returned persisted focus") + reused = cli(f"{format_name} reuse", *import_args) + _require(reused.get("reused") is True, "Dataset import did not reuse its cache") + _require( + _path_value(reused, "dataset_path", "sharded reuse") == dataset, + "Reused dataset changed target", + ) + _require(reused.get("focus_state") is None, "No-focus reuse returned persisted focus") + + fingerprint = None + selected = None + for target in (dataset, dataset / "manifest.json"): + overview = cli(f"{format_name} overview {target.name}", "overview", target) + details = _dataset_overview(overview, target, dataset, format_name) + _require( + overview["devices"] == standalone_overview["devices"], + "Dataset discovery changed real device bounds", + ) + selected = next( + (d for d in details.get("devices", []) if d.get("device_id") == device), None + ) + _require(selected is not None, "Dataset discovery lost the selected device") + slices = selected.get("slices", []) + _require( + len(slices) >= 2, + "Sharded acceptance needs at least two slices on the selected device; lower --events-per-slice", + ) + _require( + all(Path(s["db_path"]).is_file() for s in slices), "Discovered shard is absent" + ) + _require( + imported.get("slice_count") == sum(len(d["slices"]) for d in details["devices"]), + "Import/discovery slice counts disagree", + ) + _require( + fingerprint in (None, details["fingerprint"]), + "Directory and manifest identify different generations", + ) + fingerprint = details["fingerprint"] + peak = cli( + f"{format_name} peak {target.name}", + "query", + target, + "--device", + device, + "--template-use", + "memory_peak", + ) + scope = _dataset_query(peak, target, dataset, fingerprint, device) + _require(peak.get("template") == "memory_peak", "Wrong peak template") + _require( + peak["rows"] == standalone_peak["rows"], + "Dataset peaks differ from standalone CLI results", + ) + _require( + scope.get("slice_indices") == [s["index"] for s in slices], + "Peak query did not span every shard", + ) + _require( + scope.get("actual_range", {}).get("real_event_count") == selected["event_count"], + "Peak query omitted real events", + ) + + # Inspect actual events at opposite ends, then a range crossing a shard + # boundary. Never count synthetic boundary rows as real capture events. + assert selected is not None and fingerprint is not None + slices = selected["slices"] + for owner in (slices[0], slices[-1]): + event_id = owner["first_event_id"] + point = cli( + f"{format_name} point {owner['index']}", + "query", + dataset, + "--device", + device, + "--template-use", + "event", + "--params", + json.dumps({"id": event_id}), + ) + scope = _dataset_query(point, dataset, dataset, fingerprint, device) + _require( + scope.get("slice_indices") == [owner["index"]], + "Point query used the wrong owning shard", + ) + _require( + scope.get("source_coverage", {}).get("range_complete") is True, + "Point query source coverage is incomplete", + ) + _require( + [row.get("id") for row in point["rows"]] == [event_id], + "Point query returned the wrong real event", + ) + + params = json.dumps( + {"min_id": slices[0]["last_event_id"], "max_id": slices[1]["first_event_id"]} + ) + arguments = ( + "--device", + device, + "--template-use", + "event", + "--params", + params, + "--exact-total", + ) + crossing = cli(f"{format_name} cross-shard events", "query", dataset, *arguments) + reference = cli(f"{format_name} standalone events", "query", standalone, *arguments) + scope = _dataset_query(crossing, dataset, dataset, fingerprint, device) + _require( + scope.get("slice_indices") == [slices[0]["index"], slices[1]["index"]], + "Event range did not cross shards", + ) + _require( + crossing["rows"] == reference["rows"] and len(crossing["rows"]) == 2, + "Cross-shard real events differ from standalone", + ) + _require( + crossing.get("total_is_exact") is True and crossing.get("total") == 2, + "Cross-shard exact total is invalid", + ) + _require( + focus_file.read_bytes() == focus_before, + "Dataset discovery/query/import changed project focus", + ) + evidence.append( + { + "format": format_name, + "slice_count": imported["slice_count"], + "real_event_count": selected["event_count"], + "device_id": device, + "addressing": ["directory", "manifest.json"], + "reused": True, + "standalone_peak_parity": True, + "cross_shard_event_parity": True, + "focus_unchanged": True, + } + ) + return evidence + + +def _run_acceptance( + wheel: Path, fixture: Path, events_per_slice: int | None = None, wheelhouse: Path | None = None +) -> dict[str, Any]: repo_root = Path(__file__).resolve().parents[2] expected_fixture = repo_root / "tests/fixtures/snapshots/snapshot_with_empty_cache.pkl" wheel = wheel.resolve() @@ -134,6 +398,13 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: _require(wheel.suffix == ".whl", f"Expected a wheel path: {wheel}") _require(fixture == expected_fixture, f"Unexpected snapshot fixture: {fixture}") _require(fixture.is_file(), f"Snapshot fixture does not exist: {fixture}") + _require( + events_per_slice is None or type(events_per_slice) is int and events_per_slice > 0, + "--events-per-slice must be positive", + ) + if wheelhouse is not None: + wheelhouse = wheelhouse.resolve() + _require(wheelhouse.is_dir(), f"Wheelhouse does not exist: {wheelhouse}") with tempfile.TemporaryDirectory(prefix="pt-snap-agent-wheel-") as temporary: root = Path(temporary).resolve() @@ -144,10 +415,41 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: home.mkdir() cwd.mkdir() output_dir.mkdir() - venv.EnvBuilder(with_pip=True, system_site_packages=True, clear=True).create(venv_dir) venv_python, pt_snap = _venv_paths(venv_dir) - _require(venv_python.is_file(), f"Venv Python was not created: {venv_python}") env = _isolated_environment(home, cwd, venv_dir, pt_snap.parent) + _run( + [ + sys.executable, + str(repo_root / ".github/scripts/audit_wheel.py"), + "--wheel", + str(wheel), + ], + cwd=cwd, + env=env, + label="wheel/source audit", + ) + _run( + [ + sys.executable, + "-c", + "import runpy, sys; from pathlib import Path; " + "root = Path(sys.argv[1]).parent / 'fixtures/snapshots'; " + "runpy.run_path(sys.argv[1])['verify_snapshot_fixtures'](root, root / 'SHA256SUMS')", + str(repo_root / "tests/_fixture_provenance.py"), + ], + cwd=cwd, + env=env, + label="fixture provenance gate", + ) + with fixture.open("rb") as stream: + _require( + not stream.read(128).startswith(b"version https://git-lfs.github.com/spec/"), + "Snapshot fixture is an unhydrated LFS pointer", + ) + venv.EnvBuilder(with_pip=True, system_site_packages=wheelhouse is None, clear=True).create( + venv_dir + ) + _require(venv_python.is_file(), f"Venv Python was not created: {venv_python}") _run( [ @@ -155,7 +457,7 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: "-m", "pip", "install", - "--no-deps", + *(["--no-deps"] if wheelhouse is None else ["--find-links", str(wheelhouse)]), "--no-index", "--force-reinstall", str(wheel), @@ -182,6 +484,31 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: _is_inside(module_path, venv_dir), f"pt_snap_cli resolved outside the venv: {module_path}", ) + if wheelhouse is not None: + identity = _run_json( + [ + str(venv_python), + "-I", + "-c", + "import json, site, sys, typer, yaml, jinja2; " + "print(json.dumps({'prefix': sys.prefix, 'user_site': site.ENABLE_USER_SITE, " + "'modules': {m.__name__: m.__file__ for m in (typer, yaml, jinja2)}}))", + ], + cwd=cwd, + env=env, + label="offline dependency identity", + ) + _require( + Path(identity["prefix"]).resolve() == venv_dir, "Wrong installed runtime prefix" + ) + _require(identity["user_site"] is False, "User site packages were enabled") + _require( + all( + _is_inside(Path(path).resolve(), venv_dir) + for path in identity["modules"].values() + ), + "Offline dependencies resolved outside the venv", + ) bundled_root = module_path.parent / "bundled_skills" bundled_helper = bundled_root / "pt-snap-helper" / "SKILL.md" _require(bundled_helper.is_file(), f"Bundled helper is missing: {bundled_helper}") @@ -297,6 +624,10 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: installed_source == source_dir, "Installed helper source changed after installation" ) installed_text = (installed_path / "SKILL.md").read_text(encoding="utf-8") + _require( + installed_text == bundled_helper.read_text(encoding="utf-8"), + "Installed helper differs from wheel source", + ) _require( "name: pt-snap-helper" in installed_text, "Installed helper content is not readable" ) @@ -307,6 +638,47 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: ): _require(command in installed_text, f"Installed helper omits {command}") + if events_per_slice is not None: + _check_dataset_skill(installed_text, "pt-snap-helper") + diagnostic = "pt-snap-memory-peak-breakdown" + installed_diagnostic = _run_json( + [str(pt_snap), "skill", "install", diagnostic, "--target", "agents", "--json"], + cwd=cwd, + env=env, + label="diagnostic skill install", + ) + diagnostic_result = installed_diagnostic.get("results", []) + _require(len(diagnostic_result) == 1, "Invalid diagnostic skill installation") + diagnostic_path = _path_value(diagnostic_result[0], "path", "diagnostic skill install") + _require( + diagnostic_result[0].get("action") == "installed", + "Diagnostic skill was not installed", + ) + _require( + diagnostic_path == home / ".agents/skills" / diagnostic, + "Diagnostic skill escaped isolated HOME", + ) + diagnostic_text = (diagnostic_path / "SKILL.md").read_text(encoding="utf-8") + _require( + diagnostic_text + == (bundled_root / diagnostic / "SKILL.md").read_text(encoding="utf-8"), + "Installed diagnostic differs from wheel source", + ) + _check_dataset_skill(diagnostic_text, diagnostic) + for command in ( + 'pt-snap overview "" --json', + "--template-use memory_peak", + "pt-snap report peak-memory --help", + ): + _require(command in diagnostic_text, f"Installed diagnostic omits {command}") + _run( + [str(pt_snap), "report", "peak-memory", "--help"], + cwd=cwd, + env=env, + label="diagnostic report help", + clean_stderr=True, + ) + capabilities = _run_json( [str(pt_snap), "capabilities", "--json"], cwd=cwd, @@ -317,6 +689,23 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: item.get("name") for item in capabilities.get("templates", []) if isinstance(item, dict) } _require("memory_peak" in template_names, "Capabilities did not include memory_peak") + if events_per_slice is not None: + for name in ("event", "memory_peak"): + template = next((t for t in capabilities["templates"] if t.get("name") == name), {}) + _require( + template.get("dataset_support", {}).get("supported") is True, + f"Catalog does not support dataset {name}", + ) + diagnostic_entry = _skill_entry(capabilities, diagnostic, "capabilities") + _require( + diagnostic_entry.get("status") == "installed", + "Capabilities did not see the installed diagnostic", + ) + _require( + _path_value(diagnostic_entry, "source_dir", "capabilities") + == bundled_root / diagnostic, + "Diagnostic catalog source escaped wheel", + ) capability_helper = _skill_entry(capabilities, "pt-snap-helper", "capabilities") _require( capability_helper.get("status") == "installed", @@ -435,22 +824,72 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: "Memory peak query returned rows without peak values", ) + sharded = [] + if events_per_slice is not None: + sharded = _sharded_acceptance( + pt_snap, + fixture, + output_dir, + events_per_slice, + database, + overview, + query, + focus_file, + cwd=cwd, + env=env, + ) + return { + "acceptance_kind": "offline-installed-cli-and-skills", + "live_agent_execution": "not-run; requires separate approval", + "stored_baseline_scoring": "not-run; separate recorded-run grading only", + "wheel_source_audit": "passed", + "fixture_provenance": "passed", + "dependency_mode": ( + "isolated-offline-wheelhouse" if wheelhouse else "system-site-packages" + ), + "installed_package_in_venv": True, + "standalone_import_reuse_discovery_query": "passed", + "events_per_slice": events_per_slice, + "datasets": sharded, + } + + +def _positive_int(value: str) -> int: + try: + number = int(value) + except ValueError as exc: + raise argparse.ArgumentTypeError("must be a positive integer") from exc + if number <= 0: + raise argparse.ArgumentTypeError("must be a positive integer") + return number + -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser() +def _parse_args(arguments: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--wheel", required=True, type=Path) parser.add_argument("--fixture", required=True, type=Path) - return parser.parse_args() + parser.add_argument( + "--events-per-slice", + type=_positive_int, + help="Also test both sharded formats; use a capacity producing at least two slices (reviewed fixture: 4096)", + ) + parser.add_argument( + "--wheelhouse", + type=Path, + help="Install dependencies offline from this directory in a venv without system packages; default reuses system dependencies", + ) + return parser.parse_args(arguments) def main() -> int: args = _parse_args() try: - _run_acceptance(args.wheel, args.fixture) + evidence = _run_acceptance(args.wheel, args.fixture, args.events_per_slice, args.wheelhouse) except AcceptanceError as exc: print(f"clean-wheel Agent acceptance failed: {exc}", file=sys.stderr) return 1 - print("clean-wheel Agent acceptance passed") + print(json.dumps(evidence, indent=2)) + print("clean-wheel offline CLI/skills acceptance passed (live agent not run)") return 0 diff --git a/.github/workflows/AGENTS.md b/.github/workflows/AGENTS.md index 2bb0121..ab7d111 100644 --- a/.github/workflows/AGENTS.md +++ b/.github/workflows/AGENTS.md @@ -18,7 +18,7 @@ Concurrency is keyed by GitHub ref and superseded runs are cancelled. | Full PR test tier | Python 3.13; `pytest -m "not nightly"` with branch coverage, preserving the existing slow tests | | Overall coverage | `pyproject.toml` fail-under 90 with branches enabled | | Critical runtime coverage | Import backend 90; simulated allocator 88; snapshot mutator 84, checked independently | -| Python 3.13 packaging | Upload coverage, build wheel, run clean-wheel Agent acceptance | +| Python 3.13 packaging | Upload coverage, build wheel, run offline clean-wheel skill/CLI acceptance including both sharded formats | | Nightly/manual dataset acceptance | Python 3.13; provenance, wheel build and audit, then `nightly` tests including the required installed-wheel CLI/API test | Million-event deep-offset pagination tests are marked `slow`; their small diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 44b1b1b..6b7b2be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -171,7 +171,8 @@ jobs: WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') python .github/scripts/clean_wheel_agent_acceptance.py \ --wheel "$WHEEL" \ - --fixture tests/fixtures/snapshots/snapshot_with_empty_cache.pkl + --fixture tests/fixtures/snapshots/snapshot_with_empty_cache.pkl \ + --events-per-slice 4096 dataset-acceptance: name: Installed-wheel dataset acceptance diff --git a/src/AGENTS.md b/src/AGENTS.md index 7dcd7d2..30000ae 100644 --- a/src/AGENTS.md +++ b/src/AGENTS.md @@ -33,7 +33,10 @@ Source-path selection alone does not change installed version metadata. package resolution inside a temporary venv, bundled skill discovery and installation, then trusted import/reuse, overview and query. Its venv shares system dependencies but installs the supplied wheel with `--no-deps --no-index`; - this is not a fresh dependency-resolution test. See [automation](../.github/AGENTS.md). + this default is not a fresh dependency-resolution test. `--wheelhouse` instead + installs dependencies offline without system packages; `--events-per-slice` + adds both complete-dataset formats, installed diagnostic skill guidance and + cross-shard public CLI parity. See [automation](../.github/AGENTS.md). Use `pytest tests/test_package.py tests/test_bundled_skills.py` for package contracts. A source checkout alone cannot prove that a newly added asset ships. diff --git a/tests/skills/README.md b/tests/skills/README.md index 7fa4fbb..bce1cda 100644 --- a/tests/skills/README.md +++ b/tests/skills/README.md @@ -199,9 +199,49 @@ python -m tests.skills baseline \ Diagnostic fixtures remain declarative SQLite databases. The import scenario records `pt_snap.import` semantically and never materializes a pickle. -The release-gating clean-wheel check separately installs the built wheel into a -fresh venv, installs `pt-snap-helper` from package data, and completes import, -capability, overview, and peak-query discovery using JSON output. +The release-gating clean-wheel check separately audits and installs the built +wheel into a temporary venv, installs `pt-snap-helper` from package data, and +completes import, capability, overview, and peak-query discovery using genuine +CLI JSON output. By default the venv shares existing system dependencies; this +is not a fresh dependency-resolution test. + +For offline sharded CLI/skill acceptance (issue #253): + +```bash +python .github/scripts/clean_wheel_agent_acceptance.py \ + --wheel dist/.whl \ + --fixture tests/fixtures/snapshots/snapshot_with_empty_cache.pkl \ + --events-per-slice 4096 +``` + +The script verifies fixture provenance before deserialization. The capacity must +produce at least two slices on the selected device. This invocation also installs +the bundled peak diagnostic, checks its and the helper's complete-dataset +instructions, and executes native-v2 and compatibility-v1 import/reuse. It checks +both dataset directory and `manifest.json` discovery, full-device peaks against +the standalone CLI, opposite-shard point events and boundary-crossing event +ranges, source completeness, stable fingerprints, and unchanged project focus. +The approved import harness owns pickle handling; diagnostic skills still must +not import or persist focus. Filesystem installation is not proof a host loaded +the skill or an agent followed it. + +Add `--wheelhouse ` to install all dependencies with +`--no-index` into a venv without system packages. This mode verifies dependency +module paths inside the venv. Both modes isolate HOME/CWD, clear source and +resource-limit overrides, and disable user site packages. A missing offline +dependency fails rather than falling back to network installation. + +This is deterministic offline CLI and installed-skill acceptance, not live-agent +evaluation. `baseline` remains scoring of stored records. A real model runner, +its transcripts and decision quality remain incomplete and require separate +approval; this check does not connect a user's agent session or model provider. + +中文:上述命令为离线 CLI 与已安装 skills 验收;`--events-per-slice 4096` +覆盖 native-v2 / compatibility-v1 的真实多分片导入、缓存复用、目录和 manifest +发现、跨分片查询及单库结果对照。运行前会验证已审核 fixture 的来源清单。 +默认复用系统依赖;加 `--wheelhouse` 可完全离线安装依赖且不共享系统包。 +安装到磁盘不等于宿主已加载或真实 agent 已遵循技能。`baseline` 仍只评分历史记录, +真实模型执行、执行记录和决策质量仍未完成,需另行批准;不连接真实用户会话或模型服务。 Generated transcripts and reports belong under `.skill-evals/`, which is ignored by Git. Normal `pytest` runs never invoke a live model. diff --git a/tests/test_clean_wheel_agent_acceptance.py b/tests/test_clean_wheel_agent_acceptance.py new file mode 100644 index 0000000..5dd3141 --- /dev/null +++ b/tests/test_clean_wheel_agent_acceptance.py @@ -0,0 +1,326 @@ +"""Offline harness guard regressions; real installed runs remain a separate gate.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import shlex +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / ".github/scripts/clean_wheel_agent_acceptance.py" + + +@pytest.fixture +def acceptance(): + spec = importlib.util.spec_from_file_location("clean_wheel_agent_acceptance", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_sharded_mode_is_explicit_and_requires_positive_capacity(acceptance): + required = ["--wheel", "current.whl", "--fixture", "reviewed.pkl"] + args = acceptance._parse_args(required) + assert args.events_per_slice is None and args.wheelhouse is None + args = acceptance._parse_args( + [*required, "--events-per-slice", "4096", "--wheelhouse", "wheels"] + ) + assert args.events_per_slice == 4096 and args.wheelhouse == Path("wheels") + for value in ("0", "-1", "2.5", "false"): + with pytest.raises(SystemExit) as exc: + acceptance._parse_args([*required, "--events-per-slice", value]) + assert exc.value.code == 2 + + +def test_environment_clears_source_focus_hosts_and_resource_overrides( + acceptance, tmp_path, monkeypatch +): + inherited = ( + *acceptance.CLEARED_ENV, + "PYTHONHOME", + "PYTHONUSERBASE", + "PT_SNAP_QUERY_TIMEOUT", + "PT_SNAP_MAX_WORK_ROWS", + "PT_SNAP_MAX_WORK_BYTES", + "PT_SNAP_MAX_SCRATCH_DB_BYTES", + "PIP_FIND_LINKS", + "PIP_TARGET", + "PIP_REQUIREMENT", + ) + for key in inherited: + monkeypatch.setenv(key, "must-not-leak") + env = acceptance._isolated_environment( + tmp_path / "home", tmp_path / "cwd", tmp_path / "venv", tmp_path / "venv/bin" + ) + assert not set(inherited) & env.keys() + assert env["HOME"] == env["USERPROFILE"] == str(tmp_path / "home") + assert env["XDG_CONFIG_HOME"] == str(tmp_path / "home/.config") + assert env["PYTHONNOUSERSITE"] == "1" + assert env["PIP_CONFIG_FILE"] == os.devnull + assert env["PATH"].split(os.pathsep)[0] == str(tmp_path / "venv/bin") + assert os.environ["PT_SNAP_DB_PATH"] == "must-not-leak" + + +def test_ci_runs_audited_wheel_with_explicit_sharded_capacity(): + workflow = yaml.load((ROOT / ".github/workflows/test.yml").read_text(), Loader=yaml.BaseLoader) + steps = workflow["jobs"]["test"]["steps"] + by_name = {step.get("name"): step for step in steps} + step = by_name["Clean-wheel Agent acceptance"] + assert step["if"] == "matrix.python-version == '3.13'" + assert "continue-on-error" not in step + command = step["run"].split("python .github/scripts/clean_wheel_agent_acceptance.py", 1)[1] + assert shlex.split(command.replace("\\\n", " ")) == [ + "--wheel", + "$WHEEL", + "--fixture", + "tests/fixtures/snapshots/snapshot_with_empty_cache.pkl", + "--events-per-slice", + "4096", + ] + names = list(by_name) + assert ( + names.index("Build wheel") + < names.index("Audit wheel contents against source") + < names.index("Clean-wheel Agent acceptance") + ) + + +@pytest.mark.parametrize("payload", [{"ok": True, "rows": []}, {"skills": []}]) +def test_json_accepts_public_envelope_and_legacy_catalog(acceptance, tmp_path, payload): + result = acceptance._run_json( + [sys.executable, "-c", f"print({json.dumps(payload)!r})"], + cwd=tmp_path, + env=os.environ.copy(), + label="public JSON", + ) + assert result == payload + + +@pytest.mark.parametrize( + ("script", "error"), + [ + ("print('[]')", "non-object"), + ("print('log noise\\n{}')", "complete JSON"), + ("print('{')", "complete JSON"), + ("print('{\"ok\": false}')", "JSON success"), + ("import sys; print('{}'); print('warning', file=sys.stderr)", "wrote to stderr"), + ("import sys; print('{}'); sys.exit(2)", "exited with 2"), + ], +) +def test_json_rejects_hollow_success(acceptance, tmp_path, script, error): + with pytest.raises(acceptance.AcceptanceError, match=error): + acceptance._run_json( + [sys.executable, "-c", script], cwd=tmp_path, env=os.environ.copy(), label="bad JSON" + ) + + +def test_unapproved_fixture_is_rejected_before_any_execution(acceptance, tmp_path, monkeypatch): + wheel = tmp_path / "standin.whl" + wheel.write_bytes(b"not installed") + fixture = tmp_path / "arbitrary.pkl" + fixture.write_bytes(b"never loaded") + monkeypatch.setattr( + acceptance, "_run", lambda *_a, **_k: pytest.fail("executed before trust check") + ) + with pytest.raises(acceptance.AcceptanceError, match="Unexpected snapshot fixture"): + acceptance._run_acceptance(wheel, fixture, 4096) + + +@pytest.mark.parametrize("failure", ["wheel/source audit", "fixture provenance gate"]) +def test_audit_and_provenance_fail_before_install_or_import( + acceptance, tmp_path, monkeypatch, failure +): + wheel = tmp_path / "standin.whl" + wheel.write_bytes(b"guard test, never installed") + labels = [] + + def run(command, *, cwd, env, label, **kwargs): + labels.append(label) + assert cwd != ROOT + assert "PYTHONPATH" not in env + if label == failure: + raise acceptance.AcceptanceError(f"{failure} refused") + + monkeypatch.setattr(acceptance, "_run", run) + monkeypatch.setattr( + acceptance.venv, "EnvBuilder", lambda **_k: pytest.fail("venv created before guards") + ) + with pytest.raises(acceptance.AcceptanceError, match="refused"): + acceptance._run_acceptance( + wheel, ROOT / "tests/fixtures/snapshots/snapshot_with_empty_cache.pkl", 4096 + ) + assert labels == ["wheel/source audit"] + ( + ["fixture provenance gate"] if failure == "fixture provenance gate" else [] + ) + + +@pytest.mark.parametrize("name", ["pt-snap-helper", "pt-snap-memory-peak-breakdown"]) +def test_dataset_skill_guard_rejects_missing_discovery_contract(acceptance, name): + text = (ROOT / "src/pt_snap_cli/bundled_skills" / name / "SKILL.md").read_text() + acceptance._check_dataset_skill(text, name) + for token in ( + "overview.dataset.status", + "manifest_path", + "fingerprint", + "dataset_support.supported", + "scope.range_complete", + "scope.source_coverage.range_complete", + ): + with pytest.raises(acceptance.AcceptanceError, match="omits dataset guidance"): + acceptance._check_dataset_skill(text.replace(token, "removed"), name) + + +def _overview(root, *, manifest=False): + target = root / "manifest.json" if manifest else root + return { + "db_path": str(target), + "focus_source": "explicit", + "import_metadata": {"status": "available"}, + "dataset": { + "status": "complete", + "format": "pt-snap-native-v2", + "manifest_path": str(root / "manifest.json"), + "fingerprint": "generation-1", + "cross_slice_queries": True, + }, + } + + +@pytest.mark.parametrize("manifest", [False, True]) +def test_dataset_discovery_accepts_directory_and_manifest(acceptance, tmp_path, manifest): + overview = _overview(tmp_path, manifest=manifest) + assert ( + acceptance._dataset_overview( + overview, Path(overview["db_path"]), tmp_path, "pt-snap-native-v2" + ) + is overview["dataset"] + ) + + +@pytest.mark.parametrize( + ("path", "value"), + [ + (("db_path",), "/wrong"), + (("focus_source",), "project"), + (("import_metadata", "status"), "invalid"), + (("dataset", "status"), "incomplete"), + (("dataset", "format"), "single-db"), + (("dataset", "fingerprint"), ""), + (("dataset", "manifest_path"), "/wrong/manifest.json"), + (("dataset", "cross_slice_queries"), False), + ], +) +def test_dataset_discovery_rejects_unsafe_or_incomplete_evidence(acceptance, tmp_path, path, value): + overview = _overview(tmp_path) + target = overview if len(path) == 1 else overview[path[0]] + target[path[-1]] = value + with pytest.raises(acceptance.AcceptanceError): + acceptance._dataset_overview(overview, tmp_path, tmp_path, "pt-snap-native-v2") + + +def _query(root): + return { + "db_path": str(root), + "focus_source": "explicit", + "device_id": 0, + "rows": [{"id": 4096}], + "returned": 1, + "has_more": False, + "truncated": False, + "scope": { + "dataset_path": str(root), + "fingerprint": "generation-1", + "range_complete": True, + "boundary_events_included": False, + }, + } + + +def test_dataset_query_keeps_generation_and_completeness_separate(acceptance, tmp_path): + query = _query(tmp_path) + assert acceptance._dataset_query(query, tmp_path, tmp_path, "generation-1", 0) is query["scope"] + + +@pytest.mark.parametrize( + ("path", "value"), + [ + (("db_path",), "/wrong"), + (("focus_source",), "project"), + (("device_id",), 1), + (("returned",), 2), + (("rows",), None), + (("has_more",), True), + (("truncated",), True), + (("scope", "dataset_path"), "/wrong"), + (("scope", "fingerprint"), "generation-2"), + (("scope", "range_complete"), False), + (("scope", "boundary_events_included"), True), + ], +) +def test_dataset_query_rejects_misleading_success(acceptance, tmp_path, path, value): + query = _query(tmp_path) + target = query if len(path) == 1 else query[path[0]] + target[path[-1]] = value + with pytest.raises(acceptance.AcceptanceError): + acceptance._dataset_query(query, tmp_path, tmp_path, "generation-1", 0) + + +def test_one_physical_shard_cannot_pass_sharded_acceptance(acceptance, tmp_path, monkeypatch): + dataset = tmp_path / "one-shard" + dataset.mkdir() + member = dataset / "slice_00000.db" + member.write_bytes(b"never opened") + focus = tmp_path / "focus.json" + focus.write_text("unchanged") + overview = _overview(dataset) + overview["devices"] = [{"device_id": 0, "first_event_id": 0, "last_event_id": 1}] + overview["dataset"]["devices"] = [ + { + "device_id": 0, + "event_count": 2, + "slices": [ + {"index": 0, "db_path": str(member), "first_event_id": 0, "last_event_id": 1} + ], + } + ] + commands = [] + + def cli(command, **kwargs): + commands.append(command) + if command[1] == "overview": + return overview + assert command[1] == "import" and "--no-focus" in command + reused = len(commands) == 2 + return { + "dataset_path": str(dataset), + "db_path": str(dataset), + "format": "pt-snap-native-v2", + "rebuilt": not reused, + "reused": reused, + "focus_state": None, + "slice_count": 1, + } + + monkeypatch.setattr(acceptance, "_run_json", cli) + with pytest.raises(acceptance.AcceptanceError, match="at least two slices"): + acceptance._sharded_acceptance( + Path("pt-snap"), + tmp_path / "fixture.pkl", + tmp_path, + 10000, + tmp_path / "standalone.db", + overview, + {"device_id": 0}, + focus, + cwd=tmp_path, + env={}, + ) + assert [command[1] for command in commands] == ["import", "import", "overview"] + assert focus.read_text() == "unchanged"