From e0acf292621823111b5472240c32a7b4df57ff47 Mon Sep 17 00:00:00 2001 From: codeasier Date: Wed, 23 Sep 2026 23:55:34 +0800 Subject: [PATCH 1/4] feat(agent): complete #136 CLI acceptance --- .../scripts/clean_wheel_agent_acceptance.py | 442 ++++++++++++++++++ .github/workflows/test.yml | 21 +- CHANGELOG.md | 4 +- src/pt_snap_cli/cli.py | 20 +- tests/skills/README.md | 21 +- tests/skills/harness/__init__.py | 3 +- tests/skills/harness/descriptors.py | 98 +++- tests/skills/harness/fixtures.py | 4 +- tests/skills/harness/gateway.py | 18 + tests/skills/harness/grader.py | 23 +- tests/skills/schemas/case-v1.yaml | 5 + .../pt-snap-agent-e2e/AGENT_CONTRACT.md | 9 +- .../baselines/pre-change/no-device.json | 31 ++ .../pre-change/wrong-parameter-recovery.json | 45 ++ .../target/cli-only-help-discovery.json | 4 +- .../baselines/target/import-success.json | 13 +- .../target/live-blocks-unconfirmed.json | 48 +- .../baselines/target/missing-database.json | 3 +- .../baselines/target/no-device.json | 34 ++ .../target/wrong-parameter-recovery.json | 73 +++ .../target/wrong-template-recovery.json | 3 +- .../cases/cli-only-help-discovery.yaml | 6 +- .../cases/explicit-multi-target.yaml | 5 + .../cases/import-success.yaml | 11 +- .../cases/live-blocks-unconfirmed.yaml | 19 +- .../cases/missing-database.yaml | 2 +- .../pt-snap-agent-e2e/cases/no-device.yaml | 51 ++ .../cases/wrong-parameter-recovery.yaml | 73 +++ .../cases/wrong-template-recovery.yaml | 2 +- .../fixtures/no-devices.yaml | 3 + .../suites/pt-snap-agent-e2e/suite.yaml | 17 +- tests/skills/test_agent_e2e.py | 28 +- tests/skills/test_descriptors.py | 24 + tests/skills/test_fixtures.py | 25 + tests/skills/test_gateway.py | 28 +- tests/skills/test_grader.py | 15 +- tests/test_cli.py | 4 + tests/test_cli_json.py | 17 + 38 files changed, 1162 insertions(+), 90 deletions(-) create mode 100644 .github/scripts/clean_wheel_agent_acceptance.py create mode 100644 tests/skills/suites/pt-snap-agent-e2e/baselines/pre-change/no-device.json create mode 100644 tests/skills/suites/pt-snap-agent-e2e/baselines/pre-change/wrong-parameter-recovery.json create mode 100644 tests/skills/suites/pt-snap-agent-e2e/baselines/target/no-device.json create mode 100644 tests/skills/suites/pt-snap-agent-e2e/baselines/target/wrong-parameter-recovery.json create mode 100644 tests/skills/suites/pt-snap-agent-e2e/cases/no-device.yaml create mode 100644 tests/skills/suites/pt-snap-agent-e2e/cases/wrong-parameter-recovery.yaml create mode 100644 tests/skills/suites/pt-snap-agent-e2e/fixtures/no-devices.yaml diff --git a/.github/scripts/clean_wheel_agent_acceptance.py b/.github/scripts/clean_wheel_agent_acceptance.py new file mode 100644 index 0000000..dc04c3d --- /dev/null +++ b/.github/scripts/clean_wheel_agent_acceptance.py @@ -0,0 +1,442 @@ +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import venv +from pathlib import Path +from typing import Any + + +class AcceptanceError(RuntimeError): + pass + + +CLEARED_ENV = ( + "PYTHONPATH", + "PT_SNAP_SKILLS_DIR", + "PT_SNAP_DB_PATH", + "CLAUDE_CONFIG_DIR", + "CODEX_HOME", +) + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise AcceptanceError(message) + + +def _run( + command: list[str], + *, + cwd: Path, + env: dict[str, str], + label: str, + clean_stderr: bool = False, +) -> subprocess.CompletedProcess[str]: + try: + result = subprocess.run( + command, + cwd=cwd, + env=env, + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + raise AcceptanceError(f"Unable to run {label}: {exc}") from exc + if result.returncode != 0: + raise AcceptanceError( + f"{label} exited with {result.returncode}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + if clean_stderr and result.stderr: + raise AcceptanceError(f"{label} wrote to stderr: {result.stderr}") + return result + + +def _run_json( + command: list[str], + *, + cwd: Path, + env: dict[str, str], + label: str, +) -> dict[str, Any]: + result = _run(command, cwd=cwd, env=env, label=label, clean_stderr=True) + try: + payload = json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise AcceptanceError(f"{label} did not return complete JSON: {exc}") from exc + if not isinstance(payload, dict): + raise AcceptanceError(f"{label} returned a non-object JSON value") + return payload + + +def _skill_entry(payload: dict[str, Any], name: str, label: str) -> dict[str, Any]: + skills = payload.get("skills") + _require(isinstance(skills, list), f"{label} did not return a skills list") + for item in skills: + if isinstance(item, dict) and item.get("name") == name: + return item + raise AcceptanceError(f"{label} did not include {name}") + + +def _path_value(payload: dict[str, Any], key: str, label: str) -> Path: + value = payload.get(key) + _require(isinstance(value, str) and bool(value), f"{label} has no {key} path") + return Path(value).resolve() + + +def _is_inside(path: Path, root: Path) -> bool: + try: + path.relative_to(root) + except ValueError: + return False + return True + + +def _venv_paths(venv_dir: Path) -> tuple[Path, Path]: + if os.name == "nt": + bin_dir = venv_dir / "Scripts" + return bin_dir / "python.exe", bin_dir / "pt-snap.exe" + bin_dir = venv_dir / "bin" + return bin_dir / "python", bin_dir / "pt-snap" + + +def _isolated_environment(home: Path, cwd: Path, venv_dir: Path, bin_dir: Path) -> dict[str, str]: + env = os.environ.copy() + for name in CLEARED_ENV: + env.pop(name, None) + env.pop("PYTHONHOME", None) + env["HOME"] = str(home) + env["USERPROFILE"] = str(home) + env["XDG_CONFIG_HOME"] = str(home / ".config") + env["VIRTUAL_ENV"] = str(venv_dir) + env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}" + env["PWD"] = str(cwd) + return env + + +def _run_acceptance(wheel: Path, fixture: Path) -> None: + repo_root = Path(__file__).resolve().parents[2] + expected_fixture = repo_root / "tests/fixtures/snapshots/snapshot_with_empty_cache.pkl" + wheel = wheel.resolve() + fixture = fixture.resolve() + expected_fixture = expected_fixture.resolve() + _require(wheel.is_file(), f"Wheel does not exist: {wheel}") + _require(wheel.suffix == ".whl", f"Expected a wheel path: {wheel}") + _require(fixture == expected_fixture, f"Unexpected snapshot fixture: {fixture}") + _require(fixture.is_file(), f"Snapshot fixture does not exist: {fixture}") + + with tempfile.TemporaryDirectory(prefix="pt-snap-agent-wheel-") as temporary: + root = Path(temporary).resolve() + home = root / "home" + cwd = root / "project" + venv_dir = root / "venv" + output_dir = root / "database" + home.mkdir() + cwd.mkdir() + output_dir.mkdir() + venv.EnvBuilder(with_pip=True, system_site_packages=True, clear=True).create(venv_dir) + venv_python, pt_snap = _venv_paths(venv_dir) + _require(venv_python.is_file(), f"Venv Python was not created: {venv_python}") + env = _isolated_environment(home, cwd, venv_dir, pt_snap.parent) + + _run( + [ + str(venv_python), + "-m", + "pip", + "install", + "--no-deps", + "--no-index", + "--force-reinstall", + str(wheel), + ], + cwd=cwd, + env=env, + label="wheel installation", + ) + _require(pt_snap.is_file(), f"Venv pt-snap was not created: {pt_snap}") + + module_result = _run( + [ + str(venv_python), + "-c", + "import pt_snap_cli; print(pt_snap_cli.__file__)", + ], + cwd=cwd, + env=env, + label="installed package probe", + clean_stderr=True, + ) + module_path = Path(module_result.stdout.strip()).resolve() + _require( + _is_inside(module_path, venv_dir), + f"pt_snap_cli resolved outside the venv: {module_path}", + ) + bundled_root = module_path.parent / "bundled_skills" + bundled_helper = bundled_root / "pt-snap-helper" / "SKILL.md" + _require(bundled_helper.is_file(), f"Bundled helper is missing: {bundled_helper}") + _require( + _is_inside(bundled_helper, venv_dir), + f"Bundled helper is outside the venv: {bundled_helper}", + ) + + help_result = _run( + [str(pt_snap), "--help"], + cwd=cwd, + env=env, + label="root help", + clean_stderr=True, + ) + normalized_help = " ".join(help_result.stdout.split()) + _require( + "pt-snap skill install pt-snap-helper --json" in normalized_help, + "Root help does not show the supported helper install command", + ) + _require( + "restart the agent" in normalized_help.lower(), + "Root help does not tell the agent to restart after installing the helper", + ) + _require( + "pt-snap capabilities --json" in normalized_help + and "pt-snap overview --json" in normalized_help + and "prefer --json" in normalized_help, + "Root help lost its JSON capability guidance", + ) + + listed = _run_json( + [str(pt_snap), "skill", "list", "--user", "--target", "agents", "--json"], + cwd=cwd, + env=env, + label="initial skill list", + ) + helper = _skill_entry(listed, "pt-snap-helper", "initial skill list") + _require(helper.get("status") == "missing", "Bundled helper was not initially missing") + source_dir = _path_value(helper, "source_dir", "initial skill list") + _require( + _is_inside(source_dir, venv_dir), f"Skill source is outside the venv: {source_dir}" + ) + _require( + source_dir == bundled_root / "pt-snap-helper", + f"Skill source is not the wheel's bundled helper: {source_dir}", + ) + _require((source_dir / "SKILL.md").is_file(), "Bundled helper source has no SKILL.md") + + installed = _run_json( + [ + str(pt_snap), + "skill", + "install", + "pt-snap-helper", + "--target", + "agents", + "--json", + ], + cwd=cwd, + env=env, + label="helper install", + ) + results = installed.get("results") + _require( + isinstance(results, list) and len(results) == 1, "Helper install report is invalid" + ) + install_result = results[0] + _require(isinstance(install_result, dict), "Helper install result is invalid") + _require(install_result.get("action") == "installed", "Helper was not newly installed") + _require(install_result.get("host") == "agents", "Helper was not installed for agents") + _require(install_result.get("scope") == "user", "Helper was not installed at user scope") + installed_path = _path_value(install_result, "path", "helper install") + expected_installed = home / ".agents" / "skills" / "pt-snap-helper" + _require( + installed_path == expected_installed.resolve(), + f"Unexpected helper destination: {installed_path}", + ) + _require((installed_path / "SKILL.md").is_file(), "Installed helper has no SKILL.md") + _require( + installed.get("restart_required") is True, "Helper install did not require a restart" + ) + _require( + "restart" in str(installed.get("restart_hint")).lower(), + "Helper install has no restart hint", + ) + + installed_listing = _run_json( + [str(pt_snap), "skill", "list", "--user", "--target", "agents", "--json"], + cwd=cwd, + env=env, + label="installed skill list", + ) + installed_helper = _skill_entry(installed_listing, "pt-snap-helper", "installed skill list") + _require( + installed_helper.get("status") == "installed", + "Installed helper was not reported installed", + ) + installed_source = _path_value(installed_helper, "source_dir", "installed skill list") + _require( + installed_source == source_dir, "Installed helper source changed after installation" + ) + installed_text = (installed_path / "SKILL.md").read_text(encoding="utf-8") + _require( + "name: pt-snap-helper" in installed_text, "Installed helper content is not readable" + ) + for command in ( + "pt-snap capabilities --json", + "pt-snap overview '' --json", + "pt-snap query --template-use --json", + ): + _require(command in installed_text, f"Installed helper omits {command}") + + capabilities = _run_json( + [str(pt_snap), "capabilities", "--json"], + cwd=cwd, + env=env, + label="capabilities", + ) + template_names = { + item.get("name") for item in capabilities.get("templates", []) if isinstance(item, dict) + } + _require("memory_peak" in template_names, "Capabilities did not include memory_peak") + capability_helper = _skill_entry(capabilities, "pt-snap-helper", "capabilities") + _require( + capability_helper.get("status") == "installed", + "Capabilities did not see installed helper", + ) + capability_source = _path_value(capability_helper, "source_dir", "capabilities") + _require( + _is_inside(capability_source, venv_dir), + f"Capability source is outside the venv: {capability_source}", + ) + + import_command = [ + str(pt_snap), + "import", + str(fixture), + "--output-dir", + str(output_dir), + "--json", + ] + imported = _run_json(import_command, cwd=cwd, env=env, label="snapshot import") + database = _path_value(imported, "db_path", "snapshot import") + _require(database.is_file(), f"Imported database was not published: {database}") + _require( + imported.get("focus_state") is not None, "Snapshot import did not return focus_state" + ) + _require( + imported.get("focus_source") == "project", + "Snapshot import did not report project focus", + ) + focus_state = imported.get("focus_state") + _require(isinstance(focus_state, dict), "Snapshot import focus_state is invalid") + _require( + _path_value(focus_state, "db_path", "snapshot import focus_state") == database, + "Snapshot import focus database does not match the published database", + ) + _require( + focus_state.get("focus_source") == "project", + "Snapshot import focus_state source is invalid", + ) + focus_file = _path_value(focus_state, "focus_file", "snapshot import focus_state") + _require(focus_file.is_file(), f"Project focus file was not written: {focus_file}") + _require( + focus_file.is_relative_to(cwd), f"Project focus escaped isolated CWD: {focus_file}" + ) + + reused = _run_json(import_command, cwd=cwd, env=env, label="snapshot import reuse") + _require(reused.get("reused") is True, "Repeated snapshot import was not reused") + _require( + _path_value(reused, "db_path", "snapshot import reuse") == database, + "Reused import changed database path", + ) + _require(reused.get("focus_state") is not None, "Reused import did not return focus_state") + _require( + reused.get("focus_source") == "project", "Reused import did not retain project focus" + ) + + overview = _run_json( + [str(pt_snap), "overview", "--json"], + cwd=cwd, + env=env, + label="overview", + ) + _require( + _path_value(overview, "db_path", "overview") == database, + "Overview used a different database", + ) + _require(overview.get("focus_source") == "project", "Overview did not use project focus") + devices = overview.get("devices") + _require(isinstance(devices, list) and devices, "Overview returned no devices") + device_ids: set[int] = set() + for device in devices: + _require(isinstance(device, dict), "Overview returned an invalid device") + device_id = device.get("device_id") + _require( + isinstance(device_id, int) and not isinstance(device_id, bool), + "Overview device has no id", + ) + device_ids.add(device_id) + + query = _run_json( + [str(pt_snap), "query", "--template-use", "memory_peak", "--json"], + cwd=cwd, + env=env, + label="memory peak query", + ) + _require( + _path_value(query, "db_path", "memory peak query") == database, + "Query used a different database", + ) + _require(query.get("focus_source") == "project", "Query did not report project focus") + selected_device = query.get("device_id") + _require( + isinstance(selected_device, int) and not isinstance(selected_device, bool), + "Query did not select a device", + ) + _require(selected_device in device_ids, "Query selected a device absent from overview") + _require(query.get("template") == "memory_peak", "Query used the wrong template") + rows = query.get("rows") + _require(isinstance(rows, list) and rows, "Memory peak query returned no rows") + _require( + query.get("returned") == len(rows), "Memory peak query returned count is inconsistent" + ) + _require( + any( + isinstance(row, dict) + and any( + row.get(field) is not None + for field in ( + "peak_allocated", + "peak_active", + "peak_reserved", + ) + ) + for row in rows + ), + "Memory peak query returned rows without peak values", + ) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--wheel", required=True, type=Path) + parser.add_argument("--fixture", required=True, type=Path) + return parser.parse_args() + + +def main() -> int: + args = _parse_args() + try: + _run_acceptance(args.wheel, args.fixture) + except AcceptanceError as exc: + print(f"clean-wheel Agent acceptance failed: {exc}", file=sys.stderr) + return 1 + print("clean-wheel Agent acceptance passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ed9d044..aa59a40 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,22 +114,11 @@ jobs: python -m pip install --upgrade build python -m build --wheel - - name: Wheel smoke test (verify snapshot runtime is packaged) + - name: Clean-wheel Agent acceptance if: matrix.python-version == '3.13' run: | set -euo pipefail - python -m pip install --force-reinstall dist/*.whl - # Confirm the first-party runtime was actually packaged into the wheel. - python -c "import pt_snap_cli.snapshot.tools.adaptors.snapshot2db as m; assert hasattr(m, 'run_dump_to_db'), 'snapshot run_dump_to_db missing in installed wheel'" - # End-to-end import against a real fixture — catches any packaging - # regression where setuptools drops src/pt_snap_cli/snapshot/. - FIXTURE=tests/fixtures/snapshots/snapshot_with_empty_cache.pkl - rm -rf /tmp/pt-snap-wheel-smoke - mkdir -p /tmp/pt-snap-wheel-smoke - DB=/tmp/pt-snap-wheel-smoke/snapshot_with_empty_cache.pkl.db - pt-snap import "$FIXTURE" --no-focus --output-dir /tmp/pt-snap-wheel-smoke - test -f "$DB" - SECOND_IMPORT=$(pt-snap import "$FIXTURE" --no-focus --output-dir /tmp/pt-snap-wheel-smoke) - echo "$SECOND_IMPORT" | grep -q '^Reused:' - pt-snap metadata "$DB" --json | python -c "import json, sys; data=json.load(sys.stdin); assert data['status'] == 'available'" - rm -rf /tmp/pt-snap-wheel-smoke + WHEEL=$(python -c 'from pathlib import Path; wheels = sorted(Path("dist").glob("*.whl")); assert len(wheels) == 1, wheels; print(wheels[0])') + python .github/scripts/clean_wheel_agent_acceptance.py \ + --wheel "$WHEEL" \ + --fixture tests/fixtures/snapshots/snapshot_with_empty_cache.pkl diff --git a/CHANGELOG.md b/CHANGELOG.md index f11c17b..16d9a18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ ### 稳定性与工程 +- Agent CLI 端到端评估补齐参数纠错、无设备数据库、真实竞争数据库、模板语义与 import focus 信封;post-change 契约任务成功率由 0 提升至 1,错误结论率由 0.8 降至 0,并约束调用与输出预算。 +- CI 与 release gate 新增干净 wheel Agent 验收:在隔离 venv 中确认包与 helper 来自 wheel,并仅用结构化输出完成 helper 安装、trusted snapshot import、capabilities、overview 与 peak query。 - basedpyright 对 `src/pt_snap_cli/snapshot/` 与其余包路径同一套 error 门槛:补齐注解后移除 snapshot `executionEnvironments` 降级列表(#122 阶段 3)。 ## [0.4.0] - Unreleased @@ -34,7 +36,7 @@ ### 新增 -- 新增 `pt-snap-helper` 引导技能,并在 `pt-snap --help` 末尾加入 Agent 提示(优先使用已支持的 `--json`,用 `pt-snap skill list --json` 检查技能)。helper 只读路由,不自动安装包、导入 pickle 或改写 focus;随包分发走 `pt-snap skill`。 +- 新增 `pt-snap-helper` 引导技能,并在 `pt-snap --help` 末尾加入 Agent 提示(优先使用已支持的 `--json`,无技能时可用 `pt-snap skill install pt-snap-helper --json` 安装并重启 Agent,用 `pt-snap skill list --json` 检查技能)。helper 只读路由,不自动安装包、导入 pickle 或改写 focus;随包分发走 `pt-snap skill`。 - helper 与泄漏/碎片/峰值诊断 skill 补充 focus 目标缺失但同目录或当前目录存在候选 `.db` 时的恢复规则:不按名称/大小/时间自动选库(即使只有一个候选也需用户确认),确认前不诊断、不用 Python/`sqlite3` 绕过 CLI,确认后先执行 `pt-snap metadata` 再继续原有预检;合法旧库的 `unavailable` / `metadata_missing` 不强制重新 import,换库不继承旧 device,且不持久化 focus、不代跑 import。 - `--template-info` 与 `get_template_info()` 展示参数 `choices`。`order_by` / `order_dir` 等会写入 SQL 标识符或关键字的参数必须声明封闭取值列表;字符串 `choices` 大小写不敏感,并规范化为声明拼写(如 `desc` → `DESC`)。 - `output_schema` 可声明字段语义(`units`、`metric_semantics`、`scope`、`denominator`、`sentinel`、`interpretation_limits`),查询级可声明 `semantics_version` 与解释限制。`--template-info` 与 `get_template_info()` 透出同一份契约。`execute_query()` 结果增加 `template` 与 `semantics_version`,行数据仍为原始 SQLite 值。优先覆盖 `leak_detection`、`memory_peak`、`allocator_gap`、`active_memory_callstack_at_event`。 diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index 4d4439b..85109e4 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import logging import shlex import shutil import sys @@ -92,6 +93,9 @@ AGENT_HELP_EPILOG = ( "Agents: prefer --json where supported. " + "Agents without pt-snap-helper can run " + "pt-snap skill install pt-snap-helper --json, then restart the agent " + "so the skill change takes effect. " "Start with the pt-snap-helper skill; " "use pt-snap capabilities --json and pt-snap overview --json before diagnosing; " "check availability with pt-snap skill list --json." @@ -175,6 +179,17 @@ def _import_json(result: ImportResult) -> dict[str, JsonValue]: ) +def _import_snapshot_result(options: ImportOptions, *, json_output: bool) -> ImportResult: + if not json_output: + return ImportService().import_snapshot(options) + previous_disable = logging.root.manager.disable + logging.disable(logging.CRITICAL) + try: + return ImportService().import_snapshot(options) + finally: + logging.disable(previous_disable) + + def _split_json(result: SplitResult) -> dict[str, JsonValue]: return json_success( output=str(result.output), @@ -422,14 +437,15 @@ def import_snapshot( ) -> None: """Import a PyTorch memory snapshot into a SQLite database.""" try: - result = ImportService().import_snapshot( + result = _import_snapshot_result( ImportOptions( snapshot_file=snapshot_file, output_dir=output_dir, device=device, set_focus=not no_focus, force=force, - ) + ), + json_output=json_output, ) except ( ImportToolMissingError, diff --git a/tests/skills/README.md b/tests/skills/README.md index 2dcdc2b..5bd1049 100644 --- a/tests/skills/README.md +++ b/tests/skills/README.md @@ -22,10 +22,15 @@ enforced by `harness/descriptors.py` with unknown fields rejected. declare `expect_output`, a mapping that must appear in the matched call's recorded output before the action counts as matched. Actions may also declare `status` (`success` or `error`; default `success`) so recovery cases can - require structured failures. Cases may cap the total call count with + require structured failures. Nested mappings and row lists in `expect_output` + are matched recursively. Cases may cap the total call count with `expected_tools.max_calls`; zero enforces refusal-only cases. - Descriptor paths are repository-relative and cannot contain `..`. Fixture - mount paths must be normalized absolute POSIX paths under `/fixtures/`. + mount paths must be unique, normalized absolute POSIX paths under `/fixtures/`. + `additional_fixtures` can mount more read-only synthetic SnapshotDBs when a + scenario depends on competing databases or devices. A semantic operation that + must publish an artifact may declare an explicit `writable_outputs` path under + `/outputs/`; it never makes project or fixture mounts writable. - Diagnostic fixtures are declarative SQLite databases. Pickle inputs are never materialized or exposed to a diagnostic runner. @@ -40,7 +45,9 @@ python -m tests.skills validate tests/skills/suites/pt-snap-agent-e2e/suite.yaml Agent adapters should expose only semantic operations through `RecordingToolGateway`. The gateway records successful, failed, and denied -attempts and enforces the suite allowlist and call budget. Adapters remain +attempts and enforces the suite allowlist and call budget. Adapters should +raise `StructuredToolError` when a failed CLI call still has a JSON envelope; +the gateway retains that envelope in the graded trace. Adapters remain responsible for process-level isolation: a fresh temporary working directory, an isolated `HOME`, no network, a read-only project, read-only fixture mounts, and cleared focus environment variables. @@ -97,7 +104,10 @@ score. scenarios plus sibling-database recovery when focus points at a missing file. It is an `agent-cli` evaluation contract, not a shipped skill. Recorded runs live under `baselines/pre-change/` (current CLI-only behavior) and -`baselines/target/` (the intended Agent-friendly contract). +`baselines/target/` (the post-change Agent-friendly contract). The comparison +requires higher task success, lower error-conclusion rate, and bounded mean +calls/output; recovery cases may use extra calls when a structured failure must +be corrected. Comparison metrics are task success rate, mean call count, mean output bytes, and error-conclusion rate: @@ -111,6 +121,9 @@ python -m tests.skills baseline \ Diagnostic fixtures remain declarative SQLite databases. The import scenario records `pt_snap.import` semantically and never materializes a pickle. +The release-gating clean-wheel check separately installs the built wheel into a +fresh venv, installs `pt-snap-helper` from package data, and completes import, +capability, overview, and peak-query discovery using JSON output. Generated transcripts and reports belong under `.skill-evals/`, which is ignored by Git. Normal `pytest` runs never invoke a live model. diff --git a/tests/skills/harness/__init__.py b/tests/skills/harness/__init__.py index a8c4ffc..6faa80c 100644 --- a/tests/skills/harness/__init__.py +++ b/tests/skills/harness/__init__.py @@ -1,7 +1,7 @@ """Descriptor, fixture, runner, and grading primitives for skill evaluations.""" from .descriptors import DescriptorError, EvalCase, EvalSuite, SandboxPolicy, load_suite -from .gateway import RecordingToolGateway, ToolDeniedError +from .gateway import RecordingToolGateway, StructuredToolError, ToolDeniedError from .grader import GradeResult, RunRecord, ToolCall, grade_run from .metrics import BaselineSummary, CaseMetrics, collect_case_metrics, summarize_metrics @@ -15,6 +15,7 @@ "RecordingToolGateway", "RunRecord", "SandboxPolicy", + "StructuredToolError", "ToolCall", "ToolDeniedError", "collect_case_metrics", diff --git a/tests/skills/harness/descriptors.py b/tests/skills/harness/descriptors.py index 9beba7a..f39ea67 100644 --- a/tests/skills/harness/descriptors.py +++ b/tests/skills/harness/descriptors.py @@ -61,6 +61,8 @@ class EvalCase: required_evidence: tuple[tuple[str, str], ...] mandatory_objectives: tuple[str, ...] max_tool_calls: int | None = None + additional_fixtures: tuple[dict[str, Any], ...] = () + writable_outputs: tuple[str, ...] = () @dataclass(frozen=True) @@ -184,6 +186,38 @@ def _validate_dag(action_ids: set[str], edges: tuple[tuple[str, str], ...], cont raise DescriptorError(f"{context} contains a cycle") +def _load_fixture( + raw_fixture: Any, + *, + path: Path, + suite_root: Path, + context: str, +) -> dict[str, Any]: + fixture = _strict_mapping( + raw_fixture, + context=context, + required={"builder", "definition", "mount_path", "read_only"}, + allowed={"builder", "definition", "mount_path", "read_only"}, + ) + if fixture["builder"] != "synthetic-snapshotdb": + raise DescriptorError(f"{path}: only synthetic-snapshotdb fixtures are supported") + definition = _safe_path(suite_root, fixture["definition"], f"{context}.definition") + if not definition.is_file(): + raise DescriptorError(f"{path}: fixture definition does not exist: {definition}") + if fixture["read_only"] is not True: + raise DescriptorError(f"{path}: diagnostic fixtures must be read-only") + mount_path = _string(fixture["mount_path"], f"{context}.mount_path") + if not mount_path.startswith("/"): + raise DescriptorError(f"{context}.mount_path must be an absolute POSIX path") + if ".." in PurePosixPath(mount_path).parts: + raise DescriptorError(f"{context}.mount_path cannot contain '..'") + if posixpath.normpath(mount_path) != mount_path: + raise DescriptorError(f"{context}.mount_path must be a normalized POSIX path") + if not mount_path.startswith("/fixtures/"): + raise DescriptorError(f"{context}.mount_path must be under /fixtures/") + return {**fixture, "definition_path": definition} + + def _load_case( path: Path, *, @@ -215,6 +249,8 @@ def _load_case( "covers", "task", "fixture", + "additional_fixtures", + "writable_outputs", "dialog", "expected_tools", "oracle", @@ -383,33 +419,45 @@ def _load_case( f"{path}: unknown objectives: {', '.join(sorted(unknown_objectives))}" ) - fixture: dict[str, Any] | None = None - if "fixture" in data: - raw_fixture = _strict_mapping( + fixture = ( + _load_fixture( data["fixture"], + path=path, + suite_root=suite_root, context=f"{path}: fixture", - required={"builder", "definition", "mount_path", "read_only"}, - allowed={"builder", "definition", "mount_path", "read_only"}, - ) - if raw_fixture["builder"] != "synthetic-snapshotdb": - raise DescriptorError(f"{path}: only synthetic-snapshotdb fixtures are supported") - definition = _safe_path( - suite_root, raw_fixture["definition"], f"{path}: fixture.definition" ) - if not definition.is_file(): - raise DescriptorError(f"{path}: fixture definition does not exist: {definition}") - if raw_fixture["read_only"] is not True: - raise DescriptorError(f"{path}: diagnostic fixtures must be read-only") - mount_path = _string(raw_fixture["mount_path"], f"{path}: fixture.mount_path") - if not mount_path.startswith("/"): - raise DescriptorError(f"{path}: fixture.mount_path must be an absolute POSIX path") - if ".." in PurePosixPath(mount_path).parts: - raise DescriptorError(f"{path}: fixture.mount_path cannot contain '..'") - if posixpath.normpath(mount_path) != mount_path: - raise DescriptorError(f"{path}: fixture.mount_path must be a normalized POSIX path") - if not mount_path.startswith("/fixtures/"): - raise DescriptorError(f"{path}: fixture.mount_path must be under /fixtures/") - fixture = {**raw_fixture, "definition_path": definition} + if "fixture" in data + else None + ) + additional_fixtures: list[dict[str, Any]] = [] + if "additional_fixtures" in data: + raw_additional_fixtures = data["additional_fixtures"] + if not isinstance(raw_additional_fixtures, list): + raise DescriptorError(f"{path}: additional_fixtures must be a list") + for index, raw_fixture in enumerate(raw_additional_fixtures): + additional_fixtures.append( + _load_fixture( + raw_fixture, + path=path, + suite_root=suite_root, + context=f"{path}: additional_fixtures[{index}]", + ) + ) + mount_paths = [item["mount_path"] for item in additional_fixtures] + if fixture is not None: + mount_paths.append(fixture["mount_path"]) + if len(mount_paths) != len(set(mount_paths)): + raise DescriptorError(f"{path}: fixture mount paths must be unique") + writable_outputs = _string_list(data.get("writable_outputs", []), f"{path}: writable_outputs") + for output_path in writable_outputs: + if not output_path.startswith("/"): + raise DescriptorError(f"{path}: writable output must be an absolute POSIX path") + if ".." in PurePosixPath(output_path).parts: + raise DescriptorError(f"{path}: writable output cannot contain '..'") + if posixpath.normpath(output_path) != output_path: + raise DescriptorError(f"{path}: writable output must be a normalized POSIX path") + if output_path != "/outputs" and not output_path.startswith("/outputs/"): + raise DescriptorError(f"{path}: writable output must be under /outputs/") return EvalCase( id=_string(data["id"], f"{path}: id"), @@ -429,6 +477,8 @@ def _load_case( required_evidence=tuple(required_evidence), mandatory_objectives=mandatory_objectives, max_tool_calls=max_tool_calls, + additional_fixtures=tuple(additional_fixtures), + writable_outputs=writable_outputs, ) diff --git a/tests/skills/harness/fixtures.py b/tests/skills/harness/fixtures.py index 2f93090..c008c0a 100644 --- a/tests/skills/harness/fixtures.py +++ b/tests/skills/harness/fixtures.py @@ -57,8 +57,8 @@ def load_fixture_definition(path: Path) -> dict[str, Any]: raise DescriptorError(f"{path} has invalid fixture fields") if data["schema_version"] != 1 or data["kind"] != "synthetic-snapshotdb": raise DescriptorError(f"{path} must declare synthetic-snapshotdb schema version 1") - if not isinstance(data["devices"], dict) or not data["devices"]: - raise DescriptorError(f"{path}: devices must be a non-empty mapping") + if not isinstance(data["devices"], dict): + raise DescriptorError(f"{path}: devices must be a mapping") for device_id, device in data["devices"].items(): if not str(device_id).isdigit() or not isinstance(device, dict): raise DescriptorError(f"{path}: device IDs must be non-negative integers") diff --git a/tests/skills/harness/gateway.py b/tests/skills/harness/gateway.py index 3ae19f2..fcd131b 100644 --- a/tests/skills/harness/gateway.py +++ b/tests/skills/harness/gateway.py @@ -12,6 +12,12 @@ class ToolDeniedError(PermissionError): """Raised after a disallowed operation is recorded in the run trace.""" +class StructuredToolError(RuntimeError): + def __init__(self, message: str, output: Any) -> None: + super().__init__(message) + self.output = output + + ToolExecutor = Callable[[str, dict[str, Any]], Any] @@ -47,6 +53,18 @@ def call(self, operation: str, arguments: dict[str, Any]) -> Any: try: output = self.executor(operation, dict(arguments)) + except StructuredToolError as exc: + self._calls.append( + ToolCall( + call_id, + operation, + dict(arguments), + status="error", + output=exc.output, + error=f"{type(exc).__name__}: {exc}", + ) + ) + raise except Exception as exc: self._calls.append( ToolCall( diff --git a/tests/skills/harness/grader.py b/tests/skills/harness/grader.py index f84ca41..0c21df1 100644 --- a/tests/skills/harness/grader.py +++ b/tests/skills/harness/grader.py @@ -88,13 +88,30 @@ def _is_subset(expected: dict[str, Any], actual: dict[str, Any]) -> bool: return all(key in actual and actual[key] == value for key, value in expected.items()) +def _value_contains(expected: Any, actual: Any) -> bool: + if isinstance(expected, dict): + return isinstance(actual, dict) and all( + key in actual and _value_contains(value, actual[key]) for key, value in expected.items() + ) + if isinstance(expected, list): + return ( + isinstance(actual, list) + and len(expected) <= len(actual) + and all( + _value_contains(item, candidate) + for item, candidate in zip(expected, actual, strict=False) + ) + ) + return expected == actual + + def _output_contains(expected: dict[str, Any], output: Any) -> bool: if not expected: return True - if isinstance(output, dict): - return _is_subset(expected, output) + if _value_contains(expected, output): + return True if isinstance(output, list): - return any(_output_contains(expected, item) for item in output) + return any(_value_contains(expected, item) for item in output) return False diff --git a/tests/skills/schemas/case-v1.yaml b/tests/skills/schemas/case-v1.yaml index 0e433a3..f849edf 100644 --- a/tests/skills/schemas/case-v1.yaml +++ b/tests/skills/schemas/case-v1.yaml @@ -20,6 +20,11 @@ fields: builder_values: [synthetic-snapshotdb] mount_path_rule: normalized-absolute-posix-path-under-/fixtures/ read_only: true + additional_fixtures: + item_required: [builder, definition, mount_path, read_only] + rule: optional-distinct-read-only-synthetic-snapshotdb-mounts + writable_outputs: + rule: optional-distinct-normalized-absolute-paths-under-/outputs/ expected_tools: required: [actions, partial_order, forbidden] optional: [max_calls] diff --git a/tests/skills/suites/pt-snap-agent-e2e/AGENT_CONTRACT.md b/tests/skills/suites/pt-snap-agent-e2e/AGENT_CONTRACT.md index 609e2fc..f97a6f6 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/AGENT_CONTRACT.md +++ b/tests/skills/suites/pt-snap-agent-e2e/AGENT_CONTRACT.md @@ -8,16 +8,17 @@ description: > # Agent CLI end-to-end evaluation -Adapters should expose semantic operations, not raw argv. Record structured -outputs even when the current CLI still prints text. Target fields: +Adapters should expose semantic operations, not raw argv. Every machine-readable +call must request JSON and preserve the complete success or error envelope. +Target fields: | Operation | Normalized output | | --- | --- | -| `pt_snap.help` | `{mentions_helper: true, helper_skill: "pt-snap-helper"}` when help points at the helper skill | +| `pt_snap.help` | `{mentions_helper: true, helper_skill: "pt-snap-helper", install_command, restart_required}` when help provides helper onboarding | | `pt_snap.skill_list` | JSON listing; `json: true` in arguments | | failed `pt_snap.query` | `{error_code: TEMPLATE_NOT_FOUND\|INVALID_PARAMETER\|DATABASE_NOT_FOUND\|DEVICE_NOT_FOUND}` | | successful `pt_snap.query` | envelope with `db_path`, `device_id`, `returned`, `has_more`, and `rows` | -| `pt_snap.import` | `{db_path: "", focus_updated: true\|false}` | +| `pt_snap.import` | `{db_path: "", focus_state, focus_source}`; non-null `focus_state` means focus changed | Do not materialize pickle fixtures for these cases. Import is a semantic operation; `pickle.access` remains forbidden. diff --git a/tests/skills/suites/pt-snap-agent-e2e/baselines/pre-change/no-device.json b/tests/skills/suites/pt-snap-agent-e2e/baselines/pre-change/no-device.json new file mode 100644 index 0000000..bd93dcc --- /dev/null +++ b/tests/skills/suites/pt-snap-agent-e2e/baselines/pre-change/no-device.json @@ -0,0 +1,31 @@ +{ + "tool_calls": [ + { + "id": "call-1", + "operation": "pt_snap.query", + "arguments": { + "database": "/fixtures/no-devices.db", + "template": "leak_detection", + "json": true + }, + "status": "error", + "output": null, + "error": "Error: no device tables" + } + ], + "result": { + "classification": "empty-success", + "facts": { + "diagnosis_performed": false, + "error_code": "DEVICE_NOT_FOUND" + }, + "claims": [ + { + "id": "error_code", + "evidence_call_ids": ["call-1"] + } + ], + "unknowns": ["analyzable SnapshotDB device"] + }, + "final_response": "No leak found because the database returned no blocks." +} diff --git a/tests/skills/suites/pt-snap-agent-e2e/baselines/pre-change/wrong-parameter-recovery.json b/tests/skills/suites/pt-snap-agent-e2e/baselines/pre-change/wrong-parameter-recovery.json new file mode 100644 index 0000000..a50c400 --- /dev/null +++ b/tests/skills/suites/pt-snap-agent-e2e/baselines/pre-change/wrong-parameter-recovery.json @@ -0,0 +1,45 @@ +{ + "tool_calls": [ + { + "id": "call-1", + "operation": "pt_snap.query", + "arguments": { + "database": "/fixtures/events.db", + "device": 0, + "template": "leak_detection", + "params": { + "min_sze": 32 + }, + "json": true + }, + "status": "error", + "output": null, + "error": "Error: unknown template parameter min_sze" + } + ], + "result": { + "classification": "inconclusive", + "facts": { + "template": "leak_detection", + "min_size": 32, + "returned": 3, + "error_code": "INVALID_PARAMETER" + }, + "claims": [ + { + "id": "error_code", + "evidence_call_ids": ["call-1"] + }, + { + "id": "min_size", + "evidence_call_ids": ["call-1"] + }, + { + "id": "returned", + "evidence_call_ids": ["call-1"] + } + ], + "unknowns": [] + }, + "final_response": "The parameter error was unstructured and the corrected query was not retried." +} diff --git a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/cli-only-help-discovery.json b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/cli-only-help-discovery.json index 0860fb8..62ae01a 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/cli-only-help-discovery.json +++ b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/cli-only-help-discovery.json @@ -7,7 +7,9 @@ "status": "success", "output": { "mentions_helper": true, - "helper_skill": "pt-snap-helper" + "helper_skill": "pt-snap-helper", + "install_command": "pt-snap skill install pt-snap-helper --json", + "restart_required": true } }, { diff --git a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/import-success.json b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/import-success.json index a1a688e..afe0c23 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/import-success.json +++ b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/import-success.json @@ -8,8 +8,13 @@ }, "status": "success", "output": { - "db_path": "/fixtures/imported.db", - "focus_updated": true, + "db_path": "/outputs/imported.db", + "focus_source": "project", + "focus_state": { + "configured": true, + "db_path": "/outputs/imported.db", + "focus_source": "project" + }, "reused": false } } @@ -17,7 +22,7 @@ "result": { "classification": "import-complete", "facts": { - "db_path": "/fixtures/imported.db", + "db_path": "/outputs/imported.db", "focus_updated": true }, "claims": [ @@ -32,5 +37,5 @@ ], "unknowns": [] }, - "final_response": "Imported database published at /fixtures/imported.db and project focus was updated." + "final_response": "Import published /outputs/imported.db; the non-null project focus_state reports that focus was updated." } diff --git a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/live-blocks-unconfirmed.json b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/live-blocks-unconfirmed.json index 1fcb2d4..30bc445 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/live-blocks-unconfirmed.json +++ b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/live-blocks-unconfirmed.json @@ -2,6 +2,23 @@ "tool_calls": [ { "id": "call-1", + "operation": "pt_snap.template_info", + "arguments": { + "template": "leak_detection", + "json": true + }, + "status": "success", + "output": { + "template": "leak_detection", + "semantics_version": 1, + "interpretation_limits": [ + "Rows are candidates still without a free-completion event in the captured range; end-of-trace survival is not a confirmed leak.", + "Event IDs and ID distances are ordering markers, not elapsed time." + ] + } + }, + { + "id": "call-2", "operation": "pt_snap.query", "arguments": { "database": "/fixtures/live-blocks.db", @@ -10,29 +27,40 @@ "json": true }, "status": "success", - "output": [ - { - "id": 1, - "address": 8192, - "size": 4096, - "allocEventId": 1 - } - ] + "output": { + "db_path": "/fixtures/live-blocks.db", + "device_id": 0, + "returned": 1, + "has_more": false, + "rows": [ + { + "id": 1, + "address": 8192, + "size": 4096, + "allocEventId": 1 + } + ] + } } ], "result": { "classification": "unconfirmed-retention", "facts": { "device_id": 0, + "semantics_version": 1, "end_live_candidate_bytes": 4096 }, "claims": [ { - "id": "end_live_candidate_bytes", + "id": "semantics_version", "evidence_call_ids": ["call-1"] + }, + { + "id": "end_live_candidate_bytes", + "evidence_call_ids": ["call-2"] } ], "unknowns": ["whether the live block will be freed after the trace"] }, - "final_response": "A 4096-byte block is still live at the end of the trace. That is candidate retention evidence, not proof that the allocation will remain." + "final_response": "The leak_detection semantics limit and query envelope show a 4096-byte candidate still live at the end of the trace, not proof that the allocation will remain." } diff --git a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/missing-database.json b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/missing-database.json index 8364b95..14e4169 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/missing-database.json +++ b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/missing-database.json @@ -6,7 +6,8 @@ "arguments": { "database": "/fixtures/missing.db", "device": 0, - "template": "leak_detection" + "template": "leak_detection", + "json": true }, "status": "error", "output": { diff --git a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/no-device.json b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/no-device.json new file mode 100644 index 0000000..3792dff --- /dev/null +++ b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/no-device.json @@ -0,0 +1,34 @@ +{ + "tool_calls": [ + { + "id": "call-1", + "operation": "pt_snap.query", + "arguments": { + "database": "/fixtures/no-devices.db", + "template": "leak_detection", + "json": true + }, + "status": "error", + "output": { + "error_code": "DEVICE_NOT_FOUND", + "hint": "Pass --device with an id present in the database, or omit it to use the first device." + }, + "error": "DEVICE_NOT_FOUND: no devices" + } + ], + "result": { + "classification": "unavailable", + "facts": { + "diagnosis_performed": false, + "error_code": "DEVICE_NOT_FOUND" + }, + "claims": [ + { + "id": "error_code", + "evidence_call_ids": ["call-1"] + } + ], + "unknowns": ["analyzable SnapshotDB device"] + }, + "final_response": "The database contains no discoverable device, so DEVICE_NOT_FOUND is a failed lookup rather than an empty leak result." +} diff --git a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/wrong-parameter-recovery.json b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/wrong-parameter-recovery.json new file mode 100644 index 0000000..6606242 --- /dev/null +++ b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/wrong-parameter-recovery.json @@ -0,0 +1,73 @@ +{ + "tool_calls": [ + { + "id": "call-1", + "operation": "pt_snap.query", + "arguments": { + "database": "/fixtures/events.db", + "device": 0, + "template": "leak_detection", + "params": { + "min_sze": 32 + }, + "json": true + }, + "status": "error", + "output": { + "error_code": "INVALID_PARAMETER", + "hint": "Check --params against 'pt-snap query --template-info leak_detection --json'." + }, + "error": "INVALID_PARAMETER: unknown parameter min_sze" + }, + { + "id": "call-2", + "operation": "pt_snap.query", + "arguments": { + "database": "/fixtures/events.db", + "device": 0, + "template": "leak_detection", + "params": { + "min_size": 32 + }, + "json": true + }, + "status": "success", + "output": { + "db_path": "/fixtures/events.db", + "device_id": 0, + "returned": 3, + "has_more": false, + "rows": [ + {"id": 1, "size": 64}, + {"id": 2, "size": 48}, + {"id": 3, "size": 32} + ] + } + } + ], + "result": { + "classification": "recovered", + "facts": { + "template": "leak_detection", + "min_size": 32, + "returned": 3, + "error_code": "INVALID_PARAMETER" + }, + "claims": [ + { + "id": "error_code", + "evidence_call_ids": ["call-1"] + }, + { + "id": "min_size", + "evidence_call_ids": ["call-2"] + }, + { + "id": "returned", + "evidence_call_ids": ["call-2"] + } + ], + "unknowns": [] + }, + "final_response": "The undeclared min_sze parameter returned INVALID_PARAMETER; min_size=32 succeeded with three rows." +} diff --git a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/wrong-template-recovery.json b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/wrong-template-recovery.json index 232e36d..2b17d92 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/baselines/target/wrong-template-recovery.json +++ b/tests/skills/suites/pt-snap-agent-e2e/baselines/target/wrong-template-recovery.json @@ -6,7 +6,8 @@ "arguments": { "database": "/fixtures/events.db", "device": 0, - "template": "leak_detecton" + "template": "leak_detecton", + "json": true }, "status": "error", "output": { diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/cli-only-help-discovery.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/cli-only-help-discovery.yaml index 4cca6d9..ff86cc4 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/cli-only-help-discovery.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/cli-only-help-discovery.yaml @@ -15,7 +15,11 @@ expected_tools: - id: read_help operation: pt_snap.help match: {} - expect_output: {mentions_helper: true, helper_skill: pt-snap-helper} + expect_output: + mentions_helper: true + helper_skill: pt-snap-helper + install_command: pt-snap skill install pt-snap-helper --json + restart_required: true - id: list_skills operation: pt_snap.skill_list match: {json: true} diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/explicit-multi-target.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/explicit-multi-target.yaml index daf18a7..ea51689 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/explicit-multi-target.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/explicit-multi-target.yaml @@ -15,6 +15,11 @@ fixture: definition: fixtures/multi-device.yaml mount_path: /fixtures/target.db read_only: true +additional_fixtures: + - builder: synthetic-snapshotdb + definition: fixtures/events.yaml + mount_path: /fixtures/other.db + read_only: true expected_tools: actions: diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/import-success.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/import-success.yaml index ac02148..b2748ed 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/import-success.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/import-success.yaml @@ -4,6 +4,7 @@ id: import-success title: Successful import reports the published database and focus change tags: [import, json, focus] covers: [workflow.import] +writable_outputs: [/outputs] task: initial_user: > @@ -17,15 +18,19 @@ expected_tools: operation: pt_snap.import match: {json: true} expect_output: - db_path: /fixtures/imported.db - focus_updated: true + db_path: /outputs/imported.db + focus_source: project + focus_state: + configured: true + db_path: /outputs/imported.db + focus_source: project partial_order: [] forbidden: - {operation: pickle.access} oracle: facts: - db_path: /fixtures/imported.db + db_path: /outputs/imported.db focus_updated: true result: allowed_classifications: [import-complete] diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/live-blocks-unconfirmed.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/live-blocks-unconfirmed.yaml index 14f9aed..5b2733a 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/live-blocks-unconfirmed.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/live-blocks-unconfirmed.yaml @@ -18,6 +18,14 @@ fixture: expected_tools: actions: + - id: inspect_template_contract + operation: pt_snap.template_info + match: {template: leak_detection, json: true} + expect_output: + template: leak_detection + semantics_version: 1 + interpretation_limits: + - "Rows are candidates still without a free-completion event in the captured range; end-of-trace survival is not a confirmed leak." - id: inspect_candidates operation: pt_snap.query match: @@ -25,14 +33,20 @@ expected_tools: device: 0 template: leak_detection json: true - expect_output: {size: 4096} - partial_order: [] + expect_output: + db_path: /fixtures/live-blocks.db + device_id: 0 + rows: + - {id: 1, size: 4096, allocEventId: 1} + partial_order: + - {before: inspect_template_contract, after: inspect_candidates} forbidden: - {operation: pt_snap.import} oracle: facts: device_id: 0 + semantics_version: 1 end_live_candidate_bytes: 4096 result: allowed_classifications: [unconfirmed-retention] @@ -42,6 +56,7 @@ oracle: - this snapshot proves a leak - the live block is a confirmed leak required_evidence: + - {claim: semantics_version, produced_by: inspect_template_contract} - {claim: end_live_candidate_bytes, produced_by: inspect_candidates} scoring: diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/missing-database.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/missing-database.yaml index 60b10e5..5729b8e 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/missing-database.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/missing-database.yaml @@ -15,7 +15,7 @@ expected_tools: - id: observe_missing_database operation: pt_snap.query status: error - match: {database: /fixtures/missing.db, device: 0, template: leak_detection} + match: {database: /fixtures/missing.db, device: 0, template: leak_detection, json: true} expect_output: {error_code: DATABASE_NOT_FOUND} partial_order: [] forbidden: diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/no-device.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/no-device.yaml new file mode 100644 index 0000000..2285c3c --- /dev/null +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/no-device.yaml @@ -0,0 +1,51 @@ +schema_version: 1 +kind: skill-eval-case +id: no-device +title: A database without discoverable devices is a failure, not an empty result +tags: [recovery, no-device, errors] +covers: [recovery.no_device] + +task: + initial_user: > + Query leak_detection from /fixtures/no-devices.db without specifying a + device and report whether any leak was found. + +fixture: + builder: synthetic-snapshotdb + definition: fixtures/no-devices.yaml + mount_path: /fixtures/no-devices.db + read_only: true + +expected_tools: + actions: + - id: observe_no_device + operation: pt_snap.query + status: error + match: {database: /fixtures/no-devices.db, template: leak_detection, json: true} + expect_output: {error_code: DEVICE_NOT_FOUND} + partial_order: [] + forbidden: + - {operation: pt_snap.import} + +oracle: + facts: + diagnosis_performed: false + error_code: DEVICE_NOT_FOUND + result: + allowed_classifications: [unavailable] + prohibited_classifications: [empty-success, confirmed leak, unconfirmed-retention] + required_unknowns: [analyzable SnapshotDB device] + prohibited_claims: + - no leak found + - no blocks were returned + required_evidence: + - {claim: error_code, produced_by: observe_no_device} + +scoring: + mandatory_objectives: + - safety.policy + - tools.correct_path + - evidence.factual_accuracy + - evidence.traceability + - diagnosis.classification + - diagnosis.uncertainty diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-parameter-recovery.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-parameter-recovery.yaml new file mode 100644 index 0000000..e8bf01c --- /dev/null +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-parameter-recovery.yaml @@ -0,0 +1,73 @@ +schema_version: 1 +kind: skill-eval-case +id: wrong-parameter-recovery +title: Agent corrects an undeclared parameter using a structured error code +tags: [recovery, json-error, parameters] +covers: [recovery.invalid_parameter] + +task: + initial_user: > + Query leak_detection on /fixtures/events.db device 0 with a minimum size of + 32 bytes, but I typed the parameter as min_sze. If the call is invalid, + fix it from the structured error and retry. + +fixture: + builder: synthetic-snapshotdb + definition: fixtures/events.yaml + mount_path: /fixtures/events.db + read_only: true + +expected_tools: + actions: + - id: mistyped_parameter_query + operation: pt_snap.query + status: error + match: + database: /fixtures/events.db + device: 0 + template: leak_detection + params: {min_sze: 32} + json: true + expect_output: {error_code: INVALID_PARAMETER} + - id: corrected_parameter_query + operation: pt_snap.query + match: + database: /fixtures/events.db + device: 0 + template: leak_detection + params: {min_size: 32} + json: true + expect_output: + db_path: /fixtures/events.db + device_id: 0 + returned: 3 + has_more: false + partial_order: + - {before: mistyped_parameter_query, after: corrected_parameter_query} + forbidden: + - {operation: pt_snap.import} + +oracle: + facts: + template: leak_detection + min_size: 32 + returned: 3 + error_code: INVALID_PARAMETER + result: + allowed_classifications: [recovered] + prohibited_classifications: [empty-success, confirmed leak] + required_unknowns: [] + prohibited_claims: + - the undeclared parameter was accepted + required_evidence: + - {claim: error_code, produced_by: mistyped_parameter_query} + - {claim: min_size, produced_by: corrected_parameter_query} + - {claim: returned, produced_by: corrected_parameter_query} + +scoring: + mandatory_objectives: + - safety.policy + - tools.correct_path + - evidence.factual_accuracy + - evidence.traceability + - diagnosis.classification diff --git a/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-template-recovery.yaml b/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-template-recovery.yaml index 8c0c598..9e3cd6a 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-template-recovery.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/cases/wrong-template-recovery.yaml @@ -21,7 +21,7 @@ expected_tools: - id: mistyped_query operation: pt_snap.query status: error - match: {database: /fixtures/events.db, device: 0, template: leak_detecton} + match: {database: /fixtures/events.db, device: 0, template: leak_detecton, json: true} expect_output: {error_code: TEMPLATE_NOT_FOUND} - id: corrected_query operation: pt_snap.query diff --git a/tests/skills/suites/pt-snap-agent-e2e/fixtures/no-devices.yaml b/tests/skills/suites/pt-snap-agent-e2e/fixtures/no-devices.yaml new file mode 100644 index 0000000..93af67b --- /dev/null +++ b/tests/skills/suites/pt-snap-agent-e2e/fixtures/no-devices.yaml @@ -0,0 +1,3 @@ +schema_version: 1 +kind: synthetic-snapshotdb +devices: {} diff --git a/tests/skills/suites/pt-snap-agent-e2e/suite.yaml b/tests/skills/suites/pt-snap-agent-e2e/suite.yaml index f9f0f75..f2ae314 100644 --- a/tests/skills/suites/pt-snap-agent-e2e/suite.yaml +++ b/tests/skills/suites/pt-snap-agent-e2e/suite.yaml @@ -6,10 +6,11 @@ metadata: title: Agent CLI end-to-end evaluation description: > Cross-cutting Agent scenarios from issue #136: discover CLI guidance, - recover from structured errors, honor an explicit database and device, - notice truncated evidence, avoid leak overclaims, treat missing targets - as failures, and read import results from structured output. Also covers - missing-focus sibling-database recovery from issue #151. + recover from structured template and parameter errors, honor explicit + database and device targets, notice truncated evidence, avoid leak + overclaims, treat missing databases and devices as failures, and read + import results from structured output. Also covers missing-focus sibling + database recovery from issue #151. skill: path: tests/skills/suites/pt-snap-agent-e2e/AGENT_CONTRACT.md @@ -59,10 +60,12 @@ coverage: required_branches: - capability.cli_only - recovery.invalid_call + - recovery.invalid_parameter - target.explicit - evidence.completeness - interpretation.live_blocks - recovery.missing_target + - recovery.no_device - recovery.sibling_target - workflow.import @@ -80,6 +83,8 @@ defaults: allowed: - pt_snap.help - pt_snap.skill_list + - pt_snap.capabilities + - pt_snap.overview - pt_snap.metadata - pt_snap.template_info - pt_snap.query @@ -103,6 +108,8 @@ cases: path: cases/cli-only-help-discovery.yaml - id: wrong-template-recovery path: cases/wrong-template-recovery.yaml + - id: wrong-parameter-recovery + path: cases/wrong-parameter-recovery.yaml - id: explicit-multi-target path: cases/explicit-multi-target.yaml - id: truncated-query @@ -111,6 +118,8 @@ cases: path: cases/live-blocks-unconfirmed.yaml - id: missing-database path: cases/missing-database.yaml + - id: no-device + path: cases/no-device.yaml - id: sibling-database-recovery path: cases/sibling-database-recovery.yaml - id: import-success diff --git a/tests/skills/test_agent_e2e.py b/tests/skills/test_agent_e2e.py index 4ddd7ea..6e09250 100644 --- a/tests/skills/test_agent_e2e.py +++ b/tests/skills/test_agent_e2e.py @@ -17,10 +17,12 @@ CASE_IDS = ( "cli-only-help-discovery", "wrong-template-recovery", + "wrong-parameter-recovery", "explicit-multi-target", "truncated-query", "live-blocks-unconfirmed", "missing-database", + "no-device", "sibling-database-recovery", "import-success", ) @@ -51,7 +53,16 @@ def test_agent_e2e_suite_covers_issue_136_scenarios() -> None: branch for case in suite.cases for branch in case.covers } assert "pt_snap.import" in suite.allowed_operations + assert "pt_snap.capabilities" in suite.allowed_operations + assert "pt_snap.overview" in suite.allowed_operations assert "pickle.access" in suite.forbidden_operations + multi_target = suite.case("explicit-multi-target") + assert multi_target.fixture is not None + assert multi_target.fixture["mount_path"] == "/fixtures/target.db" + assert [item["mount_path"] for item in multi_target.additional_fixtures] == [ + "/fixtures/other.db" + ] + assert suite.case("import-success").writable_outputs == ("/outputs",) def test_target_baseline_is_a_perfect_score() -> None: @@ -69,15 +80,28 @@ def test_pre_change_baseline_records_current_gaps() -> None: assert summary.task_success_rate == 0.0 assert summary.passed_count == 0 - assert summary.error_conclusion_rate == 0.875 + assert summary.error_conclusion_rate == 0.8 assert summary.mean_call_count == 1.0 assert all(not grade.passed for grade, _metrics in by_id.values()) assert by_id["wrong-template-recovery"][1].error_conclusion is False + assert by_id["wrong-parameter-recovery"][1].error_conclusion is False assert by_id["live-blocks-unconfirmed"][1].error_conclusion is True assert by_id["missing-database"][1].error_conclusion is True + assert by_id["no-device"][1].error_conclusion is True assert by_id["sibling-database-recovery"][1].error_conclusion is True +def test_post_change_quality_improves_and_efficiency_stays_bounded() -> None: + suite, rows, target = _summarize(TARGET_BASELINE) + _pre_suite, _pre_rows, pre_change = _summarize(PRE_CHANGE_BASELINE) + + assert target.task_success_rate > pre_change.task_success_rate + assert target.error_conclusion_rate < pre_change.error_conclusion_rate + assert target.mean_call_count <= 1.5 + assert target.mean_output_bytes <= 4096 + assert all(metrics.call_count <= suite.max_calls for _case, _run, _grade, metrics in rows) + + def test_error_status_actions_match_structured_failures() -> None: suite = load_suite(SUITE_PATH) case = suite.case("wrong-template-recovery") @@ -126,7 +150,7 @@ def test_baseline_cli_writes_metrics(tmp_path: Path, monkeypatch, capsys) -> Non printed = json.loads(capsys.readouterr().out) assert payload == printed assert payload["task_success_rate"] == 0.0 - assert payload["error_conclusion_rate"] == 0.875 + assert payload["error_conclusion_rate"] == 0.8 assert payload["mean_output_bytes"] > 0 case_bytes = [case["output_bytes"] for case in payload["cases"]] assert payload["mean_output_bytes"] == round(sum(case_bytes) / len(case_bytes), 4) diff --git a/tests/skills/test_descriptors.py b/tests/skills/test_descriptors.py index e96ad34..fe70af2 100644 --- a/tests/skills/test_descriptors.py +++ b/tests/skills/test_descriptors.py @@ -66,6 +66,30 @@ def test_case_rejects_mount_path_traversal(tmp_path: Path) -> None: load_suite(copied_suite / "suite.yaml") +def test_case_rejects_duplicate_additional_fixture_mount(tmp_path: Path) -> None: + copied_suite = tmp_path / "suite" + copytree(SUITE_DIRECTORY, copied_suite) + case_path = copied_suite / "cases" / "allocator-cache.yaml" + data = yaml.safe_load(case_path.read_text()) + data["additional_fixtures"] = [data["fixture"]] + case_path.write_text(yaml.safe_dump(data, sort_keys=False)) + + with pytest.raises(DescriptorError, match="fixture mount paths must be unique"): + load_suite(copied_suite / "suite.yaml") + + +def test_case_rejects_writable_output_outside_output_root(tmp_path: Path) -> None: + copied_suite = tmp_path / "suite" + copytree(SUITE_DIRECTORY, copied_suite) + case_path = copied_suite / "cases" / "allocator-cache.yaml" + data = yaml.safe_load(case_path.read_text()) + data["writable_outputs"] = ["/fixtures/result.db"] + case_path.write_text(yaml.safe_dump(data, sort_keys=False)) + + with pytest.raises(DescriptorError, match="writable output must be under /outputs/"): + load_suite(copied_suite / "suite.yaml") + + def test_case_rejects_unnormalized_mount_path(tmp_path: Path) -> None: copied_suite = tmp_path / "suite" copytree(SUITE_DIRECTORY, copied_suite) diff --git a/tests/skills/test_fixtures.py b/tests/skills/test_fixtures.py index 2644353..81f9859 100644 --- a/tests/skills/test_fixtures.py +++ b/tests/skills/test_fixtures.py @@ -6,6 +6,7 @@ from pt_snap_cli.context import Context from pt_snap_cli.query.executor import QueryExecutor from pt_snap_cli.query.registry import QueryRegistry, _load_all_templates +from tests.skills.harness.descriptors import load_suite from tests.skills.harness.fixtures import build_snapshotdb, load_fixture_definition FIXTURE_DIRECTORY = Path("tests/skills/suites/pt-snap-memory-leak/fixtures") @@ -79,6 +80,30 @@ def test_agent_e2e_definitions_build_read_only_snapshotdbs( assert Context(output_path).device_ids[0] == 0 +def test_multi_target_case_materializes_competing_databases(tmp_path: Path) -> None: + suite = load_suite(Path("tests/skills/suites/pt-snap-agent-e2e/suite.yaml")) + case = suite.case("explicit-multi-target") + assert case.fixture is not None + fixtures = (case.fixture, *case.additional_fixtures) + + outputs = [ + build_snapshotdb(fixture["definition_path"], tmp_path / Path(fixture["mount_path"]).name) + for fixture in fixtures + ] + + assert [output.name for output in outputs] == ["target.db", "other.db"] + assert [Context(output).device_ids for output in outputs] == [[0, 1], [0]] + + +def test_no_device_fixture_builds_a_valid_read_only_snapshotdb(tmp_path: Path) -> None: + output_path = tmp_path / "no-devices.db" + + build_snapshotdb(E2E_FIXTURE_DIRECTORY / "no-devices.yaml", output_path) + + assert not output_path.stat().st_mode & stat.S_IWUSR + assert Context(output_path).device_ids == [] + + def test_events_fixture_has_four_trace_rows(tmp_path: Path) -> None: output_path = tmp_path / "events.db" build_snapshotdb(E2E_FIXTURE_DIRECTORY / "events.yaml", output_path) diff --git a/tests/skills/test_gateway.py b/tests/skills/test_gateway.py index 0b57b39..59f9844 100644 --- a/tests/skills/test_gateway.py +++ b/tests/skills/test_gateway.py @@ -3,7 +3,11 @@ import pytest from tests.skills.harness.descriptors import load_suite -from tests.skills.harness.gateway import RecordingToolGateway, ToolDeniedError +from tests.skills.harness.gateway import ( + RecordingToolGateway, + StructuredToolError, + ToolDeniedError, +) SUITE_PATH = Path("tests/skills/suites/pt-snap-memory-leak/suite.yaml") @@ -30,6 +34,28 @@ def executor(operation: str, arguments: dict): assert gateway.calls[1].error == "RuntimeError: query failed" +def test_gateway_records_structured_error_output() -> None: + suite = load_suite(SUITE_PATH) + + def executor(operation: str, arguments: dict): + raise StructuredToolError( + "query failed", + {"error_code": "INVALID_PARAMETER", "hint": "Check parameters."}, + ) + + gateway = RecordingToolGateway(suite, executor) + + with pytest.raises(StructuredToolError, match="query failed"): + gateway.call("pt_snap.query", {"json": True}) + + assert gateway.calls[0].status == "error" + assert gateway.calls[0].output == { + "error_code": "INVALID_PARAMETER", + "hint": "Check parameters.", + } + assert gateway.calls[0].error == "StructuredToolError: query failed" + + def test_gateway_records_and_denies_forbidden_operations() -> None: suite = load_suite(SUITE_PATH) gateway = RecordingToolGateway(suite, lambda operation, arguments: None) diff --git a/tests/skills/test_grader.py b/tests/skills/test_grader.py index dc4c025..fcbfe5a 100644 --- a/tests/skills/test_grader.py +++ b/tests/skills/test_grader.py @@ -2,11 +2,24 @@ from tests.skills.harness.artifacts import write_run_artifacts from tests.skills.harness.descriptors import load_suite -from tests.skills.harness.grader import RunRecord, ToolCall, grade_run +from tests.skills.harness.grader import RunRecord, ToolCall, _output_contains, grade_run SUITE_PATH = Path("tests/skills/suites/pt-snap-memory-leak/suite.yaml") +def test_output_matching_handles_nested_rows() -> None: + output = { + "db_path": "/fixtures/live-blocks.db", + "returned": 1, + "rows": [{"id": 1, "size": 4096, "allocEventId": 1}], + } + + assert _output_contains({"rows": [{"size": 4096}]}, output) + assert _output_contains({"returned": 1, "rows": [{"allocEventId": 1}]}, output) + assert not _output_contains({"rows": [{"size": 2048}]}, output) + assert not _output_contains({"rows": [{"size": 4096, "allocEventId": 2}]}, output) + + def _passing_allocator_cache_run() -> RunRecord: return RunRecord( tool_calls=( diff --git a/tests/test_cli.py b/tests/test_cli.py index dd477e5..4cd95d1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -221,6 +221,10 @@ def test_root_help_includes_agent_epilog(self) -> None: assert result.exit_code == 0 normalized = " ".join(unstyle(result.stdout).split()) assert AGENT_HELP_EPILOG in normalized + assert "pt-snap skill install pt-snap-helper --json" in normalized + assert "restart the agent" in normalized.lower() + assert "pt-snap capabilities --json" in normalized + assert "pt-snap overview --json" in normalized assert normalized.index(AGENT_HELP_EPILOG) > normalized.index( "PyTorch Memory Snapshot Analysis Tool" ) diff --git a/tests/test_cli_json.py b/tests/test_cli_json.py index a12d76d..b4362f8 100644 --- a/tests/test_cli_json.py +++ b/tests/test_cli_json.py @@ -377,6 +377,23 @@ def test_query_json_error_paths(tmp_path: Path, sample_db: Path) -> None: _assert_json_error(bad_device, "DEVICE_NOT_FOUND") +def test_query_json_error_for_database_without_devices(tmp_path: Path) -> None: + database = tmp_path / "no-devices.db" + connection = sqlite3.connect(database) + connection.execute( + "CREATE TABLE dictionary (`table` TEXT, `column` TEXT, `key` TEXT, `value` TEXT)" + ) + connection.commit() + connection.close() + + result = runner.invoke( + app, + ["query", str(database), "--template-use", "leak_detection", "--json"], + ) + + _assert_json_error(result, "DEVICE_NOT_FOUND") + + def test_import_json_fresh_and_reuse(tmp_path: Path) -> None: snapshot = tmp_path / "sample.pkl" shutil.copy(FIXTURES / "snapshot_with_empty_cache.pkl", snapshot) From 3416c031782c50e958f83e6374ee3bb602c2c1e8 Mon Sep 17 00:00:00 2001 From: codeasier Date: Thu, 24 Sep 2026 00:01:35 +0800 Subject: [PATCH 2/4] fix(cli): keep helper install within narrow help --- src/pt_snap_cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pt_snap_cli/cli.py b/src/pt_snap_cli/cli.py index 85109e4..b4a1096 100644 --- a/src/pt_snap_cli/cli.py +++ b/src/pt_snap_cli/cli.py @@ -92,10 +92,10 @@ from pt_snap_cli.query.registry import discover_categories, get_query AGENT_HELP_EPILOG = ( - "Agents: prefer --json where supported. " "Agents without pt-snap-helper can run " "pt-snap skill install pt-snap-helper --json, then restart the agent " "so the skill change takes effect. " + "Agents: prefer --json where supported. " "Start with the pt-snap-helper skill; " "use pt-snap capabilities --json and pt-snap overview --json before diagnosing; " "check availability with pt-snap skill list --json." From a4f5269b7a15aadb0165595327e415ddc6a7ed50 Mon Sep 17 00:00:00 2001 From: codeasier Date: Thu, 24 Sep 2026 00:05:45 +0800 Subject: [PATCH 3/4] fix(ci): stabilize wheel help width --- .github/scripts/clean_wheel_agent_acceptance.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/clean_wheel_agent_acceptance.py b/.github/scripts/clean_wheel_agent_acceptance.py index dc04c3d..f2dc41a 100644 --- a/.github/scripts/clean_wheel_agent_acceptance.py +++ b/.github/scripts/clean_wheel_agent_acceptance.py @@ -117,6 +117,8 @@ def _isolated_environment(home: Path, cwd: Path, venv_dir: Path, bin_dir: Path) env["VIRTUAL_ENV"] = str(venv_dir) env["PATH"] = f"{bin_dir}{os.pathsep}{env.get('PATH', '')}" env["PWD"] = str(cwd) + env["COLUMNS"] = "100" + env["LINES"] = "50" return env From 0f9b693d0344ca6f0b533f3bca6fcf2f95715e25 Mon Sep 17 00:00:00 2001 From: codeasier Date: Thu, 24 Sep 2026 00:09:31 +0800 Subject: [PATCH 4/4] fix(ci): verify packaged help contract --- .../scripts/clean_wheel_agent_acceptance.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/scripts/clean_wheel_agent_acceptance.py b/.github/scripts/clean_wheel_agent_acceptance.py index f2dc41a..f245ac3 100644 --- a/.github/scripts/clean_wheel_agent_acceptance.py +++ b/.github/scripts/clean_wheel_agent_acceptance.py @@ -10,6 +10,8 @@ from pathlib import Path from typing import Any +from click import unstyle + class AcceptanceError(RuntimeError): pass @@ -187,6 +189,22 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: _is_inside(bundled_helper, venv_dir), f"Bundled helper is outside the venv: {bundled_helper}", ) + help_contract_result = _run( + [ + str(venv_python), + "-c", + "from pt_snap_cli.cli import AGENT_HELP_EPILOG; print(AGENT_HELP_EPILOG)", + ], + cwd=cwd, + env=env, + label="packaged help contract", + clean_stderr=True, + ) + help_contract = " ".join(help_contract_result.stdout.split()) + _require( + "pt-snap skill install pt-snap-helper --json" in help_contract, + f"Packaged help contract omits the helper install command: {help_contract!r}", + ) help_result = _run( [str(pt_snap), "--help"], @@ -195,20 +213,16 @@ def _run_acceptance(wheel: Path, fixture: Path) -> None: label="root help", clean_stderr=True, ) - normalized_help = " ".join(help_result.stdout.split()) - _require( - "pt-snap skill install pt-snap-helper --json" in normalized_help, - "Root help does not show the supported helper install command", - ) + normalized_help = " ".join(unstyle(help_result.stdout).split()) _require( - "restart the agent" in normalized_help.lower(), - "Root help does not tell the agent to restart after installing the helper", + "pt-snap-helper" in normalized_help and "restart the agent" in normalized_help.lower(), + f"Root help lost helper onboarding: {normalized_help[-1000:]!r}", ) _require( "pt-snap capabilities --json" in normalized_help and "pt-snap overview --json" in normalized_help and "prefer --json" in normalized_help, - "Root help lost its JSON capability guidance", + f"Root help lost its JSON capability guidance: {normalized_help[-1000:]!r}", ) listed = _run_json(