diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..f49a963 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +name: Release + +on: + release: + types: [published] + +jobs: + pypi-publish: + name: Publish to PyPI + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/snapshot-tool + permissions: + # Required for PyPI Trusted Publishing (OIDC). No API token/secret is used. + id-token: write + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Build sdist and wheel + run: uv build + + - name: Publish to PyPI (trusted publishing) + run: uv publish --trusted-publishing always diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..2ae36f4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,90 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this tool does + +`snapshot-tool` is a snapshot-testing harness for [ASV](https://github.com/airspeed-velocity/asv) benchmarks. Given a benchmark directory it: + +1. Discovers ASV benchmarks (functions/methods prefixed `time_`, `timeraw_`, `mem_`, `peakmem_`, `track_`) by AST-parsing files — it never imports them at discovery time. +2. Runs each benchmark while a `sys.settrace`-based tracer captures the return value of the shallowest meaningful user-code call (stdlib + numpy/shapely internals are skipped). +3. Persists the captured return value into `<.snapshots>/snapshots.db` — a single SQLite database with a content-addressed `blobs` table (sha256 → gzipped pickle, refcounted) and a `snapshots` table. A JSON metadata sidecar is also written per `(module, class, benchmark, parameters)` tuple for downstream tooling. +4. Reruns later and compares outputs with tolerance-aware numerical comparison (pure-Python `isclose`; numpy is optional). + +The roundtrip executed by `customtest.sh` is the canonical demo: `list -> capture -> baseline -> verify`. + +## Common commands + +This project uses `uv`. Always invoke tools through `uv run` (or after `uv sync --group dev && uv pip install -e .`). + +```bash +# Install dev environment +uv sync --group dev && uv pip install -e . + +# Run the full test suite (excludes the heavy benchmark-repo roundtrips, matching CI) +uv run pytest -v --ignore=tests/test_repos/ --ignore=tests/test_cli_roundtrip.py + +# Run a single test / file +uv run pytest tests/test_comparator_comprehensive.py -v +uv run pytest tests/test_comparator_comprehensive.py::TestComparator::test_name -v + +# Run the slow real-repo roundtrip tests locally +uv run pytest tests/test_cli_roundtrip.py -v +# Filter inside the roundtrip via env vars (the same knobs CI uses): +SNAPSHOT_TOOL_FILTER='^benchmarks\.(coordinates|units)' SNAPSHOT_TOOL_TIMEOUT=10 \ + uv run pytest tests/test_cli_roundtrip.py::TestAstropyRoundtrip -x + +# Lint / format (matches the lint.yml CI job — no auto-fix in CI) +uv run ruff format --check src/ tests/ +uv run ruff check src/ tests/ + +# Manual smoke test against one of the bundled repos +bash customtest.sh # currently wired to tests/test_repos/shapely_benchmarks +``` + +## CLI surface + +Entry point: `snapshot-tool` (defined in `pyproject.toml` -> `snapshot_tool.cli:main`). Subcommands: + +- `list [--filter REGEX]` +- `capture [--filter REGEX] [--snapshot-dir DIR] [--timeout SEC]` +- `verify [--filter REGEX] [--snapshot-dir DIR] [--tolerance RTOL ATOL] [--summary summary.json] [--timeout SEC]` +- `baseline ...` — like verify but records pass/fail per test_id into `/baseline.json`. A subsequent `verify` reads it and emits a 3x3 pass/fail/skip transition matrix via `transitions.compute_transitions`. +- `clean`, `config --init|--show` + +`--filter` is a Python regex matched against `f"{module_path}.{benchmark_name}"`. + +## Architecture + +The package lives under `src/snapshot_tool/`. The pipeline is intentionally a chain of single-purpose modules; understand them in this order: + +1. **`discovery.py`** — `BenchmarkDiscovery` walks the benchmark directory and AST-parses every `.py` file (skipping `__init__.py`). It returns `BenchmarkInfo` records describing each function- or class-method benchmark, including `params`, `param_names`, whether the class has `setup` / `setup_cache`, and `needs_runtime_eval` (set when `params` is something dynamic that can't be evaluated statically — e.g. a comprehension or call). Parameter evaluation for these cases is deferred to the runner. + +2. **`tracer.py`** — `ExecutionTracer` installs a `sys.settrace` callback that filters out frames belonging to stdlib (`sys.stdlib_module_names` + a hand-curated set), numpy internals, shapely internals, dunder methods, and lambdas. It captures the *shallowest* non-meaningless return value — comments in `_handle_return` describe this as "deepest" but the code prefers shallower depths (`<= self.deepest_call.depth`). The trace result is what gets snapshotted, not the benchmark function's own return value. + +3. **`rng_patcher.py`** — `RNGPatcher` deterministically reseeds numpy (legacy + Generator API), PyTorch, and TensorFlow before every benchmark run. `BenchmarkRunner._reset_random_state` calls this on every invocation — do not bypass it, the whole point of snapshot comparison is bit-stable output. + +4. **`runner.py`** — `BenchmarkRunner` ties the above together. Key behaviors: + - Loads benchmark modules via `importlib.util` and synthesizes parent-package `ModuleType` entries in `sys.modules` so files using relative imports (`from .common import setup`) work. + - Caches loaded modules and class-level `setup_cache()` results. + - Wraps each run in a `ThreadPoolExecutor.submit(...).result(timeout=...)` so per-benchmark timeouts are enforceable (default 300 s from the CLI). On timeout the future is cancelled but the thread keeps running — be aware this can leak threads on hangs. + - For class-based benchmarks calls `setup_cache` (once), then `setup(*params)` before the benchmark method. For methods declared with parameters but invoked without, it falls back to the first parameter combination and logs a warning. + +5. **`storage.py`** — `SnapshotManager` keeps all captured values in a single SQLite database at `/snapshots.db`. The schema has two tables joined by `blob_hash`: `blobs(hash, data, refcount, raw_size, compressed_size)` content-addresses each gzipped pickle by sha256 (so benchmarks producing identical outputs share a single blob), and `snapshots(test_id PRIMARY KEY, ...)` carries per-test metadata. PRAGMAs: `journal_mode=WAL`, `synchronous=NORMAL`, `foreign_keys=ON`. A JSON metadata sidecar is still written per snapshot at `///.json` for downstream tooling — `store_snapshot` returns that path. `SnapshotManager` also owns `baseline.json` (kept as a flat JSON file) — the test_id->status map produced by `baseline` and consumed by `verify`. There is no `.pkl` / `.pkl.gz` read path; the SQLite backend is a clean break. + +6. **`comparator.py`** — `Comparator` performs tolerance-aware comparison. Numpy is imported lazily and detected via module/type-name probing (`_is_numpy_array`), so the package itself does not depend on numpy. Pure-Python `_py_isclose` mirrors `numpy.isclose` semantics for scalars. + +7. **`transitions.py`** — pure function `compute_transitions(baseline, verify)` returning the 9-cell pass/fail/skip transition matrix. Legacy status `"failed_to_pass"` is normalized to `"fail"`. + +8. **`cli.py`** — argparse front end; thin glue over the modules above. + +Public API is re-exported in `src/snapshot_tool/__init__.py`; prefer adding to `__all__` there over import-from-submodule patterns elsewhere. + +## Project conventions to respect + +- **Python 3.8+ compatibility is enforced.** `pyproject.toml` pins `target-version = "py38"` and ignores `UP007` (no `X | Y` unions). Every module uses `from __future__ import annotations`; keep this for any new file. The full test matrix in CI covers 3.8 -> 3.13. +- Ruff lints with `E,W,F,I,B,C4,UP` and ignores `E501`; formatter is the source of truth (CI runs `ruff format --check`). +- `tests/test_repos/` is excluded by `norecursedirs` in `pyproject.toml` — those directories are vendored benchmark sources (astropy, pandas, shapely), not test files. They are exercised only through `tests/test_cli_roundtrip.py`, which CI runs in dedicated `test-astropy.yml` / `test-pandas.yml` / `test-shapely.yml` jobs sharded by benchmark module regex. +- `pytest -v --strict-markers` is configured; the `slow` marker is registered for full-repo roundtrips. Don't introduce new markers without registering them. +- The package's logger is configured at import time via `configure_logging()` in `__init__.py`. Use `logging.getLogger(__name__)` in submodules — don't add new root-level handlers. +- Snapshot files are written under `.snapshots/` (or wherever `--snapshot-dir` points). `customtest.sh` blows that directory away before each run; treat it as disposable build output, not source. diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md new file mode 100644 index 0000000..931b1f0 --- /dev/null +++ b/docs/getting-started/installation.md @@ -0,0 +1,112 @@ +# Installation + +`snapshot-tool` is pure Python (3.8 → 3.13) with no required runtime dependencies. Numpy is optional — detected lazily by the comparator if your benchmarks return arrays. + +## Prerequisites + +- **Python 3.8 or newer** (3.8 through 3.13 are CI-tested) +- **[uv](https://astral.sh/uv/)** — recommended for development; the project uses `uv` for dependency management and the `uv_build` backend +- **ASV benchmarks** to test against — any directory of files with `time_*`, `timeraw_*`, `mem_*`, `peakmem_*`, or `track_*` functions/methods + +## 1. Install `uv` + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +If you'd rather use `pip`, you can skip this step — see the [pip installation](#using-pip) at the bottom. + +## 2. Clone and install + +```bash +git clone https://github.com/formula-code/snapshot-tester.git +cd snapshot-tester + +# Install the dev environment (creates .venv, installs all deps) +uv sync --group dev +uv pip install -e . +``` + +This installs the `snapshot-tool` console entry point along with the full dev toolchain: `pytest`, `ruff`, `mypy`, and the optional comparison libraries (`numpy`, `shapely`, `astropy`, `scipy`, `pandas`) used by the test suite. + +## 3. Verify + +Confirm the CLI is on your `$PATH`: + +```bash +uv run snapshot-tool --help +``` + +You should see the top-level usage with six subcommands: `list`, `capture`, `verify`, `baseline`, `clean`, `config`. + +Run the unit tests (excluding the heavy real-repo roundtrips, which is what CI does on `main`): + +```bash +uv run pytest -v --ignore=tests/test_repos/ --ignore=tests/test_cli_roundtrip.py +``` + +## 4. Smoke test against a bundled benchmark repo + +The repo vendors three real benchmark suites under `tests/test_repos/` (astropy, pandas, shapely). The `customtest.sh` script runs the canonical `list → capture → baseline → verify` roundtrip against the shapely suite: + +```bash +bash customtest.sh +``` + +You can also point the CLI at any of them directly: + +```bash +uv run snapshot-tool list tests/test_repos/shapely_benchmarks +uv run snapshot-tool capture tests/test_repos/shapely_benchmarks +uv run snapshot-tool verify tests/test_repos/shapely_benchmarks +``` + +!!! note + The `tests/test_repos/` directories are excluded from `pytest` collection (`norecursedirs` in `pyproject.toml`) — they are vendored benchmark sources, not test files. They are exercised through `tests/test_cli_roundtrip.py`, which CI runs in dedicated per-suite jobs sharded by benchmark module regex. + +## Using pip + +If you don't use `uv`: + +```bash +git clone https://github.com/formula-code/snapshot-tester.git +cd snapshot-tester +pip install -e . + +# Or directly from the repo +pip install git+https://github.com/formula-code/snapshot-tester.git +``` + +The runtime has no required dependencies. To run the bundled test suite you'll need the dev extras (`pytest`, `numpy`, `shapely`, `astropy`, etc.) — install them manually or via `uv sync --group dev`. + +## Development tasks + +The same lint commands CI runs (the `lint.yml` job runs `--check` only — no auto-fix): + +```bash +uv run ruff format --check src/ tests/ +uv run ruff check src/ tests/ +``` + +The full per-Python test matrix (3.8 → 3.13) runs in CI; locally you typically only need one interpreter: + +```bash +uv run pytest -v --ignore=tests/test_repos/ --ignore=tests/test_cli_roundtrip.py +``` + +The slow real-repo roundtrips (matching the per-suite CI jobs): + +```bash +uv run pytest tests/test_cli_roundtrip.py -v + +# Filter inside the roundtrip via env vars (the same knobs CI uses): +SNAPSHOT_TOOL_FILTER='^benchmarks\.(coordinates|units)' \ +SNAPSHOT_TOOL_TIMEOUT=10 \ + uv run pytest tests/test_cli_roundtrip.py::TestAstropyRoundtrip -x +``` + +## Next steps + +- [**CLI guide**](../guide/cli.md) — Run `list → capture → verify` against your own benchmarks. +- [Python API Quickstart](quickstart.md) — Use the modules programmatically. +- [Configuration](../guide/configuration.md) — `snapshot_config.json` and every CLI flag. diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md new file mode 100644 index 0000000..48466e6 --- /dev/null +++ b/docs/getting-started/quickstart.md @@ -0,0 +1,213 @@ +# Python API Quickstart + +This guide walks through `snapshot-tool`'s public API. Everything below is re-exported from the top-level `snapshot_tool` package. + +For day-to-day use the [CLI](../guide/cli.md) is enough — reach for the Python API when you need to embed snapshot testing inside a larger harness (e.g., a CI pre-flight, a notebook, or a custom comparison strategy). + +## Discovering benchmarks + +`BenchmarkDiscovery` walks a directory tree and AST-parses every `*.py` file. It never imports the benchmark modules at discovery time — that's deferred to the runner. + +```python +from snapshot_tool import BenchmarkDiscovery + +discovery = BenchmarkDiscovery("path/to/benchmarks") +benchmarks = discovery.discover_all() + +for b in benchmarks: + print(f"{b.module_path}.{b.name} (type={b.benchmark_type})") + if b.params: + print(f" params: {b.params}") + if b.needs_runtime_eval: + print(" params need runtime evaluation (not statically resolvable)") +``` + +Each entry is a `BenchmarkInfo` dataclass with everything the runner needs: + +| Field | Meaning | +|-------|---------| +| `name` | The benchmark function/method name (e.g., `time_compute`) | +| `module_path` | Dotted module path relative to `benchmark_dir` (e.g., `geometry.union`) | +| `benchmark_type` | `"function"` or `"method"` | +| `class_name` | For methods, the enclosing class name | +| `params` | List-of-lists of parameter values (e.g., `[[1, 10, 100], ["a", "b"]]`) | +| `param_names` | Optional list of parameter names | +| `has_setup` / `setup_method` | Whether the class has a `setup` method to call before each run | +| `has_setup_cache` | Whether the class has a `setup_cache` method (called once per class) | +| `needs_runtime_eval` | `True` if `params` contains a non-literal expression (a comprehension, call, etc.) that has to be evaluated at runtime | + +## Running a benchmark with tracing + +`BenchmarkRunner` ties discovery, RNG patching, and the tracer together: + +```python +from snapshot_tool import BenchmarkRunner + +runner = BenchmarkRunner( + benchmark_dir="path/to/benchmarks", + seed=42, # Deterministic seed applied before every run + timeout=300.0, # Per-benchmark timeout in seconds +) + +result = runner.run_benchmark(benchmarks[0]) + +if result.success: + print(f"Captured from {result.function_name} at depth {result.depth}") + print(f"Return value: {result.return_value!r}") +else: + print(f"Failed: {result.error}") +``` + +For parameterized benchmarks, generate all combinations and run each: + +```python +for params in runner.get_param_combinations(benchmark): + result = runner.run_benchmark(benchmark, params) + ... +``` + +The runner wraps each call in a `ThreadPoolExecutor.submit(...).result(timeout=...)`. On timeout the future is cancelled and a `TraceResult` with `success=False` and `error=TimeoutError(...)` is returned — but the underlying thread keeps running, so persistent hangs can leak threads. Keep timeouts realistic. + +## Storing and loading snapshots + +`SnapshotManager` writes captured values into a single SQLite database (`/snapshots.db`) — pickled, gzipped, and deduplicated by sha256 — and emits a JSON metadata sidecar next to where the snapshot would have lived under the old per-file layout. `store_snapshot` returns the sidecar path: + +```python +from snapshot_tool import SnapshotManager + +storage = SnapshotManager(".snapshots/") + +storage.store_snapshot( + benchmark_name=benchmark.name, + module_path=benchmark.module_path, + parameters=(), + param_names=None, + return_value=result.return_value, + class_name=benchmark.class_name, +) +``` + +Later, load it back: + +```python +loaded = storage.load_snapshot( + benchmark_name=benchmark.name, + module_path=benchmark.module_path, + parameters=(), + class_name=benchmark.class_name, +) +if loaded is not None: + expected_value, metadata = loaded + # metadata is a SnapshotMetadata: timestamp, git_commit, git_branch, + # python_version, platform, capture_failed, failure_reason +``` + +See [Snapshot Storage](../guide/storage.md) for the full on-disk layout. + +## Comparing outputs + +`Comparator` dispatches across numpy arrays, scalars, sequences, dicts, and custom classes — falling back to `==` and `NotImplemented`-aware skips for types it doesn't recognize. Numpy is optional; the comparator probes for it lazily. + +```python +from snapshot_tool import Comparator, ComparisonConfig + +config = ComparisonConfig( + rtol=1e-5, + atol=1e-8, + equal_nan=False, + strict_types=True, + strict_shapes=True, +) +comparator = Comparator(config) + +comparison = comparator.compare(actual=result.return_value, expected=expected_value) + +if comparison.match: + print("OK") +elif comparison.skipped: + print(f"Skipped: {comparison.details}") +else: + print(f"Failed: {comparison.error_message}") + if comparison.details: + print(f" details: {comparison.details}") +``` + +The strategy dispatch order is fixed: + +1. Serialized placeholders (`__generator__`, `__callable__`, `__unpicklable__`) → skip. +2. Serialized class instance (`__class_instance__`) → compare `__dict__` recursively. +3. Numpy arrays → shape/dtype check, then element-wise `isclose`. +4. Numeric scalars → pure-Python `isclose`. +5. Objects with a custom `__eq__` → use it (handles array-returning equality on `SkyCoord`, pandas `Series`, etc.). +6. Sequences (list/tuple) → length + element-wise recursion. +7. Dicts → key-set match + value recursion. +8. Fallback `==` with `NotImplemented`-aware skip. + +See [Comparison](../guide/comparison.md) for the full dispatch logic. + +## A complete capture/verify loop + +```python +from snapshot_tool import ( + BenchmarkDiscovery, BenchmarkRunner, SnapshotManager, + Comparator, ComparisonConfig, +) + +bench_dir = "path/to/benchmarks" +snap_dir = ".snapshots/" + +discovery = BenchmarkDiscovery(bench_dir) +runner = BenchmarkRunner(bench_dir, seed=42, timeout=60.0) +storage = SnapshotManager(snap_dir) +comparator = Comparator(ComparisonConfig(rtol=1e-5, atol=1e-8)) + +# Phase 1: capture +for b in discovery.discover_all(): + for params in runner.get_param_combinations(b): + result = runner.run_benchmark(b, params) + if result and result.success: + storage.store_snapshot( + benchmark_name=b.name, + module_path=b.module_path, + parameters=params, + param_names=b.param_names, + return_value=result.return_value, + class_name=b.class_name, + ) + +# ... change code, then ... + +# Phase 2: verify +for b in discovery.discover_all(): + for params in runner.get_param_combinations(b): + loaded = storage.load_snapshot(b.name, b.module_path, params, b.class_name) + if loaded is None: + continue + expected, _meta = loaded + result = runner.run_benchmark(b, params) + if not result or not result.success: + print(f"FAIL (runtime): {b.module_path}.{b.name} {params}") + continue + cmp = comparator.compare(result.return_value, expected) + if not cmp.match and not cmp.skipped: + print(f"FAIL: {b.module_path}.{b.name} {params}: {cmp.error_message}") +``` + +## Forcing determinism without the runner + +If you're running benchmark code outside of `BenchmarkRunner` (e.g., from a notebook), reseed all RNGs manually: + +```python +from snapshot_tool import reset_all_rngs + +reset_all_rngs(seed=42) # reseeds Python random, numpy, torch, tensorflow +``` + +`reset_all_rngs` is the lightweight version of `RNGPatcher.patch_all()` — same seed, no state tracking. See [Determinism](../guide/determinism.md) for the full story. + +## Next steps + +- [CLI](../guide/cli.md) — Reference for all six subcommands. +- [Baseline & Verify](../guide/baseline-and-verify.md) — The 3×3 transition matrix. +- [Comparison](../guide/comparison.md) — Tolerance semantics and the strategy dispatch chain. +- [Determinism](../guide/determinism.md) — How `RNGPatcher` keeps captures bit-stable. diff --git a/docs/guide/baseline-and-verify.md b/docs/guide/baseline-and-verify.md new file mode 100644 index 0000000..8212e34 --- /dev/null +++ b/docs/guide/baseline-and-verify.md @@ -0,0 +1,137 @@ +# Baseline & Verify + +`snapshot-tool` has two related comparison subcommands that produce different outputs: + +| Subcommand | What it writes | When to use | +|------------|----------------|-------------| +| `verify` | `summary.json` with per-outcome counts (`passed`, `failed`, `skipped`). Exits non-zero on failure. | Day-to-day correctness gate. | +| `baseline` | `/baseline.json` with a per-test pass/fail/skip map. Always exits 0. | Record the *starting* state so a later `verify` can compute a transition matrix. | + +When a `baseline.json` is present in the snapshot directory, `verify` reads it and augments `summary.json` with a **3×3 transition matrix**. + +## When to use which + +The intuition: `verify` answers "did anything regress?", `baseline` answers "what was the state at this point in time?". For local development you usually only need `verify`. For grading an optimization PR — where you care about which tests changed status, not just the totals — you want `baseline` followed by `verify`. + +The canonical CI flow: + +```mermaid +graph LR + A[capture
on revision R₀] --> B + B[baseline
on revision R₀] --> C + C[apply patch] --> D + D[verify
on revision R₁] --> E + E[transition matrix
in summary.json] +``` + +## The 3×3 matrix + +Both `baseline` and the per-test status pass produced by `verify` use the same vocabulary: `pass`, `fail`, `skip`. `compute_transitions(baseline, verify)` produces a count for every observed `-to-` pair. + +The cells, in `summary.json` ordering: + +| | → pass | → fail | → skip | +|--------------|--------|--------|--------| +| **from pass** | `pass-to-pass` | `pass-to-fail` | `pass-to-skip` | +| **from fail** | `fail-to-pass` | `fail-to-fail` | `fail-to-skip` | +| **from skip** | `skip-to-pass` | `skip-to-fail` | `skip-to-skip` | + +Interpreting cells: + +| Cell | What it usually means | +|------|----------------------| +| `pass-to-pass` | Stable correct test — your patch left this test intact. | +| `pass-to-fail` | **Regression** — a previously-passing test now produces a different output. | +| `pass-to-skip` | A previously-passing test crashed or got skipped on verify (e.g., raised an exception, returned an unpicklable). Worth investigating. | +| `fail-to-pass` | Your patch fixed a previously-failing test. This is the win condition for optimization PRs that include corrections. | +| `fail-to-fail` | Still failing in the same way (or differently — the matrix only counts statuses, not specific errors). | +| `fail-to-skip` | Previously failed; now skipped. Often noise — the snapshot may have been a failed-capture marker. | +| `skip-to-pass` | Previously skipped (no snapshot or failed capture); now produces a comparable output. | +| `skip-to-fail` | Skipped during baseline (likely no snapshot), but now the test produces an output that's different from the **original capture**. Diagnose by inspecting what's in `.snapshots/`. | +| `skip-to-skip` | Persistently un-snapshottable — e.g., a benchmark that returns a generator. | + +Only `test_id`s present in **both** `baseline.json` and the verify pass are counted. Tests added or removed between the two runs are silently dropped. + +## `test_id` + +A `test_id` is a stable, on-disk-path-like string identifying one `(benchmark, parameters)` row: + +``` +/[.]/ +``` + +The `param-hash` is a 16-character MD5 prefix of `repr(parameters)`. The same hashing function is used for storing snapshots, so a snapshot file and a baseline entry for the same test always agree on the path. + +Two examples: + +- A free function `time_compute()` in `benchmarks/foo.py` with no parameters: + `foo/time_compute/99914b932bd37a50` +- A method `time_union(self, n, geom_type)` on class `TimeUnion` in `benchmarks/geometry/union.py`, parameters `(100, "polygon")`: + `geometry.union/TimeUnion.time_union/` + +You don't normally construct these by hand — `SnapshotManager.get_test_id(...)` does it consistently. + +## What `baseline.json` looks like + +```json +{ + "schema": "snapshot_tool/baseline@2", + "timestamp": "2026-05-14T10:11:00.123456", + "entries": { + "geometry.union/TimeUnion.time_union/2f3a...": "pass", + "geometry.union/TimeUnion.time_union/8c1e...": "fail", + "coordinates/time_separation/99914b932bd37a50": "skip" + }, + "meta": { + "counts": {"total": 124, "pass": 118, "fail": 2, "skip": 4}, + "snapshot_dir": ".snapshots", + "benchmark_dir": "benchmarks" + } +} +``` + +The schema tag (`snapshot_tool/baseline@2`) reserves room for future formats. The current reader accepts any payload with an `entries` dict; the `meta` block is informational. + +## Status normalization + +`compute_transitions` runs every status through `_normalize_status` before matching: + +- The legacy status `"failed_to_pass"` is mapped to `"fail"`. Older baseline files used this name; newer ones use `"fail"` directly. +- Anything else passes through unchanged. In practice, the only values in the wild are `pass`, `fail`, and `skip`. + +## Wiring into CI + +A minimal optimization-grading workflow: + +```yaml +# Pseudocode for a CI job +- name: Capture and baseline on main + run: | + git checkout main + snapshot-tool capture benchmarks --snapshot-dir .snapshots + snapshot-tool baseline benchmarks --snapshot-dir .snapshots + +- name: Verify on the PR branch + run: | + git checkout ${{ github.head_ref }} + snapshot-tool verify benchmarks --snapshot-dir .snapshots --summary summary.json + +- name: Surface transition counts + run: | + jq '. | {passed, failed, "pass-to-fail", "fail-to-pass"}' summary.json +``` + +The `verify` step will exit non-zero if any test failed. If you only want to grade transitions without failing the build on `pass-to-fail`, run `verify` with `|| true` and inspect `summary.json` yourself. + +## Common pitfalls + +- **`baseline` before `capture`.** `baseline` reads existing snapshots; it doesn't create them. If you call `baseline` before `capture`, every entry will be `skip` (no snapshot) and the resulting transition matrix will be useless. +- **Different `--snapshot-dir` between `baseline` and `verify`.** Both must point at the same directory — `baseline.json` lives inside it. +- **Different `--filter` between `baseline` and `verify`.** The matrix only counts overlapping `test_id`s. Filtering differently shrinks the overlap silently. +- **Non-deterministic benchmarks.** If a benchmark returns a value that depends on randomness, on iteration order of a `set`, or on the wall clock, it will appear as `pass-to-fail` even when nothing meaningful changed. See [Determinism](determinism.md) for the RNG controls that mitigate this. Wall-clock and dict/set-order issues need fixing in the benchmark itself. + +## Next steps + +- [CLI](cli.md) — Per-subcommand reference. +- [Snapshot Storage](storage.md) — Where `baseline.json` lives and how `test_id`s are constructed. +- [Determinism](determinism.md) — Why RNG patching is non-optional for stable transitions. diff --git a/docs/guide/cli.md b/docs/guide/cli.md new file mode 100644 index 0000000..560119c --- /dev/null +++ b/docs/guide/cli.md @@ -0,0 +1,249 @@ +# CLI (`snapshot-tool`) + +`snapshot-tool` is the primary entrypoint. It discovers ASV benchmarks in a directory, captures their return values, and compares them on later runs. All operations are local — no network or database is involved. + +## Quick reference + +```bash +# 1. See what discovery finds +snapshot-tool list path/to/benchmarks + +# 2. Capture a baseline snapshot of every benchmark's output +snapshot-tool capture path/to/benchmarks --timeout 60 + +# 3. (Optional) Record current pass/fail state for transition tracking +snapshot-tool baseline path/to/benchmarks --tolerance 1e-5 1e-8 + +# 4. Verify after a code change +snapshot-tool verify path/to/benchmarks --tolerance 1e-5 1e-8 --summary summary.json + +# Show or initialize config +snapshot-tool config --show +snapshot-tool config --init +``` + +`snapshot-tool` exits non-zero on a verification failure; `0` otherwise. `baseline` always exits `0` — it only records state. + +## Global flags + +These apply to every subcommand: + +| Flag | Description | +|------|-------------| +| `--config PATH`, `-c PATH` | Path to a `snapshot_config.json` (default: `./snapshot_config.json`) | +| `--verbose`, `-v` | Verbose logging (prints comparison details and full tracebacks on errors) | +| `--quiet`, `-q` | Suppress per-benchmark `[PASS]` / `[SKIP]` lines | + +## Subcommands + +--- + +### `list` + +AST-discovers benchmarks under `benchmark_dir` and prints them. No code is imported or executed. + +```bash +snapshot-tool list [--filter REGEX] +``` + +| Flag | Description | +|------|-------------| +| `benchmark_dir` | Directory to scan recursively for `*.py` (skips `__init__.py`) | +| `--filter REGEX` | Python regex matched against `f"{module_path}.{benchmark_name}"` | + +For each benchmark, `list` prints the dotted path, its type (`function` or `method`), the setup method if any, and the number of parameter combinations. Use `--verbose` to also print the first five parameter combinations explicitly. + +```bash +snapshot-tool list tests/test_repos/shapely_benchmarks --filter '^geometry\.' +``` + +--- + +### `capture` + +Runs each discovered benchmark with tracing enabled and writes a snapshot per `(module, class, benchmark, parameter-tuple)` tuple into `--snapshot-dir`. + +```bash +snapshot-tool capture \ + [--filter REGEX] [--snapshot-dir DIR] [--timeout SEC] +``` + +| Flag | Description | Default | +|------|-------------|---------| +| `benchmark_dir` | Directory containing benchmark files | **required** | +| `--filter REGEX` | Python regex matched against `f"{module_path}.{benchmark_name}"` | — | +| `--snapshot-dir DIR` | Where to write snapshots | `.snapshots/` (from config) | +| `--timeout SEC` | Per-benchmark timeout in seconds | `300` | + +For every parameter combination, `capture`: + +1. Reseeds RNGs (`RNGPatcher.patch_all()`). +2. Loads the benchmark module via `importlib.util`, synthesizing any parent packages in `sys.modules` so relative imports work. +3. Calls `setup_cache()` once per class (cached), then `setup(*params)`, then the benchmark method. +4. Installs `sys.settrace` and runs. On return, takes the shallowest meaningful user-code return value. +5. Pickles, gzips, and content-addresses the result by sha256, then inserts (or refcounts) it into `/snapshots.db`. A JSON metadata sidecar is also written under `///.json` for downstream tooling. +6. If the benchmark raises, times out, or returns something unpicklable, a **failed capture marker** is written instead — a snapshot with `capture_failed=True` in metadata. `verify` will skip these. + +```bash +# Capture everything with a tight 30-second budget per benchmark +snapshot-tool capture tests/test_repos/shapely_benchmarks --timeout 30 + +# Only capture the coordinates and units modules +snapshot-tool capture tests/test_repos/astropy_benchmarks \ + --filter '^benchmarks\.(coordinates|units)' +``` + +!!! warning + `capture` is destructive in the sense that it overwrites existing snapshots at the same path. If you want to keep an old set of snapshots around (e.g., for A/B comparison) move the directory aside first or pass a different `--snapshot-dir`. + +--- + +### `verify` + +Re-runs each benchmark, loads the matching snapshot, and compares them with the configured tolerance. + +```bash +snapshot-tool verify \ + [--filter REGEX] [--snapshot-dir DIR] \ + [--tolerance RTOL ATOL] [--summary summary.json] [--timeout SEC] +``` + +| Flag | Description | Default | +|------|-------------|---------| +| `benchmark_dir` | Directory containing benchmark files | **required** | +| `--filter REGEX` | Python regex matched against `f"{module_path}.{benchmark_name}"` | — | +| `--snapshot-dir DIR` | Where to read snapshots from | `.snapshots/` (from config) | +| `--tolerance RTOL ATOL` | Override `rtol` and `atol` for numeric comparison | from config | +| `--summary PATH` | Where to write the JSON summary | `summary.json` | +| `--timeout SEC` | Per-benchmark timeout in seconds | `300` | + +Per-test outcomes are: + +| Outcome | Meaning | +|---------|---------| +| `[PASS]` | Comparison matched within tolerance | +| `[FAIL]` | The current return value differs, **or** the benchmark crashed during verify (but succeeded during capture — usually a real regression or environment drift) | +| `[SKIP]` | No snapshot exists, the snapshot was a failed-capture marker, or the comparison itself was unsupported (e.g., generators, callables, unpicklable values) | + +At the end, `verify` writes a JSON summary to `--summary`: + +```json +{ + "total": 124, + "passed": 118, + "failed": 2, + "skipped": 4, + "timestamp": "2026-05-14T10:14:32.018000", + "snapshot_dir": ".snapshots", + "benchmark_dir": "tests/test_repos/shapely_benchmarks" +} +``` + +If a `baseline.json` is present in `--snapshot-dir`, the summary is **augmented** with a 3×3 transition matrix derived from `compute_transitions(baseline, verify)`: + +```json +{ + ... + "pass-to-pass": 100, + "pass-to-fail": 2, + "pass-to-skip": 0, + "fail-to-pass": 1, + "fail-to-fail": 15, + "fail-to-skip": 0, + "skip-to-pass": 0, + "skip-to-fail": 0, + "skip-to-skip": 4 +} +``` + +See [Baseline & Verify](baseline-and-verify.md) for the transition semantics. + +`verify` exits with code `1` if any test was marked `FAIL`, else `0`. `SKIP` does not fail the run. + +--- + +### `baseline` + +Same execution path as `verify`, but instead of failing on mismatches it **records the pass/fail/skip status of each test_id** into `/baseline.json`. A later `verify` consumes this file to compute the transition matrix. + +```bash +snapshot-tool baseline \ + [--filter REGEX] [--snapshot-dir DIR] \ + [--tolerance RTOL ATOL] [--timeout SEC] +``` + +The flags mean exactly what they do for `verify`, except there is no `--summary` — the output is `baseline.json` inside the snapshot directory. + +A typical CI workflow: + +```bash +# On the "before" revision +snapshot-tool capture benchmarks +snapshot-tool baseline benchmarks # writes baseline.json + +# ... apply optimization patch, then on the "after" revision ... + +snapshot-tool verify benchmarks # summary.json now includes the transition matrix +``` + +`baseline` always returns `0` — it only records state. + +--- + +### `clean` + +Reports on the snapshot directory. Currently informational only. + +```bash +snapshot-tool clean [--snapshot-dir DIR] [--dry-run] +``` + +| Flag | Description | +|------|-------------| +| `--snapshot-dir DIR` | Directory to inspect (default: from config) | +| `--dry-run` | Show what *would* be deleted without deleting | + +Today `clean` prints the snapshot count and total size on disk; it does not delete files. To wipe snapshots, remove the directory yourself (`rm -rf .snapshots/`) — `customtest.sh` is the canonical example. + +--- + +### `config` + +Manage the `snapshot_config.json` file. + +```bash +snapshot-tool config --init # write default snapshot_config.json +snapshot-tool config --show # print the active config +``` + +The config file is loaded from `./snapshot_config.json` unless overridden with `--config PATH`. See [Configuration](configuration.md) for the schema and every field. + +## Filter semantics + +`--filter` is a Python regex (via `re.search`) matched against `f"{module_path}.{benchmark_name}"` — not the file path. For class-based benchmarks the benchmark name is the *method* name; the class is not part of the matched string. To filter on a class, target its module: + +```bash +# Only TimeAngularSeparation.* methods, which live in benchmarks/coordinates.py +snapshot-tool list benchmarks --filter '^coordinates\..*' +``` + +## Typical workflow + +```bash +# 1. Iterate locally on a benchmark suite +uv run snapshot-tool list benchmarks +uv run snapshot-tool capture benchmarks --timeout 30 +uv run snapshot-tool baseline benchmarks +uv run snapshot-tool verify benchmarks # baseline of yourself: should be 100% pass-to-pass + +# 2. Change code, re-verify +uv run snapshot-tool verify benchmarks # transitions tell you what flipped +``` + +For a one-shot smoke test, `customtest.sh` runs the full roundtrip against the bundled shapely repo. + +## Next steps + +- [Configuration](configuration.md) — the `snapshot_config.json` schema. +- [Baseline & Verify](baseline-and-verify.md) — what each transition cell means. +- [Discovery](discovery.md) — what `list` is actually doing under the hood. diff --git a/docs/guide/comparison.md b/docs/guide/comparison.md new file mode 100644 index 0000000..2c4b80b --- /dev/null +++ b/docs/guide/comparison.md @@ -0,0 +1,129 @@ +# Comparison + +`Comparator` decides whether an actual value matches an expected snapshot. It is type-aware, tolerance-aware, and **never** depends on numpy at import time — numpy is detected lazily and probed by module/type-name so the package works in pure-Python environments. + +## Configuration + +```python +from snapshot_tool import Comparator, ComparisonConfig + +config = ComparisonConfig( + rtol=1e-5, # relative tolerance + atol=1e-8, # absolute tolerance + equal_nan=False, # if True, NaN == NaN + strict_types=True, # error on dtype mismatches for numpy arrays + strict_shapes=True, # error on shape mismatches for numpy arrays + ignore_order=False, # (reserved; not currently honored by built-in strategies) +) + +comparator = Comparator(config) +result = comparator.compare(actual, expected) +``` + +| Field | Default | What it does | +|-------|---------|--------------| +| `rtol` | `1e-5` | Relative tolerance applied to `\|a - b\| <= atol + rtol * \|b\|` | +| `atol` | `1e-8` | Absolute tolerance, same formula | +| `equal_nan` | `False` | If `True`, two NaN values compare equal | +| `strict_types` | `True` | Numpy array `dtype` mismatch is a failure | +| `strict_shapes` | `True` | Numpy array `shape` mismatch is a failure | + +The CLI exposes `rtol` and `atol` via `--tolerance RTOL ATOL`. The remaining fields are configurable only through `snapshot_config.json` or programmatic use. + +## The result + +```python +@dataclass +class ComparisonResult: + match: bool + skipped: bool = False + tolerance_used: Optional[dict[str, float]] = None + error_message: Optional[str] = None + details: Optional[dict[str, Any]] = None +``` + +Three outcomes: + +| Outcome | `match` | `skipped` | When | +|---------|---------|-----------|------| +| Pass | `True` | `False` | Values match within tolerance | +| Skip | `True` | `True` | The expected value was a placeholder that can't be compared (a generator, callable, or unpicklable marker), or the type has no usable `__eq__` | +| Fail | `False` | `False` | Concrete mismatch — `error_message` and `details` describe what differs | + +`verify` treats `skipped=True` results as `[SKIP]` (not a failure) and only fails the run on `match=False, skipped=False`. + +## Dispatch order + +`compare()` runs through a fixed strategy chain and returns the first result that isn't `None`: + +1. **Serialized placeholders** — If `expected` is a dict tagged `__generator__`, `__callable__`, or `__unpicklable__`, the comparison is skipped. These tags come from [`SnapshotManager._serialize_value`](storage.md#serialization-of-unpicklable-values), which writes them when the captured value can't round-trip through pickle. +2. **Serialized class instance** — If `expected` is tagged `__class_instance__`, the comparator checks `__class__.__name__`, then recursively compares each attribute in the saved `__dict__` against `actual.__dict__`. Extra or missing attributes fail. +3. **`None` handling** — Both `None` → match. One `None` → fail. +4. **Numpy arrays** — If both sides are arrays: + - Shapes are compared (failure under `strict_shapes`). + - Dtypes are compared (failure under `strict_types`). + - `object` dtype arrays go through `_compare_object_arrays`, which recursively compares each element with `compare()` (so a shapely-array-of-polygons works). + - Numeric arrays are flattened and compared element-wise with the pure-Python `_py_isclose`. The result includes `max_difference`, `mean_difference`, `shape`, and `dtype` for diagnostics. +5. **Numeric scalars** — `int`/`float` and numpy scalar types. Compared with `_py_isclose`. +6. **Objects with `__eq__`** — If `type(actual) == type(expected)` and `__eq__` is defined somewhere on the MRO (not just the default `object.__eq__`), `==` is called. The result may itself be a numpy array (e.g., `astropy.coordinates.SkyCoord` returns a bool array from `__eq__`), in which case `.all()` reduces it to a single bool. Lists/tuples of arrays are handled similarly. +7. **Sequences** (list, tuple) — Length match, then element-wise recursion. The first ten mismatch indices and their messages are retained in `details["mismatches"]`. +8. **Dicts** — Key-set match, then per-key recursion. Like sequences, up to ten mismatches are retained. +9. **Fallback `==`** — Type-check first, then `actual == expected`. If the comparison itself raises (e.g., `ValueError("ambiguous truth value")` from accidentally calling `bool()` on an array), the failure is converted to a **skip** with `details["reason"]` explaining why. + +The order matters. `_compare_objects` runs before `_compare_sequences` because many "object-like" types (pandas `Series`, astropy `SkyCoord`) have `__len__` and `__getitem__` but want to be compared via their own `__eq__`, not element-by-element. + +## Scalar comparison + +Numeric scalars use `_py_isclose` — a pure-Python mirror of `numpy.isclose`: + +```python +def _py_isclose(a, b, rtol=1e-5, atol=1e-8, equal_nan=False): + # NaN handling + if equal_nan and isnan(a) and isnan(b): return True + if isnan(a) or isnan(b): return False + # Infinity: inf == inf, -inf == -inf, inf != -inf + if isinf(a) or isinf(b): return a == b + return abs(a - b) <= atol + rtol * abs(b) +``` + +The formula is **asymmetric** in `b` — that's intentional; it matches numpy's semantics. In `snapshot-tool`, `b` is always the *expected* (snapshot) value, so the tolerance scales with the magnitude of what you originally captured. + +## Numpy array comparison without importing numpy + +`Comparator` never imports numpy unconditionally. Detection is done with: + +```python +def _is_numpy_array(obj): + if HAS_NUMPY: + return isinstance(obj, np.ndarray) + obj_type = type(obj) + return obj_type.__module__ == 'numpy' and obj_type.__name__ == 'ndarray' +``` + +If numpy isn't installed, array snapshots round-trip as the placeholder representations produced by `SnapshotManager` (e.g., `__class_instance__` for object arrays) and are compared accordingly. In practice, if you have numpy-array snapshots you almost certainly have numpy installed — but the harness doesn't assume it. + +## Failure diagnostics + +Failed comparisons populate `error_message` and, where useful, `details`: + +- **Numpy arrays**: `details = {"max_difference": ..., "mean_difference": ..., "shape": ..., "dtype": ...}` +- **Sequences/dicts**: `details = {"mismatches": [(index_or_key, message), ...]}` (capped at 10) +- **Scalars**: `details = {"difference": abs(actual - expected)}` +- **Classes via fallback**: `details = {"type": ..., "reason": ..., "skipped": True}` when comparison is unsupported + +`verify` prints `error_message` on `[FAIL]` and the `details` dict when `--verbose` is set. + +## Configuring tolerance per-run + +The CLI accepts the two most important knobs directly: + +```bash +snapshot-tool verify benchmarks --tolerance 1e-4 1e-6 +``` + +For everything else (`equal_nan`, `strict_types`, `strict_shapes`), set them in `snapshot_config.json` — see [Configuration](configuration.md). + +## Next steps + +- [Snapshot Storage](storage.md) — How values are serialized and what placeholders look like on disk. +- [Baseline & Verify](baseline-and-verify.md) — How comparison outcomes roll up into the 3×3 transition matrix. diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md new file mode 100644 index 0000000..fc095ad --- /dev/null +++ b/docs/guide/configuration.md @@ -0,0 +1,149 @@ +# Configuration + +`snapshot-tool` reads its configuration from `./snapshot_config.json` by default. The file is optional — every field has a sensible default — and is loaded by `ConfigManager` at CLI startup. + +Override the path with `--config PATH` (or `-c PATH`). Override individual fields with the CLI flags described in the [CLI guide](cli.md). + +## Initialize the file + +```bash +snapshot-tool config --init +``` + +writes a default `snapshot_config.json` with every field at its built-in default: + +```json +{ + "benchmark_dir": "benchmarks/", + "snapshot_dir": ".snapshots/", + "project_dir": null, + "tolerance": { + "rtol": 1e-5, + "atol": 1e-8, + "equal_nan": false + }, + "exclude_benchmarks": [], + "trace_depth_limit": 100, + "verbose": false, + "quiet": false +} +``` + +To inspect what's currently active: + +```bash +snapshot-tool config --show +``` + +## Fields + +### Directories + +| Field | Default | What it does | +|-------|---------|--------------| +| `benchmark_dir` | `benchmarks/` | Default directory for `list` / `capture` / `verify` / `baseline` if not provided on the command line. **Currently informational** — the CLI requires `benchmark_dir` as a positional argument on every subcommand, so this is rarely used by itself. | +| `snapshot_dir` | `.snapshots/` | Where snapshots are written and read. Overridden by `--snapshot-dir`. | +| `project_dir` | `null` | Optional override for the project root. By default `BenchmarkRunner` uses `benchmark_dir.parent` and inserts it onto `sys.path` so the benchmarked package can be imported. Set this when your project root isn't the parent of the benchmark directory. | + +### Comparison tolerance + +| Field | Default | What it does | +|-------|---------|--------------| +| `tolerance.rtol` | `1e-5` | Relative tolerance for numeric comparison. Overridden by the first value of `--tolerance RTOL ATOL`. | +| `tolerance.atol` | `1e-8` | Absolute tolerance. Overridden by the second value of `--tolerance RTOL ATOL`. | +| `tolerance.equal_nan` | `false` | When `true`, two NaN values compare equal. No CLI override — set in the file. | + +The comparison formula is `|a - b| <= atol + rtol * |b|`, matching `numpy.isclose` semantics. `b` is the *expected* (snapshot) value. + +See [Comparison](comparison.md) for the full dispatch and additional knobs (`strict_types`, `strict_shapes`) available when constructing a `ComparisonConfig` programmatically. + +### Filtering + +| Field | Default | What it does | +|-------|---------|--------------| +| `exclude_benchmarks` | `[]` | List of benchmark *names* (not full paths) to skip. Each entry is either an exact match or a prefix wildcard ending in `*`. Independent of `--filter`. | + +`should_exclude_benchmark` does a simple `==` or `startswith` check against the benchmark `name` only — it doesn't see the module path. To exclude by module, use the `--filter` regex on the command line instead. + +Examples: + +```json +{ + "exclude_benchmarks": [ + "time_flaky", // exact match + "time_slow_*", // prefix wildcard + "peakmem_*" // skip all peakmem benchmarks + ] +} +``` + +### Tracing + +| Field | Default | What it does | +|-------|---------|--------------| +| `trace_depth_limit` | `100` | Maximum call depth for the [tracer](tracing.md). The tracer stops recording new frames beyond this depth to avoid infinite-recursion blowups. The default is generous for typical benchmark code; lower it if you're seeing pathological recursion. | + +This field is read into the dataclass but not currently threaded through to `ExecutionTracer`'s constructor by the CLI — the tracer always uses its built-in `max_depth=100`. The two values agree by default; if you want a different depth, instantiate `ExecutionTracer(max_depth=N)` and use the [Python API](../getting-started/quickstart.md) directly. + +### Output + +| Field | Default | What it does | +|-------|---------|--------------| +| `verbose` | `false` | Equivalent to `--verbose` / `-v`. Enables debug-level comparison detail and full tracebacks on errors. | +| `quiet` | `false` | Equivalent to `--quiet` / `-q`. Suppresses per-benchmark `[PASS]` / `[SKIP]` lines (failures are always printed). | + +CLI flags take precedence — passing `-v` on the command line forces `verbose=True` for that invocation regardless of the file. + +## CLI flag precedence + +For every overlapping field, the precedence is: + +``` +CLI flag > snapshot_config.json > built-in default +``` + +So a `verify` command like: + +```bash +snapshot-tool -v --config ./conf.json verify ./benchmarks \ + --snapshot-dir ./snapshots --tolerance 1e-4 1e-6 +``` + +resolves to: + +- `snapshot_dir = "./snapshots"` (from `--snapshot-dir`, overriding `conf.json`). +- `tolerance.rtol = 1e-4`, `tolerance.atol = 1e-6` (from `--tolerance`). +- `tolerance.equal_nan = ` (not overridable on the CLI). +- `verbose = True` (from `-v`). + +## Programmatic configuration + +```python +from snapshot_tool import ConfigManager, SnapshotConfig +from pathlib import Path + +# Load (or default if missing) +manager = ConfigManager(Path("snapshot_config.json")) +config: SnapshotConfig = manager.get_config() + +# Mutate in memory +manager.update_config(verbose=True, snapshot_dir="custom/.snapshots") + +# Persist +manager.save_config() + +# Or build one from scratch +config = SnapshotConfig( + benchmark_dir="benchmarks/", + snapshot_dir=".snapshots/", + tolerance={"rtol": 1e-4, "atol": 1e-6, "equal_nan": True}, + exclude_benchmarks=["time_flaky_*"], +) +config.save_to_file(Path("snapshot_config.json")) +``` + +## Next steps + +- [CLI](cli.md) — Every subcommand and flag. +- [Comparison](comparison.md) — Tolerance semantics and the strategy chain. +- [Snapshot Storage](storage.md) — Where snapshots live on disk. diff --git a/docs/guide/determinism.md b/docs/guide/determinism.md new file mode 100644 index 0000000..03a859d --- /dev/null +++ b/docs/guide/determinism.md @@ -0,0 +1,101 @@ +# Determinism + +Snapshot testing is only useful if the same code produces the same output twice. `BenchmarkRunner` enforces this by reseeding every known RNG before each benchmark invocation. The component responsible is `RNGPatcher`. + +## What gets reseeded + +`RNGPatcher.patch_all()` reseeds the following, in order, on every invocation: + +| Library | Calls | Notes | +|---------|-------|-------| +| **Python stdlib** | `random.seed(seed)` | Always — `random` is in the stdlib. | +| **NumPy** (legacy API) | `np.random.seed(seed)` | If numpy is importable. Affects `np.random.rand`, `np.random.randn`, etc. Compatible with numpy 1.12+ (2017). | +| **PyTorch** | `torch.manual_seed(seed)`, plus `torch.cuda.manual_seed_all(seed)` if CUDA is available | If torch is importable. | +| **TensorFlow** | `tf.random.set_seed(seed)` | If tensorflow is importable. | + +Default `seed=42`. `BenchmarkRunner` instantiates `RNGPatcher(seed=42)` and calls `patch_all()` before: + +- Every benchmark run (via `_reset_random_state()` inside `_run_benchmark_internal`). +- Every `setup_cache()` call (once per class). +- Every `setup()` call (before each parameter combination). + +## Numpy Generator API + +`RNGPatcher` reseeds the **legacy** numpy API (`np.random.seed`). The newer `numpy.random.Generator` API (`np.random.default_rng()`) is **not** patched automatically — each `Generator` instance carries its own state and is created independently. + +If a benchmark uses `default_rng()`, the same call inside `setup` or the benchmark method will produce the same output **as long as no other code is producing entropy in between**. This is usually fine because `patch_all` runs immediately before `setup`/method execution, so the import-time random state is irrelevant. + +If you have a benchmark that creates a `Generator` *outside* the setup path (e.g., at module import), pin its seed explicitly in the benchmark code — `RNGPatcher` can't reach module-level state that was already instantiated. + +## Inside `BenchmarkRunner` + +```python +class BenchmarkRunner: + def __init__(self, benchmark_dir, ..., seed=42, timeout=None): + self.rng_patcher = RNGPatcher(seed=seed) + + def _reset_random_state(self): + self.rng_patcher.patch_all() +``` + +`_reset_random_state` is called: + +- Once at the top of `_run_benchmark_internal`, before module load and parameter resolution. +- Again before `setup_cache()`, if the class has one. +- Again before `setup()`, if the class has one. + +The whole point of running it three times is to guarantee that the RNG state immediately before any user-code path is identical between capture and verify. **Do not bypass these calls** — bypassing them is exactly equivalent to making the snapshots non-reproducible. + +## Module-level constants vs runtime state + +`RNGPatcher` only handles RNG state. Other sources of nondeterminism are out of scope: + +- **Dict/set iteration order** — Python ≥ 3.7 dicts iterate insertion order; sets do not. If a benchmark returns a `set` or iterates one, the result may vary between runs. +- **Hash randomization** — `PYTHONHASHSEED` affects `hash()` for strings/bytes. Snapshots survive this if the benchmark doesn't expose hashes; if it does, set `PYTHONHASHSEED=0` (or any constant) in your environment. +- **Wall-clock time** — `datetime.now()`, `time.time()` and similar are not patched. +- **OS / hardware** — floating-point semantics can differ across BLAS implementations and CPU vector widths. `--tolerance` is the primary mitigation; if your `pass-to-fail` transitions are tiny float drifts, loosen `rtol`/`atol` before assuming a real regression. + +## Programmatic use + +The simplest entry point is `reset_all_rngs`: + +```python +from snapshot_tool import reset_all_rngs + +reset_all_rngs(seed=42) # one-shot: reseed Python random, numpy, torch, tensorflow +``` + +For longer-lived control, use the class directly: + +```python +from snapshot_tool import RNGPatcher + +patcher = RNGPatcher(seed=123) +patcher.patch_all() +# ... run code that consumes randomness ... + +# Or as a context manager +with RNGPatcher(seed=123): + do_random_stuff() +``` + +There's also `patch_all_rngs` / `unpatch_all_rngs` — these wrap a single global `_global_patcher`. The "unpatch" path doesn't actually restore the prior RNG state (you can't; it's already been consumed) — it just allows another `patch_all_rngs(...)` call to take effect. + +## Quick determinism check + +If you suspect a benchmark is non-deterministic despite the patcher: + +```bash +# Capture and verify twice. The second verify should be 100% pass-to-pass. +snapshot-tool capture benchmarks +snapshot-tool baseline benchmarks +snapshot-tool verify benchmarks +jq '."pass-to-fail"' summary.json # expect 0 +``` + +If you see `pass-to-fail` > 0 on this self-comparison, the benchmark is reading entropy from a source `RNGPatcher` doesn't cover. Common culprits: a `Generator` instantiated at import time, `os.urandom`, `secrets`, network/filesystem timestamps, dict ordering of stringly-keyed maps with `PYTHONHASHSEED` unset. + +## Next steps + +- [Tracing](tracing.md) — Why deterministic execution matters for what the tracer captures. +- [Baseline & Verify](baseline-and-verify.md) — How `pass-to-fail` cells are produced. diff --git a/docs/guide/discovery.md b/docs/guide/discovery.md new file mode 100644 index 0000000..f08bb6c --- /dev/null +++ b/docs/guide/discovery.md @@ -0,0 +1,117 @@ +# Discovery + +`BenchmarkDiscovery` finds ASV benchmarks under a directory without ever importing the benchmark modules. It AST-parses each `*.py` file (skipping `__init__.py`) and emits a `BenchmarkInfo` record per discovered benchmark. + +## What counts as a benchmark + +A function or method is a benchmark if its name starts with one of the standard ASV prefixes: + +| Prefix | ASV semantics | +|--------|---------------| +| `time_` | Wall-clock timing | +| `timeraw_` | Wall-clock timing in a fresh subprocess | +| `mem_` | Memory consumption | +| `peakmem_` | Peak memory consumption | +| `track_` | A user-defined metric | + +`snapshot-tool` discovers all five — it doesn't care about ASV's measurement type, only that the function/method exists and is callable. What `snapshot-tool` records is the **return value** captured by the [tracer](tracing.md), not the timing/memory metric. + +## What gets emitted + +Each discovered benchmark becomes a `BenchmarkInfo` dataclass: + +```python +@dataclass +class BenchmarkInfo: + name: str # e.g., "time_union" + module_path: str # dotted path relative to benchmark_dir, e.g., "geometry.union" + benchmark_type: str # "function" or "method" + class_name: Optional[str] = None + params: Optional[list[list[Any]]] = None + param_names: Optional[list[str]] = None + setup_method: Optional[str] = None + has_setup: bool = False + has_setup_cache: bool = False + method_params: Optional[list[str]] = None + needs_runtime_eval: bool = False +``` + +The `module_path` is computed by taking the file's path relative to `benchmark_dir`, stripping the `.py` extension, and replacing path separators with `.`. So `benchmarks/geometry/union.py` becomes `geometry.union`. + +## Function-level vs class-level benchmarks + +`BenchmarkDiscovery` walks each file's top-level body twice: + +1. **Functions at module scope** — `def time_*(...)` declared directly in the file. Emitted as `benchmark_type="function"`. +2. **Classes** — `class TimeFoo:` with one or more `time_*` methods. For each such method, `BenchmarkDiscovery` emits a separate `BenchmarkInfo` with `benchmark_type="method"`, and copies the class's `params`, `param_names`, `setup` / `setup_cache` attributes into every record. + +Functions inside classes are only discovered as methods; nested functions inside `def` blocks are ignored. + +## Parameters + +ASV uses class-level `params` and `param_names` attributes to express parametric benchmarks. `BenchmarkDiscovery` extracts both: + +```python +class TimeUnion: + params = ([10, 100, 1000], ["polygon", "linestring"]) + param_names = ["n", "geom_type"] + + def setup(self, n, geom_type): + ... + + def time_union(self, n, geom_type): + ... +``` + +For each method in the class, the emitted `BenchmarkInfo` carries: + +- `params = [[10, 100, 1000], ["polygon", "linestring"]]` +- `param_names = ["n", "geom_type"]` +- `setup_method = "setup"`, `has_setup = True` + +`runner.get_param_combinations(benchmark)` produces the Cartesian product `[(10, "polygon"), (10, "linestring"), (100, "polygon"), ...]`. + +### Runtime-evaluated params + +ASV allows `params` to be any Python expression — a generator, a comprehension, a function call, a module-level constant. `BenchmarkDiscovery` can only statically extract **literal** values: `ast.Constant`, lists/tuples of constants, and nested combinations of those. Anything else is flagged: + +```python +class TimeFoo: + params = [n for n in range(10)] # ← comprehension, not a literal list + # or + params = list(SOMETHING) # ← call, not a literal list +``` + +When `BenchmarkDiscovery` sees a non-literal expression in `params`, it sets `needs_runtime_eval=True` and leaves `params=None`. The runner then evaluates the class's `params` attribute at runtime (after `_load_module`) and back-fills the field before generating combinations. + +## What discovery skips + +- `__init__.py` — never parsed. +- Files that fail to parse — a warning is logged via `logger.warning(...)`; discovery continues. +- Imports — never followed. If `benchmarks/foo.py` imports from `benchmarks/bar.py`, both are scanned independently as files. +- Inheritance — a class that inherits benchmark methods from a parent class will only have its **own** `time_*` methods discovered. ASV's class-based benchmark inheritance is not modeled. + +## Programmatic use + +```python +from snapshot_tool import BenchmarkDiscovery + +discovery = BenchmarkDiscovery("benchmarks/") +all_benchmarks = discovery.discover_all() + +# Look up a specific benchmark +b = discovery.get_benchmark_by_name("time_union") + +# All benchmarks in one module +geom_benchmarks = discovery.get_benchmarks_by_module("geometry.union") + +# Generate the parameter Cartesian product for a class benchmark +combinations = discovery.generate_parameter_combinations(b) +``` + +Note that `generate_parameter_combinations` on a benchmark with `needs_runtime_eval=True` returns the placeholder `[("",)]` — use `BenchmarkRunner.get_param_combinations(b)` instead, which loads the module and back-fills `params` first. + +## Next steps + +- [Tracing](tracing.md) — How `sys.settrace` decides which return value to capture. +- [CLI guide → `list`](cli.md#list) — The user-facing wrapper around `discover_all()`. diff --git a/docs/guide/storage.md b/docs/guide/storage.md new file mode 100644 index 0000000..37c2cb3 --- /dev/null +++ b/docs/guide/storage.md @@ -0,0 +1,200 @@ +# Snapshot Storage + +`SnapshotManager` owns everything under `--snapshot-dir` (default `.snapshots/`). All captured return values live in a single **SQLite database** at `/snapshots.db`, content-addressed and gzipped for compactness. The per-test **JSON metadata sidecars** are still written to disk in their per-`(module, benchmark)` directory so downstream tooling can read them without opening the database. + +## Directory layout + +``` +.snapshots/ +├── snapshots.db # SQLite DB: blobs + snapshot rows +├── baseline.json # produced by `baseline`; consumed by `verify` +├── / +│ ├── / # function-level benchmark +│ │ └── .json # metadata sidecar (one per snapshot) +│ └── ./ # class-method benchmark +│ └── .json +``` + +- `` is the dotted import path from [discovery](discovery.md), with `.` preserved (not converted back to `/`). `benchmarks/geometry/union.py` → directory `geometry.union/`. +- `` for free functions; `.` for methods. The class is included to disambiguate same-named benchmark methods on different classes in the same module. +- `` is the first 16 hex characters of `md5(repr(parameters))`. Empty `parameters=()` always hashes to `99914b932bd37a50`. + +There are no `.pkl` or `.pkl.gz` files — everything that used to be a pickle now lives in the database. + +## SQLite schema + +Two tables, joined by `blob_hash`: + +```sql +CREATE TABLE blobs ( + hash TEXT PRIMARY KEY, -- sha256 of raw pickle bytes + data BLOB NOT NULL, -- gzipped pickle of return_value + refcount INTEGER NOT NULL DEFAULT 0, + raw_size INTEGER NOT NULL, + compressed_size INTEGER NOT NULL +); + +CREATE TABLE snapshots ( + test_id TEXT PRIMARY KEY, -- "//" + module_path TEXT NOT NULL, + benchmark_name TEXT NOT NULL, + class_name TEXT, + param_hash TEXT NOT NULL, + parameters BLOB NOT NULL, -- pickled tuple + param_names BLOB, -- pickled list[str] or NULL + blob_hash TEXT, -- NULL for failed-capture rows + capture_failed INTEGER NOT NULL DEFAULT 0, + failure_reason TEXT, + timestamp TEXT NOT NULL, + git_commit TEXT, + git_branch TEXT, + python_version TEXT, + platform TEXT, + FOREIGN KEY (blob_hash) REFERENCES blobs(hash) +); + +CREATE INDEX idx_snapshots_module ON snapshots(module_path); +CREATE INDEX idx_snapshots_module_bench ON snapshots(module_path, benchmark_name); +``` + +PRAGMAs set at open: `journal_mode = WAL`, `synchronous = NORMAL`, `foreign_keys = ON`. + +## Why content-addressing + dedup + +Each captured return value is pickled, then sha256-hashed by its raw pickle bytes. The `blobs` table is keyed by that hash, so two benchmarks producing **identical** outputs share one blob — only the `refcount` is incremented. This matters in real benchmark suites because many parameterized benchmarks emit the same trivial value (`None`, `0`, an empty list) across param combinations, and many benchmarks share computed sub-results. + +A representative roundtrip on the bundled shapely suite: + +``` +58 snapshots → 45 unique blobs (≈22% deduplicated) +raw payload size 111 MB +gzipped in SQLite 84 MB (≈24% smaller after gzip) +``` + +Compression ratio depends entirely on what your benchmarks return. Repetitive text or dictionaries shrink 3–10×; incompressible float64 numpy arrays barely move. Either way the dedup wins are independent of the data's compressibility. + +### Refcount lifecycle + +Every `store_snapshot` runs in a single SQLite transaction: + +1. Pickle the return value, compute its sha256. +2. Read the current `blob_hash` (if any) for this `test_id`. +3. Insert the new blob if novel; otherwise bump its `refcount`. +4. Upsert the `snapshots` row with the new `blob_hash`. +5. Decrement the old blob's `refcount`. Delete the blob if it falls to zero. + +`delete_snapshot` follows the same release path. Overwriting a snapshot with the same value is cheap — the old and new blob hashes match, refcount is net zero, no new bytes get written. + +## JSON metadata sidecars + +Every snapshot — successful or failed-capture — gets a JSON sidecar at `.snapshots///.json` containing the full `SnapshotMetadata` block: + +```json +{ + "benchmark_name": "time_union", + "module_path": "geometry.union", + "parameters": [100, "polygon"], + "param_names": ["n", "geom_type"], + "timestamp": "2026-05-14T10:14:32.018000", + "class_name": "TimeUnion", + "git_commit": "abc123def456", + "git_branch": "main", + "python_version": "3.12.5", + "platform": "Darwin-arm64", + "capture_failed": false, + "failure_reason": null +} +``` + +The sidecars are convenience artefacts for downstream tooling (CI summaries, dashboards) — `verify` reads metadata from SQLite, not the JSON. They're rewritten on every `store_snapshot` to stay in sync with the DB row. + +## Serialization of unpicklable values + +Not every Python value can round-trip through `pickle.dumps`/`pickle.loads`. `_serialize_value` tests the round-trip and falls back to a tagged dict when it fails: + +| Tag | Meaning | What `Comparator` does | +|-----|---------|------------------------| +| `__generator__` | A generator (has `__iter__` and `__next__`). | Skip comparison. | +| `__callable__` | A function/closure/method. The dict carries `name`, `qualname`, `module`. | Skip comparison. | +| `__unpicklable__` | Anything else whose pickle round-trip raised; stored as `__str__` plus `__type__`. | Skip comparison. | +| `__class_instance__` | A class instance whose `__dict__` could be serialized but the instance itself couldn't. Carries `__class_name__`, `__module__`, and `__dict__`. | Recursive attribute-by-attribute comparison against the actual instance's `__dict__`. | + +These placeholder dicts are pickleable, so they land in the `blobs` table just like any other captured value. + +## Failed-capture markers + +When a benchmark crashes, times out, or returns something we can't even pickle as a placeholder, `capture` calls `store_failed_capture(...)` instead of `store_snapshot(...)`. The resulting `snapshots` row has `capture_failed = 1`, `blob_hash = NULL`, and a `failure_reason` string describing what went wrong. The JSON sidecar is still written so failed captures are visible on disk without opening the DB. + +`verify` checks `capture_failed` after loading and treats it as a `[SKIP]`. If you captured against a benchmark that's flaky, the first failure goes into the DB and subsequent verifies don't re-fail on it — they skip until you re-capture. + +## `test_id` + +```python +storage.get_test_id( + module_path="geometry.union", + benchmark_name="time_union", + parameters=(100, "polygon"), + class_name="TimeUnion", +) +# → "geometry.union/TimeUnion.time_union/" +``` + +`test_id` is the primary key on the `snapshots` table, and the same string used as the directory path for the JSON sidecar (minus `.snapshots/` and `.json`). It's also the key in `baseline.json`. See [Baseline & Verify](baseline-and-verify.md) for how it's consumed. + +## Programmatic use + +```python +from snapshot_tool import SnapshotManager + +storage = SnapshotManager(".snapshots/") + +# Write +storage.store_snapshot( + benchmark_name="time_compute", + module_path="foo", + parameters=(10,), + param_names=["n"], + return_value=[1.0, 2.0, 3.0], +) + +# Read +loaded = storage.load_snapshot("time_compute", "foo", (10,)) +if loaded is not None: + value, metadata = loaded + +# Inspect +stats = storage.get_snapshot_stats() +# { +# "total_snapshots": N, +# "unique_blobs": M, # <= N when dedup hit +# "modules": [...], "benchmarks": [...], +# "oldest_snapshot": dt, "newest_snapshot": dt, +# "total_size_bytes": , +# "uncompressed_size_bytes": , +# } + +# List +for sidecar_path, meta in storage.list_snapshots(module_path="foo"): + print(sidecar_path, meta.capture_failed) + +# Delete a specific row (decrements the underlying blob refcount automatically) +storage.delete_snapshot("time_compute", "foo", (10,)) + +# Close the underlying SQLite connection (also runs on __del__) +storage.close() +``` + +`store_snapshot` / `store_failed_capture` return the **JSON sidecar path** — useful when downstream tooling wants a stable on-disk artefact to point at. The payload itself lives in `snapshots.db`. + +## Operational notes + +- **Treat `.snapshots/` as disposable build output.** `customtest.sh` `rm -rf`s it before each run. It is not source. +- **Don't commit `snapshots.db` to version control.** Binary diffs are useless, and the metadata fields (`timestamp`, `git_commit`, `platform`) drift on every run. The recommended pattern is to re-capture from a known-good commit on each CI run. +- **The DB is local-only.** No locking story for shared filesystems, no replication. SQLite + WAL handles same-process concurrency safely, but cross-process or cross-host coordination is out of scope. +- **Cross-Python-version pickles.** The pickle format can change between major Python releases. If you capture on Python 3.12 and verify on Python 3.8, expect breakage. CI matrices should capture and verify on the same interpreter. +- **No backward compatibility for `.pkl` / `.pkl.gz` directories.** The SQLite backend is a clean break — existing pickle snapshot directories are not read. Re-capture against the new schema. + +## Next steps + +- [Comparison](comparison.md) — How serialized placeholders are interpreted at verify time. +- [Baseline & Verify](baseline-and-verify.md) — How `baseline.json` is structured and used. diff --git a/docs/guide/tracing.md b/docs/guide/tracing.md new file mode 100644 index 0000000..0230469 --- /dev/null +++ b/docs/guide/tracing.md @@ -0,0 +1,115 @@ +# Tracing + +ASV benchmarks usually don't return anything — they exist to be timed, not to produce output. `snapshot-tool` extracts a return value anyway by installing a `sys.settrace` callback during execution and capturing the **shallowest meaningful user-code return value**. + +## The problem + +A typical ASV benchmark looks like this: + +```python +def time_compute(self, n): + self.solver.run(n) +``` + +There's nothing to compare here. The interesting value is what `self.solver.run(n)` returned internally — but we don't get to see it from the outside, because the function body discards it. + +Two mechanisms cooperate to capture it: + +1. **AST rewrite (preferred)** — Before execution, `BenchmarkRunner` re-parses the benchmark method's source. If the last statement is a bare expression (`self.solver.run(n)`), it rewrites it to `return self.solver.run(n)` and recompiles. This is the cheap path and works for the vast majority of ASV-style benchmarks. It's important because C/Cython-implemented internals are invisible to `sys.settrace`, so the AST rewrite is sometimes the **only** way to capture a value. +2. **`sys.settrace` (fallback / supplement)** — While the benchmark runs, a per-frame trace callback records the return value of every traced call. The shallowest meaningful one — the one closest to the benchmark itself — wins. + +If both fail, the benchmark is still reported as `success=True` with `return_value=None`. The capture is silently empty; verify will read it back as `None == None` (a pass). + +## What "meaningful" means + +`ExecutionTracer._should_trace_frame` filters frames aggressively. A frame is traced only if **all** of the following are true: + +- The frame has a `__name__` in its globals. +- The module isn't in the **stdlib**: + - Python ≥ 3.10: `sys.stdlib_module_names` is consulted (the canonical list). + - Plus a hand-curated set covering common stdlib modules and earlier Python versions (`builtins`, `os`, `pathlib`, `typing`, `re`, `json`, `io`, `threading`, `subprocess`, `argparse`, `logging`, `tempfile`, `traceback`, `pytest`, `_pytest`, ...). +- The module isn't a known third-party internal: + - **numpy** internals: `numpy._core`, `numpy.core`, `numpy.lib`, `numpy.ma`, `numpy.array_api`, `numpy.f2py`, `numpy.fft`, `numpy.linalg`, `numpy.random`, `numpy.testing`, `numpy._`, `numpy.compat`, `numpy.matrixlib`. + - **shapely** internals: `shapely.lib`, `shapely._`, `shapely.geos`, `shapely.geometry.base`. +- The function isn't a dunder method (`__init__`, `__repr__`, `__eq__`, `__len__`, `__iter__`, ...) or a lambda / synthetic name (``, anything starting with `<`). + +The list of suppressed modules is intentionally broad — we want to trace **user benchmark code and the public surface of scientific libraries**, not their internal helpers. A `shapely.geometry.Polygon.union` call gets traced; the `shapely.lib.intersection` C-binding it calls does not. + +## "Shallowest" vs "deepest" + +The dataclass field is called `deepest_call`, but the capture rule prefers **shallower** frames: + +```python +# tracer.py +if arg is not None and not self._is_meaningless_return(arg): + if self.deepest_call is None or self.current_depth <= self.deepest_call.depth: + self.deepest_call = TraceResult(...) +``` + +In other words: every meaningful return value at a depth `<=` the current best replaces it. This biases capture toward the **first user-code call from the benchmark**, not the innermost. + +!!! note + The field name and docstring say "deepest" but the comparator is `<=`, so what you actually get is the shallowest frame. This is intentional — naming hasn't been updated to match the behavior. + +## What's "meaningless" + +`_is_meaningless_return` skips returns that aren't worth snapshotting: + +- `None` — always skipped. +- Empty containers (`len(value) == 0` if `__len__` exists). +- Trivial scalar values (`0`, `0.0`, `""`) — but not `False` (booleans are always kept). + +Class instances are always kept, even if they're empty — they're typically the "real" return value. Numpy arrays, strings (non-empty), and non-zero numbers are always kept. + +## Lifecycle + +```python +tracer = ExecutionTracer(max_depth=100) +tracer.start_tracing() # sys.settrace(self._trace_calls) +try: + benchmark() +finally: + result = tracer.stop_tracing() # sys.settrace(None); returns the captured TraceResult or None +``` + +If `benchmark()` raises before any traced frame returns meaningfully, `result.success = False` and `result.error` is set. If `benchmark()` returns normally but nothing meaningful was traced (e.g., the whole call ran inside C code), `result` is `None`. + +`BenchmarkRunner` normalizes `None` to `TraceResult(success=True, return_value=None)` — it considers "no trace captured" a successful run with empty output, not a failure. + +## `TraceResult` + +```python +@dataclass +class TraceResult: + return_value: Any + function_name: str # name of the frame whose return was captured + module_name: str # module of that frame + depth: int # call depth at capture time + success: bool + error: Optional[Exception] = None +``` + +`function_name` and `module_name` are useful for debugging: they tell you exactly which function the snapshot is coming from. If a snapshot starts failing after a refactor that renames or relocates a helper, this is where you'll see it. + +## Overhead + +`sys.settrace` is meaningfully slow — every Python function call goes through your callback. `snapshot-tool` runs benchmarks under tracing only during `capture` / `verify` / `baseline`. The actual ASV timing job is unaffected; the snapshot harness is separate. + +## Programmatic use + +```python +from snapshot_tool import ExecutionTracer + +tracer = ExecutionTracer() +result = tracer.trace_execution(my_function, arg1, arg2) +if result.success: + print(result.return_value) + +print(tracer.get_trace_stats()) +# {"max_depth_reached": 7, "deepest_call_depth": 1, ...} +``` + +## Next steps + +- [Comparison](comparison.md) — How captured values are compared on `verify`. +- [Determinism](determinism.md) — Why RNGs are reseeded before every trace. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..bcd5790 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,77 @@ +# snapshot-tool + +

+ + FormulaCode Website + + + GitHub + + + ASV + +

+ +`snapshot-tool` is a snapshot-testing harness for [ASV](https://github.com/airspeed-velocity/asv) (airspeed-velocity) benchmarks. It captures the return value of each benchmark on a known-good revision, then re-runs the same benchmarks after a code change and reports which outputs drifted — with tolerance-aware numerical comparison. + +It is the correctness companion to ASV's performance numbers: ASV tells you *how fast* a change is, `snapshot-tool` tells you *whether the result is still correct*. + +## How it works + +```mermaid +graph LR + A[benchmarks/*.py] -->|AST discovery| B + B -->|sys.settrace| C + C -->|pickle + json| D + D -->|tolerance compare| E + + B["Benchmark
Discovery"] + C["Traced
Execution"] + D[".snapshots/"] + E["Verify /
Baseline"] +``` + +The pipeline runs in four phases: + +1. **Discover** — AST-parse each `*.py` under `benchmark_dir` to find ASV-prefixed functions and methods (`time_`, `timeraw_`, `mem_`, `peakmem_`, `track_`). No imports happen at discovery time. +2. **Run** — Execute each benchmark with `sys.settrace` installed, RNGs reseeded, and a per-benchmark timeout. The tracer captures the shallowest meaningful return value from user code (stdlib, numpy internals, and shapely internals are filtered out). +3. **Persist** — Write a row into a single SQLite database at `.snapshots/snapshots.db`. Payloads are pickled, gzipped, and content-addressed by sha256 so benchmarks producing identical outputs share one blob on disk. A JSON metadata sidecar is also written per `(module, class, benchmark, parameters)` tuple for downstream tooling. +4. **Compare** — On a later run, replay each benchmark and compare to the stored snapshot using a pure-Python `isclose` (`rtol` / `atol` / `equal_nan`). + +## Get started + +The full roundtrip is four commands: + +```bash +# 1. See what's discoverable +snapshot-tool list path/to/benchmarks + +# 2. Capture a baseline snapshot of every benchmark's output +snapshot-tool capture path/to/benchmarks + +# 3. (Optional) Record the current pass/fail state as a baseline for transition tracking +snapshot-tool baseline path/to/benchmarks + +# 4. After a code change, verify nothing regressed +snapshot-tool verify path/to/benchmarks --tolerance 1e-5 1e-8 +``` + +See the **[CLI guide](guide/cli.md)** for the full subcommand reference. + +## Key features + +- **Zero-instrumentation capture** — Benchmarks don't need to return anything explicitly. `sys.settrace` snaps the shallowest meaningful user-code return value, so existing ASV benchmarks that just compute-and-discard work as-is. +- **Tolerance-aware comparison** — Pure-Python `isclose` (`rtol` / `atol` / `equal_nan`) for scalars, numpy arrays (element-wise), Python sequences, dicts, and custom classes with `__eq__`. Numpy is optional — detected lazily. +- **Deterministic by construction** — Python `random`, numpy legacy and Generator APIs, PyTorch, and TensorFlow are reseeded before every run via [`RNGPatcher`](guide/determinism.md). The whole point of snapshot testing is bit-stable output. +- **Per-benchmark timeouts** — Each benchmark runs inside a `ThreadPoolExecutor.submit(...).result(timeout=...)` so hung benchmarks fail fast instead of stalling the whole run. +- **One SQLite file, deduplicated and gzipped** — Every payload is pickled, gzipped, and content-addressed by sha256 in a single `snapshots.db`. Benchmarks emitting identical outputs share a single blob via refcount. No tens-of-thousands of `.pkl` files. +- **Baseline → verify transition matrix** — Record a `baseline.json` of pass/fail/skip statuses on one revision, then `verify` against another to get a 3×3 transition matrix (`pass-to-fail`, `fail-to-pass`, etc.) — useful for grading optimization attempts in CI. +- **Python 3.8 → 3.13** — Pure standard library at runtime; numpy is detected lazily and never imported by the tool itself. + +## Quick links + +- [Installation](getting-started/installation.md) — Set up the dev environment with `uv`. +- [**CLI (`snapshot-tool`)**](guide/cli.md) — **The primary entrypoint** — full subcommand reference. +- [Python API Quickstart](getting-started/quickstart.md) — Programmatic usage of `BenchmarkDiscovery`, `BenchmarkRunner`, `SnapshotManager`, and `Comparator`. +- [Baseline & Verify](guide/baseline-and-verify.md) — The 3×3 transition matrix and how to wire it into CI. +- [Configuration](guide/configuration.md) — `snapshot_config.json` and CLI flag reference. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..1225647 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,84 @@ +site_name: snapshot-tool +site_description: Snapshot-testing harness for ASV benchmarks — capture function outputs, verify correctness after optimization. +repo_url: https://github.com/formula-code/snapshot-tester +repo_name: formula-code/snapshot-tester + +theme: + name: material + palette: + - scheme: default + primary: indigo + accent: indigo + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/brightness-4 + name: Switch to light mode + features: + - navigation.tabs + - navigation.sections + - navigation.expand + - navigation.top + - content.code.copy + - content.code.annotate + - search.suggest + - search.highlight + +validation: + links: + not_found: info + +plugins: + - search + - mkdocstrings: + handlers: + python: + paths: ["src"] + options: + docstring_style: google + show_source: true + show_root_heading: true + show_symbol_type_heading: true + members_order: source + merge_init_into_class: true + show_if_no_docstring: false + filters: + - "!^_" + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.highlight: + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.tabbed: + alternate_style: true + - toc: + permalink: true + - attr_list + - md_in_html + +nav: + - Home: index.md + - Getting Started: + - Installation: getting-started/installation.md + - CLI (snapshot-tool): guide/cli.md + - Python API Quickstart: getting-started/quickstart.md + - Configuration: guide/configuration.md + - User Guide: + - Discovery: guide/discovery.md + - Tracing: guide/tracing.md + - Comparison: guide/comparison.md + - Baseline & Verify: guide/baseline-and-verify.md + - Determinism: guide/determinism.md + - Snapshot Storage: guide/storage.md diff --git a/pyproject.toml b/pyproject.toml index 501d155..e990f03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "snapshot-tool" -version = "0.1.0" +version = "0.2.0" description = "Snapshot testing tool for ASV benchmarks - captures and compares function outputs to verify correctness after optimizations" readme = "README.md" authors = [ @@ -55,6 +55,9 @@ dev = [ [tool.ruff] line-length = 100 target-version = "py38" +# tests/test_repos/ holds vendored third-party benchmark sources (astropy, +# pandas, shapely). They are not our code and must not be linted/formatted. +extend-exclude = ["tests/test_repos"] [tool.ruff.lint] select = [ @@ -68,9 +71,23 @@ select = [ ] ignore = [ "E501", # line too long (handled by formatter) - "UP007", # Use X | Y for union types (not compatible with Python 3.8-3.9) + # The package supports Python 3.8+ and deliberately uses typing.Optional / + # typing.List with `from __future__ import annotations`. The pyupgrade rules + # that rewrite to 3.10+ syntax are therefore intentionally disabled. + "UP006", # Use `list` instead of `List` for type annotations + "UP007", # Use `X | Y` for union types + "UP035", # Deprecated typing import (List, etc.) + "UP045", # Use `X | None` for Optional ] +[tool.ruff.lint.per-file-ignores] +# __init__.py configures package logging before importing submodules so their +# module-level loggers inherit the handler — imports are deliberately not at top. +"src/snapshot_tool/__init__.py" = ["E402"] +# Test modules use a logging.basicConfig() preamble before imports, keep unused +# locals for readability, and intentionally exercise lambdas/generators. +"tests/**" = ["E402", "F841", "B007", "C401", "E731"] + [tool.mypy] python_version = "3.8" warn_return_any = true diff --git a/src/snapshot_tool/__init__.py b/src/snapshot_tool/__init__.py index ced569b..7da0502 100644 --- a/src/snapshot_tool/__init__.py +++ b/src/snapshot_tool/__init__.py @@ -4,29 +4,32 @@ This package provides tools to capture and compare function return values from benchmarks to verify correctness after optimizations. """ + from __future__ import annotations import logging import sys -__version__ = "0.1.0" +__version__ = "0.2.0" + # Configure logging for the package def configure_logging(level=logging.INFO): """Configure logging for the snapshot_tool package.""" # Configure the package-level logger - logger = logging.getLogger('snapshot_tool') + logger = logging.getLogger("snapshot_tool") if not logger.handlers: handler = logging.StreamHandler(sys.stdout) handler.setLevel(level) - formatter = logging.Formatter('%(levelname)s: %(message)s') + formatter = logging.Formatter("%(levelname)s: %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) logger.setLevel(level) return logger + # Configure logging by default configure_logging() @@ -35,7 +38,7 @@ def configure_logging(level=logging.INFO): from .comparator import Comparator, ComparisonConfig, ComparisonResult from .config import ConfigManager, SnapshotConfig from .discovery import BenchmarkDiscovery, BenchmarkInfo -from .rng_patcher import RNGPatcher, patch_all_rngs, unpatch_all_rngs, reset_all_rngs +from .rng_patcher import RNGPatcher, patch_all_rngs, reset_all_rngs, unpatch_all_rngs from .runner import BenchmarkRunner from .storage import SnapshotManager, SnapshotMetadata from .tracer import ExecutionTracer, TraceResult diff --git a/src/snapshot_tool/cli.py b/src/snapshot_tool/cli.py index 01db00b..4993c23 100644 --- a/src/snapshot_tool/cli.py +++ b/src/snapshot_tool/cli.py @@ -4,6 +4,7 @@ This module provides CLI commands for capturing and verifying snapshots of ASV benchmark outputs. """ + from __future__ import annotations import argparse @@ -180,7 +181,7 @@ def _capture_command(self, args) -> int: benchmark_dir = args.benchmark_dir snapshot_dir = self.config.get_snapshot_dir() - timeout = args.timeout if hasattr(args, 'timeout') else None + timeout = args.timeout if hasattr(args, "timeout") else None logger.info(f"Capturing snapshots from {benchmark_dir}") logger.info(f"Storing snapshots in {snapshot_dir}") @@ -196,7 +197,9 @@ def _capture_command(self, args) -> int: benchmarks = discovery.discover_all() if args.filter: - benchmarks = [b for b in benchmarks if re.search(args.filter, f"{b.module_path}.{b.name}")] + benchmarks = [ + b for b in benchmarks if re.search(args.filter, f"{b.module_path}.{b.name}") + ] captured_count = 0 @@ -231,7 +234,9 @@ def _capture_command(self, args) -> int: if result and result.error: error_type = type(result.error).__name__ error_msg = str(result.error) - failure_reason = f"{error_type}: {error_msg}" if error_msg else error_type + failure_reason = ( + f"{error_type}: {error_msg}" if error_msg else error_type + ) else: failure_reason = "Unknown error (no exception details)" @@ -243,7 +248,9 @@ def _capture_command(self, args) -> int: failure_reason=failure_reason, class_name=benchmark.class_name, ) - logger.warning(f" Failed to capture with params: {params} - {failure_reason}") + logger.warning( + f" Failed to capture with params: {params} - {failure_reason}" + ) if self.config.verbose and result and result.error: import traceback @@ -301,7 +308,7 @@ def _verify_command(self, args) -> int: benchmark_dir = args.benchmark_dir snapshot_dir = self.config.get_snapshot_dir() - timeout = args.timeout if hasattr(args, 'timeout') else None + timeout = args.timeout if hasattr(args, "timeout") else None logger.info(f"Verifying benchmarks in {benchmark_dir}") logger.info(f"Comparing against snapshots in {snapshot_dir}") @@ -329,7 +336,9 @@ def _verify_command(self, args) -> int: benchmarks = discovery.discover_all() if args.filter: - benchmarks = [b for b in benchmarks if re.search(args.filter, f"{b.module_path}.{b.name}")] + benchmarks = [ + b for b in benchmarks if re.search(args.filter, f"{b.module_path}.{b.name}") + ] total_tests = 0 passed_tests = 0 @@ -394,7 +403,9 @@ def _verify_command(self, args) -> int: if not result or not result.success: # Benchmark failed during verify but succeeded during capture # This is a real failure (non-deterministic benchmark or environment change) - logger.error(f" [FAIL] Failed to run with params: {params} (succeeded during capture)") + logger.error( + f" [FAIL] Failed to run with params: {params} (succeeded during capture)" + ) failed_tests += 1 total_tests += 1 test_id = storage.get_test_id( @@ -587,7 +598,7 @@ def _verify_command(self, args) -> int: ]: logger.info(f" {k}: {summary.get(k, 0)}") - summary_path = args.summary if hasattr(args, 'summary') else Path("summary.json") + summary_path = args.summary if hasattr(args, "summary") else Path("summary.json") try: with open(summary_path, "w") as f: json.dump(summary, f, indent=2) @@ -615,7 +626,7 @@ def _baseline_command(self, args) -> int: benchmark_dir = args.benchmark_dir snapshot_dir = self.config.get_snapshot_dir() - timeout = args.timeout if hasattr(args, 'timeout') else None + timeout = args.timeout if hasattr(args, "timeout") else None logger.info(f"Baselining benchmarks in {benchmark_dir}") logger.info(f"Using snapshots in {snapshot_dir}") @@ -641,7 +652,9 @@ def _baseline_command(self, args) -> int: discovery = BenchmarkDiscovery(benchmark_dir) benchmarks = discovery.discover_all() if args.filter: - benchmarks = [b for b in benchmarks if re.search(args.filter, f"{b.module_path}.{b.name}")] + benchmarks = [ + b for b in benchmarks if re.search(args.filter, f"{b.module_path}.{b.name}") + ] # Collect entries total = 0 @@ -678,27 +691,21 @@ def _baseline_command(self, args) -> int: if snapshot_data is None: entries[test_id] = "skip" skipped += 1 - logger.info( - f" [SKIP: NO SNAPSHOT] No snapshot for params: {params}" - ) + logger.info(f" [SKIP: NO SNAPSHOT] No snapshot for params: {params}") continue expected_value, metadata = snapshot_data if metadata.capture_failed: entries[test_id] = "skip" skipped += 1 - logger.info( - f" [SKIP: FAILED CAPTURE] Failed capture for params: {params}" - ) + logger.info(f" [SKIP: FAILED CAPTURE] Failed capture for params: {params}") continue result = runner.run_benchmark(benchmark, params) if not result or not result.success: entries[test_id] = "fail" failed += 1 - logger.error( - f" [FAIL] Failed to run for params: {params}" - ) + logger.error(f" [FAIL] Failed to run for params: {params}") continue comparison = comparator.compare(result.return_value, expected_value) @@ -782,7 +789,9 @@ def _list_command(self, args) -> int: benchmarks = discovery.discover_all() if args.filter: - benchmarks = [b for b in benchmarks if re.search(args.filter, f"{b.module_path}.{b.name}")] + benchmarks = [ + b for b in benchmarks if re.search(args.filter, f"{b.module_path}.{b.name}") + ] logger.info(f"Found {len(benchmarks)} benchmarks in {benchmark_dir}:") diff --git a/src/snapshot_tool/comparator.py b/src/snapshot_tool/comparator.py index a41e227..01699a4 100644 --- a/src/snapshot_tool/comparator.py +++ b/src/snapshot_tool/comparator.py @@ -4,6 +4,7 @@ This module compares captured outputs with stored snapshots using pure Python numerical comparisons with tolerances. """ + from __future__ import annotations import math @@ -13,6 +14,7 @@ # Optional numpy import - only used for type detection when benchmarks return numpy arrays try: import numpy as np + HAS_NUMPY = True except ImportError: np = None # type: ignore @@ -25,7 +27,7 @@ def _is_numpy_array(obj: Any) -> bool: return isinstance(obj, np.ndarray) # Fallback: check by module and type name obj_type = type(obj) - return obj_type.__module__ == 'numpy' and obj_type.__name__ == 'ndarray' + return obj_type.__module__ == "numpy" and obj_type.__name__ == "ndarray" def _is_numpy_scalar(obj: Any) -> bool: @@ -34,10 +36,12 @@ def _is_numpy_scalar(obj: Any) -> bool: return isinstance(obj, np.number) # Fallback: check by module obj_type = type(obj) - return obj_type.__module__ == 'numpy' + return obj_type.__module__ == "numpy" -def _py_isclose(a: float, b: float, rtol: float = 1e-5, atol: float = 1e-8, equal_nan: bool = False) -> bool: +def _py_isclose( + a: float, b: float, rtol: float = 1e-5, atol: float = 1e-8, equal_nan: bool = False +) -> bool: """Pure Python implementation of numpy.isclose for scalars.""" # Handle NaN and infinity values try: @@ -101,7 +105,7 @@ def compare(self, actual: Any, expected: Any) -> ComparisonResult: return ComparisonResult( match=True, skipped=True, - details="Skipped comparison for generator (cannot be pickled)" + details="Skipped comparison for generator (cannot be pickled)", ) # Handle serialized callables (functions/closures) - skip comparison @@ -170,9 +174,7 @@ def _compare_class_instance(self, actual: Any, expected: dict[str, Any]) -> Comp # Check class name and module expected_class = expected["__class_name__"] - expected_module = expected["__module__"] actual_class = actual.__class__.__name__ - actual_module = getattr(actual.__class__, "__module__", "") if actual_class != expected_class: return ComparisonResult( @@ -215,10 +217,10 @@ def _compare_numpy_arrays(self, actual: Any, expected: Any) -> Optional[Comparis return None # Get shape and dtype info - actual_shape = getattr(actual, 'shape', None) - expected_shape = getattr(expected, 'shape', None) - actual_dtype = getattr(actual, 'dtype', None) - expected_dtype = getattr(expected, 'dtype', None) + actual_shape = getattr(actual, "shape", None) + expected_shape = getattr(expected, "shape", None) + actual_dtype = getattr(actual, "dtype", None) + expected_dtype = getattr(expected, "dtype", None) # Check shapes if strict_shapes is enabled if self.config.strict_shapes and actual_shape != expected_shape: @@ -234,8 +236,10 @@ def _compare_numpy_arrays(self, actual: Any, expected: Any) -> Optional[Comparis error_message=f"Array dtypes differ: {actual_dtype} vs {expected_dtype}", ) - # Handle object arrays (like Shapely geometry arrays) - compare element-wise - if actual_dtype == object or expected_dtype == object: + # Handle object arrays (like Shapely geometry arrays) - compare element-wise. + # `dtype == object` is the idiomatic numpy object-dtype check; `is` would + # be wrong here (a dtype is not the `object` type itself). + if actual_dtype == object or expected_dtype == object: # noqa: E721 return self._compare_object_arrays(actual, expected) # Compare numeric arrays element-wise using pure Python @@ -261,7 +265,9 @@ def _compare_numpy_arrays(self, actual: Any, expected: Any) -> Optional[Comparis continue # Use pure Python isclose - if not _py_isclose(a_float, e_float, self.config.rtol, self.config.atol, self.config.equal_nan): + if not _py_isclose( + a_float, e_float, self.config.rtol, self.config.atol, self.config.equal_nan + ): all_close = False differences.append(abs(a_float - e_float)) @@ -444,7 +450,7 @@ def _compare_objects(self, actual: Any, expected: Any) -> Optional[ComparisonRes # Handle cases where __eq__ returns an array (e.g., SkyCoord, pandas Series) if _is_numpy_array(match): # Call .all() method on the array if available - if hasattr(match, 'all'): + if hasattr(match, "all"): match = bool(match.all()) else: # Fallback: iterate and check all elements @@ -454,7 +460,9 @@ def _compare_objects(self, actual: Any, expected: Any) -> Optional[ComparisonRes # Check if it contains arrays (use len() to avoid evaluating the list as boolean) if _is_numpy_array(match[0]): match = all( - (arr.all() if hasattr(arr, 'all') else bool(arr)) if _is_numpy_array(arr) else arr + (arr.all() if hasattr(arr, "all") else bool(arr)) + if _is_numpy_array(arr) + else arr for arr in match ) @@ -464,14 +472,16 @@ def _compare_objects(self, actual: Any, expected: Any) -> Optional[ComparisonRes ) except Exception as e: return ComparisonResult( - match=False, error_message=f"Object comparison failed: {e}", details={"type": actual_type.__name__} + match=False, + error_message=f"Object comparison failed: {e}", + details={"type": actual_type.__name__}, ) def _compare_fallback(self, actual: Any, expected: Any) -> Optional[ComparisonResult]: """Fallback comparison using == operator.""" try: - # Check type consistency - if type(actual) != type(expected): + # Check type consistency (strict: exact type identity, not isinstance) + if type(actual) is not type(expected): return ComparisonResult( match=False, error_message=f"Type mismatch: {type(actual).__name__} vs {type(expected).__name__}", @@ -544,7 +554,7 @@ def _is_numeric_scalar(self, value: Any) -> bool: # Check numpy scalar types if available if _is_numpy_scalar(value): # Check if it's actually a scalar (not an array) - return not hasattr(value, 'shape') or value.shape == () + return not hasattr(value, "shape") or value.shape == () return False def _is_sequence(self, value: Any) -> bool: diff --git a/src/snapshot_tool/config.py b/src/snapshot_tool/config.py index 235275b..11a1a5e 100644 --- a/src/snapshot_tool/config.py +++ b/src/snapshot_tool/config.py @@ -4,6 +4,7 @@ This module handles loading and managing configuration settings for the snapshot testing tool. """ + from __future__ import annotations import json @@ -49,12 +50,12 @@ def to_dict(self) -> dict[str, Any]: return asdict(self) @classmethod - def from_dict(cls, data: dict[str, Any]) -> "SnapshotConfig": + def from_dict(cls, data: dict[str, Any]) -> SnapshotConfig: """Create from dictionary.""" return cls(**data) @classmethod - def from_file(cls, config_path: Path) -> "SnapshotConfig": + def from_file(cls, config_path: Path) -> SnapshotConfig: """Load configuration from JSON file.""" if not config_path.exists(): return cls() diff --git a/src/snapshot_tool/discovery.py b/src/snapshot_tool/discovery.py index 25b84d4..319995f 100644 --- a/src/snapshot_tool/discovery.py +++ b/src/snapshot_tool/discovery.py @@ -4,6 +4,7 @@ This module parses ASV benchmark files to discover benchmark classes, functions, parameters, and setup methods. """ + from __future__ import annotations import ast diff --git a/src/snapshot_tool/rng_patcher.py b/src/snapshot_tool/rng_patcher.py index b721549..4b9458e 100644 --- a/src/snapshot_tool/rng_patcher.py +++ b/src/snapshot_tool/rng_patcher.py @@ -12,10 +12,11 @@ - PyTorch random number generation (if installed) - TensorFlow random number generation (if installed) """ + from __future__ import annotations import logging -from typing import Any, Optional +from typing import Optional logger = logging.getLogger(__name__) @@ -64,6 +65,7 @@ def unpatch_all(self): def _patch_python_random(self): """Patch Python's built-in random module.""" import random + random.seed(self.seed) logger.debug("Patched Python random module") @@ -71,6 +73,7 @@ def _patch_numpy_legacy(self): """Patch NumPy's legacy random API (compatible with numpy 1.12+/2017).""" try: import numpy as np + np.random.seed(self.seed) logger.debug("Patched NumPy legacy random API") except ImportError: @@ -80,6 +83,7 @@ def _patch_torch(self): """Patch PyTorch random number generation.""" try: import torch # type: ignore + torch.manual_seed(self.seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(self.seed) @@ -91,6 +95,7 @@ def _patch_tensorflow(self): """Patch TensorFlow random number generation.""" try: import tensorflow as tf # type: ignore + tf.random.set_seed(self.seed) logger.debug("Patched TensorFlow random") except ImportError: @@ -154,16 +159,19 @@ def reset_all_rngs(seed: int = 42): seed: The deterministic seed to use. """ import random + random.seed(seed) try: import numpy as np + np.random.seed(seed) except ImportError: pass try: import torch # type: ignore + torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed) @@ -172,6 +180,7 @@ def reset_all_rngs(seed: int = 42): try: import tensorflow as tf # type: ignore + tf.random.set_seed(seed) except ImportError: pass diff --git a/src/snapshot_tool/runner.py b/src/snapshot_tool/runner.py index 9ff86bb..98d9012 100644 --- a/src/snapshot_tool/runner.py +++ b/src/snapshot_tool/runner.py @@ -4,6 +4,7 @@ This module executes benchmarks with tracing enabled and handles setup methods, parameter combinations, and global variable initialization. """ + from __future__ import annotations import importlib @@ -11,7 +12,8 @@ import logging import sys import traceback -from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as FuturesTimeoutError from pathlib import Path from typing import Any, Optional @@ -44,7 +46,7 @@ def __init__( # Cache for loaded modules self._module_cache: dict[str, Any] = {} - + # Cache for setup_cache results (per class instance) self._setup_cache: dict[str, Any] = {} @@ -114,9 +116,7 @@ def _run_with_timeout( executor.shutdown(wait=True) return result except FuturesTimeoutError: - logger.warning( - f"Benchmark {benchmark.name} timed out after {self.timeout} seconds" - ) + logger.warning(f"Benchmark {benchmark.name} timed out after {self.timeout} seconds") # Cancel the future and shutdown without waiting future.cancel() executor.shutdown(wait=False) @@ -286,15 +286,16 @@ def _load_module(self, module_path: str) -> Any: # Ensure parent packages exist in sys.modules to support relative imports # in benchmark files (e.g., `from .pandas_vb_common import setup`). import types - parts = module_path.split('.') - + + parts = module_path.split(".") + # Create parent packages if they don't exist for i in range(1, len(parts)): - pkg_name = '.'.join(parts[:i]) + pkg_name = ".".join(parts[:i]) if pkg_name not in sys.modules: pkg = types.ModuleType(pkg_name) # Mark as a package by setting __path__ to the directory - pkg_dir = self.benchmark_dir / '/'.join(parts[:i]) + pkg_dir = self.benchmark_dir / "/".join(parts[:i]) if pkg_dir.exists() and pkg_dir.is_dir(): pkg.__path__ = [str(pkg_dir)] sys.modules[pkg_name] = pkg @@ -305,12 +306,12 @@ def _load_module(self, module_path: str) -> Any: raise ImportError(f"Could not load module: {module_path}") module = importlib.util.module_from_spec(spec) - + # Set __package__ attribute for relative imports to work # This is critical for modules using relative imports like `from .utils import ...` if len(parts) > 1: # Module is in a subpackage - parent is the package - module.__package__ = '.'.join(parts[:-1]) + module.__package__ = ".".join(parts[:-1]) else: # Module is at root of benchmark_dir # If benchmark_dir has __init__.py, it's a package @@ -323,22 +324,22 @@ def _load_module(self, module_path: str) -> Any: # No __init__.py, but modules might still use relative imports # Create a synthetic package based on directory name module.__package__ = self.benchmark_dir.name - + # Add module to sys.modules before execution (required for relative imports) sys.modules[module_path] = module - + # Ensure the package exists in sys.modules for relative imports to work if module.__package__ and module.__package__ not in sys.modules: pkg = types.ModuleType(module.__package__) # Set __path__ to benchmark_dir or the appropriate parent directory if len(parts) > 1: - pkg_dir = self.benchmark_dir / '/'.join(parts[:-1]) + pkg_dir = self.benchmark_dir / "/".join(parts[:-1]) else: pkg_dir = self.benchmark_dir if pkg_dir.exists() and pkg_dir.is_dir(): pkg.__path__ = [str(pkg_dir)] sys.modules[module.__package__] = pkg - + spec.loader.exec_module(module) # Cache the module @@ -513,7 +514,7 @@ def _run_method_benchmark( # Instantiate the class instance = benchmark_class() - + # Run setup_cache if it exists (once per class, cached) cached_state = None if benchmark.has_setup_cache: @@ -543,22 +544,23 @@ def _run_method_benchmark( # Check if setup expects cached state as first parameter # Inspect the setup method's signature import inspect + setup_expects_state = False if cached_state is not None: try: sig = inspect.signature(setup_method) params = list(sig.parameters.keys()) # Skip 'self' if present - if params and params[0] == 'self': + if params and params[0] == "self": params = params[1:] # Check if first parameter is 'state' or '_state' - if params and (params[0] == 'state' or params[0] == '_state'): + if params and (params[0] == "state" or params[0] == "_state"): setup_expects_state = True except (ValueError, TypeError): # If signature inspection fails, try heuristic # If we have cached_state and setup has parameters, assume it expects state pass - + if parameters: # Call setup with parameters if setup_expects_state and cached_state is not None: @@ -568,58 +570,76 @@ def _run_method_benchmark( except NotImplementedError: # Setup explicitly indicates this parameter combination is not supported # Skip this benchmark run - logger.debug(f"Setup raised NotImplementedError for parameters {parameters}, skipping") + logger.debug( + f"Setup raised NotImplementedError for parameters {parameters}, skipping" + ) return TraceResult( return_value=None, function_name=benchmark.name, module_name=benchmark.module_path, depth=0, success=False, - error=NotImplementedError(f"Parameter combination {parameters} not supported"), + error=NotImplementedError( + f"Parameter combination {parameters} not supported" + ), ) except TypeError as e: # Try without state if that fails try: setup_method(*parameters) except NotImplementedError: - logger.debug(f"Setup raised NotImplementedError for parameters {parameters}, skipping") + logger.debug( + f"Setup raised NotImplementedError for parameters {parameters}, skipping" + ) return TraceResult( return_value=None, function_name=benchmark.name, module_name=benchmark.module_path, depth=0, success=False, - error=NotImplementedError(f"Parameter combination {parameters} not supported"), + error=NotImplementedError( + f"Parameter combination {parameters} not supported" + ), ) except TypeError: - logger.warning(f"Setup method failed with state and parameters: {e}") + logger.warning( + f"Setup method failed with state and parameters: {e}" + ) elif benchmark.param_names: # Use param_names for keyword arguments try: setup_method(*parameters) except NotImplementedError: - logger.debug(f"Setup raised NotImplementedError for parameters {parameters}, skipping") + logger.debug( + f"Setup raised NotImplementedError for parameters {parameters}, skipping" + ) return TraceResult( return_value=None, function_name=benchmark.name, module_name=benchmark.module_path, depth=0, success=False, - error=NotImplementedError(f"Parameter combination {parameters} not supported"), + error=NotImplementedError( + f"Parameter combination {parameters} not supported" + ), ) except TypeError as e: try: param_dict = dict(zip(benchmark.param_names, parameters)) setup_method(**param_dict) except NotImplementedError: - logger.debug(f"Setup raised NotImplementedError for parameters {parameters}, skipping") + logger.debug( + f"Setup raised NotImplementedError for parameters {parameters}, skipping" + ) return TraceResult( return_value=None, function_name=benchmark.name, module_name=benchmark.module_path, depth=0, success=False, - error=NotImplementedError(f"Parameter combination {parameters} not supported"), + error=NotImplementedError( + f"Parameter combination {parameters} not supported" + ), ) except TypeError: logger.warning( @@ -634,14 +654,18 @@ def _run_method_benchmark( try: setup_method(*parameters) except NotImplementedError: - logger.debug(f"Setup raised NotImplementedError for parameters {parameters}, skipping") + logger.debug( + f"Setup raised NotImplementedError for parameters {parameters}, skipping" + ) return TraceResult( return_value=None, function_name=benchmark.name, module_name=benchmark.module_path, depth=0, success=False, - error=NotImplementedError(f"Parameter combination {parameters} not supported"), + error=NotImplementedError( + f"Parameter combination {parameters} not supported" + ), ) except TypeError as e: logger.warning(f"Setup method failed with parameters {parameters}: {e}") @@ -655,7 +679,7 @@ def _run_method_benchmark( try: setup_method(cached_state) except NotImplementedError: - logger.debug(f"Setup raised NotImplementedError, skipping") + logger.debug("Setup raised NotImplementedError, skipping") return TraceResult( return_value=None, function_name=benchmark.name, @@ -674,7 +698,7 @@ def _run_method_benchmark( try: setup_method() except NotImplementedError: - logger.debug(f"Setup raised NotImplementedError, skipping") + logger.debug("Setup raised NotImplementedError, skipping") return TraceResult( return_value=None, function_name=benchmark.name, @@ -693,25 +717,29 @@ def _run_method_benchmark( # Check if benchmark method expects cached state as first parameter # Inspect the benchmark method's signature import inspect + method_expects_state = False if cached_state is not None: try: sig = inspect.signature(benchmark_method) params = list(sig.parameters.keys()) # Skip 'self' if present - if params and params[0] == 'self': + if params and params[0] == "self": params = params[1:] # Check if first parameter is 'state' or '_state' - if params and (params[0] == 'state' or params[0] == '_state'): + if params and (params[0] == "state" or params[0] == "_state"): method_expects_state = True except (ValueError, TypeError): # If signature inspection fails, fall back to method_params check method_expects_state = ( - benchmark.method_params - and len(benchmark.method_params) > 0 - and (benchmark.method_params[0] == "state" or benchmark.method_params[0] == "_state") + benchmark.method_params + and len(benchmark.method_params) > 0 + and ( + benchmark.method_params[0] == "state" + or benchmark.method_params[0] == "_state" + ) ) - + # Execute the benchmark method with parameters # Rule: # - If method expects state and we have cached_state, pass state first diff --git a/src/snapshot_tool/storage.py b/src/snapshot_tool/storage.py index 389f310..6355624 100644 --- a/src/snapshot_tool/storage.py +++ b/src/snapshot_tool/storage.py @@ -1,17 +1,28 @@ """ Snapshot storage and management system. -This module handles storing and retrieving snapshots using pickle files -with an organized directory structure. +Snapshots live in a single SQLite database at ``/snapshots.db``. +The captured return values are pickled, gzipped, and stored in a content-addressed +``blobs`` table (sha256 → gzipped pickle, refcounted) so that benchmarks producing +identical outputs share a single payload on disk. Per-test metadata lives in a +``snapshots`` table that references the blob by hash. + +A JSON metadata sidecar is still written next to where the per-test entry would +have lived under the old layout (``///.json``) +because downstream tooling consumes it. + +The baseline file (``/baseline.json``) is unchanged — it is small, +human-readable, and easy to diff in CI. """ + from __future__ import annotations import gzip import hashlib import json import logging -import os import pickle +import sqlite3 from dataclasses import asdict, dataclass from datetime import datetime from pathlib import Path @@ -19,7 +30,39 @@ logger = logging.getLogger(__name__) -DEFAULT_COMPRESS_THRESHOLD_BYTES = 5 * 1024 * 1024 +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS blobs ( + hash TEXT PRIMARY KEY, + data BLOB NOT NULL, + refcount INTEGER NOT NULL DEFAULT 0, + raw_size INTEGER NOT NULL, + compressed_size INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS snapshots ( + test_id TEXT PRIMARY KEY, + module_path TEXT NOT NULL, + benchmark_name TEXT NOT NULL, + class_name TEXT, + param_hash TEXT NOT NULL, + parameters BLOB NOT NULL, + param_names BLOB, + blob_hash TEXT, + capture_failed INTEGER NOT NULL DEFAULT 0, + failure_reason TEXT, + timestamp TEXT NOT NULL, + git_commit TEXT, + git_branch TEXT, + python_version TEXT, + platform TEXT, + FOREIGN KEY (blob_hash) REFERENCES blobs(hash) +); + +CREATE INDEX IF NOT EXISTS idx_snapshots_module + ON snapshots(module_path); +CREATE INDEX IF NOT EXISTS idx_snapshots_module_bench + ON snapshots(module_path, benchmark_name); +""" @dataclass @@ -28,10 +71,10 @@ class SnapshotMetadata: benchmark_name: str module_path: str - parameters: tuple[Any, ...] - param_names: Optional[list[str]] + parameters: tuple + param_names: Optional[list] timestamp: datetime - class_name: Optional[str] = None # Added to disambiguate benchmarks with same name + class_name: Optional[str] = None git_commit: Optional[str] = None git_branch: Optional[str] = None python_version: Optional[str] = None @@ -39,67 +82,53 @@ class SnapshotMetadata: capture_failed: bool = False failure_reason: Optional[str] = None - def to_dict(self) -> dict[str, Any]: - """Convert to dictionary for JSON serialization.""" + def to_dict(self) -> dict: data = asdict(self) - # Convert datetime to string data["timestamp"] = self.timestamp.isoformat() return data @classmethod - def from_dict(cls, data: dict[str, Any]) -> "SnapshotMetadata": - """Create from dictionary.""" - # Convert timestamp string back to datetime + def from_dict(cls, data: dict) -> SnapshotMetadata: if isinstance(data["timestamp"], str): data["timestamp"] = datetime.fromisoformat(data["timestamp"]) return cls(**data) class SnapshotManager: - """Manages snapshot storage and retrieval.""" + """Manages snapshot storage in a SQLite database with deduplicated, gzipped payloads.""" + + DB_NAME = "snapshots.db" def __init__( - self, snapshot_dir: Path, *, compress_threshold_bytes: int = DEFAULT_COMPRESS_THRESHOLD_BYTES + self, + snapshot_dir, + *, + compress_threshold_bytes: int = 0, # accepted for API back-compat; unused ): + # ``compress_threshold_bytes`` is intentionally accepted but ignored — + # the SQLite backend always gzips every payload. Kept in the signature + # so callers passing it as a keyword don't crash. + del compress_threshold_bytes + self.snapshot_dir = Path(snapshot_dir) self.snapshot_dir.mkdir(parents=True, exist_ok=True) - self.compress_threshold_bytes = compress_threshold_bytes + self.db_path = self.snapshot_dir / self.DB_NAME + self._conn = sqlite3.connect(str(self.db_path)) + self._conn.execute("PRAGMA journal_mode = WAL") + self._conn.execute("PRAGMA synchronous = NORMAL") + self._conn.execute("PRAGMA foreign_keys = ON") + self._conn.executescript(_SCHEMA) + self._conn.commit() + + # ------------------------------------------------------------------ + # Baseline utilities (unchanged — baseline.json stays a flat file) + # ------------------------------------------------------------------ - # ----------------- - # Baseline utilities - # ----------------- def baseline_path(self) -> Path: - """Path to the baseline status file inside the snapshot directory.""" return self.snapshot_dir / "baseline.json" - def get_test_id( - self, - *, - module_path: str, - benchmark_name: str, - parameters: tuple[Any, ...], - class_name: Optional[str] = None, - ) -> str: - """Return a stable identifier for a benchmark + parameters. - - The identifier mirrors the on-disk snapshot layout to ensure stability - across baseline and verify runs, including parameter hashing. - """ - param_hash = self._generate_param_hash(parameters) - benchmark_dir = f"{class_name}.{benchmark_name}" if class_name else benchmark_name - # Avoid accidental path traversal by normalizing components explicitly - return f"{module_path}/{benchmark_dir}/{param_hash}" - - def write_baseline(self, entries: dict[str, str], meta: Optional[dict[str, Any]] = None) -> Path: - """Persist baseline pass/fail statuses under the snapshot directory. - - Args: - entries: Mapping of `test_id` -> one of "pass" | "fail" | "skip". - meta: Optional metadata to include (e.g., counts, dirs). - Returns: - Path to the written file. - """ - payload: dict[str, Any] = { + def write_baseline(self, entries: dict, meta: Optional[dict] = None) -> Path: + payload: dict = { "schema": "snapshot_tool/baseline@2", "timestamp": datetime.now().isoformat(), "entries": entries, @@ -113,8 +142,7 @@ def write_baseline(self, entries: dict[str, str], meta: Optional[dict[str, Any]] json.dump(payload, f, indent=2) return path - def read_baseline(self) -> Optional[dict[str, Any]]: - """Load baseline statuses if present, else None.""" + def read_baseline(self) -> Optional[dict]: path = self.baseline_path() if not path.exists(): return None @@ -125,94 +153,42 @@ def read_baseline(self) -> Optional[dict[str, Any]]: logger.warning(f"Failed to read baseline file {path}: {e}") return None - # ----------------- - # Baseline utilities - # ----------------- - def baseline_path(self) -> Path: - """Path to the baseline status file inside the snapshot directory.""" - return self.snapshot_dir / "baseline.json" - def get_test_id( self, *, module_path: str, benchmark_name: str, - parameters: tuple[Any, ...], + parameters: tuple, class_name: Optional[str] = None, ) -> str: - """Return a stable identifier for a benchmark + parameters. - - The identifier mirrors the on-disk snapshot layout to ensure stability - across baseline and verify runs, including parameter hashing. - """ + """Return a stable identifier for a (benchmark, parameters) pair.""" param_hash = self._generate_param_hash(parameters) - benchmark_dir = f"{class_name}.{benchmark_name}" if class_name else benchmark_name - # Avoid accidental path traversal by normalizing components explicitly - return f"{module_path}/{benchmark_dir}/{param_hash}" - - def write_baseline(self, entries: dict[str, str], meta: Optional[dict[str, Any]] = None) -> Path: - """Persist baseline pass/fail statuses under the snapshot directory. - - Args: - entries: Mapping of `test_id` -> one of "pass" | "fail" | "skip". - meta: Optional metadata to include (e.g., counts, dirs). - Returns: - Path to the written file. - """ - payload: dict[str, Any] = { - "schema": "snapshot_tool/baseline@2", - "timestamp": datetime.now().isoformat(), - "entries": entries, - } - if meta: - payload["meta"] = meta + bench_dir = f"{class_name}.{benchmark_name}" if class_name else benchmark_name + return f"{module_path}/{bench_dir}/{param_hash}" - path = self.baseline_path() - path.parent.mkdir(parents=True, exist_ok=True) - with open(path, "w") as f: - json.dump(payload, f, indent=2) - return path - - def read_baseline(self) -> Optional[dict[str, Any]]: - """Load baseline statuses if present, else None.""" - path = self.baseline_path() - if not path.exists(): - return None - try: - with open(path) as f: - return json.load(f) - except Exception as e: - logger.warning(f"Failed to read baseline file {path}: {e}") - return None + # ------------------------------------------------------------------ + # Public snapshot API + # ------------------------------------------------------------------ def store_snapshot( self, benchmark_name: str, module_path: str, - parameters: tuple[Any, ...], - param_names: Optional[list[str]], + parameters: tuple, + param_names: Optional[list], return_value: Any, class_name: Optional[str] = None, - metadata: Optional[dict[str, Any]] = None, + metadata: Optional[dict] = None, ) -> Path: - """Store a snapshot with its metadata.""" - - # Generate parameter hash for unique identification - param_hash = self._generate_param_hash(parameters) + """Store a snapshot and its metadata; returns the JSON sidecar path.""" - # Create directory structure: .snapshots//./ or .snapshots/// - if class_name: - benchmark_dir = f"{class_name}.{benchmark_name}" - else: - benchmark_dir = benchmark_name - base_path = self.snapshot_dir / module_path / benchmark_dir / param_hash - base_path.parent.mkdir(parents=True, exist_ok=True) + # 1. Serialize the return value (pickle + best-effort placeholder fallback) + serialized = self._serialize_value(return_value) - # Create metadata snapshot_metadata = SnapshotMetadata( benchmark_name=benchmark_name, module_path=module_path, - parameters=parameters, + parameters=tuple(parameters), param_names=param_names, class_name=class_name, timestamp=datetime.now(), @@ -223,59 +199,50 @@ def store_snapshot( **(metadata or {}), ) - # Store the snapshot data - serialized_value = self._serialize_value(return_value) - snapshot_data = {"return_value": serialized_value, "metadata": snapshot_metadata} - - # Attempt to store snapshot; if pickling still fails due to nested - # unpicklables, fall back to a placeholder structure. try: - snapshot_path = self._write_snapshot_data(snapshot_data, base_path) + self._write_row( + snapshot_metadata, + return_value=serialized, + capture_failed=False, + failure_reason=None, + ) except Exception as e: - fallback_data = { - "return_value": { - "__unpicklable__": True, - "__error__": f"Pickle failed: {e}", - }, - "metadata": snapshot_metadata, + # Pickle of the value itself failed (the serializer's placeholders + # should have prevented this; this is a belt-and-braces fallback). + placeholder = { + "__unpicklable__": True, + "__error__": f"Pickle failed: {e}", } - snapshot_path = self._write_snapshot_data(fallback_data, base_path) - - # Store metadata separately as JSON for easy inspection - metadata_path = base_path.with_suffix(".json") - with open(metadata_path, "w") as f: - json.dump(snapshot_metadata.to_dict(), f, indent=2, default=str) + self._write_row( + snapshot_metadata, + return_value=placeholder, + capture_failed=False, + failure_reason=None, + ) - return snapshot_path + return self._write_json_sidecar(snapshot_metadata) def store_failed_capture( self, benchmark_name: str, module_path: str, - parameters: tuple[Any, ...], - param_names: Optional[list[str]], - failure_reason: str, + parameters: tuple, + param_names: Optional[list], + failure_reason, class_name: Optional[str] = None, - metadata: Optional[dict[str, Any]] = None, + metadata: Optional[dict] = None, ) -> Path: - """Store a failed capture marker.""" + """Record a failed capture; returns the JSON sidecar path.""" - # Generate parameter hash for unique identification - param_hash = self._generate_param_hash(parameters) - - # Create directory structure: .snapshots//./ or .snapshots/// - if class_name: - benchmark_dir = f"{class_name}.{benchmark_name}" - else: - benchmark_dir = benchmark_name - base_path = self.snapshot_dir / module_path / benchmark_dir / param_hash - base_path.parent.mkdir(parents=True, exist_ok=True) + # Coerce non-string failure_reason (e.g., a raw Exception passed in by + # an embedding harness) to a string — the column is TEXT. + if failure_reason is not None and not isinstance(failure_reason, str): + failure_reason = f"{type(failure_reason).__name__}: {failure_reason}" - # Create metadata for failed capture snapshot_metadata = SnapshotMetadata( benchmark_name=benchmark_name, module_path=module_path, - parameters=parameters, + parameters=tuple(parameters), param_names=param_names, class_name=class_name, timestamp=datetime.now(), @@ -288,57 +255,415 @@ def store_failed_capture( **(metadata or {}), ) - # Store the failed capture marker - snapshot_data = { - "return_value": None, # No return value for failed captures - "metadata": snapshot_metadata, + self._write_row( + snapshot_metadata, + return_value=None, # no payload for failed captures + capture_failed=True, + failure_reason=failure_reason, + ) + + return self._write_json_sidecar(snapshot_metadata) + + def load_snapshot( + self, + benchmark_name: str, + module_path: str, + parameters: tuple, + class_name: Optional[str] = None, + ): + """Load (return_value, SnapshotMetadata) for a snapshot, or None if missing.""" + + test_id = self.get_test_id( + module_path=module_path, + benchmark_name=benchmark_name, + parameters=tuple(parameters), + class_name=class_name, + ) + row = self._conn.execute( + """ + SELECT s.module_path, s.benchmark_name, s.class_name, + s.parameters, s.param_names, + s.capture_failed, s.failure_reason, s.timestamp, + s.git_commit, s.git_branch, s.python_version, s.platform, + b.data + FROM snapshots s + LEFT JOIN blobs b ON b.hash = s.blob_hash + WHERE s.test_id = ? + """, + (test_id,), + ).fetchone() + + if row is None: + return None + + try: + ( + module_path_db, + benchmark_name_db, + class_name_db, + parameters_blob, + param_names_blob, + capture_failed, + failure_reason, + timestamp, + git_commit, + git_branch, + python_version, + platform, + data, + ) = row + + metadata = SnapshotMetadata( + benchmark_name=benchmark_name_db, + module_path=module_path_db, + parameters=tuple(pickle.loads(parameters_blob)), + param_names=pickle.loads(param_names_blob) + if param_names_blob is not None + else None, + class_name=class_name_db, + timestamp=datetime.fromisoformat(timestamp), + git_commit=git_commit, + git_branch=git_branch, + python_version=python_version, + platform=platform, + capture_failed=bool(capture_failed), + failure_reason=failure_reason, + ) + + if data is None: + # Failed-capture row, or value was never stored + return self._deserialize_value(None), metadata + + raw = gzip.decompress(data) + value = self._deserialize_value(pickle.loads(raw)) + return value, metadata + except Exception as e: + logger.warning(f"Failed to load snapshot {test_id}: {e}") + return None + + def is_failed_capture( + self, + benchmark_name: str, + module_path: str, + parameters: tuple, + ) -> bool: + loaded = self.load_snapshot(benchmark_name, module_path, parameters) + if loaded is None: + return False + _, metadata = loaded + return metadata.capture_failed + + def list_snapshots( + self, + module_path: Optional[str] = None, + benchmark_name: Optional[str] = None, + ): + """List all snapshots as a list of (json_sidecar_path, SnapshotMetadata) tuples.""" + + query = """ + SELECT module_path, benchmark_name, class_name, param_hash, + parameters, param_names, + capture_failed, failure_reason, timestamp, + git_commit, git_branch, python_version, platform + FROM snapshots + """ + clauses = [] + args: list = [] + if module_path: + clauses.append("module_path = ?") + args.append(module_path) + if benchmark_name: + clauses.append("benchmark_name = ?") + args.append(benchmark_name) + if clauses: + query += " WHERE " + " AND ".join(clauses) + + results = [] + for row in self._conn.execute(query, args).fetchall(): + ( + module_path_db, + benchmark_name_db, + class_name_db, + param_hash, + parameters_blob, + param_names_blob, + capture_failed, + failure_reason, + timestamp, + git_commit, + git_branch, + python_version, + platform, + ) = row + try: + metadata = SnapshotMetadata( + benchmark_name=benchmark_name_db, + module_path=module_path_db, + parameters=tuple(pickle.loads(parameters_blob)), + param_names=( + pickle.loads(param_names_blob) if param_names_blob is not None else None + ), + class_name=class_name_db, + timestamp=datetime.fromisoformat(timestamp), + git_commit=git_commit, + git_branch=git_branch, + python_version=python_version, + platform=platform, + capture_failed=bool(capture_failed), + failure_reason=failure_reason, + ) + except Exception as e: + logger.warning(f"Failed to load metadata row for {benchmark_name_db}: {e}") + continue + sidecar = self._sidecar_path( + module_path_db, benchmark_name_db, class_name_db, param_hash + ) + results.append((sidecar, metadata)) + return results + + def delete_snapshot( + self, + benchmark_name: str, + module_path: str, + parameters: tuple, + class_name: Optional[str] = None, + ) -> bool: + """Delete a snapshot row (and decrement the underlying blob refcount).""" + + test_id = self.get_test_id( + module_path=module_path, + benchmark_name=benchmark_name, + parameters=tuple(parameters), + class_name=class_name, + ) + with self._conn: + row = self._conn.execute( + "SELECT blob_hash, param_hash FROM snapshots WHERE test_id = ?", + (test_id,), + ).fetchone() + if row is None: + return False + old_blob_hash, param_hash = row + self._conn.execute("DELETE FROM snapshots WHERE test_id = ?", (test_id,)) + if old_blob_hash is not None: + self._release_blob(old_blob_hash) + + # Remove JSON sidecar if present + sidecar = self._sidecar_path(module_path, benchmark_name, class_name, param_hash) + if sidecar.exists(): + try: + sidecar.unlink() + except OSError: + pass + return True + + def get_snapshot_stats(self) -> dict: + """Aggregate stats over the snapshot DB.""" + + total_snapshots = self._conn.execute("SELECT COUNT(*) FROM snapshots").fetchone()[0] + size_row = self._conn.execute( + "SELECT COALESCE(SUM(compressed_size), 0), COALESCE(SUM(raw_size), 0), COUNT(*) " + "FROM blobs" + ).fetchone() + total_compressed, total_raw, unique_blobs = size_row + + modules = [r[0] for r in self._conn.execute("SELECT DISTINCT module_path FROM snapshots")] + benchmarks = [ + f"{r[0]}.{r[1]}" + for r in self._conn.execute( + "SELECT DISTINCT module_path, benchmark_name FROM snapshots" + ) + ] + + ts_row = self._conn.execute( + "SELECT MIN(timestamp), MAX(timestamp) FROM snapshots" + ).fetchone() + oldest_ts, newest_ts = ts_row + oldest = datetime.fromisoformat(oldest_ts) if oldest_ts else None + newest = datetime.fromisoformat(newest_ts) if newest_ts else None + + return { + "total_snapshots": total_snapshots, + "unique_blobs": unique_blobs, + "modules": modules, + "benchmarks": benchmarks, + "oldest_snapshot": oldest, + "newest_snapshot": newest, + "total_size_bytes": int(total_compressed), + "uncompressed_size_bytes": int(total_raw), } - snapshot_path = self._write_snapshot_data(snapshot_data, base_path) + def close(self) -> None: + """Close the underlying SQLite connection.""" + try: + self._conn.close() + except Exception: + pass + + def __del__(self): + self.close() - # Store metadata separately as JSON for easy inspection - metadata_path = base_path.with_suffix(".json") - with open(metadata_path, "w") as f: - json.dump(snapshot_metadata.to_dict(), f, indent=2, default=str) + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ - return snapshot_path + def _write_row( + self, + meta: SnapshotMetadata, + return_value: Any, + capture_failed: bool, + failure_reason: Optional[str], + ) -> None: + test_id = self.get_test_id( + module_path=meta.module_path, + benchmark_name=meta.benchmark_name, + parameters=meta.parameters, + class_name=meta.class_name, + ) + param_hash = self._generate_param_hash(meta.parameters) + + blob_hash: Optional[str] = None + if not capture_failed: + blob_hash = self._store_blob(return_value) + + params_blob = pickle.dumps(tuple(meta.parameters), protocol=pickle.HIGHEST_PROTOCOL) + param_names_blob = ( + pickle.dumps(list(meta.param_names), protocol=pickle.HIGHEST_PROTOCOL) + if meta.param_names is not None + else None + ) + + with self._conn: + # Decrement refcount of any previously-stored blob for this test_id + prev = self._conn.execute( + "SELECT blob_hash FROM snapshots WHERE test_id = ?", + (test_id,), + ).fetchone() + old_blob_hash = prev[0] if prev else None + + self._conn.execute( + """ + INSERT INTO snapshots ( + test_id, module_path, benchmark_name, class_name, param_hash, + parameters, param_names, + blob_hash, capture_failed, failure_reason, + timestamp, git_commit, git_branch, python_version, platform + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(test_id) DO UPDATE SET + module_path = excluded.module_path, + benchmark_name = excluded.benchmark_name, + class_name = excluded.class_name, + param_hash = excluded.param_hash, + parameters = excluded.parameters, + param_names = excluded.param_names, + blob_hash = excluded.blob_hash, + capture_failed = excluded.capture_failed, + failure_reason = excluded.failure_reason, + timestamp = excluded.timestamp, + git_commit = excluded.git_commit, + git_branch = excluded.git_branch, + python_version = excluded.python_version, + platform = excluded.platform + """, + ( + test_id, + meta.module_path, + meta.benchmark_name, + meta.class_name, + param_hash, + params_blob, + param_names_blob, + blob_hash, + 1 if capture_failed else 0, + failure_reason, + meta.timestamp.isoformat(), + meta.git_commit, + meta.git_branch, + meta.python_version, + meta.platform, + ), + ) + + if old_blob_hash is not None: + self._release_blob(old_blob_hash) + + def _store_blob(self, value: Any) -> str: + """Pickle + gzip the value, insert into ``blobs`` (or bump refcount). Return its hash.""" + raw = pickle.dumps(value, protocol=pickle.HIGHEST_PROTOCOL) + h = hashlib.sha256(raw).hexdigest() + + row = self._conn.execute("SELECT 1 FROM blobs WHERE hash = ?", (h,)).fetchone() + if row is None: + compressed = gzip.compress(raw) + self._conn.execute( + "INSERT INTO blobs (hash, data, refcount, raw_size, compressed_size) " + "VALUES (?, ?, 1, ?, ?)", + (h, compressed, len(raw), len(compressed)), + ) + else: + self._conn.execute( + "UPDATE blobs SET refcount = refcount + 1 WHERE hash = ?", + (h,), + ) + return h + + def _release_blob(self, blob_hash: str) -> None: + """Decrement a blob's refcount; delete the row if it falls to zero.""" + self._conn.execute( + "UPDATE blobs SET refcount = refcount - 1 WHERE hash = ?", + (blob_hash,), + ) + self._conn.execute( + "DELETE FROM blobs WHERE hash = ? AND refcount <= 0", + (blob_hash,), + ) + + def _sidecar_path( + self, + module_path: str, + benchmark_name: str, + class_name: Optional[str], + param_hash: str, + ) -> Path: + bench_dir = f"{class_name}.{benchmark_name}" if class_name else benchmark_name + return self.snapshot_dir / module_path / bench_dir / f"{param_hash}.json" + + def _write_json_sidecar(self, meta: SnapshotMetadata) -> Path: + param_hash = self._generate_param_hash(meta.parameters) + path = self._sidecar_path( + meta.module_path, meta.benchmark_name, meta.class_name, param_hash + ) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + json.dump(meta.to_dict(), f, indent=2, default=str) + return path + + # ------------------------------------------------------------------ + # Value serialization (unchanged placeholder semantics) + # ------------------------------------------------------------------ def _serialize_dict_safely(self, d: dict) -> dict: - """Safely serialize a dictionary, handling generators and other non-serializable values.""" result = {} for key, value in d.items(): try: - # Try to serialize the key - serialized_key = self._serialize_value(key) - # Try to serialize the value - serialized_value = self._serialize_value(value) - result[serialized_key] = serialized_value + result[self._serialize_value(key)] = self._serialize_value(value) except Exception as e: - # If we can't serialize this key-value pair, store an error message result[f"__error_{key}__"] = f"Cannot serialize: {e}" return result def _serialize_value(self, value: Any) -> Any: - """Safely serialize a value, handling class instances and generators.""" try: - # Try to pickle AND unpickle the value to ensure it's truly serializable - # This catches objects that pickle fine but fail to unpickle (e.g., version mismatches) pickled = pickle.dumps(value) - pickle.loads(pickled) # Test round-trip + pickle.loads(pickled) # round-trip test return value except Exception as e: - # Catch all exceptions during pickling/unpickling - # Common issues: PicklingError, TypeError, AttributeError, ImportError, etc. - # Check if it's a generator if hasattr(value, "__iter__") and hasattr(value, "__next__"): - # It's a generator - cannot be pickled return { "__generator__": True, "__generator_type__": type(value).__name__, "__error__": f"Cannot pickle generator: {e}", } - # Check if it's a callable (e.g., local function/closure) if callable(value): return { "__callable__": True, @@ -348,22 +673,15 @@ def _serialize_value(self, value: Any) -> Any: "module": getattr(value, "__module__", ""), } - # If pickling fails, try to create a serializable representation - # First, try to convert iterables to plain lists (for HomogeneousList, etc.) if hasattr(value, "__iter__") and not isinstance(value, (str, bytes, dict)): try: - # Try to convert to a plain list and serialize elements plain_list = [self._serialize_value(item) for item in value] - # Test if the list can be pickled pickle.dumps(plain_list) return plain_list except Exception: - # If list conversion fails, continue to other methods pass if hasattr(value, "__dict__"): - # For class instances, create a dict representation - # Safely serialize the __dict__ to avoid generator issues try: serialized_dict = self._serialize_dict_safely(value.__dict__) except Exception as dict_error: @@ -376,204 +694,28 @@ def _serialize_value(self, value: Any) -> Any: "__dict__": serialized_dict, "__error__": str(e), } - else: - # Fall back to string representation - return { - "__unpicklable__": True, - "__type__": type(value).__name__, - "__str__": str(value), - "__error__": str(e), - } + + return { + "__unpicklable__": True, + "__type__": type(value).__name__, + "__str__": str(value), + "__error__": str(e), + } def _deserialize_value(self, value: Any) -> Any: - """Deserialize a value, handling class instances and generators.""" - if isinstance(value, dict) and value.get("__generator__"): - # This is a serialized generator - cannot be deserialized - return value # Return the dict representation - - if isinstance(value, dict) and value.get("__class_instance__"): - # This is a serialized class instance - # For now, we'll return the dict representation - # In a real implementation, you might want to reconstruct the class - return value + # Tagged dicts (__generator__, __class_instance__, ...) are returned as-is; + # the Comparator interprets them. return value - def load_snapshot( - self, benchmark_name: str, module_path: str, parameters: tuple[Any, ...], class_name: Optional[str] = None - ) -> Optional[tuple[Any, SnapshotMetadata]]: - """Load a snapshot and its metadata.""" - - param_hash = self._generate_param_hash(parameters) - - # Use class name in path if provided - if class_name: - benchmark_dir = f"{class_name}.{benchmark_name}" - else: - benchmark_dir = benchmark_name - base_path = self.snapshot_dir / module_path / benchmark_dir / param_hash - snapshot_path = self._resolve_snapshot_path(base_path) - if snapshot_path is None: - return None - - try: - with self._open_snapshot_file(snapshot_path, "rb") as f: - snapshot_data = pickle.load(f) - - serialized_value = snapshot_data["return_value"] - metadata = snapshot_data["metadata"] - - # Deserialize the return value - return_value = self._deserialize_value(serialized_value) - - return return_value, metadata - - except Exception as e: - logger.warning(f"Failed to load snapshot {snapshot_path}: {e}") - return None - - def is_failed_capture( - self, benchmark_name: str, module_path: str, parameters: tuple[Any, ...] - ) -> bool: - """Check if a snapshot represents a failed capture.""" - snapshot_data = self.load_snapshot(benchmark_name, module_path, parameters) - if snapshot_data is None: - return False - - _, metadata = snapshot_data - return metadata.capture_failed - - def list_snapshots( - self, module_path: Optional[str] = None, benchmark_name: Optional[str] = None - ) -> list[tuple[Path, SnapshotMetadata]]: - """List all available snapshots.""" - snapshots = [] - - search_dir = self.snapshot_dir - if module_path: - search_dir = search_dir / module_path - if benchmark_name: - search_dir = search_dir / benchmark_name - - if not search_dir.exists(): - return snapshots - - # Find all .pkl and .pkl.gz files - snapshot_files = list(search_dir.rglob("*.pkl")) - snapshot_files.extend(search_dir.rglob("*.pkl.gz")) - for pkl_file in snapshot_files: - try: - with self._open_snapshot_file(pkl_file, "rb") as f: - snapshot_data = pickle.load(f) - metadata = snapshot_data["metadata"] - snapshots.append((pkl_file, metadata)) - except Exception as e: - logger.warning(f"Failed to load snapshot {pkl_file}: {e}") - continue - - return snapshots - - def delete_snapshot( - self, benchmark_name: str, module_path: str, parameters: tuple[Any, ...] - ) -> bool: - """Delete a specific snapshot.""" - - param_hash = self._generate_param_hash(parameters) - base_path = self.snapshot_dir / module_path / benchmark_name / param_hash - snapshot_path = self._resolve_snapshot_path(base_path) - metadata_path = base_path.with_suffix(".json") - - deleted = False - - if snapshot_path and snapshot_path.exists(): - snapshot_path.unlink() - deleted = True - - if metadata_path.exists(): - metadata_path.unlink() - - return deleted - - def _write_snapshot_data(self, snapshot_data: dict[str, Any], base_path: Path) -> Path: - """Serialize and store snapshot data, compressing if over size threshold.""" - pickled = pickle.dumps(snapshot_data) - if len(pickled) >= self.compress_threshold_bytes: - snapshot_path = base_path.with_suffix(".pkl.gz") - with gzip.open(snapshot_path, "wb") as f: - f.write(pickled) - else: - snapshot_path = base_path.with_suffix(".pkl") - with open(snapshot_path, "wb") as f: - f.write(pickled) - return snapshot_path - - def _resolve_snapshot_path(self, base_path: Path) -> Optional[Path]: - """Return existing snapshot path (compressed or plain) if present.""" - gz_path = base_path.with_suffix(".pkl.gz") - if gz_path.exists(): - return gz_path - pkl_path = base_path.with_suffix(".pkl") - if pkl_path.exists(): - return pkl_path - return None - - def _open_snapshot_file(self, path: Path, mode: str): - """Open snapshot file, handling gzip when needed.""" - if path.name.endswith(".pkl.gz"): - return gzip.open(path, mode) - return open(path, mode) - - def cleanup_empty_directories(self) -> None: - """Remove empty directories in the snapshot tree.""" - for root, dirs, files in os.walk(self.snapshot_dir, topdown=False): - for dir_name in dirs: - dir_path = Path(root) / dir_name - try: - if not any(dir_path.iterdir()): - dir_path.rmdir() - except OSError: - pass # Directory not empty or permission error - - def get_snapshot_stats(self) -> dict[str, Any]: - """Get statistics about stored snapshots.""" - snapshots = self.list_snapshots() - - stats = { - "total_snapshots": len(snapshots), - "modules": set(), - "benchmarks": set(), - "oldest_snapshot": None, - "newest_snapshot": None, - "total_size_bytes": 0, - } - - for snapshot_path, metadata in snapshots: - stats["modules"].add(metadata.module_path) - stats["benchmarks"].add(f"{metadata.module_path}.{metadata.benchmark_name}") - - if stats["oldest_snapshot"] is None or metadata.timestamp < stats["oldest_snapshot"]: - stats["oldest_snapshot"] = metadata.timestamp - - if stats["newest_snapshot"] is None or metadata.timestamp > stats["newest_snapshot"]: - stats["newest_snapshot"] = metadata.timestamp - - try: - stats["total_size_bytes"] += snapshot_path.stat().st_size - except OSError: - pass - - stats["modules"] = list(stats["modules"]) - stats["benchmarks"] = list(stats["benchmarks"]) - - return stats + # ------------------------------------------------------------------ + # Metadata helpers (git, python, platform) + # ------------------------------------------------------------------ - def _generate_param_hash(self, parameters: tuple[Any, ...]) -> str: - """Generate a hash for parameter combination.""" - # Convert parameters to a string representation for hashing - param_str = str(parameters) + def _generate_param_hash(self, parameters: tuple) -> str: + param_str = str(tuple(parameters)) return hashlib.md5(param_str.encode()).hexdigest()[:16] def _get_git_commit(self) -> Optional[str]: - """Get current git commit hash.""" try: import subprocess @@ -581,13 +723,12 @@ def _get_git_commit(self) -> Optional[str]: ["git", "rev-parse", "HEAD"], capture_output=True, text=True, timeout=5 ) if result.returncode == 0: - return result.stdout.strip()[:12] # Short hash + return result.stdout.strip()[:12] except Exception: pass return None def _get_git_branch(self) -> Optional[str]: - """Get current git branch.""" try: import subprocess @@ -601,13 +742,11 @@ def _get_git_branch(self) -> Optional[str]: return None def _get_python_version(self) -> str: - """Get Python version.""" import sys return f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" def _get_platform(self) -> str: - """Get platform information.""" import platform return f"{platform.system()}-{platform.machine()}" diff --git a/src/snapshot_tool/tracer.py b/src/snapshot_tool/tracer.py index f5b48af..267f2fb 100644 --- a/src/snapshot_tool/tracer.py +++ b/src/snapshot_tool/tracer.py @@ -4,6 +4,7 @@ This module implements a tracing mechanism using sys.settrace to capture the deepest function call's return value during benchmark execution. """ + from __future__ import annotations import sys @@ -261,34 +262,52 @@ def _should_trace_frame(self, frame: types.FrameType) -> bool: # Skip if module_name is None (can happen in some execution contexts) if module_name is None: return False - + function_name = frame.f_code.co_name # Skip excluded modules (manual list) # Check for exact match or proper module prefix (with dot separator) - if any(module_name == excluded or module_name.startswith(excluded + '.') for excluded in self.excluded_modules): + if any( + module_name == excluded or module_name.startswith(excluded + ".") + for excluded in self.excluded_modules + ): return False # Skip all standard library modules using sys.stdlib_module_names (Python 3.10+) - if hasattr(sys, 'stdlib_module_names'): + if hasattr(sys, "stdlib_module_names"): # Check if the module or any parent module is in stdlib - module_parts = module_name.split('.') + module_parts = module_name.split(".") for i in range(len(module_parts)): - partial_name = '.'.join(module_parts[:i+1]) + partial_name = ".".join(module_parts[: i + 1]) if partial_name in sys.stdlib_module_names: return False # Skip numpy internals and other common third-party library internals numpy_internal_patterns = [ - 'numpy._core', 'numpy.core', 'numpy.lib', 'numpy.ma', 'numpy.array_api', - 'numpy.f2py', 'numpy.fft', 'numpy.linalg', 'numpy.random', 'numpy.testing', - 'numpy._', 'numpy.compat', 'numpy.matrixlib' + "numpy._core", + "numpy.core", + "numpy.lib", + "numpy.ma", + "numpy.array_api", + "numpy.f2py", + "numpy.fft", + "numpy.linalg", + "numpy.random", + "numpy.testing", + "numpy._", + "numpy.compat", + "numpy.matrixlib", ] if any(module_name.startswith(pattern) for pattern in numpy_internal_patterns): return False # Skip shapely internals - only trace top-level shapely functions - shapely_internal_patterns = ['shapely.lib', 'shapely._', 'shapely.geos', 'shapely.geometry.base'] + shapely_internal_patterns = [ + "shapely.lib", + "shapely._", + "shapely.geos", + "shapely.geometry.base", + ] if any(module_name.startswith(pattern) for pattern in shapely_internal_patterns): return False @@ -345,7 +364,6 @@ def _is_class_instance(self, value: Any) -> bool: class_name = value.__class__.__name__ module_name = getattr(value.__class__, "__module__", "") or "" - # Skip built-in types and common library types if module_name in ("builtins", "types", "collections", "typing"): return False diff --git a/src/snapshot_tool/transitions.py b/src/snapshot_tool/transitions.py index ab86507..fd75974 100644 --- a/src/snapshot_tool/transitions.py +++ b/src/snapshot_tool/transitions.py @@ -5,10 +5,12 @@ of test_id->status, computes the 3x3 {pass,fail,skip}-to-{pass,fail,skip} transition counts. """ + from __future__ import annotations from typing import Dict + def _normalize_status(status: str) -> str: """Normalize status strings to one of pass|fail|skip. @@ -38,7 +40,7 @@ def compute_transitions( """ # Determine dynamic state sets from data actually present # Baseline states come only from baseline entries - baseline_states = set(_normalize_status(s) for s in baseline_entries.values()) + baseline_states = {_normalize_status(s) for s in baseline_entries.values()} # Verify states come from overlapping tests (present in both) verify_states: set[str] = set() @@ -47,9 +49,7 @@ def compute_transitions( verify_states.add(_normalize_status(verify_entries[tid])) # Initialize all pair keys seen in the data - transitions: Dict[str, int] = { - f"{a}-to-{b}": 0 for a in baseline_states for b in verify_states - } + transitions: Dict[str, int] = {f"{a}-to-{b}": 0 for a in baseline_states for b in verify_states} for test_id, b_status_raw in baseline_entries.items(): v_status_raw = verify_entries.get(test_id) diff --git a/tests/test_class_capture.py b/tests/test_class_capture.py index f304e94..597f066 100644 --- a/tests/test_class_capture.py +++ b/tests/test_class_capture.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 import logging + logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format='%(message)s') +logging.basicConfig(level=logging.INFO, format="%(message)s") """ Test script to verify class instance capture and failed capture handling. """ @@ -17,12 +18,16 @@ def test_class_instance_capture(): # Initialize components benchmark_dir = Path(__file__).parent - runner = BenchmarkRunner(benchmark_dir) + runner = BenchmarkRunner(benchmark_dir, timeout=30) storage = SnapshotManager(benchmark_dir / ".snapshots") - # Discover benchmarks + # Discover benchmarks. benchmark_dir is the whole tests/ tree, which also + # contains the vendored real-repo suites under tests/test_repos/ (~1500 + # benchmarks). This test only exercises class-instance capture, so scope it + # to the test_class_instance fixture module — otherwise the loop below would + # execute every vendored benchmark and run for hours. discovery = BenchmarkDiscovery(benchmark_dir) - benchmarks = discovery.discover_all() + benchmarks = [b for b in discovery.discover_all() if b.module_path == "test_class_instance"] logger.info(f"Found {len(benchmarks)} benchmarks") diff --git a/tests/test_cli_roundtrip.py b/tests/test_cli_roundtrip.py index 7bdc9b9..e7640e0 100644 --- a/tests/test_cli_roundtrip.py +++ b/tests/test_cli_roundtrip.py @@ -6,6 +6,7 @@ This mimics the behavior of customtest.sh. """ + import os import shutil import subprocess @@ -29,9 +30,15 @@ def snapshot_dir(): yield Path(temp_dir) shutil.rmtree(temp_dir, ignore_errors=True) + def list_snapshot_files(snapshot_dir: Path) -> list[Path]: - """List snapshot files, including compressed ones.""" - return list(snapshot_dir.rglob("*.pkl")) + list(snapshot_dir.rglob("*.pkl.gz")) + """List snapshot artefacts produced by a capture. + + Under the SQLite backend, the payloads all live in ``snapshots.db``; the + per-test JSON metadata sidecars are 1:1 with snapshot rows, so we use them + as the count. + """ + return [p for p in snapshot_dir.rglob("*.json") if p.name != "baseline.json"] def _get_cli_filter() -> Optional[str]: @@ -71,12 +78,7 @@ def run_snapshot_roundtrip(benchmark_dir: Path, snapshot_dir: Path, timeout_minu list_args.extend(["--filter", filter_pattern]) # Step 1: List benchmarks - list_result = subprocess.run( - list_args, - capture_output=True, - text=True, - timeout=60 - ) + list_result = subprocess.run(list_args, capture_output=True, text=True, timeout=60) capture_args = [ "snapshot-tool", @@ -92,10 +94,7 @@ def run_snapshot_roundtrip(benchmark_dir: Path, snapshot_dir: Path, timeout_minu # Step 2: Capture snapshots capture_result = subprocess.run( - capture_args, - capture_output=True, - text=True, - timeout=timeout_seconds + capture_args, capture_output=True, text=True, timeout=timeout_seconds ) verify_args = [ @@ -112,10 +111,7 @@ def run_snapshot_roundtrip(benchmark_dir: Path, snapshot_dir: Path, timeout_minu # Step 3: Verify snapshots verify_result = subprocess.run( - verify_args, - capture_output=True, - text=True, - timeout=timeout_seconds + verify_args, capture_output=True, text=True, timeout=timeout_seconds ) return list_result, capture_result, verify_result @@ -169,9 +165,10 @@ def assert_roundtrip_succeeds(result, step_name: str, repo_name: str): # Extract pass/fail/skip counts import re - passed_match = re.search(r'Passed:\s*(\d+)', output) - failed_match = re.search(r'Failed:\s*(\d+)', output) - skipped_match = re.search(r'Skipped:\s*(\d+)', output) + + passed_match = re.search(r"Passed:\s*(\d+)", output) + failed_match = re.search(r"Failed:\s*(\d+)", output) + skipped_match = re.search(r"Skipped:\s*(\d+)", output) if passed_match and failed_match: passed = int(passed_match.group(1)) @@ -188,9 +185,7 @@ def assert_roundtrip_succeeds(result, step_name: str, repo_name: str): ) # Ensure at least some benchmarks ran - assert total > 0, ( - f"{step_name} for {repo_name} had no benchmarks:\n{output}" - ) + assert total > 0, f"{step_name} for {repo_name} had no benchmarks:\n{output}" # Ensure at least one benchmark passed (not all skipped) assert passed > 0, ( @@ -228,7 +223,10 @@ def test_astropy_full_roundtrip(self, test_repos_dir, snapshot_dir): snapshots = list_snapshot_files(snapshot_dir) if len(snapshots) == 0: # All benchmarks were skipped - that's OK, but verify should reflect this - assert "skipped" in verify_result.stdout.lower() or "no snapshots" in verify_result.stdout.lower() + assert ( + "skipped" in verify_result.stdout.lower() + or "no snapshots" in verify_result.stdout.lower() + ) class TestPandasRoundtrip: @@ -260,7 +258,10 @@ def test_pandas_full_roundtrip(self, test_repos_dir, snapshot_dir): snapshots = list_snapshot_files(snapshot_dir) if len(snapshots) == 0: # All benchmarks were skipped - that's OK - assert "skipped" in verify_result.stdout.lower() or "no snapshots" in verify_result.stdout.lower() + assert ( + "skipped" in verify_result.stdout.lower() + or "no snapshots" in verify_result.stdout.lower() + ) class TestShapelyRoundtrip: @@ -313,12 +314,7 @@ def test_shapely_multiple_verify_passes(self, test_repos_dir, snapshot_dir): if benchmark_timeout is not None: capture_args.extend(["--timeout", str(benchmark_timeout)]) - capture_result = subprocess.run( - capture_args, - capture_output=True, - text=True, - timeout=300 - ) + capture_result = subprocess.run(capture_args, capture_output=True, text=True, timeout=300) assert_roundtrip_succeeds(capture_result, "Capture", "shapely_benchmarks") # Verify three times - all should pass with no failures @@ -335,12 +331,7 @@ def test_shapely_multiple_verify_passes(self, test_repos_dir, snapshot_dir): if benchmark_timeout is not None: verify_args.extend(["--timeout", str(benchmark_timeout)]) - verify_result = subprocess.run( - verify_args, - capture_output=True, - text=True, - timeout=300 - ) + verify_result = subprocess.run(verify_args, capture_output=True, text=True, timeout=300) assert_roundtrip_succeeds(verify_result, "Verify", "shapely_benchmarks") @@ -354,11 +345,7 @@ def test_all_repos_roundtrip(self, test_repos_dir, snapshot_dir): Test roundtrip for all three repos sequentially. This is the equivalent of running customtest.sh. """ - repos = [ - "astropy_benchmarks", - "pandas_benchmarks", - "shapely_benchmarks" - ] + repos = ["astropy_benchmarks", "pandas_benchmarks", "shapely_benchmarks"] results = {} @@ -376,31 +363,32 @@ def test_all_repos_roundtrip(self, test_repos_dir, snapshot_dir): ) results[repo_name] = { - 'list': list_result.returncode, - 'capture': capture_result.returncode, - 'verify': verify_result.returncode, - 'verify_output': verify_result.stdout + verify_result.stderr + "list": list_result.returncode, + "capture": capture_result.returncode, + "verify": verify_result.returncode, + "verify_output": verify_result.stdout + verify_result.stderr, } # All operations should succeed failed_repos = [] for repo_name, result in results.items(): - if result['list'] != 0: + if result["list"] != 0: failed_repos.append(f"{repo_name}: list failed") - if result['capture'] not in [0, 1]: + if result["capture"] not in [0, 1]: failed_repos.append(f"{repo_name}: capture crashed") - if result['verify'] != 0: + if result["verify"] != 0: failed_repos.append(f"{repo_name}: verify failed") # Check for failures in verify output - output_lower = result['verify_output'].lower() + output_lower = result["verify_output"].lower() if "failed" in output_lower and "0 failed" not in output_lower: # Look for actual failure counts import re - failure_match = re.search(r'(\d+)\s+failed', output_lower) + + failure_match = re.search(r"(\d+)\s+failed", output_lower) if failure_match and int(failure_match.group(1)) > 0: failed_repos.append(f"{repo_name}: verify had failures") - assert len(failed_repos) == 0, ( - "Some repositories failed roundtrip test:\n" + "\n".join(failed_repos) + assert len(failed_repos) == 0, "Some repositories failed roundtrip test:\n" + "\n".join( + failed_repos ) diff --git a/tests/test_comparator_comprehensive.py b/tests/test_comparator_comprehensive.py index 01a0dd0..3f1a17c 100644 --- a/tests/test_comparator_comprehensive.py +++ b/tests/test_comparator_comprehensive.py @@ -1,7 +1,9 @@ """Comprehensive tests for Comparator.""" + import logging + logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format='%(message)s') +logging.basicConfig(level=logging.INFO, format="%(message)s") import numpy as np import pytest @@ -112,9 +114,9 @@ def test_empty_arrays(self, comparator): def test_structured_arrays(self, comparator): """Test structured numpy arrays.""" - dt = np.dtype([('name', 'U10'), ('age', 'i4')]) - arr1 = np.array([('Alice', 25), ('Bob', 30)], dtype=dt) - arr2 = np.array([('Alice', 25), ('Bob', 30)], dtype=dt) + dt = np.dtype([("name", "U10"), ("age", "i4")]) + arr1 = np.array([("Alice", 25), ("Bob", 30)], dtype=dt) + arr2 = np.array([("Alice", 25), ("Bob", 30)], dtype=dt) result = comparator.compare(arr1, arr2) # Pure Python implementation can compare structured arrays element-wise @@ -246,40 +248,40 @@ class TestDictionaryComparison: def test_identical_dicts(self, comparator): """Test identical dictionaries.""" - dict1 = {'a': 1, 'b': 2, 'c': 3} - dict2 = {'a': 1, 'b': 2, 'c': 3} + dict1 = {"a": 1, "b": 2, "c": 3} + dict2 = {"a": 1, "b": 2, "c": 3} result = comparator.compare(dict1, dict2) assert result.match is True def test_different_values(self, comparator): """Test dicts with different values.""" - dict1 = {'a': 1, 'b': 2} - dict2 = {'a': 1, 'b': 3} + dict1 = {"a": 1, "b": 2} + dict2 = {"a": 1, "b": 3} result = comparator.compare(dict1, dict2) assert result.match is False def test_different_keys(self, comparator): """Test dicts with different keys.""" - dict1 = {'a': 1, 'b': 2} - dict2 = {'a': 1, 'c': 2} + dict1 = {"a": 1, "b": 2} + dict2 = {"a": 1, "c": 2} result = comparator.compare(dict1, dict2) assert result.match is False def test_nested_dicts(self, comparator): """Test nested dictionaries.""" - dict1 = {'a': {'b': {'c': 1}}} - dict2 = {'a': {'b': {'c': 1}}} + dict1 = {"a": {"b": {"c": 1}}} + dict2 = {"a": {"b": {"c": 1}}} result = comparator.compare(dict1, dict2) assert result.match is True def test_dicts_with_numpy_values(self, comparator): """Test dicts containing numpy arrays.""" - dict1 = {'arr': np.array([1, 2, 3]), 'num': 42} - dict2 = {'arr': np.array([1, 2, 3]), 'num': 42} + dict1 = {"arr": np.array([1, 2, 3]), "num": 42} + dict2 = {"arr": np.array([1, 2, 3]), "num": 42} result = comparator.compare(dict1, dict2) assert result.match is True @@ -291,16 +293,8 @@ def test_empty_dicts(self, comparator): def test_complex_nested_structures(self, comparator): """Test complex nested dict/list structures.""" - struct1 = { - 'list': [1, 2, [3, 4]], - 'dict': {'a': {'b': 'c'}}, - 'array': np.array([5, 6, 7]) - } - struct2 = { - 'list': [1, 2, [3, 4]], - 'dict': {'a': {'b': 'c'}}, - 'array': np.array([5, 6, 7]) - } + struct1 = {"list": [1, 2, [3, 4]], "dict": {"a": {"b": "c"}}, "array": np.array([5, 6, 7])} + struct2 = {"list": [1, 2, [3, 4]], "dict": {"a": {"b": "c"}}, "array": np.array([5, 6, 7])} result = comparator.compare(struct1, struct2) assert result.match is True @@ -311,6 +305,7 @@ class TestClassInstanceComparison: def test_same_class_instances_with_eq(self, comparator): """Test class instances with __eq__ implemented.""" + class Point: def __init__(self, x, y): self.x = x @@ -327,6 +322,7 @@ def __eq__(self, other): def test_different_class_instances(self, comparator): """Test different class instances.""" + class Point: def __init__(self, x, y): self.x = x @@ -343,6 +339,7 @@ def __eq__(self, other): def test_class_instances_with_dict_comparison(self, comparator): """Test comparing class instance __dict__ attributes.""" + class TestClass: def __init__(self, value): self.value = value @@ -357,6 +354,7 @@ def __init__(self, value): def test_serialized_class_instances(self, comparator): """Test comparison of actual class instance vs serialized representation.""" + # Create a real class instance class TestClass: def __init__(self, value): @@ -366,10 +364,10 @@ def __init__(self, value): # Serialized representation (as stored by SnapshotManager) serialized = { - '__class_instance__': True, - '__class_name__': 'TestClass', - '__module__': '__main__', - '__dict__': {'value': 42} + "__class_instance__": True, + "__class_name__": "TestClass", + "__module__": "__main__", + "__dict__": {"value": 42}, } result = comparator.compare(actual_instance, serialized) @@ -381,8 +379,8 @@ class TestSpecialCases: def test_generator_comparison(self, comparator): """Test comparison of generator markers.""" - gen1 = {'__generator__': True} - gen2 = {'__generator__': True} + gen1 = {"__generator__": True} + gen2 = {"__generator__": True} result = comparator.compare(gen1, gen2) # Generators should be skipped or match trivially @@ -390,8 +388,8 @@ def test_generator_comparison(self, comparator): def test_callable_comparison(self, comparator): """Test comparison of callable markers.""" - callable1 = {'__callable__': True} - callable2 = {'__callable__': True} + callable1 = {"__callable__": True} + callable2 = {"__callable__": True} result = comparator.compare(callable1, callable2) assert result.match is True @@ -470,10 +468,11 @@ def test_very_small_numbers(self, comparator): def test_deeply_nested_structures(self, comparator): """Test very deeply nested structures.""" + def create_nested(depth): if depth == 0: return 42 - return {'nested': create_nested(depth - 1)} + return {"nested": create_nested(depth - 1)} struct1 = create_nested(10) struct2 = create_nested(10) @@ -503,10 +502,10 @@ def test_unicode_in_structures(self, comparator): def test_bytes_comparison(self, comparator): """Test bytes comparison.""" - result = comparator.compare(b'hello', b'hello') + result = comparator.compare(b"hello", b"hello") assert result.match is True - result = comparator.compare(b'hello', b'world') + result = comparator.compare(b"hello", b"world") assert result.match is False def test_set_comparison(self, comparator): diff --git a/tests/test_debug_fixes.py b/tests/test_debug_fixes.py index ee6cf2e..f6df33d 100644 --- a/tests/test_debug_fixes.py +++ b/tests/test_debug_fixes.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 import logging + logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format='%(message)s') +logging.basicConfig(level=logging.INFO, format="%(message)s") """ Simple test script to verify the snapshot testing tool fixes. """ @@ -50,9 +51,7 @@ def test_simple_benchmark(): result = runner.run_benchmark(simple_benchmark) if result and result.success: - logger.info( - f"[PASS] Successfully captured return value: {type(result.return_value)}" - ) + logger.info(f"[PASS] Successfully captured return value: {type(result.return_value)}") logger.info(f" Function: {result.function_name}") logger.info(f" Module: {result.module_name}") logger.info(f" Depth: {result.depth}") @@ -102,9 +101,7 @@ def test_parameterized_benchmark(): result = runner.run_benchmark(param_benchmark, params) if result and result.success: - logger.info( - f"[PASS] Successfully captured return value: {type(result.return_value)}" - ) + logger.info(f"[PASS] Successfully captured return value: {type(result.return_value)}") logger.info(f" Function: {result.function_name}") logger.info(f" Module: {result.module_name}") logger.info(f" Depth: {result.depth}") diff --git a/tests/test_discovery_comprehensive.py b/tests/test_discovery_comprehensive.py index 1901b4a..8d7af01 100644 --- a/tests/test_discovery_comprehensive.py +++ b/tests/test_discovery_comprehensive.py @@ -1,7 +1,9 @@ """Comprehensive tests for BenchmarkDiscovery.""" + import logging + logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format='%(message)s') +logging.basicConfig(level=logging.INFO, format="%(message)s") import shutil import tempfile @@ -70,7 +72,7 @@ def track_tracker(): benchmarks = discovery.discover_all() assert len(benchmarks) == 5 - prefixes = {b.name.split('_')[0] for b in benchmarks} + prefixes = {b.name.split("_")[0] for b in benchmarks} assert prefixes == {"time", "timeraw", "mem", "peakmem", "track"} def test_function_with_docstring(self, temp_benchmark_dir, discovery): @@ -166,13 +168,13 @@ def time_benchmark(self): assert len(benchmarks) == 1 benchmark = benchmarks[0] assert benchmark.params is not None - assert benchmark.param_names == ['number', 'letter'] + assert benchmark.param_names == ["number", "letter"] # Test parameter combination generation combinations = discovery.generate_parameter_combinations(benchmark) assert len(combinations) == 6 # 3 * 2 = 6 combinations - assert (1, 'a') in combinations - assert (3, 'b') in combinations + assert (1, "a") in combinations + assert (3, "b") in combinations def test_params_without_names(self, temp_benchmark_dir, discovery): """Test parameters without explicit names.""" @@ -230,7 +232,7 @@ def time_benchmark(self): combinations = discovery.generate_parameter_combinations(benchmark) assert len(combinations) == 6 # 2 * 3 = 6 - assert ([1, 2], 'x') in combinations + assert ([1, 2], "x") in combinations class TestModuleStructure: @@ -432,4 +434,4 @@ def time_test(self): assert benchmark.class_name == "MyBenchmark" assert benchmark.has_setup is True assert benchmark.params is not None - assert benchmark.param_names == ['x'] + assert benchmark.param_names == ["x"] diff --git a/tests/test_integration.py b/tests/test_integration.py index 0c3ab90..6252563 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -1,7 +1,9 @@ """Integration tests for end-to-end workflows.""" + import logging + logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format='%(message)s') +logging.basicConfig(level=logging.INFO, format="%(message)s") import shutil import tempfile @@ -31,11 +33,7 @@ def workspace(): snapshot_dir = workspace_path / ".snapshots" snapshot_dir.mkdir() - yield { - 'root': workspace_path, - 'benchmarks': benchmark_dir, - 'snapshots': snapshot_dir - } + yield {"root": workspace_path, "benchmarks": benchmark_dir, "snapshots": snapshot_dir} shutil.rmtree(temp_dir, ignore_errors=True) @@ -46,7 +44,7 @@ class TestCaptureWorkflow: def test_capture_simple_function_benchmark(self, workspace): """Test capturing a simple function benchmark.""" # Create benchmark file - bench_file = workspace['benchmarks'] / "simple.py" + bench_file = workspace["benchmarks"] / "simple.py" bench_file.write_text(""" import numpy as np @@ -55,30 +53,30 @@ def time_simple(): """) # Discover benchmarks - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() assert len(benchmarks) == 1 # Run benchmark - runner = BenchmarkRunner(workspace['benchmarks']) + runner = BenchmarkRunner(workspace["benchmarks"]) result = runner.run_benchmark(benchmarks[0]) assert result.success is True assert isinstance(result.return_value, np.ndarray) # Store snapshot - storage = SnapshotManager(workspace['snapshots']) + storage = SnapshotManager(workspace["snapshots"]) snapshot_path = storage.store_snapshot( benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) assert snapshot_path.exists() def test_capture_parameterized_benchmark(self, workspace): """Test capturing parameterized benchmarks.""" - bench_file = workspace['benchmarks'] / "parameterized.py" + bench_file = workspace["benchmarks"] / "parameterized.py" bench_file.write_text(""" import numpy as np @@ -94,7 +92,7 @@ def time_compute(self): return self.data * self.multiplier """) - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() assert len(benchmarks) == 1 @@ -103,8 +101,8 @@ def time_compute(self): assert len(param_combinations) == 4 # 2 * 2 # Run and capture all combinations - runner = BenchmarkRunner(workspace['benchmarks']) - storage = SnapshotManager(workspace['snapshots']) + runner = BenchmarkRunner(workspace["benchmarks"]) + storage = SnapshotManager(workspace["snapshots"]) for params in param_combinations: result = runner.run_benchmark(benchmarks[0], params) @@ -115,7 +113,7 @@ def time_compute(self): module_path=benchmarks[0].module_path, parameters=params, param_names=benchmarks[0].param_names, - return_value=result.return_value + return_value=result.return_value, ) # Verify all snapshots were created @@ -124,7 +122,7 @@ def time_compute(self): def test_capture_multiple_benchmarks(self, workspace): """Test capturing multiple benchmarks in one file.""" - bench_file = workspace['benchmarks'] / "multiple.py" + bench_file = workspace["benchmarks"] / "multiple.py" bench_file.write_text(""" def time_bench1(): return 42 @@ -136,12 +134,12 @@ def time_bench3(): return {'key': 'value'} """) - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() assert len(benchmarks) == 3 - runner = BenchmarkRunner(workspace['benchmarks']) - storage = SnapshotManager(workspace['snapshots']) + runner = BenchmarkRunner(workspace["benchmarks"]) + storage = SnapshotManager(workspace["snapshots"]) # Capture all benchmarks for benchmark in benchmarks: @@ -153,7 +151,7 @@ def time_bench3(): module_path=benchmark.module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) # Verify @@ -166,7 +164,7 @@ class TestVerifyWorkflow: def test_verify_unchanged_benchmark(self, workspace): """Test verifying a benchmark that hasn't changed.""" - bench_file = workspace['benchmarks'] / "verify.py" + bench_file = workspace["benchmarks"] / "verify.py" bench_file.write_text(""" import numpy as np @@ -175,27 +173,25 @@ def time_verify(): """) # Initial capture - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() - runner = BenchmarkRunner(workspace['benchmarks']) + runner = BenchmarkRunner(workspace["benchmarks"]) result = runner.run_benchmark(benchmarks[0]) - storage = SnapshotManager(workspace['snapshots']) + storage = SnapshotManager(workspace["snapshots"]) storage.store_snapshot( benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) # Re-run and verify result2 = runner.run_benchmark(benchmarks[0]) loaded_value, _ = storage.load_snapshot( - benchmark_name=benchmarks[0].name, - module_path=benchmarks[0].module_path, - parameters=() + benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=() ) comparator = Comparator() @@ -204,26 +200,26 @@ def time_verify(): def test_detect_changed_benchmark(self, workspace): """Test detecting when a benchmark output changes.""" - bench_file = workspace['benchmarks'] / "changed.py" + bench_file = workspace["benchmarks"] / "changed.py" bench_file.write_text(""" def time_changed(): return 42 """) # Initial capture - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() - runner = BenchmarkRunner(workspace['benchmarks']) + runner = BenchmarkRunner(workspace["benchmarks"]) result = runner.run_benchmark(benchmarks[0]) - storage = SnapshotManager(workspace['snapshots']) + storage = SnapshotManager(workspace["snapshots"]) storage.store_snapshot( benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) # Modify benchmark @@ -233,17 +229,15 @@ def time_changed(): """) # Re-discover and run - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() # Need to reload module - runner = BenchmarkRunner(workspace['benchmarks']) + runner = BenchmarkRunner(workspace["benchmarks"]) result2 = runner.run_benchmark(benchmarks[0]) loaded_value, _ = storage.load_snapshot( - benchmark_name=benchmarks[0].name, - module_path=benchmarks[0].module_path, - parameters=() + benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=() ) comparator = Comparator() @@ -253,7 +247,7 @@ def time_changed(): def test_verify_with_tolerance(self, workspace): """Test verification with numerical tolerance.""" - bench_file = workspace['benchmarks'] / "tolerance.py" + bench_file = workspace["benchmarks"] / "tolerance.py" bench_file.write_text(""" import numpy as np @@ -263,28 +257,26 @@ def time_tolerance(): """) # Initial capture - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() - runner = BenchmarkRunner(workspace['benchmarks']) + runner = BenchmarkRunner(workspace["benchmarks"]) result = runner.run_benchmark(benchmarks[0]) - storage = SnapshotManager(workspace['snapshots']) + storage = SnapshotManager(workspace["snapshots"]) storage.store_snapshot( benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) # Create slightly different value new_value = np.array([1.0000001, 2.0000001, 3.0000001]) loaded_value, _ = storage.load_snapshot( - benchmark_name=benchmarks[0].name, - module_path=benchmarks[0].module_path, - parameters=() + benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=() ) # Should pass with default tolerance @@ -299,7 +291,7 @@ class TestComplexScenarios: def test_mixed_benchmark_types(self, workspace): """Test file with both functions and classes.""" - bench_file = workspace['benchmarks'] / "mixed.py" + bench_file = workspace["benchmarks"] / "mixed.py" bench_file.write_text(""" import numpy as np @@ -317,16 +309,16 @@ def time_method(self): return np.sum(self.data) """) - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() # Should find both function and methods assert len(benchmarks) == 2 types = {b.benchmark_type for b in benchmarks} - assert types == {'function', 'method'} + assert types == {"function", "method"} - runner = BenchmarkRunner(workspace['benchmarks']) - storage = SnapshotManager(workspace['snapshots']) + runner = BenchmarkRunner(workspace["benchmarks"]) + storage = SnapshotManager(workspace["snapshots"]) # Capture all for benchmark in benchmarks: @@ -340,7 +332,7 @@ def time_method(self): module_path=benchmark.module_path, parameters=params, param_names=benchmark.param_names, - return_value=result.return_value + return_value=result.return_value, ) else: result = runner.run_benchmark(benchmark) @@ -350,13 +342,13 @@ def time_method(self): module_path=benchmark.module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) def test_nested_module_structure(self, workspace): """Test benchmarks in nested directories.""" # Create nested structure - subdir = workspace['benchmarks'] / "submodule" + subdir = workspace["benchmarks"] / "submodule" subdir.mkdir() (subdir / "nested_bench.py").write_text(""" @@ -364,48 +356,46 @@ def time_nested(): return "nested_result" """) - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() assert len(benchmarks) == 1 assert "submodule" in benchmarks[0].module_path - runner = BenchmarkRunner(workspace['benchmarks']) + runner = BenchmarkRunner(workspace["benchmarks"]) result = runner.run_benchmark(benchmarks[0]) assert result.success is True def test_failed_benchmark_handling(self, workspace): """Test handling of benchmarks that fail.""" - bench_file = workspace['benchmarks'] / "failing.py" + bench_file = workspace["benchmarks"] / "failing.py" bench_file.write_text(""" def time_failing(): raise ValueError("This benchmark fails") """) - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() - runner = BenchmarkRunner(workspace['benchmarks']) + runner = BenchmarkRunner(workspace["benchmarks"]) result = runner.run_benchmark(benchmarks[0]) assert result.success is False assert result.error is not None # Store failed capture - storage = SnapshotManager(workspace['snapshots']) + storage = SnapshotManager(workspace["snapshots"]) storage.store_failed_capture( benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=(), param_names=None, - failure_reason=result.error + failure_reason=result.error, ) # Verify it's marked as failed assert storage.is_failed_capture( - benchmark_name=benchmarks[0].name, - module_path=benchmarks[0].module_path, - parameters=() + benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=() ) @@ -414,7 +404,7 @@ class TestDataTypeRoundTrip: def test_numpy_array_round_trip(self, workspace): """Test numpy array survives capture and load.""" - bench_file = workspace['benchmarks'] / "numpy_test.py" + bench_file = workspace["benchmarks"] / "numpy_test.py" bench_file.write_text(""" import numpy as np @@ -422,25 +412,23 @@ def time_numpy(): return np.array([[1, 2, 3], [4, 5, 6]]) """) - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() - runner = BenchmarkRunner(workspace['benchmarks']) + runner = BenchmarkRunner(workspace["benchmarks"]) result = runner.run_benchmark(benchmarks[0]) - storage = SnapshotManager(workspace['snapshots']) + storage = SnapshotManager(workspace["snapshots"]) storage.store_snapshot( benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) loaded, _ = storage.load_snapshot( - benchmark_name=benchmarks[0].name, - module_path=benchmarks[0].module_path, - parameters=() + benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=() ) assert isinstance(loaded, np.ndarray) @@ -450,7 +438,7 @@ def time_numpy(): def test_complex_dict_round_trip(self, workspace): """Test complex nested structure round trip.""" - bench_file = workspace['benchmarks'] / "dict_test.py" + bench_file = workspace["benchmarks"] / "dict_test.py" bench_file.write_text(""" import numpy as np @@ -463,25 +451,23 @@ def time_complex(): } """) - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() - runner = BenchmarkRunner(workspace['benchmarks']) + runner = BenchmarkRunner(workspace["benchmarks"]) result = runner.run_benchmark(benchmarks[0]) - storage = SnapshotManager(workspace['snapshots']) + storage = SnapshotManager(workspace["snapshots"]) storage.store_snapshot( benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) loaded, _ = storage.load_snapshot( - benchmark_name=benchmarks[0].name, - module_path=benchmarks[0].module_path, - parameters=() + benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, parameters=() ) comparator = Comparator() @@ -494,7 +480,7 @@ class TestParameterCombinations: def test_multiple_parameter_dimensions(self, workspace): """Test benchmarks with multiple parameter dimensions.""" - bench_file = workspace['benchmarks'] / "multi_param.py" + bench_file = workspace["benchmarks"] / "multi_param.py" bench_file.write_text(""" import numpy as np @@ -511,15 +497,15 @@ def time_multi(self): return f"{self.letter}{self.num}_{self.flag}" """) - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() param_combinations = discovery.generate_parameter_combinations(benchmarks[0]) # 3 * 2 * 2 = 12 combinations assert len(param_combinations) == 12 - runner = BenchmarkRunner(workspace['benchmarks']) - storage = SnapshotManager(workspace['snapshots']) + runner = BenchmarkRunner(workspace["benchmarks"]) + storage = SnapshotManager(workspace["snapshots"]) # Capture all combinations for params in param_combinations: @@ -531,7 +517,7 @@ def time_multi(self): module_path=benchmarks[0].module_path, parameters=params, param_names=benchmarks[0].param_names, - return_value=result.return_value + return_value=result.return_value, ) # Verify all were stored @@ -540,7 +526,7 @@ def time_multi(self): def test_parameter_uniqueness(self, workspace): """Test that different parameters create unique snapshots.""" - bench_file = workspace['benchmarks'] / "unique_params.py" + bench_file = workspace["benchmarks"] / "unique_params.py" bench_file.write_text(""" class UniqueParams: params = ([1, 2],) @@ -553,11 +539,11 @@ def time_unique(self): return self.x * 10 """) - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() - runner = BenchmarkRunner(workspace['benchmarks']) - storage = SnapshotManager(workspace['snapshots']) + runner = BenchmarkRunner(workspace["benchmarks"]) + storage = SnapshotManager(workspace["snapshots"]) param_combinations = discovery.generate_parameter_combinations(benchmarks[0]) @@ -571,7 +557,7 @@ def time_unique(self): module_path=benchmarks[0].module_path, parameters=params, param_names=benchmarks[0].param_names, - return_value=result.return_value + return_value=result.return_value, ) # Verify each parameter has unique result @@ -579,7 +565,7 @@ def time_unique(self): loaded, _ = storage.load_snapshot( benchmark_name=benchmarks[0].name, module_path=benchmarks[0].module_path, - parameters=params + parameters=params, ) assert loaded == expected_value @@ -590,7 +576,7 @@ class TestEndToEnd: def test_full_capture_verify_cycle(self, workspace): """Test a complete capture and verify cycle.""" # Create benchmarks - bench_file = workspace['benchmarks'] / "full_test.py" + bench_file = workspace["benchmarks"] / "full_test.py" bench_file.write_text(""" import numpy as np @@ -609,13 +595,13 @@ def time_param(self): """) # Phase 1: Discovery - discovery = BenchmarkDiscovery(workspace['benchmarks']) + discovery = BenchmarkDiscovery(workspace["benchmarks"]) benchmarks = discovery.discover_all() assert len(benchmarks) == 2 # Phase 2: Capture - runner = BenchmarkRunner(workspace['benchmarks']) - storage = SnapshotManager(workspace['snapshots']) + runner = BenchmarkRunner(workspace["benchmarks"]) + storage = SnapshotManager(workspace["snapshots"]) captured_count = 0 for benchmark in benchmarks: @@ -629,7 +615,7 @@ def time_param(self): module_path=benchmark.module_path, parameters=params, param_names=benchmark.param_names, - return_value=result.return_value + return_value=result.return_value, ) captured_count += 1 else: @@ -640,7 +626,7 @@ def time_param(self): module_path=benchmark.module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) captured_count += 1 @@ -659,7 +645,7 @@ def time_param(self): loaded_value, _ = storage.load_snapshot( benchmark_name=benchmark.name, module_path=benchmark.module_path, - parameters=params + parameters=params, ) comparison = comparator.compare(result.return_value, loaded_value) assert comparison.match is True @@ -670,7 +656,7 @@ def time_param(self): loaded_value, _ = storage.load_snapshot( benchmark_name=benchmark.name, module_path=benchmark.module_path, - parameters=() + parameters=(), ) comparison = comparator.compare(result.return_value, loaded_value) assert comparison.match is True @@ -693,17 +679,14 @@ def shapely_workspace(self): snapshot_dir = Path(temp_dir) / ".snapshots" snapshot_dir.mkdir() - yield { - 'benchmarks': benchmark_dir, - 'snapshots': snapshot_dir - } + yield {"benchmarks": benchmark_dir, "snapshots": snapshot_dir} # Cleanup shutil.rmtree(temp_dir, ignore_errors=True) def test_shapely_benchmark_discovery(self, shapely_workspace): """Test that we can discover shapely benchmarks.""" - discovery = BenchmarkDiscovery(shapely_workspace['benchmarks']) + discovery = BenchmarkDiscovery(shapely_workspace["benchmarks"]) benchmarks = discovery.discover_all() # Shapely has multiple benchmark classes @@ -713,7 +696,7 @@ def test_shapely_benchmark_discovery(self, shapely_workspace): benchmark_names = {b.name for b in benchmarks} # These are classes from the shapely benchmarks - expected_classes = {'PointPolygonTimeSuite', 'IOSuite', 'ConstructorsSuite'} + expected_classes = {"PointPolygonTimeSuite", "IOSuite", "ConstructorsSuite"} found_classes = {b.class_name for b in benchmarks if b.class_name} # At least some of these should be found @@ -721,22 +704,22 @@ def test_shapely_benchmark_discovery(self, shapely_workspace): def test_shapely_capture_and_verify_subset(self, shapely_workspace): """Test capturing and verifying a subset of shapely benchmarks.""" - discovery = BenchmarkDiscovery(shapely_workspace['benchmarks']) + discovery = BenchmarkDiscovery(shapely_workspace["benchmarks"]) benchmarks = discovery.discover_all() # Filter to a small, fast benchmark for testing # ConstructorsSuite.time_point is a simple microbenchmark test_benchmark = None for b in benchmarks: - if b.class_name == 'ConstructorsSuite' and b.name == 'time_point': + if b.class_name == "ConstructorsSuite" and b.name == "time_point": test_benchmark = b break if test_benchmark is None: pytest.skip("Could not find ConstructorsSuite.time_point benchmark") - runner = BenchmarkRunner(shapely_workspace['benchmarks']) - storage = SnapshotManager(shapely_workspace['snapshots']) + runner = BenchmarkRunner(shapely_workspace["benchmarks"]) + storage = SnapshotManager(shapely_workspace["snapshots"]) # Capture the benchmark result = runner.run_benchmark(test_benchmark) @@ -751,7 +734,7 @@ def test_shapely_capture_and_verify_subset(self, shapely_workspace): module_path=test_benchmark.module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) # Verify the snapshot @@ -761,7 +744,7 @@ def test_shapely_capture_and_verify_subset(self, shapely_workspace): loaded_value, _ = storage.load_snapshot( benchmark_name=test_benchmark.name, module_path=test_benchmark.module_path, - parameters=() + parameters=(), ) comparator = Comparator() @@ -770,21 +753,21 @@ def test_shapely_capture_and_verify_subset(self, shapely_workspace): def test_shapely_determinism_multiple_verifies(self, shapely_workspace): """Test that verification is deterministic - running verify twice should always pass.""" - discovery = BenchmarkDiscovery(shapely_workspace['benchmarks']) + discovery = BenchmarkDiscovery(shapely_workspace["benchmarks"]) benchmarks = discovery.discover_all() # Find a simple benchmark to test test_benchmark = None for b in benchmarks: - if b.class_name == 'ConstructorsSuite' and b.name == 'time_point': + if b.class_name == "ConstructorsSuite" and b.name == "time_point": test_benchmark = b break if test_benchmark is None: pytest.skip("Could not find ConstructorsSuite.time_point benchmark") - runner = BenchmarkRunner(shapely_workspace['benchmarks']) - storage = SnapshotManager(shapely_workspace['snapshots']) + runner = BenchmarkRunner(shapely_workspace["benchmarks"]) + storage = SnapshotManager(shapely_workspace["snapshots"]) comparator = Comparator() # Initial capture @@ -796,7 +779,7 @@ def test_shapely_determinism_multiple_verifies(self, shapely_workspace): module_path=test_benchmark.module_path, parameters=(), param_names=None, - return_value=capture_result.return_value + return_value=capture_result.return_value, ) # First verification @@ -806,7 +789,7 @@ def test_shapely_determinism_multiple_verifies(self, shapely_workspace): loaded_value1, _ = storage.load_snapshot( benchmark_name=test_benchmark.name, module_path=test_benchmark.module_path, - parameters=() + parameters=(), ) comparison1 = comparator.compare(verify1_result.return_value, loaded_value1) @@ -819,7 +802,7 @@ def test_shapely_determinism_multiple_verifies(self, shapely_workspace): loaded_value2, _ = storage.load_snapshot( benchmark_name=test_benchmark.name, module_path=test_benchmark.module_path, - parameters=() + parameters=(), ) comparison2 = comparator.compare(verify2_result.return_value, loaded_value2) @@ -832,7 +815,7 @@ def test_shapely_determinism_multiple_verifies(self, shapely_workspace): loaded_value3, _ = storage.load_snapshot( benchmark_name=test_benchmark.name, module_path=test_benchmark.module_path, - parameters=() + parameters=(), ) comparison3 = comparator.compare(verify3_result.return_value, loaded_value3) @@ -840,21 +823,21 @@ def test_shapely_determinism_multiple_verifies(self, shapely_workspace): def test_shapely_with_setup_method(self, shapely_workspace): """Test a shapely benchmark that has a setup method - uses time_distance which returns numpy arrays.""" - discovery = BenchmarkDiscovery(shapely_workspace['benchmarks']) + discovery = BenchmarkDiscovery(shapely_workspace["benchmarks"]) benchmarks = discovery.discover_all() # Find PointPolygonTimeSuite.time_distance which has setup and returns numpy array test_benchmark = None for b in benchmarks: - if b.class_name == 'PointPolygonTimeSuite' and b.name == 'time_distance': + if b.class_name == "PointPolygonTimeSuite" and b.name == "time_distance": test_benchmark = b break if test_benchmark is None: pytest.skip("Could not find PointPolygonTimeSuite.time_distance benchmark") - runner = BenchmarkRunner(shapely_workspace['benchmarks']) - storage = SnapshotManager(shapely_workspace['snapshots']) + runner = BenchmarkRunner(shapely_workspace["benchmarks"]) + storage = SnapshotManager(shapely_workspace["snapshots"]) # Run the benchmark (it should handle setup internally) result = runner.run_benchmark(test_benchmark) @@ -869,7 +852,7 @@ def test_shapely_with_setup_method(self, shapely_workspace): module_path=test_benchmark.module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) # Verify determinism - important for benchmarks with random data in setup @@ -879,7 +862,7 @@ def test_shapely_with_setup_method(self, shapely_workspace): loaded_value, _ = storage.load_snapshot( benchmark_name=test_benchmark.name, module_path=test_benchmark.module_path, - parameters=() + parameters=(), ) comparator = Comparator() @@ -888,20 +871,22 @@ def test_shapely_with_setup_method(self, shapely_workspace): def test_shapely_full_capture_and_verify_cycle(self, shapely_workspace): """Full integration test: capture shapely benchmarks and verify determinism.""" - discovery = BenchmarkDiscovery(shapely_workspace['benchmarks']) + discovery = BenchmarkDiscovery(shapely_workspace["benchmarks"]) benchmarks = discovery.discover_all() # Filter to fast benchmarks fast_benchmarks = [ - b for b in benchmarks - if b.class_name == 'ConstructorsSuite' and b.name in ['time_point', 'time_linestring_from_numpy'] + b + for b in benchmarks + if b.class_name == "ConstructorsSuite" + and b.name in ["time_point", "time_linestring_from_numpy"] ] if len(fast_benchmarks) == 0: pytest.skip("Could not find ConstructorsSuite benchmarks") - runner = BenchmarkRunner(shapely_workspace['benchmarks']) - storage = SnapshotManager(shapely_workspace['snapshots']) + runner = BenchmarkRunner(shapely_workspace["benchmarks"]) + storage = SnapshotManager(shapely_workspace["snapshots"]) comparator = Comparator() # Phase 1: Capture @@ -914,7 +899,7 @@ def test_shapely_full_capture_and_verify_cycle(self, shapely_workspace): module_path=benchmark.module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) captured_benchmarks.append(benchmark) @@ -925,37 +910,41 @@ def test_shapely_full_capture_and_verify_cycle(self, shapely_workspace): all_passed = True for benchmark in captured_benchmarks: result = runner.run_benchmark(benchmark) - assert result.success, f"Benchmark {benchmark.name} failed on verify round {verify_round + 1}" + assert result.success, ( + f"Benchmark {benchmark.name} failed on verify round {verify_round + 1}" + ) loaded_value, _ = storage.load_snapshot( - benchmark_name=benchmark.name, - module_path=benchmark.module_path, - parameters=() + benchmark_name=benchmark.name, module_path=benchmark.module_path, parameters=() ) comparison = comparator.compare(result.return_value, loaded_value) if not comparison.match: all_passed = False - logger.info(f"Round {verify_round + 1} failed for {benchmark.name}: {comparison.error_message}") + logger.info( + f"Round {verify_round + 1} failed for {benchmark.name}: {comparison.error_message}" + ) - assert all_passed, f"Verification round {verify_round + 1} should pass (determinism check)" + assert all_passed, ( + f"Verification round {verify_round + 1} should pass (determinism check)" + ) def test_shapely_with_random_data_determinism(self, shapely_workspace): """Test that benchmarks using np.random produce deterministic results with seed management.""" - discovery = BenchmarkDiscovery(shapely_workspace['benchmarks']) + discovery = BenchmarkDiscovery(shapely_workspace["benchmarks"]) benchmarks = discovery.discover_all() # Find PointPolygonTimeSuite.time_distance which uses random data test_benchmark = None for b in benchmarks: - if b.class_name == 'PointPolygonTimeSuite' and b.name == 'time_distance': + if b.class_name == "PointPolygonTimeSuite" and b.name == "time_distance": test_benchmark = b break if test_benchmark is None: pytest.skip("Could not find PointPolygonTimeSuite.time_distance benchmark") - runner = BenchmarkRunner(shapely_workspace['benchmarks']) + runner = BenchmarkRunner(shapely_workspace["benchmarks"]) comparator = Comparator() # Run 3 times - should be identical due to seed reset @@ -980,13 +969,13 @@ def test_shapely_with_random_data_determinism(self, shapely_workspace): def test_shapely_multiple_benchmarks_verify(self, shapely_workspace): """Test running multiple shapely benchmarks and verifying all.""" - discovery = BenchmarkDiscovery(shapely_workspace['benchmarks']) + discovery = BenchmarkDiscovery(shapely_workspace["benchmarks"]) benchmarks = discovery.discover_all() target_benchmarks = [ - ('ConstructorsSuite', 'time_point'), - ('ConstructorsSuite', 'time_linestring_from_numpy'), - ('ConstructorsSuite', 'time_linearring_from_numpy'), + ("ConstructorsSuite", "time_point"), + ("ConstructorsSuite", "time_linestring_from_numpy"), + ("ConstructorsSuite", "time_linearring_from_numpy"), ] selected_benchmarks = [] @@ -999,8 +988,8 @@ def test_shapely_multiple_benchmarks_verify(self, shapely_workspace): if len(selected_benchmarks) < 2: pytest.skip("Could not find enough benchmarks") - runner = BenchmarkRunner(shapely_workspace['benchmarks']) - storage = SnapshotManager(shapely_workspace['snapshots']) + runner = BenchmarkRunner(shapely_workspace["benchmarks"]) + storage = SnapshotManager(shapely_workspace["snapshots"]) comparator = Comparator() # Capture all @@ -1014,7 +1003,7 @@ def test_shapely_multiple_benchmarks_verify(self, shapely_workspace): module_path=benchmark.module_path, parameters=(), param_names=None, - return_value=result.return_value + return_value=result.return_value, ) assert len(capture_results) >= 2, "Should capture at least 2 benchmarks" @@ -1027,9 +1016,7 @@ def test_shapely_multiple_benchmarks_verify(self, shapely_workspace): assert result.success is True loaded_value, _ = storage.load_snapshot( - benchmark_name=benchmark.name, - module_path=benchmark.module_path, - parameters=() + benchmark_name=benchmark.name, module_path=benchmark.module_path, parameters=() ) comparison = comparator.compare(result.return_value, loaded_value) diff --git a/tests/test_minimal_debug.py b/tests/test_minimal_debug.py index 5c3c32f..daca893 100644 --- a/tests/test_minimal_debug.py +++ b/tests/test_minimal_debug.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 import logging + logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format='%(message)s') +logging.basicConfig(level=logging.INFO, format="%(message)s") """ Test script using minimal benchmarks to verify the snapshot testing tool fixes. """ diff --git a/tests/test_rng_patcher.py b/tests/test_rng_patcher.py index b841915..7940e76 100644 --- a/tests/test_rng_patcher.py +++ b/tests/test_rng_patcher.py @@ -60,7 +60,9 @@ def test_numpy_legacy_random_determinism(self): patcher.unpatch_all() - @pytest.mark.skip(reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility") + @pytest.mark.skip( + reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility" + ) def test_numpy_generator_pcg64_determinism(self): """Test that NumPy Generator with PCG64 produces deterministic results.""" patcher = RNGPatcher(seed=12345) @@ -78,7 +80,9 @@ def test_numpy_generator_pcg64_determinism(self): patcher.unpatch_all() - @pytest.mark.skip(reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility") + @pytest.mark.skip( + reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility" + ) def test_numpy_generator_mt19937_determinism(self): """Test that NumPy Generator with MT19937 produces deterministic results.""" patcher = RNGPatcher(seed=12345) @@ -96,7 +100,9 @@ def test_numpy_generator_mt19937_determinism(self): patcher.unpatch_all() - @pytest.mark.skip(reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility") + @pytest.mark.skip( + reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility" + ) def test_numpy_generator_philox_determinism(self): """Test that NumPy Generator with Philox produces deterministic results.""" patcher = RNGPatcher(seed=12345) @@ -114,7 +120,9 @@ def test_numpy_generator_philox_determinism(self): patcher.unpatch_all() - @pytest.mark.skip(reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility") + @pytest.mark.skip( + reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility" + ) def test_numpy_generator_sfc64_determinism(self): """Test that NumPy Generator with SFC64 produces deterministic results.""" patcher = RNGPatcher(seed=12345) @@ -132,7 +140,9 @@ def test_numpy_generator_sfc64_determinism(self): patcher.unpatch_all() - @pytest.mark.skip(reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility") + @pytest.mark.skip( + reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility" + ) def test_context_manager(self): """Test that the context manager properly patches and unpatches.""" # Generate without patching @@ -157,7 +167,9 @@ def test_context_manager(self): # Values before and after should be the same (same seed, unpatched) np.testing.assert_array_equal(values_before, values_after) - @pytest.mark.skip(reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility") + @pytest.mark.skip( + reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility" + ) def test_different_seeds_produce_different_results(self): """Test that different seeds produce different results.""" patcher1 = RNGPatcher(seed=12345) @@ -175,7 +187,9 @@ def test_different_seeds_produce_different_results(self): # Different seeds should produce different results assert not np.array_equal(values1, values2) - @pytest.mark.skip(reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility") + @pytest.mark.skip( + reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility" + ) def test_global_patch_function(self): """Test the global patch_all_rngs function.""" patch_all_rngs(seed=12345) @@ -203,7 +217,9 @@ def test_reset_all_rngs_function(self): np.testing.assert_array_equal(values1, values2) - @pytest.mark.skip(reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility") + @pytest.mark.skip( + reason="Modern Generator API (numpy 1.17+) not supported for 2017 compatibility" + ) def test_unpatch_restores_original_behavior(self): """Test that unpatching restores original RNG behavior.""" # Create a generator with a specific seed diff --git a/tests/test_snapshot_tool.py b/tests/test_snapshot_tool.py index 73e2426..a49c16e 100644 --- a/tests/test_snapshot_tool.py +++ b/tests/test_snapshot_tool.py @@ -6,8 +6,9 @@ """ import logging + logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format='%(message)s') +logging.basicConfig(level=logging.INFO, format="%(message)s") import sys from pathlib import Path diff --git a/tests/test_storage_comprehensive.py b/tests/test_storage_comprehensive.py index 7e1ec50..831b2c1 100644 --- a/tests/test_storage_comprehensive.py +++ b/tests/test_storage_comprehensive.py @@ -1,7 +1,9 @@ """Comprehensive tests for SnapshotManager.""" + import logging + logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO, format='%(message)s') +logging.basicConfig(level=logging.INFO, format="%(message)s") import shutil import tempfile @@ -34,16 +36,19 @@ def test_store_simple_snapshot(self, manager): """Test storing a simple snapshot.""" return_value = {"test": "data", "number": 42} - snapshot_path = manager.store_snapshot( + sidecar_path = manager.store_snapshot( benchmark_name="test_bench", module_path="test_module", parameters=(1, 2, 3), param_names=["a", "b", "c"], - return_value=return_value + return_value=return_value, ) - assert snapshot_path.exists() - assert snapshot_path.suffix == ".pkl" + # store_snapshot returns the JSON metadata sidecar path; the payload + # lives in the SQLite DB. + assert sidecar_path.exists() + assert sidecar_path.suffix == ".json" + assert manager.db_path.exists() def test_load_simple_snapshot(self, manager): """Test loading a simple snapshot.""" @@ -54,13 +59,11 @@ def test_load_simple_snapshot(self, manager): module_path="test_module", parameters=(1, 2, 3), param_names=["a", "b", "c"], - return_value=return_value + return_value=return_value, ) loaded = manager.load_snapshot( - benchmark_name="test_bench", - module_path="test_module", - parameters=(1, 2, 3) + benchmark_name="test_bench", module_path="test_module", parameters=(1, 2, 3) ) assert loaded is not None @@ -78,13 +81,11 @@ def test_store_and_load_numpy_array(self, manager): module_path="test_module", parameters=(), param_names=None, - return_value=arr + return_value=arr, ) loaded_value, _ = manager.load_snapshot( - benchmark_name="array_bench", - module_path="test_module", - parameters=() + benchmark_name="array_bench", module_path="test_module", parameters=() ) assert isinstance(loaded_value, np.ndarray) @@ -97,13 +98,11 @@ def test_metadata_stored(self, manager): module_path="test_module", parameters=(1,), param_names=["x"], - return_value=42 + return_value=42, ) _, metadata = manager.load_snapshot( - benchmark_name="meta_bench", - module_path="test_module", - parameters=(1,) + benchmark_name="meta_bench", module_path="test_module", parameters=(1,) ) assert metadata.benchmark_name == "meta_bench" @@ -125,7 +124,7 @@ def test_different_parameters_different_snapshots(self, manager): module_path="test_module", parameters=(1, 2), param_names=["a", "b"], - return_value="result_1_2" + return_value="result_1_2", ) manager.store_snapshot( @@ -133,19 +132,15 @@ def test_different_parameters_different_snapshots(self, manager): module_path="test_module", parameters=(3, 4), param_names=["a", "b"], - return_value="result_3_4" + return_value="result_3_4", ) loaded1, _ = manager.load_snapshot( - benchmark_name="param_bench", - module_path="test_module", - parameters=(1, 2) + benchmark_name="param_bench", module_path="test_module", parameters=(1, 2) ) loaded2, _ = manager.load_snapshot( - benchmark_name="param_bench", - module_path="test_module", - parameters=(3, 4) + benchmark_name="param_bench", module_path="test_module", parameters=(3, 4) ) assert loaded1 == "result_1_2" @@ -158,13 +153,11 @@ def test_no_parameters(self, manager): module_path="test_module", parameters=(), param_names=None, - return_value="result" + return_value="result", ) loaded, _ = manager.load_snapshot( - benchmark_name="no_param_bench", - module_path="test_module", - parameters=() + benchmark_name="no_param_bench", module_path="test_module", parameters=() ) assert loaded == "result" @@ -178,13 +171,11 @@ def test_complex_parameter_types(self, manager): module_path="test_module", parameters=params, param_names=["list", "str", "float"], - return_value="result" + return_value="result", ) loaded, _ = manager.load_snapshot( - benchmark_name="complex_params", - module_path="test_module", - parameters=params + benchmark_name="complex_params", module_path="test_module", parameters=params ) assert loaded == "result" @@ -195,6 +186,7 @@ class TestUnpicklableObjects: def test_generator_serialization(self, manager): """Test storing generators.""" + def gen(): yield 1 yield 2 @@ -207,18 +199,16 @@ def gen(): module_path="test_module", parameters=(), param_names=None, - return_value=generator + return_value=generator, ) loaded, _ = manager.load_snapshot( - benchmark_name="gen_bench", - module_path="test_module", - parameters=() + benchmark_name="gen_bench", module_path="test_module", parameters=() ) # Should be stored as a marker dict assert isinstance(loaded, dict) - assert loaded.get('__generator__') is True + assert loaded.get("__generator__") is True def test_lambda_serialization(self, manager): """Test storing lambda functions.""" @@ -229,21 +219,20 @@ def test_lambda_serialization(self, manager): module_path="test_module", parameters=(), param_names=None, - return_value=func + return_value=func, ) loaded, _ = manager.load_snapshot( - benchmark_name="lambda_bench", - module_path="test_module", - parameters=() + benchmark_name="lambda_bench", module_path="test_module", parameters=() ) # Should be stored as a callable marker assert isinstance(loaded, dict) - assert loaded.get('__callable__') is True + assert loaded.get("__callable__") is True def test_class_instance_serialization(self, manager): """Test storing class instances that can't be pickled easily.""" + class TestClass: def __init__(self, value): self.value = value @@ -258,13 +247,11 @@ def __init__(self, value): module_path="test_module", parameters=(), param_names=None, - return_value=obj + return_value=obj, ) loaded, _ = manager.load_snapshot( - benchmark_name="class_bench", - module_path="test_module", - parameters=() + benchmark_name="class_bench", module_path="test_module", parameters=() ) # Check if it was serialized as marker or actual object @@ -275,27 +262,22 @@ def __init__(self, value): def test_nested_unpicklable(self, manager): """Test nested structures with unpicklable objects.""" + def gen(): yield 1 - data = { - 'number': 42, - 'generator': gen(), - 'list': [1, 2, 3] - } + data = {"number": 42, "generator": gen(), "list": [1, 2, 3]} manager.store_snapshot( benchmark_name="nested_unpickle", module_path="test_module", parameters=(), param_names=None, - return_value=data + return_value=data, ) loaded, _ = manager.load_snapshot( - benchmark_name="nested_unpickle", - module_path="test_module", - parameters=() + benchmark_name="nested_unpickle", module_path="test_module", parameters=() ) # When dict contains unpicklable objects, storage may fall back to placeholder @@ -313,14 +295,12 @@ def test_store_failed_capture(self, manager): module_path="test_module", parameters=(1, 2), param_names=["a", "b"], - failure_reason="Test error" + failure_reason="Test error", ) # Check if failed marker exists is_failed = manager.is_failed_capture( - benchmark_name="failed_bench", - module_path="test_module", - parameters=(1, 2) + benchmark_name="failed_bench", module_path="test_module", parameters=(1, 2) ) assert is_failed is True @@ -332,13 +312,11 @@ def test_is_failed_capture_false_for_success(self, manager): module_path="test_module", parameters=(), param_names=None, - return_value=42 + return_value=42, ) is_failed = manager.is_failed_capture( - benchmark_name="success_bench", - module_path="test_module", - parameters=() + benchmark_name="success_bench", module_path="test_module", parameters=() ) assert is_failed is False @@ -350,13 +328,11 @@ def test_load_failed_capture_returns_none(self, manager): module_path="test_module", parameters=(), param_names=None, - failure_reason="Test error" + failure_reason="Test error", ) loaded = manager.load_snapshot( - benchmark_name="failed_bench", - module_path="test_module", - parameters=() + benchmark_name="failed_bench", module_path="test_module", parameters=() ) # Should return None or handle gracefully @@ -379,7 +355,7 @@ def test_list_single_snapshot(self, manager): module_path="module1", parameters=(), param_names=None, - return_value=42 + return_value=42, ) snapshots = manager.list_snapshots() @@ -395,7 +371,7 @@ def test_list_multiple_snapshots(self, manager): module_path=f"module{i}", parameters=(i,), param_names=["x"], - return_value=i + return_value=i, ) snapshots = manager.list_snapshots() @@ -408,7 +384,7 @@ def test_list_snapshots_by_module(self, manager): module_path="moduleA", parameters=(), param_names=None, - return_value=1 + return_value=1, ) manager.store_snapshot( @@ -416,7 +392,7 @@ def test_list_snapshots_by_module(self, manager): module_path="moduleB", parameters=(), param_names=None, - return_value=2 + return_value=2, ) # If listing supports filtering @@ -434,23 +410,27 @@ def test_snapshot_path_structure(self, manager, temp_snapshot_dir): module_path="my_module", parameters=(1, 2), param_names=["a", "b"], - return_value=42 + return_value=42, ) - # Should create path like: .snapshots/my_module/test_bench/.pkl + # Payload lives in /snapshots.db; the per-test JSON + # sidecar still lives at ///.json + # because downstream tooling consumes it. + assert (temp_snapshot_dir / "snapshots.db").exists() + module_dir = temp_snapshot_dir / "my_module" assert module_dir.exists() bench_dir = module_dir / "test_bench" assert bench_dir.exists() - # Should have .pkl and .json files - pkl_files = list(bench_dir.glob("*.pkl")) json_files = list(bench_dir.glob("*.json")) - - assert len(pkl_files) >= 1 assert len(json_files) >= 1 + # No pickle files anywhere — everything in SQLite. + assert list(temp_snapshot_dir.rglob("*.pkl")) == [] + assert list(temp_snapshot_dir.rglob("*.pkl.gz")) == [] + def test_parameter_hashing_consistency(self, manager): """Test that same parameters always produce same hash.""" params = (1, 2, 3) @@ -460,7 +440,7 @@ def test_parameter_hashing_consistency(self, manager): module_path="test_module", parameters=params, param_names=["a", "b", "c"], - return_value="first" + return_value="first", ) path2 = manager.store_snapshot( @@ -468,7 +448,7 @@ def test_parameter_hashing_consistency(self, manager): module_path="test_module", parameters=params, param_names=["a", "b", "c"], - return_value="second" + return_value="second", ) # Should overwrite the same file @@ -487,23 +467,24 @@ def test_very_large_snapshot(self, manager): module_path="test_module", parameters=(), param_names=None, - return_value=large_array + return_value=large_array, ) loaded, _ = manager.load_snapshot( - benchmark_name="large_bench", - module_path="test_module", - parameters=() + benchmark_name="large_bench", module_path="test_module", parameters=() ) assert np.array_equal(loaded, large_array) - def test_snapshot_compression_for_large_files(self, temp_snapshot_dir): - """Test that large snapshots are compressed and still loadable.""" - manager = SnapshotManager(temp_snapshot_dir, compress_threshold_bytes=1) - data = {"message": "compressed"} + def test_payload_is_compressed_in_db(self, temp_snapshot_dir): + """Every payload is gzipped in the SQLite ``blobs`` table — there is no + size threshold under the SQLite backend.""" + import sqlite3 + + manager = SnapshotManager(temp_snapshot_dir) + data = {"message": "compressed" * 1000} # large enough to benefit from gzip - snapshot_path = manager.store_snapshot( + manager.store_snapshot( benchmark_name="compress_bench", module_path="test_module", parameters=(), @@ -511,17 +492,51 @@ def test_snapshot_compression_for_large_files(self, temp_snapshot_dir): return_value=data, ) - assert snapshot_path.exists() - assert snapshot_path.name.endswith(".pkl.gz") + # Compressed bytes are strictly smaller than raw bytes for repetitive data. + with sqlite3.connect(str(manager.db_path)) as conn: + row = conn.execute("SELECT compressed_size, raw_size FROM blobs").fetchone() + compressed, raw = row + assert compressed > 0 and raw > 0 + assert compressed < raw loaded, _ = manager.load_snapshot( benchmark_name="compress_bench", module_path="test_module", parameters=(), ) - assert loaded == data + def test_identical_payloads_are_deduplicated(self, temp_snapshot_dir): + """Two benchmarks producing identical return values share a single blob.""" + import sqlite3 + + manager = SnapshotManager(temp_snapshot_dir) + same_value = [1, 2, 3, 4, 5] + + manager.store_snapshot( + benchmark_name="bench_a", + module_path="modA", + parameters=(), + param_names=None, + return_value=same_value, + ) + manager.store_snapshot( + benchmark_name="bench_b", + module_path="modB", + parameters=(), + param_names=None, + return_value=same_value, + ) + + with sqlite3.connect(str(manager.db_path)) as conn: + blob_count = conn.execute("SELECT COUNT(*) FROM blobs").fetchone()[0] + refcount = conn.execute("SELECT refcount FROM blobs").fetchone()[0] + snapshot_count = conn.execute("SELECT COUNT(*) FROM snapshots").fetchone()[0] + + assert blob_count == 1 + assert refcount == 2 + assert snapshot_count == 2 + def test_special_characters_in_names(self, manager): """Test handling special characters in benchmark names.""" # Some characters might need escaping @@ -530,13 +545,11 @@ def test_special_characters_in_names(self, manager): module_path="my.module", parameters=(), param_names=None, - return_value=42 + return_value=42, ) loaded, _ = manager.load_snapshot( - benchmark_name="test-bench.v2", - module_path="my.module", - parameters=() + benchmark_name="test-bench.v2", module_path="my.module", parameters=() ) assert loaded == 42 @@ -568,13 +581,11 @@ def test_nested_directory_creation(self, manager): module_path="deeply/nested/module/path", parameters=(), param_names=None, - return_value=42 + return_value=42, ) loaded, _ = manager.load_snapshot( - benchmark_name="bench", - module_path="deeply/nested/module/path", - parameters=() + benchmark_name="bench", module_path="deeply/nested/module/path", parameters=() ) assert loaded == 42 @@ -588,7 +599,7 @@ def test_overwrite_existing_snapshot(self, manager): module_path="test_module", parameters=params, param_names=["a", "b"], - return_value="first" + return_value="first", ) manager.store_snapshot( @@ -596,13 +607,11 @@ def test_overwrite_existing_snapshot(self, manager): module_path="test_module", parameters=params, param_names=["a", "b"], - return_value="second" + return_value="second", ) loaded, _ = manager.load_snapshot( - benchmark_name="overwrite_test", - module_path="test_module", - parameters=params + benchmark_name="overwrite_test", module_path="test_module", parameters=params ) # Should have the second value @@ -611,9 +620,7 @@ def test_overwrite_existing_snapshot(self, manager): def test_load_nonexistent_snapshot(self, manager): """Test loading a snapshot that doesn't exist.""" loaded = manager.load_snapshot( - benchmark_name="nonexistent", - module_path="test_module", - parameters=() + benchmark_name="nonexistent", module_path="test_module", parameters=() ) assert loaded is None @@ -626,13 +633,13 @@ def test_empty_return_value(self, manager): module_path="test_module", parameters=(), param_names=None, - return_value=value + return_value=value, ) loaded, _ = manager.load_snapshot( benchmark_name=f"empty_{type(value).__name__}", module_path="test_module", - parameters=() + parameters=(), ) assert loaded == value @@ -648,13 +655,11 @@ def test_git_commit_in_metadata(self, manager): module_path="test_module", parameters=(), param_names=None, - return_value=42 + return_value=42, ) _, metadata = manager.load_snapshot( - benchmark_name="git_test", - module_path="test_module", - parameters=() + benchmark_name="git_test", module_path="test_module", parameters=() ) # git_commit might be None if not in a git repo @@ -667,13 +672,11 @@ def test_platform_info_in_metadata(self, manager): module_path="test_module", parameters=(), param_names=None, - return_value=42 + return_value=42, ) _, metadata = manager.load_snapshot( - benchmark_name="platform_test", - module_path="test_module", - parameters=() + benchmark_name="platform_test", module_path="test_module", parameters=() ) assert metadata.platform is not None or metadata.platform is None @@ -692,14 +695,12 @@ def test_multiple_snapshots_same_benchmark(self, manager): module_path="test_module", parameters=(i,), param_names=["x"], - return_value=i * 2 + return_value=i * 2, ) # All should be loadable for i in range(10): loaded, _ = manager.load_snapshot( - benchmark_name="concurrent_bench", - module_path="test_module", - parameters=(i,) + benchmark_name="concurrent_bench", module_path="test_module", parameters=(i,) ) assert loaded == i * 2 diff --git a/tests/test_transitions.py b/tests/test_transitions.py index 2de0ddd..f37017e 100644 --- a/tests/test_transitions.py +++ b/tests/test_transitions.py @@ -1,8 +1,9 @@ from __future__ import annotations -from snapshot_tool.transitions import compute_transitions import random +from snapshot_tool.transitions import compute_transitions + def test_compute_transitions_basic_identity(): baseline = { @@ -42,11 +43,11 @@ def test_compute_transitions_mixed_and_legacy(): "E": "pass", } verify = { - "A": "fail", # pass->fail - "B": "pass", # fail->pass - "C": "pass", # legacy fail->pass - "D": "fail", # skip->fail - "F": "pass", # new test not in baseline; ignored + "A": "fail", # pass->fail + "B": "pass", # fail->pass + "C": "pass", # legacy fail->pass + "D": "fail", # skip->fail + "F": "pass", # new test not in baseline; ignored } out = compute_transitions(baseline, verify) assert out.get("pass-to-fail", 0) == 1 diff --git a/uv.lock b/uv.lock index 307127a..9cb1a36 100644 --- a/uv.lock +++ b/uv.lock @@ -2028,7 +2028,7 @@ wheels = [ [[package]] name = "snapshot-tool" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } [package.dev-dependencies]