perf: add CPython bench gate, IC depth, and JIT range loops

.cargo/config.toml

@@ -9,6 +9,9 @@ xtest = "test --workspace --all-targets"
 xcheck = "check --workspace --all-targets --all-features"
 xclippy = "clippy --workspace --all-targets --all-features -- -D warnings"
 xfmt = "fmt --all"
+# RFC 0058 — the benchmark lane. Requires a release `weavepy` binary
+# (`cargo build --release -p weavepy-cli`) next to the bench binary.
+xbench = "run --release -p weavepy-bench --"
 
 [build]
 # Faster incremental builds for local dev. CI overrides via env if needed.

.github/workflows/ci.yml

@@ -107,6 +107,47 @@ jobs:
           path: target/regrtest/
           if-no-files-found: warn
 
+  bench:
+    name: bench gate (blocking, ${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    # RFC 0058 — the benchmark lane. Gates on WeavePy/CPython *ratios*
+    # against baselines/bench.json (host-independent, unlike absolute
+    # times); the 25% threshold absorbs shared-runner noise. The
+    # markdown report lands in the job summary so every PR shows its
+    # ratio table.
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: Build weavepy CLI + bench harness
+        run: cargo build --release -p weavepy-cli -p weavepy-bench
+      - name: Run bench gate
+        run: |
+          target/release/weavepy-bench gate --pct=25 \
+            --weavepy=target/release/weavepy \
+            | tee bench-report.md
+      - name: Append bench report to job summary
+        if: always()
+        run: |
+          {
+            echo "## WeavePy bench"
+            echo
+            cat bench-report.md || echo "(no report produced)"
+          } >> "$GITHUB_STEP_SUMMARY"
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: bench-report-${{ matrix.os }}
+          path: bench-report.md
+          if-no-files-found: warn
+
   conformance:
     name: cpython conformance (non-blocking)
     runs-on: ubuntu-latest

crates/weavepy-bench/Cargo.toml

@@ -4,7 +4,7 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true
 publish = false
-description = "RFC 0021 — pyperformance-shaped microbench harness for WeavePy."
+description = "RFC 0058 — pyperformance-shaped benchmark lane for WeavePy."
 
 [lib]
 path = "src/lib.rs"
@@ -13,19 +13,13 @@ path = "src/lib.rs"
 name = "weavepy-bench"
 path = "src/main.rs"
 
+# The harness is subprocess-only (RFC 0058 WS1): it times the built
+# `weavepy` binary and the host CPython symmetrically, so it does not
+# link the interpreter crates at all. The `--jit` column requires the
+# `weavepy` binary itself to be built with `--features weavepy-cli/jit`.
 [dependencies]
-weavepy = { workspace = true }
-weavepy-compiler = { workspace = true }
-weavepy-parser = { workspace = true }
-weavepy-vm = { workspace = true }
-
 serde = { workspace = true }
 serde_json = { workspace = true }
 
-[features]
-default = []
-# RFC 0032 — run the bench harness with the tier-2 JIT compiled in.
-jit = ["weavepy/jit", "weavepy-vm/jit"]
-
 [lints]
 workspace = true

crates/weavepy-bench/README.md

@@ -1,44 +1,65 @@
 # weavepy-bench
 
-RFC 0021 — `pyperformance`-shaped microbench harness for WeavePy.
+RFC 0058 — the `pyperformance`-shaped benchmark lane for WeavePy
+(supersedes the RFC 0021 harness).
+
+The harness times each fixture's `bench(n)` under **both** the built
+`weavepy` binary and the host CPython, as subprocesses with an
+identical `WEAVEPY_BENCH_WORK`. Fixtures self-time the bench region
+with `time.perf_counter_ns()` and print `WEAVEPY_BENCH_NS=<int>`, so
+startup / parse / import cost is excluded symmetrically (the dedicated
+`startup` fixture measures full subprocess wall time instead).
+
+The tracked baseline (`baselines/bench.json`) stores medians for both
+interpreters plus the WeavePy/CPython **ratio** per fixture and the
+suite geometric mean. `gate` compares ratios — host-independent,
+unlike absolute nanoseconds — and fails on regressions beyond a
+threshold, like the regrtest and ecosystem lanes' `--check`. CI runs
+`gate --pct=25` on ubuntu + macos (the `bench` job).
 
 The crate is excluded from `default-members` so `cargo build` /
-`cargo test --workspace` doesn't pull it in. Opt in with `-p
-weavepy-bench` when you want to run the benches.
+`cargo test --workspace` doesn't pull it in. Opt in with
+`-p weavepy-bench`.
 
 ## Usage
 
 ```bash
-# Run all fixtures, print a markdown report.
-cargo run -p weavepy-bench -- run
+# The harness needs the binary under test next to it.
+cargo build --release -p weavepy-cli -p weavepy-bench
 
-# Skip the host CPython subprocess (faster on CI without python3).
-cargo run -p weavepy-bench -- run --no-cpython
+# Run all fixtures, print a markdown report (ratio column + geomean).
+cargo xbench run
 
-# Print the report as JSON instead of markdown.
-cargo run -p weavepy-bench -- run --json
+# Compare current ratios against the baseline; exit non-zero on
+# regression beyond 10% (default threshold).
+cargo xbench gate
+cargo xbench gate --pct=25
 
 # Refresh the baseline JSON tracked at `baselines/bench.json`.
-cargo run -p weavepy-bench -- run --update-baseline
+cargo xbench run --update-baseline
 
-# Compare current run against the baseline; exit non-zero on
-# regression beyond 10% (default threshold).
-cargo run -p weavepy-bench -- gate
-cargo run -p weavepy-bench -- gate --pct=15
-```
+# Point at explicit interpreters.
+cargo xbench run --weavepy=target/release/weavepy --python=python3.13
+
+# Add a WEAVEPY_JIT=1 column (reported, never gated). The binary must
+# be built with the tier-2 JIT compiled in:
+cargo build --release -p weavepy-cli --features weavepy-cli/jit
+cargo xbench run --jit
 
-Run with `--release` for representative numbers — the dev profile
-is far slower than what CI / shipped binaries see.
+# Print the report as JSON instead of markdown.
+cargo xbench run --json
+```
 
 ## Adding a fixture
 
-1. Drop `fixtures/foo.py`. The file should:
-   - Import `os`.
-   - Define a `bench(n)` callable that runs the workload `n` times.
-   - Have a `if __name__ == "__main__":` block that reads
-     `WEAVEPY_BENCH_WORK` from the environment so the runner can
-     parameterize CPython runs.
+1. Drop `fixtures/foo.py`. The file must:
+   - Define a `bench(n)` callable that runs the workload scaled by `n`.
+   - End with the standard self-timing block (copy it from any
+     fixture): read `WEAVEPY_BENCH_WORK`, time `bench(n)` with
+     `time.perf_counter_ns()`, print `WEAVEPY_BENCH_NS=<int>`.
 2. Add `"foo"` to `FIXTURES` in `src/fixtures.rs`.
-3. Pick a default `work` parameter in `default_work(...)`.
-4. Run `cargo run -p weavepy-bench -- run --update-baseline` and
-   inspect the diff before committing.
+3. Pick a `default_work(...)` value sized so the **CPython** leg takes
+   ~25–65 ms.
+4. Run `cargo xbench run --update-baseline` and inspect the diff
+   before committing. The gate fails fixtures that have no baseline
+   row, so the baseline refresh ships in the same change.