ci(bench): stabilize HS256 smoke benchmark (#71)

RELEASING.md

@@ -20,6 +20,12 @@ This checklist is for maintainers publishing **0.4.x** (and later) to PyPI via t
    mkdocs build --strict -f docs-site/mkdocs.yml
    ```
 
+   Optional extended benchmark (slower, not required for every PR):
+
+   ```bash
+   OXYJWT_BENCHMARK=1 python -m pytest -m benchmark tests/test_benchmark_jwt_libraries.py
+   ```
+
 4. **Smoke import** (after `maturin develop`):
 
    ```bash

docs-site/docs/benchmarks.md

@@ -12,14 +12,14 @@ OxyJWT is optimized for throughput on typical JWT workloads. Numbers depend on C
 
 ## Reference ratios (HS256 smoke parameters)
 
-Measured on a typical Linux dev machine with `maturin develop --release`, 50 iterations, 1 round, warmup 8 (same as CI smoke):
+Measured on a typical Linux dev machine with `maturin develop --release`, 50 iterations, 3 rounds (median timing), warmup 8 (CI smoke):
 
 | Operation | OxyJWT (ops/s) | PyJWT (ops/s) | OxyJWT / PyJWT |
 |-----------|----------------|---------------|----------------|
 | encode | ~400k+ | ~130k+ | ~3× |
 | decode | ~160k+ | ~115k+ | ~1.4× |
 
-CI asserts **≥75%** of PyJWT for both operations so large regressions fail without requiring absolute ops/s parity across runners.
+CI asserts **≥75%** of PyJWT (median ops/s across rounds) for both operations so large regressions fail without requiring absolute ops/s parity across runners.
 
 ## Running comparisons locally
 
@@ -37,6 +37,17 @@ python3 -m venv .venv
   --markdown benchmark-results/local.bench.md
 ```
 
+Fairer RSA/EdDSA comparison against PyJWT (cached competitor keys):
+
+```bash
+.venv/bin/python scripts/compare_jwt_libraries.py \
+  --algorithms RS256,EdDSA \
+  --iterations 1000 \
+  --rounds 3 \
+  --competitor-key-mode cached \
+  --markdown benchmark-results/local-cached.bench.md
+```
+
 Raw JSON/Markdown outputs are gitignored; keep them local or attach them to release notes as needed.
 
 ## CI artifacts
@@ -49,6 +60,12 @@ Main [CI](https://github.com/QueryaHub/OxyJWT/blob/main/.github/workflows/ci.yml
 
 - **Metric:** operations per second (encode and decode measured separately).
 - **Warmup:** reduces JIT and allocator noise; see script defaults.
-- **Fairness:** each library uses its supported key types; unsupported pairs are recorded as zero throughput in the script output.
+- **Key preparation (`--competitor-key-mode`):**
+  - **`pem` (default)** — used by CI smoke and the extended pytest sweep. The harness builds one signing/verification key per library before timing. OxyJWT uses `EncodingKey` / `DecodingKey` parsed from PEM once; PyJWT, Authlib, and python-jose receive PEM `str`/`bytes` and may parse that material inside each timed call. This matches “pass a PEM string to the library” usage but can understate competitor throughput on RSA/EC/EdDSA.
+  - **`cached`** — competitors that support it receive preloaded `cryptography` key objects (same idea as holding parsed keys in application code). Use this for fairer asymmetric comparisons and for release-note / weekly benchmark artifacts.
+- **HMAC (HS\*)** — both modes pass the same raw secret; key-mode differences are negligible.
+- Unsupported library/algorithm pairs are recorded as zero throughput in the script output.
+
+For asymmetric algorithms, prefer reporting **both** modes or explicitly label which mode was used. The headline table in the root README was measured with defaults that favor OxyJWT on RSA unless noted otherwise.
 
 Always compare on your own target hardware before choosing a library for production latency budgets.

tests/test_benchmark_jwt_libraries.py

@@ -23,7 +23,9 @@
 # HS256 smoke: minimum OxyJWT/PyJWT throughput ratio (tightened from 0.25 = 4× slack).
 _MIN_OXY_VS_PYJWT_ENCODE_RATIO = 0.75
 _MIN_OXY_VS_PYJWT_DECODE_RATIO = 0.75
-_MIN_OXYJWT_OPS_PER_SECOND = 500
+_SMOKE_ITERATIONS = 50
+_SMOKE_ROUNDS = 3
+_SMOKE_WARMUP = 8
 
 # Extended sweep: looser floor vs PyJWT when present (asymmetric crypto is noisier in CI).
 _MIN_OXY_VS_PYJWT_EXTENDED_RATIO = 0.5
@@ -48,24 +50,48 @@ def _ops_for(
     return None
 
 
+def _median_ops(result: object) -> float:
+    iterations = int(result.iterations)  # type: ignore[attr-defined]
+    median_seconds = float(result.median_seconds)  # type: ignore[attr-defined]
+    assert median_seconds > 0
+    return iterations / median_seconds
+
+
 def _assert_oxyjwt_hs256_smoke(results: list[object], *, mod: object) -> None:
-    oxy_enc = _ops_for(results, "OxyJWT", "encode", mod=mod)
-    oxy_dec = _ops_for(results, "OxyJWT", "decode", mod=mod)
-    assert oxy_enc is not None and oxy_dec is not None
-    assert oxy_enc > _MIN_OXYJWT_OPS_PER_SECOND, f"encode too slow: {oxy_enc:.0f} ops/s"
-    assert oxy_dec > _MIN_OXYJWT_OPS_PER_SECOND, f"decode too slow: {oxy_dec:.0f} ops/s"
+    oxy_enc = next(
+        r for r in results if r.library == "OxyJWT" and r.operation == "encode"  # type: ignore[attr-defined]
+    )
+    oxy_dec = next(
+        r for r in results if r.library == "OxyJWT" and r.operation == "decode"  # type: ignore[attr-defined]
+    )
+    oxy_enc_ops = _median_ops(oxy_enc)
+    oxy_dec_ops = _median_ops(oxy_dec)
 
     py_enc = _ops_for(results, "PyJWT", "encode", mod=mod)
     py_dec = _ops_for(results, "PyJWT", "decode", mod=mod)
     if py_enc is not None and py_enc > 0:
-        assert oxy_enc >= py_enc * _MIN_OXY_VS_PYJWT_ENCODE_RATIO, (
-            f"HS256 encode: OxyJWT {oxy_enc:.0f} ops/s vs PyJWT {py_enc:.0f} ops/s "
-            f"(need >={_MIN_OXY_VS_PYJWT_ENCODE_RATIO:.0%} of PyJWT)"
+        py_enc_median = _median_ops(
+            next(
+                r
+                for r in results
+                if r.library == "PyJWT" and r.operation == "encode"  # type: ignore[attr-defined]
+            )
+        )
+        assert oxy_enc_ops >= py_enc_median * _MIN_OXY_VS_PYJWT_ENCODE_RATIO, (
+            f"HS256 encode: OxyJWT median {oxy_enc_ops:.0f} ops/s vs PyJWT "
+            f"{py_enc_median:.0f} ops/s (need >={_MIN_OXY_VS_PYJWT_ENCODE_RATIO:.0%})"
         )
     if py_dec is not None and py_dec > 0:
-        assert oxy_dec >= py_dec * _MIN_OXY_VS_PYJWT_DECODE_RATIO, (
-            f"HS256 decode: OxyJWT {oxy_dec:.0f} ops/s vs PyJWT {py_dec:.0f} ops/s "
-            f"(need >={_MIN_OXY_VS_PYJWT_DECODE_RATIO:.0%} of PyJWT)"
+        py_dec_median = _median_ops(
+            next(
+                r
+                for r in results
+                if r.library == "PyJWT" and r.operation == "decode"  # type: ignore[attr-defined]
+            )
+        )
+        assert oxy_dec_ops >= py_dec_median * _MIN_OXY_VS_PYJWT_DECODE_RATIO, (
+            f"HS256 decode: OxyJWT median {oxy_dec_ops:.0f} ops/s vs PyJWT "
+            f"{py_dec_median:.0f} ops/s (need >={_MIN_OXY_VS_PYJWT_DECODE_RATIO:.0%})"
         )
 
 
@@ -76,9 +102,9 @@ def test_benchmark_hs256_smoke_vs_competitors() -> None:
 
     mod = _load_compare_module()
     results, _skipped = mod.run_benchmark(
-        iterations=50,
-        rounds=1,
-        warmup=8,
+        iterations=_SMOKE_ITERATIONS,
+        rounds=_SMOKE_ROUNDS,
+        warmup=_SMOKE_WARMUP,
         selected_algorithms={"HS256"},
         competitor_key_mode="pem",
     )