From 446da75665115d703311f29ff0e72aca3dddb747 Mon Sep 17 00:00:00 2001 From: meandmytram Date: Sat, 12 Sep 2026 13:56:36 -0400 Subject: [PATCH 01/11] Warn at import when NumPy's LAPACK is Accelerate, and document the OpenBLAS wheel On the macOS arm64 NumPy 2.x wheels, which link Apple's Accelerate framework, the decoders' rank-deficient, wide-spectrum matrices made numpy.linalg.qr die with SIGBUS and numpy.linalg.svd trip malloc's heap-corruption check inside dgesdd (five crash reports from the [[72,12,6]] investigation), and a chi_max=400 decode returned wrong verdicts while every unit test passed. The OpenBLAS build of the same NumPy version (the macosx_11_0_arm64 wheel) runs the same reproducers clean. The backend now warns once at import when Accelerate is found behind NumPy (MDOPT_ALLOW_ACCELERATE=1 silences it) and the README gives the two commands that install the OpenBLAS wheel. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- README.md | 20 +++++++++++++++++ mdopt/backend/array.py | 43 +++++++++++++++++++++++++++++++++++++ tests/backend/test_array.py | 29 +++++++++++++++++++++++++ 3 files changed, 92 insertions(+) diff --git a/README.md b/README.md index bb70d3ec..e469f691 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,26 @@ Otherwise, you can clone the repository and use [poetry](https://python-poetry.o poetry install ``` +### A note on NumPy's BLAS on Apple silicon + +The macOS arm64 wheels of NumPy 2.x link Apple's Accelerate framework. On +mdopt's matrices (rank-deficient, with singular values spanning many orders +of magnitude) Accelerate's LAPACK corrupted memory: `numpy.linalg.qr` died +with SIGBUS, `numpy.linalg.svd` tripped malloc's heap check, and a +bivariate-bicycle decode returned wrong verdicts while every test passed. +mdopt warns at import when it finds Accelerate behind NumPy. The OpenBLAS +build of the same NumPy version is the `macosx_11_0_arm64` wheel: + +```bash +pip download "numpy==$(python -c 'import numpy; print(numpy.__version__)')" \ + --platform macosx_11_0_arm64 --only-binary=:all: --no-deps -d /tmp/numpy-openblas +pip install --force-reinstall --no-deps /tmp/numpy-openblas/*.whl +``` + +Set `MDOPT_ALLOW_ACCELERATE=1` to silence the warning if you must keep +Accelerate. Use `OMP_NUM_THREADS=1` (or `OPENBLAS_NUM_THREADS=1`) for the +per-process BLAS of worker pools. + ## Minimal example ```python diff --git a/mdopt/backend/array.py b/mdopt/backend/array.py index 2a13512a..b4ba1b09 100644 --- a/mdopt/backend/array.py +++ b/mdopt/backend/array.py @@ -60,6 +60,49 @@ def _load_backend(): GPU = _xp.__name__ == "cupy" +def _lapack_vendor(numpy_module) -> str: + """The LAPACK library NumPy was built against, lower-cased ('' if unknown).""" + try: + deps = numpy_module.show_config(mode="dicts")["Build Dependencies"] + return str(deps["lapack"]["name"]).lower() + except Exception: # pylint: disable=broad-except + return "" + + +def _warn_if_accelerate(numpy_module) -> None: + """Warn once when NumPy's LAPACK is Apple's Accelerate framework. + + On the macOS arm64 wheels of NumPy 2.x (which link Accelerate) the + decoders' rank-deficient, wide-spectrum matrices made ``linalg.qr`` die + with SIGBUS and ``linalg.svd`` trip malloc's heap-corruption check + inside dgesdd, and a [[72,12,6]] decode at chi_max=400 returned wrong + verdicts while every unit test passed. The OpenBLAS build of the same + NumPy version has none of this; it is the ``macosx_11_0_arm64`` wheel:: + + pip download numpy== --platform macosx_11_0_arm64 \ + --only-binary=:all: --no-deps -d /tmp/numpy-openblas + pip install --force-reinstall --no-deps /tmp/numpy-openblas/*.whl + + Set MDOPT_ALLOW_ACCELERATE=1 to silence the warning. + """ + if os.getenv("MDOPT_ALLOW_ACCELERATE") == "1": + return + if "accelerate" in _lapack_vendor(numpy_module): + warnings.warn( + "NumPy is built against Apple's Accelerate LAPACK, which corrupted " + "memory and produced wrong decoding verdicts on mdopt's matrices " + "(see mdopt.backend.array._warn_if_accelerate). Install the " + "OpenBLAS build of NumPy (the macosx_11_0_arm64 wheel) or set " + "MDOPT_ALLOW_ACCELERATE=1 to silence this warning.", + RuntimeWarning, + stacklevel=2, + ) + + +if not GPU: + _warn_if_accelerate(_xp) + + # ---------------------------------------------------------------------- # Introspection helpers # ---------------------------------------------------------------------- diff --git a/tests/backend/test_array.py b/tests/backend/test_array.py index 472ba3d8..8473f008 100644 --- a/tests/backend/test_array.py +++ b/tests/backend/test_array.py @@ -88,3 +88,32 @@ def test_stream_is_a_context_manager_on_numpy(monkeypatch): with module.stream(): pass module.synchronize() + + +def test_accelerate_lapack_warns_unless_allowed(monkeypatch): + """A NumPy built against Accelerate triggers a RuntimeWarning at import + time; MDOPT_ALLOW_ACCELERATE=1 silences it; other vendors are silent.""" + import warnings + from types import SimpleNamespace + + from mdopt.backend import array as backend + + def fake_numpy(vendor): + return SimpleNamespace( + show_config=lambda mode="dicts": { + "Build Dependencies": { + "lapack": {"name": vendor}, + "blas": {"name": vendor}, + } + } + ) + + monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) + with pytest.warns(RuntimeWarning, match="Accelerate"): + backend._warn_if_accelerate(fake_numpy("accelerate")) + with warnings.catch_warnings(): + warnings.simplefilter("error") + backend._warn_if_accelerate(fake_numpy("scipy-openblas")) + monkeypatch.setenv("MDOPT_ALLOW_ACCELERATE", "1") + backend._warn_if_accelerate(fake_numpy("accelerate")) + assert backend._lapack_vendor(SimpleNamespace()) == "" From 0eb84e281d66710729e6aab712366c80e91b740d Mon Sep 17 00:00:00 2001 From: meandmytram Date: Sat, 12 Sep 2026 14:20:08 -0400 Subject: [PATCH 02/11] Warn for an Accelerate-linked SciPy as well, and document both OpenBLAS wheels The SVD helpers call SciPy's LAPACK too, and the SciPy macOS arm64 wheels link Accelerate like NumPy's; with only NumPy switched to OpenBLAS the reduced SVD path still returned a wrong factorisation (reconstruction error 0.15) or crashed, and with both switched it passes repeatedly. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- README.md | 12 ++++++++---- mdopt/backend/array.py | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index e469f691..6942cc5d 100644 --- a/README.md +++ b/README.md @@ -39,13 +39,17 @@ mdopt's matrices (rank-deficient, with singular values spanning many orders of magnitude) Accelerate's LAPACK corrupted memory: `numpy.linalg.qr` died with SIGBUS, `numpy.linalg.svd` tripped malloc's heap check, and a bivariate-bicycle decode returned wrong verdicts while every test passed. -mdopt warns at import when it finds Accelerate behind NumPy. The OpenBLAS -build of the same NumPy version is the `macosx_11_0_arm64` wheel: +The SciPy macOS wheels link Accelerate as well, and mdopt's SVD helpers +call SciPy's LAPACK too. mdopt warns at import when it finds Accelerate +behind either library. The OpenBLAS builds of the same versions are the +`macosx_11_0_arm64` NumPy wheel and the `macosx_12_0_arm64` SciPy wheel: ```bash pip download "numpy==$(python -c 'import numpy; print(numpy.__version__)')" \ - --platform macosx_11_0_arm64 --only-binary=:all: --no-deps -d /tmp/numpy-openblas -pip install --force-reinstall --no-deps /tmp/numpy-openblas/*.whl + --platform macosx_11_0_arm64 --only-binary=:all: --no-deps -d /tmp/openblas-wheels +pip download "scipy==$(python -c 'import scipy; print(scipy.__version__)')" \ + --platform macosx_12_0_arm64 --only-binary=:all: --no-deps -d /tmp/openblas-wheels +pip install --force-reinstall --no-deps /tmp/openblas-wheels/*.whl ``` Set `MDOPT_ALLOW_ACCELERATE=1` to silence the warning if you must keep diff --git a/mdopt/backend/array.py b/mdopt/backend/array.py index b4ba1b09..173eaa5e 100644 --- a/mdopt/backend/array.py +++ b/mdopt/backend/array.py @@ -99,8 +99,28 @@ def _warn_if_accelerate(numpy_module) -> None: ) +def _warn_if_scipy_accelerate() -> None: + """The same warning for SciPy, whose LAPACK the SVD helpers also call.""" + if os.getenv("MDOPT_ALLOW_ACCELERATE") == "1": + return + try: + scipy = importlib.import_module("scipy") + except ImportError: + return + if "accelerate" in _lapack_vendor(scipy): + warnings.warn( + "SciPy is built against Apple's Accelerate LAPACK (see " + "mdopt.backend.array._warn_if_accelerate); install the OpenBLAS " + "build (the macosx_12_0_arm64 wheel) or set " + "MDOPT_ALLOW_ACCELERATE=1 to silence this warning.", + RuntimeWarning, + stacklevel=2, + ) + + if not GPU: _warn_if_accelerate(_xp) + _warn_if_scipy_accelerate() # ---------------------------------------------------------------------- From 365c91db17a3b812b802ff723c5fd4de21d2fc50 Mon Sep 17 00:00:00 2001 From: meandmytram Date: Sun, 13 Sep 2026 13:33:04 -0400 Subject: [PATCH 03/11] Detect SciPy 1.9's LAPACK too, and say which macOS wheels link Accelerate SciPy 1.10 and later report their LAPACK through show_config(mode="dicts"), but SciPy 1.9, which the dependency floor allows, has show() without a mode argument, so the SciPy check silently reported an unknown vendor there. The helper now falls back to __config__.get_info("lapack_opt") in that case, and two SciPy-shaped test doubles cover both APIs. The README said the macOS arm64 wheels link Accelerate; only the macosx_14_0_arm64 wheels do (checked NumPy 2.4.6, SciPy 1.15.3 and 1.18.1), which pip picks on macOS 14 and later, while the macosx_11_0_arm64 NumPy and macosx_12_0_arm64 SciPy wheels bundle OpenBLAS. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- README.md | 19 +++++++------- mdopt/backend/array.py | 23 +++++++++++++--- tests/backend/test_array.py | 52 +++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 6942cc5d..cf9105f9 100644 --- a/README.md +++ b/README.md @@ -34,15 +34,16 @@ poetry install ### A note on NumPy's BLAS on Apple silicon -The macOS arm64 wheels of NumPy 2.x link Apple's Accelerate framework. On -mdopt's matrices (rank-deficient, with singular values spanning many orders -of magnitude) Accelerate's LAPACK corrupted memory: `numpy.linalg.qr` died -with SIGBUS, `numpy.linalg.svd` tripped malloc's heap check, and a -bivariate-bicycle decode returned wrong verdicts while every test passed. -The SciPy macOS wheels link Accelerate as well, and mdopt's SVD helpers -call SciPy's LAPACK too. mdopt warns at import when it finds Accelerate -behind either library. The OpenBLAS builds of the same versions are the -`macosx_11_0_arm64` NumPy wheel and the `macosx_12_0_arm64` SciPy wheel: +On Apple silicon running macOS 14 or later, pip installs the NumPy and +SciPy wheels tagged `macosx_14_0_arm64`, which link Apple's Accelerate +framework. On mdopt's matrices (rank-deficient, with singular values +spanning many orders of magnitude) Accelerate's LAPACK corrupted memory: +`numpy.linalg.qr` died with SIGBUS, `numpy.linalg.svd` tripped malloc's +heap check, and a bivariate-bicycle decode returned wrong verdicts while +every test passed. mdopt's SVD helpers call both libraries, and mdopt warns +at import when it finds Accelerate behind either one. The same versions are +also published as OpenBLAS builds, the `macosx_11_0_arm64` NumPy wheel and +the `macosx_12_0_arm64` SciPy wheel, which install on the same machines: ```bash pip download "numpy==$(python -c 'import numpy; print(numpy.__version__)')" \ diff --git a/mdopt/backend/array.py b/mdopt/backend/array.py index 173eaa5e..ace4a686 100644 --- a/mdopt/backend/array.py +++ b/mdopt/backend/array.py @@ -60,13 +60,30 @@ def _load_backend(): GPU = _xp.__name__ == "cupy" -def _lapack_vendor(numpy_module) -> str: - """The LAPACK library NumPy was built against, lower-cased ('' if unknown).""" +def _lapack_vendor(module) -> str: + """The LAPACK library a NumPy or SciPy module was built against, lower-cased. + + NumPy and SciPy 1.10 and later report it through + ``show_config(mode="dicts")``. SciPy 1.9 (a numpy.distutils build) has no + ``mode`` argument and exposes its link information through + ``__config__.get_info`` instead; an Accelerate link there is reported as + ``"accelerate"``. Returns ``""`` when neither source is available. + """ try: - deps = numpy_module.show_config(mode="dicts")["Build Dependencies"] + deps = module.show_config(mode="dicts")["Build Dependencies"] return str(deps["lapack"]["name"]).lower() + except TypeError: + pass # show_config without a mode argument: fall through to get_info + except Exception: # pylint: disable=broad-except + return "" + try: + info = module.__config__.get_info("lapack_opt") except Exception: # pylint: disable=broad-except return "" + text = " ".join(str(value) for value in dict(info).values()).lower() + if "accelerate" in text or "veclib" in text: + return "accelerate" + return text def _warn_if_accelerate(numpy_module) -> None: diff --git a/tests/backend/test_array.py b/tests/backend/test_array.py index 8473f008..38b44721 100644 --- a/tests/backend/test_array.py +++ b/tests/backend/test_array.py @@ -117,3 +117,55 @@ def fake_numpy(vendor): monkeypatch.setenv("MDOPT_ALLOW_ACCELERATE", "1") backend._warn_if_accelerate(fake_numpy("accelerate")) assert backend._lapack_vendor(SimpleNamespace()) == "" + + +def test_scipy_shaped_config_is_detected(monkeypatch): + """SciPy 1.10+ reports its LAPACK through show_config(mode="dicts"), with + the vendor capitalised ("Accelerate"); the SciPy check must warn on it.""" + from types import SimpleNamespace + + from mdopt.backend import array as backend + + def show_config(mode="stdout"): + if mode == "stdout": + return None + return { + "Build Dependencies": { + "blas": {"name": "Accelerate"}, + "lapack": {"name": "Accelerate"}, + } + } + + monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) + monkeypatch.setitem(sys.modules, "scipy", SimpleNamespace(show_config=show_config)) + with pytest.warns(RuntimeWarning, match="SciPy"): + backend._warn_if_scipy_accelerate() + + +def test_scipy_without_show_config_mode_falls_back_to_get_info(monkeypatch): + """SciPy 1.9 has show() without a mode argument; its link information + comes from __config__.get_info, which must still detect Accelerate and + stay silent for OpenBLAS.""" + from types import SimpleNamespace + + from mdopt.backend import array as backend + + def show(): + return None + + def scipy_with(info): + return SimpleNamespace( + show_config=show, __config__=SimpleNamespace(get_info=lambda name: info) + ) + + monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) + accelerate = scipy_with({"extra_link_args": ["-Wl,-framework", "-Wl,Accelerate"]}) + openblas = scipy_with({"libraries": ["openblas", "openblas"], "language": "c"}) + assert backend._lapack_vendor(accelerate) == "accelerate" + monkeypatch.setitem(sys.modules, "scipy", accelerate) + with pytest.warns(RuntimeWarning, match="SciPy"): + backend._warn_if_scipy_accelerate() + monkeypatch.setitem(sys.modules, "scipy", openblas) + with warnings.catch_warnings(): + warnings.simplefilter("error") + backend._warn_if_scipy_accelerate() From 0ae40a696b051b883dc948392a351bffe596c638 Mon Sep 17 00:00:00 2001 From: meandmytram Date: Sun, 13 Sep 2026 17:18:28 -0400 Subject: [PATCH 04/11] Allow the Accelerate import warnings in the no-warnings backend test test_defaults_to_numpy asserted that importing the backend warns about nothing, which fails on exactly the builds the new warning targets: the macOS CI runners install the Accelerate-linked NumPy and SciPy wheels, and all three macOS test jobs failed on it. The test now allows RuntimeWarnings that mention Accelerate and still fails on any other warning. Checked with NumPy and SciPy made to report Accelerate: the backend tests and the full suite pass. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- tests/backend/test_array.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/backend/test_array.py b/tests/backend/test_array.py index 38b44721..10d43d8a 100644 --- a/tests/backend/test_array.py +++ b/tests/backend/test_array.py @@ -45,11 +45,23 @@ def get_device_count(): def test_defaults_to_numpy(monkeypatch): - """With no MDOPT_BACKEND set, NumPy is used and nothing is warned about.""" + """With no MDOPT_BACKEND set, NumPy is used and nothing unexpected is warned about. + + The import deliberately warns when NumPy or SciPy links Apple's Accelerate + framework, which the stock macOS 14+ wheels do (the macOS CI runners among + them); those warnings are allowed, any other warning is not. + """ module, caught = _reload_backend(monkeypatch, None, None) assert module.GPU is False assert module.backend_name() == "numpy" - assert not caught + unexpected = [ + w + for w in caught + if not ( + issubclass(w.category, RuntimeWarning) and "Accelerate" in str(w.message) + ) + ] + assert not unexpected, [str(w.message) for w in unexpected] def test_falls_back_when_cupy_missing(monkeypatch): From 6983106dfb354ecdeb6ebbced9f692c3f15f9f6b Mon Sep 17 00:00:00 2001 From: meandmytram Date: Sun, 13 Sep 2026 18:25:14 -0400 Subject: [PATCH 05/11] Warn about an Accelerate-linked NumPy or SciPy on every backend The import-time checks were skipped when CuPy was selected, yet the GPU path still runs NumPy's linalg and BLAS on the host (orthogonality-centre moves, host-side contractions) and SciPy's LAPACK in the SVD fallbacks and in qr, so a GPU user on the stock macOS wheels was exposed to the same corruption without a warning. Both checks now run regardless of the backend. The CuPy-selection test allows the Accelerate warnings, and a new test checks that they fire with CuPy selected. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- mdopt/backend/array.py | 8 +++++--- tests/backend/test_array.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/mdopt/backend/array.py b/mdopt/backend/array.py index ace4a686..553c1538 100644 --- a/mdopt/backend/array.py +++ b/mdopt/backend/array.py @@ -135,9 +135,11 @@ def _warn_if_scipy_accelerate() -> None: ) -if not GPU: - _warn_if_accelerate(_xp) - _warn_if_scipy_accelerate() +# Both checks run on every backend: with CuPy selected the orthogonality-centre +# moves and host-side contractions still call NumPy's linalg and BLAS, and the +# SVD fallbacks and qr call SciPy's LAPACK. +_warn_if_accelerate(importlib.import_module("numpy")) +_warn_if_scipy_accelerate() # ---------------------------------------------------------------------- diff --git a/tests/backend/test_array.py b/tests/backend/test_array.py index 10d43d8a..ddd8f72f 100644 --- a/tests/backend/test_array.py +++ b/tests/backend/test_array.py @@ -91,7 +91,14 @@ def test_uses_cupy_when_device_present(monkeypatch): module, caught = _reload_backend(monkeypatch, "cupy", _fake_cupy(num_devices=1)) assert module.GPU is True assert module.backend_name() == "cupy" - assert not caught + unexpected = [ + w + for w in caught + if not ( + issubclass(w.category, RuntimeWarning) and "Accelerate" in str(w.message) + ) + ] + assert not unexpected, [str(w.message) for w in unexpected] def test_stream_is_a_context_manager_on_numpy(monkeypatch): @@ -181,3 +188,30 @@ def scipy_with(info): with warnings.catch_warnings(): warnings.simplefilter("error") backend._warn_if_scipy_accelerate() + + +def test_accelerate_warnings_fire_on_the_gpu_backend(monkeypatch): + """With CuPy selected, NumPy and SciPy LAPACK still run on the host (centre + moves, host-side contractions, the SVD fallbacks, qr), so an + Accelerate-linked build must still be warned about.""" + import numpy + import scipy + + def faked(real): + def show_config(mode="stdout"): + if mode == "stdout": + return None + config = real(mode="dicts") + config["Build Dependencies"]["lapack"]["name"] = "accelerate" + return config + + return show_config + + monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) + monkeypatch.setattr(numpy, "show_config", faked(numpy.show_config)) + monkeypatch.setattr(scipy, "show_config", faked(scipy.show_config)) + module, caught = _reload_backend(monkeypatch, "cupy", _fake_cupy(num_devices=1)) + assert module.GPU is True + messages = [str(w.message) for w in caught] + assert any("NumPy is built against" in m for m in messages), messages + assert any("SciPy is built against" in m for m in messages), messages From 9065231463b7ce0d18adc0bed6c12450945edfe0 Mon Sep 17 00:00:00 2001 From: meandmytram Date: Mon, 14 Sep 2026 10:57:32 -0400 Subject: [PATCH 06/11] Give the GPU-backend warning test a synthetic config The test faked show_config by delegating to the installed function with mode="dicts", which SciPy 1.9 (still allowed by the dependency floor and handled by _lapack_vendor) does not accept; there the helper fell back to the real OpenBLAS config and the expected SciPy warning never fired. The fake now returns a synthetic modern config and no longer touches the installed API. Checked with real OpenBLAS, with NumPy and SciPy reporting Accelerate, and with SciPy's show_config replaced by the 1.9 signature. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- tests/backend/test_array.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/backend/test_array.py b/tests/backend/test_array.py index ddd8f72f..9d553c34 100644 --- a/tests/backend/test_array.py +++ b/tests/backend/test_array.py @@ -197,19 +197,21 @@ def test_accelerate_warnings_fire_on_the_gpu_backend(monkeypatch): import numpy import scipy - def faked(real): - def show_config(mode="stdout"): - if mode == "stdout": - return None - config = real(mode="dicts") - config["Build Dependencies"]["lapack"]["name"] = "accelerate" - return config - - return show_config + def show_config(mode="stdout"): + # A synthetic modern config, independent of the installed versions' + # show_config API (SciPy 1.9 has no mode argument). + if mode == "stdout": + return None + return { + "Build Dependencies": { + "blas": {"name": "accelerate"}, + "lapack": {"name": "accelerate"}, + } + } monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) - monkeypatch.setattr(numpy, "show_config", faked(numpy.show_config)) - monkeypatch.setattr(scipy, "show_config", faked(scipy.show_config)) + monkeypatch.setattr(numpy, "show_config", show_config) + monkeypatch.setattr(scipy, "show_config", show_config) module, caught = _reload_backend(monkeypatch, "cupy", _fake_cupy(num_devices=1)) assert module.GPU is True messages = [str(w.message) for w in caught] From 01990a2450c8e70b03d1b4af3904602f5a5570f0 Mon Sep 17 00:00:00 2001 From: meandmytram Date: Mon, 14 Sep 2026 19:06:21 -0400 Subject: [PATCH 07/11] Say at which bond dimensions the Accelerate fault broke BB decoding The README described the failure only as "a bivariate-bicycle decode returned wrong verdicts". It now gives the scale: on the [[72,12,6]] code in the natural qubit order at chi_max=400, where the SVDs act on matrices of about 800x1600, 21 of 22 shots with a non-trivial error decoded wrongly while mdopt still reduced its SVDs by QR first, against 3 of 24 on OpenBLAS; the same test shot decoded correctly at chi_max 64, 128 and 256, which is noted as no guarantee because the fault depends on heap state. The SIGBUS matrix size (636x304) is given as well. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- README.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index cf9105f9..32d1732d 100644 --- a/README.md +++ b/README.md @@ -38,10 +38,17 @@ On Apple silicon running macOS 14 or later, pip installs the NumPy and SciPy wheels tagged `macosx_14_0_arm64`, which link Apple's Accelerate framework. On mdopt's matrices (rank-deficient, with singular values spanning many orders of magnitude) Accelerate's LAPACK corrupted memory: -`numpy.linalg.qr` died with SIGBUS, `numpy.linalg.svd` tripped malloc's -heap check, and a bivariate-bicycle decode returned wrong verdicts while -every test passed. mdopt's SVD helpers call both libraries, and mdopt warns -at import when it finds Accelerate behind either one. The same versions are +`numpy.linalg.qr` died with SIGBUS on a 636x304 matrix, and +`numpy.linalg.svd` tripped malloc's heap check. How much this showed in the +decoding depended on the bond dimension. On the [[72,12,6]] +bivariate-bicycle code in the natural qubit order, at `chi_max=400`, where +the SVDs act on matrices of about 800x1600, 21 of 22 shots with a +non-trivial error decoded wrongly while mdopt still reduced its SVDs by QR +first, against 3 of 24 on OpenBLAS, and every unit test still passed. The +same test shot decoded correctly at `chi_max` 64, 128 and 256. Not seeing +the fault at those smaller bond dimensions is no guarantee, because it +depends on the state of the heap. mdopt's SVD helpers call both libraries, +and mdopt warns at import when it finds Accelerate behind either one. The same versions are also published as OpenBLAS builds, the `macosx_11_0_arm64` NumPy wheel and the `macosx_12_0_arm64` SciPy wheel, which install on the same machines: From 0e73e331c1ad554bb9c35dc43a7c7b8568345d9f Mon Sep 17 00:00:00 2001 From: meandmytram Date: Mon, 14 Sep 2026 19:27:49 -0400 Subject: [PATCH 08/11] Correct figures in the comments and docs added by #543 An audit of the claims #543 added, against the result files and fresh measurements, found five wrong figures: - svd() comment and the convergence test's docstring: the chi_max=400 [[72,12,6]] failure was 21 of 22 shots with a non-trivial error, not "24 of 24" (that count mixed repeated runs of one shot with a second). - svd() comment: the pre-reduction saved at most 7% and nothing measurable on most workloads, rather than "0-7%". - _to_numpy docstring: the per-call cupy import cost about 11,000 failed imports per decode, between roughly a tenth and a third of a decode (surface_bitflip 4-13%, dem_d3 about a third, timed on pre-#543 main), not a flat "~18%". - _contract_cached docstring: a shape-keyed cache misses 1-2% of zip-up calls on the DEM workloads and 23% on the RCM surface workload, not "7-14% on large codes". - test_convergence docstring: the test takes about 20 minutes on an idle laptop, not 30-60 minutes per decode. The benchmark README's classical_ldpc row now says three LDPC codes. Claims checked and left as they are: Apple M5 10 cores, the ~1700-site DEM chain (1677 mechanisms), the DEM workload parameters, the opt_einsum overhead at small chi (42% of a zip step at chi=32), and agreement of pre- and post-#543 decoders at chi=1e5 (1.6e-14 on 22 posteriors). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- benchmarks/README.md | 2 +- mdopt/contractor/contractor.py | 5 +++-- mdopt/utils/utils.py | 11 +++++++---- tests/decoding/test_convergence.py | 7 ++++--- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index b72896e7..57229d1f 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -10,7 +10,7 @@ and a change is only accepted if the fingerprints still match. | `surface_bitflip` | 5x5 surface code, bit-flip noise, 6 shots, χ = 64, natural qubit order | verdicts exact, posterior entries within 1e-2 | | `css_optimised` | the same code under `qubit_order_strategy="Optimised"` (reverse Cuthill-McKee) | as above | | `shor_depolarising` | Shor code, depolarising noise, 40 shots, χ = 128: the dense-readout path | as above | -| `classical_ldpc` | random (3,4) LDPC code, XOR constraints + dephasing-DMRG readout | overlaps exact to 1e-10 | +| `classical_ldpc` | three random (3,4) LDPC codes, XOR constraints + dephasing-DMRG readout | overlaps exact to 1e-10 | | `dmrg_ground_state` | DMRG on a 24-site transverse-field Ising chain | energy exact to 1e-10 | | `dem_d3` | circuit-level detector error model, d=3 r=3 p=0.8%, eight busiest syndromes, χ = 32 | verdicts exact, class masses within 1e-2 | | `dem_d5` | the same at d=5 r=5 p=0.5%, busiest syndrome, χ = 32 | as above | diff --git a/mdopt/contractor/contractor.py b/mdopt/contractor/contractor.py index 25f297a9..79d23de8 100644 --- a/mdopt/contractor/contractor.py +++ b/mdopt/contractor/contractor.py @@ -23,8 +23,9 @@ def _contract_cached(subscripts, path, backend, *tensors): rebuilds path metadata on every call even when ``optimize`` is explicit. An expression with an explicit path is shape-independent, so it is cached per (subscripts, path) only -- keying on operand shapes made - truncation's data-dependent bond dimensions miss 7-14% of calls on - large codes -- and built from whatever shapes the first call carries. + truncation's data-dependent bond dimensions miss the cache (1-2% of the + zip-up calls on the circuit-level DEM workloads, 23% on the RCM + surface-code one) -- and built from whatever shapes the first call carries. """ key = (subscripts, path) expression = _EXPRESSIONS.get(key) diff --git a/mdopt/utils/utils.py b/mdopt/utils/utils.py index 667aaa90..e5f3189e 100644 --- a/mdopt/utils/utils.py +++ b/mdopt/utils/utils.py @@ -16,7 +16,8 @@ def _to_numpy(a): Resolved through the backend's own transfer, which honours MDOPT_BACKEND and the CUDA device probe; a per-call ``import cupy`` here once cost - ~18% of a decoding run. + about 11,000 failed imports per decode, between roughly a tenth and + a third of the decode depending on the workload. """ host = xp.to_host(a) try: @@ -93,12 +94,14 @@ def svd( # (the macOS 14+ arm64 wheels) it corrupted memory on the # decoders' rank-deficient matrices: numpy.linalg.qr died # with SIGBUS, and a [[72,12,6]] decode at chi_max=400 - # returned wrong verdicts on 24 of 24 shots while every - # unit test and benchmark fingerprint still passed; + # returned wrong verdicts on 21 of 22 shots with a + # non-trivial error, while every unit test and benchmark + # fingerprint still passed; # - with OpenBLAS it is memory-safe but not equivalent under # truncation: it moved a chi_max=64 surface-code posterior # entry by 0.017, beyond the benchmark suite's tolerance; - # - it saved 0-7% on the benchmark workloads. + # - it saved at most 7% on the benchmark workloads, and + # nothing measurable on most of them. # Reintroducing it needs `benchmarks/bench_suite.py --check` # and tests/decoding/test_convergence.py to pass. # No finiteness pre-scan: a non-finite input makes the SVD diff --git a/tests/decoding/test_convergence.py b/tests/decoding/test_convergence.py index 25c5ed98..7d72bc6d 100644 --- a/tests/decoding/test_convergence.py +++ b/tests/decoding/test_convergence.py @@ -29,9 +29,10 @@ def test_bb_72_12_6_natural_order_single_error_converges_in_chi(): This instance exposed the QR pre-reduction of ``svd`` (since removed): with it, the decode returned a flat or wrongly peaked - posterior at chi_max=400 on NumPy/Accelerate builds -- 24 of 24 shots - -- while every unit test and benchmark fingerprint still passed. Each - chi_max=400 decode takes 30-60 minutes on a laptop. + posterior at chi_max=400 on NumPy/Accelerate builds, as did 21 of the 22 + shots with a non-trivial error in the natural-order experiment, while + every unit test and benchmark fingerprint still passed. The test takes + about 20 minutes on an idle laptop. """ code = create_bb_code(6, 6, "x**3 + y + y**2", "y**3 + x + x**2") error = "I" * 58 + "Z" + "I" * 13 From 5225048d2508b864875edd690225e9a50bfaa8d3 Mon Sep 17 00:00:00 2001 From: meandmytram Date: Mon, 14 Sep 2026 19:35:37 -0400 Subject: [PATCH 09/11] Say which wheels link Accelerate and when the wrong verdicts occurred The _warn_if_accelerate docstring said the macOS arm64 wheels of NumPy 2.x link Accelerate; only the macOS 14+ wheels (macosx_14_0_arm64) do, while the macosx_11_0_arm64 wheels are OpenBLAS builds. The docstring and the NumPy warning message also attributed the wrong chi_max=400 verdicts to Accelerate alone, but those occurred while mdopt still reduced its SVDs by QR first; with the plain SVD the checked chi_max=400 shots decoded correctly even on Accelerate. The memory corruption itself (SIGBUS in linalg.qr, malloc's heap check in dgesdd) does not depend on that path, so the warning stays; its wording now matches the README. Also reflows an over-long README line. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- README.md | 7 ++++--- mdopt/backend/array.py | 20 ++++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 32d1732d..398bcc12 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,10 @@ first, against 3 of 24 on OpenBLAS, and every unit test still passed. The same test shot decoded correctly at `chi_max` 64, 128 and 256. Not seeing the fault at those smaller bond dimensions is no guarantee, because it depends on the state of the heap. mdopt's SVD helpers call both libraries, -and mdopt warns at import when it finds Accelerate behind either one. The same versions are -also published as OpenBLAS builds, the `macosx_11_0_arm64` NumPy wheel and -the `macosx_12_0_arm64` SciPy wheel, which install on the same machines: +and mdopt warns at import when it finds Accelerate behind either one. The +same versions are also published as OpenBLAS builds, the `macosx_11_0_arm64` +NumPy wheel and the `macosx_12_0_arm64` SciPy wheel, which install on the +same machines: ```bash pip download "numpy==$(python -c 'import numpy; print(numpy.__version__)')" \ diff --git a/mdopt/backend/array.py b/mdopt/backend/array.py index 553c1538..f261cd20 100644 --- a/mdopt/backend/array.py +++ b/mdopt/backend/array.py @@ -89,12 +89,15 @@ def _lapack_vendor(module) -> str: def _warn_if_accelerate(numpy_module) -> None: """Warn once when NumPy's LAPACK is Apple's Accelerate framework. - On the macOS arm64 wheels of NumPy 2.x (which link Accelerate) the - decoders' rank-deficient, wide-spectrum matrices made ``linalg.qr`` die - with SIGBUS and ``linalg.svd`` trip malloc's heap-corruption check - inside dgesdd, and a [[72,12,6]] decode at chi_max=400 returned wrong - verdicts while every unit test passed. The OpenBLAS build of the same - NumPy version has none of this; it is the ``macosx_11_0_arm64`` wheel:: + On the NumPy 2.x wheels for macOS 14+ on Apple silicon + (``macosx_14_0_arm64``, which link Accelerate) the decoders' + rank-deficient, wide-spectrum matrices made ``linalg.qr`` die with SIGBUS + and ``linalg.svd`` trip malloc's heap-corruption check inside dgesdd. + While mdopt still reduced its SVDs by QR first, this also turned a + [[72,12,6]] decode at chi_max=400 into wrong verdicts (21 of 22 shots with + a non-trivial error) while every unit test passed. The OpenBLAS build of + the same NumPy version has none of this; it is the ``macosx_11_0_arm64`` + wheel:: pip download numpy== --platform macosx_11_0_arm64 \ --only-binary=:all: --no-deps -d /tmp/numpy-openblas @@ -107,8 +110,9 @@ def _warn_if_accelerate(numpy_module) -> None: if "accelerate" in _lapack_vendor(numpy_module): warnings.warn( "NumPy is built against Apple's Accelerate LAPACK, which corrupted " - "memory and produced wrong decoding verdicts on mdopt's matrices " - "(see mdopt.backend.array._warn_if_accelerate). Install the " + "memory on mdopt's matrices and, with an earlier SVD code path, led " + "to wrong decoding verdicts (see " + "mdopt.backend.array._warn_if_accelerate). Install the " "OpenBLAS build of NumPy (the macosx_11_0_arm64 wheel) or set " "MDOPT_ALLOW_ACCELERATE=1 to silence this warning.", RuntimeWarning, From c1c8f0ac18ce52148452468d8fc6875ef740490c Mon Sep 17 00:00:00 2001 From: meandmytram Date: Mon, 14 Sep 2026 19:46:44 -0400 Subject: [PATCH 10/11] Fix a garbled docstring command, a stale range and a redundant import - The trailing backslash in _warn_if_accelerate's docstring was a line continuation inside a normal string literal, so the two-line pip command rendered as one run-on line; it is escaped now. - The _to_numpy docstring's "between roughly a tenth and a third" left out the measured low end; it now says about 4% on surface_bitflip to about a third on dem_d3, matching the measurements. - test_accelerate_lapack_warns_unless_allowed re-imported warnings, which the module already imports. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- mdopt/backend/array.py | 2 +- mdopt/utils/utils.py | 4 ++-- tests/backend/test_array.py | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/mdopt/backend/array.py b/mdopt/backend/array.py index f261cd20..464d9320 100644 --- a/mdopt/backend/array.py +++ b/mdopt/backend/array.py @@ -99,7 +99,7 @@ def _warn_if_accelerate(numpy_module) -> None: the same NumPy version has none of this; it is the ``macosx_11_0_arm64`` wheel:: - pip download numpy== --platform macosx_11_0_arm64 \ + pip download numpy== --platform macosx_11_0_arm64 \\ --only-binary=:all: --no-deps -d /tmp/numpy-openblas pip install --force-reinstall --no-deps /tmp/numpy-openblas/*.whl diff --git a/mdopt/utils/utils.py b/mdopt/utils/utils.py index e5f3189e..7dde4e06 100644 --- a/mdopt/utils/utils.py +++ b/mdopt/utils/utils.py @@ -16,8 +16,8 @@ def _to_numpy(a): Resolved through the backend's own transfer, which honours MDOPT_BACKEND and the CUDA device probe; a per-call ``import cupy`` here once cost - about 11,000 failed imports per decode, between roughly a tenth and - a third of the decode depending on the workload. + about 11,000 failed imports per decode: from about 4% of the decode on + surface_bitflip to about a third on dem_d3. """ host = xp.to_host(a) try: diff --git a/tests/backend/test_array.py b/tests/backend/test_array.py index 9d553c34..5c906d06 100644 --- a/tests/backend/test_array.py +++ b/tests/backend/test_array.py @@ -112,7 +112,6 @@ def test_stream_is_a_context_manager_on_numpy(monkeypatch): def test_accelerate_lapack_warns_unless_allowed(monkeypatch): """A NumPy built against Accelerate triggers a RuntimeWarning at import time; MDOPT_ALLOW_ACCELERATE=1 silences it; other vendors are silent.""" - import warnings from types import SimpleNamespace from mdopt.backend import array as backend From 8e32ce040c083bb6a68f0383b5e52c214c3781bf Mon Sep 17 00:00:00 2001 From: meandmytram Date: Mon, 14 Sep 2026 19:58:54 -0400 Subject: [PATCH 11/11] Raise the SciPy floor to 1.15.0 and drop the unreachable SciPy 1.9 fallback The dependency table allowed scipy>=1.9.2 next to numpy>=2.3.0, but no SciPy before 1.15 can be installed with NumPy 2.3: SciPy 1.9 to 1.12 require numpy<1.29, and 1.13 and 1.14 require numpy<2.3 (PyPI metadata). SciPy 1.15.0 allows numpy<2.5 and Python >=3.10, matching this package's NumPy and Python floors, so it is the real minimum. poetry.lock already locks SciPy 1.15.3; only its content hash changes. With the floor raised, the vendor check's fallback for SciPy 1.9's configuration API (show() without a mode argument, __config__.get_info) could never run, so it and its test are removed; an unreadable configuration still reports an unknown vendor instead of raising, which a test now checks. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01S6mbxc9eJDQMVx7tjvYwud --- mdopt/backend/array.py | 18 +++--------------- poetry.lock | 4 ++-- pyproject.toml | 2 +- tests/backend/test_array.py | 38 ++++++------------------------------- 4 files changed, 12 insertions(+), 50 deletions(-) diff --git a/mdopt/backend/array.py b/mdopt/backend/array.py index 464d9320..1a9e944f 100644 --- a/mdopt/backend/array.py +++ b/mdopt/backend/array.py @@ -63,27 +63,15 @@ def _load_backend(): def _lapack_vendor(module) -> str: """The LAPACK library a NumPy or SciPy module was built against, lower-cased. - NumPy and SciPy 1.10 and later report it through - ``show_config(mode="dicts")``. SciPy 1.9 (a numpy.distutils build) has no - ``mode`` argument and exposes its link information through - ``__config__.get_info`` instead; an Accelerate link there is reported as - ``"accelerate"``. Returns ``""`` when neither source is available. + Read from ``show_config(mode="dicts")``, which every NumPy and SciPy + release mdopt supports provides (NumPy 2.3+, SciPy 1.15+). Returns ``""`` + when the configuration cannot be read. """ try: deps = module.show_config(mode="dicts")["Build Dependencies"] return str(deps["lapack"]["name"]).lower() - except TypeError: - pass # show_config without a mode argument: fall through to get_info except Exception: # pylint: disable=broad-except return "" - try: - info = module.__config__.get_info("lapack_opt") - except Exception: # pylint: disable=broad-except - return "" - text = " ".join(str(value) for value in dict(info).values()).lower() - if "accelerate" in text or "veclib" in text: - return "accelerate" - return text def _warn_if_accelerate(numpy_module) -> None: diff --git a/poetry.lock b/poetry.lock index fa7bc7a9..3d82c815 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.3 and should not be changed by hand. [[package]] name = "accessible-pygments" @@ -4627,4 +4627,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.11,<4.0" -content-hash = "dd4a5e7d24bc35695c8f673dd13db5bd2ad2b19cd8c638a876466d4bcbcc89f2" +content-hash = "011aaabbb7188b795e0fbb3cf9c3b0bcc63ee44aaf486d05b3365665c405ed19" diff --git a/pyproject.toml b/pyproject.toml index 79c240c9..5f513554 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ ] dependencies = [ - "scipy>=1.9.2,<2.0.0", + "scipy>=1.15.0,<2.0.0", "opt-einsum>=3.3.0,<4.0.0", "more-itertools>=8.12,<11.0", "threadpoolctl>=3.6.0,<4.0.0", diff --git a/tests/backend/test_array.py b/tests/backend/test_array.py index 5c906d06..bfd13d25 100644 --- a/tests/backend/test_array.py +++ b/tests/backend/test_array.py @@ -134,11 +134,14 @@ def fake_numpy(vendor): backend._warn_if_accelerate(fake_numpy("scipy-openblas")) monkeypatch.setenv("MDOPT_ALLOW_ACCELERATE", "1") backend._warn_if_accelerate(fake_numpy("accelerate")) + # An unreadable configuration (no show_config, or one without the dicts + # mode) reports an unknown vendor instead of raising. assert backend._lapack_vendor(SimpleNamespace()) == "" + assert backend._lapack_vendor(SimpleNamespace(show_config=lambda: None)) == "" def test_scipy_shaped_config_is_detected(monkeypatch): - """SciPy 1.10+ reports its LAPACK through show_config(mode="dicts"), with + """SciPy reports its LAPACK through show_config(mode="dicts"), with the vendor capitalised ("Accelerate"); the SciPy check must warn on it.""" from types import SimpleNamespace @@ -160,35 +163,6 @@ def show_config(mode="stdout"): backend._warn_if_scipy_accelerate() -def test_scipy_without_show_config_mode_falls_back_to_get_info(monkeypatch): - """SciPy 1.9 has show() without a mode argument; its link information - comes from __config__.get_info, which must still detect Accelerate and - stay silent for OpenBLAS.""" - from types import SimpleNamespace - - from mdopt.backend import array as backend - - def show(): - return None - - def scipy_with(info): - return SimpleNamespace( - show_config=show, __config__=SimpleNamespace(get_info=lambda name: info) - ) - - monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) - accelerate = scipy_with({"extra_link_args": ["-Wl,-framework", "-Wl,Accelerate"]}) - openblas = scipy_with({"libraries": ["openblas", "openblas"], "language": "c"}) - assert backend._lapack_vendor(accelerate) == "accelerate" - monkeypatch.setitem(sys.modules, "scipy", accelerate) - with pytest.warns(RuntimeWarning, match="SciPy"): - backend._warn_if_scipy_accelerate() - monkeypatch.setitem(sys.modules, "scipy", openblas) - with warnings.catch_warnings(): - warnings.simplefilter("error") - backend._warn_if_scipy_accelerate() - - def test_accelerate_warnings_fire_on_the_gpu_backend(monkeypatch): """With CuPy selected, NumPy and SciPy LAPACK still run on the host (centre moves, host-side contractions, the SVD fallbacks, qr), so an @@ -197,8 +171,8 @@ def test_accelerate_warnings_fire_on_the_gpu_backend(monkeypatch): import scipy def show_config(mode="stdout"): - # A synthetic modern config, independent of the installed versions' - # show_config API (SciPy 1.9 has no mode argument). + # A synthetic config, independent of what the installed NumPy and + # SciPy report. if mode == "stdout": return None return {