Warn when NumPy links Accelerate; document the OpenBLAS wheel

README.md

@@ -32,6 +32,31 @@ Otherwise, you can clone the repository and use [poetry](https://python-poetry.o
 poetry install
 ```
 
+### A note on NumPy's BLAS on Apple silicon
+
+On Apple silicon running macOS 14 or later, pip installs the NumPy and
+SciPy wheels tagged `macosx_14_0_arm64`, which link Apple's Accelerate
+framework. On mdopt's matrices (rank-deficient, with singular values
+spanning many orders of magnitude) Accelerate's LAPACK corrupted memory:
+`numpy.linalg.qr` died with SIGBUS, `numpy.linalg.svd` tripped malloc's
+heap check, and a bivariate-bicycle decode returned wrong verdicts while
+every test passed. mdopt's SVD helpers call both libraries, and mdopt warns
+at import when it finds Accelerate behind either one. The same versions are
+also published as OpenBLAS builds, the `macosx_11_0_arm64` NumPy wheel and
+the `macosx_12_0_arm64` SciPy wheel, which install on the same machines:
+
+```bash
+pip download "numpy==$(python -c 'import numpy; print(numpy.__version__)')" \
+    --platform macosx_11_0_arm64 --only-binary=:all: --no-deps -d /tmp/openblas-wheels
+pip download "scipy==$(python -c 'import scipy; print(scipy.__version__)')" \
+    --platform macosx_12_0_arm64 --only-binary=:all: --no-deps -d /tmp/openblas-wheels
+pip install --force-reinstall --no-deps /tmp/openblas-wheels/*.whl
+```
+
+Set `MDOPT_ALLOW_ACCELERATE=1` to silence the warning if you must keep
+Accelerate. Use `OMP_NUM_THREADS=1` (or `OPENBLAS_NUM_THREADS=1`) for the
+per-process BLAS of worker pools.
+
 ## Minimal example
 
 ```python

mdopt/backend/array.py

@@ -60,6 +60,88 @@ def _load_backend():
 GPU = _xp.__name__ == "cupy"
 
 
+def _lapack_vendor(module) -> str:
+    """The LAPACK library a NumPy or SciPy module was built against, lower-cased.
+
+    NumPy and SciPy 1.10 and later report it through
+    ``show_config(mode="dicts")``. SciPy 1.9 (a numpy.distutils build) has no
+    ``mode`` argument and exposes its link information through
+    ``__config__.get_info`` instead; an Accelerate link there is reported as
+    ``"accelerate"``. Returns ``""`` when neither source is available.
+    """
+    try:
+        deps = module.show_config(mode="dicts")["Build Dependencies"]
+        return str(deps["lapack"]["name"]).lower()
+    except TypeError:
+        pass  # show_config without a mode argument: fall through to get_info
+    except Exception:  # pylint: disable=broad-except
+        return ""
+    try:
+        info = module.__config__.get_info("lapack_opt")
+    except Exception:  # pylint: disable=broad-except
+        return ""
+    text = " ".join(str(value) for value in dict(info).values()).lower()
+    if "accelerate" in text or "veclib" in text:
+        return "accelerate"
+    return text
+
+
+def _warn_if_accelerate(numpy_module) -> None:
+    """Warn once when NumPy's LAPACK is Apple's Accelerate framework.
+
+    On the macOS arm64 wheels of NumPy 2.x (which link Accelerate) the
+    decoders' rank-deficient, wide-spectrum matrices made ``linalg.qr`` die
+    with SIGBUS and ``linalg.svd`` trip malloc's heap-corruption check
+    inside dgesdd, and a [[72,12,6]] decode at chi_max=400 returned wrong
+    verdicts while every unit test passed. The OpenBLAS build of the same
+    NumPy version has none of this; it is the ``macosx_11_0_arm64`` wheel::
+
+        pip download numpy==<version> --platform macosx_11_0_arm64 \
+            --only-binary=:all: --no-deps -d /tmp/numpy-openblas
+        pip install --force-reinstall --no-deps /tmp/numpy-openblas/*.whl
+
+    Set MDOPT_ALLOW_ACCELERATE=1 to silence the warning.
+    """
+    if os.getenv("MDOPT_ALLOW_ACCELERATE") == "1":
+        return
+    if "accelerate" in _lapack_vendor(numpy_module):
+        warnings.warn(
+            "NumPy is built against Apple's Accelerate LAPACK, which corrupted "
+            "memory and produced wrong decoding verdicts on mdopt's matrices "
+            "(see mdopt.backend.array._warn_if_accelerate). Install the "
+            "OpenBLAS build of NumPy (the macosx_11_0_arm64 wheel) or set "
+            "MDOPT_ALLOW_ACCELERATE=1 to silence this warning.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
+
+def _warn_if_scipy_accelerate() -> None:
+    """The same warning for SciPy, whose LAPACK the SVD helpers also call."""
+    if os.getenv("MDOPT_ALLOW_ACCELERATE") == "1":
+        return
+    try:
+        scipy = importlib.import_module("scipy")
+    except ImportError:
+        return
+    if "accelerate" in _lapack_vendor(scipy):
+        warnings.warn(
+            "SciPy is built against Apple's Accelerate LAPACK (see "
+            "mdopt.backend.array._warn_if_accelerate); install the OpenBLAS "
+            "build (the macosx_12_0_arm64 wheel) or set "
+            "MDOPT_ALLOW_ACCELERATE=1 to silence this warning.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
+
+# Both checks run on every backend: with CuPy selected the orthogonality-centre
+# moves and host-side contractions still call NumPy's linalg and BLAS, and the
+# SVD fallbacks and qr call SciPy's LAPACK.
+_warn_if_accelerate(importlib.import_module("numpy"))
+_warn_if_scipy_accelerate()
+
+
 # ----------------------------------------------------------------------
 # Introspection helpers
 # ----------------------------------------------------------------------

tests/backend/test_array.py

@@ -45,11 +45,23 @@ def get_device_count():
 
 
 def test_defaults_to_numpy(monkeypatch):
-    """With no MDOPT_BACKEND set, NumPy is used and nothing is warned about."""
+    """With no MDOPT_BACKEND set, NumPy is used and nothing unexpected is warned about.
+
+    The import deliberately warns when NumPy or SciPy links Apple's Accelerate
+    framework, which the stock macOS 14+ wheels do (the macOS CI runners among
+    them); those warnings are allowed, any other warning is not.
+    """
     module, caught = _reload_backend(monkeypatch, None, None)
     assert module.GPU is False
     assert module.backend_name() == "numpy"
-    assert not caught
+    unexpected = [
+        w
+        for w in caught
+        if not (
+            issubclass(w.category, RuntimeWarning) and "Accelerate" in str(w.message)
+        )
+    ]
+    assert not unexpected, [str(w.message) for w in unexpected]
 
 
 def test_falls_back_when_cupy_missing(monkeypatch):
@@ -79,7 +91,14 @@ def test_uses_cupy_when_device_present(monkeypatch):
     module, caught = _reload_backend(monkeypatch, "cupy", _fake_cupy(num_devices=1))
     assert module.GPU is True
     assert module.backend_name() == "cupy"
-    assert not caught
+    unexpected = [
+        w
+        for w in caught
+        if not (
+            issubclass(w.category, RuntimeWarning) and "Accelerate" in str(w.message)
+        )
+    ]
+    assert not unexpected, [str(w.message) for w in unexpected]
 
 
 def test_stream_is_a_context_manager_on_numpy(monkeypatch):
@@ -88,3 +107,111 @@ def test_stream_is_a_context_manager_on_numpy(monkeypatch):
     with module.stream():
         pass
     module.synchronize()
+
+
+def test_accelerate_lapack_warns_unless_allowed(monkeypatch):
+    """A NumPy built against Accelerate triggers a RuntimeWarning at import
+    time; MDOPT_ALLOW_ACCELERATE=1 silences it; other vendors are silent."""
+    import warnings
+    from types import SimpleNamespace
+
+    from mdopt.backend import array as backend
+
+    def fake_numpy(vendor):
+        return SimpleNamespace(
+            show_config=lambda mode="dicts": {
+                "Build Dependencies": {
+                    "lapack": {"name": vendor},
+                    "blas": {"name": vendor},
+                }
+            }
+        )
+
+    monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False)
+    with pytest.warns(RuntimeWarning, match="Accelerate"):
+        backend._warn_if_accelerate(fake_numpy("accelerate"))
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        backend._warn_if_accelerate(fake_numpy("scipy-openblas"))
+        monkeypatch.setenv("MDOPT_ALLOW_ACCELERATE", "1")
+        backend._warn_if_accelerate(fake_numpy("accelerate"))
+    assert backend._lapack_vendor(SimpleNamespace()) == ""
+
+
+def test_scipy_shaped_config_is_detected(monkeypatch):
+    """SciPy 1.10+ reports its LAPACK through show_config(mode="dicts"), with
+    the vendor capitalised ("Accelerate"); the SciPy check must warn on it."""
+    from types import SimpleNamespace
+
+    from mdopt.backend import array as backend
+
+    def show_config(mode="stdout"):
+        if mode == "stdout":
+            return None
+        return {
+            "Build Dependencies": {
+                "blas": {"name": "Accelerate"},
+                "lapack": {"name": "Accelerate"},
+            }
+        }
+
+    monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False)
+    monkeypatch.setitem(sys.modules, "scipy", SimpleNamespace(show_config=show_config))
+    with pytest.warns(RuntimeWarning, match="SciPy"):
+        backend._warn_if_scipy_accelerate()
+
+
+def test_scipy_without_show_config_mode_falls_back_to_get_info(monkeypatch):
+    """SciPy 1.9 has show() without a mode argument; its link information
+    comes from __config__.get_info, which must still detect Accelerate and
+    stay silent for OpenBLAS."""
+    from types import SimpleNamespace
+
+    from mdopt.backend import array as backend
+
+    def show():
+        return None
+
+    def scipy_with(info):
+        return SimpleNamespace(
+            show_config=show, __config__=SimpleNamespace(get_info=lambda name: info)
+        )
+
+    monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False)
+    accelerate = scipy_with({"extra_link_args": ["-Wl,-framework", "-Wl,Accelerate"]})
+    openblas = scipy_with({"libraries": ["openblas", "openblas"], "language": "c"})
+    assert backend._lapack_vendor(accelerate) == "accelerate"
+    monkeypatch.setitem(sys.modules, "scipy", accelerate)
+    with pytest.warns(RuntimeWarning, match="SciPy"):
+        backend._warn_if_scipy_accelerate()
+    monkeypatch.setitem(sys.modules, "scipy", openblas)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error")
+        backend._warn_if_scipy_accelerate()
+
+
+def test_accelerate_warnings_fire_on_the_gpu_backend(monkeypatch):
+    """With CuPy selected, NumPy and SciPy LAPACK still run on the host (centre
+    moves, host-side contractions, the SVD fallbacks, qr), so an
+    Accelerate-linked build must still be warned about."""
+    import numpy
+    import scipy
+
+    def faked(real):
+        def show_config(mode="stdout"):
+            if mode == "stdout":
+                return None
+            config = real(mode="dicts")
+            config["Build Dependencies"]["lapack"]["name"] = "accelerate"
+            return config
+
+        return show_config
+
+    monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False)
+    monkeypatch.setattr(numpy, "show_config", faked(numpy.show_config))
+    monkeypatch.setattr(scipy, "show_config", faked(scipy.show_config))
+    module, caught = _reload_backend(monkeypatch, "cupy", _fake_cupy(num_devices=1))
+    assert module.GPU is True
+    messages = [str(w.message) for w in caught]
+    assert any("NumPy is built against" in m for m in messages), messages
+    assert any("SciPy is built against" in m for m in messages), messages