diff --git a/README.md b/README.md index bb70d3ec..cf9105f9 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,31 @@ Otherwise, you can clone the repository and use [poetry](https://python-poetry.o poetry install ``` +### A note on NumPy's BLAS on Apple silicon + +On Apple silicon running macOS 14 or later, pip installs the NumPy and +SciPy wheels tagged `macosx_14_0_arm64`, which link Apple's Accelerate +framework. On mdopt's matrices (rank-deficient, with singular values +spanning many orders of magnitude) Accelerate's LAPACK corrupted memory: +`numpy.linalg.qr` died with SIGBUS, `numpy.linalg.svd` tripped malloc's +heap check, and a bivariate-bicycle decode returned wrong verdicts while +every test passed. mdopt's SVD helpers call both libraries, and mdopt warns +at import when it finds Accelerate behind either one. The same versions are +also published as OpenBLAS builds, the `macosx_11_0_arm64` NumPy wheel and +the `macosx_12_0_arm64` SciPy wheel, which install on the same machines: + +```bash +pip download "numpy==$(python -c 'import numpy; print(numpy.__version__)')" \ + --platform macosx_11_0_arm64 --only-binary=:all: --no-deps -d /tmp/openblas-wheels +pip download "scipy==$(python -c 'import scipy; print(scipy.__version__)')" \ + --platform macosx_12_0_arm64 --only-binary=:all: --no-deps -d /tmp/openblas-wheels +pip install --force-reinstall --no-deps /tmp/openblas-wheels/*.whl +``` + +Set `MDOPT_ALLOW_ACCELERATE=1` to silence the warning if you must keep +Accelerate. Use `OMP_NUM_THREADS=1` (or `OPENBLAS_NUM_THREADS=1`) for the +per-process BLAS of worker pools. + ## Minimal example ```python diff --git a/mdopt/backend/array.py b/mdopt/backend/array.py index 2a13512a..553c1538 100644 --- a/mdopt/backend/array.py +++ b/mdopt/backend/array.py @@ -60,6 +60,88 @@ def _load_backend(): GPU = _xp.__name__ == "cupy" +def _lapack_vendor(module) -> str: + """The LAPACK library a NumPy or SciPy module was built against, lower-cased. + + NumPy and SciPy 1.10 and later report it through + ``show_config(mode="dicts")``. SciPy 1.9 (a numpy.distutils build) has no + ``mode`` argument and exposes its link information through + ``__config__.get_info`` instead; an Accelerate link there is reported as + ``"accelerate"``. Returns ``""`` when neither source is available. + """ + try: + deps = module.show_config(mode="dicts")["Build Dependencies"] + return str(deps["lapack"]["name"]).lower() + except TypeError: + pass # show_config without a mode argument: fall through to get_info + except Exception: # pylint: disable=broad-except + return "" + try: + info = module.__config__.get_info("lapack_opt") + except Exception: # pylint: disable=broad-except + return "" + text = " ".join(str(value) for value in dict(info).values()).lower() + if "accelerate" in text or "veclib" in text: + return "accelerate" + return text + + +def _warn_if_accelerate(numpy_module) -> None: + """Warn once when NumPy's LAPACK is Apple's Accelerate framework. + + On the macOS arm64 wheels of NumPy 2.x (which link Accelerate) the + decoders' rank-deficient, wide-spectrum matrices made ``linalg.qr`` die + with SIGBUS and ``linalg.svd`` trip malloc's heap-corruption check + inside dgesdd, and a [[72,12,6]] decode at chi_max=400 returned wrong + verdicts while every unit test passed. The OpenBLAS build of the same + NumPy version has none of this; it is the ``macosx_11_0_arm64`` wheel:: + + pip download numpy== --platform macosx_11_0_arm64 \ + --only-binary=:all: --no-deps -d /tmp/numpy-openblas + pip install --force-reinstall --no-deps /tmp/numpy-openblas/*.whl + + Set MDOPT_ALLOW_ACCELERATE=1 to silence the warning. + """ + if os.getenv("MDOPT_ALLOW_ACCELERATE") == "1": + return + if "accelerate" in _lapack_vendor(numpy_module): + warnings.warn( + "NumPy is built against Apple's Accelerate LAPACK, which corrupted " + "memory and produced wrong decoding verdicts on mdopt's matrices " + "(see mdopt.backend.array._warn_if_accelerate). Install the " + "OpenBLAS build of NumPy (the macosx_11_0_arm64 wheel) or set " + "MDOPT_ALLOW_ACCELERATE=1 to silence this warning.", + RuntimeWarning, + stacklevel=2, + ) + + +def _warn_if_scipy_accelerate() -> None: + """The same warning for SciPy, whose LAPACK the SVD helpers also call.""" + if os.getenv("MDOPT_ALLOW_ACCELERATE") == "1": + return + try: + scipy = importlib.import_module("scipy") + except ImportError: + return + if "accelerate" in _lapack_vendor(scipy): + warnings.warn( + "SciPy is built against Apple's Accelerate LAPACK (see " + "mdopt.backend.array._warn_if_accelerate); install the OpenBLAS " + "build (the macosx_12_0_arm64 wheel) or set " + "MDOPT_ALLOW_ACCELERATE=1 to silence this warning.", + RuntimeWarning, + stacklevel=2, + ) + + +# Both checks run on every backend: with CuPy selected the orthogonality-centre +# moves and host-side contractions still call NumPy's linalg and BLAS, and the +# SVD fallbacks and qr call SciPy's LAPACK. +_warn_if_accelerate(importlib.import_module("numpy")) +_warn_if_scipy_accelerate() + + # ---------------------------------------------------------------------- # Introspection helpers # ---------------------------------------------------------------------- diff --git a/tests/backend/test_array.py b/tests/backend/test_array.py index 472ba3d8..9d553c34 100644 --- a/tests/backend/test_array.py +++ b/tests/backend/test_array.py @@ -45,11 +45,23 @@ def get_device_count(): def test_defaults_to_numpy(monkeypatch): - """With no MDOPT_BACKEND set, NumPy is used and nothing is warned about.""" + """With no MDOPT_BACKEND set, NumPy is used and nothing unexpected is warned about. + + The import deliberately warns when NumPy or SciPy links Apple's Accelerate + framework, which the stock macOS 14+ wheels do (the macOS CI runners among + them); those warnings are allowed, any other warning is not. + """ module, caught = _reload_backend(monkeypatch, None, None) assert module.GPU is False assert module.backend_name() == "numpy" - assert not caught + unexpected = [ + w + for w in caught + if not ( + issubclass(w.category, RuntimeWarning) and "Accelerate" in str(w.message) + ) + ] + assert not unexpected, [str(w.message) for w in unexpected] def test_falls_back_when_cupy_missing(monkeypatch): @@ -79,7 +91,14 @@ def test_uses_cupy_when_device_present(monkeypatch): module, caught = _reload_backend(monkeypatch, "cupy", _fake_cupy(num_devices=1)) assert module.GPU is True assert module.backend_name() == "cupy" - assert not caught + unexpected = [ + w + for w in caught + if not ( + issubclass(w.category, RuntimeWarning) and "Accelerate" in str(w.message) + ) + ] + assert not unexpected, [str(w.message) for w in unexpected] def test_stream_is_a_context_manager_on_numpy(monkeypatch): @@ -88,3 +107,113 @@ def test_stream_is_a_context_manager_on_numpy(monkeypatch): with module.stream(): pass module.synchronize() + + +def test_accelerate_lapack_warns_unless_allowed(monkeypatch): + """A NumPy built against Accelerate triggers a RuntimeWarning at import + time; MDOPT_ALLOW_ACCELERATE=1 silences it; other vendors are silent.""" + import warnings + from types import SimpleNamespace + + from mdopt.backend import array as backend + + def fake_numpy(vendor): + return SimpleNamespace( + show_config=lambda mode="dicts": { + "Build Dependencies": { + "lapack": {"name": vendor}, + "blas": {"name": vendor}, + } + } + ) + + monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) + with pytest.warns(RuntimeWarning, match="Accelerate"): + backend._warn_if_accelerate(fake_numpy("accelerate")) + with warnings.catch_warnings(): + warnings.simplefilter("error") + backend._warn_if_accelerate(fake_numpy("scipy-openblas")) + monkeypatch.setenv("MDOPT_ALLOW_ACCELERATE", "1") + backend._warn_if_accelerate(fake_numpy("accelerate")) + assert backend._lapack_vendor(SimpleNamespace()) == "" + + +def test_scipy_shaped_config_is_detected(monkeypatch): + """SciPy 1.10+ reports its LAPACK through show_config(mode="dicts"), with + the vendor capitalised ("Accelerate"); the SciPy check must warn on it.""" + from types import SimpleNamespace + + from mdopt.backend import array as backend + + def show_config(mode="stdout"): + if mode == "stdout": + return None + return { + "Build Dependencies": { + "blas": {"name": "Accelerate"}, + "lapack": {"name": "Accelerate"}, + } + } + + monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) + monkeypatch.setitem(sys.modules, "scipy", SimpleNamespace(show_config=show_config)) + with pytest.warns(RuntimeWarning, match="SciPy"): + backend._warn_if_scipy_accelerate() + + +def test_scipy_without_show_config_mode_falls_back_to_get_info(monkeypatch): + """SciPy 1.9 has show() without a mode argument; its link information + comes from __config__.get_info, which must still detect Accelerate and + stay silent for OpenBLAS.""" + from types import SimpleNamespace + + from mdopt.backend import array as backend + + def show(): + return None + + def scipy_with(info): + return SimpleNamespace( + show_config=show, __config__=SimpleNamespace(get_info=lambda name: info) + ) + + monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) + accelerate = scipy_with({"extra_link_args": ["-Wl,-framework", "-Wl,Accelerate"]}) + openblas = scipy_with({"libraries": ["openblas", "openblas"], "language": "c"}) + assert backend._lapack_vendor(accelerate) == "accelerate" + monkeypatch.setitem(sys.modules, "scipy", accelerate) + with pytest.warns(RuntimeWarning, match="SciPy"): + backend._warn_if_scipy_accelerate() + monkeypatch.setitem(sys.modules, "scipy", openblas) + with warnings.catch_warnings(): + warnings.simplefilter("error") + backend._warn_if_scipy_accelerate() + + +def test_accelerate_warnings_fire_on_the_gpu_backend(monkeypatch): + """With CuPy selected, NumPy and SciPy LAPACK still run on the host (centre + moves, host-side contractions, the SVD fallbacks, qr), so an + Accelerate-linked build must still be warned about.""" + import numpy + import scipy + + def show_config(mode="stdout"): + # A synthetic modern config, independent of the installed versions' + # show_config API (SciPy 1.9 has no mode argument). + if mode == "stdout": + return None + return { + "Build Dependencies": { + "blas": {"name": "accelerate"}, + "lapack": {"name": "accelerate"}, + } + } + + monkeypatch.delenv("MDOPT_ALLOW_ACCELERATE", raising=False) + monkeypatch.setattr(numpy, "show_config", show_config) + monkeypatch.setattr(scipy, "show_config", show_config) + module, caught = _reload_backend(monkeypatch, "cupy", _fake_cupy(num_devices=1)) + assert module.GPU is True + messages = [str(w.message) for w in caught] + assert any("NumPy is built against" in m for m in messages), messages + assert any("SciPy is built against" in m for m in messages), messages