From d1ed40daa649042f4b1292ab2509235fe7dca148 Mon Sep 17 00:00:00 2001 From: Darren Garnier Date: Sun, 26 Jul 2026 21:32:58 +1200 Subject: [PATCH] Feature: build a pip-installable MDSplus "local" wheel Build a small MDSplus wheel, at install time, into /python/wheelhouse. Its only payload is a MDSplus.pth (+ a _mdsplus_bootstrap.py it imports) that points a Python environment at this install's python/MDSplus package -- a redirect, not a copy. Gated behind ENABLE_PYTHON_LOCAL_WHEEL (default ON). Why a wheel instead of PYTHONPATH=$MDSPLUS_DIR/python: setting PYTHONPATH makes `import MDSplus` work but leaves the environment with no record that MDSplus is installed. Shipping it as a real distribution means: - MDSplus's dependencies (numpy, ...) are pulled in automatically by the resolver instead of being wrangled out of band; - it installs into a venv / uv-managed env and just works; - other pyprojects can depend on `MDSplus` and resolve it from the local wheelhouse (pip/uv --find-links, or a flat uv index) -- no per-project or per-CI PYTHONPATH plumbing; - being a redirect, it always imports the MDSplus that matches the installed C libraries. At import time the bootstrap back-determines MDSPLUS_DIR from the install the wheel was built for and sets up its environment -- convenient for testing: - startup is minimal: set MDSPLUS_DIR to the baked install (or $MDSPLUS_DIR if it was relocated) and prepend its python/ to sys.path; - on the first `import MDSplus`, if MDSPLUS_DIR was rewritten or the env is not sourced, a one-shot import hook sources the install's setup.sh (honoring local envsyms) and merges the resulting environment; - it never spawns a process at plain startup and can never break the interpreter; MDSPLUS_LOCAL_WHEEL_DISABLE=1 turns it off. The wheel is assembled by make_wheel.py using only the standard library (no build backend, no pip build isolation, no network); its dependencies and metadata are copied from python/MDSplus/pyproject.toml so tooling reports the genuine package info. Co-Authored-By: Claude Opus 4.8 (1M context) --- python/CMakeLists.txt | 10 ++ python/local_wheel/CMakeLists.txt | 40 +++++ python/local_wheel/MDSplus.pth | 1 + python/local_wheel/README.md | 114 +++++++++++++ python/local_wheel/_mdsplus_bootstrap.py.in | 98 +++++++++++ python/local_wheel/gen_wheel.cmake.in | 22 +++ python/local_wheel/make_wheel.py | 175 ++++++++++++++++++++ 7 files changed, 460 insertions(+) create mode 100644 python/local_wheel/CMakeLists.txt create mode 100644 python/local_wheel/MDSplus.pth create mode 100644 python/local_wheel/README.md create mode 100644 python/local_wheel/_mdsplus_bootstrap.py.in create mode 100644 python/local_wheel/gen_wheel.cmake.in create mode 100644 python/local_wheel/make_wheel.py diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 1ab9d14905..c9c239b599 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -1,2 +1,12 @@ add_subdirectory(MDSplus) + +mdsplus_option( + ENABLE_PYTHON_LOCAL_WHEEL BOOL + "Build a pip-installable 'local' wheel whose only payload is a .pth file that points at the installed python/MDSplus directory." + DEFAULT ON +) + +if(ENABLE_PYTHON_LOCAL_WHEEL) + add_subdirectory(local_wheel) +endif() diff --git a/python/local_wheel/CMakeLists.txt b/python/local_wheel/CMakeLists.txt new file mode 100644 index 0000000000..54a7167fac --- /dev/null +++ b/python/local_wheel/CMakeLists.txt @@ -0,0 +1,40 @@ +# Build the MDSplus "local" wheel -- a pip-installable wheel whose only payload +# is a MDSplus.pth (+ a _mdsplus_bootstrap.py it imports) that makes +# `import MDSplus` resolve to the system-installed python/MDSplus package. +# +# The wheel is built at INSTALL time (install(SCRIPT) below) rather than build +# time, so the real install prefix -- ${CMAKE_INSTALL_PREFIX} as resolved at +# install, honoring `cmake --install --prefix ...` -- can be baked into the +# bootstrap and the wheel tied to this install. It is written to +# /python/wheelhouse so it can be consumed via pip --find-links. +# +# At runtime the bootstrap (which runs at EVERY interpreter startup) does the +# minimum, and never spawns a process or sources setup.sh: +# 1. resolve the install -- the baked prefix if its python/ dir still exists, +# else $MDSPLUS_DIR; +# 2. set MDSPLUS_DIR to it (deterministically -- so the package and its C +# libraries always resolve to the same install) for the library loader; +# 3. prepend its python/ dir to sys.path so this install wins. +# +# The wheel is assembled directly by make_wheel.py (standard library only) -- no +# build backend, no pip build isolation, no network access. Its dependencies are +# copied from python/MDSplus/pyproject.toml so they never drift. + +set(_source_toml ${CMAKE_SOURCE_DIR}/python/MDSplus/pyproject.toml) +set(_builder ${CMAKE_CURRENT_SOURCE_DIR}/make_wheel.py) +set(_pth ${CMAKE_CURRENT_SOURCE_DIR}/MDSplus.pth) +set(_bootstrap_tmpl ${CMAKE_CURRENT_SOURCE_DIR}/_mdsplus_bootstrap.py.in) + +configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/gen_wheel.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/gen_wheel.cmake + @ONLY +) + +install(SCRIPT ${CMAKE_CURRENT_BINARY_DIR}/gen_wheel.cmake) + +# Ship the rationale/usage docs next to the wheel in the wheelhouse. +install( + FILES ${CMAKE_CURRENT_SOURCE_DIR}/README.md + DESTINATION python/wheelhouse +) diff --git a/python/local_wheel/MDSplus.pth b/python/local_wheel/MDSplus.pth new file mode 100644 index 0000000000..c6607449dc --- /dev/null +++ b/python/local_wheel/MDSplus.pth @@ -0,0 +1 @@ +import _mdsplus_bootstrap diff --git a/python/local_wheel/README.md b/python/local_wheel/README.md new file mode 100644 index 0000000000..16bec9ec5f --- /dev/null +++ b/python/local_wheel/README.md @@ -0,0 +1,114 @@ +# MDSplus local wheelhouse + +This directory holds a pip-installable **MDSplus** wheel that points a Python +environment at *this* MDSplus installation's `python/` package (via a small +`.pth`/bootstrap redirect — it does **not** copy the package). It is placed +here so it can be consumed as a normal package source, e.g. + +```sh +pip install --find-links "$MDSPLUS_DIR/python/wheelhouse" MDSplus +``` + +## Why a wheel instead of `PYTHONPATH=$MDSPLUS_DIR/python`? + +Setting `PYTHONPATH` makes `import MDSplus` work, but the environment has **no +record** that MDSplus is installed. That has real costs: + +- **Dependencies aren't handled.** MDSplus needs `numpy` (and optional extras). + With `PYTHONPATH` you have to install those yourself, by hand, everywhere. +- **Package managers can't see it.** pip / uv / poetry can't include MDSplus in + a lockfile, can't detect version conflicts, and can't reason about it at all. +- **Other projects can't depend on it.** A downstream `pyproject.toml` cannot + declare a dependency on `MDSplus` and have it resolved. + +Installing this wheel registers MDSplus as a **real distribution** (with proper +metadata and `Requires-Dist`), so instead: + +- **Dependencies are pulled in automatically.** Installing `MDSplus` brings in + `numpy` (and the `widgets` extra on request) through the normal resolver — no + separate, out-of-band dependency wrangling. +- **It works in a venv / uv-managed env.** `pip install` (or uv) MDSplus into an + isolated environment and it "just works." +- **Other pyprojects can depend on it.** A consuming project lists `MDSplus` as a + dependency and resolves it from this wheelhouse — no per-project or per-CI + `PYTHONPATH` plumbing. +- **No copy, always in sync.** The wheel is a redirect to this install's + `python/` dir, so you import the MDSplus that matches the installed C + libraries; upgrading the install upgrades what the environment imports. + +## Using it + +### pip + +```sh +# MDSplus resolves from the wheelhouse; its deps (numpy, ...) resolve from PyPI. +# Do NOT pass --no-index -- that would block downloading the dependencies. +pip install --find-links "$MDSPLUS_DIR/python/wheelhouse" MDSplus +``` + +### uv (CLI) + +```sh +uv pip install --find-links "$MDSPLUS_DIR/python/wheelhouse" MDSplus +``` + +### uv (inside another project's `pyproject.toml`) + +Two equivalent routes. Both point at the wheelhouse **directory** (no +version-specific wheel filename to track) and take a **literal** path — uv does +not expand environment variables in `pyproject.toml`/`uv.toml` +(see astral-sh/uv#10096). + +**Simple — `find-links`** (the wheelhouse is searched alongside PyPI): + +```toml +[project] +dependencies = ["MDSplus"] + +[tool.uv] +find-links = ["/usr/local/mdsplus/python/wheelhouse"] +``` + +**Precise — a flat index pinned to MDSplus** (MDSplus comes *only* from the +wheelhouse; everything else, including numpy, from PyPI): + +```toml +[project] +dependencies = ["MDSplus"] + +[[tool.uv.index]] +name = "mdsplus" +url = "/usr/local/mdsplus/python/wheelhouse" +format = "flat" # a directory of wheels, i.e. pip's --find-links +explicit = true # only used for packages pinned to it + +[tool.uv.sources] +MDSplus = { index = "mdsplus" } +``` + +**Relocatable / env-driven:** since `pyproject.toml` paths are literal, to key +off `$MDSPLUS_DIR` use the `UV_FIND_LINKS` environment variable instead +(comma-separated, equivalent to `--find-links`): + +```sh +export UV_FIND_LINKS="$MDSPLUS_DIR/python/wheelhouse" +uv sync # or: uv add MDSplus, uv pip install MDSplus, ... +``` + +## How it works + +The wheel ships only a `MDSplus.pth` that imports a small `_mdsplus_bootstrap` +module. Because it runs at *every* interpreter startup, the startup path is +deliberately minimal and can never fail: it resolves the install (the baked +prefix if it is still present, else `$MDSPLUS_DIR`), sets `MDSPLUS_DIR` to it, +and prepends its `python/` dir to `sys.path` — no subprocess. + +The full MDSplus environment is set up lazily on the first `import MDSplus`, and +only when needed (`MDSPLUS_DIR` was rewritten, or the environment isn't sourced +yet): a one-shot import hook sources that install's `setup.sh` — honoring local +`envsyms` — and merges the resulting environment. Set +`MDSPLUS_LOCAL_WHEEL_DISABLE=1` to turn the bootstrap off without uninstalling. + +Its dependencies and metadata are copied from the real +`python/MDSplus/pyproject.toml`, so tools like `pip show MDSplus` report the +genuine package information. diff --git a/python/local_wheel/_mdsplus_bootstrap.py.in b/python/local_wheel/_mdsplus_bootstrap.py.in new file mode 100644 index 0000000000..ed5683ff95 --- /dev/null +++ b/python/local_wheel/_mdsplus_bootstrap.py.in @@ -0,0 +1,98 @@ +# Imported from MDSplus.pth at every interpreter startup: keep the startup path +# minimal and never let it raise. It sets MDSPLUS_DIR + prepends python/ to +# sys.path, and (only if the env isn't already sourced) registers a one-shot +# import hook that sources setup.sh on the first `import MDSplus` -- deferred so +# startup stays cheap and can't fork-bomb (the env-dump child never imports +# MDSplus). MDSPLUS_LOCAL_WHEEL_DISABLE=1 disables everything. +# +# @MDSPLUS_PREFIX@ is substituted (as a repr literal) by make_wheel.py. +import os +import sys + +_PREFIX = @MDSPLUS_PREFIX@ + +_BEGIN = "<>" +_END = "<>" + + +def _source_setup(mdsdir): + # Source mdsdir/setup.sh once and merge the resulting environment (so local + # envsyms includes/sources are honored). MDSPLUS_DIR is already exported, so + # the shell inherits it; the env-dump child never imports MDSplus. + setup = os.path.join(mdsdir, "setup.sh") + if not os.path.isfile(setup): + return + import json + import subprocess + + def q(s): # POSIX single-quote escape (py2/py3) + return "'" + s.replace("'", "'\\''") + "'" + + dump = ("import os,json,sys;" + "sys.stdout.write(%r+json.dumps(dict(os.environ))+%r)" % (_BEGIN, _END)) + script = ". %s >/dev/null 2>&1; exec %s -c %s" % ( + q(setup), q(sys.executable), q(dump)) + try: + out = subprocess.check_output(["/bin/sh", "-c", script]).decode( + "utf-8", "replace") + i, j = out.find(_BEGIN), out.find(_END) + env = json.loads(out[i + len(_BEGIN):j]) if 0 <= i < j else {} + except Exception: + return + for k, v in env.items(): + if os.environ.get(k) != v: # apply only the delta + os.environ[k] = v + + +class _SetupHook(object): + # One-shot: populate the full MDSplus environment on the first + # `import MDSplus`, then remove itself and defer to the real import. + def __init__(self, mdsdir): + self._mdsdir = mdsdir + + def _fire(self, name): + if name != "MDSplus": + return + try: + sys.meta_path.remove(self) + except ValueError: + pass + _source_setup(self._mdsdir) + + def find_spec(self, name, path=None, target=None): # py3 + self._fire(name) + return None + + def find_module(self, name, path=None): # py2 + self._fire(name) + return None + + +def _activate(prefix): + if os.environ.get("MDSPLUS_LOCAL_WHEEL_DISABLE"): + return + # baked install if still present, else whatever the environment points at + if os.path.isdir(os.path.join(prefix, "python")): + mdsdir = prefix + else: + mdsdir = os.environ.get("MDSPLUS_DIR") + if not mdsdir: + return + pydir = os.path.join(mdsdir, "python") + if not os.path.isdir(pydir): + return + # keep MDSPLUS_DIR and sys.path on the same install (no half state) + changed = os.environ.get("MDSPLUS_DIR") != mdsdir + os.environ["MDSPLUS_DIR"] = mdsdir + if pydir not in sys.path: + sys.path.insert(0, pydir) # prepend: this install wins + # Source setup.sh on first `import MDSplus` if we rewrote MDSPLUS_DIR (a + # stale MDS_PATH from a different install won't do) or it isn't sourced yet. + if changed or not os.environ.get("MDS_PATH"): + sys.meta_path.insert(0, _SetupHook(mdsdir)) + + +try: + _activate(_PREFIX) +except Exception: + pass # a .pth must never break interpreter startup diff --git a/python/local_wheel/gen_wheel.cmake.in b/python/local_wheel/gen_wheel.cmake.in new file mode 100644 index 0000000000..718df33721 --- /dev/null +++ b/python/local_wheel/gen_wheel.cmake.in @@ -0,0 +1,22 @@ +# Configured by CMake (@ONLY) and run at install time via install(SCRIPT). +# +# ${CMAKE_INSTALL_PREFIX} and $ENV{DESTDIR} are evaluated at INSTALL time (they +# survive @ONLY substitution), so the wheel captures the real install prefix -- +# honoring `cmake --install --prefix ...`. That prefix is baked into the +# bootstrap so the wheel is tied to this install. +set(_prefix "${CMAKE_INSTALL_PREFIX}") +# A wheelhouse dir so the wheel is usable via `pip install --find-links +# /python/wheelhouse MDSplus` from other packages. +set(_outdir "$ENV{DESTDIR}${_prefix}/python/wheelhouse") + +message(STATUS "MDSplus local wheel: baking prefix ${_prefix}, writing to ${_outdir}") + +execute_process( + COMMAND "@Python_EXECUTABLE@" "@_builder@" + "@_source_toml@" "@_pth@" "@_bootstrap_tmpl@" + "${_outdir}" "@RELEASE_VERSION@" "${_prefix}" + RESULT_VARIABLE _rc +) +if(NOT _rc EQUAL 0) + message(FATAL_ERROR "Failed to build MDSplus local wheel (exit ${_rc})") +endif() diff --git a/python/local_wheel/make_wheel.py b/python/local_wheel/make_wheel.py new file mode 100644 index 0000000000..dbd0390448 --- /dev/null +++ b/python/local_wheel/make_wheel.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python +"""Build the MDSplus "local" wheel using only the standard library. + +A wheel is just a zip with a -.dist-info/ directory, so there is +no need for a build backend (hatchling/setuptools), pip build isolation, or any +network access. The wheel ships two files, both at the archive root so they land +in site-packages: + + MDSplus.pth -> "import _mdsplus_bootstrap" + _mdsplus_bootstrap.py -> resolves this install at interpreter startup + +Installing the wheel makes `import MDSplus` resolve to the system-installed +python/MDSplus package. This is built at install time so the real install prefix +can be baked into the bootstrap (see the CMake install script). + +Runtime dependencies are copied from python/MDSplus/pyproject.toml so they never +drift from the real package. + +Usage: + make_wheel.py SOURCE_TOML PTH_FILE BOOTSTRAP_TEMPLATE OUTDIR VERSION PREFIX +""" +import base64 +import glob +import hashlib +import os +import sys +import zipfile + +# Universal pure-Python wheel: filename carries the compressed "py2.py3-none-any" +# tag, which the WHEEL metadata expands to one Tag line each. +NAME = "MDSplus" +DISTNAME = "mdsplus" # normalized name used in the filename and dist-info dir +TAG = "py2.py3-none-any" +WHEEL_TAGS = ("py2-none-any", "py3-none-any") +ZIP_DATE = (2020, 1, 1, 0, 0, 0) # fixed, for reproducible archives + + +def load_project(path): + try: + import tomllib as toml_mod + except ImportError: + try: + import tomli as toml_mod + except ImportError: + sys.exit( + "make_wheel.py: reading %s needs Python 3.11+ (tomllib) or the " + "'tomli' package installed for %s" % (path, sys.executable) + ) + with open(path, "rb") as fp: + return toml_mod.load(fp).get("project", {}) + + +def record_hash(data): + digest = hashlib.sha256(data).digest() + return "sha256=" + base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + + +def build_metadata(project, version): + """Mirror the real package's [project] metadata so that `pip show`, + importlib.metadata, etc. report the genuine MDSplus info -- not a stub. + Only the version is overridden (with the CMake build version).""" + lines = [ + "Metadata-Version: 2.1", + "Name: %s" % NAME, + "Version: %s" % version, + ] + + if project.get("description"): + lines.append("Summary: %s" % project["description"]) + if project.get("requires-python"): + lines.append("Requires-Python: %s" % project["requires-python"]) + + license = project.get("license") + if isinstance(license, dict): + license = license.get("text") + if license: + lines.append("License: %s" % license) + + # authors -> Author-email ("Name ") and/or Author. + author_emails, author_names = [], [] + for author in project.get("authors", []): + name, email = author.get("name"), author.get("email") + if email and name: + author_emails.append("%s <%s>" % (name, email)) + elif email: + author_emails.append(email) + elif name: + author_names.append(name) + if author_emails: + lines.append("Author-email: %s" % ", ".join(author_emails)) + if author_names: + lines.append("Author: %s" % ", ".join(author_names)) + + if project.get("keywords"): + lines.append("Keywords: %s" % ",".join(project["keywords"])) + for classifier in project.get("classifiers", []): + lines.append("Classifier: %s" % classifier) + for label, url in project.get("urls", {}).items(): + lines.append("Project-URL: %s, %s" % (label, url)) + + for dep in project.get("dependencies", []): + lines.append("Requires-Dist: %s" % dep) + for extra, extra_deps in project.get("optional-dependencies", {}).items(): + lines.append("Provides-Extra: %s" % extra) + for dep in extra_deps: + lines.append("Requires-Dist: %s; extra == '%s'" % (dep, extra)) + return ("\n".join(lines) + "\n").encode("utf-8") + + +def build_wheel_metadata(): + lines = [ + "Wheel-Version: 1.0", + "Generator: mdsplus-cmake make_wheel.py", + "Root-Is-Purelib: true", + ] + lines += ["Tag: %s" % tag for tag in WHEEL_TAGS] + return ("\n".join(lines) + "\n").encode("utf-8") + + +def main(): + ( + source_toml, + pth_file, + bootstrap_template, + outdir, + version, + prefix, + ) = sys.argv[1:7] + + project = load_project(source_toml) + distinfo = "%s-%s.dist-info" % (DISTNAME, version) + + with open(pth_file, "rb") as fp: + pth_data = fp.read() + + # Bake this install's prefix into the bootstrap as a valid Python literal. + with open(bootstrap_template, "r") as fp: + bootstrap = fp.read().replace("@MDSPLUS_PREFIX@", repr(prefix)) + + # (arcname, bytes) for every member except RECORD itself. + members = [ + ("%s.pth" % NAME, pth_data), + ("_mdsplus_bootstrap.py", bootstrap.encode("utf-8")), + ("%s/METADATA" % distinfo, build_metadata(project, version)), + ("%s/WHEEL" % distinfo, build_wheel_metadata()), + ] + + record_lines = [ + "%s,%s,%d" % (arc, record_hash(data), len(data)) for arc, data in members + ] + record_lines.append("%s/RECORD,," % distinfo) # RECORD lists itself, unhashed + members.append( + ("%s/RECORD" % distinfo, ("\n".join(record_lines) + "\n").encode("utf-8")) + ) + + if not os.path.isdir(outdir): + os.makedirs(outdir) + + # Remove any stale MDSplus wheels so only the current one remains. + for pattern in ("mdsplus-*.whl", "MDSplus-*.whl"): + for old in glob.glob(os.path.join(outdir, pattern)): + os.remove(old) + + wheel_path = os.path.join(outdir, "%s-%s-%s.whl" % (DISTNAME, version, TAG)) + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as whl: + for arc, data in members: + info = zipfile.ZipInfo(arc, date_time=ZIP_DATE) + info.external_attr = 0o644 << 16 + whl.writestr(info, data) + + sys.stdout.write(wheel_path + "\n") + + +if __name__ == "__main__": + main()