From 2c51f2cc43b54c0f684a41948cf5214343c75eb1 Mon Sep 17 00:00:00 2001 From: T-Karu-smaecs Date: Sat, 19 Sep 2026 14:50:06 +0200 Subject: [PATCH] fix: ship kernel runtime headers in wheels (0.2.1) PyPI 0.2.0 installs could not JIT @Thread: shared_host.hpp lived only in the monorepo cpp/ tree and build.py skipped missing includes silently. Bundle headers + sync_bridge under cthreads/_native, resolve that path first, and fail loud if neither layout exists. Add unit/integration regression tests. --- pyproject.toml | 4 +- scripts/retarget_gpu_wheel.py | 8 +- src/cthreads/cpp/CMakeLists.txt | 13 ++ src/cthreads/python/cthreads/build.py | 110 +++++++++++-- tests/integration/.gitignore | 11 ++ .../integration/test_wheel_native_headers.py | 152 ++++++++++++++++++ tests/unit/test_build_native_headers.py | 102 ++++++++++++ 7 files changed, 380 insertions(+), 20 deletions(-) create mode 100644 tests/integration/.gitignore create mode 100644 tests/integration/test_wheel_native_headers.py create mode 100644 tests/unit/test_build_native_headers.py diff --git a/pyproject.toml b/pyproject.toml index 4a48a0c..d4ef922 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ build-backend = "scikit_build_core.build" [project] readme = "ReadMe.md" name = "cthreads" -version = "0.2.0" +version = "0.2.1" description = "Compile @Threadable / @Thread Python into native C++ kernels and run them off the GIL." requires-python = ">=3.10" license = { file = "LICENSE" } @@ -37,7 +37,7 @@ Documentation = "https://github.com/K-T0BIAS/CThreads/tree/main/docs" [project.optional-dependencies] -test = ["pytest>=8"] +test = ["pytest>=8", "build"] dev = ["pytest>=8", "build", "scikit-build-core>=0.10"] [tool.scikit-build] diff --git a/scripts/retarget_gpu_wheel.py b/scripts/retarget_gpu_wheel.py index c521b29..bfdb873 100644 --- a/scripts/retarget_gpu_wheel.py +++ b/scripts/retarget_gpu_wheel.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 """Retarget this tree to build/publish the cthreads-gpu PyPI distribution. -GPU-enabled wheels are a separate PyPI project (``cthreads-gpu``) with the same -import path ``cthreads``. Prefer ``pip install cthreads-gpu`` for GPU; do not -install ``cthreads`` and ``cthreads-gpu`` together (they both ship ``_ext``). +GPU-enabled wheels are a separate PyPI project (`cthreads-gpu`) with the same +import path `cthreads`. Prefer `pip install cthreads-gpu` for GPU; do not +install `cthreads` and `cthreads-gpu` together (they both ship `_ext`). -Run from the repo root before cibuildwheel / ``python -m build`` for the GPU job. +Run from the repo root before cibuildwheel / `python -m build` for the GPU job. """ from __future__ import annotations diff --git a/src/cthreads/cpp/CMakeLists.txt b/src/cthreads/cpp/CMakeLists.txt index 0d71548..9ff81d1 100644 --- a/src/cthreads/cpp/CMakeLists.txt +++ b/src/cthreads/cpp/CMakeLists.txt @@ -135,6 +135,19 @@ else() ) endif() +# Kernel DLL build needs these next to the installed package (PyPI wheels). +# Editable installs resolve headers from the monorepo cpp/ tree via build.py. +# Layout: cthreads/_native/headers/... and cthreads/_native/runtime/sync_bridge.cpp +# so sync_bridge's `#include "../headers/sync/syncState.hpp"` stays valid. +if(DEFINED SKBUILD AND NOT (DEFINED SKBUILD_STATE AND SKBUILD_STATE STREQUAL "editable")) + install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/headers/" + DESTINATION cthreads/_native/headers + ) + install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/runtime/sync_bridge.cpp" + DESTINATION cthreads/_native/runtime + ) +endif() + message(STATUS "Python: ${Python_EXECUTABLE} (${Python_VERSION})") if(DEFINED SKBUILD_STATE) message(STATUS "SKBUILD_STATE: ${SKBUILD_STATE}") diff --git a/src/cthreads/python/cthreads/build.py b/src/cthreads/python/cthreads/build.py index 7db57cc..9e27e0c 100644 --- a/src/cthreads/python/cthreads/build.py +++ b/src/cthreads/python/cthreads/build.py @@ -18,6 +18,98 @@ BINARY_STEM = "cthreads_kernels" +# Installed wheel layout (see CMakeLists.txt install rules): +# site-packages/cthreads/_native/headers/shared_host.hpp +# site-packages/cthreads/_native/runtime/sync_bridge.cpp +# Editable / monorepo layout: +# .../python/cthreads/build.py -> .../cpp/headers , .../cpp/runtime +_NATIVE_DIRNAME = "_native" + + +def _package_dir() -> Path: + """Directory containing this package's `build.py` (installed or editable).""" + return Path(__file__).resolve().parent + + +def _packaged_native_root() -> Path | None: + """ + Return `cthreads/_native` when the wheel-installed kernel assets exist. + + #### Returns + - Path | None = native root, or None when not a packaged install + """ + root = _package_dir() / _NATIVE_DIRNAME + if (root / "headers" / "shared_host.hpp").is_file(): + return root + return None + + +def _monorepo_cpp_root() -> Path | None: + """ + Return `.../cthreads/cpp` for editable/source checkouts. + + `build.py` lives at `.../python/cthreads/build.py`; cpp is a sibling of + `python/`. + + #### Returns + - Path | None = cpp root, or None when the tree is not present + """ + # .../python/cthreads/build.py -> parents[2] == .../cthreads (src/cthreads) + cpp = _package_dir().parent.parent / "cpp" + if (cpp / "headers" / "shared_host.hpp").is_file(): + return cpp + return None + + +def runtime_headers_dir() -> Path: + """ + Directory that must be on the kernel compile include path. + + Prefers wheel-installed `_native/headers`, then the monorepo `cpp/headers`. + Raises if neither exists — silent omission caused PyPI/Colab kernel builds to + fail with missing `shared_host.hpp`. + + #### Returns + - Path = include directory containing `shared_host.hpp` + + #### Raises + - RuntimeError = bundled headers are missing from this install + """ + packaged = _packaged_native_root() + if packaged is not None: + return packaged / "headers" + mono = _monorepo_cpp_root() + if mono is not None: + return mono / "headers" + raise RuntimeError( + "cthreads kernel runtime headers are missing from this install. " + "Expected either:\n" + f" - {_package_dir() / _NATIVE_DIRNAME / 'headers' / 'shared_host.hpp'}\n" + " (PyPI / wheel install), or\n" + f" - {_package_dir().parent.parent / 'cpp' / 'headers' / 'shared_host.hpp'}\n" + " (editable / source checkout).\n" + "Reinstall from a wheel built with current CMake install rules, or use " + "an editable install from the full repository." + ) + + +def sync_bridge_source() -> Path | None: + """ + Path to `sync_bridge.cpp` when present (optional link input). + + #### Returns + - Path | None = source file, or None if this install has no bridge + """ + packaged = _packaged_native_root() + if packaged is not None: + p = packaged / "runtime" / "sync_bridge.cpp" + return p if p.is_file() else None + mono = _monorepo_cpp_root() + if mono is not None: + p = mono / "runtime" / "sync_bridge.cpp" + return p if p.is_file() else None + return None + def _locate_vs_cl() -> str | None: pf86 = os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)") @@ -127,20 +219,10 @@ def _collect_sources_and_includes() -> tuple[list[Path], list[Path]]: include_dirs.add(unit.hpp_path.resolve().parent) thread_dirs.add(unit.hpp_path.resolve().parent) - # Bundled runtime headers: .../python/cthreads/V2/build.py -> .../cpp/headers - runtime_headers = ( - Path(__file__).resolve().parent.parent.parent / "cpp" / "headers" - ) - if runtime_headers.is_dir(): - include_dirs.add(runtime_headers) - - sync_bridge = ( - Path(__file__).resolve().parent.parent.parent - / "cpp" - / "runtime" - / "sync_bridge.cpp" - ) - if sync_bridge.is_file(): + # Bundled runtime headers + optional sync_bridge (wheel _native/ or monorepo cpp/). + include_dirs.add(runtime_headers_dir()) + sync_bridge = sync_bridge_source() + if sync_bridge is not None: sources.append(sync_bridge) for thread_dir in thread_dirs: diff --git a/tests/integration/.gitignore b/tests/integration/.gitignore new file mode 100644 index 0000000..0bf994e --- /dev/null +++ b/tests/integration/.gitignore @@ -0,0 +1,11 @@ +# >>> cthreads (auto) +__Thread__/ +__Threadable__/ +__Gpu__/ +.cthreads_cache.json +cthreads_kernels.dll +cthreads_kernels.so +cthreads_kernels.lib +libcthreads_kernels.so +libcthreads_kernels.dylib +# <<< cthreads (auto) diff --git a/tests/integration/test_wheel_native_headers.py b/tests/integration/test_wheel_native_headers.py new file mode 100644 index 0000000..4567b9a --- /dev/null +++ b/tests/integration/test_wheel_native_headers.py @@ -0,0 +1,152 @@ +""" +Integration: wheel must ship kernel native headers; packaged layout must compile. + +These catch the PyPI/Colab regression where `@Thread` failed with +`shared_host.hpp: No such file or directory` after `pip install cthreads`. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + +import pytest + +build_mod = __import__("cthreads.build", fromlist=["*"]) +from cthreads import Thread, thread +from helpers import skip_if_kernel_runtime_error + +REPO_ROOT = Path(__file__).resolve().parents[2] +CPP_HEADERS = REPO_ROOT / "src" / "cthreads" / "cpp" / "headers" +SYNC_BRIDGE = REPO_ROOT / "src" / "cthreads" / "cpp" / "runtime" / "sync_bridge.cpp" + + +@pytest.mark.integration +def test_installed_package_ships_native_headers_or_monorepo_fallback(): + """ + Wheel installs must expose `cthreads/_native/headers``. + Editable checkouts may use monorepo ``cpp/headers`` instead. + """ + packaged = build_mod._packaged_native_root() + mono = build_mod._monorepo_cpp_root() + assert packaged is not None or mono is not None, ( + "neither wheel _native/ nor monorepo cpp/headers found — " + "kernel builds cannot succeed" + ) + headers = build_mod.runtime_headers_dir() + assert (headers / "shared_host.hpp").is_file() + + +@pytest.mark.integration +def test_non_editable_install_requires_packaged_native_headers(): + """ + cibuildwheel / ``pip install`` layout: monorepo ``cpp/`` is not next to + ``build.py``, so ``_native/headers/shared_host.hpp`` must be in the wheel. + """ + if build_mod._monorepo_cpp_root() is not None: + pytest.skip("editable / source tree — monorepo headers are enough") + packaged = build_mod._packaged_native_root() + assert packaged is not None, ( + "wheel install missing cthreads/_native/headers/shared_host.hpp — " + "this is the PyPI/Colab shared_host.hpp regression" + ) + assert (packaged / "runtime" / "sync_bridge.cpp").is_file() + + +@pytest.mark.integration +def test_built_wheel_contains_native_headers(tmp_path: Path): + """ + ``python -m build --wheel`` must install ``cthreads/_native/headers/...``. + + Soft-skips when the toolchain/`build` frontend is unavailable. Set + ``CTHREADS_REQUIRE_WHEEL_HEADERS=1`` to fail hard (manual / release gate). + cibuildwheel coverage is ``test_non_editable_install_requires_packaged_native_headers``. + """ + require = bool(os.environ.get("CTHREADS_REQUIRE_WHEEL_HEADERS")) + try: + import build as _build_frontend # noqa: F401 + except ImportError: + if require: + pytest.fail("python `build` package required to verify wheel header install") + pytest.skip("python `build` package not installed") + + out = tmp_path / "dist" + out.mkdir() + proc = subprocess.run( + [sys.executable, "-m", "build", "--wheel", "--outdir", str(out)], + cwd=str(REPO_ROOT), + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + detail = (proc.stderr or proc.stdout or "")[-3000:] + if require: + pytest.fail(f"wheel build failed (headers packaging unverified):\n{detail}") + pytest.skip( + "wheel build failed in this environment " + f"(need scikit-build toolchain):\n{detail}" + ) + wheels = list(out.glob("*.whl")) + assert wheels, "no wheel produced" + with zipfile.ZipFile(wheels[0]) as zf: + names = zf.namelist() + assert any( + n.replace("\\", "/").endswith("cthreads/_native/headers/shared_host.hpp") + for n in names + ), f"shared_host.hpp missing from wheel; sample entries: {names[:20]}" + assert any( + n.replace("\\", "/").endswith("cthreads/_native/runtime/sync_bridge.cpp") + for n in names + ), "sync_bridge.cpp missing from wheel" + + +@pytest.mark.integration +def test_thread_compiles_when_only_packaged_native_layout_visible( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """ + Hide the monorepo ``cpp/`` path and expose only a wheel-like ``_native/`` + tree next to a fake ``build.py``. Kernel link must still find headers. + """ + if not CPP_HEADERS.is_dir() or not SYNC_BRIDGE.is_file(): + pytest.skip("monorepo cpp assets missing") + + pkg = tmp_path / "cthreads" + native = pkg / "_native" + shutil.copytree(CPP_HEADERS, native / "headers") + (native / "runtime").mkdir(parents=True) + shutil.copy2(SYNC_BRIDGE, native / "runtime" / "sync_bridge.cpp") + fake_build_py = pkg / "build.py" + fake_build_py.write_text("# fake package build module path\n", encoding="utf-8") + + monkeypatch.setattr(build_mod, "__file__", str(fake_build_py)) + + # Confirm monorepo fallback is not used for this process's locator. + assert build_mod._packaged_native_root() is not None + assert build_mod.runtime_headers_dir() == (native / "headers").resolve() + + work = tmp_path / "work" + work.mkdir() + monkeypatch.chdir(work) + + @Thread + def add_one(n: int, out: list[int]) -> None: + i: int = 0 + while i < n: + out[i] = out[i] + 1 + i = i + 1 + + out: list[int] = [0, 1, 2, 3] + try: + thread(add_one, len(out), out).join() + except RuntimeError as exc: + skip_if_kernel_runtime_error(exc) + raise + + assert out == [1, 2, 3, 4] diff --git a/tests/unit/test_build_native_headers.py b/tests/unit/test_build_native_headers.py new file mode 100644 index 0000000..cd1ea6e --- /dev/null +++ b/tests/unit/test_build_native_headers.py @@ -0,0 +1,102 @@ +""" +Regression tests for kernel runtime header discovery (wheel vs editable). + +PyPI 0.2.0 wheels shipped `_ext` but not `cpp/headers`. `build.py` then only +looked at a monorepo-relative path and silently skipped includes, so Colab / +`pip install cthreads` failed with missing `shared_host.hpp`. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +build_mod = __import__("cthreads.build", fromlist=["*"]) + + +def test_runtime_headers_dir_finds_shared_host_in_editable_checkout(): + headers = build_mod.runtime_headers_dir() + assert (headers / "shared_host.hpp").is_file() + assert (headers / "sync" / "syncState.hpp").is_file() + + +def test_sync_bridge_source_present_in_editable_checkout(): + bridge = build_mod.sync_bridge_source() + assert bridge is not None + assert bridge.is_file() + assert bridge.name == "sync_bridge.cpp" + + +def test_packaged_native_layout_is_preferred(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """ + Simulate a wheel install: headers live under cthreads/_native/, not cpp/. + """ + pkg = tmp_path / "cthreads" + native_headers = pkg / "_native" / "headers" + native_headers.mkdir(parents=True) + (native_headers / "shared_host.hpp").write_text("#pragma once\n", encoding="utf-8") + sync = native_headers / "sync" + sync.mkdir() + (sync / "syncState.hpp").write_text("#pragma once\n", encoding="utf-8") + runtime = pkg / "_native" / "runtime" + runtime.mkdir(parents=True) + (runtime / "sync_bridge.cpp").write_text( + '#include "../headers/sync/syncState.hpp"\n', + encoding="utf-8", + ) + fake_build_py = pkg / "build.py" + fake_build_py.write_text("# fake\n", encoding="utf-8") + + monkeypatch.setattr(build_mod, "__file__", str(fake_build_py)) + + headers = build_mod.runtime_headers_dir() + assert headers == native_headers.resolve() + bridge = build_mod.sync_bridge_source() + assert bridge is not None + assert bridge == (runtime / "sync_bridge.cpp").resolve() + + +def test_missing_headers_raise_instead_of_silent_skip( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + """ + Old bug: missing dir was ignored and compile failed later with a cryptic + missing include. Locator must fail fast with an actionable message. + """ + pkg = tmp_path / "cthreads" + pkg.mkdir() + fake_build_py = pkg / "build.py" + fake_build_py.write_text("# fake\n", encoding="utf-8") + monkeypatch.setattr(build_mod, "__file__", str(fake_build_py)) + + with pytest.raises(RuntimeError, match="runtime headers are missing"): + build_mod.runtime_headers_dir() + + +def test_collect_sources_adds_headers_include_with_shared_host(): + """ + Collecting includes must inject the runtime headers dir (kernel meta always + pulls shared_host.hpp), even when no ThreadUnits are registered yet. + """ + _sources, include_dirs = build_mod._collect_sources_and_includes() + assert any((Path(d) / "shared_host.hpp").is_file() for d in include_dirs) + assert build_mod.sync_bridge_source() is not None + + +def test_cmake_installs_native_headers_into_wheel_layout(): + """ + Guard the CMake install rule that puts headers into cthreads/_native/. + Deleting this silently reintroduces the PyPI/Colab shared_host.hpp bug. + """ + cmake = ( + Path(__file__).resolve().parents[2] + / "src" + / "cthreads" + / "cpp" + / "CMakeLists.txt" + ) + text = cmake.read_text(encoding="utf-8") + assert "cthreads/_native/headers" in text + assert "cthreads/_native/runtime" in text + assert "sync_bridge.cpp" in text