fix: ship kernel headers in wheels so @Thread works after pip (0.2.1)

pyproject.toml

@@ -8,7 +8,7 @@ build-backend = "scikit_build_core.build"
 [project]
 readme = "ReadMe.md"
 name = "cthreads"
-version = "0.2.0"
+version = "0.2.1"
 description = "Compile @Threadable / @Thread Python into native C++ kernels and run them off the GIL."
 requires-python = ">=3.10"
 license = { file = "LICENSE" }
@@ -37,7 +37,7 @@ Documentation = "https://github.com/K-T0BIAS/CThreads/tree/main/docs"
 
 
 [project.optional-dependencies]
-test = ["pytest>=8"]
+test = ["pytest>=8", "build"]
 dev = ["pytest>=8", "build", "scikit-build-core>=0.10"]
 
 [tool.scikit-build]

scripts/retarget_gpu_wheel.py

@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 """Retarget this tree to build/publish the cthreads-gpu PyPI distribution.
 
-GPU-enabled wheels are a separate PyPI project (``cthreads-gpu``) with the same
-import path ``cthreads``. Prefer ``pip install cthreads-gpu`` for GPU; do not
-install ``cthreads`` and ``cthreads-gpu`` together (they both ship ``_ext``).
+GPU-enabled wheels are a separate PyPI project (`cthreads-gpu`) with the same
+import path `cthreads`. Prefer `pip install cthreads-gpu` for GPU; do not
+install `cthreads` and `cthreads-gpu` together (they both ship `_ext`).
 
-Run from the repo root before cibuildwheel / ``python -m build`` for the GPU job.
+Run from the repo root before cibuildwheel / `python -m build` for the GPU job.
 """
 
 from __future__ import annotations

src/cthreads/cpp/CMakeLists.txt

@@ -135,6 +135,19 @@ else()
     )
 endif()
 
+# Kernel DLL build needs these next to the installed package (PyPI wheels).
+# Editable installs resolve headers from the monorepo cpp/ tree via build.py.
+# Layout: cthreads/_native/headers/... and cthreads/_native/runtime/sync_bridge.cpp
+# so sync_bridge's `#include "../headers/sync/syncState.hpp"` stays valid.
+if(DEFINED SKBUILD AND NOT (DEFINED SKBUILD_STATE AND SKBUILD_STATE STREQUAL "editable"))
+    install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/headers/"
+        DESTINATION cthreads/_native/headers
+    )
+    install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/runtime/sync_bridge.cpp"
+        DESTINATION cthreads/_native/runtime
+    )
+endif()
+
 message(STATUS "Python:  ${Python_EXECUTABLE} (${Python_VERSION})")
 if(DEFINED SKBUILD_STATE)
     message(STATUS "SKBUILD_STATE: ${SKBUILD_STATE}")

src/cthreads/python/cthreads/build.py

@@ -18,6 +18,98 @@
 
 BINARY_STEM = "cthreads_kernels"
 
+# Installed wheel layout (see CMakeLists.txt install rules):
+#   site-packages/cthreads/_native/headers/shared_host.hpp
+#   site-packages/cthreads/_native/runtime/sync_bridge.cpp
+# Editable / monorepo layout:
+#   .../python/cthreads/build.py  ->  .../cpp/headers , .../cpp/runtime
+_NATIVE_DIRNAME = "_native"
+
+
+def _package_dir() -> Path:
+    """Directory containing this package's `build.py` (installed or editable)."""
+    return Path(__file__).resolve().parent
+
+
+def _packaged_native_root() -> Path | None:
+    """
+    Return `cthreads/_native` when the wheel-installed kernel assets exist.
+
+    #### Returns
+    - Path | None = native root, or None when not a packaged install
+    """
+    root = _package_dir() / _NATIVE_DIRNAME
+    if (root / "headers" / "shared_host.hpp").is_file():
+        return root
+    return None
+
+
+def _monorepo_cpp_root() -> Path | None:
+    """
+    Return `.../cthreads/cpp` for editable/source checkouts.
+
+    `build.py` lives at `.../python/cthreads/build.py`; cpp is a sibling of
+    `python/`.
+
+    #### Returns
+    - Path | None = cpp root, or None when the tree is not present
+    """
+    # .../python/cthreads/build.py -> parents[2] == .../cthreads (src/cthreads)
+    cpp = _package_dir().parent.parent / "cpp"
+    if (cpp / "headers" / "shared_host.hpp").is_file():
+        return cpp
+    return None
+
+
+def runtime_headers_dir() -> Path:
+    """
+    Directory that must be on the kernel compile include path.
+
+    Prefers wheel-installed `_native/headers`, then the monorepo `cpp/headers`.
+    Raises if neither exists — silent omission caused PyPI/Colab kernel builds to
+    fail with missing `shared_host.hpp`.
+
+    #### Returns
+    - Path = include directory containing `shared_host.hpp`
+
+    #### Raises
+    - RuntimeError = bundled headers are missing from this install
+    """
+    packaged = _packaged_native_root()
+    if packaged is not None:
+        return packaged / "headers"
+    mono = _monorepo_cpp_root()
+    if mono is not None:
+        return mono / "headers"
+    raise RuntimeError(
+        "cthreads kernel runtime headers are missing from this install. "
+        "Expected either:\n"
+        f"  - {_package_dir() / _NATIVE_DIRNAME / 'headers' / 'shared_host.hpp'}\n"
+        "    (PyPI / wheel install), or\n"
+        f"  - {_package_dir().parent.parent / 'cpp' / 'headers' / 'shared_host.hpp'}\n"
+        "    (editable / source checkout).\n"
+        "Reinstall from a wheel built with current CMake install rules, or use "
+        "an editable install from the full repository."
+    )
+
+
+def sync_bridge_source() -> Path | None:
+    """
+    Path to `sync_bridge.cpp` when present (optional link input).
+
+    #### Returns
+    - Path | None = source file, or None if this install has no bridge
+    """
+    packaged = _packaged_native_root()
+    if packaged is not None:
+        p = packaged / "runtime" / "sync_bridge.cpp"
+        return p if p.is_file() else None
+    mono = _monorepo_cpp_root()
+    if mono is not None:
+        p = mono / "runtime" / "sync_bridge.cpp"
+        return p if p.is_file() else None
+    return None
+
 
 def _locate_vs_cl() -> str | None:
     pf86 = os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")
@@ -127,20 +219,10 @@ def _collect_sources_and_includes() -> tuple[list[Path], list[Path]]:
         include_dirs.add(unit.hpp_path.resolve().parent)
         thread_dirs.add(unit.hpp_path.resolve().parent)
 
-    # Bundled runtime headers: .../python/cthreads/V2/build.py -> .../cpp/headers
-    runtime_headers = (
-        Path(__file__).resolve().parent.parent.parent / "cpp" / "headers"
-    )
-    if runtime_headers.is_dir():
-        include_dirs.add(runtime_headers)
-
-    sync_bridge = (
-        Path(__file__).resolve().parent.parent.parent
-        / "cpp"
-        / "runtime"
-        / "sync_bridge.cpp"
-    )
-    if sync_bridge.is_file():
+    # Bundled runtime headers + optional sync_bridge (wheel _native/ or monorepo cpp/).
+    include_dirs.add(runtime_headers_dir())
+    sync_bridge = sync_bridge_source()
+    if sync_bridge is not None:
         sources.append(sync_bridge)
 
     for thread_dir in thread_dirs:

tests/integration/.gitignore

@@ -0,0 +1,11 @@
+# >>> cthreads (auto)
+__Thread__/
+__Threadable__/
+__Gpu__/
+.cthreads_cache.json
+cthreads_kernels.dll
+cthreads_kernels.so
+cthreads_kernels.lib
+libcthreads_kernels.so
+libcthreads_kernels.dylib
+# <<< cthreads (auto)

tests/integration/test_wheel_native_headers.py

@@ -0,0 +1,152 @@
+"""
+Integration: wheel must ship kernel native headers; packaged layout must compile.
+
+These catch the PyPI/Colab regression where `@Thread` failed with
+`shared_host.hpp: No such file or directory` after `pip install cthreads`.
+"""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+import sys
+import zipfile
+from pathlib import Path
+
+import pytest
+
+build_mod = __import__("cthreads.build", fromlist=["*"])
+from cthreads import Thread, thread
+from helpers import skip_if_kernel_runtime_error
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+CPP_HEADERS = REPO_ROOT / "src" / "cthreads" / "cpp" / "headers"
+SYNC_BRIDGE = REPO_ROOT / "src" / "cthreads" / "cpp" / "runtime" / "sync_bridge.cpp"
+
+
+@pytest.mark.integration
+def test_installed_package_ships_native_headers_or_monorepo_fallback():
+    """
+    Wheel installs must expose `cthreads/_native/headers``.
+    Editable checkouts may use monorepo ``cpp/headers`` instead.
+    """
+    packaged = build_mod._packaged_native_root()
+    mono = build_mod._monorepo_cpp_root()
+    assert packaged is not None or mono is not None, (
+        "neither wheel _native/ nor monorepo cpp/headers found — "
+        "kernel builds cannot succeed"
+    )
+    headers = build_mod.runtime_headers_dir()
+    assert (headers / "shared_host.hpp").is_file()
+
+
+@pytest.mark.integration
+def test_non_editable_install_requires_packaged_native_headers():
+    """
+    cibuildwheel / ``pip install`` layout: monorepo ``cpp/`` is not next to
+    ``build.py``, so ``_native/headers/shared_host.hpp`` must be in the wheel.
+    """
+    if build_mod._monorepo_cpp_root() is not None:
+        pytest.skip("editable / source tree — monorepo headers are enough")
+    packaged = build_mod._packaged_native_root()
+    assert packaged is not None, (
+        "wheel install missing cthreads/_native/headers/shared_host.hpp — "
+        "this is the PyPI/Colab shared_host.hpp regression"
+    )
+    assert (packaged / "runtime" / "sync_bridge.cpp").is_file()
+
+
+@pytest.mark.integration
+def test_built_wheel_contains_native_headers(tmp_path: Path):
+    """
+    ``python -m build --wheel`` must install ``cthreads/_native/headers/...``.
+
+    Soft-skips when the toolchain/`build` frontend is unavailable. Set
+    ``CTHREADS_REQUIRE_WHEEL_HEADERS=1`` to fail hard (manual / release gate).
+    cibuildwheel coverage is ``test_non_editable_install_requires_packaged_native_headers``.
+    """
+    require = bool(os.environ.get("CTHREADS_REQUIRE_WHEEL_HEADERS"))
+    try:
+        import build as _build_frontend  # noqa: F401
+    except ImportError:
+        if require:
+            pytest.fail("python `build` package required to verify wheel header install")
+        pytest.skip("python `build` package not installed")
+
+    out = tmp_path / "dist"
+    out.mkdir()
+    proc = subprocess.run(
+        [sys.executable, "-m", "build", "--wheel", "--outdir", str(out)],
+        cwd=str(REPO_ROOT),
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+    )
+    if proc.returncode != 0:
+        detail = (proc.stderr or proc.stdout or "")[-3000:]
+        if require:
+            pytest.fail(f"wheel build failed (headers packaging unverified):\n{detail}")
+        pytest.skip(
+            "wheel build failed in this environment "
+            f"(need scikit-build toolchain):\n{detail}"
+        )
+    wheels = list(out.glob("*.whl"))
+    assert wheels, "no wheel produced"
+    with zipfile.ZipFile(wheels[0]) as zf:
+        names = zf.namelist()
+    assert any(
+        n.replace("\\", "/").endswith("cthreads/_native/headers/shared_host.hpp")
+        for n in names
+    ), f"shared_host.hpp missing from wheel; sample entries: {names[:20]}"
+    assert any(
+        n.replace("\\", "/").endswith("cthreads/_native/runtime/sync_bridge.cpp")
+        for n in names
+    ), "sync_bridge.cpp missing from wheel"
+
+
+@pytest.mark.integration
+def test_thread_compiles_when_only_packaged_native_layout_visible(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+):
+    """
+    Hide the monorepo ``cpp/`` path and expose only a wheel-like ``_native/``
+    tree next to a fake ``build.py``. Kernel link must still find headers.
+    """
+    if not CPP_HEADERS.is_dir() or not SYNC_BRIDGE.is_file():
+        pytest.skip("monorepo cpp assets missing")
+
+    pkg = tmp_path / "cthreads"
+    native = pkg / "_native"
+    shutil.copytree(CPP_HEADERS, native / "headers")
+    (native / "runtime").mkdir(parents=True)
+    shutil.copy2(SYNC_BRIDGE, native / "runtime" / "sync_bridge.cpp")
+    fake_build_py = pkg / "build.py"
+    fake_build_py.write_text("# fake package build module path\n", encoding="utf-8")
+
+    monkeypatch.setattr(build_mod, "__file__", str(fake_build_py))
+
+    # Confirm monorepo fallback is not used for this process's locator.
+    assert build_mod._packaged_native_root() is not None
+    assert build_mod.runtime_headers_dir() == (native / "headers").resolve()
+
+    work = tmp_path / "work"
+    work.mkdir()
+    monkeypatch.chdir(work)
+
+    @Thread
+    def add_one(n: int, out: list[int]) -> None:
+        i: int = 0
+        while i < n:
+            out[i] = out[i] + 1
+            i = i + 1
+
+    out: list[int] = [0, 1, 2, 3]
+    try:
+        thread(add_one, len(out), out).join()
+    except RuntimeError as exc:
+        skip_if_kernel_runtime_error(exc)
+        raise
+
+    assert out == [1, 2, 3, 4]