diff --git a/.github/workflows/scripts/qd_build/cmake.py b/.github/workflows/scripts/qd_build/cmake.py index 42a799462e..5d7a4efcb0 100644 --- a/.github/workflows/scripts/qd_build/cmake.py +++ b/.github/workflows/scripts/qd_build/cmake.py @@ -30,6 +30,8 @@ def __init__(self, environ_name): self.option_definitions = { "CMAKE_EXPORT_COMPILE_COMMANDS": ("Generate compile_commands.json", False, ""), } + # CMAKE_ARGS entries we don't parse into definitions, kept verbatim for writeback(). + self.passthrough = "" self.finalized = False @@ -42,6 +44,12 @@ def collect_options(self, *files: str) -> None: def parse_initial_args(self) -> None: args = os.environ.get(self.environ_name, "") + # DEF_RE only understands `-D[:BOOL]=` entries (the options we manage and + # render below). Everything else in CMAKE_ARGS -- CMake generators (`-GNinja`), typed cache + # entries (`-DCMAKE_BUILD_TYPE:STRING=Debug`), lowercase-named or space-containing defines -- + # must be preserved verbatim, else writeback() would silently drop it before scikit-build-core + # sees CMAKE_ARGS. Stash the unparsed remainder (matched entries blanked out) and re-emit it. + self.passthrough = " ".join(DEF_RE.sub(" ", args).split()) for name, value in DEF_RE.findall(args): self.set(name, value) @@ -129,13 +137,17 @@ def print_summary(self, rendered) -> None: def writeback(self) -> None: rendered = self.render() self.print_summary(rendered) - value = " ".join([v for _, v, _ in rendered]) + # Rendered options first, then the verbatim passthrough captured in parse_initial_args. The + # two sets are disjoint (parsed entries are blanked out of passthrough), so there is no + # duplicate-cache-var ambiguity for CMake to resolve. + parts = [v for _, v, _ in rendered] + if self.passthrough: + parts.append(self.passthrough) + value = " ".join(parts) + # CMAKE_ARGS is scikit-build-core's standard CMake-args passthrough, and it is also what we parse on input, so + # writing it back makes the environment exported by `build.py wheel`, `-w`, and `--shell` directly usable by the + # build with no further bridging. os.environ[self.environ_name] = value - # scikit-build-core reads CMake args from CMAKE_ARGS, not the legacy QUADRANTS_CMAKE_ARGS. - # Mirror the rendered args there so the environment exported by `build.py wheel`, `-w`, and - # `--shell` is directly usable -- no manual `export CMAKE_ARGS="$QUADRANTS_CMAKE_ARGS"` step. - if self.environ_name == "QUADRANTS_CMAKE_ARGS": - os.environ["CMAKE_ARGS"] = value self.finalized = True def __setitem__(self, name: str, value: Union[str, bool]) -> None: @@ -145,10 +157,10 @@ def __getitem__(self, name: str) -> Union[str, bool]: return self.definitions[name] -cmake_args = CMakeArgsManager("QUADRANTS_CMAKE_ARGS") +cmake_args = CMakeArgsManager("CMAKE_ARGS") -@banner("Parsing QUADRANTS_CMAKE_ARGS") +@banner("Parsing CMAKE_ARGS") def _init_cmake_args(): cmake_args.collect_options("CMakeLists.txt", *glob.glob("cmake/*.cmake")) cmake_args.parse_initial_args() diff --git a/.github/workflows/scripts/qd_build/entry.py b/.github/workflows/scripts/qd_build/entry.py index 0baf28d43f..54ba96746c 100644 --- a/.github/workflows/scripts/qd_build/entry.py +++ b/.github/workflows/scripts/qd_build/entry.py @@ -23,7 +23,7 @@ # -- code -- @banner("Build Quadrants Wheel") def build_wheel(python: Command) -> None: - # cmake_args.writeback() populates both QUADRANTS_CMAKE_ARGS and (for scikit-build-core) CMAKE_ARGS. + # cmake_args.writeback() renders the effective options into CMAKE_ARGS for scikit-build-core. cmake_args.writeback() plat = None diff --git a/.github/workflows/scripts_new/clang_tidy/2_build.sh b/.github/workflows/scripts_new/clang_tidy/2_build.sh index 0b9c0adfd1..f449be7bf3 100644 --- a/.github/workflows/scripts_new/clang_tidy/2_build.sh +++ b/.github/workflows/scripts_new/clang_tidy/2_build.sh @@ -2,5 +2,5 @@ set -ex -export QUADRANTS_CMAKE_ARGS="-DCMAKE_EXPORT_COMPILE_COMMANDS=ON" +export CMAKE_ARGS="-DCMAKE_EXPORT_COMPILE_COMMANDS=ON" ./build.py wheel diff --git a/.github/workflows/scripts_new/linux/2_build.sh b/.github/workflows/scripts_new/linux/2_build.sh index 769ca9d2f9..01d3646387 100644 --- a/.github/workflows/scripts_new/linux/2_build.sh +++ b/.github/workflows/scripts_new/linux/2_build.sh @@ -8,5 +8,5 @@ export PATH=${LLVM_DIR}/bin:$PATH which clang clang --version -export QUADRANTS_CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON" +export CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON" ./build.py wheel diff --git a/.github/workflows/scripts_new/macosx/2_build.sh b/.github/workflows/scripts_new/macosx/2_build.sh index 6101239513..d609a8c2f2 100644 --- a/.github/workflows/scripts_new/macosx/2_build.sh +++ b/.github/workflows/scripts_new/macosx/2_build.sh @@ -2,5 +2,5 @@ set -ex -export QUADRANTS_CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_METAL:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON" +export CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_METAL:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON" ./build.py wheel diff --git a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh index 7791d70918..3c030bb56f 100644 --- a/.github/workflows/scripts_new/manylinux_wheel/2_build.sh +++ b/.github/workflows/scripts_new/manylinux_wheel/2_build.sh @@ -19,7 +19,7 @@ echo "Detected platform: $PLATFORM" export PATH="$PWD/taichi-llvm-15.0.7-linux-${PLATFORM}/bin:$PATH" # Taichi build options -export QUADRANTS_CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON" +export CMAKE_ARGS="-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON" # GCC toolset include paths inc_base="/opt/rh/gcc-toolset-14/root/usr/include/c++/14" diff --git a/.github/workflows/scripts_new/win/2_build.ps1 b/.github/workflows/scripts_new/win/2_build.ps1 index 54510a822e..30539fb779 100644 --- a/.github/workflows/scripts_new/win/2_build.ps1 +++ b/.github/workflows/scripts_new/win/2_build.ps1 @@ -2,5 +2,5 @@ $ErrorActionPreference = "Stop" Set-PSDebug -Trace 1 trap { Write-Error $_; exit 1 } -$env:GSTAICHI_CMAKE_ARGS = "-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON" +$env:CMAKE_ARGS = "-DQD_WITH_VULKAN:BOOL=ON -DQD_WITH_AMDGPU:BOOL=ON -DQD_WITH_CUDA:BOOL=ON -DQD_BUILD_TESTS:BOOL=ON" python build.py diff --git a/docs/source/user_guide/contributing.md b/docs/source/user_guide/contributing.md index 2c0b7a8279..5c38529d54 100644 --- a/docs/source/user_guide/contributing.md +++ b/docs/source/user_guide/contributing.md @@ -1,4 +1,4 @@ -# Contributing to quadrants +# Advanced: Contributing to quadrants ## Good practice reminder @@ -49,7 +49,7 @@ uv pip install --group dev --group test `build.py` is a python script to automatically set up the build environment for you before invoking the build commands: -* `LLVM libraries`: downloads an archive for `LLVM` libraries, decompresses it and sets `LLVM_DIR`. +* `LLVM libraries`: downloads an archive for [LLVM](https://llvm.org/) libraries (a library for building compilers), decompresses it and sets `LLVM_DIR`. * `clang`: depending on the platform, download `clang` or just check if available with the right version. `build.py` can be used at least two ways: @@ -57,7 +57,7 @@ uv pip install --group dev --group test * `build.py wheel` to build the wheel (via [scikit-build-core](https://scikit-build-core.readthedocs.io/en/latest/), i.e. `pip wheel`) * `build.py --shell` to enter a shell with environment variables set up as with `build.py wheel` in order to let you invoke yourself the commands. -For incremental development, do an editable install (scikit-build-core "redirect" mode: the compiled core is installed and rebuilt on demand, Python edits are live): +For incremental development, do an editable install ([scikit-build-core](https://scikit-build-core.readthedocs.io/en/latest/) "redirect" mode: the compiled core is installed and rebuilt on demand, Python edits are live): ``` ./build.py --shell # run a new shell with environment variables @@ -72,7 +72,7 @@ source env.sh pip install --no-build-isolation -e . -Ceditable.rebuild=true ``` -`build.py` exports both the legacy `QUADRANTS_CMAKE_ARGS` and the `CMAKE_ARGS` that scikit-build-core actually reads, so sourcing `env.sh` (or using `--shell`) is enough -- no manual `export CMAKE_ARGS="$QUADRANTS_CMAKE_ARGS"` step is needed. +`build.py` reads and exports `CMAKE_ARGS` (scikit-build-core's CMake-args passthrough), so sourcing `env.sh` (or using `--shell`) is enough to make the configured options available to the build. ## Building the package for release purposes @@ -84,13 +84,13 @@ To build the release package: We use `cmake` to build the C++ core. scikit-build-core puts the CMake build tree under `build/{wheel_tag}`, where the wheel tag encodes the Python version and host platform. For example: `build/cp310-cp310-linux_x86_64`. -You can modify the cmake options to your liking in order to enable or disable some features you need or don't need. To discover them, you can use `ccmake`: +You can modify the cmake options to your liking in order to enable or disable some features you need or don't need. To discover them, you can use [ccmake](https://cmake.org/cmake/help/latest/manual/ccmake.1.html): ``` ccmake build/cp310-cp310-linux_x86_64 ``` -You could then set the environment variable `CMAKE_ARGS` (scikit-build-core's CMake-args passthrough) to configure the build. `build.py` also accepts the legacy `QUADRANTS_CMAKE_ARGS` and forwards it to `CMAKE_ARGS`. For instance, to disable the CUDA and AMDGPU backends: +You could then set the environment variable `CMAKE_ARGS` (scikit-build-core's CMake-args passthrough) to configure the build. `build.py` reads it, layers on the toolchain options it manages, and exports it back. For instance, to disable the CUDA and AMDGPU backends: ``` export CMAKE_ARGS="-DQD_WITH_CUDA=OFF -DQD_WITH_AMDGPU=OFF" @@ -102,7 +102,7 @@ To direct `cmake` where to look at for some dependencies, for example `LLVM`, yo # using an env var export LLVM_DIR="/path/to/llvm/" # or with a cmake option -export QUADRANTS_CMAKE_ARGS="$QUADRANTS_CMAKE_ARGS -DLLVM_ROOT=/path/to/llvm" +export CMAKE_ARGS="$CMAKE_ARGS -DLLVM_ROOT=/path/to/llvm" ``` ### Building with the AMD GPU backend (Linux) @@ -114,22 +114,6 @@ The AMD GPU backend is Linux-only (it is force-disabled on macOS and Windows) an CMAKE_ARGS="-DQD_WITH_AMDGPU=ON -DQD_WITH_CUDA=OFF" pip install --no-build-isolation -e . -v ``` -## Advanced usage - -### CI Convention about compilers/LLVM - -Quadrants comprises at least three important parts: - -1. `quadrants` host runtime: Made with a mix of Python and C++. The C++ core is compiled using the OS default C/C++ compiler. -2. `quadrants` device runtime (bitcode): C++ code compiled using `clang++` from the distribution/OS. Using `clang++` is required as it has to support the same targets as `LLVM`. -3. `LLVM` libraries used by host runtime: statically or dynamically linked, used to lower the kernel's final IR to machine code on the host. The CI uses an LLVM version compiled from source. - -### Building LLVM for debugging it - -Sometimes, it could be useful to have a `LLVM` version that allows to print intermediate passes or with debug symbols to find out where and why LLVM fails (for example, when Instruction Selection fails). To do so you would have to build LLVM by yourself. If so, you should take some inspiration from our [CI pipeline to build LLVM](https://github.com/Genesis-Embodied-AI/quadrants-sdk-builds/blob/main/.github/workflows/llvm-ci.yml) to tweak a little bit to your liking (and not enable/disable options that would create discrepancies). - -You can then use `LLVM_DIR` to point to the `LLVM` build directory. - ## CI checks Pull requests are validated by several CI jobs. Most run automatically; a failing check blocks merge. @@ -227,3 +211,19 @@ quadrants/program/legacy_stream.cpp 42 -42 The `0` in the LoC column for the two new files reflects that both files did not exist before this PR (their pre-PR code-line count is 0). The `42 -42` row for `legacy_stream.cpp` is a fully-deleted file: 42 code lines existed before this PR and all 42 were removed. This check is delayed by 30 minutes, to avoid running repeatedly if multiple commits pushed with a short delay between each. + +## Advanced + +### CI Convention about compilers/LLVM + +Quadrants comprises at least three important parts: + +1. `quadrants` host runtime: Made with a mix of Python and C++. The C++ core is compiled using the OS default C/C++ compiler. +2. `quadrants` device runtime (bitcode): C++ code compiled using `clang++` from the distribution/OS. Using `clang++` is required as it has to support the same targets as `LLVM`. +3. `LLVM` libraries used by host runtime: statically or dynamically linked, used to lower the kernel's final IR to machine code on the host. The CI uses an LLVM version compiled from source. + +### Building LLVM for debugging it + +Sometimes, it could be useful to have a `LLVM` version that allows to print intermediate passes or with debug symbols to find out where and why LLVM fails (for example, when Instruction Selection fails). To do so you would have to build LLVM by yourself. If so, you should take some inspiration from our [CI pipeline to build LLVM](https://github.com/Genesis-Embodied-AI/quadrants-sdk-builds/blob/main/.github/workflows/llvm-ci.yml) to tweak a little bit to your liking (and not enable/disable options that would create discrepancies). + +You can then use `LLVM_DIR` to point to the `LLVM` build directory.