Add CPU decoding and pipeline components

.github/workflows/cpu.yml

@@ -0,0 +1,40 @@
+name: cpu
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  cpu:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y cmake ninja-build pkg-config liburing-dev libzstd-dev libblosc-dev
+          python -m pip install uv pytest pytest-cov numpy
+      - name: Build without CUDA
+        run: |
+          cmake --preset cpu -DDAMACY_PYTHON=ON -DPython_EXECUTABLE="$(command -v python)" -DCMAKE_DISABLE_FIND_PACKAGE_CUDAToolkit=ON
+          cmake --build build -j2
+      - name: Test
+        run: ctest --test-dir build --output-on-failure --timeout 180 -j2
+      - name: Check native dependencies
+        env:
+          PYTHONPATH: build/python
+        run: |
+          python - <<'PY'
+          import subprocess
+          from damacy import _native
+          assert not _native.CUDA_ENABLED
+          dependencies = subprocess.check_output(['ldd', _native.__file__], text=True).lower()
+          print(dependencies)
+          assert all(name not in dependencies for name in ('libcuda', 'libcudart', 'libnvcomp'))
+          PY

.github/workflows/tsan.yml

@@ -3,10 +3,6 @@
 # unsynchronized access to shared state surfaces as a TSan report rather than
 # a flaky hang.
 #
-# The test itself is pure C, but the test binary links transitively
-# against libcuda via the shared test fixture / zarr / store libs, so
-# it needs the NVIDIA driver visible at load time. Uses
-# --device=nvidia.com/gpu=all.
 
 name: tsan
 
@@ -50,4 +46,5 @@ jobs:
             --security-opt seccomp=unconfined \
             -e TSAN_OPTIONS="suppressions=/workspace/damacy/tests/tsan-suppressions.txt halt_on_error=1 second_deadlock_stack=1" \
             "$IMAGE" \
+            setarch --addr-no-randomize \
             ctest --test-dir build -L tsan --output-on-failure

CMakeLists.txt

@@ -15,11 +15,16 @@ include(Warnings)
 include(Helpers)
 include(Fuzz) # declares DAMACY_FUZZ and applies fuzz-mode flags
 
+option(DAMACY_CUDA "Build the CUDA executor" ON)
+if(DAMACY_FUZZ)
+    set(DAMACY_CUDA OFF)
+endif()
+
 find_package(Threads REQUIRED)
 find_package(PkgConfig REQUIRED)
 pkg_check_modules(LIBURING REQUIRED IMPORTED_TARGET liburing)
 
-if(NOT DAMACY_FUZZ)
+if(DAMACY_CUDA)
     if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
         set(CMAKE_CUDA_ARCHITECTURES
             "75-real;80-real;86-real;89-real;90-real;90-virtual"
@@ -48,6 +53,12 @@ option(
 # inline no-ops via static inlines in src/nvtx/nvtx.h. Defaults ON for
 # non-fuzz builds; src/CMakeLists.txt forces OFF under DAMACY_FUZZ.
 option(DAMACY_NVTX_ENABLED "Enable NVTX timeline instrumentation" ON)
+if(NOT DAMACY_CUDA)
+    set(DAMACY_NVTX_ENABLED OFF)
+    if(DAMACY_ENABLE_GDS)
+        message(FATAL_ERROR "DAMACY_ENABLE_GDS requires DAMACY_CUDA")
+    endif()
+endif()
 
 # DAMACY_ENABLE_GDS — gates the cuFile/GDS reader backend
 # (src/store/store_fs_gds.c). Defaults OFF so developers/CI without
@@ -57,7 +68,7 @@ option(DAMACY_ENABLE_GDS "Build cuFile/GDS reader backend" OFF)
 # cufile.h is needed at compile time for typedefs; libcufile.so.0 is
 # dlopen'd at runtime (see src/store/store_fs_gds.c), not linked. Gated
 # on DAMACY_ENABLE_GDS so non-GDS builds don't require cufile.h.
-if(NOT DAMACY_FUZZ AND DAMACY_ENABLE_GDS)
+if(DAMACY_CUDA AND DAMACY_ENABLE_GDS)
     find_package(CuFile REQUIRED)
 endif()
 
@@ -87,6 +98,11 @@ if(DAMACY_TSAN AND NOT DAMACY_FUZZ)
     target_link_options(warnings INTERFACE -fsanitize=thread)
 endif()
 
+if(NOT DAMACY_FUZZ)
+    pkg_check_modules(ZSTD REQUIRED IMPORTED_TARGET libzstd)
+    pkg_check_modules(BLOSC REQUIRED IMPORTED_TARGET blosc)
+endif()
+
 add_subdirectory(src)
 add_subdirectory(bench)
 
@@ -95,7 +111,7 @@ add_subdirectory(bench)
 # `cmake --build build --target asmbench` etc. The directory may be
 # pruned from build contexts that don't ship experiments (e.g. Docker).
 if(
-    NOT DAMACY_FUZZ
+    DAMACY_CUDA
     AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/experiments/asm/CMakeLists.txt"
 )
     add_subdirectory(experiments/asm)

CMakePresets.json

@@ -7,7 +7,15 @@
       "binaryDir": "${sourceDir}/build",
       "cacheVariables": {
         "CMAKE_EXPORT_COMPILE_COMMANDS": "ON",
-        "CMAKE_BUILD_TYPE": "RelWithDebInfo"
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo",
+        "DAMACY_CUDA": "ON"
+      }
+    },
+    {
+      "name": "cpu",
+      "inherits": "default",
+      "cacheVariables": {
+        "DAMACY_CUDA": "OFF"
       }
     },
     {

Dockerfile

@@ -45,12 +45,15 @@ RUN apt-get update \
         ca-certificates \
         curl \
         xz-utils \
+        util-linux \
         cmake \
         ninja-build \
         pkg-config \
         python3 \
         python3-dev \
         liburing-dev \
+        libzstd-dev \
+        libblosc-dev \
         libnuma1 \
         libmount1 \
         libudev1 \
@@ -81,7 +84,7 @@ ENV PATH=/opt/venv/bin:${PATH} \
 # pytest drives python/tests/* via the python_pytest ctest target;
 # installed alongside scikit-build-core so the cmake configure can
 # detect it and register the test.
-RUN uv pip install scikit-build-core pytest pytest-cov
+RUN uv pip install scikit-build-core pytest pytest-cov numpy
 
 # ----- build configuration ---------------------------------------------------
 # Override at `docker build` time via --build-arg to produce a coverage
@@ -137,7 +140,7 @@ RUN ctest --test-dir build --output-on-failure -E "test_damacy|test_assemble|pyt
 # so the editable install resolves `damacy._native` without rebuilding.
 # Skipped under TSan (the .so isn't built; the install would 404).
 RUN if [ "${DAMACY_TSAN}" != "ON" ]; then \
-        cp build/python/_native*.so python/damacy/ && \
+        cp build/python/damacy/_native*.so python/damacy/ && \
         uv pip install --no-deps --no-build-isolation -e .; \
     fi
 

README.md

@@ -7,15 +7,67 @@
 [![bench](https://github.com/nclack/damacy/actions/workflows/bench.yml/badge.svg)](https://nclack.github.io/damacy/throughput/)
 [![docs](https://github.com/nclack/damacy/actions/workflows/docs.yml/badge.svg)](https://nclack.github.io/damacy/)
 
-High-speed streamed assembly of nD tensors from zarr sources to GPU.
+Streamed assembly of n-dimensional tensors from Zarr sources into RAM or GPU memory.
 
-Damacy reads sharded [NGFF](https://ngff.openmicroscopy.org/) [zarr
-v3](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html) stores
-directly on the GPU: per-shard chunk indexing, parallel host I/O, in-flight
-GPU-side decompression (zstd, blosc1-zstd), and assembly of each batch
-as a DLPack-ready device tensor.
+Damacy loads array metadata and shard indexes, plans the chunks needed for a
+batch, then reads, decodes, and assembles them with a CPU or CUDA executor.
+Both paths support raw bytes, zstd, and Blosc-zstd and return contiguous,
+DLPack-compatible tensors. CPU builds have no CUDA or nvCOMP dependency.
 
-## Quick start
+Source URIs identify concrete Zarr v3 arrays, including arrays inside
+[NGFF](https://ngff.openmicroscopy.org/) multiscale images. Queries currently
+select rectangular regions in array coordinates. Index queries, transformed
+crops, and automatic NGFF level selection are planned extensions.
+
+## CPU quick start
+
+```python
+import damacy
+import numpy as np
+
+metadata_reader = damacy.FileMetadataReader(concurrency=64)
+chunk_reader = damacy.FileReader(workers=8, max_inflight_reads=4096)
+metadata = damacy.ZarrMetadata(
+    reader=metadata_reader,
+    cache=damacy.MetadataCache(array_entries=256, shard_index_entries=8192),
+)
+planner = damacy.ChunkPlanner(metadata=metadata, limits=damacy.PlanLimits())
+executor = damacy.CpuExecutor(
+    reader=chunk_reader,
+    limits=damacy.CpuLimits(max_memory_bytes=1 << 30, decode_workers=8),
+)
+
+with damacy.Pipeline(
+    planner=planner,
+    executor=executor,
+    output=damacy.BatchSpec(samples=2, shape=(64, 256, 256), dtype="f32"),
+    queues=damacy.QueueLimits(lookahead_samples=4, prepared_batches=2),
+) as pipeline:
+    pipeline.push([
+        damacy.Sample(uri=uri, aabb=[(0, 64), (0, 256), (0, 256)])
+        for uri in ["/data/image-1.zarr/0", "/data/image-2.zarr/0"]
+    ])
+    with pipeline.pop() as batch:
+        array = np.from_dlpack(batch)
+        print(array.shape)
+        del array
+```
+
+The metadata provider loads descriptions on demand from the pushed URIs;
+the chunk reader loads the encoded byte ranges that planning produces.
+`np.from_dlpack` shares the output buffer. A live array keeps that buffer
+occupied even after its `Batch` is released; use `.copy()` to retain data
+while allowing the pool to reuse its storage.
+
+See [Pipeline composition](docs/pipeline.md) for the complete component,
+limit, ownership, and build contracts. Build a CPU Python package from source
+with the Linux dependencies `liburing`, `libzstd`, and `libblosc` installed:
+
+```sh
+pip install . --config-settings=cmake.define.DAMACY_CUDA=OFF
+```
+
+## CUDA quick start
 
 ```python
 import random
@@ -25,13 +77,9 @@ import torch
 cfg = damacy.Config(
     samples_per_batch=8,
     sample_shape=(64, 256, 256),
-    # Resource caps are fixed at construction; nothing grows after.
     max_gpu_memory_bytes=1 << 30,  # primary GPU budget
     dtype="bf16",                  # source dtype is cast on assemble
-    # One pipeline binds to one GPU. Omit `device=` to capture the
-    # current CUDA context (handy single-GPU; PyTorch sets one up
-    # implicitly). For multi-GPU pass `device=local_rank` — see
-    # https://nclack.github.io/damacy/distributed/
+    device=0,
 )
 
 # A Sample names an absolute uri and a per-axis half-open AABB into
@@ -99,7 +147,7 @@ Damacy reads zarr v3 (sharded and non-sharded). What's recognized today:
 | Array versions | v3 | v2 stores are not read |
 | Layout (sharded) | `sharding_indexed` | with `index_location` either `"start"` or `"end"` (default) |
 | Layout (non-sharded) | yes | each chunk is a separate file at `c/<i>/<j>/...` |
-| Inner / chunk codec | `bytes` (passthrough), `zstd`, `blosc` (cname=`zstd`) | `blosc` with `cname=lz4`/`lz4hc` is recognized at parse time and rejected at planning |
+| Inner / chunk codec | `bytes` (passthrough), `zstd`, `blosc` (cname=`zstd`) | `blosc` with `cname=lz4`/`lz4hc` is recognized at parse time and rejected by the executor |
 | Sharding index codec | `bytes` + `crc32c` | the shard index itself; the data codec is separate |
 | Missing chunks | yes — read as `fill_value` | per zarr v3 spec; sharded "empty" entries (`offset==nbytes==2^64−1`) and missing chunk files both route here |
 
@@ -113,22 +161,22 @@ If you have data that uses one of the unsupported codecs and you'd like it added
 
 ## Runtime dependencies
 
-Damacy links the core CUDA and Linux async-I/O pieces directly. Optional
-features dlopen their backends lazily, so a damacy binary loads on any host
-with the required core libraries; optional features simply turn off when their
-library is not present.
+All builds require Linux async metadata I/O and CPU codec libraries. CUDA
+builds additionally link the NVIDIA driver and nvCOMP. Build with
+`DAMACY_CUDA=OFF` to import and run on a host without a CUDA driver.
 
-| Library | Required at runtime | What you lose if missing | How damacy finds it |
-|---|---|---|---|
-| `libcuda.so.1` | always | nothing — damacy cannot run without it | NVIDIA driver install (`/run/opengl-driver/lib`, `/usr/lib/x86_64-linux-gnu`, …) |
-| `liburing.so` | always on Linux builds | nothing — async metadata stat/open/read/close uses io_uring | normal dynamic loader (`LD_LIBRARY_PATH`, `ld.so.cache`, rpath, …) |
-| `libnuma.so.1` | optional | NUMA pinning of pinned-host slabs, bulk I/O workers, scheduler thread, and metadata io_uring driver thread (single-socket hosts: no effect) | `dlopen` via dynamic loader (`LD_LIBRARY_PATH`, `ld.so.cache`) |
-| `libcufile.so.0` | optional | `damacy_config.enable_gds = DAMACY_GDS_ON` — direct read of compressed chunks into device memory via NVIDIA GPUDirect Storage | `dlopen` via dynamic loader; ships with the CUDA toolkit and with nvidia-fs. Requires a build with `-DDAMACY_ENABLE_GDS=ON` (default OFF) |
-| `libmount.so.1`, `libudev.so.1` | required *if and only if* using GDS | cuFile dlopen's these at driver init even in compat mode | dynamic loader |
+| Library | Used by | How it is loaded |
+|---|---|---|
+| `liburing` | CPU and CUDA: async metadata I/O | normal dynamic loader |
+| `libzstd`, `libblosc` | CPU decoding, included in both builds | normal dynamic loader |
+| `libcuda.so.1`, nvCOMP | CUDA builds | driver loader; nvCOMP may be linked statically |
+| `libnuma.so.1` | Optional CUDA placement and host affinity | `dlopen`; absence disables placement |
+| `libcufile.so.0` | Optional CUDA GPUDirect Storage | `dlopen`; requires `DAMACY_ENABLE_GDS=ON` |
+| `libmount.so.1`, `libudev.so.1` | cuFile initialization when GDS is used | dynamic loader |
 
 Metadata reads require a Linux kernel with the io_uring operations damacy uses:
 `STATX`, `OPENAT2`, `READ`, and `CLOSE`. If the kernel does not advertise
-those operations, `damacy_create` fails instead of falling back to a thread
+those operations, pipeline construction fails instead of falling back to a thread
 pool.
 
 GDS notes:

bench/CMakeLists.txt

@@ -10,7 +10,9 @@ target_link_libraries(
     PRIVATE damacy json json_writer slice strbuf warnings
 )
 # nvcomp_static drags in libstdc++/libgcc_s; fold them into the binary too.
-target_link_options(damacy_bench PRIVATE -static-libstdc++ -static-libgcc)
+if(DAMACY_CUDA)
+    target_link_options(damacy_bench PRIVATE -static-libstdc++ -static-libgcc)
+endif()
 
 add_executable(metadata_store_async_bench metadata_store_async_bench.c)
 target_link_libraries(
@@ -19,8 +21,10 @@ target_link_libraries(
 )
 
 # Mount-ceiling probe; replays DAMACY_TRACE_READS captures (see file header).
-add_executable(preadreplay preadreplay.c)
-target_link_libraries(
-    preadreplay
-    PRIVATE warnings Threads::Threads CUDA::cuda_driver
-)
+if(DAMACY_CUDA)
+    add_executable(preadreplay preadreplay.c)
+    target_link_libraries(
+        preadreplay
+        PRIVATE warnings Threads::Threads CUDA::cuda_driver
+    )
+endif()