Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,10 @@ jobs:

- name: Test
shell: micromamba-shell {0}
run: >
ctest --test-dir build
--output-on-failure
-E "(s3)"
run: |
uv sync --script tests/validate_zarr.py
uv sync --script tests/validate_ngff_readback.py
ctest --test-dir build --output-on-failure -E "(s3)"

windows-tests:
runs-on: windows-latest
Expand Down Expand Up @@ -143,6 +143,8 @@ jobs:
run: |
eval "$(micromamba shell hook --shell bash)"
micromamba activate ci
uv sync --script tests/validate_zarr.py
uv sync --script tests/validate_ngff_readback.py
ctest --test-dir build --output-on-failure -E "(s3)"

coverage:
Expand Down
2 changes: 1 addition & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ install(
# integration path. cudart_static / nvcomp are baked into the chucky archive
# but a bare -l flag cannot locate them without -L into the toolkit.
set(CHUCKY_VARIANT_DESCRIPTION "CPU only")
set(CHUCKY_PC_LIBS_PRIVATE "-lm -llz4 -lzstd -laws-c-s3 -ldl -lrt -lpthread")
set(CHUCKY_PC_LIBS_PRIVATE "-lm -lblosc -llz4 -lzstd -laws-c-s3 -ldl -lrt -lpthread")
if(CHUCKY_ENABLE_GPU)
set(CHUCKY_VARIANT_DESCRIPTION "GPU + CPU")
string(APPEND CHUCKY_PC_LIBS_PRIVATE " -lcudart_static -lnvcomp")
Expand Down
4 changes: 4 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,10 @@ ENV CMAKE_PREFIX_PATH="/opt/nvcomp:/opt/aws"
WORKDIR /src
COPY . .

# Install independent readback readers before CTest runs.
RUN uv sync --script tests/validate_zarr.py \
&& uv sync --script tests/validate_ngff_readback.py

RUN cmake --preset docker -G Ninja \
-DCMAKE_BUILD_TYPE=Release \
&& cmake --build build
4 changes: 4 additions & 0 deletions Dockerfile.coverage
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,9 @@ ENV CMAKE_PREFIX_PATH="/opt/aws"
WORKDIR /src
COPY . .

# Install independent readback readers before CTest runs.
RUN uv sync --script tests/validate_zarr.py \
&& uv sync --script tests/validate_ngff_readback.py

RUN cmake --preset coverage \
&& cmake --build build
4 changes: 4 additions & 0 deletions Dockerfile.cpu
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ ENV CMAKE_PREFIX_PATH="/opt/aws"
WORKDIR /src
COPY . .

# Install independent readback readers before CTest runs.
RUN uv sync --script tests/validate_zarr.py \
&& uv sync --script tests/validate_ngff_readback.py

RUN cmake --preset cpu-only \
-DCMAKE_BUILD_TYPE=Release \
&& cmake --build build
5 changes: 2 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,8 @@ up to 32 LOD levels. Internal layout supports up to 64 dimensions.

- **aws-c-s3** — Amazon S3 client library for S3 storage backend
- **lz4**, **zstd** — compression libraries
- **c-blosc** (optional) — enables CPU Blosc-LZ4/Blosc-Zstd and the GPU
Blosc interoperability tests. GPU Blosc encoding uses nvCOMP and does not
require c-blosc.
- **c-blosc** — CPU Blosc-LZ4/Blosc-Zstd compression and GPU Blosc
interoperability tests. GPU Blosc encoding uses nvCOMP.
- **CMake** (3.18+) + **Ninja** — build system

The default build targets SM 100 (Blackwell). For other GPUs, set
Expand Down
22 changes: 10 additions & 12 deletions bench/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ foreach(src IN LISTS BENCH_STREAM_SOURCES)
endforeach()

find_package(Python3 COMPONENTS Interpreter QUIET)
if(Python3_Interpreter_FOUND AND HAVE_BLOSC)
if(Python3_Interpreter_FOUND)
add_test(
NAME test-bench-cli
COMMAND
Expand Down Expand Up @@ -128,17 +128,15 @@ if(Python3_Interpreter_FOUND)
list(APPEND MEASUREMENT_BACKENDS gpu)
endif()
foreach(backend IN LISTS MEASUREMENT_BACKENDS)
if(HAVE_BLOSC)
add_test(
NAME test-bench_image_readback_${backend}
COMMAND
${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/test_bench_image_readback.py
$<TARGET_FILE:bench_stream_microscopy> ${backend}
)
set_tests_properties(test-bench_image_readback_${backend}
PROPERTIES TIMEOUT 600)
endif()
add_test(
NAME test-bench_image_readback_${backend}
COMMAND
${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/test_bench_image_readback.py
$<TARGET_FILE:bench_stream_microscopy> ${backend}
)
set_tests_properties(test-bench_image_readback_${backend}
PROPERTIES TIMEOUT 600)
add_test(
NAME test-bench_images_${backend}
COMMAND
Expand Down
12 changes: 2 additions & 10 deletions cmake/Blosc.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@
#
# Tries CONFIG mode first (covers vcpkg, Conan, system CMake configs),
# then falls back to the bundled FindBlosc.cmake module.
# If neither finds blosc, HAVE_BLOSC is OFF and CPU Blosc is unavailable.
# GPU Blosc uses nvCOMP and remains available in GPU builds.
# Required for CPU compression and GPU Blosc interoperability tests.

find_package(blosc CONFIG QUIET)
if(blosc_FOUND)
Expand All @@ -30,12 +29,5 @@ if(blosc_FOUND)
endif()

if(NOT TARGET Blosc::Blosc)
find_package(Blosc QUIET)
endif()

if(TARGET Blosc::Blosc)
set(HAVE_BLOSC ON)
else()
set(HAVE_BLOSC OFF)
message(STATUS "Blosc not found — CPU blosc codecs disabled")
find_package(Blosc REQUIRED MODULE)
endif()
4 changes: 1 addition & 3 deletions cmake/chuckyConfig.cmake.in
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@ endmacro()

_chucky_find_dep_config_or_module(lz4 Lz4)
_chucky_find_dep_config_or_module(zstd Zstd)
if(@HAVE_BLOSC@)
_chucky_find_dep_config_or_module(blosc Blosc)
endif()
_chucky_find_dep_config_or_module(blosc Blosc)

find_dependency(aws-c-s3 CONFIG)

Expand Down
4 changes: 2 additions & 2 deletions docs/guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ internal conventions, and key concepts.
| Target | Source | Purpose |
|--------|--------|---------|
| `transpose_cpu` | `src/cpu/transpose.cpp` | OpenMP scatter |
| `compress_cpu` | `src/cpu/compress.c`, `src/cpu/compress_blosc.c` | zstd / lz4 and optional c-blosc compression (CPU) |
| `compress_cpu` | `src/cpu/compress.c`, `src/cpu/compress_blosc.c` | zstd / lz4 / c-blosc compression (CPU) |
| `aggregate_cpu` | `src/cpu/aggregate.c` | Shard packing (CPU) |
| `lod_cpu` | `src/cpu/lod.cpp` | CPU LOD scatter + reduce |
| `stream_cpu` | `src/cpu/stream.c` | CPU pipeline orchestrator |
Expand Down Expand Up @@ -128,7 +128,7 @@ Both backends implement the same pipeline stages behind the same
| Stage | GPU | CPU |
|-------|-----|-----|
| Scatter | CUDA kernel | OpenMP parallel loop |
| Compress | nvCOMP LZ4/Zstd, raw or Blosc-framed | libzstd / liblz4 / optional c-blosc |
| Compress | nvCOMP LZ4/Zstd, raw or Blosc-framed | libzstd / liblz4 / c-blosc |
| Aggregate | CUDA kernel | Sequential packing |
| LOD | CUDA kernels (templated on dtype) | C++ templates + OpenMP |
| Orchestration | 4 CUDA streams, double-buffered | Single-threaded pipeline |
Expand Down
3 changes: 1 addition & 2 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -109,16 +109,15 @@ list(

# Conditional deps via generator expressions — empty branches are dropped at
# generation time without referencing missing targets.
set(_with_blosc $<BOOL:${HAVE_BLOSC}>)
set(_with_gpu $<BOOL:${CHUCKY_ENABLE_GPU}>)
set(_chucky_public_deps
${_math_lib}
Threads::Threads
Lz4::Lz4
Zstd::Zstd
Blosc::Blosc
AWS::aws-c-s3
$<$<PLATFORM_ID:Windows>:psapi>
$<${_with_blosc}:Blosc::Blosc>
$<${_with_gpu}:CUDA::cuda_driver>
$<${_with_gpu}:CUDA::cudart_static>
$<${_with_gpu}:nvcomp::nvcomp>
Expand Down
11 changes: 2 additions & 9 deletions src/cpu/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,9 @@ add_src_lib(transpose_cpu SOURCES transpose.cpp transpose.h
LINKS index_ops threadpool
)

add_src_lib(compress_cpu SOURCES compress.c compress.h compress_blosc.h
LINKS codec_types threadpool Lz4::Lz4 Zstd::Zstd chucky_log
add_src_lib(compress_cpu SOURCES compress.c compress.h compress_blosc.c compress_blosc.h
LINKS codec_types threadpool Lz4::Lz4 Zstd::Zstd Blosc::Blosc chucky_log
)
if(HAVE_BLOSC)
target_sources(compress_cpu PRIVATE compress_blosc.c)
target_link_libraries(compress_cpu PUBLIC Blosc::Blosc)
target_compile_definitions(compress_cpu PUBLIC HAVE_BLOSC)
else()
target_sources(compress_cpu PRIVATE compress_blosc_stub.c)
endif()

add_src_lib(aggregate_cpu SOURCES aggregate.c aggregate.h
LINKS index_ops threadpool chucky_log
Expand Down
44 changes: 0 additions & 44 deletions src/cpu/compress_blosc_stub.c

This file was deleted.

29 changes: 22 additions & 7 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,7 @@ add_test_exe(test_ngff_flush_cascade test_ngff_flush_cascade.c stream_cpu writer
add_test_exe(test_multiarray_cpu test_multiarray_cpu.c multiarray_cpu stream_cpu writer test_shard_sink chucky_log)

# GPU multi-array stream test
add_gpu_test_exe(test_multiarray_gpu test_multiarray_gpu.c LINKS CUDA::cuda_driver CUDA::cudart_static multiarray_gpu stream writer test_shard_sink)
if(CHUCKY_ENABLE_GPU AND HAVE_BLOSC)
target_compile_definitions(test_multiarray_gpu PRIVATE HAVE_BLOSC)
target_link_libraries(test_multiarray_gpu PRIVATE Blosc::Blosc)
endif()
add_gpu_test_exe(test_multiarray_gpu test_multiarray_gpu.c LINKS CUDA::cuda_driver CUDA::cudart_static multiarray_gpu stream writer test_shard_sink Blosc::Blosc)
# A cross-array ordering cycle would otherwise strand the GPU test runner.
if(CHUCKY_ENABLE_GPU)
set_tests_properties(test-test_multiarray_gpu PROPERTIES TIMEOUT 60)
Expand All @@ -178,14 +174,24 @@ add_test_exe(test_multi_append test_multi_append.c stream_cpu writer test_shard_
# CPU stream + zarr store integration (exercises async IO fencing)
add_test_exe(test_cpu_zarr_fs test_cpu_zarr_fs.c stream_cpu test_zarr_helpers test_counting_sink writer Zstd::Zstd test_platform test_shard_verify)

# Zarr readback validation via zarr-python (requires uv)
# Zarr readback validation via zarr-python and TensorStore (requires uv)
add_test_exe(test_zarr_readback test_zarr_readback.c stream_cpu test_zarr_helpers writer test_platform chucky_log)
target_compile_definitions(
test_zarr_readback
PRIVATE SOURCE_DIR="${CMAKE_SOURCE_DIR}"
)
set_tests_properties(test-test_zarr_readback PROPERTIES SKIP_RETURN_CODE 77)

# Full NGFF pyramid readback, sharing the fixture and codec matrix across backends.
add_test_exe(test_ngff_readback_cpu test_ngff_readback.c
stream_cpu test_zarr_helpers writer test_platform chucky_log
)
target_compile_definitions(
test_ngff_readback_cpu
PRIVATE SOURCE_DIR="${CMAKE_SOURCE_DIR}"
)
set_tests_properties(test-test_ngff_readback_cpu PROPERTIES TIMEOUT 180)

# Cross-validate GPU vs CPU (byte-exact shard comparison with CODEC_NONE)
add_gpu_test_exe(test_cross_validate test_cross_validate.c LINKS CUDA::cuda_driver CUDA::cudart_static stream stream_cpu writer test_data test_shard_verify)

Expand All @@ -201,7 +207,7 @@ target_link_libraries(test_data PUBLIC stream_config writer chucky_log)
# Tests with zstd
add_gpu_test_exe(test_stream test_stream.c LINKS CUDA::cuda_driver CUDA::cudart_static stream Zstd::Zstd index_ops_util test_shard_sink test_gpu_helpers test_shard_verify test_voxel_encode)
add_gpu_test_exe(test_compress test_compress.c LINKS CUDA::cuda_driver CUDA::cudart_static stream Zstd::Zstd Lz4::Lz4)
if(CHUCKY_ENABLE_GPU AND HAVE_BLOSC)
if(CHUCKY_ENABLE_GPU)
add_gpu_test_exe(test_compress_blosc_gpu test_compress_blosc_gpu.c
LINKS CUDA::cuda_driver CUDA::cudart_static stream Blosc::Blosc
)
Expand All @@ -217,6 +223,15 @@ if(CHUCKY_ENABLE_GPU AND HAVE_BLOSC)
test-test_zarr_readback_gpu
PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120
)
add_gpu_test_exe(test_ngff_readback_gpu test_ngff_readback.c
LINKS CUDA::cuda_driver CUDA::cudart_static stream test_zarr_helpers
writer test_platform chucky_log
)
target_compile_definitions(
test_ngff_readback_gpu
PRIVATE TEST_NGFF_READBACK_GPU SOURCE_DIR="${CMAKE_SOURCE_DIR}"
)
set_tests_properties(test-test_ngff_readback_gpu PROPERTIES TIMEOUT 180)
endif()

add_test_exe(test_json_writer test_json_writer.c zarr_metadata ngff_metadata attr_set json_validate strbuf chucky_log)
Expand Down
6 changes: 0 additions & 6 deletions tests/test_compress_cpu.c
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@
#include "threadpool/threadpool.h"
#include "util/prelude.h"

#ifdef HAVE_BLOSC
#include <blosc.h>
#endif
#include <lz4.h>
#include <stdlib.h>
#include <string.h>
Expand Down Expand Up @@ -259,7 +257,6 @@ test_nthreads_1(void)
return 1;
}

#ifdef HAVE_BLOSC
static int
test_codec_blosc(enum compression_codec id, const char* name)
{
Expand Down Expand Up @@ -421,7 +418,6 @@ test_blosc_explicit_blocks(void)
free(decoded);
return 1;
}
#endif // HAVE_BLOSC

int
main(int ac, char* av[])
Expand All @@ -441,11 +437,9 @@ main(int ac, char* av[])
rc |= test_codec_lz4();
rc |= test_codec_zstd();
rc |= test_nthreads_1();
#ifdef HAVE_BLOSC
rc |= test_codec_blosc(CODEC_BLOSC_LZ4, "blosc_lz4");
rc |= test_codec_blosc(CODEC_BLOSC_ZSTD, "blosc_zstd");
rc |= test_blosc_explicit_blocks();
#endif

threadpool_free(g_pool);
threadpool_free(g_pool1);
Expand Down
Loading
Loading