diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0421d928..8c57b71a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -97,10 +97,10 @@ jobs: - name: Test shell: micromamba-shell {0} - run: > - ctest --test-dir build - --output-on-failure - -E "(s3)" + run: | + uv sync --script tests/validate_zarr.py + uv sync --script tests/validate_ngff_readback.py + ctest --test-dir build --output-on-failure -E "(s3)" windows-tests: runs-on: windows-latest @@ -143,6 +143,8 @@ jobs: run: | eval "$(micromamba shell hook --shell bash)" micromamba activate ci + uv sync --script tests/validate_zarr.py + uv sync --script tests/validate_ngff_readback.py ctest --test-dir build --output-on-failure -E "(s3)" coverage: diff --git a/CMakeLists.txt b/CMakeLists.txt index 080b5080..3f1673bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ install( # integration path. cudart_static / nvcomp are baked into the chucky archive # but a bare -l flag cannot locate them without -L into the toolkit. set(CHUCKY_VARIANT_DESCRIPTION "CPU only") -set(CHUCKY_PC_LIBS_PRIVATE "-lm -llz4 -lzstd -laws-c-s3 -ldl -lrt -lpthread") +set(CHUCKY_PC_LIBS_PRIVATE "-lm -lblosc -llz4 -lzstd -laws-c-s3 -ldl -lrt -lpthread") if(CHUCKY_ENABLE_GPU) set(CHUCKY_VARIANT_DESCRIPTION "GPU + CPU") string(APPEND CHUCKY_PC_LIBS_PRIVATE " -lcudart_static -lnvcomp") diff --git a/Dockerfile b/Dockerfile index 37927514..8b5cffd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,6 +64,10 @@ ENV CMAKE_PREFIX_PATH="/opt/nvcomp:/opt/aws" WORKDIR /src COPY . . +# Install independent readback readers before CTest runs. +RUN uv sync --script tests/validate_zarr.py \ + && uv sync --script tests/validate_ngff_readback.py + RUN cmake --preset docker -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ && cmake --build build diff --git a/Dockerfile.coverage b/Dockerfile.coverage index 97a91baf..0e7b48c0 100644 --- a/Dockerfile.coverage +++ b/Dockerfile.coverage @@ -50,5 +50,9 @@ ENV CMAKE_PREFIX_PATH="/opt/aws" WORKDIR /src COPY . . +# Install independent readback readers before CTest runs. +RUN uv sync --script tests/validate_zarr.py \ + && uv sync --script tests/validate_ngff_readback.py + RUN cmake --preset coverage \ && cmake --build build diff --git a/Dockerfile.cpu b/Dockerfile.cpu index 0dce31ee..79f4414a 100644 --- a/Dockerfile.cpu +++ b/Dockerfile.cpu @@ -45,6 +45,10 @@ ENV CMAKE_PREFIX_PATH="/opt/aws" WORKDIR /src COPY . . +# Install independent readback readers before CTest runs. +RUN uv sync --script tests/validate_zarr.py \ + && uv sync --script tests/validate_ngff_readback.py + RUN cmake --preset cpu-only \ -DCMAKE_BUILD_TYPE=Release \ && cmake --build build diff --git a/README.md b/README.md index 6ecf715e..c519c15b 100644 --- a/README.md +++ b/README.md @@ -52,9 +52,8 @@ up to 32 LOD levels. Internal layout supports up to 64 dimensions. - **aws-c-s3** — Amazon S3 client library for S3 storage backend - **lz4**, **zstd** — compression libraries -- **c-blosc** (optional) — enables CPU Blosc-LZ4/Blosc-Zstd and the GPU - Blosc interoperability tests. GPU Blosc encoding uses nvCOMP and does not - require c-blosc. +- **c-blosc** — CPU Blosc-LZ4/Blosc-Zstd compression and GPU Blosc + interoperability tests. GPU Blosc encoding uses nvCOMP. - **CMake** (3.18+) + **Ninja** — build system The default build targets SM 100 (Blackwell). For other GPUs, set diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index df5668a1..4ab53397 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -95,7 +95,7 @@ foreach(src IN LISTS BENCH_STREAM_SOURCES) endforeach() find_package(Python3 COMPONENTS Interpreter QUIET) -if(Python3_Interpreter_FOUND AND HAVE_BLOSC) +if(Python3_Interpreter_FOUND) add_test( NAME test-bench-cli COMMAND @@ -128,17 +128,15 @@ if(Python3_Interpreter_FOUND) list(APPEND MEASUREMENT_BACKENDS gpu) endif() foreach(backend IN LISTS MEASUREMENT_BACKENDS) - if(HAVE_BLOSC) - add_test( - NAME test-bench_image_readback_${backend} - COMMAND - ${Python3_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/test_bench_image_readback.py - $ ${backend} - ) - set_tests_properties(test-bench_image_readback_${backend} - PROPERTIES TIMEOUT 600) - endif() + add_test( + NAME test-bench_image_readback_${backend} + COMMAND + ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/test_bench_image_readback.py + $ ${backend} + ) + set_tests_properties(test-bench_image_readback_${backend} + PROPERTIES TIMEOUT 600) add_test( NAME test-bench_images_${backend} COMMAND diff --git a/cmake/Blosc.cmake b/cmake/Blosc.cmake index b1be9dac..88102ca5 100644 --- a/cmake/Blosc.cmake +++ b/cmake/Blosc.cmake @@ -2,8 +2,7 @@ # # Tries CONFIG mode first (covers vcpkg, Conan, system CMake configs), # then falls back to the bundled FindBlosc.cmake module. -# If neither finds blosc, HAVE_BLOSC is OFF and CPU Blosc is unavailable. -# GPU Blosc uses nvCOMP and remains available in GPU builds. +# Required for CPU compression and GPU Blosc interoperability tests. find_package(blosc CONFIG QUIET) if(blosc_FOUND) @@ -30,12 +29,5 @@ if(blosc_FOUND) endif() if(NOT TARGET Blosc::Blosc) - find_package(Blosc QUIET) -endif() - -if(TARGET Blosc::Blosc) - set(HAVE_BLOSC ON) -else() - set(HAVE_BLOSC OFF) - message(STATUS "Blosc not found — CPU blosc codecs disabled") + find_package(Blosc REQUIRED MODULE) endif() diff --git a/cmake/chuckyConfig.cmake.in b/cmake/chuckyConfig.cmake.in index 51596428..1435b072 100644 --- a/cmake/chuckyConfig.cmake.in +++ b/cmake/chuckyConfig.cmake.in @@ -22,9 +22,7 @@ endmacro() _chucky_find_dep_config_or_module(lz4 Lz4) _chucky_find_dep_config_or_module(zstd Zstd) -if(@HAVE_BLOSC@) - _chucky_find_dep_config_or_module(blosc Blosc) -endif() +_chucky_find_dep_config_or_module(blosc Blosc) find_dependency(aws-c-s3 CONFIG) diff --git a/docs/guide.md b/docs/guide.md index 0ff544ac..1a86bf3c 100644 --- a/docs/guide.md +++ b/docs/guide.md @@ -53,7 +53,7 @@ internal conventions, and key concepts. | Target | Source | Purpose | |--------|--------|---------| | `transpose_cpu` | `src/cpu/transpose.cpp` | OpenMP scatter | -| `compress_cpu` | `src/cpu/compress.c`, `src/cpu/compress_blosc.c` | zstd / lz4 and optional c-blosc compression (CPU) | +| `compress_cpu` | `src/cpu/compress.c`, `src/cpu/compress_blosc.c` | zstd / lz4 / c-blosc compression (CPU) | | `aggregate_cpu` | `src/cpu/aggregate.c` | Shard packing (CPU) | | `lod_cpu` | `src/cpu/lod.cpp` | CPU LOD scatter + reduce | | `stream_cpu` | `src/cpu/stream.c` | CPU pipeline orchestrator | @@ -128,7 +128,7 @@ Both backends implement the same pipeline stages behind the same | Stage | GPU | CPU | |-------|-----|-----| | Scatter | CUDA kernel | OpenMP parallel loop | -| Compress | nvCOMP LZ4/Zstd, raw or Blosc-framed | libzstd / liblz4 / optional c-blosc | +| Compress | nvCOMP LZ4/Zstd, raw or Blosc-framed | libzstd / liblz4 / c-blosc | | Aggregate | CUDA kernel | Sequential packing | | LOD | CUDA kernels (templated on dtype) | C++ templates + OpenMP | | Orchestration | 4 CUDA streams, double-buffered | Single-threaded pipeline | diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f915878c..2056b584 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -109,16 +109,15 @@ list( # Conditional deps via generator expressions — empty branches are dropped at # generation time without referencing missing targets. -set(_with_blosc $) set(_with_gpu $) set(_chucky_public_deps ${_math_lib} Threads::Threads Lz4::Lz4 Zstd::Zstd + Blosc::Blosc AWS::aws-c-s3 $<$:psapi> - $<${_with_blosc}:Blosc::Blosc> $<${_with_gpu}:CUDA::cuda_driver> $<${_with_gpu}:CUDA::cudart_static> $<${_with_gpu}:nvcomp::nvcomp> diff --git a/src/cpu/CMakeLists.txt b/src/cpu/CMakeLists.txt index bd737929..b0acc68c 100644 --- a/src/cpu/CMakeLists.txt +++ b/src/cpu/CMakeLists.txt @@ -2,16 +2,9 @@ add_src_lib(transpose_cpu SOURCES transpose.cpp transpose.h LINKS index_ops threadpool ) -add_src_lib(compress_cpu SOURCES compress.c compress.h compress_blosc.h - LINKS codec_types threadpool Lz4::Lz4 Zstd::Zstd chucky_log +add_src_lib(compress_cpu SOURCES compress.c compress.h compress_blosc.c compress_blosc.h + LINKS codec_types threadpool Lz4::Lz4 Zstd::Zstd Blosc::Blosc chucky_log ) -if(HAVE_BLOSC) - target_sources(compress_cpu PRIVATE compress_blosc.c) - target_link_libraries(compress_cpu PUBLIC Blosc::Blosc) - target_compile_definitions(compress_cpu PUBLIC HAVE_BLOSC) -else() - target_sources(compress_cpu PRIVATE compress_blosc_stub.c) -endif() add_src_lib(aggregate_cpu SOURCES aggregate.c aggregate.h LINKS index_ops threadpool chucky_log diff --git a/src/cpu/compress_blosc_stub.c b/src/cpu/compress_blosc_stub.c deleted file mode 100644 index d5e548c1..00000000 --- a/src/cpu/compress_blosc_stub.c +++ /dev/null @@ -1,44 +0,0 @@ -#include "cpu/compress_blosc.h" - -#include "util/prelude.h" - -int -compress_blosc_validate(struct codec_config codec) -{ - if (!codec_is_blosc(codec.id)) - return 1; - log_error("blosc codec requested but not compiled in"); - return 1; -} - -size_t -compress_blosc_max_output_size(size_t chunk_bytes) -{ - return chunk_bytes + 16; -} - -int -compress_blosc(struct codec_config codec, - const void* src, - size_t input_stride, - void* dst, - size_t max_output_size, - size_t* comp_sizes, - size_t chunk_bytes, - size_t batch_size, - size_t bytes_per_element, - struct threadpool* pool) -{ - (void)codec; - (void)src; - (void)input_stride; - (void)dst; - (void)max_output_size; - (void)comp_sizes; - (void)chunk_bytes; - (void)batch_size; - (void)bytes_per_element; - (void)pool; - log_error("blosc codec requested but not compiled in"); - return 1; -} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 40793a1b..4d655dba 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -162,11 +162,7 @@ add_test_exe(test_ngff_flush_cascade test_ngff_flush_cascade.c stream_cpu writer add_test_exe(test_multiarray_cpu test_multiarray_cpu.c multiarray_cpu stream_cpu writer test_shard_sink chucky_log) # GPU multi-array stream test -add_gpu_test_exe(test_multiarray_gpu test_multiarray_gpu.c LINKS CUDA::cuda_driver CUDA::cudart_static multiarray_gpu stream writer test_shard_sink) -if(CHUCKY_ENABLE_GPU AND HAVE_BLOSC) - target_compile_definitions(test_multiarray_gpu PRIVATE HAVE_BLOSC) - target_link_libraries(test_multiarray_gpu PRIVATE Blosc::Blosc) -endif() +add_gpu_test_exe(test_multiarray_gpu test_multiarray_gpu.c LINKS CUDA::cuda_driver CUDA::cudart_static multiarray_gpu stream writer test_shard_sink Blosc::Blosc) # A cross-array ordering cycle would otherwise strand the GPU test runner. if(CHUCKY_ENABLE_GPU) set_tests_properties(test-test_multiarray_gpu PROPERTIES TIMEOUT 60) @@ -178,7 +174,7 @@ add_test_exe(test_multi_append test_multi_append.c stream_cpu writer test_shard_ # CPU stream + zarr store integration (exercises async IO fencing) add_test_exe(test_cpu_zarr_fs test_cpu_zarr_fs.c stream_cpu test_zarr_helpers test_counting_sink writer Zstd::Zstd test_platform test_shard_verify) -# Zarr readback validation via zarr-python (requires uv) +# Zarr readback validation via zarr-python and TensorStore (requires uv) add_test_exe(test_zarr_readback test_zarr_readback.c stream_cpu test_zarr_helpers writer test_platform chucky_log) target_compile_definitions( test_zarr_readback @@ -186,6 +182,16 @@ target_compile_definitions( ) set_tests_properties(test-test_zarr_readback PROPERTIES SKIP_RETURN_CODE 77) +# Full NGFF pyramid readback, sharing the fixture and codec matrix across backends. +add_test_exe(test_ngff_readback_cpu test_ngff_readback.c + stream_cpu test_zarr_helpers writer test_platform chucky_log +) +target_compile_definitions( + test_ngff_readback_cpu + PRIVATE SOURCE_DIR="${CMAKE_SOURCE_DIR}" +) +set_tests_properties(test-test_ngff_readback_cpu PROPERTIES TIMEOUT 180) + # Cross-validate GPU vs CPU (byte-exact shard comparison with CODEC_NONE) add_gpu_test_exe(test_cross_validate test_cross_validate.c LINKS CUDA::cuda_driver CUDA::cudart_static stream stream_cpu writer test_data test_shard_verify) @@ -201,7 +207,7 @@ target_link_libraries(test_data PUBLIC stream_config writer chucky_log) # Tests with zstd add_gpu_test_exe(test_stream test_stream.c LINKS CUDA::cuda_driver CUDA::cudart_static stream Zstd::Zstd index_ops_util test_shard_sink test_gpu_helpers test_shard_verify test_voxel_encode) add_gpu_test_exe(test_compress test_compress.c LINKS CUDA::cuda_driver CUDA::cudart_static stream Zstd::Zstd Lz4::Lz4) -if(CHUCKY_ENABLE_GPU AND HAVE_BLOSC) +if(CHUCKY_ENABLE_GPU) add_gpu_test_exe(test_compress_blosc_gpu test_compress_blosc_gpu.c LINKS CUDA::cuda_driver CUDA::cudart_static stream Blosc::Blosc ) @@ -217,6 +223,15 @@ if(CHUCKY_ENABLE_GPU AND HAVE_BLOSC) test-test_zarr_readback_gpu PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 120 ) + add_gpu_test_exe(test_ngff_readback_gpu test_ngff_readback.c + LINKS CUDA::cuda_driver CUDA::cudart_static stream test_zarr_helpers + writer test_platform chucky_log + ) + target_compile_definitions( + test_ngff_readback_gpu + PRIVATE TEST_NGFF_READBACK_GPU SOURCE_DIR="${CMAKE_SOURCE_DIR}" + ) + set_tests_properties(test-test_ngff_readback_gpu PROPERTIES TIMEOUT 180) endif() add_test_exe(test_json_writer test_json_writer.c zarr_metadata ngff_metadata attr_set json_validate strbuf chucky_log) diff --git a/tests/test_compress_cpu.c b/tests/test_compress_cpu.c index 2e1f0b90..0b4f8247 100644 --- a/tests/test_compress_cpu.c +++ b/tests/test_compress_cpu.c @@ -3,9 +3,7 @@ #include "threadpool/threadpool.h" #include "util/prelude.h" -#ifdef HAVE_BLOSC #include -#endif #include #include #include @@ -259,7 +257,6 @@ test_nthreads_1(void) return 1; } -#ifdef HAVE_BLOSC static int test_codec_blosc(enum compression_codec id, const char* name) { @@ -421,7 +418,6 @@ test_blosc_explicit_blocks(void) free(decoded); return 1; } -#endif // HAVE_BLOSC int main(int ac, char* av[]) @@ -441,11 +437,9 @@ main(int ac, char* av[]) rc |= test_codec_lz4(); rc |= test_codec_zstd(); rc |= test_nthreads_1(); -#ifdef HAVE_BLOSC rc |= test_codec_blosc(CODEC_BLOSC_LZ4, "blosc_lz4"); rc |= test_codec_blosc(CODEC_BLOSC_ZSTD, "blosc_zstd"); rc |= test_blosc_explicit_blocks(); -#endif threadpool_free(g_pool); threadpool_free(g_pool1); diff --git a/tests/test_multiarray_gpu.c b/tests/test_multiarray_gpu.c index 812b3aaa..34fd79a1 100644 --- a/tests/test_multiarray_gpu.c +++ b/tests/test_multiarray_gpu.c @@ -9,9 +9,7 @@ #include #include -#ifdef HAVE_BLOSC #include -#endif #define SHARD_CAP (1 << 20) #define SINK_N_SHARDS 16 @@ -27,7 +25,6 @@ test_sink_shard_count(const struct test_shard_sink* s) return count; } -#ifdef HAVE_BLOSC static int verify_one_blosc_chunk(const struct test_shard_sink* sink, size_t chunk_bytes, @@ -68,7 +65,6 @@ verify_one_blosc_chunk(const struct test_shard_sink* sink, free(recovered); return result; } -#endif // Helper: 2D config with given dtype. Shape 4x4, chunk 2x2, 2 shards along // dim0, 1 along dim1 (cps 1x2). epoch_elements = 8. @@ -231,7 +227,6 @@ test_switch_at_epoch_boundary(void) return 1; } -#ifdef HAVE_BLOSC static int test_blosc_rebind(void) { @@ -313,7 +308,6 @@ test_blosc_rebind(void) log_error(" FAIL"); return 1; } -#endif // ---- Test: switch mid-epoch rejected ---- @@ -1610,9 +1604,7 @@ main(int ac, char* av[]) int ret = 0; ret |= test_basic_two_array(); ret |= test_switch_at_epoch_boundary(); -#ifdef HAVE_BLOSC ret |= test_blosc_rebind(); -#endif ret |= test_switch_mid_epoch_rejected(); ret |= test_flush_all(); ret |= test_one_array_failure_spares_the_others(); diff --git a/tests/test_ngff_readback.c b/tests/test_ngff_readback.c new file mode 100644 index 00000000..4878804c --- /dev/null +++ b/tests/test_ngff_readback.c @@ -0,0 +1,191 @@ +// Write the same NGFF fixture with either backend, then read it independently. +#include "test_platform.h" +#include "test_readback_codecs.h" +#include "test_zarr_helpers.h" +#include "util/prelude.h" +#include "writer.buffered.h" + +#ifdef TEST_NGFF_READBACK_GPU +#include "gpu/prelude.cuda.h" +#include "stream.gpu.h" +#define stream_type tile_stream_gpu +#define stream_create tile_stream_gpu_create +#define stream_writer tile_stream_gpu_writer +#define stream_destroy tile_stream_gpu_destroy +#else +#include "stream.cpu.h" +#define stream_type tile_stream_cpu +#define stream_create tile_stream_cpu_create +#define stream_writer tile_stream_cpu_writer +#define stream_destroy tile_stream_cpu_destroy +#endif + +#include +#include +#include + +#define NT 5 +#define NY 512 +#define NX 512 +#define NLEVELS 3 + +static int +write_pyramid(const char* path, struct codec_config codec, int buffered) +{ + int err = 1; + const size_t frame_bytes = NY * NX * sizeof(uint16_t); + const size_t total_bytes = (NT + (buffered == 2)) * frame_bytes; + // Preserve the GPU readback coverage for unaligned caller buffers. + uint8_t* allocation = malloc(total_bytes + 1); + struct test_zarr_multiscale sink = { 0 }; + struct stream_type* stream = NULL; + struct buffered_writer* adapter = NULL; + CHECK(Done, allocation); + uint8_t* src = allocation + 1; + for (int t = 0; t < NT + (buffered == 2); ++t) + for (int y = 0; y < NY; ++y) + for (int x = 0; x < NX; ++x) { + // Positive affine ramp: all spatial block means are exact integers. + const uint16_t value = (uint16_t)(1 + 1024 * t + 8 * y + 4 * x); + const size_t i = ((size_t)t * NY + y) * NX + x; + memcpy(src + i * sizeof(value), &value, sizeof(value)); + } + + struct dimension dims[3]; + dims_create(dims, "tyx", (uint64_t[]){ buffered == 2 ? NT : 0, NY, NX }); + dims_set_chunk_sizes(dims, 3, (uint64_t[]){ 1, 128, 128 }); + // Two frames per shard: two complete shards and a final partial shard. + for (int d = 0; d < 3; ++d) + dims[d].chunks_per_shard = 2; + dims[1].downsample = dims[2].downsample = 1; + const struct ngff_axis axes[] = { + { .type = ngff_axis_time, .unit = "second", .scale = 0.25 }, + { .type = ngff_axis_space, .unit = "micrometer", .scale = 0.5 }, + { .type = ngff_axis_space, .unit = "micrometer", .scale = 0.75 }, + }; + CHECK( + Done, + test_zarr_multiscale_open( + &sink, path, "pyramid", dims, 3, dtype_u16, NLEVELS, codec, axes, 0) == + 0); + const struct tile_stream_configuration config = { + .buffer_capacity_bytes = frame_bytes, + .dtype = dtype_u16, + .rank = 3, + .dimensions = dims, + .codec = codec, + .reduce_method = lod_reduce_mean, + .max_nlod = NLEVELS, + .epochs_per_batch = 1, + .metadata_update_interval_s = 0, + .max_threads = 2, + }; + stream = stream_create(&config, test_zarr_multiscale_as_shard_sink(&sink)); + CHECK(Done, stream); + struct writer* writer = stream_writer(stream); + if (buffered) { + struct buffered_writer_config buffering = { 2 * 1023 * 3, 2 * 997 }; + // Accept an extra frame so the bounded stream's unconsumed suffix is + // deterministic, just as in the single-array readback tests. + if (buffered == 2) + buffering = (struct buffered_writer_config){ total_bytes, 0 }; + adapter = buffered_writer_create(writer, &buffering); + CHECK(Done, adapter); + writer = buffered_writer_as_writer(adapter); + } + if (buffered == 2) { + CHECK(Done, + writer_append_wait(writer, (struct slice){ src, src + total_bytes }) + .error == 0); + memset(src, 0xFF, total_bytes); + } else { + for (int t = 0; t < NT; ++t) { + uint8_t* frame = src + t * frame_bytes; + CHECK( + Done, + writer_append_wait(writer, (struct slice){ frame, frame + frame_bytes }) + .error == 0); + memset(frame, 0xFF, frame_bytes); + } + } + CHECK(Done, writer_flush(writer).error == (buffered == 2)); + CHECK(Done, writer_close(writer).error == (buffered == 2)); + CHECK(Done, ngff_multiscale_flush(sink.ms) == 0); + if (buffered == 2) { + const struct buffered_writer_stats stats = + buffered_writer_get_stats(adapter); + CHECK(Done, stats.downstream_finished && stats.failed); + CHECK(Done, stats.forwarded_bytes == NT * frame_bytes); + CHECK(Done, stats.abandoned_bytes == frame_bytes); + } + err = 0; +Done: + buffered_writer_destroy(adapter); + stream_destroy(stream); + test_zarr_multiscale_close(&sink); + free(allocation); + return err; +} + +int +main(void) +{ + // Reader dependencies are required; missing uv must fail CI, not skip it. + if (system("uv --version > " NULL_DEV " 2>&1") != 0) { + log_error("uv is required for TensorStore/NGFF readback"); + return 1; + } + int err = 1; + char tmpdir[256] = { 0 }; +#ifdef TEST_NGFF_READBACK_GPU + CUcontext context = 0; + CUdevice device; + CU(Done, cuInit(0)); + CU(Done, cuDeviceGet(&device, 0)); + CU(Done, cu_ctx_create(&context, 0, device)); +#endif + CHECK(Done, test_tmpdir_create(tmpdir, sizeof(tmpdir)) == 0); + size_t n_codecs, written = 0; + const struct test_readback_codec* codecs = test_readback_codecs(&n_codecs); + for (int buffered = 0; buffered <= 2; ++buffered) { + for (size_t i = 0; i < n_codecs; ++i) { + char path[512]; + snprintf(path, + sizeof(path), + "%s/%s%s", + tmpdir, + codecs[i].name, + buffered == 2 ? "_buffered_limit" + : buffered ? "_buffered" + : ""); + CHECK(Done, test_mkdir(path) == 0); + log_info("Writing NGFF %s", path); + CHECK(Done, write_pyramid(path, codecs[i].codec, buffered) == 0); + ++written; + } + } + char command[2048]; + snprintf(command, + sizeof(command), + "uv run \"" SOURCE_DIR "/tests/validate_ngff_readback.py\"" + " \"%s\" %d %d %d %zu", + tmpdir, + NT, + NY, + NX, + written); + CHECK(Done, system(command) == 0); + err = 0; +Done: + if (tmpdir[0]) { + if (err) + log_error("Readback failed; output preserved at %s", tmpdir); + else + test_tmpdir_remove(tmpdir); + } +#ifdef TEST_NGFF_READBACK_GPU + if (context) + cuCtxDestroy(context); +#endif + return err; +} diff --git a/tests/test_readback_codecs.h b/tests/test_readback_codecs.h new file mode 100644 index 00000000..67617d70 --- /dev/null +++ b/tests/test_readback_codecs.h @@ -0,0 +1,63 @@ +// Shared codec matrix for single-array and NGFF reader interoperability. +#pragma once + +#include "types.codec.h" +#include + +struct test_readback_codec +{ + const char* name; + struct codec_config codec; +}; + +static inline const struct test_readback_codec* +test_readback_codecs(size_t* count) +{ + static const struct test_readback_codec cases[] = { + // lz4 omitted: no zarr v3 LZ4 codec spec; zarr-python can't read it. + { "none", { .id = CODEC_NONE } }, + { "zstd", { .id = CODEC_ZSTD } }, + { "blosc_lz4_noshuffle", + { .id = CODEC_BLOSC_LZ4, + .level = 5, + .shuffle = CODEC_SHUFFLE_NONE, + .blosc_block_bytes = 16 * 1024 } }, + { "blosc_lz4_shuffle", + { .id = CODEC_BLOSC_LZ4, + .level = 5, + .shuffle = CODEC_SHUFFLE_BYTE, + .blosc_block_bytes = 16 * 1024 } }, + { "blosc_lz4_bitshuffle", + { .id = CODEC_BLOSC_LZ4, + .level = 5, + .shuffle = CODEC_SHUFFLE_BIT, + .blosc_block_bytes = 16 * 1024 } }, + { "blosc_zstd_noshuffle", + { .id = CODEC_BLOSC_ZSTD, + .level = 5, + .shuffle = CODEC_SHUFFLE_NONE, + .blosc_block_bytes = 16 * 1024 } }, + { "blosc_zstd_shuffle", + { .id = CODEC_BLOSC_ZSTD, + .level = 5, + .shuffle = CODEC_SHUFFLE_BYTE, + .blosc_block_bytes = 16 * 1024 } }, + { "blosc_zstd_bitshuffle", + { .id = CODEC_BLOSC_ZSTD, + .level = 5, + .shuffle = CODEC_SHUFFLE_BIT, + .blosc_block_bytes = 16 * 1024 } }, + { "blosc_lz4_unaligned_blocks", + { .id = CODEC_BLOSC_LZ4, + .level = 5, + .shuffle = CODEC_SHUFFLE_BYTE, + .blosc_block_bytes = 4097 } }, + { "blosc_zstd_unaligned_blocks", + { .id = CODEC_BLOSC_ZSTD, + .level = 5, + .shuffle = CODEC_SHUFFLE_BIT, + .blosc_block_bytes = 4097 } }, + }; + *count = sizeof(cases) / sizeof(cases[0]); + return cases; +} diff --git a/tests/test_zarr_readback.c b/tests/test_zarr_readback.c index 61b67c15..9b6c59bd 100644 --- a/tests/test_zarr_readback.c +++ b/tests/test_zarr_readback.c @@ -1,9 +1,9 @@ -// Test that zarr output is readable by zarr-python. +// Test that zarr output is readable by zarr-python and TensorStore. // Writes zarr stores with various codecs, then runs a Python validation script. -#include "cpu/compress_blosc.h" #include "stream.cpu.h" #include "test_platform.h" +#include "test_readback_codecs.h" #include "test_zarr_helpers.h" #include "util/prelude.h" #include "writer.buffered.h" @@ -21,9 +21,6 @@ static int write_zarr(const char* store_path, struct codec_config codec, int buffered) { - if (codec_is_blosc(codec.id) && compress_blosc_validate(codec)) - return 2; - const int total = (NT + (buffered == 2)) * NY * NX; uint16_t* src = (uint16_t*)malloc((size_t)total * sizeof(uint16_t)); CHECK(Fail, src); @@ -108,30 +105,12 @@ main(void) char tmpdir[256]; CHECK(Fail, test_tmpdir_create(tmpdir, sizeof(tmpdir)) == 0); - struct - { - const char* name; - struct codec_config codec; - } codecs[] = { - // lz4 omitted: no zarr v3 LZ4 codec spec; zarr-python can't read it. - { "none", { .id = CODEC_NONE } }, - { "zstd", { .id = CODEC_ZSTD } }, - { "blosc_lz4", - { .id = CODEC_BLOSC_LZ4, - .level = 5, - .shuffle = CODEC_SHUFFLE_BYTE, - .blosc_block_bytes = 16 * 1024 } }, - { "blosc_zstd", - { .id = CODEC_BLOSC_ZSTD, - .level = 5, - .shuffle = CODEC_SHUFFLE_BYTE, - .blosc_block_bytes = 16 * 1024 } }, - }; - int n_codecs = (int)(sizeof(codecs) / sizeof(codecs[0])); + size_t n_codecs; + const struct test_readback_codec* codecs = test_readback_codecs(&n_codecs); int err = 0; for (int buffered = 0; buffered <= 2; ++buffered) { - for (int i = 0; i < n_codecs; ++i) { + for (size_t i = 0; i < n_codecs; ++i) { char store[512]; snprintf(store, sizeof(store), @@ -144,11 +123,6 @@ main(void) CHECK(Cleanup, test_mkdir(store) == 0); log_info("Writing %s ...", codecs[i].name); int wrc = write_zarr(store, codecs[i].codec, buffered); - if (wrc == 2) { // blosc not available - log_info(" skipped: %s (codec not available)", codecs[i].name); - test_tmpdir_remove(store); - continue; - } if (wrc) { log_error(" write failed: %s", codecs[i].name); err = 1; diff --git a/tests/test_zarr_readback_gpu.c b/tests/test_zarr_readback_gpu.c index 3a4ee0e1..eda03010 100644 --- a/tests/test_zarr_readback_gpu.c +++ b/tests/test_zarr_readback_gpu.c @@ -1,8 +1,9 @@ -// Write GPU Blosc stores and validate them through zarr-python/numcodecs. +// Write GPU stores and validate them through zarr-python and TensorStore. #include "gpu/prelude.cuda.h" #include "stream.gpu.h" #include "test_platform.h" +#include "test_readback_codecs.h" #include "test_zarr_helpers.h" #include "util/prelude.h" #include "writer.buffered.h" @@ -111,58 +112,12 @@ main(void) char tmpdir[256]; CHECK(RunFail, test_tmpdir_create(tmpdir, sizeof(tmpdir)) == 0); - const struct - { - const char* name; - struct codec_config codec; - } cases[] = { - { "none", { .id = CODEC_NONE } }, - { "zstd", { .id = CODEC_ZSTD } }, - { "blosc_lz4_noshuffle", - { .id = CODEC_BLOSC_LZ4, - .level = 5, - .shuffle = CODEC_SHUFFLE_NONE, - .blosc_block_bytes = 16 * 1024 } }, - { "blosc_lz4_shuffle", - { .id = CODEC_BLOSC_LZ4, - .level = 5, - .shuffle = CODEC_SHUFFLE_BYTE, - .blosc_block_bytes = 16 * 1024 } }, - { "blosc_lz4_bitshuffle", - { .id = CODEC_BLOSC_LZ4, - .level = 5, - .shuffle = CODEC_SHUFFLE_BIT, - .blosc_block_bytes = 16 * 1024 } }, - { "blosc_zstd_noshuffle", - { .id = CODEC_BLOSC_ZSTD, - .level = 5, - .shuffle = CODEC_SHUFFLE_NONE, - .blosc_block_bytes = 16 * 1024 } }, - { "blosc_zstd_shuffle", - { .id = CODEC_BLOSC_ZSTD, - .level = 5, - .shuffle = CODEC_SHUFFLE_BYTE, - .blosc_block_bytes = 16 * 1024 } }, - { "blosc_zstd_bitshuffle", - { .id = CODEC_BLOSC_ZSTD, - .level = 5, - .shuffle = CODEC_SHUFFLE_BIT, - .blosc_block_bytes = 16 * 1024 } }, - { "blosc_lz4_unaligned_blocks", - { .id = CODEC_BLOSC_LZ4, - .level = 5, - .shuffle = CODEC_SHUFFLE_BYTE, - .blosc_block_bytes = 4097 } }, - { "blosc_zstd_unaligned_blocks", - { .id = CODEC_BLOSC_ZSTD, - .level = 5, - .shuffle = CODEC_SHUFFLE_BIT, - .blosc_block_bytes = 4097 } }, - }; + size_t n_codecs; + const struct test_readback_codec* cases = test_readback_codecs(&n_codecs); int error = 0; for (int buffered = 0; buffered <= 2; ++buffered) { - for (size_t i = 0; i < countof(cases); ++i) { + for (size_t i = 0; i < n_codecs; ++i) { char store[512]; snprintf(store, sizeof(store), diff --git a/tests/validate_ngff_readback.py b/tests/validate_ngff_readback.py new file mode 100644 index 00000000..91227e0d --- /dev/null +++ b/tests/validate_ngff_readback.py @@ -0,0 +1,167 @@ +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "zarr>=3", +# "numcodecs", +# "tensorstore>=0.1.76", +# "ome-zarr-models>=1.6", +# ] +# /// +"""Read every NGFF level and exercise the reader's missing/corrupt-data checks. + +The affine reference and metadata checks are shared by the CPU/GPU fixtures +and can also be used by the streaming S3 readback test. +""" + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np +import zarr + +from validate_ome_ngff import validate_store as validate_ome_store +from validate_zarr import validate_tensorstore + + +def expected_level(level: int, shape: tuple[int, int, int]) -> np.ndarray: + """Analytic mean of each spatial block of the input affine ramp. + + Mean coordinates of a block of width s are s*i + (s-1)/2. Applying + coefficients 8 and 4 gives a combined offset of 6*(s-1), exactly integral. + This reference neither reads L0 nor uses Chucky's downsampling routines. + """ + nt, ny, nx = shape + scale = 2**level + t = np.arange(nt, dtype=np.uint32)[:, None, None] + y = np.arange(ny // scale, dtype=np.uint32)[None, :, None] + x = np.arange(nx // scale, dtype=np.uint32)[None, None, :] + return (1 + 1024 * t + 8 * scale * y + 4 * scale * x + 6 * (scale - 1)).astype( + np.uint16 + ) + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def validate_pyramid(group_path: Path, shape: tuple[int, int, int]) -> None: + require(validate_ome_store(group_path), f"Invalid NGFF metadata: {group_path}") + metadata = json.loads((group_path / "zarr.json").read_text()) + ome = metadata["attributes"]["ome"] + require(ome["version"] == "0.5", "Expected OME-NGFF 0.5") + require(len(ome["multiscales"]) == 1, "Expected one multiscale image") + multiscale = ome["multiscales"][0] + require( + multiscale["axes"] + == [ + {"name": "t", "type": "time", "unit": "second"}, + {"name": "y", "type": "space", "unit": "micrometer"}, + {"name": "x", "type": "space", "unit": "micrometer"}, + ], + "Incorrect axis names, order, types, or units", + ) + datasets = multiscale["datasets"] + require(len(datasets) == 3, "Expected three resolution levels") + require(len({d["path"] for d in datasets}) == 3, "Duplicate dataset paths") + group = zarr.open_group(str(group_path), mode="r") + for level, dataset in enumerate(datasets): + scale = 2**level + require( + dataset["coordinateTransformations"] + == [{"type": "scale", "scale": [0.25, 0.5 * scale, 0.75 * scale]}], + f"Incorrect coordinate transform at level {level}", + ) + # Follow the metadata's paths rather than assuming names like 0/1/2. + path = dataset["path"] + expected = expected_level(level, shape) + arr = group[path] + require(arr.shape == expected.shape, f"{path}: incorrect zarr-python shape") + require(arr.dtype == expected.dtype, f"{path}: incorrect zarr-python dtype") + np.testing.assert_array_equal(arr[:], expected, err_msg=f"zarr-python: {path}") + validate_tensorstore(group_path / path, expected) + print(f" PASS {group_path.parent.name}: all three NGFF levels", file=sys.stderr) + + +def expect_rejected(label, check): + try: + check() + except (ValueError, AssertionError): + print(f" PASS rejected {label}", file=sys.stderr) + return + raise AssertionError(f"Validator accepted {label}") + + +def check_reader_failures(group_path: Path, shape: tuple[int, int, int]) -> None: + """Prove that every level is read, including the final partial shard.""" + metadata_path = group_path / "zarr.json" + original_metadata = metadata_path.read_bytes() + metadata = json.loads(original_metadata) + datasets = metadata["attributes"]["ome"]["multiscales"][0]["datasets"] + for level, dataset in enumerate(datasets): + array_path = group_path / dataset["path"] + expected = expected_level(level, shape) + shard = sorted(p for p in (array_path / "c").rglob("*") if p.is_file())[-1] + original = shard.read_bytes() + try: + shard.unlink() + expect_rejected( + f"missing level {level} shard", + lambda: validate_tensorstore(array_path, expected), + ) + for offset, label in ((0, "payload"), (-1, "index checksum")): + corrupt = bytearray(original) + corrupt[offset] ^= 1 + shard.write_bytes(corrupt) + expect_rejected( + f"corrupt level {level} {label}", + lambda: validate_tensorstore(array_path, expected), + ) + finally: + shard.write_bytes(original) + + # A valid but incorrect physical scale must fail our semantic checks. + try: + datasets[2]["coordinateTransformations"][0]["scale"][1] *= 2 + metadata_path.write_text(json.dumps(metadata)) + expect_rejected("incorrect NGFF scale", lambda: validate_pyramid(group_path, shape)) + finally: + metadata_path.write_bytes(original_metadata) + + # A renamed level still works when its advertised path is updated. + metadata = json.loads(original_metadata) + dataset = metadata["attributes"]["ome"]["multiscales"][0]["datasets"][2] + original_path = group_path / dataset["path"] + renamed_path = group_path / "coarse" + original_path.rename(renamed_path) + try: + dataset["path"] = "coarse" + metadata_path.write_text(json.dumps(metadata)) + validate_pyramid(group_path, shape) + finally: + renamed_path.rename(original_path) + metadata_path.write_bytes(original_metadata) + validate_pyramid(group_path, shape) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + parser.add_argument("nt", type=int) + parser.add_argument("ny", type=int) + parser.add_argument("nx", type=int) + parser.add_argument("expected_stores", type=int) + args = parser.parse_args() + shape = (args.nt, args.ny, args.nx) + stores = sorted(p for p in args.directory.iterdir() if p.is_dir()) + require(len(stores) == args.expected_stores > 0, "Missing readback fixtures") + for store in stores: + validate_pyramid(store / "pyramid", shape) + # CODEC_NONE makes payload corruption independent of decoder behavior. + check_reader_failures(args.directory / "none" / "pyramid", shape) + + +if __name__ == "__main__": + main() diff --git a/tests/validate_zarr.py b/tests/validate_zarr.py index 1ec3c4fe..ba250207 100644 --- a/tests/validate_zarr.py +++ b/tests/validate_zarr.py @@ -3,6 +3,7 @@ # dependencies = [ # "zarr>=3", # "numcodecs", +# "tensorstore>=0.1.76", # ] # /// """Validate zarr stores written by test_zarr_readback. @@ -10,16 +11,38 @@ Usage: uv run tests/validate_zarr.py Each subdirectory of is a zarr store with array "0". -Validates shape, dtype, and data values for each codec variant. +Validates shape, dtype, and data values with zarr-python and TensorStore. """ import sys from pathlib import Path import numpy as np +import tensorstore as ts import zarr +def validate_tensorstore(array_path: Path, expected: np.ndarray) -> None: + """Read a complete array without treating absent chunks as valid fill data.""" + arr = ts.open( + { + "driver": "zarr3", + "kvstore": {"driver": "file", "path": array_path.resolve().as_posix() + "/"}, + "fill_missing_data_reads": False, + }, + open=True, + read=True, + write=False, + recheck_cached_metadata=True, + recheck_cached_data=True, + ).result() + if tuple(arr.shape) != expected.shape: + raise ValueError(f"{array_path}: shape {arr.shape} != {expected.shape}") + if arr.dtype.numpy_dtype != expected.dtype: + raise ValueError(f"{array_path}: dtype {arr.dtype} != {expected.dtype}") + np.testing.assert_array_equal(arr.read().result(), expected, err_msg=str(array_path)) + + def validate_store(store_path: Path, nt: int, ny: int, nx: int) -> bool: name = store_path.name array_path = store_path / "0" @@ -59,7 +82,13 @@ def validate_store(store_path: Path, nt: int, ny: int, nx: int) -> bool: ) return False - print(f" PASS {name}", file=sys.stderr) + try: + validate_tensorstore(array_path, expected) + except Exception as e: + print(f" FAIL {name}: TensorStore: {e}", file=sys.stderr) + return False + + print(f" PASS {name}: zarr-python and TensorStore", file=sys.stderr) return True