diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bfeeb0..149169b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,16 +6,33 @@ on: pull_request: jobs: + core-smoke: + # A real lean install: no default dev group and therefore no zarr, image, or + # torch stack. This protects the documented `pip install raw2features` path. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v7 + with: + python-version: "3.12" + - name: Install (core only) + run: uv sync --locked --no-default-groups + - name: Smoke import and CLI + run: | + uv run --no-sync python -c "import raw2features, sys; assert 'torch' not in sys.modules; assert 'zarr' not in sys.modules" + uv run --no-sync raw2features --help + lint-and-test: - # Core-only: lint + run the suite with no heavy extras installed. Proves the - # package imports and the core paths work without torch/zarr (heavy-dep tests - # self-skip), so an end user who installs just the core isn't broken. Run across - # every supported Python (matches the requires-python / classifiers contract). + # Default developer environment: lint + the offline suite across every + # supported Python. The dev group intentionally includes the light Zarr/image + # test stack; the separate core-smoke job protects the dependency-lean install. runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v5 with: @@ -23,8 +40,8 @@ jobs: - uses: astral-sh/setup-uv@v7 with: python-version: ${{ matrix.python-version }} - - name: Install (core only) - run: uv sync + - name: Install (developer test environment) + run: uv sync --locked - name: Lint run: uv run ruff check . - name: Test (core) @@ -45,6 +62,34 @@ jobs: with: python-version: "3.12" - name: Install (full stack, CPU torch) - run: uv sync --extra zarr --extra image --extra torch --extra models --extra stain + run: uv sync --locked --extra zarr --extra image --extra torch --extra models --extra stain - name: Test (with torch) run: uv run pytest -m "not slow and not network" -q + + package-smoke: + # Validate exactly what a release would upload, including clean installs from + # both artifacts rather than importing from the checkout. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@v7 + with: + python-version: "3.12" + - name: Build and check artifacts + run: | + uv build + uvx --from twine twine check dist/* + - name: Smoke wheel + run: | + uv venv .venv-wheel + uv pip install --python .venv-wheel/bin/python dist/*.whl + .venv-wheel/bin/python -c "import importlib.metadata, raw2features; assert raw2features.__version__ == importlib.metadata.version('raw2features')" + .venv-wheel/bin/raw2features --help + - name: Smoke source distribution + run: | + uv venv .venv-sdist + uv pip install --python .venv-sdist/bin/python dist/*.tar.gz + .venv-sdist/bin/python -c "import importlib.metadata, raw2features; assert raw2features.__version__ == importlib.metadata.version('raw2features')" + .venv-sdist/bin/raw2features --help diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f51ade..8b85530 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,7 @@ # Workflow: release.yml Environment: pypi # (A *pending* publisher lets the very first release create the project on PyPI.) # 2. Cut a release whose tag matches the version in pyproject.toml, e.g.: -# gh release create v0.1.0 --generate-notes +# gh release create v0.1.1 --generate-notes # This workflow then builds the sdist + wheel and uploads them automatically. name: Release @@ -28,8 +28,27 @@ jobs: contents: read # actions/checkout needs read access (required while the repo is private) steps: - uses: actions/checkout@v5 + with: + persist-credentials: false - uses: astral-sh/setup-uv@v7 + with: + python-version: "3.12" + - name: Verify tag matches package version + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: >- + python -c "import os, pathlib, tomllib; + version = tomllib.loads(pathlib.Path('pyproject.toml').read_text())['project']['version']; + expected = f'v{version}'; actual = os.environ['RELEASE_TAG']; + assert actual == expected, f'release tag {actual!r} does not match {expected!r}'" - name: Build sdist + wheel run: uv build + - name: Check and smoke wheel + run: | + uvx --from twine twine check dist/* + uv venv .venv-release-smoke + uv pip install --python .venv-release-smoke/bin/python dist/*.whl + .venv-release-smoke/bin/python -c "import importlib.metadata, raw2features; assert raw2features.__version__ == importlib.metadata.version('raw2features')" + .venv-release-smoke/bin/raw2features --help - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e6f6cf..76e240b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,75 @@ Notable changes to raw2features, newest first. This project follows [Semantic Versioning](https://semver.org). +## [0.1.1] - 2026-07-16 + +This is a focused correctness, reproducibility, and interoperability release. It keeps +the v0.1 store layout and geometry identity while making resume decisions stricter and +multi-grid workflows fully additive. + +### Correctness and reproducibility + +- Redact command-line secrets and persist credential-free canonical source URIs while + still supporting query-authenticated HTTP Zarr stores in memory. +- Bind stores and receipts to their source and requested output, reject duplicate cohort + output IDs before dispatch, make receipt writes atomic, and propagate worker-startup + failures to the command exit status. +- Enforce immutable model revisions at load time. Patch and slide outputs now carry a + per-model fingerprint covering their effective weights, preprocessing, pooling, + dimension, resolved AMP, and loader contract; a stale or legacy-unfingerprinted output + is recomputed without changing `grid_hash` or unrelated model arrays. +- Verify composite checkpoints before deserialisation, validate expected embedding + dimensions during completion checks, and mark SEAL experimental until its upstream + base-model fetch can be pinned end to end. +- Make colliding human-readable grid labels unique, produce explicitly requested + thumbnails/QC/GeoJSON/slide embeddings when missing, and validate receipts against the + requested grid rather than unioning same-named models across grids. +- Harden partial-store recovery and conformance validation, preserve per-axis scale in + slide-relative STAMP coordinates, place clipped border pixels at their true offsets, + reject invalid AMP/batch/geometry settings before model loading, and avoid attributing + an enclosing unrelated Git checkout to an installed wheel. + +### Models and interoperability + +- Add pinned, forward-validated H0-mini, KEEP, OpenMidnight, and OpenPath patch encoders, + with their licenses, access requirements, preprocessing, and weight hashes recorded. +- Correct cached OME-Zarr reads for noncanonical spatial axis order. The reader accepts + x/y in either order (with an optional channel axis) and warns whenever another + non-singleton axis is reduced to index zero. +- Refresh the development lock to current `ngff-zarr`/`ome-zarr` releases while retaining + the existing declared compatibility floor. +- Choose the float-pixel `[0,1]` versus `[0,255]` convention once per opened slide instead + of independently for every patch. + +### API, documentation, and release safety + +- Bless `embed_slide` as the high-level Python entry point while retaining `run_slide` as + the explicit single-grid primitive, resolve injected embedders from their own model + specifications, and make inline `-s/--slide-encoder` requests produce missing outputs + on otherwise-complete runs. +- Correct the multi-grid, additive-append, readback, and SLURM documentation, and use the + compact 4x PNG project banner so GitHub and PyPI render it consistently. +- Add a genuine dependency-lean import/CLI check, built-wheel and source-distribution + smoke installs, package metadata validation, and a release tag/version gate. + +### Compatibility and migration + +- Existing v0.1 stores remain readable; the embeddings schema and geometry-only + `grid_hash` are unchanged. +- The first request for a model output created before fingerprints were introduced will + recompute that model in place. The old header is deliberately not treated as proof that + pre-v0.1.1 revision pins were enforced. +- Plain local output IDs remain basename-based for v0.1 compatibility. Source comparison + prevents accidental reuse, and manifest preflight rejects duplicate derived IDs. +- Credentials already persisted by v0.1.0 are not rewritten by this release and should be + rotated separately if the store was shared. + +Tracked by [#6](https://github.com/CraigMyles/raw2features/issues/6); this release resolves +[#2](https://github.com/CraigMyles/raw2features/issues/2), +[#3](https://github.com/CraigMyles/raw2features/issues/3), +[#4](https://github.com/CraigMyles/raw2features/issues/4), and +[#5](https://github.com/CraigMyles/raw2features/issues/5). + ## [0.1.0] - 2026-06-30 First public release. @@ -33,3 +102,4 @@ What 0.1.0 provides: harness, and six runnable tutorial notebooks. [0.1.0]: https://github.com/CraigMyles/raw2features/releases/tag/v0.1.0 +[0.1.1]: https://github.com/CraigMyles/raw2features/compare/v0.1.0...v0.1.1 diff --git a/CITATION.cff b/CITATION.cff index 70fa82c..7e5ec8e 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -20,4 +20,4 @@ keywords: - foundation models - feature extraction license: MIT -version: "0.1.0" +version: "0.1.1" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 15ef054..0200dc7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,14 +1,22 @@ # Contributing to raw2features -raw2features is meant to be built *with* the community. The five seams - readers, -segmenters, patchers, embedders, sinks - are plugin points: you can add a model or -a storage backend by shipping a small package, no fork required. +raw2features is meant to be built *with* the community. Its six implementation seams - +readers, segmenters, patchers, embedders, sinks, and slide embedders - are Python +entry-point plugin groups. A third-party package can add an implementation or backend +without forking raw2features. + +The patch-model names accepted by the CLI are deliberately narrower: they come from the +provenance-complete registry bundled with raw2features. An external +`raw2features.embedders` entry point adds a loader family, not a new `--model` row by +itself. Python callers can pass external `Embedder` instances explicitly through +`embed_slide(..., embedders=[...])`; proposing a built-in CLI model still means adding a +reviewed registry row here. ## Dev setup ```bash uv sync --extra zarr --extra image --extra torch --extra models -uv run pytest -m "not slow" +uv run pytest -m "not slow and not network" uv run ruff check ``` @@ -18,28 +26,36 @@ Every model entry must be **provenance-complete** - its preprocessing is transcribed from the model's authoritative card/repo, and should never be guessed. If the preprocessing is uncertain, the model is not added. -1. Add a row to `models/registry.yaml` with **all** required fields: +1. Add a row to `src/raw2features/embedders/registry.yaml` with **all** required fields: - `source` (HF repo id / `torchvision://…` / local path) - - `weights_sha256` (verified at load; load fails loudly on mismatch) + - `weights_sha256`, `weights_revision`, and `weights_filename` (immutable artifact + identity; verified at load where raw2features handles the bytes) - `transform_source_url` (the model card / paper / repo the transform came from) - `input_size`, `mean`, `std`, `resize`, `interpolation` - - `embedding_dim`, `pooling` (`cls` / `mean` / `cls_mean_cat` / `pooled`) - - `license`, `gated` (HuggingFace gating?) + - `embedding_dim`, `pooling`, model family/constructor parameters, and any + recommended extraction MPP/patch size + - `license`, `gated` (Hugging Face access gate?), paper/source metadata, and DOI 2. If the model loads via timm/torchvision with a standard transform, that's all - the generic `TimmEmbedder` handles it. Only write a new `Embedder` subclass for bespoke loaders (e.g. CTransPath's modified Swin). -3. Register it: add an entry-point under `[project.entry-points."raw2features.embedders"]` - (in this repo) or ship it from your own package's entry-points. +3. If it needs a new loader family, register that implementation under + `[project.entry-points."raw2features.embedders"]`. Models using an existing family + need only the registry row and tests. 4. Add a test (a `MockEmbedder`-style shape/contract test is fine; a real-weights test should be marked `slow`). See `docs/adding_a_model.md` for a worked example. -## Adding a reader / segmenter / sink +## Adding another implementation plugin -Same shape: subclass the seam's ABC in `src/raw2features//base.py`, register -via entry-points. A missing optional dependency must degrade gracefully (the -plugin simply does not appear) - never break discovery. +Subclass the relevant ABC and register it through the matching +`raw2features.` entry-point group. The guaranteed discovery contract is small: +an importable entry point appears in `available()` and is returned by `get()`; an entry +point that cannot import (for example because its own optional dependency is absent) is +skipped without breaking discovery. The seam ABC defines the runtime methods and return +types. raw2features does not claim that an arbitrary third-party object conforms merely +because its entry point imports, so plugin packages should add their own end-to-end +contract test. ## Conventions diff --git a/README.md b/README.md index 156da9a..08fb023 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # raw2features

- raw2features: OME-Zarr whole-slide image in, patch-level and slide-level foundation-model embeddings out + raw2features: OME-Zarr whole-slide image in, patch-level and slide-level foundation-model embeddings out

Read a whole-slide image in **OME-Zarr / OME-NGFF** and emit **patch- and slide-level @@ -42,9 +42,11 @@ store too. (e.g. default 0.5 µm/px @ 224 px) by downsampling from the nearest finer pyramid level such that embeddings are comparable across slides and datasets. -- **Modular implementation.** `Reader`, `segmenter`, `patcher`, `embedder` and `sink` are - plugin seams exposed via Python entry-points: add a model or backend by - shipping a package. +- **Modular implementation.** Reader, segmenter, patcher, patch-embedder, + slide-embedder, and sink implementations are plugin seams exposed through Python + entry points. Entry points add implementations/families; the CLI's patch-model names + come from the bundled provenance registry, while Python callers can inject external + embedder instances through `embed_slide(..., embedders=[...])`. - **FAIR & provenance-first.** Stable models' weights are pinned to an **immutable HuggingFace revision** (or a sha256-pinned URL), with preprocessing sourced from each model's card. Every output records that provenance plus a 1:1 @@ -83,7 +85,7 @@ and the matching extra's comment in `pyproject.toml`. **Development** (from a clone, with [uv](https://docs.astral.sh/uv/)): ```bash -uv sync # core +uv sync --no-default-groups # dependency-lean core only uv sync --extra zarr --extra image --extra torch --extra models # full stack ``` diff --git a/assets/raw2features-diagram.png b/assets/raw2features-diagram.png new file mode 100644 index 0000000..a758a43 Binary files /dev/null and b/assets/raw2features-diagram.png differ diff --git a/docs/INTEROP.md b/docs/INTEROP.md index 244a928..2123a6b 100644 --- a/docs/INTEROP.md +++ b/docs/INTEROP.md @@ -119,10 +119,13 @@ disagree on dataset names and coordinate units, so pick the one your downstream | read by | CLAM, TRIDENT, TITAN, THREADS, CHIEF, Patho-Bench | KatherLab STAMP | We have every field for both natively (level-0 coords + `level0_patch` for TRIDENT; -`mpp_level0` to convert px→µm and `patch_px × achieved_mpp` for STAMP's `tile_size_um`). -The schemas are transcribed from each project's own source and written with an independent -`h5py` writer - file formats are not copyrightable, and no project's code (e.g. CLAM's -GPL writer) is copied. +per-axis `source.scale_um` to convert px→µm, and `patch_px × achieved_mpp` for STAMP's +`tile_size_um`). STAMP coordinates remain relative to the WSI scan's top-left, so an +NGFF physical/stage origin in `source.level0_translation_um` is deliberately not added. +Legacy stores without per-axis scale fall back to isotropic `mpp_level0`. The schemas +are transcribed from each project's own source and written with an independent `h5py` +writer - file formats are not copyrightable, and no project's code (e.g. CLAM's GPL +writer) is copied. ### Verified loading in each tool - a point-in-time claim diff --git a/docs/usage.md b/docs/usage.md index e7f0d60..f7a89f8 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -41,10 +41,11 @@ OME-Zarr ──reader──▶ segmenter ──▶ patcher ──▶ embedde | **segmenter** | `otsu` (or `--no-seg`) | low-res tissue mask so only tissue patches are kept | | **patcher** | `grid` | tiles a regular grid at an **exact** target MPP | | **embedder** | `resnet50` (repeatable) | patch → feature vector; one or many, run together | +| **slide_embedder** | none | patch-feature matrix → optional slide-level vector | | **sink** | `zarr` | writes `coords`/`grid_index`/`mask`/`features/` + provenance | List what's installed for any component: `raw2features list embedders` (or `readers`, -`segmenters`, `patchers`, `sinks`). +`segmenters`, `patchers`, `sinks`, `slide_embedders`). --- @@ -75,7 +76,7 @@ on another (e.g. an x86 cluster). Verify with `raw2features version`. | `raw2features export-h5 ` | export features to HDF5 (one `.h5` per model); needs the `[h5]` extra | | `raw2features validate-store ` | check a store conforms to the embeddings-store spec | | `raw2features models` | list the model registry (name, dim, gated, source) | -| `raw2features list ` | list installed plugins for a component type (readers, segmenters, …) | +| `raw2features list ` | list installed plugins for a component type (readers, segmenters, patchers, embedders, sinks, slide_embedders) | | `raw2features version` | print the version | Every option below is also visible via `raw2features --help`. @@ -135,8 +136,9 @@ Any token passed on the command line is redacted from stored command provenance. | Option | Default | Meaning | |--------|---------|---------| | `-m, --model` | `resnet50` | model name; **repeatable** for multi-model (`-f`/`--feature-extractor` are aliases) | -| `--mpp` | per-model | target µm/px; default = each model's **recommended** MPP (0.5 for most pathology FMs, 1.0 for scale-agnostic). Pass a value to override (collapses all models to one grid) | -| `--patch-size` | per-model | patch side in px; default = each model's recommended size (224 / 448 CONCH / 512 conch_v1_5). Pass to force one size for all | +| `--mpp` | per-model | target µm/px; default = each model's **recommended** MPP (0.5 for most pathology FMs, 1.0 for scale-agnostic). A value overrides the scale while retaining each model's recommended patch size; combine it with `--patch-size` to force one shared grid | +| `--source-mpp` | source metadata | level-0 source µm/px override for an uncalibrated OME-Zarr whose spatial axes declare no physical unit; normally leave unset | +| `--patch-size` | per-model | patch side in px; default = each model's recommended size (224 / 448 CONCH / 512 conch_v1_5). A value overrides the size; combine it with `--mpp` to force one shared grid | | `--step` | = patch | stride in output px; `< patch` gives overlap | | `--no-seg` | off | tile the whole slide (skip tissue masking) | | `--segmenter` | `otsu` | tissue segmenter to use | @@ -211,6 +213,18 @@ Otherwise the patch is resampled and `achieved_mpp == target`. `--snap-to-level` forces a native level read (faster, MPP = that level); `--allow-upsample` permits going finer than level 0. `raw2features info` prints the exact plan. +### Float source pixels + +OME-NGFF float arrays do not necessarily declare whether RGB values use a normalized +`[0, 1]` range or a byte-like `[0, 255]` range. The OME-Zarr reader therefore chooses +one convention when the slide is opened and reuses it for every patch and pyramid level; +it never rescales one patch differently because that patch happens to be locally dark. +The decision uses the array fill value and a deterministic 3×3 lattice of level-0 +chunks (up to the first three channels): any finite observed value above 1 selects the +byte-like convention, otherwise the slide is treated as normalized. This bounded sample +avoids scanning an entire WSI at open. `read_region_channels` remains native-dtype and +unscaled for multiplex models. + ### Segmentation The default `otsu` segmenter runs Otsu-on-saturation + morphology on a low-res @@ -222,8 +236,9 @@ are pluggable via the `segmenters` seam - see ### Multi-model fan-out (decode-once) -With several `-f` flags, each patch is **read and decoded once** and reused across -every model in the batch - the slow part (IO + decode) is not repeated per model. +With several `-m` flags (`-f` is an alias), each patch is **read and decoded once** +and reused across every model in the batch - the slow part (IO + decode) is not +repeated per model. Each model applies **its own** preprocessing (mean/std/input size, sourced from its model card), and writes into its own `features/` array. @@ -269,18 +284,23 @@ re-runs are **additive and idempotent - no `--receipts-dir` required**: - A requested model already present and output-validated against its current model fingerprint is **skipped**. - A **missing** model is embedded and **appended in place**: `features/` is - added next to the existing arrays, decoded once at the store's existing patch - coordinates. Existing models and `coords` are left untouched. -- If **all** requested models are already present, the run does nothing - (`status: skipped`). -- If a store exists but its **patch geometry differs** (different `--mpp`, - `--patch-size`, segmentation, …), `embed` **errors** rather than mixing - incompatible patches. Use `--force` (rebuild from scratch) or a new output dir. + added next to the existing arrays when it shares that grid's geometry. Existing + models and `coords` are left untouched. +- A requested geometry not already in the store is added as another + `grids//` group. A difference in `--mpp`, `--patch-size`, segmentation, or + another geometry field is **not** an error and does not wipe existing grids or + mix incompatible rows in one grid. +- If **all** requested models and auxiliary outputs are already present, no patch + features are recomputed (a model-only run returns `status: skipped`). + +Use `--force` only when you intentionally want to rebuild the whole store from +scratch. ```bash raw2features embed slide.zarr out/ -m uni # writes features/uni raw2features embed slide.zarr out/ -m virchow2 # ADDS features/virchow2; uni untouched raw2features embed slide.zarr out/ -m uni -m virchow2 # both present -> nothing to do +raw2features embed slide.zarr out/ -m conch_v1_5 # ADDS its 0.5/512 grid; other grids untouched ``` The shared extraction/storage grid is identified by a **`grid_hash`**, recorded in @@ -311,13 +331,15 @@ model set: ``` reader, models (order-independent), segmenter / --no-seg, --mpp, --patch-size, ---step, --tissue-threshold, --features-dtype, --snap-to-level, --mpp-tolerance, ---allow-upsample, --amp +--source-mpp (when set), --step, --tissue-threshold, --features-dtype, +--stain-norm, --snap-to-level, --mpp-tolerance, --allow-upsample, --amp ``` Runtime-only knobs do **not** change the hash: -`--device`, `--batch-size`, `--emit-geojson`, `--emit-thumbnail`, -`--thumbnail-mpp`, `--max-px`, `--output-zarr-format`, `--force`. +`--device`, `--devices`, `--batch-size`, `--read-workers`, `--read-block`, +`--compile`, `--emit-geojson`, `--emit-thumbnail`, `--thumbnail-mpp`, `--max-px`, +`--slide-encoder`, `--qc`, `--qc-stain-norm`, `--qc-artifact-mpp`, +`--output-zarr-format`, `--force`. `--force` is the exception for control flow: it deliberately bypasses skipping and rebuilds the target without changing the content identity. @@ -341,15 +363,20 @@ Pass `--force` to ignore an existing store and rebuild it from scratch (all requested models recomputed, the store overwritten). It also bypasses a valid receipt's fast path. Deleting the output store (and any receipt) has the same effect. -### Important: side-products are skipped too +### Auxiliary outputs are produce-if-missing -Thumbnails and geojson are written only on a **fresh build**, not when a model is -appended to an existing store (the patches and mask are unchanged, so there is -nothing new to draw). And a fully-complete slide is skipped before any work, so -re-running `embed --emit-thumbnail` (or `--emit-geojson`) on it produces -**nothing new**. To add a thumbnail to an already-embedded slide, use the -standalone [`thumbnail`](#8-thumbnail--previews--qc-overlays) command (geojson -emits only during a fresh `embed`). +An explicit `--emit-thumbnail`, `--emit-geojson`, `--qc`, or `-s/--slide-encoder` +request is checked against the matching grid even when its patch features are already +complete. Missing outputs are produced without re-embedding patches; existing valid +outputs are left alone. These runtime products do not change the patch-grid identity or +receipt content hash. The standalone [`thumbnail`](#8-thumbnail--previews--qc-overlays) +and `slide-embed` commands remain convenient when you only want to add those products. + +In a multi-grid invocation, a slide encoder runs on the grid containing its required +patch encoder (for example, TITAN uses the `conch_v1_5` grid). Model-agnostic pooling +encoders run on each requested grid. Standalone `slide-embed` can infer an unambiguous +grid from the required patch model; otherwise pass `--grid` (and `--patch-model` when +needed). --- @@ -420,6 +447,7 @@ rows are also rejected; a per-output lock across independent invocations is not ```bash cd raw2features && uv sync --extra zarr --extra image --extra torch --extra models +# Add any model-specific extras / pinned git packages from docs/MODELS.md. export SLIDE_DIR=/path/to/raw # dir of *.zarr stores (top-level only) export OUT_DIR=/path/to/embeddings export MODELS="uni resnet50" # space-separated; each becomes a -m flag @@ -427,7 +455,11 @@ export HF_TOKEN=hf_... # only if MODELS includes a gated mode bash slurm/preflight.sh # validates venv/arch, models/HF, slide count mkdir -p logs # SLURM opens --output here before the job runs -N=$(ls -d "$SLIDE_DIR"/*.zarr | wc -l) +shopt -s nullglob +SLIDES=("$SLIDE_DIR"/*.zarr) +shopt -u nullglob +N=${#SLIDES[@]} +(( N > 0 )) || { echo "no *.zarr slides found" >&2; exit 1; } sbatch --array=0-$((N-1))%64 --export=ALL,SLIDE_DIR,OUT_DIR,MODELS,HF_TOKEN slurm/embed_array.sbatch ``` @@ -464,12 +496,16 @@ Notes: .patches.geojson # optional (--emit-geojson): per-patch polygons for QuPath .thumbnail.png # optional (--emit-thumbnail / thumbnail cmd) .thumbnail.overlay.png # optional QC overlay +..patches.geojson # corresponding sidecar for each additional grid +..thumbnail.overlay.png # corresponding overlay for each additional grid ``` A store nests its patch sets under `grids//` (one child per extraction geometry - `mpp0.5_px224`, etc.); a one-geometry run is just a single grid. Open one grid with the reader helper (`raw2features.core.store.open_grid(store)` returns the sole grid, or pass a -key), and `export`/`info` default to the sole grid (a multi-grid store wants `--grid`). +key), and `export`/`info` default to the sole grid (a multi-grid store wants `--grid`). The +first-created grid keeps the backward-compatible unsuffixed GeoJSON/overlay names; later +grids use their key as shown above. The plain thumbnail is shared by the slide. **Spatial-provenance contract (within a grid).** Row `i` refers to the same patch across **every** array of that grid: `coords[i]`, `grid_index[i]`, and `features/[i]` are @@ -479,7 +515,8 @@ pixel extent), any embedding is invertible to the exact slide region **Each grid's `.zattrs["raw2features"]`** records the full, self-describing header: `schema_version`, `provenance` (version, CLI, git SHA, host, GPU), `thumbnail` (if emitted), -`source` (uri, ngff_version, reader, mpp_level0, level_dimensions, level_downsamples), +`source` (uri, ngff_version, reader, mpp_level0, dimensions/downsamples, source axes, +per-axis physical scale, and source translation/origin), `patching` (target/achieved MPP, patch_px, read_level, read_px, resample, level0_patch, level0_step, n_patches, grid_shape, `coords_convention: level0_xy`), `segmentation`, and per-model `models.` (source, embedding_dim, input_size, pooling, mean, std, @@ -490,8 +527,10 @@ models, grid_hash}`) for discovery without opening each grid. Read it back: ```python -import zarr -g = zarr.open_group("out/.embeddings.zarr", mode="r") +from raw2features.core.store import open_grid + +# Omit the key only when the store contains exactly one grid. +g = open_grid("out/.embeddings.zarr", key="mpp0.5_px224") print(g["features/uni"].shape) # (N, 1024) print(dict(g.attrs)["raw2features"]["patching"]["achieved_mpp"]) ``` @@ -500,17 +539,20 @@ print(dict(g.attrs)["raw2features"]["patching"]["achieved_mpp"]) ## 11. FAQ / gotchas -**`embed --emit-thumbnail` produced no thumbnail.** The slide was already complete -(receipt + matching config) so the run was skipped before any work. Use the -standalone `raw2features thumbnail … --overlay` to add thumbnails to -already-embedded slides, or force a re-run (§7). +**`embed --emit-thumbnail` produced no thumbnail.** A missing thumbnail should be +produced even when the patch arrays are complete, while an existing valid thumbnail is +left unchanged. Check the command's error and output path; in a multi-grid store, +non-primary overlays have the grid key in their filename. The standalone +`raw2features thumbnail … --overlay` command is also available when only a preview is +needed. -**Adding a model didn't append - it recomputed everything.** Correct: the model -set is part of the config hash, so a different set is a different run and the store -is rewritten. Run all the models you want in a single `embed` invocation. +**What happens when I add a model?** A model with the same extraction geometry is +appended to that grid. A model with a different geometry gets a new grid. Existing +valid arrays are preserved; only a missing, damaged, legacy-unfingerprinted, or +stale-fingerprint model output is recomputed. **`verify` always says incomplete.** It must be given the **same content-affecting -flags** as the `embed` that produced the store (same `-f` models, `--mpp`, +flags** as the `embed` that produced the store (same `-m` models, `--mpp`, `--patch-size`, etc.) so the config hash matches. Pass the same output directory as `--out-dir` so the receipt is also bound to the intended target store. diff --git a/notebooks/01_quickstart.ipynb b/notebooks/01_quickstart.ipynb index 5a06f58..cc4a766 100644 --- a/notebooks/01_quickstart.ipynb +++ b/notebooks/01_quickstart.ipynb @@ -43,7 +43,7 @@ "IN_COLAB = \"google.colab\" in sys.modules\n", "HAVE_PKG = importlib.util.find_spec(\"raw2features\") is not None\n", "EXTRAS = \"zarr,image,torch,models\"\n", - "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.0) for reproducibility\n", + "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.1) for reproducibility\n", "if IN_COLAB or not HAVE_PKG:\n", " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", SPEC])\n", "\n", diff --git a/notebooks/02_visual_walkthrough.ipynb b/notebooks/02_visual_walkthrough.ipynb index 8e37a76..bff7f6e 100644 --- a/notebooks/02_visual_walkthrough.ipynb +++ b/notebooks/02_visual_walkthrough.ipynb @@ -52,7 +52,7 @@ "HAVE_PKG = importlib.util.find_spec(\"raw2features\") is not None\n", "\n", "EXTRAS = \"zarr,image,torch,models\"\n", - "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.0) for reproducibility\n", + "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.1) for reproducibility\n", "if IN_COLAB or not HAVE_PKG:\n", " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", SPEC])\n", "\n", diff --git a/notebooks/03_the_embeddings_store.ipynb b/notebooks/03_the_embeddings_store.ipynb index a85892e..d58d89d 100644 --- a/notebooks/03_the_embeddings_store.ipynb +++ b/notebooks/03_the_embeddings_store.ipynb @@ -44,7 +44,7 @@ "IN_COLAB = \"google.colab\" in sys.modules\n", "HAVE_PKG = importlib.util.find_spec(\"raw2features\") is not None\n", "EXTRAS = \"zarr,image,torch,models\"\n", - "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.0) for reproducibility\n", + "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.1) for reproducibility\n", "if IN_COLAB or not HAVE_PKG:\n", " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", SPEC])\n", "\n", diff --git a/notebooks/04_spatialdata.ipynb b/notebooks/04_spatialdata.ipynb index bd18a3d..00684d2 100644 --- a/notebooks/04_spatialdata.ipynb +++ b/notebooks/04_spatialdata.ipynb @@ -53,9 +53,9 @@ "HAVE_PKG = importlib.util.find_spec(\"raw2features\") is not None\n", "\n", "# SpatialData export needs the [spatialdata] extra (spatialdata + geopandas) on top of\n", - "# the resnet50 stack. After a PyPI release prefer a hard pin: raw2features[...]==0.1.0\n", + "# the resnet50 stack. After a PyPI release prefer a hard pin: raw2features[...]==0.1.1\n", "EXTRAS = \"zarr,image,torch,models,spatialdata\"\n", - "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.0) for reproducibility\n", + "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.1) for reproducibility\n", "\n", "if IN_COLAB or not HAVE_PKG:\n", " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", SPEC])\n", diff --git a/notebooks/05_quality_control.ipynb b/notebooks/05_quality_control.ipynb index a237d2d..38b91e2 100644 --- a/notebooks/05_quality_control.ipynb +++ b/notebooks/05_quality_control.ipynb @@ -51,7 +51,7 @@ "IN_COLAB = \"google.colab\" in sys.modules\n", "HAVE_PKG = importlib.util.find_spec(\"raw2features\") is not None\n", "EXTRAS = \"zarr,image,torch,models,grandqc\"\n", - "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.0) for reproducibility\n", + "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.1) for reproducibility\n", "if IN_COLAB or not HAVE_PKG:\n", " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", SPEC])\n", "\n", diff --git a/notebooks/06_cohort_quality_control.ipynb b/notebooks/06_cohort_quality_control.ipynb index bbab502..059fd2e 100644 --- a/notebooks/06_cohort_quality_control.ipynb +++ b/notebooks/06_cohort_quality_control.ipynb @@ -46,7 +46,7 @@ "IN_COLAB = \"google.colab\" in sys.modules\n", "HAVE_PKG = importlib.util.find_spec(\"raw2features\") is not None\n", "EXTRAS = \"zarr,image,torch,models,grandqc\"\n", - "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.0) for reproducibility\n", + "SPEC = f\"raw2features[{EXTRAS}]\" # PyPI; pin a version (==0.1.1) for reproducibility\n", "if IN_COLAB or not HAVE_PKG:\n", " subprocess.check_call([sys.executable, \"-m\", \"pip\", \"install\", \"-q\", SPEC])\n", "\n", diff --git a/pyproject.toml b/pyproject.toml index afbf0ea..cb23d46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "raw2features" -version = "0.1.0" +version = "0.1.1" description = "Read OME-Zarr whole-slide images and emit patch- and slide-level foundation-model embeddings - storage backend and model independently swappable." readme = "README.md" authors = [{ name = "Craig Myles", email = "me@craig.im" }] @@ -25,6 +25,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering :: Bio-Informatics", "Topic :: Scientific/Engineering :: Image Processing", "Topic :: Scientific/Engineering :: Medical Science Apps.", diff --git a/slurm/README.md b/slurm/README.md index f7cc526..dc6b283 100644 --- a/slurm/README.md +++ b/slurm/README.md @@ -8,12 +8,16 @@ per slide (or per batch of slides) - picking up where it left off if anything is You don't need to be a SLURM expert. You set a few environment variables (where the slides are, where output goes, which models), adjust the `#SBATCH` lines at the top of a script to match your cluster (partition, GPU, time limit), and submit it. Everything else is handled. +Install any model-specific extras or pinned git packages listed in +[`docs/MODELS.md`](../docs/MODELS.md) before preflight; the check validates model names and +HuggingFace authentication, but deliberately does not download every model on the login +node. ## Which script do I use? | Script | What it does | Use it when | |--------|--------------|-------------| -| **`preflight.sh`** | A read-only sanity check, run on the **login node**: is the venv working, do the slides exist, is HuggingFace access OK? | **Always run this first** - it catches a misconfiguration before you spend GPU hours discovering it. | +| **`preflight.sh`** | A read-only sanity check, run on the **login node**: is the venv working, do the slides exist, are model names known, and is HuggingFace authentication present when needed? | **Always run this first** - it catches a misconfiguration before you spend GPU hours discovering it. It cannot confirm that every model gate has been accepted. | | **`embed_array.sbatch`** | One cluster job **per slide**. Simplest mapping; one log file per slide. | The default. Small-to-medium cohorts. | | **`embed_cohort.sbatch`** | One cluster job **per batch ("shard") of slides**, loading the models **once** per job instead of once per slide. | Large cohorts, where re-loading the model for every slide would waste time. | @@ -36,7 +40,11 @@ bash slurm/preflight.sh # 3. Submit one job per slide (array size = number of slides). mkdir -p logs -N=$(ls -d "$SLIDE_DIR"/*.zarr | wc -l) +shopt -s nullglob +SLIDES=("$SLIDE_DIR"/*.zarr) +shopt -u nullglob +N=${#SLIDES[@]} +(( N > 0 )) || { echo "no *.zarr slides found" >&2; exit 1; } sbatch --array=0-$((N-1))%64 \ --export=ALL,SLIDE_DIR,OUT_DIR,MODELS,HF_TOKEN \ slurm/embed_array.sbatch @@ -52,7 +60,14 @@ top of each script. The ones you're most likely to touch: - `MODELS` - which encoders to run (space-separated). - `MPP` - extraction scale; leave blank to use each model's recommended scale. -- `AMP` - precision (`bf16` is a good default on modern GPUs; `auto` uses each model's card). +- `SOURCE_MPP` - level-0 source calibration for OME-Zarr inputs whose spatial axes + declare no physical unit; leave blank for normally calibrated sources. The array + script passes it identically to `verify` and `embed`. +- `PATCH` - patch side; leave blank to use each model's recommended size. Set it only + when you intentionally want one size for every model. +- `STEP` - stride; leave blank for one patch-width (no overlap). +- `AMP` - precision; keep `auto` to use each model's declared inference precision. Set + `bf16` explicitly only when the selected models and GPUs support it. - `BATCH_SIZE` - lower it if a GPU runs out of memory. - `READ_BLOCK` - read patches in NxN blocks; bigger means fewer, larger reads, which helps on a slow shared filesystem (try `16`) at the cost of host RAM. diff --git a/slurm/embed_array.sbatch b/slurm/embed_array.sbatch index bc55787..9c8df13 100755 --- a/slurm/embed_array.sbatch +++ b/slurm/embed_array.sbatch @@ -23,7 +23,11 @@ # export MODELS="uni resnet50" # gated models need HF_TOKEN exported # export HF_TOKEN=hf_... # only if MODELS includes a gated one # mkdir -p logs -# N=$(ls -d "$SLIDE_DIR"/*.zarr 2>/dev/null | wc -l) +# shopt -s nullglob +# SLIDES=("$SLIDE_DIR"/*.zarr) +# shopt -u nullglob +# N=${#SLIDES[@]} +# (( N > 0 )) || { echo "no *.zarr slides found" >&2; exit 1; } # sbatch --array=0-$((N-1))%64 \ # --export=ALL,SLIDE_DIR,OUT_DIR,MODELS,HF_TOKEN \ # slurm/embed_array.sbatch @@ -52,12 +56,13 @@ export LC_COLLATE=C : "${RECEIPTS_DIR:=$OUT_DIR/receipts}" : "${REPO_DIR:=$HOME/raw2features}" : "${VENV:=$REPO_DIR/.venv}" -: "${MODELS:=uni resnet50}" # space-separated; each becomes a -f flag +: "${MODELS:=uni resnet50}" # space-separated; each becomes a -m flag : "${MPP:=}" # empty => auto (each model's recommended MPP) -: "${PATCH:=224}" +: "${SOURCE_MPP:=}" # level-0 MPP override for uncalibrated sources +: "${PATCH:=}" # empty => each model's recommended patch size : "${STEP:=}" # empty => no overlap (step == patch) : "${TISSUE_THRESHOLD:=0.1}" -: "${AMP:=bf16}" +: "${AMP:=auto}" # each model's declared inference precision : "${BATCH_SIZE:=256}" : "${READ_BLOCK:=8}" # read patches in NxN blocks (1=off); bump to 16 on a latency-bound shared FS : "${EMIT_GEOJSON:=0}" # 1 => also write .patches.geojson @@ -91,28 +96,29 @@ set +u source "$VENV/bin/activate" set -u -# ---- repeated -f flags + the content-affecting flags shared by verify+embed +# ---- repeated -m flags + the content-affecting flags shared by verify+embed # These MUST be identical for `verify` and `embed` so the config hash matches. MODEL_FLAGS=() -for m in $MODELS; do MODEL_FLAGS+=(-f "$m"); done - -# Only pin --mpp when MPP is set; empty => let the CLI auto-resolve the model's -# recommended MPP (identical for verify + embed, so the config hash still matches). -MPP_FLAG=() -[ -n "$MPP" ] && MPP_FLAG=(--mpp "$MPP") +for m in $MODELS; do MODEL_FLAGS+=(-m "$m"); done COMMON=( "${MODEL_FLAGS[@]}" --reader omezarr - "${MPP_FLAG[@]}" - --patch-size "$PATCH" --tissue-threshold "$TISSUE_THRESHOLD" --amp "$AMP" ) +# Add optional geometry overrides directly to COMMON. Avoid expanding empty arrays: +# Bash 3.2 treats those as unbound under `set -u` (still common on login nodes). +[[ -n "$MPP" ]] && COMMON+=(--mpp "$MPP") +[[ -n "$SOURCE_MPP" ]] && COMMON+=(--source-mpp "$SOURCE_MPP") +[[ -n "$PATCH" ]] && COMMON+=(--patch-size "$PATCH") [[ -n "$STEP" ]] && COMMON+=(--step "$STEP") # ---- skip-if-complete (validates the actual output, not just the receipt) - -if raw2features verify "$SLIDE" --receipts-dir "$RECEIPTS_DIR" --out-dir "$OUT_DIR" "${COMMON[@]}" --quiet; then +# An explicit auxiliary-output request must reach `embed`: its own completion path +# produces a missing GeoJSON without re-embedding otherwise-complete patch arrays. +if [[ "$EMIT_GEOJSON" != "1" ]] && \ + raw2features verify "$SLIDE" --receipts-dir "$RECEIPTS_DIR" --out-dir "$OUT_DIR" "${COMMON[@]}" --quiet; then echo "[$(date -u +%FT%TZ)] already complete; skipping ${SLIDE}" exit 0 fi diff --git a/slurm/embed_cohort.sbatch b/slurm/embed_cohort.sbatch index c4c82b3..cf03357 100644 --- a/slurm/embed_cohort.sbatch +++ b/slurm/embed_cohort.sbatch @@ -31,16 +31,18 @@ set -euo pipefail : "${SLIDE_DIR:?set SLIDE_DIR to the directory holding the slide stores}" : "${OUT_DIR:?set OUT_DIR for the *.embeddings.zarr outputs}" -: "${NUM_SHARDS:?set NUM_SHARDS = the array size (0-$((NUM_SHARDS-1)))}" +: "${NUM_SHARDS:?set NUM_SHARDS to the submitted array size}" : "${GLOB:=*.zarr}" : "${RECEIPTS_DIR:=$OUT_DIR/receipts}" : "${REPO_DIR:=$HOME/raw2features}" : "${VENV:=$REPO_DIR/.venv}" : "${MODELS:=uni resnet50}" : "${MPP:=}" # empty => auto (each model's recommended MPP) -: "${PATCH:=224}" +: "${SOURCE_MPP:=}" # level-0 MPP override for uncalibrated sources +: "${PATCH:=}" # empty => each model's recommended patch size +: "${STEP:=}" # empty => no overlap (step == patch) : "${TISSUE_THRESHOLD:=0.1}" -: "${AMP:=auto}" # auto = each model's card precision (fp32 for current models) +: "${AMP:=auto}" # each model's declared inference precision : "${READ_WORKERS:=16}" # 16 suits a latency-bound shared FS; 8 on local NVMe : "${READ_BLOCK:=8}" # read patches in NxN blocks (1=off); 16 helps on a latency-bound shared FS, costs host RAM : "${BATCH_SIZE:=256}" @@ -53,24 +55,25 @@ source "$VENV/bin/activate" set -u MODEL_FLAGS=() -for m in $MODELS; do MODEL_FLAGS+=(-f "$m"); done - -# Only pin --mpp when MPP is set; empty => auto-resolve each model's recommended MPP. -MPP_FLAG=() -[ -n "$MPP" ] && MPP_FLAG=(--mpp "$MPP") +for m in $MODELS; do MODEL_FLAGS+=(-m "$m"); done echo "[$(date -u +%FT%TZ)] shard ${SLURM_ARRAY_TASK_ID:-0}/${NUM_SHARDS} on $(hostname)" -srun raw2features embed-many "$SLIDE_DIR" "$OUT_DIR" \ - --glob "$GLOB" \ - "${MODEL_FLAGS[@]}" \ - "${MPP_FLAG[@]}" \ - --patch-size "$PATCH" \ - --tissue-threshold "$TISSUE_THRESHOLD" \ - --amp "$AMP" \ - --read-workers "$READ_WORKERS" \ - --read-block "$READ_BLOCK" \ - --batch-size "$BATCH_SIZE" \ - --receipts-dir "$RECEIPTS_DIR" \ - --shard-index "${SLURM_ARRAY_TASK_ID:-0}" \ +CMD=( + raw2features embed-many "$SLIDE_DIR" "$OUT_DIR" + --glob "$GLOB" + "${MODEL_FLAGS[@]}" + --tissue-threshold "$TISSUE_THRESHOLD" + --amp "$AMP" + --read-workers "$READ_WORKERS" + --read-block "$READ_BLOCK" + --batch-size "$BATCH_SIZE" + --receipts-dir "$RECEIPTS_DIR" + --shard-index "${SLURM_ARRAY_TASK_ID:-0}" --num-shards "$NUM_SHARDS" +) +[[ -n "$MPP" ]] && CMD+=(--mpp "$MPP") +[[ -n "$SOURCE_MPP" ]] && CMD+=(--source-mpp "$SOURCE_MPP") +[[ -n "$PATCH" ]] && CMD+=(--patch-size "$PATCH") +[[ -n "$STEP" ]] && CMD+=(--step "$STEP") +srun "${CMD[@]}" echo "[$(date -u +%FT%TZ)] shard ${SLURM_ARRAY_TASK_ID:-0} finished" diff --git a/slurm/preflight.sh b/slurm/preflight.sh index 3c02caa..69d350b 100755 --- a/slurm/preflight.sh +++ b/slurm/preflight.sh @@ -1,7 +1,7 @@ #!/bin/bash # raw2features SLURM pre-flight: run this on your cluster's LOGIN node before # submitting embed_array.sbatch. It validates the environment so you don't burn -# a whole array on a misconfiguration (wrong-arch venv, missing HF access, empty +# a whole array on a misconfiguration (wrong-arch venv, missing HF login, empty # slide dir, ...). Read-only except for `mkdir -p` of OUT_DIR/logs. # # New here? slurm/README.md explains the cluster workflow + which script to use. @@ -56,27 +56,31 @@ else fi echo "[3] models ($MODELS) + HuggingFace access" -if command -v python >/dev/null 2>&1; then - GATED=$(python - </dev/null +if ! command -v python >/dev/null 2>&1; then + bad "python is unavailable after venv activation" +elif GATED=$(R2F_PREFLIGHT_MODELS="$MODELS" python - <<'PY' 2>/dev/null +import os from raw2features.embedders.model_registry import load_registry -reg = load_registry(); req = "$MODELS".split() +reg = load_registry(); req = os.environ["R2F_PREFLIGHT_MODELS"].split() unknown = [m for m in req if m not in reg] gated = [m for m in req if m in reg and reg[m].gated] print("UNKNOWN " + " ".join(unknown)) print("GATED " + " ".join(gated)) PY -) +); then UNK=$(sed -n 's/^UNKNOWN //p' <<<"$GATED"); GAT=$(sed -n 's/^GATED //p' <<<"$GATED") [[ -n "${UNK// }" ]] && bad "unknown model(s): $UNK (see 'raw2features models')" || ok "all models known" if [[ -n "${GAT// }" ]]; then if WHO=$(python -c "from huggingface_hub import whoami; print(whoami()['name'])" 2>/dev/null); then - ok "gated models [$GAT] -> HF authenticated as '$WHO' (ensure access is granted on the model pages)" + ok "gated models [$GAT] -> HF authenticated as '$WHO' (authentication only; ensure each model gate is accepted)" else - bad "gated models [$GAT] but no HF login -- export HF_TOKEN (and pass it via --export) or 'huggingface-cli login'" + bad "gated models [$GAT] but no HF login -- export HF_TOKEN (and pass it via --export) or run 'hf auth login'" fi else ok "no gated models requested (no HF token needed)" fi +else + bad "could not load the raw2features model registry in this environment" fi echo "[4] output + logs" diff --git a/src/raw2features/__init__.py b/src/raw2features/__init__.py index deea934..29e2df4 100644 --- a/src/raw2features/__init__.py +++ b/src/raw2features/__init__.py @@ -1,15 +1,18 @@ """raw2features - read OME-Zarr WSIs and emit patch-level FM embeddings. -The storage backend (reader) and the model (embedder) are independently -swappable via six plugin seams: readers, segmenters, patchers, embedders, sinks, -slide_embedders (see ``raw2features.core.plugins.SEAMS``). Third parties add a -backend or model by shipping a package that registers an entry-point in the -matching ``raw2features.`` group - no fork required. Those group names are -the public plugin contract. +The storage backend (reader) and model implementation are independently swappable +via six plugin seams: readers, segmenters, patchers, embedders, sinks, and +slide_embedders (see ``raw2features.core.plugins.SEAMS``). Third parties add an +implementation through the matching ``raw2features.`` entry-point group. +For embedders, that entry point is a loader family used by the packaged model +registry; it does not add a new CLI ``--model`` name. Programmatic callers can pass +external patch-model instances through ``embed_slide(..., embedders=[...])``. +Those group names and the explicit ``embedders=`` hook are the public plugin contract. Public API (stable import surface - couple to these, not to deep module paths): -* ``run_slide(slide_path, out_dir, cfg)`` / ``RunConfig`` - embed one slide. +* ``embed_slide(slide_path, out_dir, cfg)`` - high-level per-model geometry runner. +* ``run_slide(slide_path, out_dir, cfg)`` / ``RunConfig`` - single-grid primitive. * ``register`` / ``available`` / ``get`` - the plugin registry. * ``write_patches_geojson`` - patch grid -> QuPath-readable GeoJSON. * ``validate_store`` - conformance-check an embeddings store against docs/SPEC.md. @@ -22,10 +25,11 @@ from typing import TYPE_CHECKING -__version__ = "0.1.0" +__version__ = "0.1.1" __all__ = [ "__version__", + "embed_slide", "run_slide", "RunConfig", "register", @@ -37,14 +41,14 @@ if TYPE_CHECKING: # import-time only for type checkers; never imported at runtime from raw2features.core.plugins import available, get, register - from raw2features.pipeline.runner import RunConfig, run_slide + from raw2features.pipeline.runner import RunConfig, embed_slide, run_slide from raw2features.sinks.zarr_sink import write_patches_geojson from raw2features.spec import validate_store def __getattr__(name: str): """Resolve a blessed public name on first access (PEP 562 lazy re-export).""" - if name in ("run_slide", "RunConfig"): + if name in ("embed_slide", "run_slide", "RunConfig"): from raw2features.pipeline import runner return getattr(runner, name) diff --git a/src/raw2features/cli/_validation.py b/src/raw2features/cli/_validation.py new file mode 100644 index 0000000..72921b3 --- /dev/null +++ b/src/raw2features/cli/_validation.py @@ -0,0 +1,55 @@ +"""Shared validation for content/runtime options exposed by several commands.""" + +from __future__ import annotations + +import math + +import typer + +AMP_CHOICES = ("auto", "fp32", "bf16", "fp16") + + +def validate_amp(value: str) -> None: + """Reject an unknown precision before model discovery or output mutation.""" + + if value not in AMP_CHOICES: + choices = ", ".join(AMP_CHOICES) + raise typer.BadParameter(f"must be one of: {choices}", param_hint="--amp") + + +def validate_batch_size(value: int) -> None: + """Reject empty/negative batches before they reach the embedding loop.""" + + if value <= 0: + raise typer.BadParameter("must be greater than zero", param_hint="--batch-size") + + +def validate_positive_float(value: float | None, param_hint: str) -> None: + """Reject zero, negative, NaN, and infinite physical scales.""" + + if value is not None and (not math.isfinite(value) or value <= 0): + raise typer.BadParameter( + "must be finite and greater than zero", param_hint=param_hint + ) + + +def validate_positive_int(value: int | None, param_hint: str) -> None: + """Reject zero or negative pixel sizes/strides.""" + + if value is not None and value <= 0: + raise typer.BadParameter("must be greater than zero", param_hint=param_hint) + + +def validate_geometry( + *, + mpp: float | None = None, + patch_size: int | None = None, + step: int | None = None, + source_mpp: float | None = None, +) -> None: + """Validate shared CLI geometry before discovery, loading, or output mutation.""" + + validate_positive_float(mpp, "--mpp") + validate_positive_int(patch_size, "--patch-size") + validate_positive_int(step, "--step") + validate_positive_float(source_mpp, "--source-mpp") diff --git a/src/raw2features/cli/benchmark.py b/src/raw2features/cli/benchmark.py index 18ae059..9992424 100644 --- a/src/raw2features/cli/benchmark.py +++ b/src/raw2features/cli/benchmark.py @@ -26,6 +26,8 @@ slide_id_from_path, ) +from ._validation import validate_amp, validate_batch_size, validate_geometry + def _print_run(slide_id: str, rep: dict, idx: int) -> None: typer.echo( @@ -43,11 +45,16 @@ def _print_run(slide_id: str, rep: dict, idx: int) -> None: def benchmark( slides: list[str] = typer.Argument(..., help="OME-Zarr slide path(s)."), feature_extractor: list[str] = typer.Option( - ["resnet50"], "--model", "-m", "--feature-extractor", "-f", - help="Model(s); repeatable. (--feature-extractor/-f are aliases.)" + ["resnet50"], + "--model", + "-m", + "--feature-extractor", + "-f", + help="Model(s); repeatable. (--feature-extractor/-f are aliases.)", ), mpp: float | None = typer.Option( - None, "--mpp", + None, + "--mpp", help="Target µm/px; default = the model's recommended scale (like embed), so " "the benchmark profiles at the same scale production runs at.", ), @@ -76,8 +83,10 @@ def benchmark( 8, "--read-workers", help="Concurrent patch-decode threads (read parallelism)." ), read_block: int = typer.Option( - 1, "--read-block", help="Read patches in NxN blocks (1=off). Bigger helps " - "remote/slow storage, costs RAM; bit-identical. Try 8 local / 16 remote." + 1, + "--read-block", + help="Read patches in NxN blocks (1=off). Bigger helps " + "remote/slow storage, costs RAM; bit-identical. Try 8 local / 16 remote.", ), compile: bool = typer.Option( False, @@ -97,6 +106,9 @@ def benchmark( hf_token: str | None = typer.Option(None, "--hf-token", envvar="HF_TOKEN"), ) -> None: """Profile the embed pipeline and report per-stage timing + throughput.""" + validate_amp(amp) + validate_batch_size(batch_size) + validate_geometry(mpp=mpp, patch_size=patch_size) if hf_token: os.environ["HF_TOKEN"] = hf_token os.environ["HUGGING_FACE_HUB_TOKEN"] = hf_token diff --git a/src/raw2features/cli/embed.py b/src/raw2features/cli/embed.py index 5cba14a..ae5a444 100644 --- a/src/raw2features/cli/embed.py +++ b/src/raw2features/cli/embed.py @@ -11,6 +11,8 @@ from raw2features.pipeline.runner import RunConfig, embed_slide from raw2features.viz import DEFAULT_THUMBNAIL_MPP +from ._validation import validate_amp, validate_batch_size, validate_geometry + def embed( slide: str = typer.Argument(..., help="Path to an OME-Zarr store."), @@ -18,8 +20,12 @@ def embed( ..., help="Output directory for the embeddings zarr." ), feature_extractor: list[str] = typer.Option( - ["resnet50"], "--model", "-m", "--feature-extractor", "-f", - help="Model(s); repeatable. (--feature-extractor/-f are aliases.)" + ["resnet50"], + "--model", + "-m", + "--feature-extractor", + "-f", + help="Model(s); repeatable. (--feature-extractor/-f are aliases.)", ), mpp: float | None = typer.Option( None, @@ -122,7 +128,7 @@ def embed( [], "--qc", help="Per-patch QC producer(s) writing grids//qc// (e.g. grandqc). " - "Needs the producer's extra, e.g. pip install \"raw2features[grandqc]\".", + 'Needs the producer\'s extra, e.g. pip install "raw2features[grandqc]".', ), qc_stain_norm: str | None = typer.Option( None, @@ -157,6 +163,11 @@ def embed( store in place and skips models already computed (no WSI re-read for them). Use ``--force`` to overwrite the store instead. """ + validate_amp(amp) + validate_batch_size(batch_size) + validate_geometry( + mpp=mpp, patch_size=patch_size, step=step, source_mpp=source_mpp + ) if hf_token: os.environ["HF_TOKEN"] = hf_token os.environ["HUGGING_FACE_HUB_TOKEN"] = hf_token @@ -175,6 +186,8 @@ def embed( # Resolve and preview the per-model extraction geometry (one grid per group). groups = resolve_geometry(models, mpp, patch_size, geometry_config) + if not groups: + raise typer.BadParameter("at least one model/extraction is required") if len(groups) > 1: plan = " · ".join( f"{','.join(g.models)} -> {g.mpp:g}/{g.patch_px}" for g in groups @@ -186,14 +199,18 @@ def embed( f"geometry: {','.join(g0.models)} -> {g0.mpp:g}/{g0.patch_px} ({g0.source})" ) + # RunConfig carries one concrete grid geometry for the run_slide primitive. + # embed_slide replaces it for every resolved group, so seed it with a real group + # rather than treating an otherwise meaningful 1.0/224 geometry as a sentinel. + representative = groups[0] cfg = RunConfig( models=models, reader=reader, segmenter=segmenter, no_seg=no_seg, - target_mpp=mpp if mpp is not None else 1.0, # per-group geometry overrides this + target_mpp=representative.mpp, source_mpp=source_mpp, - patch_px=patch_size if patch_size is not None else 224, + patch_px=representative.patch_px, step_px=step, tissue_threshold=tissue_threshold, features_dtype=features_dtype, diff --git a/src/raw2features/cli/embed_many.py b/src/raw2features/cli/embed_many.py index 4eacb01..43c5937 100644 --- a/src/raw2features/cli/embed_many.py +++ b/src/raw2features/cli/embed_many.py @@ -15,6 +15,7 @@ from __future__ import annotations import glob as globmod +import math import os import sys import time @@ -35,6 +36,8 @@ slide_id_from_path, ) +from ._validation import validate_amp, validate_batch_size, validate_geometry + class _WorkerStartupError(RuntimeError): """One or more slide-parallel workers could not load their models.""" @@ -54,8 +57,12 @@ def embed_many( "*.zarr", "--glob", help="Top-level glob for slides (never recurses)." ), feature_extractor: list[str] = typer.Option( - ["resnet50"], "--model", "-m", "--feature-extractor", "-f", - help="Model(s); repeatable. (--feature-extractor/-f are aliases.)" + ["resnet50"], + "--model", + "-m", + "--feature-extractor", + "-f", + help="Model(s); repeatable. (--feature-extractor/-f are aliases.)", ), mpp: float | None = typer.Option( None, @@ -81,22 +88,27 @@ def embed_many( tissue_threshold: float = typer.Option(0.1, "--tissue-threshold"), features_dtype: str = typer.Option("float16", "--features-dtype"), stain_norm: str | None = typer.Option( - None, "--stain-norm", + None, + "--stain-norm", help="Stain-normalize each patch before embedding (macenko|reinhard|vahadane).", ), slide_encoder: list[str] = typer.Option( - [], "--slide-encoder", "-s", + [], + "--slide-encoder", + "-s", help="Slide-level encoder(s) to run after patch embedding (e.g. titan).", ), qc: list[str] = typer.Option( [], "--qc", help="Per-patch QC producer(s) writing qc// (e.g. grandqc)." ), qc_stain_norm: str | None = typer.Option( - None, "--qc-stain-norm", + None, + "--qc-stain-norm", help="Normalize the QC input first (macenko|reinhard|vahadane).", ), qc_artifact_mpp: str = typer.Option( - "1.5", "--qc-artifact-mpp", + "1.5", + "--qc-artifact-mpp", help="GrandQC artifact µm/px (1.0|1.5|2.0); coarser = less focus-sensitive.", ), device: str = typer.Option( @@ -122,7 +134,8 @@ def embed_many( help="auto | fp32 | bf16 | fp16 (auto = each model's card precision)", ), read_workers: int = typer.Option( - 8, "--read-workers", + 8, + "--read-workers", help="Concurrent patch-decode threads (16 suits a shared/parallel FS).", ), read_block: int = typer.Option( @@ -151,17 +164,24 @@ def embed_many( receipts_dir: str | None = typer.Option(None, "--receipts-dir"), force: bool = typer.Option(False, "--force"), config: str | None = typer.Option( - None, "--config", + None, + "--config", help="YAML extraction plan ('extractions:' list of {model, mpp?, patch_px?}).", ), manifest: str | None = typer.Option( - None, "--manifest", + None, + "--manifest", help="CSV of slides (path[,source_mpp]) instead of globbing slide_dir; " "relative paths resolve against slide_dir.", ), hf_token: str | None = typer.Option(None, "--hf-token", envvar="HF_TOKEN"), ) -> None: """Warm worker: build the models once and embed this shard's slides.""" + validate_amp(amp) + validate_batch_size(batch_size) + validate_geometry( + mpp=mpp, patch_size=patch_size, step=step, source_mpp=source_mpp + ) from raw2features.slide_embedders.model_registry import ( validate_slide_encoder_names, ) @@ -186,7 +206,13 @@ def embed_many( if manifest: from raw2features.core.config import load_manifest - rows = load_manifest(manifest) + try: + rows = load_manifest(manifest) + except (OSError, ValueError) as exc: + typer.echo( + redact_uri_credentials(f"Error: invalid manifest: {exc}"), err=True + ) + raise typer.Exit(1) from exc try: all_rows = _resolve_manifest_sources(rows, slide_dir) except Exception as exc: # noqa: BLE001 - do not echo malformed credentials @@ -203,6 +229,11 @@ def embed_many( where = manifest or os.path.join(slide_dir, glob) typer.echo(f"Error: no slides found ({where})", err=True) raise typer.Exit(1) + try: + _validate_source_mpps(all_rows) + except ValueError as exc: + typer.echo(f"Error: {exc}", err=True) + raise typer.Exit(1) from exc try: _validate_unique_output_ids(all_rows) except ValueError as exc: @@ -226,14 +257,21 @@ def embed_many( else: models = list(feature_extractor) + from raw2features.embedders.model_registry import resolve_geometry + + groups = resolve_geometry(models, mpp, patch_size, geometry_config) + if not groups: + raise typer.BadParameter("at least one model/extraction is required") + representative = groups[0] + cfg = RunConfig( models=models, reader=reader, segmenter=segmenter, no_seg=no_seg, - target_mpp=mpp if mpp is not None else 1.0, # per-group geometry overrides this + target_mpp=representative.mpp, source_mpp=source_mpp, - patch_px=patch_size if patch_size is not None else 224, + patch_px=representative.patch_px, step_px=step, tissue_threshold=tissue_threshold, features_dtype=features_dtype, @@ -260,13 +298,28 @@ def embed_many( try: if len(device_list) > 1: done, skipped, failed = _embed_shard_parallel( - shard, out_dir, cfg, device_list, receipts_dir, cli, force, - mpp, patch_size, geometry_config, + shard, + out_dir, + cfg, + device_list, + receipts_dir, + cli, + force, + mpp, + patch_size, + geometry_config, ) else: done, skipped, failed = _embed_shard_serial( - shard, out_dir, cfg, receipts_dir, cli, force, - mpp, patch_size, geometry_config, + shard, + out_dir, + cfg, + receipts_dir, + cli, + force, + mpp, + patch_size, + geometry_config, ) except _WorkerStartupError as exc: typer.echo(f"Error: {exc}", err=True) @@ -338,6 +391,26 @@ def _validate_unique_output_ids(rows: list[dict]) -> None: raise ValueError("\n".join(lines)) +def _validate_source_mpps(rows: list[dict]) -> None: + """Validate every per-slide calibration before sharding or loading models.""" + + for index, row in enumerate(rows, 1): + if "source_mpp" not in row: + continue + try: + value = float(row["source_mpp"]) + except (TypeError, ValueError) as exc: + raise ValueError( + f"manifest row {index} source_mpp must be a finite number greater " + "than zero" + ) from exc + if not math.isfinite(value) or value <= 0: + raise ValueError( + f"manifest row {index} source_mpp must be a finite number greater " + "than zero" + ) + + def _with_source_mpp(cfg: RunConfig, row: dict) -> RunConfig: """Apply a manifest row's per-slide ``source_mpp`` override (else cfg unchanged).""" if "source_mpp" in row: diff --git a/src/raw2features/cli/slide_embed.py b/src/raw2features/cli/slide_embed.py index d070012..256ecfc 100644 --- a/src/raw2features/cli/slide_embed.py +++ b/src/raw2features/cli/slide_embed.py @@ -67,15 +67,13 @@ def slide_embed( import zarr from raw2features.core.device import resolve_device - from raw2features.core.store import grid_for_model, grid_keys, open_grid from raw2features.slide_embedders.encoding import ( encode_slide_embedding, - resolve_slide_patch_model, + resolve_slide_grid, slide_embedding_is_complete, write_slide_embedding, ) from raw2features.slide_embedders.model_registry import ( - get_slide_spec, validate_slide_encoder_names, ) @@ -103,44 +101,13 @@ def slide_embed( # consolidated metadata predates any slide/ group we add here. Reading the # live metadata avoids stale-key KeyErrors; we re-consolidate at the end. root = zarr.open_group(path, mode="r+", use_consolidated=False) - keys = grid_keys(root) - if not keys: - typer.echo( - "Error: store has no grids/ group; expected a v0.1 embeddings store.", - err=True, - ) - raise typer.Exit(1) for slide_model_name in slide_encoder: try: - if grid is not None: - if grid not in keys: - raise ValueError( - f"Unknown grid {grid!r}. Available grid keys: {keys}" - ) - selected_grid = grid - elif len(keys) == 1: - selected_grid = keys[0] - elif patch_model is not None: - selected_grid = grid_for_model(root, patch_model) - else: - required = get_slide_spec(slide_model_name).patch_encoder - if required == "any": - raise ValueError( - f"Slide encoder {slide_model_name!r} is model-agnostic and " - f"the store has multiple grids {keys}; pass --grid or " - "--patch-model to choose one." - ) - selected_grid = grid_for_model(root, required) - group = open_grid(root, selected_grid) - except (KeyError, ValueError) as exc: - typer.echo(f"Error: {exc}", err=True) - raise typer.Exit(1) from exc - - try: - selected_patch_model = resolve_slide_patch_model( - group, + selected_grid, group, selected_patch_model = resolve_slide_grid( + root, slide_model_name, + grid=grid, patch_model=patch_model, ) except (KeyError, ValueError) as exc: diff --git a/src/raw2features/cli/verify.py b/src/raw2features/cli/verify.py index 7b8453a..83cd409 100644 --- a/src/raw2features/cli/verify.py +++ b/src/raw2features/cli/verify.py @@ -20,6 +20,8 @@ slide_id_from_path, ) +from ._validation import validate_amp, validate_geometry + def verify( slide: str = typer.Argument(...), @@ -35,6 +37,11 @@ def verify( mpp: float | None = typer.Option( None, "--mpp", help="Target µm/px (default: model recommended; matches embed)." ), + source_mpp: float | None = typer.Option( + None, + "--source-mpp", + help="Source level-0 µm/px override (must match the embed run).", + ), patch_size: int | None = typer.Option( None, "--patch-size", help="Patch size in px (default: model recommended)." ), @@ -62,7 +69,12 @@ def verify( quiet: bool = typer.Option(False, "--quiet"), ) -> None: """Exit 0 if the slide is already complete & output-validated, else exit 1.""" + validate_amp(amp) + validate_geometry( + mpp=mpp, patch_size=patch_size, step=step, source_mpp=source_mpp + ) from raw2features.core.device import resolve_device + from raw2features.embedders.model_registry import resolve_geometry from raw2features.pipeline.runner import resolve_run geometry_config = None @@ -73,13 +85,33 @@ def verify( models = list(dict.fromkeys(e["model"] for e in geometry_config)) else: models = list(feature_extractor) + # Verification has no programmatic external-embedder instance from which to + # derive a current contract or geometry. Fail closed with the established clean + # diagnostic before the geometry resolver rejects the unknown name. + from raw2features.embedders.model_registry import load_registry + + registered = load_registry() + unknown = [model for model in models if model not in registered] + if unknown: + if not quiet: + typer.echo( + "cannot derive the current output contract for unregistered " + f"model(s) {', '.join(unknown)}; incomplete", + err=True, + ) + raise typer.Exit(code=1) + groups = resolve_geometry(models, mpp, patch_size, geometry_config) + if not groups: + raise typer.BadParameter("at least one model/extraction is required") + representative = groups[0] cfg = RunConfig( models=models, reader=reader, segmenter=segmenter, no_seg=no_seg, - target_mpp=mpp if mpp is not None else 1.0, # per-group geometry overrides this - patch_px=patch_size if patch_size is not None else 224, + target_mpp=representative.mpp, + source_mpp=source_mpp, + patch_px=representative.patch_px, step_px=step, tissue_threshold=tissue_threshold, features_dtype=features_dtype, @@ -113,18 +145,6 @@ def verify( if expected_path is not None else None ) - from raw2features.embedders.model_registry import load_registry - - registered = load_registry() - unknown = [model for model in models if model not in registered] - if unknown: - if not quiet: - typer.echo( - "cannot derive the current output contract for unregistered " - f"model(s) {', '.join(unknown)}; incomplete", - err=True, - ) - raise typer.Exit(code=1) contracts = expected_model_contracts(cfg) expected_grid_models = { group_cfg.grid_hash(): list(group_cfg.models) for group_cfg in group_cfgs diff --git a/src/raw2features/core/config.py b/src/raw2features/core/config.py index cd5af2e..a59cb84 100644 --- a/src/raw2features/core/config.py +++ b/src/raw2features/core/config.py @@ -16,6 +16,30 @@ from __future__ import annotations +import math + + +def _positive_float(value, *, field: str) -> float: + try: + number = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} must be a finite number greater than zero") from exc + if not math.isfinite(number) or number <= 0: + raise ValueError(f"{field} must be a finite number greater than zero") + return number + + +def _positive_int(value, *, field: str) -> int: + if isinstance(value, bool): + raise ValueError(f"{field} must be an integer greater than zero") + try: + integer = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} must be an integer greater than zero") from exc + if integer != value or integer <= 0: + raise ValueError(f"{field} must be an integer greater than zero") + return integer + def load_extractions(path: str) -> list[dict]: """Parse an extraction-plan config; return the validated ``extractions`` list. @@ -42,9 +66,13 @@ def load_extractions(path: str) -> list[dict]: ) entry: dict = {"model": str(e["model"])} if e.get("mpp") is not None: - entry["mpp"] = float(e["mpp"]) + entry["mpp"] = _positive_float( + e["mpp"], field=f"{path}: extractions[{i}].mpp" + ) if e.get("patch_px") is not None: - entry["patch_px"] = int(e["patch_px"]) + entry["patch_px"] = _positive_int( + e["patch_px"], field=f"{path}: extractions[{i}].patch_px" + ) out.append(entry) return out @@ -75,14 +103,18 @@ def load_manifest(path: str) -> list[dict]: row: dict = {"path": p} sm = (r.get("source_mpp") or "").strip() if sm: - row["source_mpp"] = float(sm) + row["source_mpp"] = _positive_float( + sm, field=f"{path}: source_mpp" + ) out.append(row) else: for ln in lines: parts = [c.strip() for c in ln.split(",")] row = {"path": parts[0]} if len(parts) > 1 and parts[1]: - row["source_mpp"] = float(parts[1]) + row["source_mpp"] = _positive_float( + parts[1], field=f"{path}: source_mpp" + ) out.append(row) if not out: raise ValueError(f"{path}: manifest has no slide paths") diff --git a/src/raw2features/core/provenance.py b/src/raw2features/core/provenance.py index c046a75..5628eb6 100644 --- a/src/raw2features/core/provenance.py +++ b/src/raw2features/core/provenance.py @@ -7,15 +7,16 @@ from __future__ import annotations -import os import platform import re import shlex import subprocess import sys +import tomllib from collections.abc import Sequence from datetime import UTC, datetime from importlib.metadata import PackageNotFoundError, version +from pathlib import Path from raw2features.core.uris import ( is_qualified_uri, @@ -44,16 +45,51 @@ } +def _is_raw2features_checkout(root: Path, module_file: Path) -> bool: + """True only when *root* is the repository containing this source package. + + ``git`` walks through parent directories. Without this boundary, a wheel in a + virtual environment nested under an unrelated checkout inherits that checkout's + commit. A real raw2features source checkout has both its own project metadata and + this module under the repository's ``src/raw2features`` (or a supported flat + ``raw2features`` package layout). + """ + + package_roots = (root / "src" / "raw2features", root / "raw2features") + if not any(module_file.is_relative_to(path) for path in package_roots): + return False + try: + with (root / "pyproject.toml").open("rb") as fh: + project_name = tomllib.load(fh).get("project", {}).get("name") + except (OSError, tomllib.TOMLDecodeError): + return False + normalized = str(project_name or "").lower().replace("_", "-") + return normalized == "raw2features" + + def _git_sha() -> str | None: try: - out = subprocess.run( + module_file = Path(__file__).resolve() + common = { + "capture_output": True, + "text": True, + "timeout": 5, + "cwd": str(module_file.parent), + } + top = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + **common, + ) + if top.returncode != 0 or not top.stdout.strip(): + return None + root = Path(top.stdout.strip()).resolve() + if not _is_raw2features_checkout(root, module_file): + return None + head = subprocess.run( ["git", "rev-parse", "HEAD"], - capture_output=True, - text=True, - timeout=5, - cwd=os.path.dirname(os.path.abspath(__file__)), + **common, ) - return out.stdout.strip() or None if out.returncode == 0 else None + return head.stdout.strip() or None if head.returncode == 0 else None except Exception: # noqa: BLE001 return None @@ -91,9 +127,7 @@ def sanitize_cli(cli: str) -> str: """Redact secret option values and credentials embedded in source URIs.""" redacted = _SECRET_OPTION_RE.sub( - lambda match: ( - f"{match.group('flag')}{match.group('separator')}" - ), + lambda match: f"{match.group('flag')}{match.group('separator')}", str(cli), ) return redact_uri_credentials(redacted) diff --git a/src/raw2features/embedders/model_registry.py b/src/raw2features/embedders/model_registry.py index f2e8765..17b979e 100644 --- a/src/raw2features/embedders/model_registry.py +++ b/src/raw2features/embedders/model_registry.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.resources +import math +from collections.abc import Mapping from dataclasses import dataclass import yaml @@ -62,12 +64,51 @@ def get_spec(name: str) -> ModelSpec: # Scale-agnostic fallback when no requested model declares a recommended MPP. DEFAULT_TARGET_MPP = 1.0 -# Extraction patch size (px) for models that declare none (scale-agnostic / unknown). +# Extraction patch size (px) for registered scale-agnostic models that declare none. DEFAULT_PATCH_PX = 224 +def _spec_catalog(specs: Mapping[str, ModelSpec] | None = None) -> dict[str, ModelSpec]: + """Packaged registry plus any explicitly injected model specifications.""" + + catalog = load_registry() + catalog.update(specs or {}) + return catalog + + +def _require_known_models( + models: list[str], catalog: Mapping[str, ModelSpec] +) -> None: + unknown = sorted(set(models) - set(catalog)) + if unknown: + available = sorted(catalog) + raise ValueError( + f"model(s) {', '.join(unknown)} not in the registry or supplied model " + f"specifications; available: {available}" + ) + + +def _positive_mpp(value: float, *, field: str = "mpp") -> float: + value = float(value) + if not math.isfinite(value) or value <= 0: + raise ValueError(f"{field} must be finite and greater than zero") + return value + + +def _positive_patch_px(value: int, *, field: str = "patch_px") -> int: + if isinstance(value, bool): + raise ValueError(f"{field} must be an integer greater than zero") + integer = int(value) + if integer != value or integer <= 0: + raise ValueError(f"{field} must be an integer greater than zero") + return integer + + def resolve_target_mpp( - models: list[str], requested: float | None + models: list[str], + requested: float | None, + *, + specs: Mapping[str, ModelSpec] | None = None, ) -> tuple[float, str]: """Resolve the patch-extraction MPP for a run, returning ``(mpp, source)``. @@ -77,19 +118,21 @@ def resolve_target_mpp( if no requested model declares one (scale-agnostic, e.g. resnet50 / dinov2) it falls back to :data:`DEFAULT_TARGET_MPP` (``"auto-default"``). Models that *disagree* raise - one run extracts patches at one scale, so the caller must - pass ``--mpp`` to pick. Unknown names (e.g. test mocks) contribute nothing - rather than erroring, so they resolve to the default. + pass ``--mpp`` to pick. Programmatic external model names are accepted only + when their authoritative :class:`ModelSpec` is supplied through ``specs``; + unknown uninjected names fail instead of silently receiving generic geometry. - Pure in ``(models, requested)`` so every CLI (embed / embed-many / verify) - resolves identically and their config hashes line up. + Pure in its inputs so every CLI (embed / embed-many / verify) resolves + identically and their config hashes line up. """ + registry = _spec_catalog(specs) + _require_known_models(models, registry) if requested is not None: - return float(requested), "explicit" - registry = load_registry() + return _positive_mpp(requested), "explicit" recs = { name: float(registry[name].recommended_mpp) for name in models - if name in registry and registry[name].recommended_mpp + if registry[name].recommended_mpp } distinct = sorted(set(recs.values())) if len(distinct) == 1: @@ -110,10 +153,10 @@ class GeometryGroup: A "grid" in the store: every model in ``models`` is decoded once at this geometry (decode-once fan-out) and writes ``features/`` against the same coords. The field of view is ``patch_px * mpp`` µm. ``source`` records how the geometry was - chosen -- ``"explicit"`` (a global --mpp/--patch-size collapse), ``"config"`` (a - per-model config file), ``"recommended"`` (the models' card geometry), or - ``"default"`` (the scale-agnostic ``DEFAULT_TARGET_MPP`` / ``DEFAULT_PATCH_PX`` - fallback). + chosen -- ``"explicit"`` (a command-line mpp and/or patch-size override), + ``"config"`` (a per-model config file), ``"recommended"`` (the models' card + geometry), or ``"default"`` (the scale-agnostic ``DEFAULT_TARGET_MPP`` / + ``DEFAULT_PATCH_PX`` fallback). """ mpp: float @@ -121,6 +164,10 @@ class GeometryGroup: models: tuple[str, ...] source: str + def __post_init__(self) -> None: + _positive_mpp(self.mpp) + _positive_patch_px(self.patch_px) + def _dedup(names: list[str]) -> tuple[str, ...]: """First-seen-order de-duplication (a model can't appear twice in one grid).""" @@ -130,21 +177,33 @@ def _dedup(names: list[str]) -> tuple[str, ...]: return tuple(seen) -def model_geometry(name: str) -> tuple[float, int, str]: +def model_geometry( + name: str, *, specs: Mapping[str, ModelSpec] | None = None +) -> tuple[float, int, str]: """The default ``(mpp, patch_px, source)`` one model is extracted at. ``recommended_mpp`` (card-sourced) sets the scale; ``recommended_patch_px`` (or ``input_size``, via :attr:`ModelSpec.extract_px`) the size. Models with no ``recommended_mpp`` are scale-agnostic and fall back to :data:`DEFAULT_TARGET_MPP`. - Unknown names (e.g. test mocks) resolve to the scale-agnostic default rather than - raising, mirroring :func:`resolve_target_mpp`. + Programmatic external models must be provided through ``specs``; an unknown name + without a specification is rejected rather than silently receiving 1.0/224. """ - spec = load_registry().get(name) + catalog = _spec_catalog(specs) + _require_known_models([name], catalog) + spec = catalog.get(name) if spec is None: - return DEFAULT_TARGET_MPP, DEFAULT_PATCH_PX, "default" + raise AssertionError("known model unexpectedly missing from the catalog") if spec.recommended_mpp: - return float(spec.recommended_mpp), spec.extract_px, "recommended" - return DEFAULT_TARGET_MPP, spec.extract_px, "default" + return ( + _positive_mpp(spec.recommended_mpp, field=f"{name}.recommended_mpp"), + _positive_patch_px(spec.extract_px, field=f"{name}.extract_px"), + "recommended", + ) + return ( + DEFAULT_TARGET_MPP, + _positive_patch_px(spec.extract_px, field=f"{name}.extract_px"), + "default", + ) def resolve_geometry( @@ -152,6 +211,8 @@ def resolve_geometry( requested_mpp: float | None = None, requested_patch_px: int | None = None, config: list[dict] | None = None, + *, + specs: Mapping[str, ModelSpec] | None = None, ) -> list[GeometryGroup]: """Group requested models into extraction geometries -- one grid per group. @@ -160,26 +221,43 @@ def resolve_geometry( * ``config`` -- a list of ``{model, mpp?, patch_px?}`` entries: explicit per-model geometry, missing values falling back to the model's defaults. Supports the SAME model at several geometries (distinct groups -- the MPP-ablation case). - * an explicit ``requested_mpp`` and/or ``requested_patch_px`` -- a GLOBAL collapse: - every model onto one grid at that geometry (today's behaviour, and the "all at - one scale" switch). A bare ``requested_patch_px`` resolves the shared mpp via - :func:`resolve_target_mpp` (which raises if the cards disagree). + * both ``requested_mpp`` and ``requested_patch_px`` -- a GLOBAL collapse: every + model onto one grid at that geometry. + * a bare ``requested_mpp`` -- override scale while retaining each model's + recommended extraction size; models with different patch sizes get separate + grids at the requested scale. A bare ``requested_patch_px`` resolves the shared + mpp via :func:`resolve_target_mpp` (which raises if the cards disagree). * neither -- PER-MODEL geometry: each model at its own geometry ``(recommended_mpp or default, recommended_patch_px or input_size)``, grouped by identical geometry in request order. Models that disagree no longer raise; they get their own grids. - Pure in its arguments so every CLI resolves identically (config hashes line up). + ``specs`` supplies authoritative metadata for programmatic external embedders. + Unknown names without a registry entry or injected spec fail instead of silently + receiving the generic 1.0/224 geometry. Pure in its arguments so every CLI + resolves identically (config hashes line up). """ if config is not None: - return _groups_from_config(config) + return _groups_from_config(config, specs=specs) + catalog = _spec_catalog(specs) + _require_known_models(models, catalog) + if requested_mpp is not None and requested_patch_px is None: + mpp = _positive_mpp(requested_mpp) + groups: dict[int, list[str]] = {} + for model in models: + _default_mpp, patch_px, _source = model_geometry(model, specs=specs) + groups.setdefault(patch_px, []).append(model) + return [ + GeometryGroup(mpp, patch_px, _dedup(group_models), "explicit") + for patch_px, group_models in groups.items() + ] if requested_mpp is not None or requested_patch_px is not None: mpp = ( - float(requested_mpp) + _positive_mpp(requested_mpp) if requested_mpp is not None - else resolve_target_mpp(models, None)[0] + else resolve_target_mpp(models, None, specs=specs)[0] ) px = ( - int(requested_patch_px) + _positive_patch_px(requested_patch_px) if requested_patch_px is not None else DEFAULT_PATCH_PX ) @@ -187,7 +265,7 @@ def resolve_geometry( groups: dict[tuple[float, int], list[str]] = {} src: dict[tuple[float, int], str] = {} for m in models: - mpp, px, s = model_geometry(m) + mpp, px, s = model_geometry(m, specs=specs) groups.setdefault((mpp, px), []).append(m) src.setdefault((mpp, px), s) return [ @@ -196,7 +274,9 @@ def resolve_geometry( ] -def _groups_from_config(extractions: list[dict]) -> list[GeometryGroup]: +def _groups_from_config( + extractions: list[dict], *, specs: Mapping[str, ModelSpec] | None = None +) -> list[GeometryGroup]: """Build geometry groups from a parsed config: ``[{model, mpp?, patch_px?}, ...]``. Each entry resolves mpp/patch_px (explicit, else the model's defaults); entries with @@ -206,9 +286,9 @@ def _groups_from_config(extractions: list[dict]) -> list[GeometryGroup]: groups: dict[tuple[float, int], list[str]] = {} for e in extractions: name = e["model"] - d_mpp, d_px, _ = model_geometry(name) - mpp = float(e.get("mpp", d_mpp)) - px = int(e.get("patch_px", d_px)) + d_mpp, d_px, _ = model_geometry(name, specs=specs) + mpp = _positive_mpp(e.get("mpp", d_mpp)) + px = _positive_patch_px(e.get("patch_px", d_px)) groups.setdefault((mpp, px), []).append(name) return [ GeometryGroup(mpp, px, _dedup(ms), "config") @@ -217,7 +297,12 @@ def _groups_from_config(extractions: list[dict]) -> list[GeometryGroup]: def build_embedder(name: str) -> Embedder: - """Resolve a model name -> its family Embedder class -> a ready instance.""" + """Build a packaged-registry model with its family implementation plugin. + + An ``embedders`` entry point supplies a loader family; it does not by itself add + *name* to the CLI model catalogue. Programmatic callers may instead pass external + :class:`Embedder` instances through ``embed_slide(..., embedders=[...])``. + """ from raw2features.core import plugins spec = get_spec(name) diff --git a/src/raw2features/export/h5.py b/src/raw2features/export/h5.py index c7dd3c4..03c5fa7 100644 --- a/src/raw2features/export/h5.py +++ b/src/raw2features/export/h5.py @@ -118,18 +118,32 @@ def export_h5( raise FileExistsError(f"{path} exists; pass overwrite=True") with h5py.File(path, "w") as fh: if layout in ("trident", "clam"): - _write_trident(fh, feats, coords, level0_patch, achieved_mpp, - mpp_level0, model, - coord_dtype=np.int32 if layout == "clam" else np.int64) + _write_trident( + fh, + feats, + coords, + level0_patch, + achieved_mpp, + mpp_level0, + model, + coord_dtype=np.int32 if layout == "clam" else np.int64, + ) else: - _write_stamp(fh, feats, coords, patch_px, achieved_mpp, mpp_level0, - model, header) + _write_stamp(fh, feats, coords, patch_px, achieved_mpp, model, header) written.append(path) return written -def _write_trident(fh, feats, coords, level0_patch, achieved_mpp, mpp_level0, model, - coord_dtype=np.int64): +def _write_trident( + fh, + feats, + coords, + level0_patch, + achieved_mpp, + mpp_level0, + model, + coord_dtype=np.int64, +): """CLAM / TRIDENT / TITAN layout: features + level-0-pixel coords + attrs. ``coord_dtype`` is int64 for ``trident`` and int32 for the ``clam`` alias (CLAM's @@ -148,13 +162,34 @@ def _write_trident(fh, feats, coords, level0_patch, achieved_mpp, mpp_level0, mo fh["features"].attrs["encoder"] = model -def _write_stamp(fh, feats, coords, patch_px, achieved_mpp, mpp_level0, model, header): - """KatherLab STAMP layout: feats (fp16) + micron coords + physical-size attrs.""" - if not mpp_level0: +def _write_stamp(fh, feats, coords, patch_px, achieved_mpp, model, header): + """KatherLab STAMP layout: feats (fp16) + slide-relative micron coords.""" + source = dict(header.get("source", {})) + scale_um = source.get("scale_um") + if ( + isinstance(scale_um, dict) + and scale_um.get("x") is not None + and scale_um.get("y") is not None + ): + sx, sy = float(scale_um["x"]), float(scale_um["y"]) + else: + # Legacy stores carry only the isotropic scalar and imply a zero origin. + mpp_level0 = source.get("mpp_level0") + if not mpp_level0: + raise ValueError( + "STAMP layout needs source.scale_um or source.mpp_level0 to " + "convert px -> microns" + ) + sx = sy = float(mpp_level0) + + if sx <= 0 or sy <= 0: raise ValueError( - "STAMP layout needs source.mpp_level0 to convert px -> microns" + "STAMP layout needs positive source.scale_um/source.mpp_level0" ) - coords_um = coords.astype(np.float32) * float(mpp_level0) # level-0 px -> µm + # STAMP coordinates are measured from the top-left of the WSI scan. An NGFF + # translation describes the source's physical/stage origin, so adding it here + # would shift STAMP heatmaps relative to the pixels they are drawn over. + coords_um = coords.astype(np.float32) * np.asarray([sx, sy], dtype=np.float32) fh.create_dataset("feats", data=feats.astype(np.float16)) fh.create_dataset("coords", data=coords_um) fh.attrs["unit"] = "um" diff --git a/src/raw2features/pipeline/runner.py b/src/raw2features/pipeline/runner.py index f23705f..a063c00 100644 --- a/src/raw2features/pipeline/runner.py +++ b/src/raw2features/pipeline/runner.py @@ -6,6 +6,7 @@ from __future__ import annotations +import math import os import queue import threading @@ -31,7 +32,11 @@ ) from raw2features.embedders.model_registry import build_embedder, resolve_geometry from raw2features.patcher.grid import GridPatcher, resample_patch -from raw2features.sinks.zarr_sink import ZarrSink, write_patches_geojson +from raw2features.sinks.zarr_sink import ( + ZarrSink, + _grid_scaffold_is_usable, + write_patches_geojson, +) from .receipt import ( SCHEMA_VERSION, @@ -47,6 +52,28 @@ _AMP = {"bf16": "bfloat16", "fp16": "float16", "fp32": "float32", None: "float32"} +def _positive_float(value, *, field: str) -> float: + try: + number = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} must be a finite number greater than zero") from exc + if not math.isfinite(number) or number <= 0: + raise ValueError(f"{field} must be a finite number greater than zero") + return number + + +def _positive_int(value, *, field: str) -> int: + if isinstance(value, bool): + raise ValueError(f"{field} must be an integer greater than zero") + try: + integer = int(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field} must be an integer greater than zero") from exc + if integer != value or integer <= 0: + raise ValueError(f"{field} must be an integer greater than zero") + return integer + + @dataclass class RunConfig: """Everything needed to embed one slide. Content-affecting fields feed the @@ -118,6 +145,16 @@ class RunConfig: # focus-sensitive; raise it for a softer-scanned cohort that over-reads as blur. qc_artifact_mpp: str = "1.5" + def __post_init__(self) -> None: + """Reject invalid geometry before hashing, model loading, or store writes.""" + + self.target_mpp = _positive_float(self.target_mpp, field="target_mpp") + self.patch_px = _positive_int(self.patch_px, field="patch_px") + if self.step_px is not None: + self.step_px = _positive_int(self.step_px, field="step_px") + if self.source_mpp is not None: + self.source_mpp = _positive_float(self.source_mpp, field="source_mpp") + # Single source of truth for which fields affect the output (and thus the # config hash) vs. runtime-only knobs. ``test_config_integrity`` asserts these # partition every RunConfig field, so a new field must be classified. @@ -239,11 +276,7 @@ def device_list(self) -> list[str]: except Exception: # noqa: BLE001 - no torch/cuda visible -> single device n = 0 return [f"cuda:{i}" for i in range(n)] if n > 1 else [single] - out = [ - resolve_device(d.strip()) - for d in self.devices.split(",") - if d.strip() - ] + out = [resolve_device(d.strip()) for d in self.devices.split(",") if d.strip()] return out or [single] @@ -271,8 +304,13 @@ def run_slide( force: bool = False, profiler=None, allow_hashless_legacy: bool = True, + _expected_model_contracts: dict[str, dict] | None = None, ) -> dict: - """Embed one slide. Returns a summary dict. + """Embed one slide into exactly one configured grid. Returns a summary dict. + + This is the single-grid primitive. Library callers that want registry-recommended + per-model geometry (and therefore potentially several grids) should use the + top-level :func:`raw2features.embed_slide` entry point instead. Resume / additive behaviour: * A validated 'complete' receipt for this exact request short-circuits @@ -330,12 +368,19 @@ def run_slide( if store_exists: _assert_store_source(out_path, expected_source) - model_contracts = _expected_contracts_with_factory_probe( - cfg, - embedders, - embedder_factory if multi_device else None, - devices=devices, - ) + if _expected_model_contracts is None: + model_contracts = _expected_contracts_with_factory_probe( + cfg, + embedders, + embedder_factory if multi_device else None, + devices=devices, + ) + else: + model_contracts = dict(_expected_model_contracts) + if set(model_contracts) != set(cfg.models): + raise ValueError( + "Pre-resolved model contracts do not match this grid's models." + ) # Fast path: a validated 'complete' receipt for this exact config (incl. the # model set), source, and requested output. ``--force`` deliberately bypasses it. @@ -366,6 +411,7 @@ def run_slide( expected_model_contracts=model_contracts, compatible_grid_hashes=cfg.compatible_legacy_grid_hashes(), allow_hashless_legacy=allow_hashless_legacy, + require_mask=not cfg.no_seg, ) # append: a grid of THIS geometry already exists -> add the missing models to it. # Otherwise we create a fresh store (no store yet) or ADD a new grid to an existing @@ -728,9 +774,7 @@ def embedder_factory(device: str) -> list: thumbnail_tissue, _ = _segment( reader, cfg, - _stored_grid_segmenter( - out_path, grid_key_existing - ), + _stored_grid_segmenter(out_path, grid_key_existing), ) thumb_meta = _write_grid_thumbnail( reader, @@ -880,6 +924,8 @@ def resolve_run( requested_mpp: float | None = None, requested_patch_px: int | None = None, geometry_config: list[dict] | None = None, + *, + model_specs: dict | None = None, ): """``(groups, group_cfgs, run_hash)`` for a request. @@ -889,8 +935,14 @@ def resolve_run( identically to the embed that produced the store. """ groups = resolve_geometry( - cfg.models, requested_mpp, requested_patch_px, geometry_config + cfg.models, + requested_mpp, + requested_patch_px, + geometry_config, + specs=model_specs, ) + if not groups: + raise ValueError("at least one model/extraction is required") group_cfgs = [ replace(cfg, models=list(g.models), target_mpp=g.mpp, patch_px=g.patch_px) for g in groups @@ -915,16 +967,26 @@ def embed_slide( ) -> dict: """Resolve per-model geometry and embed each group as its own grid in one store. - The user-facing entry point. Without an mpp/patch override, models that recommend - different geometries are each extracted at their own ``(mpp, patch_px)`` into a - separate ``grids//``; an explicit override collapses them onto one grid (see + The high-level public entry point. Without an mpp/patch override, models that + recommend different geometries are each extracted at their own + ``(mpp, patch_px)`` into a separate ``grids//``. An mpp-only override + retains each model's extraction size; supplying both mpp and patch size + collapses them onto one grid (see :func:`raw2features.embedders.model_registry.resolve_geometry`). It drives the - single-grid :func:`run_slide` once per group: the first writes the store, each later - group *finds-or-creates* its grid (never a wipe). ``force`` wipes once, on the first - group. A single per-slide receipt records the whole request for fast skip. + single-grid :func:`run_slide` once per group: the first writes the store, each + later group *finds-or-creates* its grid (never a wipe). ``force`` wipes once, on + the first group. A single per-slide receipt records the whole request for fast + skip. ``embedders`` (a warm worker's pre-built set for all of ``cfg.models``) is passed - through to every group; ``run_slide`` selects each group's subset. + through to every group; ``run_slide`` selects each group's subset. Their + :class:`~raw2features.embedders.base.ModelSpec` objects also drive geometry for + models outside the packaged registry. When only ``embedder_factory`` is supplied, + it is probed once for the same specifications and output contracts. High-level + overrides belong in ``requested_mpp`` / ``requested_patch_px`` (or + ``geometry_config``); + ``RunConfig.target_mpp`` and ``RunConfig.patch_px`` remain the concrete geometry + consumed by the single-grid :func:`run_slide` primitive. """ from raw2features.slide_embedders.model_registry import ( validate_slide_encoder_names, @@ -946,13 +1008,26 @@ def embed_slide( raise ValueError( "Source URI is malformed and no safe output ID can be derived." ) from None + model_specs = {embedder.name: embedder.spec for embedder in (embedders or [])} + probed_contracts: dict[str, dict] | None = None + if not model_specs and embedder_factory is not None: + # A factory can be the only source for a programmatic external model. Probe it + # once before geometry resolution, then pass the resulting contracts into each + # run_slide call so the same loaded copy is not probed a second time. + model_specs, probed_contracts = _probe_factory_contracts( + cfg, embedder_factory, devices + ) groups, group_cfgs, run_hash = resolve_run( - cfg, requested_mpp, requested_patch_px, geometry_config + cfg, + requested_mpp, + requested_patch_px, + geometry_config, + model_specs=model_specs, ) out_path = os.path.join(out_dir, f"{slide_id}.embeddings.zarr") grids = {grid_key(g.mpp, g.patch_px): list(g.models) for g in groups} expected_output = f"file://{os.path.abspath(out_path)}" - model_contracts = _expected_contracts_with_factory_probe( + model_contracts = probed_contracts or _expected_contracts_with_factory_probe( cfg, embedders, embedder_factory, @@ -1010,23 +1085,29 @@ def embed_slide( # requested geometry, but cannot identify which of several grids it # represents. A multi-grid request therefore leaves it untouched. allow_hashless_legacy=(len(group_cfgs) == 1), + _expected_model_contracts={ + model: model_contracts[model] for model in gc.models + }, ) ) - elapsed = round(time.time() - started, 2) grids = { (result.get("grid") or grid_key(group.mpp, group.patch_px)): list(group.models) for group, result in zip(groups, results, strict=True) } status = "skipped" if all(r["status"] == "skipped" for r in results) else "complete" - # Coverage: each requested slide encoder must have run on some grid (the one holding - # its patch encoder). One that ran nowhere has no grid with its patch model -> clear - # error rather than a silent omission. - if cfg.slide_encoders and status != "skipped": + # Coverage: first use outputs produced on this request's grids. If a specific + # encoder's patch model lives only on another, already-existing grid, discover it + # with the same selection path as standalone `slide-embed` and produce it there. + # An explicit -s request never returns success without a complete output. + if cfg.slide_encoders: ran: set[str] = set() for r in results: ran |= set(r.get("slide_embeddings") or {}) missing = [s for s in cfg.slide_encoders if s not in ran] + if missing: + ran |= set(_run_slide_encoders_from_store(out_path, missing, cfg.device)) + missing = [s for s in cfg.slide_encoders if s not in ran] if missing: from raw2features.slide_embedders.model_registry import get_slide_spec @@ -1041,6 +1122,8 @@ def embed_slide( "their patch encoder -- embed that patch model (at its geometry) first." ) + elapsed = round(time.time() - started, 2) + # Record the explicit job knobs (extraction plan, stain norm) so the run replays # from one artifact and the store is self-describing about how it was made. if status != "skipped" and (geometry_config is not None or cfg.stain_norm): @@ -1165,11 +1248,25 @@ def _expected_contracts_with_factory_probe( factory = embedder_factory or ( lambda device: _build_embedders_on(cfg, cfg.models, device) ) + _specs, contracts = _probe_factory_contracts(cfg, factory, worker_devices) + return contracts + + +def _probe_factory_contracts( + cfg: RunConfig, + factory, + worker_devices: list[str], +) -> tuple[dict[str, object], dict[str, dict]]: + """Probe a factory once for external specs and the shared worker contract.""" + + if not worker_devices: + raise ValueError("At least one patch worker device is required.") probe = factory(worker_devices[0]) try: contracts = _expected_contracts_for_devices(cfg, probe, worker_devices) _assert_loaded_model_contracts(probe, contracts) - return contracts + specs = {embedder.name: embedder.spec for embedder in probe} + return specs, contracts finally: for embedder in probe: embedder.unload() @@ -1390,8 +1487,16 @@ def _warn_read_block_memory(reader, cfg, read_px, read_workers, multichannel) -> ) -def _decode_batch(reader, batch, read_level, read_px, patch_px, executor=None, - multichannel=False, read_block=1): +def _decode_batch( + reader, + batch, + read_level, + read_px, + patch_px, + executor=None, + multichannel=False, + read_block=1, +): """Decode one batch's patch list once at the store geometry (read + resample). Read read_px px then resample to exactly patch_px -> the stored patch is at the @@ -1413,7 +1518,13 @@ def _decode_batch(reader, batch, read_level, read_px, patch_px, executor=None, """ if read_block > 1: return _decode_batch_blocked( - reader, batch, read_level, read_px, patch_px, executor, multichannel, + reader, + batch, + read_level, + read_px, + patch_px, + executor, + multichannel, side=read_block, ) if executor is None: @@ -1510,8 +1621,18 @@ def _group_by_transform(embedders: list) -> list[list]: def _embed_patches( - reader, coords, read_level, read_px, patch_px, embedders, sink, cfg, prof=None, - multichannel=False, normalizer=None, device=None, + reader, + coords, + read_level, + read_px, + patch_px, + embedders, + sink, + cfg, + prof=None, + multichannel=False, + normalizer=None, + device=None, ): """Decode each patch once at the store geometry and embed it with every extractor in *embedders*, writing ``features/`` in batches. @@ -1622,8 +1743,14 @@ def _producer() -> None: batch = coords[start : start + cfg.batch_size] with prof.stage("read"): patches = _decode_batch( - reader, batch, read_level, read_px, patch_px, executor, - multichannel, read_block=cfg.read_block, + reader, + batch, + read_level, + read_px, + patch_px, + executor, + multichannel, + read_block=cfg.read_block, ) if normalizer is not None: # per-patch stain norm (brightfield) patches = [normalizer(p) for p in patches] @@ -1662,7 +1789,9 @@ def _producer() -> None: for emb in group: with prof.stage("gpu"): feats = ( - emb.embed_batch(batch_tensor).numpy().astype(cfg.features_dtype) + emb.embed_batch(batch_tensor) + .numpy() + .astype(cfg.features_dtype) ) with prof.stage("write"): write_block(emb.name, start, feats) @@ -1817,6 +1946,7 @@ def _inspect_store( expected_model_contracts: dict[str, dict] | None = None, compatible_grid_hashes: tuple[str, ...] = (), allow_hashless_legacy: bool = True, + require_mask: bool = False, ): """Find the grid matching ``expected_grid_hash``: ``(key | None, n, valid_models)``. @@ -1842,6 +1972,26 @@ def _inspect_store( def _result_for(k: str): g = root[GRIDS][k] try: + header = dict(g.attrs.get("raw2features", {})) + patching = header.get("patching", {}) + patching = patching if isinstance(patching, dict) else {} + declared_n = patching.get("n_patches") + expected_n = declared_n if isinstance(declared_n, int) else None + grid_shape = patching.get("grid_shape") + expected_mask_shape = ( + tuple(int(v) for v in grid_shape) + if require_mask + and isinstance(grid_shape, (list, tuple)) + and len(grid_shape) == 2 + else None + ) + if not _grid_scaffold_is_usable( + g, + expected_n=expected_n, + require_mask=require_mask, + expected_mask_shape=expected_mask_shape, + ): + return None n = int(g["coords"].shape[0]) except Exception: # noqa: BLE001 - skip an unreadable grid return None @@ -1957,9 +2107,7 @@ def _missing_qc_tools( group = open_grid(root, key) n_patches = int(group["coords"].shape[0]) present = { - tool - for tool in requested - if _qc_tool_complete(group, tool, n_patches) + tool for tool in requested if _qc_tool_complete(group, tool, n_patches) } except Exception: # noqa: BLE001 - unreadable auxiliary state is missing return list(requested) @@ -2030,9 +2178,7 @@ def _thumbnail_files_complete( ) -def _stored_thumbnail_metadata( - out_path: str, key: str | None = None -) -> dict | None: +def _stored_thumbnail_metadata(out_path: str, key: str | None = None) -> dict | None: """Read live root or per-grid thumbnail metadata, even when an asset is missing.""" try: @@ -2113,9 +2259,7 @@ def _stored_grid_segmenter(out_path: str, key: str | None) -> str | None: return None -def _grid_geojson_path( - out_dir: str, slide_id: str, key: str, primary_key: str -) -> str: +def _grid_geojson_path(out_dir: str, slide_id: str, key: str, primary_key: str) -> str: """Backward-compatible primary path; namespaced path for every other grid.""" filename = ( @@ -2126,9 +2270,7 @@ def _grid_geojson_path( return os.path.join(out_dir, filename) -def _grid_thumbnail_overlay_name( - slide_id: str, key: str, primary_key: str -) -> str: +def _grid_thumbnail_overlay_name(slide_id: str, key: str, primary_key: str) -> str: return ( f"{slide_id}.thumbnail.overlay.png" if key == primary_key @@ -2190,9 +2332,7 @@ def _stored_grid_summary( root = zarr.open_group(out_path, mode="r", use_consolidated=False) keys = grid_keys(root) stored = { - key: dict(root[GRIDS][key].attrs.get("raw2features", {})).get( - "grid_hash" - ) + key: dict(root[GRIDS][key].attrs.get("raw2features", {})).get("grid_hash") for key in keys } result: dict[str, list[str]] = {} @@ -2202,9 +2342,7 @@ def _stored_grid_summary( match = None for candidate in dict.fromkeys(candidates): hits = [ - key - for key in keys - if key not in used and stored[key] == candidate + key for key in keys if key not in used and stored[key] == candidate ] if len(hits) == 1: match = hits[0] @@ -2249,7 +2387,13 @@ def _slide_encoders_for(names: list[str], available: list[str]) -> list[str]: def _run_qc( - qc_tools, reader, sink, coords, level0_patch, device, stain_norm=None, + qc_tools, + reader, + sink, + coords, + level0_patch, + device, + stain_norm=None, artifact_mpp="1.5", ) -> None: """Run requested QC producers on the active grid -> write ``qc//`` scores. @@ -2338,6 +2482,76 @@ def _run_slide_encoders( return results +def _run_slide_encoders_from_store( + out_path: str, + slide_encoder_names: list[str], + device: str, +) -> dict[str, str]: + """Produce missing slide outputs by discovering compatible grids in a store. + + This is the high-level inline fallback for patch features written by an earlier + request. Grid and patch-model selection is shared with standalone ``slide-embed``; + no WSI is reopened and complete outputs are left untouched. + """ + import zarr + + from raw2features.slide_embedders.encoding import ( + encode_slide_embedding, + resolve_slide_grid, + slide_embedding_is_complete, + write_slide_embedding, + ) + + root = zarr.open_group(out_path, mode="r+", use_consolidated=False) + results: dict[str, str] = {} + wrote = False + for slide_model_name in slide_encoder_names: + try: + selected_grid, group, patch_model = resolve_slide_grid( + root, + slide_model_name, + ) + except (KeyError, ValueError): + # Coverage is checked by embed_slide after every requested grid and this + # fallback have been tried. Selection failure here means only that no + # compatible pre-existing grid was found; model-load/encode errors below + # still propagate rather than being mistaken for a missing grid. + continue + if slide_embedding_is_complete( + group, + slide_model_name, + patch_model=patch_model, + device=device, + ): + results[slide_model_name] = ( + f"grids/{selected_grid}/slide/{slide_model_name}" + ) + continue + encoding = encode_slide_embedding( + group, + slide_model_name, + device, + patch_model=patch_model, + ) + if encoding is None: + continue + write_slide_embedding( + group, + slide_model_name, + encoding.vector, + encoding.provenance, + ) + wrote = True + results[slide_model_name] = f"grids/{selected_grid}/slide/{slide_model_name}" + + if wrote: + try: + zarr.consolidate_metadata(root.store) + except Exception: # noqa: BLE001 - consolidation is rebuildable metadata + pass + return results + + def _models_header( embedders: list, model_contracts: dict[str, dict] | None = None, @@ -2378,8 +2592,17 @@ def _models_header( def _build_header( - reader, grid, seg_meta, embedders, slide_id, n, thumbnail, grid_hash, prov, - panel_meta=None, model_contracts=None, + reader, + grid, + seg_meta, + embedders, + slide_id, + n, + thumbnail, + grid_hash, + prov, + panel_meta=None, + model_contracts=None, ) -> dict: header = { "schema_version": SCHEMA_VERSION, diff --git a/src/raw2features/readers/omezarr.py b/src/raw2features/readers/omezarr.py index 58e5a67..ae0c0fe 100644 --- a/src/raw2features/readers/omezarr.py +++ b/src/raw2features/readers/omezarr.py @@ -62,11 +62,11 @@ class _ChunkCache: """Thread-safe LRU cache of decompressed 2D (y, x) chunk planes. Keyed by ``(level, c_index, chunk_y, chunk_x)``; the value is the decompressed - chunk plane (uint8/source dtype, edge-clipped to the array bounds) exactly as - a single chunk-aligned zarr slice returns it. Adjacent ~224 px patches read in - grid order overlap the same 512x512 chunks, so caching the decompressed plane - lets the second patch reuse it instead of re-decompressing - and an RGB read - touches three channel chunks, all cached. + chunk plane (uint8/source dtype, edge-clipped to the array bounds), normalised + from the source axis order to ``(y, x)``. Adjacent ~224 px patches read in grid + order overlap the same 512x512 chunks, so caching the decompressed plane lets + the second patch reuse it instead of re-decompressing - and an RGB read touches + three channel chunks, all cached. The reads run on the pipeline's parallel read-worker pool, so get/put are guarded by a lock. ``capacity`` bounds the cache to N planes (LRU eviction); @@ -171,6 +171,11 @@ def __init__(self, path: str) -> None: # the on-disk chunk grid for whatever layout the store actually uses. self._chunk_hw: list[tuple[int, int]] = [] self._chunk_cache = _ChunkCache(_chunk_cache_capacity()) + # Float pixels have no dtype range to distinguish normalized [0, 1] from + # byte-like [0, 255]. This multiplier is inferred once from level 0 at open + # (255 or 1), then reused for every region and pyramid level. ``None`` means + # the level-0 dtype is not floating and keeps the integer path untouched. + self._float_to_uint8_scale: float | None = None # Query-authenticated HTTP paths use a custom read-only store so the raw # query is attached after every metadata/chunk key. Cache one per child root. self._source_stores: dict[str, object] = {} @@ -258,6 +263,7 @@ def open(self) -> OmeZarrReader: (int(a.chunks[yi]), int(a.chunks[xi])) # type: ignore[attr-defined] for a in self._arrays ] + self._float_to_uint8_scale = self._infer_float_to_uint8_scale(self._arrays[0]) self._channel_names = self._read_omero_channels(ms_path) self._warn_plane_collapse(self._arrays[0].shape) # type: ignore[attr-defined] self._chunk_cache.clear() @@ -265,27 +271,28 @@ def open(self) -> OmeZarrReader: return self def _warn_plane_collapse(self, level0_shape: tuple[int, ...]) -> None: - """Warn if the source has a t or z extent > 1 (only plane 0 is read). + """Warn for every extra axis with extent > 1 (only index 0 is read). - The reader serves a single 2D (y, x[, c]) plane per patch, indexing any time - or depth axis at 0. A multi-timepoint or z-stack source is therefore silently - reduced to its first plane - fine if intended, a trap if not - so we surface it - once at open rather than embedding plane 0 without a word. + The reader serves a single 2D ``(y, x[, c])`` plane per patch, indexing every + axis other than x, y, and c at 0. Any non-singleton extra axis is therefore + reduced to its first plane - fine if intended, a trap if not - so we surface + it once at open rather than embedding plane 0 without a word. """ - for axis in ("t", "z"): - if axis in self._dims: - extent = int(level0_shape[self._dims.index(axis)]) - if extent > 1: - warnings.warn( - f"{source_uri(self.path)}: source has {axis}={extent}; only " - f"{axis}=0 is " - "read (the reader serves one 2D plane per patch).", - stacklevel=2, - ) + for pos, axis in enumerate(self._dims): + if axis in {"x", "y", "c"}: + continue + extent = int(level0_shape[pos]) + if extent > 1: + warnings.warn( + f"{source_uri(self.path)}: source has {axis}={extent}; only " + f"{axis}=0 is read (the reader serves one 2D plane per patch).", + stacklevel=2, + ) def close(self) -> None: self._arrays = [] self._chunk_hw = [] + self._float_to_uint8_scale = None self._chunk_cache.clear() for store in self._source_stores.values(): store.close() # type: ignore[attr-defined] @@ -430,8 +437,12 @@ def read_region_channels(self, region: Region) -> np.ndarray: nc = block.shape[2] if block.shape[0] != h or block.shape[1] != w: out = np.zeros((h, w, nc), dtype=block.dtype) - ph, pw = min(block.shape[0], h), min(block.shape[1], w) - out[:ph, :pw, :] = block[:ph, :pw, :] + dst_y = min(h, max(0, -y0)) + dst_x = min(w, max(0, -x0)) + ph = min(block.shape[0], h - dst_y) + pw = min(block.shape[1], w - dst_x) + if ph > 0 and pw > 0: + out[dst_y : dst_y + ph, dst_x : dst_x + pw, :] = block[:ph, :pw, :] block = out return np.ascontiguousarray(block) @@ -452,7 +463,7 @@ def _read_omero_channels(self, ms_path: str) -> list[str] | None: def _read_hwc( self, arr: object, level: int, x0: int, y0: int, w: int, h: int ) -> np.ndarray: - """Slice a 2-5D level array and normalise to (h, w, 3) uint8. + """Slice a multidimensional level array and normalise to (h, w, 3) uint8. The (y, x) window is assembled from a decompressed-chunk cache (see ``_read_block_cached``) so adjacent/overlapping patches reuse already- @@ -480,13 +491,20 @@ def _read_hwc( # Convert to 8-bit RGB by rescaling on the source dtype range -- never a # truncating ``astype(uint8)``, which would wrap uint16/float pixels # mod 256 and silently corrupt them (e.g. uint16 40000 -> 64). - block = self._to_uint8(block) + block = self._to_uint8( + block, + float_scale=self._float_to_uint8_scale, + ) # Pad to the requested size if the read was clipped at a border. if block.shape[0] != h or block.shape[1] != w: out = np.full((h, w, 3), 255, dtype=np.uint8) - ph, pw = min(block.shape[0], h), min(block.shape[1], w) - out[:ph, :pw, :] = block[:ph, :pw, :] + dst_y = min(h, max(0, -y0)) + dst_x = min(w, max(0, -x0)) + ph = min(block.shape[0], h - dst_y) + pw = min(block.shape[1], w - dst_x) + if ph > 0 and pw > 0: + out[dst_y : dst_y + ph, dst_x : dst_x + pw, :] = block[:ph, :pw, :] block = out return np.ascontiguousarray(block) @@ -496,8 +514,9 @@ def _read_block_cached( """Assemble the (y, x[, c]) window from cached decompressed chunk planes. Returns ``(block, sliced)`` where ``block`` is byte-identical to a direct - ``arr[t0, :, z0, y0:y0+h, x0:x0+w]`` slice (scalar t/z axes dropped, axes in - ``self._dims`` order) and ``sliced`` lists the surviving non-scalar dims. + source-order slice with every extra axis indexed at 0 (and the complete c, + y, and x ranges retained). ``sliced`` lists those surviving dimensions in + ``self._dims`` order. The window is built by copying from per-(channel, chunk_y, chunk_x) planes pulled from ``self._chunk_cache``; a miss decompresses exactly one chunk via @@ -515,8 +534,8 @@ def _read_block_cached( clip_h = max(0, y_hi - y_lo) clip_w = max(0, x_hi - x_lo) - # The surviving (non-scalar) dims in self._dims order; t/z/... collapse to a - # single plane just as the direct slice's scalar 0-index did. + # The surviving dimensions in self._dims order; every extra axis collapses + # to a single plane just as the direct slice's scalar 0-index did. ci = None sliced: list[str] = [] for pos, d in enumerate(self._dims): @@ -572,9 +591,10 @@ def _read_chunk_plane( ) -> np.ndarray: """Decompress one chunk-aligned (y, x) plane for a single channel. - Builds a full index over ``self._dims`` (scalar axes -> 0, the c axis -> the - single channel ``c_val``, y/x -> the chunk's bounds) so the result is a 2D - ``(y, x)`` array, exactly the chunk's pixels the multi-chunk slice would use. + Builds a full index over ``self._dims`` (extra axes -> 0, the c axis -> the + single channel ``c_val``, y/x -> the chunk's bounds), then explicitly + transposes the two surviving spatial dimensions from source order to + ``(y, x)``. This keeps the cache contract independent of source axis order. """ idx: list[object] = [] for d in self._dims: @@ -586,7 +606,13 @@ def _read_chunk_plane( idx.append(c_val) else: idx.append(0) - return np.asarray(arr[tuple(idx)]) # type: ignore[index] + plane = np.asarray(arr[tuple(idx)]) # type: ignore[index] + spatial_order = [d for d in self._dims if d in {"y", "x"}] + plane = np.transpose( + plane, + (spatial_order.index("y"), spatial_order.index("x")), + ) + return np.ascontiguousarray(plane) @staticmethod def _place( @@ -600,7 +626,9 @@ def _place( """Write a 2D (y, x) sub-window into ``block`` at the right axis positions. ``block`` axes follow ``nonscalar`` (dims order). The c axis (if present) is - indexed by the scalar ``c_pos``; y/x take the destination slices. + indexed by the scalar ``c_pos``; y/x take the destination slices. The cached + ``sub`` plane is always ``(y, x)``, so it is transposed back to the block's + source spatial order for assignment when necessary. """ sel: list[object] = [] for d in nonscalar: @@ -610,11 +638,90 @@ def _place( sel.append(dst_y) else: # "x" sel.append(dst_x) - block[tuple(sel)] = sub + spatial_order = [d for d in nonscalar if d in {"y", "x"}] + source_order = ("y", "x") + sub_in_block_order = np.transpose( + sub, + tuple(source_order.index(d) for d in spatial_order), + ) + block[tuple(sel)] = sub_in_block_order + + def _infer_float_to_uint8_scale(self, level0: object) -> float | None: + """Choose one float convention for the opened slide from level zero. + + OME-NGFF does not require float arrays to declare whether pixels use + normalized ``[0, 1]`` or byte-like ``[0, 255]`` values. Scanning a whole + level-zero WSI at open would be prohibitively expensive, so inspect a + deterministic 3x3 lattice of complete source chunks (up to the first three + channels), plus the array fill value. Any finite value above 1 selects the + byte-like convention; otherwise the slide is treated as normalized. + + The important invariant is that this bounded, slide-level decision is made + once. It is never revised from a later patch or lower pyramid level, so two + regions from one float slide cannot receive different scaling merely because + their local value ranges differ. + """ + dtype = np.dtype(level0.dtype) # type: ignore[attr-defined] + if not np.issubdtype(dtype, np.floating): + return None + + fill_value = getattr(level0, "fill_value", None) + try: + fill = float(fill_value) + except (TypeError, ValueError): + fill = float("nan") + if np.isfinite(fill) and fill > 1.0: + return 1.0 + + height = self._level_dims[0].height + width = self._level_dims[0].width + if height <= 0 or width <= 0: + return 255.0 + chunk_h, chunk_w = self._chunk_hw[0] + chunk_rows = (height + chunk_h - 1) // chunk_h + chunk_cols = (width + chunk_w - 1) // chunk_w + sample_rows = sorted({0, (chunk_rows - 1) // 2, chunk_rows - 1}) + sample_cols = sorted({0, (chunk_cols - 1) // 2, chunk_cols - 1}) + + if "c" in self._dims: + ci = self._dims.index("c") + channels: tuple[int | None, ...] = tuple( + range(min(3, int(level0.shape[ci]))) # type: ignore[attr-defined] + ) + else: + channels = (None,) + + for chunk_y in sample_rows: + y0 = chunk_y * chunk_h + y1 = min(y0 + chunk_h, height) + for chunk_x in sample_cols: + x0 = chunk_x * chunk_w + x1 = min(x0 + chunk_w, width) + for channel in channels: + sample = self._read_chunk_plane( + level0, + channel, + y0, + y1, + x0, + x1, + ) + finite = sample[np.isfinite(sample)] + if finite.size and float(finite.max()) > 1.0: + return 1.0 + return 255.0 @staticmethod - def _to_uint8(block: np.ndarray) -> np.ndarray: - """Rescale an image to 8-bit by its dtype range (never wrap mod 256).""" + def _to_uint8( + block: np.ndarray, + *, + float_scale: float | None = None, + ) -> np.ndarray: + """Rescale an image to 8-bit by its dtype range (never wrap mod 256). + + ``float_scale`` is the slide-level convention selected at open. Leaving it + unset preserves the standalone helper's historical local-range behaviour. + """ if block.dtype == np.uint8: return block if np.issubdtype(block.dtype, np.unsignedinteger): @@ -627,11 +734,17 @@ def _to_uint8(block: np.ndarray) -> np.ndarray: scaled = np.rint(np.clip(block.astype(np.float32), 0, None) * scale) return np.clip(scaled, 0, 255).astype(np.uint8) if np.issubdtype(block.dtype, np.floating): - # Float images are usually [0, 1] (normalized) or already [0, 255]; pick the - # convention from the data range (NaNs treated as 0). + # Float images are usually [0, 1] (normalized) or already [0, 255]. The + # reader supplies one level-zero decision; the fallback is retained for + # backwards-compatible direct calls to this internal helper. b = np.nan_to_num(block.astype(np.float32)) - if b.size and float(b.max()) <= 1.0: - b = b * 255.0 + scale = float_scale + if scale is None: + scale = 255.0 if b.size and float(b.max()) <= 1.0 else 1.0 + if scale != 1.0: + # Clip before multiplying so +inf (converted to float32 max by + # nan_to_num) saturates without an overflow warning. + b = np.clip(b, 0, 255.0 / scale) * scale return np.clip(np.rint(b), 0, 255).astype(np.uint8) raise NotImplementedError( f"unsupported OME-Zarr pixel dtype {block.dtype!r}; expected uint8, " diff --git a/src/raw2features/sinks/zarr_sink.py b/src/raw2features/sinks/zarr_sink.py index b221bb3..bd6d9f5 100644 --- a/src/raw2features/sinks/zarr_sink.py +++ b/src/raw2features/sinks/zarr_sink.py @@ -90,6 +90,104 @@ def stored_hash(key: str): ) +def _grid_scaffold_is_usable( + group, + *, + expected_n: int | None = None, + require_mask: bool = False, + expected_mask_shape: tuple[int, ...] | None = None, +) -> bool: + """Whether a grid has the structural scaffold needed for safe model append. + + Missing model arrays are deliberately allowed: adding those arrays is the normal + resume path. A malformed coordinate scaffold or an existing feature column with a + different row count cannot be repaired model-by-model because it breaks the grid's + 1:1 row identity, so the matching grid must instead be rebuilt as a unit. + """ + + try: + if "coords" not in group: + return False + coords = group["coords"] + if ( + getattr(coords, "ndim", None) != 2 + or tuple(coords.shape[1:]) != (2,) + or not np.issubdtype(coords.dtype, np.integer) + ): + return False + n = int(coords.shape[0]) + if expected_n is not None and n != int(expected_n): + return False + + # grid_index is optional in the public format, but when present it must retain + # the same row identity as coords. + if "grid_index" in group: + grid_index = group["grid_index"] + if ( + getattr(grid_index, "ndim", None) != 2 + or tuple(grid_index.shape) != (n, 2) + or not np.issubdtype(grid_index.dtype, np.integer) + ): + return False + + if "features" not in group or not hasattr(group["features"], "keys"): + return False + features = group["features"] + for model in features.keys(): + array = features[model] + if getattr(array, "ndim", None) != 2 or int(array.shape[0]) != n: + return False + + # A mask is required only for a segmented extraction. Unsegmented and legacy + # stores may validly omit it. + if require_mask: + if "mask" not in group: + return False + mask = group["mask"] + if getattr(mask, "ndim", None) != 2: + return False + if expected_mask_shape is not None and tuple(mask.shape) != tuple( + expected_mask_shape + ): + return False + except Exception: # noqa: BLE001 - an unreadable scaffold is not appendable + return False + return True + + +def _unappendable_grid_to_rebuild( + root, + base: str, + grid_hash: str | None, + *, + expected_n: int, + required_mask_shape: tuple[int, ...] | None, +) -> str | None: + """Return the matching target when its grid scaffold cannot be appended safely. + + A same-hash grid is authoritative even when a label collision gave it a suffix. + The sole hashless legacy grid remains the one explicitly-supported compatibility + case. Other hashless/mismatched grids are unrelated and must be preserved. + """ + + if GRIDS not in root: + return None + keys = list(root[GRIDS].keys()) + for key in keys: + g = root[GRIDS][key] + stored_hash = dict(g.attrs.get("raw2features", {})).get("grid_hash") + is_target = stored_hash == grid_hash and grid_hash is not None + is_sole_legacy_target = len(keys) == 1 and key == base and stored_hash is None + if (is_target or is_sole_legacy_target) and not _grid_scaffold_is_usable( + g, + expected_n=expected_n, + require_mask=required_mask_shape is not None, + expected_mask_shape=required_mask_shape, + ): + return key + return None + + @register("sinks", "zarr") class ZarrSink(Sink): """One ``grids//`` per geometry: coords + grid_index + mask + features.""" @@ -138,7 +236,24 @@ def create( else: # use_consolidated=False: read live metadata (a prior run consolidated it). root = zarr.open_group(path, mode="r+", use_consolidated=False) - actual_grid = _select_grid_label(root, grid, header.get("grid_hash")) + rebuild = _unappendable_grid_to_rebuild( + root, + grid, + header.get("grid_hash"), + expected_n=n_patches, + required_mask_shape=( + tuple(np.asarray(grid_tissue).shape) + if grid_tissue is not None + else None + ), + ) + if rebuild is not None: + # Delete only the unusable target. The complete replacement inputs + # are already in hand, and all unrelated grids remain untouched. + del root[GRIDS][rebuild] + actual_grid = rebuild + else: + actual_grid = _select_grid_label(root, grid, header.get("grid_hash")) rh = dict(root.attrs.get("raw2features", {})) rh.setdefault("schema_version", header.get("schema_version")) for k in ("source", "provenance", "segmentation", "thumbnail"): @@ -505,9 +620,7 @@ def write_patches_geojson( try: # Honour the process umask/default ACL, matching a normal output # file rather than mkstemp's fixed 0600 permissions. - fd = os.open( - temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666 - ) + fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o666) break except FileExistsError: temporary = None diff --git a/src/raw2features/slide_embedders/encoding.py b/src/raw2features/slide_embedders/encoding.py index 2106d7d..b4a55d3 100644 --- a/src/raw2features/slide_embedders/encoding.py +++ b/src/raw2features/slide_embedders/encoding.py @@ -165,6 +165,52 @@ def slide_embedding_is_complete( return bool(vector.size and np.isfinite(vector).all() and (vector != 0).any()) +def resolve_slide_grid( + root, + slide_model: str, + *, + grid: str | None = None, + patch_model: str | None = None, +): + """Select the store grid and patch model for one slide encoder request. + + This is the shared selection contract for standalone ``slide-embed`` and inline + ``embed -s`` fallback. A specific slide model is discovered by its required patch + encoder across all grids. Model-agnostic encoders require an explicit choice when + the store has several grids. + """ + from raw2features.core.store import grid_for_model, grid_keys, open_grid + from raw2features.slide_embedders.model_registry import get_slide_spec + + keys = grid_keys(root) + if not keys: + raise ValueError("store has no grids/ group; expected a v0.1 embeddings store") + if grid is not None: + if grid not in keys: + raise ValueError(f"Unknown grid {grid!r}. Available grid keys: {keys}") + selected_grid = grid + elif len(keys) == 1: + selected_grid = keys[0] + elif patch_model is not None: + selected_grid = grid_for_model(root, patch_model) + else: + required = get_slide_spec(slide_model).patch_encoder + if required == "any": + raise ValueError( + f"Slide encoder {slide_model!r} is model-agnostic and the store has " + f"multiple grids {keys}; pass --grid or --patch-model to choose one." + ) + selected_grid = grid_for_model(root, required) + + group = open_grid(root, selected_grid) + selected_patch_model = resolve_slide_patch_model( + group, + slide_model, + patch_model=patch_model, + ) + return selected_grid, group, selected_patch_model + + def resolve_slide_patch_model( group, slide_model: str, diff --git a/src/raw2features/slide_embedders/model_registry.py b/src/raw2features/slide_embedders/model_registry.py index 85c3f42..3493604 100644 --- a/src/raw2features/slide_embedders/model_registry.py +++ b/src/raw2features/slide_embedders/model_registry.py @@ -2,8 +2,9 @@ Built-in slide encoders live in the ``slide_encoders`` section of ``embedders/registry.yaml`` (the same file as patch encoders, to keep all -model provenance in one place). Third-party slide encoders can be added -via the ``raw2features.slide_embedders`` entry-point group. +model provenance in one place). The ``raw2features.slide_embedders`` entry-point +group supplies implementations for names in that packaged registry; an entry point +alone does not extend the CLI slide-encoder catalogue. """ from __future__ import annotations diff --git a/src/raw2features/spec.py b/src/raw2features/spec.py index 55a92bb..eb26510 100644 --- a/src/raw2features/spec.py +++ b/src/raw2features/spec.py @@ -18,10 +18,17 @@ SPEC_VERSION = "0.1" _INT_DTYPES = ( - "int8", "int16", "int32", "int64", - "uint8", "uint16", "uint32", "uint64", + "int8", + "int16", + "int32", + "int64", + "uint8", + "uint16", + "uint32", + "uint64", ) + def _load_header_schema(version: str | None) -> dict | None: """Load the packaged JSON Schema for a store's ``schema_version`` (or ``None``).""" if not version: @@ -87,7 +94,11 @@ def validate_store(path: str) -> list[str]: from raw2features.core.store import GRIDS, grid_keys try: - root = zarr.open_group(path, mode="r") + # Validation is a correctness decision, so inspect the authoritative live + # hierarchy. Consolidated metadata is only a rebuildable listing cache and + # can describe arrays that a crashed replacement has since removed (or omit + # arrays added after the last consolidation). + root = zarr.open_group(path, mode="r", use_consolidated=False) except Exception as exc: # noqa: BLE001 - any open failure is one clear violation return [f"cannot open {path!r} as a zarr group: {exc}"] @@ -130,8 +141,19 @@ def _validate_grid(g, header: dict) -> list[str]: out.append("coords must carry attr units='level0_px'") n = int(c.shape[0]) + patching = header.get("patching") + patching = patching if isinstance(patching, dict) else {} + declared_n = patching.get("n_patches") + if n is not None and isinstance(declared_n, int) and declared_n != n: + out.append(f"header.patching.n_patches {declared_n} != coords length {n}") + + hdr_models = header.get("models") + hdr_models = hdr_models if isinstance(hdr_models, dict) else {} + if "features" not in g: out.append("required group 'features' is missing") + for m in hdr_models: + out.append(f"header.models.{m} has no features/{m} array") else: from raw2features.embedders.fingerprint import ( output_fingerprints_equal, @@ -139,11 +161,12 @@ def _validate_grid(g, header: dict) -> list[str]: ) feats = g["features"] - hdr_models = header.get("models") - hdr_models = hdr_models if isinstance(hdr_models, dict) else {} models = list(feats.keys()) if not models: out.append("'features' group has no model arrays") + for m in hdr_models: + if m not in feats: + out.append(f"header.models.{m} has no features/{m} array") for m in models: a = feats[m] if a.ndim != 2: @@ -175,9 +198,7 @@ def _validate_grid(g, header: dict) -> list[str]: # valid and agree; resume separately requires them for completion. if array_fp is not None or header_fp is not None: if not valid_output_fingerprint(array_fp): - out.append( - f"features/{m} has no valid output_fingerprint attr" - ) + out.append(f"features/{m} has no valid output_fingerprint attr") if not valid_output_fingerprint(header_fp): out.append( f"header.models.{m}.output_fingerprint is missing " diff --git a/tests/test_additive_embed.py b/tests/test_additive_embed.py index 71cfc27..66b8286 100644 --- a/tests/test_additive_embed.py +++ b/tests/test_additive_embed.py @@ -69,7 +69,7 @@ def _make_store( g.attrs["raw2features"] = header root.attrs["raw2features"] = { "source": {"uri": source}, - "grids": {key: {"models": [m for m, _ in models], "grid_hash": grid_hash}} + "grids": {key: {"models": [m for m, _ in models], "grid_hash": grid_hash}}, } return path @@ -189,6 +189,36 @@ def test_inspect_store_absent_is_not_appendable(tmp_path): assert key is None and n == 0 and valid == [] +def test_inspect_store_missing_features_group_is_not_appendable(tmp_path): + path = _make_store(str(tmp_path / "out"), grid_hash="GH") + del open_grid(path, mode="r+")["features"] + + assert _inspect_store(path, "GH", ["a"]) == (None, 0, []) + + +@pytest.mark.parametrize("damage", ["header_only", "grid_index", "feature_rows"]) +def test_inspect_store_rejects_structurally_incomplete_grid(tmp_path, damage): + path = _make_store(str(tmp_path / "out"), grid_hash="GH") + g = open_grid(path, mode="r+") + if damage == "header_only": + del g["coords"] + del g["features"] + elif damage == "grid_index": + g.create_array("grid_index", shape=(3, 2), dtype="int32") + else: + del g["features"]["a"] + g["features"].create_array("a", shape=(3, 4), dtype="float16") + + assert _inspect_store(path, "GH", ["a"]) == (None, 0, []) + + +def test_inspect_store_requires_mask_only_for_segmented_config(tmp_path): + path = _make_store(str(tmp_path / "out"), grid_hash="GH") + + assert _inspect_store(path, "GH", ["a"])[0] == "mpp1_px64" + assert _inspect_store(path, "GH", ["a"], require_mask=True) == (None, 0, []) + + def test_inspect_store_rejects_ambiguous_hashless_multigrid(tmp_path): path = _make_store(str(tmp_path / "out"), grid_hash=None) root = zarr.open_group(path, mode="r+") @@ -266,9 +296,7 @@ def test_store_source_binding_checks_root_and_every_grid(tmp_path): root = zarr.open_group(p, mode="r+") other = root[GRIDS].require_group("mpp2_px64") - other.attrs["raw2features"] = { - "source": {"uri": "file:///source/B.zarr"} - } + other.attrs["raw2features"] = {"source": {"uri": "file:///source/B.zarr"}} with pytest.raises(ValueError, match="Refusing to reuse existing store"): _assert_store_source(p, "file:///source/A.zarr") @@ -347,6 +375,72 @@ def test_open_append_never_clobbers_present_model(tmp_path): assert (np.asarray(g["features"]["a"][:]) == 5.0).all() +@pytest.mark.parametrize( + "damage", + [ + "missing_features", + "header_only", + "grid_index", + "feature_rows", + "missing_required_mask", + ], +) +def test_create_rebuilds_unappendable_target_and_preserves_other_grids( + tmp_path, damage +): + out = str(tmp_path / "out") + path = _make_store(out, "s", n=4, models=(("a", 4),), grid_hash="GH") + root = zarr.open_group(path, mode="r+", use_consolidated=False) + target = open_grid(root, "mpp1_px64") + if damage in {"missing_features", "header_only"}: + del target["features"] + if damage == "header_only": + del target["coords"] + if damage == "grid_index": + target.create_array("grid_index", shape=(3, 2), dtype="int32") + if damage == "feature_rows": + del target["features"]["a"] + target["features"].create_array("a", shape=(3, 4), dtype="float16") + other = root[GRIDS].create_group("unrelated") + sentinel = other.create_array("sentinel", shape=(1,), dtype="int8") + sentinel[:] = 7 + other.attrs["raw2features"] = {"grid_hash": "OTHER"} + + coords = np.arange(8, dtype="int32").reshape(4, 2) + header = { + "grid_hash": "GH", + "source": {"uri": "file:///source/s.zarr"}, + "patching": {"target_mpp": 1.0, "patch_px": 64, "n_patches": 4}, + } + sink = ZarrSink() + key = sink.create( + out, + "s", + grid="mpp1_px64", + fresh=False, + n_patches=4, + coords=coords, + grid_index=coords, + grid_tissue=( + np.ones((2, 2), dtype="float32") + if damage == "missing_required_mask" + else None + ), + model_dims={"a": 4}, + header=header, + ) + sink.write_block("a", 0, np.ones((4, 4), dtype="float16")) + sink.close() + + root = zarr.open_group(path, mode="r", use_consolidated=False) + assert key == "mpp1_px64" + assert set(root[GRIDS].keys()) == {"mpp1_px64", "unrelated"} + np.testing.assert_array_equal(root[GRIDS]["unrelated"]["sentinel"][:], [7]) + assert np.isfinite(open_grid(root, "mpp1_px64")["features"]["a"][:]).all() + if damage == "missing_required_mask": + assert open_grid(root, "mpp1_px64")["mask"].shape == (2, 2) + + # -- end-to-end (needs torch) -------------------------------------------------- @@ -356,7 +450,8 @@ def test_embed_then_add_model_is_additive(synthetic_ngff, tmp_path): common = dict(no_seg=True, target_mpp=0.5, patch_px=64, device="cpu", amp="fp32") sa = run_slide( - synthetic_ngff, out, + synthetic_ngff, + out, RunConfig(models=["mockA"], **common), embedders=[MockEmbedder(dim=8, name="mockA", bias=1.0)], ) @@ -368,7 +463,8 @@ def test_embed_then_add_model_is_additive(synthetic_ngff, tmp_path): assert a_before.shape[1] == 8 sb = run_slide( - synthetic_ngff, out, + synthetic_ngff, + out, RunConfig(models=["mockB"], **common), embedders=[MockEmbedder(dim=5, name="mockB", bias=2.0)], ) @@ -384,6 +480,47 @@ def test_embed_then_add_model_is_additive(synthetic_ngff, tmp_path): assert np.isfinite(b).all() +@pytest.mark.skipif(not _TORCH, reason="torch not installed") +def test_missing_features_group_rebuilds_only_the_target_grid(synthetic_ngff, tmp_path): + out = str(tmp_path / "out") + common = dict(no_seg=True, patch_px=64, device="cpu", amp="fp32") + target_cfg = RunConfig(models=["mockA"], target_mpp=0.5, **common) + other_cfg = RunConfig(models=["mockB"], target_mpp=1.0, **common) + first = run_slide( + synthetic_ngff, + out, + target_cfg, + embedders=[MockEmbedder(dim=8, name="mockA", bias=1.0)], + ) + run_slide( + synthetic_ngff, + out, + other_cfg, + embedders=[MockEmbedder(dim=5, name="mockB", bias=2.0)], + ) + path = first["output_uri"].removeprefix("file://") + root = zarr.open_group(path, mode="r+", use_consolidated=False) + other_before = np.asarray(open_grid(root, "mpp1_px64")["features"]["mockB"][:]) + del open_grid(root, "mpp0.5_px64")["features"] + + rebuilt = run_slide( + synthetic_ngff, + out, + target_cfg, + embedders=[MockEmbedder(dim=8, name="mockA", bias=3.0)], + ) + + assert rebuilt["status"] == "complete" + root = zarr.open_group(path, mode="r", use_consolidated=False) + assert set(root[GRIDS].keys()) == {"mpp0.5_px64", "mpp1_px64"} + assert np.isfinite( + np.asarray(open_grid(root, "mpp0.5_px64")["features"]["mockA"][:]) + ).all() + np.testing.assert_array_equal( + open_grid(root, "mpp1_px64")["features"]["mockB"][:], other_before + ) + + @pytest.mark.skipif(not _TORCH, reason="torch not installed") def test_embed_rerun_same_model_skips(synthetic_ngff, tmp_path): out = str(tmp_path / "out") @@ -597,8 +734,12 @@ def test_embed_slide_writes_multiple_grids_in_one_run(synthetic_ngff, tmp_path): ] embedders = [MockEmbedder(dim=8, name="mockA"), MockEmbedder(dim=5, name="mockB")] summary = embed_slide( - synthetic_ngff, out, cfg, geometry_config=geom, - embedders=embedders, receipts_dir=rec, + synthetic_ngff, + out, + cfg, + geometry_config=geom, + embedders=embedders, + receipts_dir=rec, ) assert summary["status"] == "complete" assert set(summary["grids"]) == {"mpp0.5_px64", "mpp1_px64"} @@ -609,8 +750,12 @@ def test_embed_slide_writes_multiple_grids_in_one_run(synthetic_ngff, tmp_path): # whole-request receipt -> an identical re-run skips both grids again = embed_slide( - synthetic_ngff, out, cfg, geometry_config=geom, - embedders=embedders, receipts_dir=rec, + synthetic_ngff, + out, + cfg, + geometry_config=geom, + embedders=embedders, + receipts_dir=rec, ) assert again["status"] == "skipped" @@ -674,7 +819,10 @@ def test_embed_slide_runs_pooling_encoder_on_each_grid(synthetic_ngff, tmp_path) encoder is whatever that grid holds) -> slide/mean under both grids.""" out = str(tmp_path / "out") cfg = RunConfig( - models=["mockA", "mockB"], no_seg=True, device="cpu", amp="fp32", + models=["mockA", "mockB"], + no_seg=True, + device="cpu", + amp="fp32", slide_encoders=["mean"], ) geom = [ @@ -683,7 +831,11 @@ def test_embed_slide_runs_pooling_encoder_on_each_grid(synthetic_ngff, tmp_path) ] embs = [MockEmbedder(dim=8, name="mockA"), MockEmbedder(dim=5, name="mockB")] summary = embed_slide( - synthetic_ngff, out, cfg, geometry_config=geom, embedders=embs, + synthetic_ngff, + out, + cfg, + geometry_config=geom, + embedders=embs, ) assert summary["status"] == "complete" root = zarr.open_group(summary["output_uri"].removeprefix("file://"), mode="r") diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 95073b8..24de8a5 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -7,8 +7,10 @@ from __future__ import annotations import os +from importlib import import_module import pytest +from click.utils import strip_ansi from typer.testing import CliRunner from conftest import MockEmbedder, build_ngff_v04 @@ -22,15 +24,161 @@ _TORCH = False +def test_embed_cli_seeds_runconfig_from_resolved_geometry(tmp_path, monkeypatch): + """The high-level CLI must not encode "unset" as the real 1.0/224 geometry.""" + embed_module = import_module("raw2features.cli.embed") + captured = {} + + def fake_embed_slide(slide, out_dir, cfg, **kwargs): + captured.update(cfg=cfg, kwargs=kwargs) + return {"status": "complete"} + + monkeypatch.setattr(embed_module, "embed_slide", fake_embed_slide) + result = CliRunner().invoke( + app, + [ + "embed", + str(tmp_path / "unused.zarr"), + str(tmp_path / "out"), + "-m", + "conch_v1_5", + "--mpp", + "1.0", + ], + ) + + assert result.exit_code == 0, result.output + assert (captured["cfg"].target_mpp, captured["cfg"].patch_px) == (1.0, 512) + assert captured["kwargs"]["requested_mpp"] == 1.0 + assert captured["kwargs"]["requested_patch_px"] is None + + +def test_verify_cli_hashes_the_source_mpp_override(tmp_path, monkeypatch): + verify_module = import_module("raw2features.cli.verify") + captured = {} + + def fake_contracts(cfg): + captured["source_mpp"] = cfg.source_mpp + return {} + + monkeypatch.setattr(verify_module, "expected_model_contracts", fake_contracts) + monkeypatch.setattr(verify_module, "is_complete", lambda *args, **kwargs: True) + result = CliRunner().invoke( + app, + [ + "verify", + str(tmp_path / "slide.zarr"), + "--receipts-dir", + str(tmp_path / "receipts"), + "--source-mpp", + "0.37", + "--device", + "cpu", + "--quiet", + ], + ) + + assert result.exit_code == 0, result.output + assert captured["source_mpp"] == pytest.approx(0.37) + + +@pytest.mark.parametrize( + "args", + [ + ["embed", "slide.zarr", "out"], + ["embed-many", "slides", "out"], + ["benchmark", "slide.zarr"], + ["verify", "slide.zarr", "--receipts-dir", "receipts"], + ], +) +def test_commands_reject_unknown_amp_before_doing_work(args): + result = CliRunner().invoke(app, [*args, "--amp", "fast16"]) + output = strip_ansi(result.output) + + assert result.exit_code == 2 + assert "--amp" in output + assert "auto, fp32, bf16, fp16" in output + + +@pytest.mark.parametrize( + "args", + [ + ["embed", "slide.zarr", "out"], + ["embed-many", "slides", "out"], + ["benchmark", "slide.zarr"], + ], +) +@pytest.mark.parametrize("batch_size", ["0", "-3"]) +def test_embedding_commands_reject_nonpositive_batch_size(args, batch_size): + result = CliRunner().invoke(app, [*args, "--batch-size", batch_size]) + output = strip_ansi(result.output) + + assert result.exit_code == 2 + assert "--batch-size" in output + assert "greater than zero" in output + + +@pytest.mark.parametrize( + ("command", "option"), + [ + ("embed", "--mpp"), + ("embed", "--source-mpp"), + ("embed", "--patch-size"), + ("embed", "--step"), + ("embed-many", "--mpp"), + ("embed-many", "--source-mpp"), + ("embed-many", "--patch-size"), + ("embed-many", "--step"), + ("verify", "--mpp"), + ("verify", "--source-mpp"), + ("verify", "--patch-size"), + ("verify", "--step"), + ("benchmark", "--mpp"), + ("benchmark", "--patch-size"), + ], +) +def test_geometry_options_reject_zero_before_work(command, option): + positional = { + "embed": ["slide.zarr", "out"], + "embed-many": ["slides", "out"], + "verify": ["slide.zarr", "--receipts-dir", "receipts"], + "benchmark": ["slide.zarr"], + }[command] + result = CliRunner().invoke(app, [command, *positional, option, "0"]) + output = strip_ansi(result.output) + + assert result.exit_code == 2 + assert option in output + assert "greater than zero" in output + + +def test_mpp_option_rejects_nonfinite_value_before_work(): + result = CliRunner().invoke( + app, ["embed", "slide.zarr", "out", "--mpp", "nan"] + ) + output = strip_ansi(result.output) + + assert result.exit_code == 2 + assert "--mpp" in output + assert "finite and greater than zero" in output + + def _build_store(tmp_path) -> str: from raw2features.pipeline.runner import RunConfig, embed_slide slide = build_ngff_v04(str(tmp_path / "S.zarr")) out = str(tmp_path / "out") - cfg = RunConfig(models=["mock"], no_seg=True, target_mpp=0.5, patch_px=64, - device="cpu", amp="fp32") - embed_slide(slide, out, cfg, - embedders=[MockEmbedder(dim=8, input_size=64, name="mock")]) + cfg = RunConfig( + models=["mock"], + no_seg=True, + target_mpp=0.5, + patch_px=64, + device="cpu", + amp="fp32", + ) + embed_slide( + slide, out, cfg, embedders=[MockEmbedder(dim=8, input_size=64, name="mock")] + ) return os.path.join(out, "S.embeddings.zarr") @@ -58,8 +206,21 @@ def test_verify_cli_exits_1_when_not_complete(tmp_path): receipts = str(tmp_path / "receipts") # empty -> nothing is complete os.makedirs(receipts, exist_ok=True) r = CliRunner().invoke( - app, ["verify", slide, "--receipts-dir", receipts, "-f", "mock", - "--no-seg", "--mpp", "0.5", "--patch-size", "64", "--quiet"], + app, + [ + "verify", + slide, + "--receipts-dir", + receipts, + "-f", + "mock", + "--no-seg", + "--mpp", + "0.5", + "--patch-size", + "64", + "--quiet", + ], ) assert r.exit_code == 1 diff --git a/tests/test_config.py b/tests/test_config.py index 9ade7e2..2f81b1f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -38,6 +38,18 @@ def test_load_extractions_rejects_entry_without_model(tmp_path): load_extractions(str(p)) +@pytest.mark.parametrize( + "geometry", ["mpp: 0", "mpp: .nan", "patch_px: 0", "patch_px: -2"] +) +def test_load_extractions_rejects_nonpositive_or_nonfinite_geometry( + tmp_path, geometry +): + p = tmp_path / "bad.yaml" + p.write_text(f"extractions:\n - {{model: uni, {geometry}}}\n") + with pytest.raises(ValueError, match="greater than zero"): + load_extractions(str(p)) + + # -- slide manifest ------------------------------------------------------------ @@ -90,3 +102,13 @@ def test_load_manifest_rejects_non_numeric_source_mpp(tmp_path): p.write_text("path,source_mpp\n/a/x.zarr,notanumber\n") with pytest.raises(ValueError): # float("notanumber") fails loudly load_manifest(str(p)) + + +@pytest.mark.parametrize("source_mpp", ["0", "-0.5", "nan", "inf"]) +def test_load_manifest_rejects_nonpositive_or_nonfinite_source_mpp( + tmp_path, source_mpp +): + p = tmp_path / "m.csv" + p.write_text(f"path,source_mpp\n/a/x.zarr,{source_mpp}\n") + with pytest.raises(ValueError, match="source_mpp.*greater than zero"): + load_manifest(str(p)) diff --git a/tests/test_config_integrity.py b/tests/test_config_integrity.py index 50634cb..50f333f 100644 --- a/tests/test_config_integrity.py +++ b/tests/test_config_integrity.py @@ -12,6 +12,8 @@ import inspect from dataclasses import fields +import pytest + from raw2features.cli.embed import embed from raw2features.cli.embed_many import embed_many from raw2features.cli.verify import verify @@ -47,6 +49,21 @@ def test_content_hash_is_pinned(): ) +@pytest.mark.parametrize( + ("field", "value"), + [ + ("target_mpp", 0), + ("target_mpp", float("nan")), + ("source_mpp", -0.5), + ("patch_px", 0), + ("step_px", -1), + ], +) +def test_runconfig_rejects_invalid_geometry_at_construction(field, value): + with pytest.raises(ValueError, match=field): + RunConfig(models=["resnet50"], **{field: value}) + + def test_content_hash_is_model_order_independent(): a = RunConfig(models=["uni", "resnet50"]).content_hash() b = RunConfig(models=["resnet50", "uni"]).content_hash() diff --git a/tests/test_coordinate_frame.py b/tests/test_coordinate_frame.py index 0d9da17..84fa639 100644 --- a/tests/test_coordinate_frame.py +++ b/tests/test_coordinate_frame.py @@ -157,8 +157,8 @@ def test_apply_source_mpp_lets_an_uncalibrated_source_proceed(tmp_path): r.apply_source_mpp(0.0) -def test_plane_collapse_warns_on_t_or_z_stack(synthetic_ngff): - # dims are (t, c, z, y, x); a >1 extent on t or z means only plane 0 is read. +def test_plane_collapse_warns_on_non_singleton_fixture_extra_axes(synthetic_ngff): + # The fixture's extra axes are t and z; every extra axis is indexed at zero. with OmeZarrReader(synthetic_ngff) as r: for shape, token in ( ((2, 3, 1, 300, 200), "t=2"), diff --git a/tests/test_embed_many.py b/tests/test_embed_many.py index 9586e75..4198f64 100644 --- a/tests/test_embed_many.py +++ b/tests/test_embed_many.py @@ -21,6 +21,21 @@ _TORCH = False +@pytest.fixture(autouse=True) +def _register_cli_mock_model(monkeypatch): + """CLI tests need an explicit model spec; unknown names now fail closed.""" + + from raw2features.embedders import model_registry + + registry = model_registry.load_registry() + mock_spec = MockEmbedder(dim=8, input_size=64, name="mock").spec + monkeypatch.setattr( + model_registry, + "load_registry", + lambda: {**registry, "mock": mock_spec}, + ) + + # -- sharding (torch-free) ----------------------------------------------------- @@ -103,6 +118,37 @@ def test_embed_many_rejects_duplicate_output_ids_before_model_load( assert "models must not be loaded" not in result.output +@pytest.mark.parametrize("source_mpp", ["0", "-0.5", "nan", "inf"]) +def test_embed_many_rejects_invalid_manifest_source_mpp_before_model_load( + tmp_path, monkeypatch, source_mpp +): + import raw2features.cli.embed_many as em + + manifest = tmp_path / "manifest.csv" + manifest.write_text(f"path,source_mpp\nS.zarr,{source_mpp}\n") + monkeypatch.setattr( + em, + "load_embedders", + lambda *args, **kwargs: pytest.fail("models must not be loaded"), + ) + + result = CliRunner().invoke( + app, + [ + "embed-many", + str(tmp_path), + str(tmp_path / "out"), + "--manifest", + str(manifest), + ], + ) + + assert result.exit_code == 1 + assert "source_mpp" in result.output + assert "greater than zero" in result.output + assert "models must not be loaded" not in result.output + + def test_duplicate_preflight_never_prints_remote_credentials(tmp_path, monkeypatch): import raw2features.cli.embed_many as em diff --git a/tests/test_export_h5.py b/tests/test_export_h5.py index 42f77bd..eb34640 100644 --- a/tests/test_export_h5.py +++ b/tests/test_export_h5.py @@ -24,18 +24,28 @@ def _make_store(tmp_path, slide_id="slideA", models=("mock",)): header = { "schema_version": 1, "patching": { - "level0_patch": _LEVEL0_PATCH, "patch_px": _PATCH_PX, - "achieved_mpp": _ACHIEVED_MPP, "target_mpp": _ACHIEVED_MPP, - "read_level": 0, "coords_convention": "level0_xy", "n_patches": 3, + "level0_patch": _LEVEL0_PATCH, + "patch_px": _PATCH_PX, + "achieved_mpp": _ACHIEVED_MPP, + "target_mpp": _ACHIEVED_MPP, + "read_level": 0, + "coords_convention": "level0_xy", + "n_patches": 3, }, "source": {"uri": "file:///x", "slide_id": slide_id, "mpp_level0": _MPP_LEVEL0}, "models": {m: {"embedding_dim": 4} for m in models}, } sink = ZarrSink() sink.create( - str(tmp_path), slide_id, grid="mpp0.5_px224", n_patches=3, coords=_COORDS, - grid_index=_COORDS, grid_tissue=np.ones(3, dtype=np.float32), - model_dims={m: 4 for m in models}, header=header, + str(tmp_path), + slide_id, + grid="mpp0.5_px224", + n_patches=3, + coords=_COORDS, + grid_index=_COORDS, + grid_tissue=np.ones(3, dtype=np.float32), + model_dims={m: 4 for m in models}, + header=header, ) for m in models: sink.write_block(m, 0, _FEATS.astype(np.float16)) @@ -90,6 +100,29 @@ def test_stamp_layout(tmp_path): assert fh.attrs["extractor"] == "mock" +def test_stamp_honours_per_axis_scale_but_remains_slide_relative(tmp_path): + import h5py + import zarr + + from raw2features.core.store import open_grid + + store = _make_store(tmp_path) + g = open_grid(store, mode="r+") + header = dict(g.attrs["raw2features"]) + header["source"]["mpp_level0"] = 0.3 # scalar mean must not mask anisotropy + header["source"]["scale_um"] = {"x": 0.2, "y": 0.4} + header["source"]["level0_translation_um"] = {"x": 10.0, "y": 20.0} + g.attrs["raw2features"] = header + zarr.consolidate_metadata(store) + + (path,) = export_h5(store, str(tmp_path / "out"), layout="stamp") + with h5py.File(path, "r") as fh: + # STAMP defines coords from the scan's top-left, so the NGFF physical/stage + # origin must not be added even though it remains in native store metadata. + expected = _COORDS.astype(np.float32) * np.array([0.2, 0.4]) + np.testing.assert_allclose(fh["coords"][:], expected, atol=1e-5) + + def test_multi_model_writes_one_file_each(tmp_path): store = _make_store(tmp_path, models=("uni", "resnet50")) paths = export_h5(store, str(tmp_path / "out"), layout="trident") @@ -108,8 +141,13 @@ def _hdr(slide_id, mpp, px, l0): return { "schema_version": "0.1", "patching": { - "achieved_mpp": mpp, "target_mpp": mpp, "patch_px": px, "level0_patch": l0, - "read_level": 0, "coords_convention": "level0_xy", "n_patches": 3, + "achieved_mpp": mpp, + "target_mpp": mpp, + "patch_px": px, + "level0_patch": l0, + "read_level": 0, + "coords_convention": "level0_xy", + "n_patches": 3, }, "source": {"uri": "file:///x", "slide_id": slide_id, "mpp_level0": _MPP_LEVEL0}, "models": {"mock": {"embedding_dim": 4}}, @@ -123,15 +161,33 @@ def test_grid_selector_required_on_multigrid(tmp_path): slide = "s" # Two grids in one store: gridA (0.5/224), gridB (1.0/256), both holding 'mock'. a = ZarrSink() - a.create(str(tmp_path), slide, grid="mpp0.5_px224", fresh=True, n_patches=3, - coords=_COORDS, grid_index=_COORDS, grid_tissue=np.ones(3, np.float32), - model_dims={"mock": 4}, header=_hdr(slide, 0.5, 224, 448)) + a.create( + str(tmp_path), + slide, + grid="mpp0.5_px224", + fresh=True, + n_patches=3, + coords=_COORDS, + grid_index=_COORDS, + grid_tissue=np.ones(3, np.float32), + model_dims={"mock": 4}, + header=_hdr(slide, 0.5, 224, 448), + ) a.write_block("mock", 0, _FEATS.astype(np.float16)) a.close() b = ZarrSink() - b.create(str(tmp_path), slide, grid="mpp1_px256", fresh=False, n_patches=3, - coords=_COORDS, grid_index=_COORDS, grid_tissue=np.ones(3, np.float32), - model_dims={"mock": 4}, header=_hdr(slide, 1.0, 256, 256)) + b.create( + str(tmp_path), + slide, + grid="mpp1_px256", + fresh=False, + n_patches=3, + coords=_COORDS, + grid_index=_COORDS, + grid_tissue=np.ones(3, np.float32), + model_dims={"mock": 4}, + header=_hdr(slide, 1.0, 256, 256), + ) b.write_block("mock", 0, _FEATS.astype(np.float16)) b.close() store = str(tmp_path / f"{slide}.embeddings.zarr") diff --git a/tests/test_multi_gpu.py b/tests/test_multi_gpu.py index 2e9ee4e..ca026e0 100644 --- a/tests/test_multi_gpu.py +++ b/tests/test_multi_gpu.py @@ -52,6 +52,21 @@ def embed_batch(self, batch): return (v.unsqueeze(1).repeat(1, self.spec.embedding_dim) + self._bias).cpu() +@pytest.fixture(autouse=True) +def _register_cli_mock_model(monkeypatch): + """CLI tests declare their mock model spec explicitly; unknown names fail closed.""" + + from raw2features.embedders import model_registry + + registry = model_registry.load_registry() + mock_spec = PosMockEmbedder(dim=8, input_size=64, name="mock").spec + monkeypatch.setattr( + model_registry, + "load_registry", + lambda: {**registry, "mock": mock_spec}, + ) + + def _ramp2d_store(path: str, *, h: int = 256, w: int = 192, mpp0: float = 0.5) -> str: """An OME-NGFF v0.4 store whose pixels encode a 2-D ramp ``(x + y)``. diff --git a/tests/test_output_fingerprints.py b/tests/test_output_fingerprints.py index 4f556f5..3271c54 100644 --- a/tests/test_output_fingerprints.py +++ b/tests/test_output_fingerprints.py @@ -387,13 +387,18 @@ def factory(device): amp="fp32", batch_size=2, ), - requested_mpp=0.5, - requested_patch_px=64, embedder_factory=factory, ) group = open_grid(result["output_uri"]) assert group["features"]["custom"].shape[1] == 8 - assert len(calls) >= 4 # high-level probe, header copy, and two workers + patching = dict(group.attrs["raw2features"])["patching"] + # The external spec is scale-agnostic (1.0 default) with a 64-pixel input. The + # high-level resolver must use that spec; RunConfig's 1.0/224 remains run_slide's + # concrete single-grid geometry rather than an implicit high-level override. + assert (patching["target_mpp"], patching["patch_px"]) == (1.0, 64) + # One shared high-level spec/contract probe, one header copy, and two workers. + # In particular, run_slide must not repeat the contract probe. + assert len(calls) == 4 @pytest.mark.skipif(not _TORCH, reason="torch not installed") diff --git a/tests/test_plugins.py b/tests/test_plugins.py new file mode 100644 index 0000000..17bb4da --- /dev/null +++ b/tests/test_plugins.py @@ -0,0 +1,59 @@ +"""The supported implementation-plugin discovery contract.""" + +from __future__ import annotations + +import pytest + +from raw2features.core import plugins + + +class _EntryPoint: + def __init__(self, name, value=None, error: Exception | None = None): + self.name = name + self._value = value + self._error = error + + def load(self): + if self._error is not None: + raise self._error + return self._value + + +@pytest.fixture +def isolated_plugins(monkeypatch): + registry = {seam: dict(values) for seam, values in plugins._REGISTRY.items()} + monkeypatch.setattr(plugins, "_REGISTRY", registry) + + def install(*entry_points): + monkeypatch.setattr( + plugins, + "entry_points", + lambda *, group: list(entry_points), + ) + + return install + + +def test_importable_entry_point_is_available_and_resolved(isolated_plugins): + implementation = object() + isolated_plugins(_EntryPoint("third_party_reader", implementation)) + + assert "third_party_reader" in plugins.available("readers") + assert plugins.get("readers", "third_party_reader") is implementation + + +def test_entry_point_load_failure_is_skipped(isolated_plugins): + isolated_plugins(_EntryPoint("broken_reader", error=ImportError("optional dep"))) + + assert "broken_reader" not in plugins.available("readers") + with pytest.raises(KeyError, match="broken_reader.*not found"): + plugins.get("readers", "broken_reader") + + +def test_in_process_registration_precedes_same_named_entry_point(isolated_plugins): + local = object() + external = object() + plugins.register("readers", "precedence_probe")(local) + isolated_plugins(_EntryPoint("precedence_probe", external)) + + assert plugins.get("readers", "precedence_probe") is local diff --git a/tests/test_provenance.py b/tests/test_provenance.py index efa0a37..57d3c99 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -4,12 +4,56 @@ import json import shlex +from types import SimpleNamespace import pytest from raw2features.core import provenance +def test_git_sha_accepts_the_raw2features_source_checkout(tmp_path, monkeypatch): + root = tmp_path / "raw2features" + module_file = root / "src/raw2features/core/provenance.py" + module_file.parent.mkdir(parents=True) + module_file.touch() + (root / "pyproject.toml").write_text( + '[project]\nname = "raw2features"\n', encoding="utf-8" + ) + calls = [] + + def fake_run(command, **kwargs): + calls.append((command, kwargs)) + value = f"{root}\n" if "--show-toplevel" in command else "abc123\n" + return SimpleNamespace(returncode=0, stdout=value) + + monkeypatch.setattr(provenance, "__file__", str(module_file)) + monkeypatch.setattr(provenance.subprocess, "run", fake_run) + + assert provenance._git_sha() == "abc123" + assert [call[0][-1] for call in calls] == ["--show-toplevel", "HEAD"] + + +def test_git_sha_rejects_an_unrelated_enclosing_repository(tmp_path, monkeypatch): + outer = tmp_path / "unrelated" + module_file = outer / ".venv/lib/site-packages/raw2features/core/provenance.py" + module_file.parent.mkdir(parents=True) + module_file.touch() + (outer / "pyproject.toml").write_text( + '[project]\nname = "some-other-project"\n', encoding="utf-8" + ) + calls = [] + + def fake_run(command, **kwargs): + calls.append(command) + return SimpleNamespace(returncode=0, stdout=f"{outer}\n") + + monkeypatch.setattr(provenance, "__file__", str(module_file)) + monkeypatch.setattr(provenance.subprocess, "run", fake_run) + + assert provenance._git_sha() is None + assert calls == [["git", "rev-parse", "--show-toplevel"]] + + @pytest.mark.parametrize( "cli", [ @@ -34,8 +78,7 @@ def test_sanitize_cli_redacts_authenticated_uri_and_keeps_semantic_selector(): ) got = provenance.sanitize_cli(cli) assert got == ( - "raw2features embed https://example.org/image.zarr?series=2 " - "out --model uni" + "raw2features embed https://example.org/image.zarr?series=2 out --model uni" ) assert "password" not in got and "R2F_SECRET" not in got diff --git a/tests/test_public_api.py b/tests/test_public_api.py index ff89d58..3cec80f 100644 --- a/tests/test_public_api.py +++ b/tests/test_public_api.py @@ -14,7 +14,7 @@ import pytest PUBLIC = ( - "run_slide", "RunConfig", "register", "available", "get", + "embed_slide", "run_slide", "RunConfig", "register", "available", "get", "write_patches_geojson", "validate_store", ) @@ -27,6 +27,13 @@ def test_public_names_resolve(): assert set(PUBLIC).issubset(set(raw2features.__all__)) +def test_slide_entry_points_state_their_supported_abstraction(): + import raw2features + + assert "high-level public entry point" in raw2features.embed_slide.__doc__ + assert "single-grid primitive" in raw2features.run_slide.__doc__ + + def test_unknown_attr_raises(): import raw2features diff --git a/tests/test_reader_omezarr.py b/tests/test_reader_omezarr.py index 8c743fd..709d9a4 100644 --- a/tests/test_reader_omezarr.py +++ b/tests/test_reader_omezarr.py @@ -9,6 +9,7 @@ from urllib.parse import urlsplit import numpy as np +import pytest import zarr from conftest import build_ngff_v04 @@ -20,9 +21,7 @@ def test_query_auth_is_attached_to_every_http_zarr_request(tmp_path): root = tmp_path / "remote.zarr" zarr.open_group(str(root), mode="w", zarr_format=2) build_ngff_v04(str(root / "0"), sizes=((64, 64), (32, 32))) - expected_query = ( - "token=R2F%2FSECRET%2BVALUE%3D&series=1&series=2&empty=" - ) + expected_query = "token=R2F%2FSECRET%2BVALUE%3D&series=1&series=2&empty=" seen = [] class AuthenticatedHandler(http.server.SimpleHTTPRequestHandler): @@ -65,14 +64,13 @@ def log_message(self, *args): assert any("/remote.zarr/0/0/" in urlsplit(path).path for path in seen) assert any( (request_path := urlsplit(path).path).startswith("/remote.zarr/0/0/") - and re.fullmatch(r"\d+(?:\.\d+)+", request_path.rsplit("/", 1)[-1]) - is not None + and re.fullmatch(r"\d+(?:\.\d+)+", request_path.rsplit("/", 1)[-1]) is not None for path in seen ) def _direct_slice_block(r: OmeZarrReader, level: int, x0: int, y0: int, w: int, h: int): - """Reference: the pre-cache direct multi-chunk zarr slice (scalar t/z dropped).""" + """Reference direct slice, with every extra axis indexed at zero.""" arr = r._arrays[level] idx: list[object] = [] for d in r._dims: @@ -87,6 +85,74 @@ def _direct_slice_block(r: OmeZarrReader, level: int, x0: int, y0: int, w: int, return np.asarray(arr[tuple(idx)]) +def _write_single_level_ngff( + path, + *, + data: np.ndarray, + chunks: tuple[int, ...], + axes: list[dict[str, str]], + scale: list[float], +) -> str: + """Write a minimal offline NGFF v0.4 store with an explicit axis layout.""" + g = zarr.open_group(str(path), mode="w", zarr_format=2) + arr = g.create_array("0", shape=data.shape, chunks=chunks, dtype=data.dtype) + arr[...] = data + g.attrs["multiscales"] = [ + { + "version": "0.4", + "name": "axis-order-regression", + "axes": axes, + "datasets": [ + { + "path": "0", + "coordinateTransformations": [{"type": "scale", "scale": scale}], + } + ], + } + ] + return str(path) + + +def _write_float_pyramid_ngff( + path, + *, + level0: np.ndarray, + level1: np.ndarray, +) -> str: + """Write a two-level c,y,x float pyramid for scaling regressions.""" + g = zarr.open_group(str(path), mode="w", zarr_format=2) + for key, data in (("0", level0), ("1", level1)): + chunks = (1, min(4, data.shape[1]), min(4, data.shape[2])) + arr = g.create_array(key, shape=data.shape, chunks=chunks, dtype=data.dtype) + arr[...] = data + g.attrs["multiscales"] = [ + { + "version": "0.4", + "name": "float-scaling-regression", + "axes": [ + {"name": "c", "type": "channel"}, + {"name": "y", "type": "space", "unit": "micrometer"}, + {"name": "x", "type": "space", "unit": "micrometer"}, + ], + "datasets": [ + { + "path": "0", + "coordinateTransformations": [ + {"type": "scale", "scale": [1.0, 0.5, 0.5]} + ], + }, + { + "path": "1", + "coordinateTransformations": [ + {"type": "scale", "scale": [1.0, 1.0, 1.0]} + ], + }, + ], + } + ] + return str(path) + + def test_metadata_mpp_dims_downsamples(synthetic_ngff): with OmeZarrReader(synthetic_ngff) as r: assert r.mpp == 0.5 @@ -122,6 +188,37 @@ def test_border_read_is_padded_to_requested_size(synthetic_ngff): assert np.all(img[:, 20:, :] == 255) +def test_negative_translated_border_padding_keeps_source_offset(tmp_path): + """A translated level's clipped pixels land after the top/left padding.""" + store = build_ngff_v04( + str(tmp_path / "translated-border.zarr"), sizes=((8, 10), (4, 5)) + ) + root = zarr.open_group(store, mode="r+") + attrs = dict(root.attrs) + datasets = attrs["multiscales"][0]["datasets"] + # Level 1 is 1 µm/px. A (+2,+1) µm source translation maps level-0 (0,0) + # to level-1 index (-2,-1), so the returned 4x4 window needs two white/zero + # columns and one white/zero row before the clipped source data. + datasets[1]["coordinateTransformations"].append( + {"type": "translation", "translation": [0.0, 0.0, 0.0, 1.0, 2.0]} + ) + root.attrs.update(attrs) + + region = Region(level=1, location=Point(0, 0), size=Size(4, 4)) + with OmeZarrReader(store) as reader: + assert reader.read_level_mapping(1)[2:] == (-2.0, -1.0) + native = reader.read_region_channels(region) + rgb = reader.read_region(region) + + source = np.asarray(root["1"][0, :, 0, :3, :2]).transpose(1, 2, 0) + expected_native = np.zeros((4, 4, 3), dtype=np.uint8) + expected_native[1:, 2:, :] = source + expected_rgb = np.full((4, 4, 3), 255, dtype=np.uint8) + expected_rgb[1:, 2:, :] = source + np.testing.assert_array_equal(native, expected_native) + np.testing.assert_array_equal(rgb, expected_rgb) + + def test_exact_mpp_plan_via_reader(synthetic_ngff): with OmeZarrReader(synthetic_ngff) as r: plan = r.level_for_mpp(1.0, 224) # mpp0=0.5 -> level 1 is exactly 1.0 @@ -169,6 +266,71 @@ def test_cached_read_region_equals_uncached(synthetic_ngff): assert np.array_equal(cached_again, uncached) +def test_cached_read_normalises_noncanonical_x_c_y_axis_order(tmp_path): + """An x,c,y source is read with correct orientation across unequal chunks.""" + height, width = 9, 13 + yy, xx = np.indices((height, width)) + expected = np.stack( + [xx + 10 * yy, 80 + xx + 10 * yy, 160 + xx + 10 * yy], axis=-1 + ).astype(np.uint8) + # Source order is deliberately x,c,y rather than a conventional c,y,x. Unequal + # dimensions and chunks make an accidental x/y swap observable. + source = np.transpose(expected, (1, 2, 0)) + store = _write_single_level_ngff( + tmp_path / "x-c-y.zarr", + data=source, + chunks=(5, 1, 3), + axes=[ + {"name": "x", "type": "space", "unit": "micrometer"}, + {"name": "c", "type": "channel"}, + {"name": "y", "type": "space", "unit": "micrometer"}, + ], + scale=[0.5, 1.0, 0.5], + ) + region = Region(level=0, location=Point(3, 2), size=Size(8, 6)) + + with OmeZarrReader(store) as reader: + assert reader.axes == ("x", "c", "y") + native = reader.read_region_channels(region) + rgb = reader.read_region(region) + assert reader._chunk_cache.misses > 1 + + wanted = expected[2:8, 3:11, :] + np.testing.assert_array_equal(native, wanted) + np.testing.assert_array_equal(rgb, wanted) + + +def test_non_singleton_custom_axis_warns_and_reads_index_zero(tmp_path): + """Every extent-bearing extra axis warns; its index-zero plane is selected.""" + height, width = 7, 11 + yy, xx = np.indices((height, width)) + phase0 = np.stack( + [xx + 10 * yy, 70 + xx + 10 * yy, 140 + xx + 10 * yy], axis=0 + ).astype(np.uint8) + phase1 = np.full_like(phase0, 255) + store = _write_single_level_ngff( + tmp_path / "custom-axis.zarr", + data=np.stack([phase0, phase1], axis=0), + chunks=(1, 1, 3, 5), + axes=[ + {"name": "phase", "type": "time", "unit": "second"}, + {"name": "c", "type": "channel"}, + {"name": "y", "type": "space", "unit": "micrometer"}, + {"name": "x", "type": "space", "unit": "micrometer"}, + ], + scale=[1.0, 1.0, 0.5, 0.5], + ) + + with pytest.warns(UserWarning, match=r"phase=2; only phase=0 is read"): + with OmeZarrReader(store) as reader: + got = reader.read_region_channels( + Region(level=0, location=Point(2, 1), size=Size(7, 5)) + ) + + wanted = np.transpose(phase0[:, 1:6, 2:9], (1, 2, 0)) + np.testing.assert_array_equal(got, wanted) + + def test_overlapping_reads_hit_the_cache(synthetic_ngff): """Adjacent/overlapping reads reuse decompressed chunks (cache hits > 0).""" with OmeZarrReader(synthetic_ngff) as r: @@ -218,9 +380,71 @@ def test_to_uint8_handles_unsigned_signed_and_float(): assert (f(np.full((1, 1, 3), 65535, np.uint16)) == 255).all() # uint16 rescaled # signed int: negatives clamp to 0, positive rescales by dtype max np.testing.assert_array_equal( - f(np.array([[[-5, 32767, 0]]], np.int16))[0, 0], [0, 255, 0]) + f(np.array([[[-5, 32767, 0]]], np.int16))[0, 0], [0, 255, 0] + ) # float in [0, 1] -> *255; float already in [0, 255] -> clip passthrough np.testing.assert_array_equal( - f(np.array([[[0.0, 1.0, 0.5]]], np.float32))[0, 0], [0, 255, 128]) + f(np.array([[[0.0, 1.0, 0.5]]], np.float32))[0, 0], [0, 255, 128] + ) np.testing.assert_array_equal( - f(np.array([[[0.0, 255.0, 300.0]]], np.float32))[0, 0], [0, 255, 255]) + f(np.array([[[0.0, 255.0, 300.0]]], np.float32))[0, 0], [0, 255, 255] + ) + + +def test_float_byte_range_is_chosen_once_from_level_zero(tmp_path): + """A locally dark patch must not be mistaken for normalized float pixels.""" + level0 = np.full((3, 12, 12), 0.75, dtype=np.float32) + # A byte-range value in a distant level-zero chunk establishes the convention. + level0[:, -2:, -2:] = 200.0 + level1 = np.full((3, 6, 6), 0.75, dtype=np.float32) + store = _write_float_pyramid_ngff( + tmp_path / "float-byte.zarr", + level0=level0, + level1=level1, + ) + + with OmeZarrReader(store) as reader: + assert reader._float_to_uint8_scale == 1.0 + dark_level0 = reader.read_region( + Region(level=0, location=Point(0, 0), size=Size(3, 3)) + ) + dark_level1 = reader.read_region( + Region(level=1, location=Point(0, 0), size=Size(3, 3)) + ) + native = reader.read_region_channels( + Region(level=0, location=Point(0, 0), size=Size(3, 3)) + ) + + # Per-region inference used to turn these local 0.75 values into 191. The + # level-zero byte-range decision now applies to both levels, while the native + # multiplex-style read remains untouched. + assert np.all(dark_level0 == 1) + assert np.all(dark_level1 == 1) + assert native.dtype == np.float32 + assert np.all(native == np.float32(0.75)) + + +def test_float_unit_range_decision_is_reused_by_lower_levels(tmp_path): + """Lower-level local extrema cannot revise a normalized slide's convention.""" + level0 = np.full((3, 12, 12), 0.5, dtype=np.float32) + level0[:, -2:, -2:] = 1.0 + # Deliberate out-of-contract sentinel: if level 1 reclassified itself from its + # local max, this would become 2 instead of saturating under level zero's choice. + level1 = np.full((3, 6, 6), 2.0, dtype=np.float32) + store = _write_float_pyramid_ngff( + tmp_path / "float-unit.zarr", + level0=level0, + level1=level1, + ) + + with OmeZarrReader(store) as reader: + assert reader._float_to_uint8_scale == 255.0 + normalized = reader.read_region( + Region(level=0, location=Point(0, 0), size=Size(3, 3)) + ) + lower_level = reader.read_region( + Region(level=1, location=Point(0, 0), size=Size(3, 3)) + ) + + assert np.all(normalized == 128) + assert np.all(lower_level == 255) diff --git a/tests/test_release_metadata.py b/tests/test_release_metadata.py new file mode 100644 index 0000000..2c38105 --- /dev/null +++ b/tests/test_release_metadata.py @@ -0,0 +1,22 @@ +"""Release metadata stays synchronized across the package and citation files.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +import yaml + +import raw2features + +ROOT = Path(__file__).resolve().parents[1] + + +def test_release_versions_match() -> None: + project = tomllib.loads((ROOT / "pyproject.toml").read_text()) + citation = yaml.safe_load((ROOT / "CITATION.cff").read_text()) + version = project["project"]["version"] + + assert raw2features.__version__ == version + assert citation["version"] == version + assert f"## [{version}]" in (ROOT / "CHANGELOG.md").read_text() diff --git a/tests/test_resolve_geometry.py b/tests/test_resolve_geometry.py index f5982ac..f8fbbfa 100644 --- a/tests/test_resolve_geometry.py +++ b/tests/test_resolve_geometry.py @@ -2,15 +2,18 @@ With no `--mpp`/`--patch-size`, each model is extracted at its own card geometry (`recommended_mpp`, `recommended_patch_px` or `input_size`); models that disagree -get SEPARATE grids instead of raising. An explicit `--mpp`/`--patch-size` collapses -everything onto one grid; a config controls geometry per model (incl. repeats). +get SEPARATE grids instead of raising. A bare `--mpp` preserves model patch sizes; +supplying both `--mpp` and `--patch-size` collapses everything onto one grid. A config +controls geometry per model (incl. repeats). Pure in (models, mpp, patch_px, config) so every CLI resolves identically. """ from __future__ import annotations +import pytest + +from raw2features.embedders.base import ModelSpec from raw2features.embedders.model_registry import ( - DEFAULT_PATCH_PX, DEFAULT_TARGET_MPP, get_spec, model_geometry, @@ -41,14 +44,44 @@ def test_scale_agnostic_baseline_defaults(): assert model_geometry("resnet50") == (DEFAULT_TARGET_MPP, 224, "default") -def test_unknown_model_falls_back_to_default_geometry(): - assert model_geometry("not_a_real_model") == ( - DEFAULT_TARGET_MPP, - DEFAULT_PATCH_PX, - "default", +def _external_spec( + *, name="external", recommended_mpp=0.75, recommended_patch_px=96 +): + return ModelSpec( + name=name, + family="external", + source="external://weights", + embedding_dim=8, + input_size=64, + pooling="cls", + mean=(0.5, 0.5, 0.5), + std=(0.5, 0.5, 0.5), + transform_source_url="https://example.org/external", + license="MIT", + gated=False, + recommended_mpp=recommended_mpp, + recommended_patch_px=recommended_patch_px, ) +def test_unknown_model_requires_an_injected_specification(): + with pytest.raises(ValueError, match="not in the registry or supplied"): + model_geometry("not_a_real_model") + + +def test_external_model_geometry_comes_from_its_injected_specification(): + spec = _external_spec() + assert model_geometry("external", specs={"external": spec}) == ( + 0.75, + 96, + "recommended", + ) + groups = resolve_geometry(["external"], specs={"external": spec}) + assert [(g.mpp, g.patch_px, g.models) for g in groups] == [ + (0.75, 96, ("external",)) + ] + + # --- per-model grouping (no override) -------------------------------------- @@ -87,15 +120,24 @@ def test_duplicate_model_names_dedup_within_a_grid(): assert len(groups) == 1 and groups[0].models == ("uni",) -# --- global collapse (explicit override) ----------------------------------- +def test_high_level_run_resolution_rejects_an_empty_request(): + from raw2features.pipeline.runner import RunConfig, resolve_run + + with pytest.raises(ValueError, match="at least one model/extraction is required"): + resolve_run(RunConfig(models=[])) + +# --- explicit geometry overrides ------------------------------------------ -def test_explicit_mpp_collapses_all_to_one_grid(): + +def test_explicit_mpp_preserves_each_models_recommended_patch_size(): groups = resolve_geometry(["uni", "conch", "retccl"], requested_mpp=0.5) - assert len(groups) == 1 - assert (groups[0].mpp, groups[0].patch_px) == (0.5, DEFAULT_PATCH_PX) - assert set(groups[0].models) == {"uni", "conch", "retccl"} - assert groups[0].source == "explicit" + assert _geom_by_models(groups) == { + frozenset({"uni"}): (0.5, 224), + frozenset({"conch"}): (0.5, 448), + frozenset({"retccl"}): (0.5, 256), + } + assert all(group.source == "explicit" for group in groups) def test_explicit_mpp_and_patch_size_collapse(): diff --git a/tests/test_resolve_mpp.py b/tests/test_resolve_mpp.py index 28f9ed8..204aa67 100644 --- a/tests/test_resolve_mpp.py +++ b/tests/test_resolve_mpp.py @@ -1,9 +1,10 @@ """`resolve_target_mpp` - auto-select the patch-extraction MPP from the model cards. -Explicit always wins; a single shared `recommended_mpp` is used; scale-agnostic / -unknown models fall back to the default; models that disagree raise (one run = one -scale). The resolver is pure in (models, requested) so embed/embed-many/verify all -resolve identically and their config hashes line up. +Explicit always wins for known models; a single shared `recommended_mpp` is used; +scale-agnostic models fall back to the default; unknown models require an injected +specification; models that disagree raise (one run = one scale). The resolver is pure +in its inputs so embed/embed-many/verify all resolve identically and their config +hashes line up. """ from __future__ import annotations @@ -44,11 +45,22 @@ def test_foundation_plus_scale_agnostic_uses_the_foundation_mpp(): assert resolve_target_mpp(["uni", "resnet50"], None) == (0.5, "auto") -def test_unknown_model_is_ignored_not_an_error(): - assert resolve_target_mpp(["mock_not_in_registry"], None) == ( - DEFAULT_TARGET_MPP, - "auto-default", +def test_unknown_model_requires_an_injected_spec_even_with_explicit_mpp(): + with pytest.raises(ValueError, match="not in the registry or supplied"): + resolve_target_mpp(["mock_not_in_registry"], None) + with pytest.raises(ValueError, match="not in the registry or supplied"): + resolve_target_mpp(["mock_not_in_registry"], 0.5) + + +def test_external_model_spec_participates_in_mpp_resolution(): + spec = ModelSpec( + name="external", family="x", source="x", embedding_dim=1, input_size=32, + pooling="cls", mean=(0.0,), std=(1.0,), transform_source_url="x", + license="MIT", gated=False, recommended_mpp=0.75, ) + assert resolve_target_mpp( + ["external"], None, specs={"external": spec} + ) == (0.75, "auto") def test_conflicting_recommendations_raise(monkeypatch): diff --git a/tests/test_runner_mock.py b/tests/test_runner_mock.py index 32b5c02..3964a4a 100644 --- a/tests/test_runner_mock.py +++ b/tests/test_runner_mock.py @@ -2,6 +2,8 @@ from __future__ import annotations +from dataclasses import replace + import numpy as np import pytest @@ -11,6 +13,44 @@ from raw2features.pipeline.runner import RunConfig, run_slide +def test_high_level_external_embedder_uses_its_spec_geometry(tmp_path, monkeypatch): + """The documented external-instance hook must not silently use 1.0/224.""" + + from raw2features.pipeline import runner + + external = MockEmbedder(dim=8, input_size=64, name="external") + external.spec = replace( + external.spec, + recommended_mpp=0.75, + recommended_patch_px=96, + ) + seen = [] + + def fake_run_slide(slide_path, out_dir, cfg, **kwargs): + seen.append((cfg.target_mpp, cfg.patch_px, tuple(cfg.models))) + return {"status": "complete", "grid": "external-grid"} + + monkeypatch.setattr(runner, "run_slide", fake_run_slide) + # These fields are the concrete geometry for run_slide. The high-level API uses + # model specs unless requested_mpp/requested_patch_px are passed explicitly. + cfg = RunConfig( + models=["external"], + target_mpp=0.5, + patch_px=64, + no_seg=True, + device="cpu", + amp="fp32", + ) + runner.embed_slide( + str(tmp_path / "source.zarr"), + str(tmp_path / "out"), + cfg, + embedders=[external], + ) + + assert seen == [(0.75, 96, ("external",))] + + def test_runner_e2e_mock_no_seg(synthetic_ngff, tmp_path): cfg = RunConfig( models=["mock"], diff --git a/tests/test_slide_embedders.py b/tests/test_slide_embedders.py index 4a3e8d9..acdf926 100644 --- a/tests/test_slide_embedders.py +++ b/tests/test_slide_embedders.py @@ -280,6 +280,33 @@ def test_build_slide_embedder_resolves_pool(): # -- inline runner integration ------------------------------------------------- +def test_store_fallback_discovers_specific_patch_model( + tmp_path, recording_slide_embedder +): + from raw2features.pipeline.runner import _run_slide_encoders_from_store + + path = _write_slide_store( + tmp_path, + { + "mpp0.5_px64": ( + 128, + {"conch_v1_5": np.full((3, 8), 2.0, np.float32)}, + ), + "mpp1_px64": ( + 64, + {"resnet50": np.full((3, 5), 3.0, np.float32)}, + ), + }, + ) + + result = _run_slide_encoders_from_store(path, ["titan"], "cpu") + + assert result == {"titan": "grids/mpp0.5_px64/slide/titan"} + assert "titan" in open_grid(path, "mpp0.5_px64")["slide"] + assert "slide" not in open_grid(path, "mpp1_px64") + assert recording_slide_embedder["loads"] == 1 + + @pytest.mark.skipif(not _TORCH, reason="torch not installed") def test_embed_with_inline_slide_encoder(synthetic_ngff, tmp_path): cfg = RunConfig( @@ -384,6 +411,114 @@ def test_inline_slide_encoder_is_produced_after_complete_receipt( assert recording_slide_embedder["loads"] == 1 +@pytest.mark.skipif(not _TORCH, reason="torch not installed") +def test_embed_slide_cannot_succeed_when_explicit_encoder_produces_nothing( + synthetic_ngff, tmp_path, monkeypatch +): + """An explicit inline ``-s`` request is success only if an output is present.""" + import raw2features.pipeline.runner as runner + + monkeypatch.setattr( + runner, + "run_slide", + lambda *args, **kwargs: { + "status": "complete", + "grid": "mpp0.5_px64", + "slide_embeddings": {}, + }, + ) + monkeypatch.setattr( + runner, + "_run_slide_encoders_from_store", + lambda *args, **kwargs: {}, + ) + cfg = RunConfig( + models=["mock"], + no_seg=True, + device="cpu", + amp="fp32", + slide_encoders=["mean"], + ) + + with pytest.raises(ValueError, match="slide encoder.*mean.*found no grid"): + runner.embed_slide( + synthetic_ngff, + str(tmp_path / "out"), + cfg, + geometry_config=[{"model": "mock", "mpp": 0.5, "patch_px": 64}], + embedders=[MockEmbedder(dim=8, input_size=64, name="mock")], + ) + + +@pytest.mark.skipif(not _TORCH, reason="torch not installed") +def test_inline_specific_encoder_missing_patch_grid_fails_clearly( + synthetic_ngff, tmp_path +): + """A missing required patch model is a final coverage error, not a raw lookup.""" + from raw2features.pipeline.runner import embed_slide + + with pytest.raises( + ValueError, + match=r"slide encoder\(s\) titan \(needs conch_v1_5\) found no grid", + ): + embed_slide( + synthetic_ngff, + str(tmp_path / "out"), + RunConfig( + models=["resnet50"], + no_seg=True, + device="cpu", + amp="fp32", + slide_encoders=["titan"], + ), + geometry_config=[{"model": "resnet50", "mpp": 1.0, "patch_px": 64}], + embedders=[MockEmbedder(dim=5, input_size=64, name="resnet50")], + ) + + +@pytest.mark.skipif(not _TORCH, reason="torch not installed") +def test_inline_specific_encoder_discovers_compatible_existing_grid( + synthetic_ngff, tmp_path, recording_slide_embedder +): + """``embed -s`` can consume a compatible grid from an earlier patch run.""" + from raw2features.core.store import grid_for_model + from raw2features.pipeline.runner import embed_slide + + out = str(tmp_path / "out") + first = embed_slide( + synthetic_ngff, + out, + RunConfig(models=["conch_v1_5"], no_seg=True, device="cpu", amp="fp32"), + geometry_config=[ + {"model": "conch_v1_5", "mpp": 0.5, "patch_px": 64} + ], + embedders=[ + MockEmbedder(dim=8, input_size=64, name="conch_v1_5") + ], + ) + produced = embed_slide( + synthetic_ngff, + out, + RunConfig( + models=["resnet50"], + no_seg=True, + device="cpu", + amp="fp32", + slide_encoders=["titan"], + ), + geometry_config=[{"model": "resnet50", "mpp": 1.0, "patch_px": 64}], + embedders=[MockEmbedder(dim=5, input_size=64, name="resnet50")], + ) + + root = zarr.open_group(first["output_uri"].removeprefix("file://"), mode="r") + conch_grid = open_grid(root, grid_for_model(root, "conch_v1_5")) + resnet_grid = open_grid(root, grid_for_model(root, "resnet50")) + assert produced["status"] == "complete" + assert "titan" in conch_grid["slide"] + assert "slide" not in resnet_grid or "titan" not in resnet_grid["slide"] + assert recording_slide_embedder["loads"] == 1 + + @pytest.mark.skipif(not _TORCH, reason="torch not installed") def test_runner_threads_patch_size_lv0(synthetic_ngff, tmp_path, monkeypatch): """Position-aware encoders (TITAN) need patch_size_lv0; the runner must pass the diff --git a/tests/test_spec_conformance.py b/tests/test_spec_conformance.py index 52a3152..64172c5 100644 --- a/tests/test_spec_conformance.py +++ b/tests/test_spec_conformance.py @@ -17,16 +17,33 @@ def _model_meta(feat_dim): """A fully-populated per-model provenance block (every SPEC-required key).""" return { - "source": "hf-hub:org/model", "embedding_dim": feat_dim, "input_size": 224, - "pooling": "cls", "mean": [0.5, 0.5, 0.5], "std": [0.5, 0.5, 0.5], - "interpolation": "bilinear", "transform_source_url": "https://example/card", - "license": "MIT", "gated": False, "weights_sha256": None, - "weights_revision": "deadbeef", "doi": None, + "source": "hf-hub:org/model", + "embedding_dim": feat_dim, + "input_size": 224, + "pooling": "cls", + "mean": [0.5, 0.5, 0.5], + "std": [0.5, 0.5, 0.5], + "interpolation": "bilinear", + "transform_source_url": "https://example/card", + "license": "MIT", + "gated": False, + "weights_sha256": None, + "weights_revision": "deadbeef", + "doi": None, } -def _store(tmp_path, *, name="s", header="default", coords_n=2, feat_n=2, feat_dim=4, - model="mock", model_in_header=True): +def _store( + tmp_path, + *, + name="s", + header="default", + coords_n=2, + feat_n=2, + feat_dim=4, + model="mock", + model_in_header=True, +): """Build a store by hand so each test can break exactly one rule.""" path = str(tmp_path / f"{name}.embeddings.zarr") key = "mpp0.5_px224" @@ -35,22 +52,37 @@ def _store(tmp_path, *, name="s", header="default", coords_n=2, feat_n=2, feat_d header = { "schema_version": SPEC_VERSION, "source": { - "uri": "file:///x", "slide_id": name, "mpp_level0": 0.25, - "ngff_version": "0.4", "reader": "omezarr", - "level_dimensions": [[1024, 1024]], "level_downsamples": [1.0], + "uri": "file:///x", + "slide_id": name, + "mpp_level0": 0.25, + "ngff_version": "0.4", + "reader": "omezarr", + "level_dimensions": [[1024, 1024]], + "level_downsamples": [1.0], }, "patching": { - "target_mpp": 0.5, "achieved_mpp": 0.5, "patch_px": 224, - "level0_patch": 448, "level0_step": 448, "read_level": 0, - "step_out_px": 224, "n_patches": coords_n, "grid_shape": [1, coords_n], + "target_mpp": 0.5, + "achieved_mpp": 0.5, + "patch_px": 224, + "level0_patch": 448, + "level0_step": 448, + "read_level": 0, + "step_out_px": 224, + "n_patches": coords_n, + "grid_shape": [1, coords_n], "coords_convention": "level0_xy", }, "models": ({model: _model_meta(feat_dim)} if model_in_header else {}), "grid_hash": "abc123", "provenance": { - "raw2features_version": "0", "created_utc": "1970-01-01T00:00:00Z", - "cli": "raw2features embed", "git_sha": None, "host": "h", - "arch": "x86_64", "platform": "test", "python": "3.12", + "raw2features_version": "0", + "created_utc": "1970-01-01T00:00:00Z", + "cli": "raw2features embed", + "git_sha": None, + "host": "h", + "arch": "x86_64", + "platform": "test", + "python": "3.12", }, } # Root discovery header + grids index; the authoritative header is per grid. @@ -58,8 +90,11 @@ def _store(tmp_path, *, name="s", header="default", coords_n=2, feat_n=2, feat_d "schema_version": SPEC_VERSION, "grids": { key: { - "target_mpp": 0.5, "patch_px": 224, "n_patches": coords_n, - "models": [model] if model_in_header else [], "grid_hash": "abc123", + "target_mpp": 0.5, + "patch_px": 224, + "n_patches": coords_n, + "models": [model] if model_in_header else [], + "grid_hash": "abc123", } }, } @@ -80,6 +115,17 @@ def test_valid_store_conforms(tmp_path): assert validate_store(_store(tmp_path)) == [] +def test_validation_reads_live_metadata_not_stale_consolidated_view(tmp_path): + path = _store(tmp_path) + zarr.consolidate_metadata(path) + root = zarr.open_group(path, mode="r+", use_consolidated=False) + del root[GRIDS]["mpp0.5_px224"]["coords"] + + violations = validate_store(path) + + assert any("required array 'coords' is missing" in item for item in violations) + + def test_missing_header(tmp_path): v = validate_store(_store(tmp_path, header=None)) assert any("raw2features" in x for x in v) @@ -90,6 +136,30 @@ def test_length_mismatch_breaks_1to1(tmp_path): assert any("1:1 invariant" in x for x in v) +def test_header_patch_count_must_match_coords(tmp_path): + path = _store(tmp_path, coords_n=2) + g = open_grid(path, mode="r+") + h = dict(g.attrs)["raw2features"] + h["patching"]["n_patches"] = 3 + g.attrs["raw2features"] = h + + v = validate_store(path) + assert any("header.patching.n_patches 3 != coords length 2" in item for item in v) + + +def test_every_header_model_must_have_a_feature_array(tmp_path): + path = _store(tmp_path) + g = open_grid(path, mode="r+") + h = dict(g.attrs)["raw2features"] + h["models"]["missing"] = _model_meta(7) + g.attrs["raw2features"] = h + + v = validate_store(path) + assert any( + "header.models.missing has no features/missing array" in item for item in v + ) + + def test_model_without_header_entry(tmp_path): v = validate_store(_store(tmp_path, model_in_header=False)) assert any("no header.models" in x for x in v) @@ -227,8 +297,15 @@ def test_built_header_aligns_with_schema(synthetic_ngff): with OmeZarrReader(synthetic_ngff) as r: grid = GridPatcher(target_mpp=0.5, patch_px=64).build_grid(r) header = _build_header( - r, grid, {"segmenter": "none"}, [MockEmbedder().load()], - "s", 0, None, "gridhash", provenance.capture("test"), + r, + grid, + {"segmenter": "none"}, + [MockEmbedder().load()], + "s", + 0, + None, + "gridhash", + provenance.capture("test"), ) schema = _load_header_schema(SPEC_VERSION) jsonschema.Draft202012Validator(schema).validate(header) # raises on any mismatch @@ -240,9 +317,18 @@ def test_real_pipeline_output_conforms(synthetic_ngff, tmp_path): from conftest import MockEmbedder from raw2features.pipeline.runner import RunConfig, run_slide - cfg = RunConfig(models=["mock"], segmenter="otsu", target_mpp=0.5, patch_px=64, - tissue_threshold=0.0, device="cpu", amp="fp32", batch_size=8) - summary = run_slide(synthetic_ngff, str(tmp_path / "out"), cfg, - embedders=[MockEmbedder()]) + cfg = RunConfig( + models=["mock"], + segmenter="otsu", + target_mpp=0.5, + patch_px=64, + tissue_threshold=0.0, + device="cpu", + amp="fp32", + batch_size=8, + ) + summary = run_slide( + synthetic_ngff, str(tmp_path / "out"), cfg, embedders=[MockEmbedder()] + ) store = summary["output_uri"].removeprefix("file://") assert validate_store(store) == [], validate_store(store) diff --git a/uv.lock b/uv.lock index 6e606a1..c5b8e65 100644 --- a/uv.lock +++ b/uv.lock @@ -625,7 +625,7 @@ wheels = [ [[package]] name = "dask" -version = "2026.1.1" +version = "2026.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, @@ -637,9 +637,9 @@ dependencies = [ { name = "pyyaml" }, { name = "toolz" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1e/46/61ecde57bac647ca7eb6ffef8dcd90af6c1c649020874cd7fd8738003d62/dask-2026.1.1.tar.gz", hash = "sha256:12b1dbb0d6e92f287feb4076871600b2fba3a843d35ff214776ada5e9e7a1529", size = 10994732, upload-time = "2026-01-16T12:35:30.258Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/39/cbd21c9133d02b4e60899ed466ad5e876553ea68ebffee6209e7dcafc8d4/dask-2026.7.1.tar.gz", hash = "sha256:5727484427665f051e86bf87d021a64d6411141cdc8a20bfe3c1ad2968cc06b7", size = 11548794, upload-time = "2026-07-14T01:06:22.46Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e9/4b/9cc373120658a2516aa5f6dcdde631c95d714b876d29ad8f8e009d793f3f/dask-2026.1.1-py3-none-any.whl", hash = "sha256:146b0ef2918eb581e06139183a88801b4a8c52d7c37758a91f8c3b75c54b0e15", size = 1481492, upload-time = "2026-01-16T12:35:22.602Z" }, + { url = "https://files.pythonhosted.org/packages/7e/5f/7c22733da92b3a6cc4dddcaa8731089d213c2790bbc997e51c429a4e8f8b/dask-2026.7.1-py3-none-any.whl", hash = "sha256:985ffd6c5e9d7979ede515e84ae8d39b647d6aa64f77600f15714ff65f578fe6", size = 1496882, upload-time = "2026-07-14T01:06:20.341Z" }, ] [package.optional-dependencies] @@ -702,28 +702,27 @@ wheels = [ [[package]] name = "distributed" -version = "2026.1.1" +version = "2026.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "cloudpickle" }, - { name = "dask" }, - { name = "jinja2" }, - { name = "locket" }, - { name = "msgpack" }, - { name = "packaging" }, - { name = "psutil" }, - { name = "pyyaml" }, - { name = "sortedcontainers" }, - { name = "tblib" }, - { name = "toolz" }, - { name = "tornado" }, - { name = "urllib3" }, - { name = "zict" }, + { name = "click", marker = "python_full_version >= '3.12'" }, + { name = "cloudpickle", marker = "python_full_version >= '3.12'" }, + { name = "dask", marker = "python_full_version >= '3.12'" }, + { name = "jinja2", marker = "python_full_version >= '3.12'" }, + { name = "locket", marker = "python_full_version >= '3.12'" }, + { name = "msgpack", marker = "python_full_version >= '3.12'" }, + { name = "packaging", marker = "python_full_version >= '3.12'" }, + { name = "psutil", marker = "python_full_version >= '3.12'" }, + { name = "pyyaml", marker = "python_full_version >= '3.12'" }, + { name = "sortedcontainers", marker = "python_full_version >= '3.12'" }, + { name = "tblib", marker = "python_full_version >= '3.12'" }, + { name = "toolz", marker = "python_full_version >= '3.12'" }, + { name = "tornado", marker = "python_full_version >= '3.12'" }, + { name = "zict", marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/45/19/0c13efdffc55cb311594f66c1c8d36a3c4711e427c820155fb9c59138b5e/distributed-2026.1.1.tar.gz", hash = "sha256:3d2709a43912797df3c345af3bb333bbf1a386ec1e9e6a134e5f050521373dbd", size = 2102870, upload-time = "2026-01-16T12:34:58.258Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/e4/567125e6f2f504a6b1e37e3aecd936cf61b662c09e6e79270bd37c93f703/distributed-2026.7.1.tar.gz", hash = "sha256:f7c48ab6961734874521574d75898466703bce93f22f0cff5872c7e4ac6425d4", size = 2461209, upload-time = "2026-07-14T01:20:55.349Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/dc/6d709bcf8fed7611d8a510aeed23b0436cf6db5b61e63c8eb8451eb0d4d8/distributed-2026.1.1-py3-none-any.whl", hash = "sha256:506759b1ed88e45e12ba65e2a429de9911862db55d27dd8bb293c6268430374e", size = 1008417, upload-time = "2026-01-16T12:34:55.535Z" }, + { url = "https://files.pythonhosted.org/packages/3c/4c/369f0b5665afa8ad6fb1862da979dbf61a650ea2b99b5dccda8953aa327c/distributed-2026.7.1-py3-none-any.whl", hash = "sha256:64eaf4406a04ad26acdc5b8e32fad6bac522d07539225a530bab49610b0114a9", size = 1012001, upload-time = "2026-07-14T01:20:53.684Z" }, ] [[package]] @@ -1958,7 +1957,7 @@ wheels = [ [[package]] name = "ngff-zarr" -version = "0.20.3" +version = "0.38.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "dask", extra = ["array"] }, @@ -1974,9 +1973,9 @@ dependencies = [ { name = "zarr", version = "3.1.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "zarr", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/98/2d/11634237253a2234046173627811a98f2d1b4bd21444e4aaba39a83053c3/ngff_zarr-0.20.3.tar.gz", hash = "sha256:c072f1b92c95acf928407d7886b85afbf81b2728782d75a422d70323bee66582", size = 220546, upload-time = "2026-01-26T18:31:00.905Z" } +sdist = { url = "https://files.pythonhosted.org/packages/07/3a/ea45a8fb52516f35eb86fab997145c0babdc0751edd8bc1686422a3956f2/ngff_zarr-0.38.0.tar.gz", hash = "sha256:ca719ad7b4da194986de8360c193e4f5068c06caa349c18c302873830a582104", size = 566671, upload-time = "2026-07-09T20:27:29.169Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/36/0a/baa6733a2cbc0c08495aad756772d0c3b6d74b317518d0f17c2dccae07ef/ngff_zarr-0.20.3-py3-none-any.whl", hash = "sha256:9a74fc5dec9fb60a617ade98f28f9b03348afa5c5b93f48d744ae7e9e2a60482", size = 106186, upload-time = "2026-01-26T18:30:59.614Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ba/659fb680390f648e28e256659640e44fbe02b03e12cccdd20b14827e16f3/ngff_zarr-0.38.0-py3-none-any.whl", hash = "sha256:2f07ca50ad3b912938e4777ce2de9b8f3350f370bf73702143018933f73a0e4a", size = 195036, upload-time = "2026-07-09T20:27:27.59Z" }, ] [[package]] @@ -2285,24 +2284,78 @@ wheels = [ [[package]] name = "ome-zarr" -version = "0.15.0" +version = "0.16.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version <= '3.11' and sys_platform == 'win32'", + "python_full_version <= '3.11' and sys_platform == 'emscripten'", + "python_full_version <= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] dependencies = [ - { name = "aiohttp" }, - { name = "dask" }, - { name = "deprecated" }, - { name = "fsspec", extra = ["s3"] }, - { name = "numpy" }, - { name = "rangehttpserver" }, - { name = "requests" }, - { name = "scikit-image" }, - { name = "toolz" }, - { name = "zarr", version = "3.1.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "aiohttp", marker = "python_full_version <= '3.11'" }, + { name = "dask", marker = "python_full_version <= '3.11'" }, + { name = "deprecated", marker = "python_full_version <= '3.11'" }, + { name = "fsspec", extra = ["s3"], marker = "python_full_version <= '3.11'" }, + { name = "numpy", marker = "python_full_version <= '3.11'" }, + { name = "rangehttpserver", marker = "python_full_version <= '3.11'" }, + { name = "requests", marker = "python_full_version <= '3.11'" }, + { name = "scikit-image", marker = "python_full_version <= '3.11'" }, + { name = "toolz", marker = "python_full_version <= '3.11'" }, + { name = "zarr", version = "3.1.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/45/b5b9933ec62028453dc841d25ea66d65405d77d2eb5bb34043621b0c294c/ome_zarr-0.16.0.tar.gz", hash = "sha256:0d31f3409c0f4ad67f04b9a4e9e8855e6e6b040e84d5c9be1095638f90d6eeb9", size = 86252, upload-time = "2026-04-14T09:43:32.354Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/81/4c41331f59e64b2a5eaec1c03819d5791e4dde321694fbda661f873184da/ome_zarr-0.16.0-py3-none-any.whl", hash = "sha256:a34a5558905af526afd343105804b6d291e97f949cb71f6b3d269a6a7112b599", size = 46410, upload-time = "2026-04-14T09:43:31.092Z" }, +] + +[[package]] +name = "ome-zarr" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "aiohttp", marker = "python_full_version > '3.11'" }, + { name = "dask", marker = "python_full_version > '3.11'" }, + { name = "deprecated", marker = "python_full_version > '3.11'" }, + { name = "fsspec", extra = ["s3"], marker = "python_full_version > '3.11'" }, + { name = "numpy", marker = "python_full_version > '3.11'" }, + { name = "ome-zarr-models", marker = "python_full_version > '3.11'" }, + { name = "rangehttpserver", marker = "python_full_version > '3.11'" }, + { name = "requests", marker = "python_full_version > '3.11'" }, + { name = "scikit-image", marker = "python_full_version > '3.11'" }, + { name = "toolz", marker = "python_full_version > '3.11'" }, + { name = "zarr", version = "3.1.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.11' and python_full_version < '3.12'" }, + { name = "zarr", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/59/98/f39cf0ec00ad64996ed2d2d6059d15a30c7b8f559cd2a216589885001a6a/ome_zarr-0.18.0.tar.gz", hash = "sha256:5cc0fc682b0c177fa32e54f4479b2436f79850a171e83ad4c6b08fec32e66cd0", size = 106039, upload-time = "2026-06-17T16:22:07.76Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/be/2cf9389c9a7b80ceefae88bd713afa6a4b488cb47e524e7dadae44260efd/ome_zarr-0.18.0-py3-none-any.whl", hash = "sha256:aaf6ea6ac5790623290210efc566ad49f92da170b50b07e9113b6b06a0870cc5", size = 57947, upload-time = "2026-06-17T16:22:06.732Z" }, +] + +[[package]] +name = "ome-zarr-models" +version = "1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic", marker = "python_full_version > '3.11'" }, + { name = "pydantic-zarr", version = "0.9.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.11' and python_full_version < '3.12'" }, + { name = "pydantic-zarr", version = "0.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "zarr", version = "3.1.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.11' and python_full_version < '3.12'" }, { name = "zarr", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8b/ee/0e99c62dd3d453c9fb84d8c8cc9f64e255c2d7638546836516282fae2be5/ome_zarr-0.15.0.tar.gz", hash = "sha256:e9479d3a9849c194a616355290f6ae574a7d4402559c5f7005be20cf81945435", size = 81497, upload-time = "2026-04-08T11:57:25.475Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3d/e7/c37b8896a24eabc76d6a5e7855f722792de5bdf3f5ccc562001738be4f7e/ome_zarr_models-1.6.tar.gz", hash = "sha256:00735ad939d2b5f6a4f583cc19af1e8a5247fa1e93f1f08191143cd913b9633a", size = 106913, upload-time = "2026-03-10T10:14:28.369Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/71/159de9929fe98326ada41393d2133fda3f9869ea10bf94996c48846ff68f/ome_zarr-0.15.0-py3-none-any.whl", hash = "sha256:99789b0c4c27fbd197ab1a8bb3ae58b1c7b4e72d0d6b95e9dda6e6416623d3eb", size = 45200, upload-time = "2026-04-08T11:57:24.399Z" }, + { url = "https://files.pythonhosted.org/packages/5b/fc/7650df09dce0edfbaa1f2898c5eb0e0523ae5964dadb83855beb52166505/ome_zarr_models-1.6-py3-none-any.whl", hash = "sha256:d855a6fe885aa2613f64cd124ff8453d75854531bb3282eeaa916e976c7bcf66", size = 64251, upload-time = "2026-03-10T10:14:27.177Z" }, ] [[package]] @@ -2996,6 +3049,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071, upload-time = "2026-05-06T13:38:08.682Z" }, ] +[[package]] +name = "pydantic-zarr" +version = "0.9.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", marker = "python_full_version > '3.11' and python_full_version < '3.12'" }, + { name = "packaging", marker = "python_full_version > '3.11' and python_full_version < '3.12'" }, + { name = "pydantic", marker = "python_full_version > '3.11' and python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/19/ad4f22b7a395408c90cc0f213d70cf5b9ed5f3b0db2b319cacff93b66850/pydantic_zarr-0.9.2.tar.gz", hash = "sha256:59f536cbc456ff4dc64d887d651476e2a6a2ca9f3ea13228dce4e3aeb1cf00b9", size = 80379, upload-time = "2026-03-18T14:38:30.766Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/f4/93a623ab5d2e1c20de0b43e60d6bd8df7704b39e88f5424b4191391a40d9/pydantic_zarr-0.9.2-py3-none-any.whl", hash = "sha256:8009b661cdce3e8283c26d281975432841a39fc8af8309f4ff66722e43850c6f", size = 52952, upload-time = "2026-03-18T14:38:29.684Z" }, +] + +[[package]] +name = "pydantic-zarr" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "numpy", marker = "python_full_version >= '3.12'" }, + { name = "packaging", marker = "python_full_version >= '3.12'" }, + { name = "pydantic", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/57/aac9e500abe15719b7406d55871fc8787e7d0ebf83de5f98d44471139f70/pydantic_zarr-0.10.0.tar.gz", hash = "sha256:996b5b4225fb05757e7fff831ee7a75c61f54514a4c2493c7b79635bade7b415", size = 80374, upload-time = "2026-04-23T14:23:12.575Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/05/04af0ca9c5747c95591a549509ebbb0a986155015de59f1e4c984e3af9ab/pydantic_zarr-0.10.0-py3-none-any.whl", hash = "sha256:d15769dab43346b070051fdf6e83509d7f55259dd00c6d12be2400935117e230", size = 52809, upload-time = "2026-04-23T14:23:11.011Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -3277,7 +3371,7 @@ wheels = [ [[package]] name = "raw2features" -version = "0.1.0" +version = "0.1.1" source = { editable = "." } dependencies = [ { name = "jsonschema" }, @@ -3296,7 +3390,8 @@ all = [ { name = "huggingface-hub" }, { name = "ngff-zarr" }, { name = "numcodecs" }, - { name = "ome-zarr" }, + { name = "ome-zarr", version = "0.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.11'" }, + { name = "ome-zarr", version = "0.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.11'" }, { name = "opencv-python-headless" }, { name = "pillow" }, { name = "safetensors" }, @@ -3446,7 +3541,8 @@ seal = [ ] spatialdata = [ { name = "geopandas" }, - { name = "spatialdata" }, + { name = "spatialdata", version = "0.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "spatialdata", version = "0.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] stain = [ { name = "scikit-learn" }, @@ -3475,7 +3571,8 @@ zarr = [ { name = "fsspec", extra = ["http"] }, { name = "ngff-zarr" }, { name = "numcodecs" }, - { name = "ome-zarr" }, + { name = "ome-zarr", version = "0.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.11'" }, + { name = "ome-zarr", version = "0.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.11'" }, { name = "zarr", version = "3.1.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "zarr", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] @@ -4421,45 +4518,101 @@ wheels = [ [[package]] name = "spatialdata" -version = "0.7.3" +version = "0.7.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform == 'win32'", + "python_full_version <= '3.11' and sys_platform == 'win32'", + "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform == 'emscripten'", + "python_full_version <= '3.11' and sys_platform == 'emscripten'", + "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version <= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] dependencies = [ - { name = "anndata", version = "0.12.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and sys_platform != 'win32') or (python_full_version <= '3.11' and sys_platform == 'win32')" }, - { name = "anndata", version = "0.12.16", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version > '3.11' and sys_platform == 'win32')" }, - { name = "annsel" }, - { name = "click" }, - { name = "dask" }, - { name = "dask-image" }, - { name = "datashader" }, - { name = "distributed" }, - { name = "fsspec", extra = ["http", "s3"] }, - { name = "geopandas" }, - { name = "multiscale-spatial-image" }, - { name = "networkx" }, - { name = "numba" }, - { name = "numpy" }, - { name = "ome-zarr" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version > '3.11' and sys_platform == 'win32')" }, - { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.14' and sys_platform != 'win32') or (python_full_version <= '3.11' and sys_platform == 'win32')" }, - { name = "pooch" }, - { name = "pyarrow" }, - { name = "rich" }, - { name = "scikit-image" }, - { name = "scipy" }, - { name = "setuptools" }, - { name = "shapely" }, - { name = "spatial-image" }, - { name = "typing-extensions" }, - { name = "universal-pathlib" }, - { name = "xarray" }, + { name = "anndata", version = "0.12.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and sys_platform != 'win32') or (python_full_version <= '3.11' and sys_platform == 'win32')" }, + { name = "anndata", version = "0.12.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform == 'win32'" }, + { name = "annsel", marker = "python_full_version < '3.12'" }, + { name = "click", marker = "python_full_version < '3.12'" }, + { name = "dask", marker = "python_full_version < '3.12'" }, + { name = "dask-image", marker = "python_full_version < '3.12'" }, + { name = "datashader", marker = "python_full_version < '3.12'" }, + { name = "fsspec", extra = ["http", "s3"], marker = "python_full_version < '3.12'" }, + { name = "geopandas", marker = "python_full_version < '3.12'" }, + { name = "multiscale-spatial-image", marker = "python_full_version < '3.12'" }, + { name = "networkx", marker = "python_full_version < '3.12'" }, + { name = "numba", marker = "python_full_version < '3.12'" }, + { name = "numpy", marker = "python_full_version < '3.12'" }, + { name = "ome-zarr", version = "0.16.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version <= '3.11'" }, + { name = "ome-zarr", version = "0.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.11' and python_full_version < '3.12'" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version > '3.11' and python_full_version < '3.12' and sys_platform == 'win32'" }, + { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and sys_platform != 'win32') or (python_full_version <= '3.11' and sys_platform == 'win32')" }, + { name = "pooch", marker = "python_full_version < '3.12'" }, + { name = "pyarrow", marker = "python_full_version < '3.12'" }, + { name = "rich", marker = "python_full_version < '3.12'" }, + { name = "scikit-image", marker = "python_full_version < '3.12'" }, + { name = "scipy", marker = "python_full_version < '3.12'" }, + { name = "setuptools", marker = "python_full_version < '3.12'" }, + { name = "shapely", marker = "python_full_version < '3.12'" }, + { name = "spatial-image", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, + { name = "universal-pathlib", marker = "python_full_version < '3.12'" }, + { name = "xarray", marker = "python_full_version < '3.12'" }, + { name = "xarray-schema", marker = "python_full_version < '3.12'" }, { name = "xarray-spatial", version = "0.5.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, - { name = "xarray-spatial", version = "0.10.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "zarr", version = "3.1.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/ae/0330a4bb68945fee2ae485a7318f51cc76911073189db4136effba023426/spatialdata-0.7.0.tar.gz", hash = "sha256:5a0a12d33e9682fe5193abe3430d8bfa106db62440423ae2d508d8a73ad264cc", size = 347530, upload-time = "2026-01-30T12:46:41.777Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/0e/4ed0def70624873f8a4feedd4183c1d090aab55e5d82f36fb3ddef4886cb/spatialdata-0.7.0-py3-none-any.whl", hash = "sha256:6df869089462994330e7b8d9b29dd2e3dbc02c09ce8d91b83a234911012ef3b0", size = 192054, upload-time = "2026-01-30T12:46:40.169Z" }, +] + +[[package]] +name = "spatialdata" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'win32'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", +] +dependencies = [ + { name = "anndata", version = "0.12.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'win32'" }, + { name = "anndata", version = "0.12.16", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version >= '3.12' and sys_platform == 'win32')" }, + { name = "annsel", marker = "python_full_version >= '3.12'" }, + { name = "click", marker = "python_full_version >= '3.12'" }, + { name = "dask", marker = "python_full_version >= '3.12'" }, + { name = "dask-image", marker = "python_full_version >= '3.12'" }, + { name = "datashader", marker = "python_full_version >= '3.12'" }, + { name = "distributed", marker = "python_full_version >= '3.12'" }, + { name = "fsspec", extra = ["http", "s3"], marker = "python_full_version >= '3.12'" }, + { name = "geopandas", marker = "python_full_version >= '3.12'" }, + { name = "multiscale-spatial-image", marker = "python_full_version >= '3.12'" }, + { name = "networkx", marker = "python_full_version >= '3.12'" }, + { name = "numba", marker = "python_full_version >= '3.12'" }, + { name = "numpy", marker = "python_full_version >= '3.12'" }, + { name = "ome-zarr", version = "0.18.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.14' and sys_platform != 'win32') or (python_full_version >= '3.12' and sys_platform == 'win32')" }, + { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'win32'" }, + { name = "pooch", marker = "python_full_version >= '3.12'" }, + { name = "pyarrow", marker = "python_full_version >= '3.12'" }, + { name = "rich", marker = "python_full_version >= '3.12'" }, + { name = "scikit-image", marker = "python_full_version >= '3.12'" }, + { name = "scipy", marker = "python_full_version >= '3.12'" }, + { name = "setuptools", marker = "python_full_version >= '3.12'" }, + { name = "shapely", marker = "python_full_version >= '3.12'" }, + { name = "spatial-image", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.12'" }, + { name = "universal-pathlib", marker = "python_full_version >= '3.12'" }, + { name = "xarray", marker = "python_full_version >= '3.12'" }, + { name = "xarray-spatial", version = "0.10.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "zarr", version = "3.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3c/50/248fae79a62f3d9dffbe70e9b66be6e3f8aaf77caa20a8c931a79256b689/spatialdata-0.7.3.tar.gz", hash = "sha256:6054d66d00e19af96368a897faa0b369f3a25dbf9e0371f091129f8522ee6986", size = 359317, upload-time = "2026-05-06T16:02:42.446Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b7/70/e575bc26b0410cf1349f5115b55d8d119edd6b2cf18e32ab84e222ee17e3/spatialdata-0.8.0.tar.gz", hash = "sha256:14143788a1de83302f8e83a29dd723b2d4e82f5b130d840409822fbf3a778e25", size = 369880, upload-time = "2026-07-02T20:42:20.597Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/bd/76cd29c982ed8f06db839411f1f44c521bf36b8714074652368353856b44/spatialdata-0.7.3-py3-none-any.whl", hash = "sha256:15e0df647f149a8be70cef6b866dcf033c71890440583bca8bb1c20c37566e6c", size = 199244, upload-time = "2026-05-06T16:02:40.994Z" }, + { url = "https://files.pythonhosted.org/packages/cf/08/f5cac3e58d1bb092bfb6015104e6cce925bbe10a2cae44428c790d4e5d06/spatialdata-0.8.0-py3-none-any.whl", hash = "sha256:75139b0df0e27c4573141a8dbe6b3301b47231571505926ad6a633c046ec1010", size = 202596, upload-time = "2026-07-02T20:42:19.268Z" }, ] [[package]] @@ -4992,6 +5145,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bc/ea/bc1de04d06b7c59fc3ff647a11fa248bf80af5a6227647a31c6250c32ce6/xarray_dataclass-3.0.0-py3-none-any.whl", hash = "sha256:7956f29005671b2acaa0da700d8f359105dce84ef84f76ce2dd05b76dba3e826", size = 16862, upload-time = "2025-07-30T20:26:30.696Z" }, ] +[[package]] +name = "xarray-schema" +version = "0.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", marker = "python_full_version < '3.12'" }, + { name = "xarray", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/3d/e0652fad05696e34897f085d4c3115dc4bedb1fb8c1827d6c79ce74095a6/xarray-schema-0.0.3.tar.gz", hash = "sha256:9c6c760489c0690a70394b2ad1368b32f8fa1333911c361b4adf249384212920", size = 15010, upload-time = "2022-04-05T23:20:58.786Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/6d/f585a27b380ee987619b5617c0ca672a71a4345b67cfedbb6299750ce845/xarray_schema-0.0.3-py3-none-any.whl", hash = "sha256:aa6f856626b2e100213ba290407797464608b2555bb8e0b26093a97fe1ba38ce", size = 10019, upload-time = "2022-04-05T23:20:57.301Z" }, +] + [[package]] name = "xarray-spatial" version = "0.5.3"