diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c91b4..db48651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,8 +6,9 @@ Notable changes to raw2features, newest first. This project follows ## [0.2.0] - Unreleased This release extends named-channel multiplex support with a `channelwise` strategy, adds -four patch encoders, and includes a set of correctness and release-safety fixes. The v0.1 -store layout and existing brightfield workflows remain unchanged. +several pinned patch and slide encoders, and includes a set of correctness and +release-safety fixes. The v0.1 store layout and existing brightfield workflows remain +unchanged. ### Correctness and reproducibility @@ -44,6 +45,14 @@ store layout and existing brightfield workflows remain unchanged. and binds every physical index into grid identity; `--no-seg` remains supported. - Add pinned, forward-validated H0-mini, KEEP, OpenMidnight, and OpenPath patch encoders, with their licenses, access requirements, preprocessing, and weight hashes recorded. +- Add the paired GigaPath-Flash tile and slide encoders with immutable weight and loader + pins, and align original GigaPath's default extraction with its released whole-slide + path: non-overlapping 256-pixel tiles on a 256-pixel lattice, followed by a centre crop + to 224. LongNet receives the stored level-0 coordinates and uses each tile's level-0 + extent as its runtime positional divisor, normalizing raw2features' coordinate units + into the tile-index frame of upstream's default 256-pixel tiling. +- Repair HIPT downloads by replacing the unavailable unofficial mirror with Mahmood Lab's + commit-pinned, SHA-verified Git LFS checkpoint and record its Commons Clause accurately. - Correct cached OME-Zarr reads for noncanonical spatial axis order. The reader accepts x/y in either order (with an optional channel axis) and warns whenever another non-singleton axis is reduced to index zero. Unnamed `omero.channels` entries now @@ -72,6 +81,10 @@ store layout and existing brightfield workflows remain unchanged. intentional value change from v0.1 and therefore creates a new grid. Compatible historical native-multiplex grids remain appendable only when the original OME panel proves that the v0.1 selector used the same single physical channel. +- Original GigaPath now defaults to the released 256-pixel non-overlapping extraction + lattice followed by a centre crop to 224. Existing 224-pixel grids remain readable and + distinct; an unmodified default request creates or selects the correctly keyed + 256-pixel grid. - The first request for a model output created before fingerprints were introduced will recompute that model in place. The old header is deliberately not treated as proof that pre-v0.2.0 revision pins were enforced. diff --git a/docs/MODELS.md b/docs/MODELS.md index 94e5424..7113904 100644 --- a/docs/MODELS.md +++ b/docs/MODELS.md @@ -26,6 +26,7 @@ read and decide on. raw2features makes no commercial-use determination. | `virchow` | timm | 2560 | 224 | 0.5 | stated | Apache-2.0 | yes | [HF](https://huggingface.co/paige-ai/Virchow) · [paper](https://doi.org/10.1038/s41591-024-03141-0) | | `virchow2` | timm | 2560 | 224 | 0.5² | stated | CC-BY-NC-ND-4.0 | yes | [HF](https://huggingface.co/paige-ai/Virchow2) · [paper](https://arxiv.org/abs/2408.00738) | | `gigapath` | timm | 1536 | 224³ | 0.5 | stated | Apache-2.0 ⁴ | yes | [HF](https://huggingface.co/prov-gigapath/prov-gigapath) · [GH](https://github.com/prov-gigapath/prov-gigapath) · [paper](https://doi.org/10.1038/s41586-024-07441-w) | +| `gigapath_flash` | timm | 384 | 224³ | 0.5 | stated | Apache-2.0 ⁴ | yes | [HF](https://huggingface.co/prov-gigapath/prov-gigapath-flash) · [GH](https://github.com/prov-gigapath/prov-gigapath) · [paper](https://doi.org/10.48550/arXiv.2607.18218) | | `conch` | conch | 512 | 448 | 0.5 | conv. | CC-BY-NC-ND-4.0 | yes | [HF](https://huggingface.co/MahmoodLab/CONCH) · [GH](https://github.com/mahmoodlab/CONCH) · [paper](https://doi.org/10.1038/s41591-024-02856-4) | | `conch_v1_5` | conch_v1_5 | 768 | 448⁵ | 0.5 | mag→ | CC-BY-NC-ND-4.0 | yes | [HF](https://huggingface.co/MahmoodLab/conchv1_5) · [GH](https://github.com/mahmoodlab/TITAN) · [paper](https://arxiv.org/abs/2411.19666) | | `h_optimus_0` | timm | 1536 | 224 | 0.5 | stated | Apache-2.0 | yes | [HF](https://huggingface.co/bioptimus/H-optimus-0) · [GH](https://github.com/bioptimus/releases) | @@ -85,6 +86,7 @@ Each needs the matching patch encoder run first (`-f `). | `chief` | `ctranspath` (768) | 768 | GPL-3.0 ◊ | no | [GH](https://github.com/hms-dbmi/CHIEF) · [paper](https://doi.org/10.1038/s41586-024-07894-z) | | `tangle` | `uni` (1024) | 512 | CC-BY-NC-ND-4.0 ✦ | no | [GH](https://github.com/mahmoodlab/TANGLE) · [paper](https://arxiv.org/abs/2405.11618) | | `gigapath_slide` | `gigapath` (1536) | 768 | Apache-2.0 ⁴ ✪ | yes | [HF](https://huggingface.co/prov-gigapath/prov-gigapath) · [GH](https://github.com/prov-gigapath/prov-gigapath) · [paper](https://doi.org/10.1038/s41586-024-07441-w) | +| `gigapath_flash_slide` | `gigapath_flash` (384) | 384 | Apache-2.0 ⁴ ✪ | yes | [HF](https://huggingface.co/prov-gigapath/prov-gigapath-flash) · [GH](https://github.com/prov-gigapath/prov-gigapath) · [paper](https://doi.org/10.48550/arXiv.2607.18218) | TITAN runs on CONCH v1.5 patch features, **not** UNI: `raw2features embed slide.ome.zarr out -f conch_v1_5 --patch-size 512 -s titan`. @@ -110,12 +112,21 @@ re-implementation; the weights are on Google Drive (no HF mirror), fetched by th Drive file id and verified against a pinned SHA-256. `tangle` needs the `[tangle]` extra (`gdown`). -✪ `gigapath_slide` needs **`flash-attn`** (no SDPA fallback). Prebuilt wheels exist for +✪ `gigapath_slide` and `gigapath_flash_slide` need **`flash-attn`** (no SDPA fallback). +Prebuilt wheels exist for x86 + torch 2.7-2.9; for torch 2.6 or aarch64 torch ≥2.10, build it from source (needs -`nvcc`). GPU-validated with a source-built flash-attn (torch 2.12). Where flash-attn isn't installed, `gigapath_slide` is simply unavailable -(its test skips cleanly), so the rest of the toolkit is unaffected. Needs the +`nvcc`). GPU-validated with a source-built flash-attn (torch 2.12). Where flash-attn is +not installed, these slide encoders are unavailable (their tests skip cleanly), so the +rest of the toolkit is unaffected. Both need the `[gigapath_slide]` extra + the gigapath git package + `fairscale` + a flash-attn matching -your torch/ABI. +your torch/ABI. The Flash tile encoder also needs the pinned git package to register its +architecture, but it does not itself use flash-attn: + +```bash +pip install "raw2features[gigapath_slide]" +pip install --no-deps fairscale "git+https://github.com/prov-gigapath/prov-gigapath.git@9d42a60babe04359978d5ad2eb94e7b3bcf9ca39" +pip install flash-attn # slide encoders only; use a wheel/build matching torch + CUDA +``` ### Legend @@ -148,8 +159,18 @@ your torch/ABI. 2. Multi-scale training - `virchow2` (2.0 / 1.0 / 0.5 / 0.25 µm/px), `midnight` (2 / 1 / 0.5 / 0.25 µm/px), the `lunit_*` family (20× = 0.5 **and** 40× = 0.25 µm/px). `0.5` is one of several training scales; raw2features defaults to it. -3. `gigapath` - tiles are 256 px at 0.5 µm/px, centre-cropped to 224 at inference. -4. `gigapath` - the `LICENSE` file is Apache-2.0; the model card additionally states "any +3. `gigapath` / `gigapath_flash` - the released whole-slide path extracts + non-overlapping 256 px tiles at 0.5 µm/px on a 256 px lattice, then centre-crops + each tile to 224 px for the tile encoder. The crop discards a 16 px margin on every + side; those margins are not seen by the encoder. Coordinates remain those of the + original 256 px tiles. raw2features passes the stored level-0 coordinates unchanged + to the paired LongNet and sets its runtime coordinate divisor to the tile's level-0 + extent. This normalizes raw2features' coordinate units into the tile-index frame of + upstream's default 256 px tiling; model construction remains at the upstream 256 px + default. Overlapping grids remain supported and every token reaches LongNet, although + origins within one full-tile cell share its tile-sized positional encoding. +4. `gigapath` / `gigapath_flash` - the `LICENSE` file is Apache-2.0; the model cards + additionally state "any deployed use case … commercial or otherwise … is out of scope" and restricts use to research/reproducibility. Both apply - read them and decide. 5. Larger / different tiles - extract `conch_v1_5` at 512 px @ 20× (resized to 448: diff --git a/docs/MODEL_LICENSES.md b/docs/MODEL_LICENSES.md index 1ccedf0..1ec2cb4 100644 --- a/docs/MODEL_LICENSES.md +++ b/docs/MODEL_LICENSES.md @@ -17,7 +17,8 @@ table below lists each model's weights licence. |---|---|---| | `resnet50` | BSD-3-Clause (torchvision) | no | | `dinov2` | Apache-2.0 | no | -| `gigapath` | Apache-2.0 | yes (login) | +| `gigapath` | Apache-2.0 | yes (terms acceptance) | +| `gigapath_flash` | Apache-2.0 | yes (terms acceptance) | | `h_optimus_0` | Apache-2.0 | yes (login) | | `h0_mini` | CC-BY-NC-ND-4.0 | yes (approval) | | `gpfm` | MIT (HF metadata) / CC-BY-NC-ND-4.0 (repo `LICENSE`) - conflict, verify | no | @@ -63,6 +64,7 @@ below. `mean` / `max` / `meanmax` are weightless pooling baselines (MIT). | `prism` | CC-BY-NC-ND-4.0 | yes | | `madeleine` | MIT (HF card) / CC-BY-NC-ND-4.0 (GitHub `LICENSE`) - conflict, verify | yes | | `feather_conch_v15` / `feather_uni_v2` / `feather_uni` | CC-BY-NC-ND-4.0 | yes | +| `gigapath_slide` / `gigapath_flash_slide` | Apache-2.0 | yes (terms acceptance) | | `chief` | GPL-3.0 (weights; aggregator code MIT) | no (Google Drive) | | `tangle` | CC-BY-NC-ND-4.0 | no (Google Drive) | diff --git a/pyproject.toml b/pyproject.toml index b2c2f14..46a620e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,6 +97,7 @@ feather_conch_v15 = "raw2features.slide_embedders.feather:FeatherSlideEmbedder" feather_uni_v2 = "raw2features.slide_embedders.feather:FeatherSlideEmbedder" feather_uni = "raw2features.slide_embedders.feather:FeatherSlideEmbedder" gigapath_slide = "raw2features.slide_embedders.gigapath_slide:GigapathSlideEmbedder" +gigapath_flash_slide = "raw2features.slide_embedders.gigapath_slide:GigapathSlideEmbedder" chief = "raw2features.slide_embedders.chief:CHIEFSlideEmbedder" tangle = "raw2features.slide_embedders.tangle:TangleSlideEmbedder" @@ -158,14 +159,15 @@ prism = ["raw2features[torch,models]", "environs", "sacremoses"] # package (post-install, like conch): # pip install git+https://github.com/mahmoodlab/MADELEINE.git@419287dc60a57296d959840b893481019c4f0d21 madeleine = ["raw2features[torch,models]"] -# Prov-GigaPath slide encoder (LongNet over gigapath 1536-d tile features). Two post-install -# steps (all --no-deps so they can't repin torch): the gigapath git package provides -# slide_encoder.create_model and vendors torchscale (the embedder aliases it as top-level -# `torchscale`, so no separate PyPI torchscale); and flash-attn is a HARD dep with no SDPA +# Prov-GigaPath original + Flash tile/slide encoders. Two post-install steps (all +# --no-deps so they can't repin torch): the gigapath git package registers the Flash tile +# architecture, provides slide_encoder.create_model, and vendors torchscale (the slide +# embedder aliases it as top-level `torchscale`); flash-attn is a HARD slide dependency +# with no SDPA # fallback, so install a wheel matching your torch/ABI - or source-build it (needs nvcc; ~15-20 min). Prebuilt wheels: x86 + torch 2.7-2.9; for torch 2.6 # or aarch64 torch >=2.10, build from source. Needs an Ampere+/Blackwell GPU + accepted # prov-gigapath/prov-gigapath gate (HF_TOKEN): -# pip install --no-deps fairscale "git+https://github.com/prov-gigapath/prov-gigapath.git@3505f87e197d167522be491bb3f18fb5a08ca584" +# pip install --no-deps fairscale "git+https://github.com/prov-gigapath/prov-gigapath.git@9d42a60babe04359978d5ad2eb94e7b3bcf9ca39" # pip install flash-attn (prebuilt wheel matching torch/ABI, or --no-build-isolation source build) gigapath_slide = ["raw2features[torch,models]"] # CHIEF slide encoder (Wang et al., Nature 2024): gated-attention MIL over CTransPath. Its diff --git a/src/raw2features/embedders/base.py b/src/raw2features/embedders/base.py index fd24203..a69829e 100644 --- a/src/raw2features/embedders/base.py +++ b/src/raw2features/embedders/base.py @@ -1,11 +1,11 @@ """``Embedder`` - the foundation-model seam, and the ``ModelSpec`` it consumes. -Design note: the patcher delivers patches at *exactly* the model's -input size and target MPP, so embedders **normalise only** - they do not apply the -model card's eval-time Resize/CenterCrop, which would re-scale the patch and -distort the MPP we worked to hit exactly. The normalisation mean/std are sourced -from each model's card (recorded in the registry with ``transform_source_url``); -we never guess them. +Design note: the patcher delivers patches at the registry's extraction size and +target MPP. Most embedders then normalize only. A small number of models declare a +larger extraction tile and a source-backed resize or centre crop to their fixed input; +that preprocessing is explicit in the model specification and output fingerprint. The +normalization mean/std are sourced from each model's card (recorded in the registry with +``transform_source_url``); we never guess them. """ from __future__ import annotations @@ -146,8 +146,16 @@ class ModelSpec: doi: str | None = None # Resolved strategy contract for a derived multiplex output. Ordinary registry # models leave this unset, preserving their v0.1 fingerprints byte-for-byte. Kept - # last so adding it does not shift any pre-existing positional constructor fields. + # after the pre-existing fields so it does not shift their positional constructors. multiplex: dict[str, Any] | None = None + # Optional model-specific eval crop. When set, the shared transform first resizes + # the square patch to ``round(input_size / crop_pct)`` and then applies + # ``crop_mode`` to reach ``input_size``. + crop_pct: float | None = None + crop_mode: str | None = None + # Optional upstream modules that register custom timm architectures. These are + # packaged registry values, never user-supplied runtime imports. + registration_modules: tuple[str, ...] = () @property def extract_px(self) -> int: @@ -202,6 +210,8 @@ def transform_signature(self) -> tuple: self.spec.mean, self.spec.std, self.spec.interpolation, + self.spec.crop_pct, + self.spec.crop_mode, ) @property diff --git a/src/raw2features/embedders/fingerprint.py b/src/raw2features/embedders/fingerprint.py index e6974f7..5b45374 100644 --- a/src/raw2features/embedders/fingerprint.py +++ b/src/raw2features/embedders/fingerprint.py @@ -114,7 +114,7 @@ CONCH_PACKAGE_REVISION = "141cc09c7d4ff33d8eda562bd75169b457f71a62" KRONOS_PACKAGE_REVISION = "48979362386c8440c934954be3d88ccfa74d6f36" MUSK_PACKAGE_REVISION = "714b666969c1911e5efe70d991140a21030f4ef3" -GIGAPATH_PACKAGE_REVISION = "3505f87e197d167522be491bb3f18fb5a08ca584" +GIGAPATH_PACKAGE_REVISION = "9d42a60babe04359978d5ad2eb94e7b3bcf9ca39" MADELEINE_PACKAGE_REVISION = "419287dc60a57296d959840b893481019c4f0d21" BIOMEDCLIP_TEXT_REPO = "microsoft/BiomedNLP-BiomedBERT-base-uncased-abstract" BIOMEDCLIP_TEXT_REVISION = "d673b8835373c6fa116d6d8006b33d48734e305d" @@ -423,6 +423,10 @@ def _patch_constructor(spec: ModelSpec) -> dict[str, Any]: }, } constructor.update(family_contracts.get(spec.family, {"entrypoint": spec.family})) + if spec.registration_modules: + constructor["registration_modules"] = list(spec.registration_modules) + if "gigapath.tile_encoder" in spec.registration_modules: + constructor["construction_package_revision"] = GIGAPATH_PACKAGE_REVISION if spec.name == "biomedclip": constructor["nested_text_config"] = { "repo": BIOMEDCLIP_TEXT_REPO, @@ -518,6 +522,13 @@ def patch_output_fingerprint(spec: ModelSpec, resolved_amp: str) -> dict[str, An }, "composite": _seal_composite(spec) if spec.family == "seal" else None, } + if spec.crop_pct is not None: + payload["preprocessing"].update( + { + "crop_pct": float(spec.crop_pct), + "crop_mode": spec.crop_mode, + } + ) return make_output_fingerprint(payload) @@ -637,10 +648,18 @@ def _slide_constructor(spec) -> dict[str, Any]: }, "gigapath_slide": { "entrypoint": "gigapath.slide_encoder.create_model", - "architecture": "gigapath_slide_enc12l768d", - "patch_dim": 1536, + "architecture": spec.architecture or "gigapath_slide_enc12l768d", + "patch_dim": int(spec.patch_dim), "global_pool": True, - "tile_size_source": "patching.level0_patch", + "constructor_tile_size": 256, + "slide_ngrids": 1000, + "runtime_position_divisor_source": "patching.level0_patch", + "coords_frame": "stored_level0_xy", + "coords_transform": "none", + "position_mapping": ( + "floor(x / divisor) * slide_ngrids " + "+ floor(y / divisor) + 1" + ), "coords_dtype": "float32", "features_dtype": "float32", "all_layer_embed": True, diff --git a/src/raw2features/embedders/model_registry.py b/src/raw2features/embedders/model_registry.py index 17b979e..1495e53 100644 --- a/src/raw2features/embedders/model_registry.py +++ b/src/raw2features/embedders/model_registry.py @@ -51,6 +51,9 @@ def load_registry() -> dict[str, ModelSpec]: experimental=bool(d.get("experimental", False)), notes=d.get("notes", ""), doi=d.get("doi"), + crop_pct=d.get("crop_pct"), + crop_mode=d.get("crop_mode"), + registration_modules=tuple(d.get("registration_modules") or ()), ) return specs diff --git a/src/raw2features/embedders/registry.yaml b/src/raw2features/embedders/registry.yaml index 8b93ebe..f6517c8 100644 --- a/src/raw2features/embedders/registry.yaml +++ b/src/raw2features/embedders/registry.yaml @@ -212,8 +212,11 @@ gigapath: family: timm source: "hf-hub:prov-gigapath/prov-gigapath" recommended_mpp: 0.5 # paper Methods state 0.5 µm/px / 20x (10.1038/s41586-024-07441-w); card silent; single-scale + recommended_patch_px: 256 # official tiling is 256 px at 20x, then centre-crop to 224 embedding_dim: 1536 input_size: 224 + crop_pct: 0.875 + crop_mode: center pooling: cls # num_classes=0 -> forward returns the [B,1536] pooled feature mean: [0.485, 0.456, 0.406] std: [0.229, 0.224, 0.225] @@ -223,9 +226,9 @@ gigapath: weights_filename: "pytorch_model.bin" weights_revision: "eba85dd46097c3eedfcc2a3a9a930baecb6bcc19" doi: "10.1038/s41586-024-07441-w" - license: "Apache-2.0 (Prov-GigaPath; open, gated only by HF login)" + license: "Apache-2.0 (Prov-GigaPath; gated, model-card use terms apply)" gated: true - inference_amp: fp32 # card gives no precision recommendation -> faithful fp32 + inference_amp: fp16 # authors' released tile-inference pipeline uses CUDA fp16 autocast timm_kwargs: num_classes: 0 dynamic_img_size: true @@ -234,8 +237,45 @@ gigapath: timm.create_model("hf-hub:prov-gigapath/prov-gigapath"). num_classes=0 -> the forward returns the pooled 1536-d tile embedding directly (pooling: cls is a pass-through for the already-pooled 2-D output). ImageNet norm, 224 px, bicubic - per the card. Weights are Apache-2.0 (gated only by HF login), so unlike the - CC-BY-NC-ND models its embeddings are unrestricted. + per the card, after the documented 256 px tile is centre-cropped to 224. Weights + are Apache-2.0; the model card separately limits intended use and excludes deployed + use, so users must review both the licence and access terms. + +gigapath_flash: + family: timm + source: "gigapath_tile_enc_dinov2s" + recommended_mpp: 0.5 # paper evaluates non-overlapping 256 px tiles at 20x + recommended_patch_px: 256 + embedding_dim: 384 + input_size: 224 + crop_pct: 0.875 + crop_mode: center + pooling: cls + mean: [0.485, 0.456, 0.406] + std: [0.229, 0.224, 0.225] + interpolation: bicubic + transform_source_url: "https://huggingface.co/prov-gigapath/prov-gigapath-flash" + weights_sha256: "ea5e36c2b3593b073868f91b9016559a620f0d6752e3861cb2bb0b5312238473" + weights_filename: "pytorch_model.bin" + weights_revision: "f5c3a709ad81d3a5605bb2eb38cd5c2e53da0375" + doi: "10.48550/arXiv.2607.18218" + license: "Apache-2.0 (Prov-GigaPath-Flash; gated, research/deployment terms in model card)" + gated: true + inference_amp: fp16 # authors' released tile-inference pipeline uses CUDA fp16 autocast + registration_modules: ["gigapath.tile_encoder"] + checkpoint: + repo: "prov-gigapath/prov-gigapath-flash" + filename: "pytorch_model.bin" + strict: true + timm_kwargs: + num_classes: 0 + notes: >- + Prov-GigaPath-Flash tile encoder, a DINOv2-small ViT-S/16 with SwiGLU and + LayerScale. Importing the pinned gigapath package registers the upstream + gigapath_tile_enc_dinov2s architecture; raw2features then downloads the immutable + checkpoint, verifies its SHA-256 before deserialisation, and loads it strictly. + The official 256 px tile at 20x is centre-cropped to the fixed 224 px input and + produces the pooled 384-d CLS embedding consumed by gigapath_flash_slide. conch: family: conch # optional family: needs the `conch` pip extra @@ -766,8 +806,7 @@ hipt: source: "vit_small_patch16_224" # 224 base (checkpoint pos_embed=197); pos_embed is interpolated to the 256px grid at forward recommended_mpp: 0.5 # 20x - repo+paper (patch dirs 'extracted_mag20x_...'); 0.5 by convention; single-scale checkpoint: - repo: "JWonderLand/HIPT_unofficial" - filename: "vit256_small_dino.pth" + url: "https://media.githubusercontent.com/media/mahmoodlab/HIPT/780fafaed2e5b112bc1ed6e78852af1fe6714342/HIPT_4K/Checkpoints/vit256_small_dino.pth" state_dict_key: "teacher" strip_prefixes: ["module.", "backbone."] strict: false # DINO projection-head keys are absent from the backbone @@ -780,9 +819,9 @@ hipt: transform_source_url: "https://github.com/mahmoodlab/HIPT/blob/master/HIPT_4K/hipt_model_utils.py" weights_sha256: "6960cd5a8657dc8bb214671aa0c6dbd3f5b698e84386884955836487ddc89e24" weights_filename: "vit256_small_dino.pth" - weights_revision: "3529ea3d95a390e210768ca4db29fa7cae39ded9" + weights_revision: "780fafaed2e5b112bc1ed6e78852af1fe6714342" doi: "10.48550/arXiv.2206.02647" - license: "Apache-2.0 WITH Commons Clause (Mahmood Lab HIPT; non-commercial)" + license: "Apache-2.0 with Commons Clause (Mahmood Lab HIPT; selling the software is restricted)" gated: false inference_amp: fp32 timm_kwargs: @@ -794,8 +833,9 @@ hipt: forward, exactly as HIPT's own get_vit256 does. Loads the 'teacher' sub-dict of the ~700MB DINO checkpoint (strip module./backbone., strict=False -> only backbone tensors consumed). 0.5 norm (not ImageNet), 256 px @ 0.5 µm/px. - Via the JWonderLand mirror, byte-identical to upstream GitHub LFS (sha256-verified before - load). Apache-2.0 + Commons Clause (non-commercial). Patch tower only (no ViT-4K aggregation). + Loaded from Mahmood Lab's official commit-pinned Git LFS asset and SHA-256 verified + before deserialisation. Apache-2.0 with Commons Clause; read the selling restriction + before use. Patch tower only (no ViT-4K aggregation). mstar: family: timm @@ -1352,6 +1392,7 @@ slide_encoders: embedding_dim: 768 patch_encoder: gigapath # GigaPath tile encoder (1536-d) - already in registry patch_dim: 1536 + architecture: "gigapath_slide_enc12l768d" gated: true license: "Apache-2.0 (HF card; research-only caveat in prose)" transform_source_url: "https://huggingface.co/prov-gigapath/prov-gigapath" @@ -1362,12 +1403,44 @@ slide_encoders: notes: >- LongNet slide encoder over GigaPath (1536-d) patch features + level-0 coords. Load via gigapath.slide_encoder.create_model(ckpt, "gigapath_slide_enc12l768d", 1536, - global_pool=True); model.tile_size = patch_size_lv0; model(feats[1,N,1536], - coords[1,N,2], all_layer_embed=True)[-1] -> 768-d. Needs the [gigapath_slide] extra + + global_pool=True); model(feats[1,N,1536], coords[1,N,2], + all_layer_embed=True)[-1] -> 768-d. Needs the [gigapath_slide] extra + the gigapath git package (vendors torchscale; aliased) + fairscale + a flash-attn wheel matching your torch/ABI (no SDPA fallback) + accepted prov-gigapath/prov-gigapath gate. - Tested end-to-end on a Blackwell GPU. `-f gigapath - -s gigapath_slide`. + Constructs LongNet with its upstream tile_size=256 default, passes stored level-0 + coordinates unchanged, then sets only its runtime positional divisor to the + store's level-0 tile extent. This normalizes raw2features' coordinate units into + the tile-index frame of upstream's default 256-pixel tiling. Tested end-to-end on a + Blackwell GPU. + `-f gigapath -s gigapath_slide`. + + # Lightweight Prov-GigaPath-Flash LongNet over the matched 384-d Flash tile + # features and level-0 coordinates. Uses the same loader and flash-attn stack. + gigapath_flash_slide: + family: gigapath_slide + source: "hf-hub:prov-gigapath/prov-gigapath-flash" + embedding_dim: 384 + patch_encoder: gigapath_flash + patch_dim: 384 + architecture: "gigapath_slide_enc12l384d" + gated: true + license: "Apache-2.0 (HF card; gated, research/deployment terms in model card)" + transform_source_url: "https://huggingface.co/prov-gigapath/prov-gigapath-flash" + weights_revision: "f5c3a709ad81d3a5605bb2eb38cd5c2e53da0375" + weights_sha256: "1539cdcb0dae4628ccdcee9910991d541a254c1647aec0a0225a20257b590cba" + weights_filename: "slide_encoder.pth" + doi: "10.48550/arXiv.2607.18218" + notes: >- + GigaPath-Flash 12-layer LongNet over matched 384-d gigapath_flash patch + embeddings plus level-0 coordinates. Loaded from the immutable Hugging Face + revision after SHA-256 verification, using + gigapath.slide_encoder.create_model(checkpoint, + "gigapath_slide_enc12l384d", 384, global_pool=True). The last mean-pooled + layer is a 384-d slide vector. Needs the [gigapath_slide] extra, pinned gigapath + package, fairscale, flash-attn, and accepted model access. Passes stored level-0 + coordinates unchanged and sets only LongNet's runtime positional divisor to the + store's level-0 tile extent. Run with + `-f gigapath_flash -s gigapath_flash_slide`. # CHIEF (Wang et al., Nature 2024) - gated-attention MIL over CTransPath. Weights are # Drive-hosted (no HF mirror); fetched by their public Drive file id + verified SHA. diff --git a/src/raw2features/embedders/timm_embedder.py b/src/raw2features/embedders/timm_embedder.py index 1a6ace7..6401a27 100644 --- a/src/raw2features/embedders/timm_embedder.py +++ b/src/raw2features/embedders/timm_embedder.py @@ -42,6 +42,16 @@ def load( dtype: torch.dtype | None = None, compile: bool = False, ) -> TimmEmbedder: + for module in self.spec.registration_modules: + try: + importlib.import_module(module) + except ImportError as exc: + raise ImportError( + f"{self.spec.name} requires the optional architecture module " + f"{module!r}; install the model-specific dependencies documented " + "in raw2features' model catalogue" + ) from exc + import timm import torch diff --git a/src/raw2features/embedders/transforms.py b/src/raw2features/embedders/transforms.py index 34ec55f..db56202 100644 --- a/src/raw2features/embedders/transforms.py +++ b/src/raw2features/embedders/transforms.py @@ -1,9 +1,9 @@ """Patch -> model-input tensor. The runner delivers patches at exactly ``patch_px`` and the target MPP. Here we -normalise and apply the per-model resize to ``spec.input_size`` -- a no-op in the -common case where ``patch_px == input_size``, and the model-fit step when a user -samples at a ``patch_px`` that differs from the model's fixed input size. +normalize and apply the per-model resize or declared centre crop to +``spec.input_size``. This is a no-op in the common case where ``patch_px == +input_size`` and no crop is configured. """ from __future__ import annotations @@ -53,7 +53,31 @@ def to_model_tensor(patch_hwc_uint8: np.ndarray, spec: ModelSpec) -> torch.Tenso img = patch_hwc_uint8 if img.ndim != 3 or img.shape[2] != 3: raise ValueError(f"expected HWC RGB patch, got shape {img.shape}") - if img.shape[0] != spec.input_size or img.shape[1] != spec.input_size: + if spec.crop_pct is not None: + if spec.crop_mode != "center": + raise ValueError( + f"unsupported crop_mode {spec.crop_mode!r}; expected 'center'" + ) + if not 0 < float(spec.crop_pct) <= 1: + raise ValueError("crop_pct must be greater than zero and at most one") + resize_px = int(round(spec.input_size / float(spec.crop_pct))) + if resize_px < spec.input_size: + raise ValueError("crop_pct produced a resize smaller than input_size") + if img.shape[0] != resize_px or img.shape[1] != resize_px: + from PIL import Image + + img = np.asarray( + Image.fromarray(img).resize( + (resize_px, resize_px), + _pil_resample(spec.interpolation), + ) + ) + offset = (resize_px - spec.input_size) // 2 + img = img[ + offset : offset + spec.input_size, + offset : offset + spec.input_size, + ] + elif img.shape[0] != spec.input_size or img.shape[1] != spec.input_size: from PIL import Image img = np.asarray( @@ -93,11 +117,11 @@ def to_model_batch( if not patches_hwc_uint8: raise ValueError("to_model_batch requires at least one patch") - needs_resize = any( + needs_cpu_transform = spec.crop_pct is not None or any( p.shape[0] != spec.input_size or p.shape[1] != spec.input_size for p in patches_hwc_uint8 ) - if needs_resize: + if needs_cpu_transform: # Keep the (rare) resize on the CPU PIL path (model's own interpolation) for # exact equivalence with the per-patch transform; stack and move once. The # normalise is folded in. diff --git a/src/raw2features/slide_embedders/base.py b/src/raw2features/slide_embedders/base.py index 41a2136..98154a9 100644 --- a/src/raw2features/slide_embedders/base.py +++ b/src/raw2features/slide_embedders/base.py @@ -49,6 +49,9 @@ class SlideModelSpec: weights_revision: str | None = None # Exact artifact identified by ``weights_sha256``; None for weight-free pools. weights_filename: str | None = None + # Upstream architecture where one loader family serves several compatible sizes. + # ``patch_dim`` supplies the constructor's input-channel width. + architecture: str | None = None class SlideEmbedder(ABC): @@ -86,10 +89,9 @@ def encode( that use spatial position. May be ``None`` for pooling models. patch_size_lv0: Side of one patch in level-0 pixels (the store's - ``patching.level0_patch``) - i.e. the spacing between adjacent - patch coordinates at level 0. Position-aware encoders (e.g. - TITAN, which builds a feature grid from ``coords`` and this - spacing) require it; pooling models ignore it. + ``patching.level0_patch``). Some position-aware encoders use it with + ``coords`` to construct their spatial representation; pooling models + and encoders with a fixed upstream coordinate lattice ignore it. Returns ------- diff --git a/src/raw2features/slide_embedders/gigapath_slide.py b/src/raw2features/slide_embedders/gigapath_slide.py index 163e4f7..3ade486 100644 --- a/src/raw2features/slide_embedders/gigapath_slide.py +++ b/src/raw2features/slide_embedders/gigapath_slide.py @@ -1,4 +1,4 @@ -"""Prov-GigaPath slide encoder (LongNet) - gated. +"""Prov-GigaPath and GigaPath-Flash slide encoders (LongNet) - gated. The GigaPath slide encoder needs ``flash-attn``, a hard dependency with no SDPA fallback. Prebuilt wheels exist for x86 with torch 2.7-2.9; other combinations (torch @@ -7,22 +7,25 @@ model. Needs the ``[gigapath_slide]`` extra + the gigapath git package + ``fairscale`` + a matching flash-attn. -The GigaPath slide encoder is a 12-layer LongNet (dilated-attention ViT) that -aggregates the slide's GigaPath tile features into one 768-d vector. It is -position-aware: it indexes a learned 2-D positional grid from the patch -``coords`` and the level-0 patch spacing (``patch_size_lv0``), so it needs both -- not just the feature matrix. Gated: an ``HF_TOKEN`` with accepted access to -``prov-gigapath/prov-gigapath`` is required (same gate as the patch encoder). +The paired slide encoders are 12-layer LongNet models (dilated-attention ViTs). +The original maps 1536-d GigaPath tile features to 768 dimensions; Flash maps +384-d Flash tile features to 384 dimensions. Both are position-aware: they index +a fixed 2-D sine-cosine positional grid from patch ``coords``. raw2features stores +coordinates in source level-0 pixels, so it sets LongNet's runtime coordinate divisor +to the level-0 extent of one extracted tile. This normalizes those coordinates into +the tile-index frame of upstream's default 256-pixel tiling. The encoder therefore +needs the coordinates and tile extent as well as the feature matrix. Gated model +access is required for the corresponding Hugging Face repository. Load and forward follow the authors' own pipeline (``gigapath.pipeline.run_inference_with_slide_encoder``): from gigapath.slide_encoder import create_model - model = create_model(ckpt, "gigapath_slide_enc12l768d", 1536, global_pool=True) - model.tile_size = patch_size_lv0 # level-0 patch spacing -> grid step + model = create_model(ckpt, architecture, patch_dim, global_pool=True) + model.tile_size = patch_size_lv0 # runtime coords_to_pos divisor with torch.autocast("cuda", torch.float16), torch.inference_mode(): - out = model(features[1,N,1536], coords[1,N,2], all_layer_embed=True) - slide_vec = out[-1] # last_layer_embed -> [1, 768] + out = model(features[1,N,D], coords[1,N,2], all_layer_embed=True) + slide_vec = out[-1] # last_layer_embed -> [1, output_dim] ``global_pool=True`` is the authors' recommended setting: the demo notes the CLS token is *not* trained during slide pretraining, so the slide vector is the @@ -30,9 +33,10 @@ HEAD with no revision, so we download the pinned commit ourselves and hand it the local path (keeps the weights sha-pinnable). -The expected patch encoder is **GigaPath** (``-f gigapath``, 1536-d), and -``patch_size_lv0`` is our store's ``patching.level0_patch`` - exactly the -level-0 spacing GigaPath's grid expects. +The registry binds each slide model to its matching patch encoder. LongNet is +constructed with its upstream 256-pixel default, which determines its dilated-attention +segment lengths. raw2features then passes level-0 coordinates unchanged and adapts only +the runtime ``coords_to_pos`` divisor to that coordinate frame. Needs the ``[gigapath_slide]`` extra plus two post-install steps: the gigapath git package (with ``fairscale``; it vendors torchscale, which ``load`` aliases as @@ -47,6 +51,8 @@ from __future__ import annotations +import operator + import numpy as np from raw2features.core.plugins import register @@ -69,12 +75,14 @@ "LongNet slide encoder over GigaPath (1536-d) patch features + level-0 coords. " "Gated - HF token with accepted prov-gigapath/prov-gigapath access required." ), + architecture="gigapath_slide_enc12l768d", ) +@register("slide_embedders", "gigapath_flash_slide") @register("slide_embedders", "gigapath_slide") class GigapathSlideEmbedder(SlideEmbedder): - """GigaPath LongNet slide encoder (Prov-GigaPath, Nature 2024).""" + """GigaPath LongNet slide encoder for original and Flash checkpoints.""" def __init__(self) -> None: super().__init__(_SPEC) @@ -101,24 +109,11 @@ def load(self, device: str = "cuda", dtype=None) -> GigapathSlideEmbedder: ' pip install "raw2features[gigapath_slide]"\n' " pip install --no-deps fairscale " '"git+https://github.com/prov-gigapath/prov-gigapath.git@' - '3505f87e197d167522be491bb3f18fb5a08ca584"\n' + '9d42a60babe04359978d5ad2eb94e7b3bcf9ca39"\n' " + a prebuilt flash-attn wheel for your torch+CUDA " "(hard dep, no SDPA fallback; Ampere+ GPU at runtime)" ) from exc - # gigapath builds its LongNet segment_length as str(list()); - # under numpy>=2 those render as "np.int64(..)", and torchscale's config then - # eval()s that string with numpy not in scope -> NameError. raw2features needs - # numpy>=2, so expose np in the exact module that runs the eval before - # create_model. (The torchscale alias loads config under the name - # `torchscale.architecture.config`; patch the module EncoderConfig lives in.) - import numpy as _np - from torchscale.architecture.config import EncoderConfig - - _cfg_mod = sys.modules[EncoderConfig.__module__] - if not hasattr(_cfg_mod, "np"): - _cfg_mod.np = _np - # create_model's hf_hub path forces the repo HEAD (no revision); download the # pinned commit ourselves and pass the local file so the weights stay pinnable. ckpt = download_pinned_hf_file( @@ -127,7 +122,13 @@ def load(self, device: str = "cuda", dtype=None) -> GigapathSlideEmbedder: self.spec.weights_revision, ) verify_sha256(ckpt, self.spec.weights_sha256, what=self.spec.name) - model = create_model(ckpt, "gigapath_slide_enc12l768d", 1536, global_pool=True) + architecture = self.spec.architecture or "gigapath_slide_enc12l768d" + model = create_model( + ckpt, + architecture, + int(self.spec.patch_dim), + global_pool=True, + ) model.eval().to(device) self._model = model self._device = device @@ -151,16 +152,43 @@ def encode( if patch_size_lv0 is None: raise ValueError( "GigaPath's slide encoder needs patch_size_lv0 (the store's " - "patching.level0_patch): the level-0 spacing between adjacent patches" + "patching.level0_patch): the level-0 extent of one extracted tile " + "used as its positional-grid divisor" + ) + try: + if isinstance(patch_size_lv0, (bool, np.bool_)): + raise TypeError + tile_extent_lv0 = int(operator.index(patch_size_lv0)) + except (TypeError, OverflowError) as exc: + raise ValueError("patch_size_lv0 must be a positive integer") from exc + if tile_extent_lv0 <= 0: + raise ValueError("patch_size_lv0 must be a positive integer") + + coords_array = np.ascontiguousarray(coords, dtype=np.float32) + if coords_array.ndim != 2 or coords_array.shape[1] != 2: + raise ValueError("GigaPath patch coords must have shape (N, 2)") + if not np.isfinite(coords_array).all(): + raise ValueError("GigaPath patch coords must contain only finite values") + + slide_ngrids = int(self._model.slide_ngrids) + cells = np.floor(coords_array / tile_extent_lv0) + if np.any(cells < 0) or np.any(cells >= slide_ngrids): + raise ValueError( + "GigaPath patch coords fall outside its positional grid after " + f"division by patch_size_lv0={tile_extent_lv0}; each axis must map " + f"to a cell in [0, {slide_ngrids - 1}]" ) - # LongNet maps a coord to a grid cell by floor(coord / tile_size). Set tile_size - # to the level-0 patch spacing so our level-0 coords land on the grid. - self._model.tile_size = int(patch_size_lv0) + # LongNet maps a coordinate to a grid cell with floor(coord / tile_size). + # Stored coordinates are source level-0 pixels, so the level-0 extent of one + # extracted tile is the divisor corresponding to upstream's 256-pixel + # target-resolution frame. Overlapping tokens may share a positional cell; + # LongNet still receives every token. + self._model.tile_size = tile_extent_lv0 feat = torch.from_numpy(np.ascontiguousarray(features, dtype=np.float32)) - coords_t = torch.from_numpy(np.ascontiguousarray(coords, dtype=np.float32)) - feat = feat.unsqueeze(0).to(self._device) # [1, N, 1536] + coords_t = torch.from_numpy(coords_array) + feat = feat.unsqueeze(0).to(self._device) # [1, N, patch_dim] coords_t = coords_t.unsqueeze(0).to(self._device) # [1, N, 2] use_amp = self._device.startswith("cuda") diff --git a/src/raw2features/slide_embedders/model_registry.py b/src/raw2features/slide_embedders/model_registry.py index 3493604..b62e20b 100644 --- a/src/raw2features/slide_embedders/model_registry.py +++ b/src/raw2features/slide_embedders/model_registry.py @@ -61,6 +61,7 @@ def build_slide_embedder(name: str) -> SlideEmbedder: emb = cls() # Overwrite spec from registry so provenance is always registry-sourced. emb.spec = spec + emb.name = spec.name return emb diff --git a/tests/test_embedder_registry.py b/tests/test_embedder_registry.py index 55ee794..22fc039 100644 --- a/tests/test_embedder_registry.py +++ b/tests/test_embedder_registry.py @@ -87,3 +87,32 @@ def test_transform_batch_resize_fallback_matches_stacked_transform(): batched = emb.transform_batch(patches, "cpu") assert tuple(batched.shape) == (3, 3, 224, 224) assert torch.equal(stacked, batched) + + +@pytest.mark.parametrize("name", ["gigapath", "gigapath_flash"]) +def test_gigapath_transform_center_crops_the_official_256_tile(name): + """The official 256px GigaPath tile becomes its central 224px model input.""" + emb = build_embedder(name) + patch = np.zeros((256, 256, 3), dtype=np.uint8) + patch[16:240, 16:240] = 255 + + transformed = emb.transform(patch) + + assert tuple(transformed.shape) == (3, 224, 224) + expected = torch.tensor( + [ + (1.0 - mean) / std + for mean, std in zip(emb.spec.mean, emb.spec.std, strict=True) + ] + ).view(3, 1, 1) + assert torch.allclose(transformed, expected.expand_as(transformed)) + + +@pytest.mark.parametrize("name", ["gigapath", "gigapath_flash"]) +def test_gigapath_crop_batch_matches_individual_transform(name): + emb = build_embedder(name) + rng = np.random.RandomState(2) + patches = [rng.randint(0, 256, (256, 256, 3), dtype=np.uint8) for _ in range(2)] + stacked = torch.stack([emb.transform(patch) for patch in patches]) + batched = emb.transform_batch(patches, "cpu") + assert torch.equal(stacked, batched) diff --git a/tests/test_gigapath_flash.py b/tests/test_gigapath_flash.py new file mode 100644 index 0000000..fbad702 --- /dev/null +++ b/tests/test_gigapath_flash.py @@ -0,0 +1,220 @@ +"""GigaPath-Flash registry and loader contracts, without downloading weights.""" + +from __future__ import annotations + +import importlib +import sys +from types import ModuleType + +import numpy as np +import pytest + +torch = pytest.importorskip("torch", reason="torch not installed") + +from raw2features.embedders.fingerprint import ( # noqa: E402 + patch_output_fingerprint, + slide_output_fingerprint, +) +from raw2features.embedders.model_registry import build_embedder, get_spec # noqa: E402 +from raw2features.embedders.timm_embedder import TimmEmbedder # noqa: E402 +from raw2features.slide_embedders.model_registry import ( # noqa: E402 + build_slide_embedder, + get_slide_spec, +) + + +def test_gigapath_flash_registry_pair_is_matched_and_pinned(): + # Register the source-tree implementation explicitly. Test environments may + # still expose entry-point metadata from the previously installed release. + import raw2features.slide_embedders.gigapath_slide # noqa: F401 + + tile = get_spec("gigapath_flash") + slide = get_slide_spec("gigapath_flash_slide") + + assert isinstance(build_embedder("gigapath_flash"), TimmEmbedder) + assert tile.source == "gigapath_tile_enc_dinov2s" + assert tile.embedding_dim == slide.patch_dim == slide.embedding_dim == 384 + assert slide.patch_encoder == tile.name + assert slide.architecture == "gigapath_slide_enc12l384d" + assert tile.weights_revision == slide.weights_revision + assert tile.weights_filename == "pytorch_model.bin" + assert slide.weights_filename == "slide_encoder.pth" + assert tile.checkpoint["strict"] is True + assert tile.registration_modules == ("gigapath.tile_encoder",) + + slide_embedder = build_slide_embedder("gigapath_flash_slide") + assert slide_embedder.name == "gigapath_flash_slide" + assert slide_embedder.spec is slide + + fingerprint = slide_output_fingerprint( + slide, + patch_model=tile.name, + patch_output_fingerprint=patch_output_fingerprint(tile, "fp16"), + patch_dim=tile.embedding_dim, + resolved_amp="fp16", + ) + constructor = fingerprint["payload"]["loader"]["constructor"] + assert constructor["constructor_tile_size"] == 256 + assert constructor["slide_ngrids"] == 1000 + assert ( + constructor["runtime_position_divisor_source"] == "patching.level0_patch" + ) + assert constructor["coords_frame"] == "stored_level0_xy" + assert constructor["coords_transform"] == "none" + assert constructor["position_mapping"] == ( + "floor(x / divisor) * slide_ngrids + floor(y / divisor) + 1" + ) + assert "tile_size" not in constructor + + +@pytest.mark.parametrize("patch_size_lv0", [256, 512]) +def test_gigapath_slide_forward_normalizes_level0_coordinate_frame( + patch_size_lv0, +): + import raw2features.slide_embedders.gigapath_slide as gigapath_slide + + captured: dict[str, torch.Tensor] = {} + + class RecordingLongNet: + tile_size = 0 + slide_ngrids = 1000 + + def __call__(self, features, coords, all_layer_embed=False): + captured["features"] = features.detach().cpu().clone() + captured["coords"] = coords.detach().cpu().clone() + cells = torch.floor(coords / self.tile_size) + captured["positions"] = ( + cells[..., 0] * self.slide_ngrids + cells[..., 1] + 1 + ).long() + captured["all_layer_embed"] = all_layer_embed + return [torch.ones((1, 384), dtype=torch.float32)] + + features = np.arange(8 * 384, dtype=np.float32).reshape(8, 384) + coords = np.asarray( + [ + [0, 0], + [patch_size_lv0 // 2, 0], + [patch_size_lv0 - 1, 0], + [patch_size_lv0, 0], + [patch_size_lv0 + 1, 0], + [2 * patch_size_lv0, 0], + [0, patch_size_lv0], + [patch_size_lv0, patch_size_lv0], + ], + dtype=np.float32, + ) + model = RecordingLongNet() + embedder = gigapath_slide.GigapathSlideEmbedder() + embedder.spec = get_slide_spec("gigapath_flash_slide") + embedder._model = model + embedder._device = "cpu" + + output = embedder.encode(features, coords, patch_size_lv0=patch_size_lv0) + + assert model.tile_size == patch_size_lv0 + assert captured["features"].shape == (1, 8, 384) + assert captured["coords"].shape == (1, 8, 2) + assert captured["positions"].tolist() == [ + [1, 1, 1, 1001, 1001, 2001, 2, 1002] + ] + assert captured["all_layer_embed"] is True + np.testing.assert_array_equal(captured["features"][0].numpy(), features) + np.testing.assert_array_equal(captured["coords"][0].numpy(), coords) + assert output.shape == (384,) + + +@pytest.mark.parametrize("patch_size_lv0", [None, 0, -256, True, 256.5]) +def test_gigapath_slide_requires_positive_integer_level0_tile_extent( + patch_size_lv0, +): + import raw2features.slide_embedders.gigapath_slide as gigapath_slide + + embedder = gigapath_slide.GigapathSlideEmbedder() + embedder.spec = get_slide_spec("gigapath_flash_slide") + embedder._model = type("LongNet", (), {"slide_ngrids": 1000})() + embedder._device = "cpu" + + with pytest.raises(ValueError, match="patch_size_lv0|positive integer"): + embedder.encode( + np.zeros((1, 384), dtype=np.float32), + np.zeros((1, 2), dtype=np.float32), + patch_size_lv0=patch_size_lv0, + ) + + +@pytest.mark.parametrize( + "coords", + [ + [[-1, 0]], + [[0, -1]], + [[256_000, 0]], + [[0, 256_000]], + [[np.nan, 0]], + ], +) +def test_gigapath_slide_rejects_coords_outside_positional_grid(coords): + import raw2features.slide_embedders.gigapath_slide as gigapath_slide + + embedder = gigapath_slide.GigapathSlideEmbedder() + embedder.spec = get_slide_spec("gigapath_flash_slide") + embedder._model = type("LongNet", (), {"slide_ngrids": 1000})() + embedder._device = "cpu" + + with pytest.raises(ValueError, match="positional grid|finite"): + embedder.encode( + np.zeros((1, 384), dtype=np.float32), + np.asarray(coords, dtype=np.float32), + patch_size_lv0=256, + ) + + +def test_timm_registration_module_is_imported_before_checkpoint_construction( + monkeypatch, +): + events: list[object] = [] + real_import = importlib.import_module + + def import_module(name): + if name == "gigapath.tile_encoder": + events.append(("import", name)) + return ModuleType(name) + return real_import(name) + + class Model: + def eval(self): + return self + + def to(self, device): + events.append(("to", device)) + return self + + timm = ModuleType("timm") + + def create_model(name, **kwargs): + events.append(("create", name, kwargs)) + return Model() + + timm.create_model = create_model + monkeypatch.setitem(sys.modules, "timm", timm) + monkeypatch.setattr( + "raw2features.embedders.timm_embedder.importlib.import_module", + import_module, + ) + monkeypatch.setattr( + TimmEmbedder, + "_load_checkpoint", + lambda self, model, checkpoint: events.append(("load", checkpoint)), + ) + + embedder = build_embedder("gigapath_flash") + embedder.load(device="cpu", dtype=torch.float32) + + assert events[:3] == [ + ("import", "gigapath.tile_encoder"), + ( + "create", + "gigapath_tile_enc_dinov2s", + {"pretrained": False, "num_classes": 0}, + ), + ("load", get_spec("gigapath_flash").checkpoint), + ] diff --git a/tests/test_loader_revision_pinning.py b/tests/test_loader_revision_pinning.py index 788af53..6db6ecc 100644 --- a/tests/test_loader_revision_pinning.py +++ b/tests/test_loader_revision_pinning.py @@ -674,6 +674,49 @@ def test_gigapath_bad_digest_never_reaches_create_model(monkeypatch, tmp_path): emb.load(device="cpu") +def test_gigapath_flash_slide_uses_the_registry_architecture(monkeypatch, tmp_path): + import raw2features.slide_embedders.gigapath_slide as gigapath_slide + from raw2features.slide_embedders.model_registry import get_slide_spec + + checkpoint = tmp_path / "slide_encoder.pth" + checkpoint.write_bytes(b"pinned GigaPath-Flash slide weights") + calls = [] + downloads = [] + + def create_model(*args, **kwargs): + calls.append((args, kwargs)) + return _Model() + + _fake_gigapath_stack(monkeypatch, create_model) + + def file_download(**kwargs): + downloads.append(kwargs) + return str(checkpoint) + + _fake_hub(monkeypatch, file_download=file_download) + emb = gigapath_slide.GigapathSlideEmbedder() + emb.spec = replace( + get_slide_spec("gigapath_flash_slide"), + weights_sha256=_digest(checkpoint.read_bytes()), + ) + emb.load(device="cpu") + + assert calls == [ + ( + (str(checkpoint), "gigapath_slide_enc12l384d", 384), + {"global_pool": True}, + ) + ] + assert downloads == [ + { + "repo_id": "prov-gigapath/prov-gigapath-flash", + "filename": "slide_encoder.pth", + "revision": emb.spec.weights_revision, + "cache_dir": None, + } + ] + + @pytest.mark.parametrize( "model_name", ["quiltnet", "conch", "kronos", "seal_conch", "seal_univ2"] ) diff --git a/tests/test_output_fingerprints.py b/tests/test_output_fingerprints.py index ba87f80..aae7d23 100644 --- a/tests/test_output_fingerprints.py +++ b/tests/test_output_fingerprints.py @@ -123,6 +123,8 @@ def test_patch_fingerprint_is_canonical_and_covers_the_output_contract(monkeypat replace(base, timm_kwargs={"num_classes": 0, "dynamic_img_size": True}), replace(base, reg_tokens=4), replace(base, modality="multiplex"), + replace(base, crop_pct=0.875, crop_mode="center"), + replace(base, registration_modules=("example.architectures",)), ] assert all( patch_output_fingerprint(candidate, "bf16")["digest"] != original["digest"] @@ -134,6 +136,50 @@ def test_patch_fingerprint_is_canonical_and_covers_the_output_contract(monkeypat assert patch_output_fingerprint(base, "bf16")["digest"] != original["digest"] +def test_optional_crop_and_registration_contracts_do_not_touch_other_models(): + ordinary = patch_output_fingerprint(_spec(), "fp32")["payload"] + assert "crop_pct" not in ordinary["preprocessing"] + assert "crop_mode" not in ordinary["preprocessing"] + assert "registration_modules" not in ordinary["loader"]["constructor"] + + configured = patch_output_fingerprint( + _spec( + crop_pct=0.875, + crop_mode="center", + registration_modules=("gigapath.tile_encoder",), + ), + "fp32", + )["payload"] + assert configured["preprocessing"]["crop_pct"] == 0.875 + assert configured["preprocessing"]["crop_mode"] == "center" + assert configured["loader"]["constructor"]["registration_modules"] == [ + "gigapath.tile_encoder" + ] + assert configured["loader"]["constructor"]["construction_package_revision"] == ( + "9d42a60babe04359978d5ad2eb94e7b3bcf9ca39" + ) + + +def test_hipt_fingerprint_records_the_official_checkpoint_source(): + from raw2features.embedders.model_registry import get_spec + + fingerprint = patch_output_fingerprint(get_spec("hipt"), "fp32") + effective = fingerprint["payload"]["checkpoint"]["effective"] + assert effective == { + "repo": None, + "filename": "vit256_small_dino.pth", + "url": ( + "https://media.githubusercontent.com/media/mahmoodlab/HIPT/" + "780fafaed2e5b112bc1ed6e78852af1fe6714342/HIPT_4K/Checkpoints/" + "vit256_small_dino.pth" + ), + "mechanism": "explicit_checkpoint", + } + assert fingerprint["payload"]["checkpoint"]["weights_revision"] == ( + "780fafaed2e5b112bc1ed6e78852af1fe6714342" + ) + + def test_fingerprint_never_persists_uri_credentials_and_ignores_rotation(): first = _spec( checkpoint={ diff --git a/tests/test_registry_contract.py b/tests/test_registry_contract.py index 5777012..2ee5227 100644 --- a/tests/test_registry_contract.py +++ b/tests/test_registry_contract.py @@ -35,6 +35,15 @@ def test_every_patch_spec_is_complete_and_consistent(): assert spec.recommended_patch_px is None or spec.recommended_patch_px > 0, ( f"{name}: recommended_patch_px must be positive or None" ) + assert spec.crop_pct is None or 0 < spec.crop_pct <= 1, ( + f"{name}: crop_pct must be in (0, 1] or None" + ) + assert spec.crop_mode in ({None} if spec.crop_pct is None else {"center"}), ( + f"{name}: crop_mode must be 'center' exactly when crop_pct is set" + ) + assert all(spec.registration_modules), ( + f"{name}: registration_modules must not contain blank module names" + ) # extract_px is always a usable patch size (falls back to input_size). assert spec.extract_px > 0, f"{name}: extract_px must be positive" # FAIR findability: a resolvable DOI, unless the model is an open-weights @@ -87,10 +96,29 @@ def test_optional_fields_round_trip_through_loader(): ) assert get_spec("uni").recommended_mpp == 0.5 assert get_spec("resnet50").recommended_mpp is None - # recommended_patch_px is set only where extraction px != input_size (conch_v1_5); - # otherwise None, and extract_px falls back to input_size. + # recommended_patch_px is set where extraction px differs from the fixed model + # input (for example CONCH v1.5 and the centre-cropped GigaPath tiles). assert get_spec("conch_v1_5").recommended_patch_px == 512 assert get_spec("conch_v1_5").extract_px == 512 + assert get_spec("gigapath").recommended_patch_px == 256 + assert get_spec("gigapath").crop_pct == 0.875 + assert get_spec("gigapath_flash").registration_modules == ( + "gigapath.tile_encoder", + ) + assert get_spec("gigapath_flash").checkpoint == { + "repo": "prov-gigapath/prov-gigapath-flash", + "filename": "pytorch_model.bin", + "strict": True, + } + assert get_spec("hipt").checkpoint.get("repo") is None + assert get_spec("hipt").checkpoint["url"].startswith( + "https://media.githubusercontent.com/media/mahmoodlab/HIPT/" + ) + assert get_spec("hipt").mean == (0.5, 0.5, 0.5) + assert get_spec("hipt").std == (0.5, 0.5, 0.5) + assert "Commons Clause" in get_spec("hipt").license + assert get_spec("gigapath").inference_amp == "fp16" + assert get_spec("gigapath_flash").inference_amp == "fp16" assert get_spec("uni").recommended_patch_px is None assert get_spec("uni").extract_px == get_spec("uni").input_size == 224 # doi is cherry-picked by the loader too -> guard it survives. diff --git a/tests/test_registry_revisions_network.py b/tests/test_registry_revisions_network.py index b5399dd..ab99c3e 100644 --- a/tests/test_registry_revisions_network.py +++ b/tests/test_registry_revisions_network.py @@ -13,6 +13,7 @@ import os import re +import urllib.request from collections import defaultdict from collections.abc import Mapping from dataclasses import dataclass @@ -68,8 +69,10 @@ def _hf_repo(spec) -> str | None: def _is_non_hf_pin(spec) -> bool: """Identify the registry's deliberately non-Hub revision identifiers.""" checkpoint = getattr(spec, "checkpoint", None) or {} - return str(spec.source).startswith("torchvision://") or ( - bool(checkpoint.get("url")) and spec.weights_revision == "pretrained-weights" + revision = str(spec.weights_revision) + return str(spec.source).startswith("torchvision://") or bool( + checkpoint.get("url") + and (revision == "pretrained-weights" or _FULL_COMMIT.fullmatch(revision)) ) @@ -112,7 +115,13 @@ def _registry_hf_pins() -> list[_HubPin]: def _new_model_artifact_pins() -> list[_HubArtifactPin]: registry = load_registry() pins: list[_HubArtifactPin] = [] - for name in ("h0_mini", "keep", "openmidnight", "openpath"): + for name in ( + "h0_mini", + "keep", + "openmidnight", + "openpath", + "gigapath_flash", + ): spec = registry[name] repo = _hf_repo(spec) assert repo is not None, ( @@ -130,6 +139,16 @@ def _new_model_artifact_pins() -> list[_HubArtifactPin]: sha256=str(spec.weights_sha256), ) ) + slide = load_slide_registry()["gigapath_flash_slide"] + pins.append( + _HubArtifactPin( + name=slide.name, + repo=_hf_repo(slide) or "", + revision=str(slide.weights_revision), + filename=str(slide.weights_filename), + sha256=str(slide.weights_sha256), + ) + ) return pins @@ -219,6 +238,21 @@ def test_new_model_artifact_checksum_matches_huggingface_metadata(pin: _HubArtif ) +@pytest.mark.network +def test_hipt_official_git_lfs_pointer_matches_registry(): + """The lightweight official pointer proves the 704 MB asset's immutable hash.""" + + spec = load_registry()["hipt"] + pointer_url = ( + "https://raw.githubusercontent.com/mahmoodlab/HIPT/" + f"{spec.weights_revision}/HIPT_4K/Checkpoints/{spec.weights_filename}" + ) + with urllib.request.urlopen(pointer_url, timeout=15) as response: # noqa: S310 + pointer = response.read().decode("ascii") + assert f"oid sha256:{spec.weights_sha256}" in pointer + assert "size 704238867" in pointer + + @pytest.mark.network @pytest.mark.parametrize( ("kind", "name", "source"), @@ -226,6 +260,7 @@ def test_new_model_artifact_checksum_matches_huggingface_metadata(pin: _HubArtif ("patch", "seal_conch", "hf-hub:MahmoodLab/SEAL"), ("patch", "seal_univ2", "hf-hub:MahmoodLab/SEAL"), ("slide", "gigapath_slide", None), + ("slide", "gigapath_flash_slide", None), ], ) def test_direct_loader_file_checksum_matches_registry(kind, name, source): diff --git a/tests/test_resolve_geometry.py b/tests/test_resolve_geometry.py index f8fbbfa..f7dc242 100644 --- a/tests/test_resolve_geometry.py +++ b/tests/test_resolve_geometry.py @@ -40,6 +40,16 @@ def test_conch_v1_5_extracts_larger_then_resizes(): assert model_geometry("conch_v1_5") == (0.5, 512, "recommended") +@pytest.mark.parametrize("name", ["gigapath", "gigapath_flash"]) +def test_gigapath_models_extract_256_then_center_crop(name): + spec = get_spec(name) + assert spec.input_size == 224 + assert spec.recommended_patch_px == 256 + assert spec.crop_pct == 0.875 + assert spec.crop_mode == "center" + assert model_geometry(name) == (0.5, 256, "recommended") + + def test_scale_agnostic_baseline_defaults(): assert model_geometry("resnet50") == (DEFAULT_TARGET_MPP, 224, "default") diff --git a/tests/test_slide_embedders.py b/tests/test_slide_embedders.py index 80a1fdb..af00b9a 100644 --- a/tests/test_slide_embedders.py +++ b/tests/test_slide_embedders.py @@ -150,7 +150,7 @@ def test_patch_registry_ignores_slide_encoders_section(): assert "slide_encoders" not in patch_reg assert set(patch_reg) == { "resnet50", "dinov2", "uni", "uni2_h", "path_orchestra", "virchow2", - "gigapath", "conch", + "gigapath", "gigapath_flash", "conch", "conch_v1_5", "h_optimus_0", "h0_mini", "gpfm", "midnight", "openmidnight", "openpath", "ctranspath", "hibou_l", "hibou_b", "kronos", "phikon", "phikon_v2",