From cf109b3ce3a5c9dd6f8f2e65c69fc685206e035d Mon Sep 17 00:00:00 2001 From: eroell Date: Wed, 29 Oct 2025 18:09:36 +0100 Subject: [PATCH 01/25] CI check --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bec413cd..6ee8b56d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "requests", "rich", "xarray", - "zarr<3", + "zarr", ] optional-dependencies.dev = [ "pre-commit", From 07ea6e4e49eb7d2c45db83c196b7e28af0b48d75 Mon Sep 17 00:00:00 2001 From: eroell Date: Wed, 29 Oct 2025 18:18:28 +0100 Subject: [PATCH 02/25] loosen in optional vitessce dependency as well --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6ee8b56d..54d46345 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,6 @@ optional-dependencies.torch = [ optional-dependencies.vitessce = [ "vitessce[all]>=3.4", # the actual dependency - "zarr>=2.18.7,<3", # vitessce does not support zarr>=3 ] # https://docs.pypi.org/project_metadata/#project-urls urls.Documentation = "https://ehrdata.readthedocs.io/" From 7b8a289731ea1f0f718723057039d8921bc547cc Mon Sep 17 00:00:00 2001 From: eroell Date: Wed, 29 Oct 2025 19:33:33 +0100 Subject: [PATCH 03/25] debug why we don't get a bug --- .github/workflows/test.yaml | 2 ++ pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3b8ad5c1..31643274 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -73,6 +73,8 @@ jobs: cache-dependency-glob: pyproject.toml - name: create hatch environment run: uvx hatch env create ${{ matrix.env.name }} + - name: Show installed packages + run: uvx hatch run ${{ matrix.env.name }}:pip list - name: run tests using hatch env: MPLBACKEND: agg diff --git a/pyproject.toml b/pyproject.toml index 54d46345..23d4e724 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "requests", "rich", "xarray", - "zarr", + "zarr>=3", ] optional-dependencies.dev = [ "pre-commit", From c09e1e651394ff17130169500aa04828f7e60b49 Mon Sep 17 00:00:00 2001 From: eroell Date: Thu, 30 Oct 2025 14:54:08 +0100 Subject: [PATCH 04/25] store commit; breaks --- src/ehrdata/io/zarr.py | 110 +++++++++++++++++++++++++++++++---------- tests/io/test_zarr.py | 31 ++++++------ 2 files changed, 100 insertions(+), 41 deletions(-) diff --git a/src/ehrdata/io/zarr.py b/src/ehrdata/io/zarr.py index 4ce8d9a3..be8df7b9 100644 --- a/src/ehrdata/io/zarr.py +++ b/src/ehrdata/io/zarr.py @@ -1,11 +1,13 @@ from __future__ import annotations +import warnings from pathlib import Path from typing import TYPE_CHECKING import anndata as ad import zarr +import ehrdata as ed from ehrdata._logger import logger from ehrdata.io._array_casting import _cast_arrays_dtype_to_float_or_str_if_nonnumeric_object, _cast_variables_to_float @@ -36,30 +38,56 @@ def read_zarr( >>> ed.io.write_zarr("mimic_2.zarr", edata) >>> edata_from_zarr = ed.io.read_zarr("mimic_2.zarr") """ - import ehrdata as ed from ehrdata import EHRData + # TODO: check that anndata can be read + # TODO: check for backwards compatibility and announce clear version of when it stops + # TODO: check that ehrdata can be read + if isinstance(filename, Path): filename = str(filename) f = filename if isinstance(filename, zarr.Group) else zarr.open(filename, mode="r") - dictionary_for_init = {k: ad.io.read_elem(f[k]) for k, v in dict(f).items() if not k.startswith("raw.")} - - edata = EHRData(**dictionary_for_init) - - if harmonize_missing_values: - ed.harmonize_missing_values(edata) - logger.info("Harmonizing missing values of X") - - for key in edata.layers: - ed.harmonize_missing_values(edata, layer=key) - logger.info(f"Harmonizing missing values of layer {key}") - - if cast_variables_to_float: - _cast_variables_to_float(edata) - - return edata + if "ehrdata" in f.attrs: + if "anndata" in f: + dictionary_for_init = { + k: ad.io.read_elem(f["anndata"][k]) for k, v in dict(f["anndata"]).items() if not k.startswith("raw.") + } + else: + err = "The zarr store does not contain the 'anndata' group." + raise ValueError(err) + if "tem" in f: + dictionary_for_init["tem"] = ad.io.read_elem(f["tem"]) + else: + warnings.warn("The zarr store does not contain the 'tem' group.", stacklevel=2) + else: + warnings.warn( + "The zarr store does not contain an ehrdata attribute. This is might not be a valid ehrdata Zarr store, and the store might not be readable or be wrongly interpeted.", + stacklevel=2, + ) + if "anndata" in f.attrs: + warnings.warn( + "The zarr store is an AnnData store, which can be read but might not support all ehrdata features.", + stacklevel=2, + ) + + dictionary_for_init = {k: ad.io.read_elem(f[k]) for k, v in dict(f).items() if not k.startswith("raw.")} + + edata = EHRData(**dictionary_for_init) + + if harmonize_missing_values: + ed.harmonize_missing_values(edata) + logger.info("Harmonizing missing values of X") + + for key in edata.layers: + ed.harmonize_missing_values(edata, layer=key) + logger.info(f"Harmonizing missing values of layer {key}") + + if cast_variables_to_float: + _cast_variables_to_float(edata) + + return edata def write_zarr( @@ -71,14 +99,14 @@ def write_zarr( ) -> None: """Write :class:`~ehrdata.EHRData` objects to disk. - To write to a `.zarr` file, `X`, `R`, and `layers` cannot be written as `object` dtype. + To write to a `.zarr` file, `X`, and `layers` cannot be written as `object` dtype. If any of these fields is of `object` dtype, it this function will attempt to cast it to a numeric dtype; if this fails, the field will be casted to a `str` dtype. Args: edata: Central data object. filename: Name of the output file, can also be prefixed with relative or absolute path to save the file to. - chunks: Chunk shape, passed to :meth:`zarr.Group.create_dataset` for `Zarr` version 2, or to :meth:`zarr.Group.create_array` for `Zarr` version 3. - convert_strings_to_categoricals: Convert columns of `str` dtype in `.obs` and `.var` to `categorical` dtype. + chunks: Chunk shape, passed to :meth:`zarr.Group.create_array` for `Zarr` version 3. + convert_strings_to_categoricals: Convert columns of `str` dtype in `.obs` and `.var` and `.tem` to `categorical` dtype. Examples: >>> import ehrdata as ed @@ -86,13 +114,41 @@ def write_zarr( >>> ed.io.write_zarr("mimic_2.zarr", edata) """ filename = Path(filename) - edata = _cast_arrays_dtype_to_float_or_str_if_nonnumeric_object(edata) - ad.AnnData(edata).write_zarr( - filename, - chunks=chunks, - convert_strings_to_categoricals=convert_strings_to_categoricals, + # TODO: add test that checks these fields + # TODO: ensure this is "canonical" and what anndata is doing + store = zarr.open(filename, mode="w") + store.attrs["ehrdata_version"] = ed.__version__ + store.attrs["ehrdata_type"] = "ehrdata" + + # while adata.write_zarr supports convert_strings_to_categoricals, ad.io.write_elem does not + # so we need to convert the strings to categoricals ourselves + # TODO: figure out what anndata is doing + # map all columns of edata.obs to categorical dtype if they are of str dtype + # def _convert_to_categorical(df: pd.DataFrame) -> pd.DataFrame: + # for column in df.columns: + # if df[column].dtype == "str": + # df[column] = df[column].astype("category") + # return df + + adata = ad.AnnData(edata) + + # TODO: test + if convert_strings_to_categoricals: + # inplace conversion + adata.strings_to_categoricals(adata.obs) + adata.strings_to_categoricals(adata.var) + adata.strings_to_categoricals(edata.tem) + # adata.obs = edata.obs if not convert_strings_to_categoricals else _convert_to_categorical(edata.obs) + # adata.var = edata.var if not convert_strings_to_categoricals else _convert_to_categorical(edata.var) + # adata.tem = edata.tem if not convert_strings_to_categoricals else _convert_to_categorical(edata.tem) + + ad.io.write_elem( + store, + "anndata", + adata, # this will store everything but the .tem field + dataset_kwargs={"chunks": chunks}, ) - f = zarr.open(filename, mode="a") - ad.io.write_elem(f, "tem", edata.tem) + + ad.io.write_elem(store, "tem", edata.tem) diff --git a/tests/io/test_zarr.py b/tests/io/test_zarr.py index cf62c9ec..51446d89 100644 --- a/tests/io/test_zarr.py +++ b/tests/io/test_zarr.py @@ -70,8 +70,12 @@ def test_write_zarr_basic(edata_name, request, tmp_path): write_zarr(edata, store_path) zarr_file = zarr.open(store_path, mode="r") - # Note that R is not included in the list because it is just a value of the .layers field assert set(dict(zarr_file).keys()) == { + "anndata", + "tem", + } + + assert set(dict(zarr_file["anndata"]).keys()) == { "X", "obs", "var", @@ -82,28 +86,27 @@ def test_write_zarr_basic(edata_name, request, tmp_path): "varp", "raw", "uns", - "tem", } - assert np.array_equal(ad.io.read_elem(zarr_file["X"]).astype(str), edata.X.astype(str)) + assert np.array_equal(ad.io.read_elem(zarr_file["anndata"]["X"]).astype(str), edata.X.astype(str)) - pd.testing.assert_frame_equal(ad.io.read_elem(zarr_file["obs"]), edata.obs) - pd.testing.assert_frame_equal(ad.io.read_elem(zarr_file["var"]), edata.var) + pd.testing.assert_frame_equal(ad.io.read_elem(zarr_file["anndata"]["obs"]), edata.obs) + pd.testing.assert_frame_equal(ad.io.read_elem(zarr_file["anndata"]["var"]), edata.var) pd.testing.assert_frame_equal(ad.io.read_elem(zarr_file["tem"]), edata.tem) for key in edata.obsm: - assert key in ad.io.read_elem(zarr_file["obsm"]) - assert np.array_equal(ad.io.read_elem(zarr_file["obsm"][key]), edata.obsm[key]) + assert key in ad.io.read_elem(zarr_file["anndata"]["obsm"]) + assert np.array_equal(ad.io.read_elem(zarr_file["anndata"]["obsm"][key]), edata.obsm[key]) for key in edata.varm: - assert key in ad.io.read_elem(zarr_file["varm"]) - assert np.array_equal(ad.io.read_elem(zarr_file["varm"][key]), edata.varm[key]) + assert key in ad.io.read_elem(zarr_file["anndata"]["varm"]) + assert np.array_equal(ad.io.read_elem(zarr_file["anndata"]["varm"][key]), edata.varm[key]) for key in edata.obsp: - assert key in ad.io.read_elem(zarr_file["obsp"]) - assert np.array_equal(ad.io.read_elem(zarr_file["obsp"][key]), edata.obsp[key]) + assert key in ad.io.read_elem(zarr_file["anndata"]["obsp"]) + assert np.array_equal(ad.io.read_elem(zarr_file["anndata"]["obsp"][key]), edata.obsp[key]) for key in edata.varp: - assert key in ad.io.read_elem(zarr_file["varp"]) - assert np.array_equal(ad.io.read_elem(zarr_file["varp"][key]), edata.varp[key]) + assert key in ad.io.read_elem(zarr_file["anndata"]["varp"]) + assert np.array_equal(ad.io.read_elem(zarr_file["anndata"]["varp"][key]), edata.varp[key]) for key in edata.uns: - assert key in ad.io.read_elem(zarr_file["uns"]) + assert key in ad.io.read_elem(zarr_file["anndata"]["uns"]) @pytest.mark.parametrize("edata_name", ["edata_333", "edata_basic_with_tem_full", "edata_nonnumeric_missing_330"]) From 485a23402a9cf2a17f0b6a7a87190a6a7a66c678 Mon Sep 17 00:00:00 2001 From: eroell Date: Fri, 31 Oct 2025 21:51:47 +0100 Subject: [PATCH 05/25] new read_zarr and write_zarr --- CHANGELOG.md | 5 +- src/ehrdata/core/constants.py | 2 + src/ehrdata/io/zarr.py | 48 +-- .../data/toy_zarr/adata_basic.zarr/.zmetadata | 382 ++++++++++++++++++ .../toy_zarr/adata_basic.zarr/raw/.zarray | 10 + .../toy_zarr/adata_basic.zarr/raw/.zattrs | 4 + tests/data/toy_zarr/adata_basic.zarr/raw/0 | Bin 0 -> 8 bytes .../varp/var_level_connectivities/0.0 | Bin 144 -> 144 bytes .../edata_basic_with_tem.zarr/.zattrs | 4 + .../edata_basic_with_tem.zarr/anndata/.zattrs | 4 + .../{layers => anndata}/.zgroup | 0 .../{ => anndata}/X/.zarray | 0 .../{ => anndata}/X/.zattrs | 0 .../{ => anndata}/X/0.0 | Bin .../{ => anndata}/layers/.zattrs | 0 .../{obs => anndata/layers}/.zgroup | 0 .../{ => anndata}/layers/other_layer/.zarray | 0 .../{ => anndata}/layers/other_layer/.zattrs | 0 .../{ => anndata}/layers/other_layer/0.0 | Bin .../layers/tem_data}/.zarray | 0 .../layers/tem_data}/.zattrs | 0 .../layers/tem_data}/0.0.0 | Bin .../{ => anndata}/obs/.zattrs | 0 .../{obsm => anndata/obs}/.zgroup | 0 .../anndata}/obs/_index/.zarray | 8 +- .../obs/_index}/.zattrs | 0 .../anndata/obs/_index/0 | Bin 0 -> 45 bytes .../_index => anndata/obs/survival}/.zarray | 0 .../_index => anndata/obs/survival}/.zattrs | 0 .../{ => anndata}/obs/survival/0 | Bin .../{ => anndata}/obsm/.zattrs | 0 .../{obsp => anndata/obsm}/.zgroup | 0 .../obsm/obs_level_representation/.zarray | 0 .../obsm/obs_level_representation}/.zattrs | 0 .../obsm/obs_level_representation/0.0 | Bin .../{ => anndata}/obsp/.zattrs | 0 .../{uns => anndata/obsp}/.zgroup | 0 .../obsp/obs_level_connectivities/.zarray | 0 .../obsp/obs_level_connectivities}/.zattrs | 0 .../obsp/obs_level_connectivities/0.0 | Bin .../anndata/raw/.zarray | 10 + .../anndata/raw/.zattrs | 4 + .../edata_basic_with_tem.zarr/anndata/raw/0 | Bin 0 -> 8 bytes .../{ => anndata}/uns/.zattrs | 0 .../{var => anndata/uns}/.zgroup | 0 .../{ => anndata}/uns/information/.zarray | 0 .../uns/information}/.zattrs | 0 .../{ => anndata}/uns/information/0 | Bin .../{ => anndata}/var/.zattrs | 0 .../{varm => anndata/var}/.zgroup | 0 .../variables => anndata/var/_index}/.zarray | 0 .../anndata/var/_index}/.zattrs | 0 .../anndata/var/_index/0 | Bin 0 -> 40 bytes .../anndata}/var/variables/.zarray | 0 .../anndata}/var/variables/.zattrs | 0 .../{ => anndata}/var/variables/0 | Bin .../{ => anndata}/varm/.zattrs | 0 .../{varp => anndata/varm}/.zgroup | 0 .../varm/var_level_representation/.zarray | 0 .../varm/var_level_representation}/.zattrs | 0 .../varm/var_level_representation/0.0 | Bin .../{ => anndata}/varp/.zattrs | 0 .../anndata/varp}/.zgroup | 0 .../varp/var_level_connectivities/.zarray | 0 .../varp/var_level_connectivities}/.zattrs | 0 .../anndata/varp/var_level_connectivities/0.0 | Bin 0 -> 144 bytes .../edata_basic_with_tem.zarr/obs/_index/0 | Bin 56 -> 0 bytes .../tem/_index/.zarray | 8 +- .../tem/_index/.zattrs | 2 +- .../edata_basic_with_tem.zarr/tem/_index/0 | Bin 32 -> 30 bytes .../edata_basic_with_tem.zarr/var/_index/0 | Bin 48 -> 0 bytes .../varp/var_level_connectivities/0.0 | Bin 144 -> 0 bytes .../edata_sparse_with_tem.zarr/.zattrs | 4 + .../anndata/.zattrs | 4 + .../{layers => anndata}/.zgroup | 0 .../{ => anndata}/X/.zattrs | 0 .../{layers/other_layer => anndata/X}/.zgroup | 0 .../{ => anndata}/X/data/.zarray | 0 .../{ => anndata}/X/data/0 | Bin .../{ => anndata}/X/indices/.zarray | 0 .../{ => anndata}/X/indices/0 | Bin .../{ => anndata}/X/indptr/.zarray | 0 .../{ => anndata}/X/indptr/0 | Bin .../{ => anndata}/layers/.zattrs | 0 .../{obs => anndata/layers}/.zgroup | 0 .../{ => anndata}/layers/other_layer/.zattrs | 0 .../layers/other_layer}/.zgroup | 0 .../layers/other_layer/data/.zarray | 0 .../{ => anndata}/layers/other_layer/data/0 | Bin .../layers/other_layer/indices/.zarray | 0 .../layers/other_layer/indices/0 | Bin .../layers/other_layer/indptr/.zarray | 0 .../{ => anndata}/layers/other_layer/indptr/0 | Bin .../layers/tem_data}/.zarray | 0 .../anndata/layers/tem_data}/.zattrs | 0 .../layers/tem_data}/0.0.0 | Bin .../{ => anndata}/obs/.zattrs | 0 .../{obsp => anndata/obs}/.zgroup | 0 .../survival => anndata/obs/_index}/.zarray | 8 +- .../anndata/obs/_index/.zattrs | 4 + .../anndata/obs/_index/0 | Bin 0 -> 45 bytes .../anndata}/obs/survival/.zarray | 0 .../anndata/obs/survival}/.zattrs | 0 .../{ => anndata}/obs/survival/0 | Bin .../{ => anndata}/obsm/.zattrs | 0 .../{uns => anndata/obsm}/.zgroup | 0 .../obsm/obs_level_representation/.zarray | 0 .../obsm/obs_level_representation}/.zattrs | 0 .../obsm/obs_level_representation/0.0 | Bin .../{ => anndata}/obsp/.zattrs | 0 .../{var => anndata/obsp}/.zgroup | 0 .../obsp/obs_level_connectivities/.zarray | 0 .../obsp/obs_level_connectivities}/.zattrs | 0 .../obsp/obs_level_connectivities/0.0 | Bin .../anndata/raw/.zarray | 10 + .../anndata/raw/.zattrs | 4 + .../edata_sparse_with_tem.zarr/anndata/raw/0 | Bin 0 -> 8 bytes .../{ => anndata}/uns/.zattrs | 0 .../{varm => anndata/uns}/.zgroup | 0 .../{ => anndata}/uns/information/.zarray | 0 .../anndata/uns/information/.zattrs | 4 + .../{ => anndata}/uns/information/0 | Bin .../{ => anndata}/var/.zattrs | 0 .../{varp => anndata/var}/.zgroup | 0 .../anndata}/var/_index/.zarray | 8 +- .../anndata/var/_index/.zattrs | 4 + .../anndata/var/_index/0 | Bin 0 -> 40 bytes .../_index => anndata/var/variables}/.zarray | 8 +- .../anndata/var/variables/.zattrs | 4 + .../{ => anndata}/var/variables/0 | Bin .../{ => anndata}/varm/.zattrs | 0 .../anndata/varm/.zgroup | 3 + .../varm/var_level_representation/.zarray | 0 .../varm/var_level_representation}/.zattrs | 0 .../varm/var_level_representation/0.0 | Bin .../{ => anndata}/varp/.zattrs | 0 .../anndata/varp/.zgroup | 3 + .../varp/var_level_connectivities/.zarray | 0 .../varp/var_level_connectivities}/.zattrs | 0 .../anndata/varp/var_level_connectivities/0.0 | Bin 0 -> 144 bytes .../edata_sparse_with_tem.zarr/obs/_index/0 | Bin 56 -> 0 bytes .../obsp/obs_level_connectivities/.zattrs | 4 - .../tem/_index/.zarray | 8 +- .../tem/_index/.zattrs | 2 +- .../edata_sparse_with_tem.zarr/tem/_index/0 | Bin 32 -> 30 bytes .../var/_index/.zattrs | 4 - .../edata_sparse_with_tem.zarr/var/_index/0 | Bin 48 -> 0 bytes .../varm/var_level_representation/.zattrs | 4 - .../varp/var_level_connectivities/.zattrs | 4 - .../varp/var_level_connectivities/0.0 | Bin 144 -> 0 bytes ...zarr_h5ad_test_files_creation_reference.py | 26 +- tests/io/test_zarr.py | 54 ++- 152 files changed, 578 insertions(+), 87 deletions(-) create mode 100644 tests/data/toy_zarr/adata_basic.zarr/.zmetadata create mode 100644 tests/data/toy_zarr/adata_basic.zarr/raw/.zarray create mode 100644 tests/data/toy_zarr/adata_basic.zarr/raw/.zattrs create mode 100644 tests/data/toy_zarr/adata_basic.zarr/raw/0 create mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/.zattrs create mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/.zattrs rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{layers => anndata}/.zgroup (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/X/.zarray (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/X/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/X/0.0 (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/layers/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{obs => anndata/layers}/.zgroup (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/layers/other_layer/.zarray (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/layers/other_layer/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/layers/other_layer/0.0 (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{layers/tem_layer => anndata/layers/tem_data}/.zarray (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{layers/tem_layer => anndata/layers/tem_data}/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{layers/tem_layer => anndata/layers/tem_data}/0.0.0 (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/obs/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{obsm => anndata/obs}/.zgroup (100%) rename tests/data/toy_zarr/{edata_sparse_with_tem.zarr => edata_basic_with_tem.zarr/anndata}/obs/_index/.zarray (74%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{uns/information => anndata/obs/_index}/.zattrs (100%) create mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/obs/_index/0 rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{obs/_index => anndata/obs/survival}/.zarray (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{obs/_index => anndata/obs/survival}/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/obs/survival/0 (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/obsm/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{obsp => anndata/obsm}/.zgroup (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/obsm/obs_level_representation/.zarray (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{obs/survival => anndata/obsm/obs_level_representation}/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/obsm/obs_level_representation/0.0 (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/obsp/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{uns => anndata/obsp}/.zgroup (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/obsp/obs_level_connectivities/.zarray (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{obsm/obs_level_representation => anndata/obsp/obs_level_connectivities}/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/obsp/obs_level_connectivities/0.0 (100%) create mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/raw/.zarray create mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/raw/.zattrs create mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/raw/0 rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/uns/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{var => anndata/uns}/.zgroup (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/uns/information/.zarray (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{var/variables => anndata/uns/information}/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/uns/information/0 (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/var/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{varm => anndata/var}/.zgroup (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{var/variables => anndata/var/_index}/.zarray (100%) rename tests/data/toy_zarr/{edata_sparse_with_tem.zarr/uns/information => edata_basic_with_tem.zarr/anndata/var/_index}/.zattrs (100%) create mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/_index/0 rename tests/data/toy_zarr/{edata_sparse_with_tem.zarr => edata_basic_with_tem.zarr/anndata}/var/variables/.zarray (100%) rename tests/data/toy_zarr/{edata_sparse_with_tem.zarr => edata_basic_with_tem.zarr/anndata}/var/variables/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/var/variables/0 (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/varm/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{varp => anndata/varm}/.zgroup (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/varm/var_level_representation/.zarray (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{obsp/obs_level_connectivities => anndata/varm/var_level_representation}/.zattrs (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/varm/var_level_representation/0.0 (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/varp/.zattrs (100%) rename tests/data/toy_zarr/{edata_sparse_with_tem.zarr/X => edata_basic_with_tem.zarr/anndata/varp}/.zgroup (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{ => anndata}/varp/var_level_connectivities/.zarray (100%) rename tests/data/toy_zarr/edata_basic_with_tem.zarr/{var/_index => anndata/varp/var_level_connectivities}/.zattrs (100%) create mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/var_level_connectivities/0.0 delete mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/obs/_index/0 delete mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/var/_index/0 delete mode 100644 tests/data/toy_zarr/edata_basic_with_tem.zarr/varp/var_level_connectivities/0.0 create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/.zattrs create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/.zattrs rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{layers => anndata}/.zgroup (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/X/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{layers/other_layer => anndata/X}/.zgroup (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/X/data/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/X/data/0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/X/indices/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/X/indices/0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/X/indptr/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/X/indptr/0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/layers/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{obs => anndata/layers}/.zgroup (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/layers/other_layer/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{obsm => anndata/layers/other_layer}/.zgroup (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/layers/other_layer/data/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/layers/other_layer/data/0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/layers/other_layer/indices/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/layers/other_layer/indices/0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/layers/other_layer/indptr/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/layers/other_layer/indptr/0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{layers/tem_layer => anndata/layers/tem_data}/.zarray (100%) rename tests/data/toy_zarr/{edata_basic_with_tem.zarr/varm/var_level_representation => edata_sparse_with_tem.zarr/anndata/layers/tem_data}/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{layers/tem_layer => anndata/layers/tem_data}/0.0.0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/obs/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{obsp => anndata/obs}/.zgroup (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{obs/survival => anndata/obs/_index}/.zarray (74%) create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/obs/_index/.zattrs create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/obs/_index/0 rename tests/data/toy_zarr/{edata_basic_with_tem.zarr => edata_sparse_with_tem.zarr/anndata}/obs/survival/.zarray (100%) rename tests/data/toy_zarr/{edata_basic_with_tem.zarr/varp/var_level_connectivities => edata_sparse_with_tem.zarr/anndata/obs/survival}/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/obs/survival/0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/obsm/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{uns => anndata/obsm}/.zgroup (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/obsm/obs_level_representation/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{layers/tem_layer => anndata/obsm/obs_level_representation}/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/obsm/obs_level_representation/0.0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/obsp/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{var => anndata/obsp}/.zgroup (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/obsp/obs_level_connectivities/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{obs/_index => anndata/obsp/obs_level_connectivities}/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/obsp/obs_level_connectivities/0.0 (100%) create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/raw/.zarray create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/raw/.zattrs create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/raw/0 rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/uns/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{varm => anndata/uns}/.zgroup (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/uns/information/.zarray (100%) create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/information/.zattrs rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/uns/information/0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/var/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{varp => anndata/var}/.zgroup (100%) rename tests/data/toy_zarr/{edata_basic_with_tem.zarr => edata_sparse_with_tem.zarr/anndata}/var/_index/.zarray (74%) create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/_index/.zattrs create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/_index/0 rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{var/_index => anndata/var/variables}/.zarray (74%) create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/variables/.zattrs rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/var/variables/0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/varm/.zattrs (100%) create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/varm/.zgroup rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/varm/var_level_representation/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{obs/survival => anndata/varm/var_level_representation}/.zattrs (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/varm/var_level_representation/0.0 (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/varp/.zattrs (100%) create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/varp/.zgroup rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{ => anndata}/varp/var_level_connectivities/.zarray (100%) rename tests/data/toy_zarr/edata_sparse_with_tem.zarr/{obsm/obs_level_representation => anndata/varp/var_level_connectivities}/.zattrs (100%) create mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/varp/var_level_connectivities/0.0 delete mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/obs/_index/0 delete mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/obsp/obs_level_connectivities/.zattrs delete mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/var/_index/.zattrs delete mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/var/_index/0 delete mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/varm/var_level_representation/.zattrs delete mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/varp/var_level_connectivities/.zattrs delete mode 100644 tests/data/toy_zarr/edata_sparse_with_tem.zarr/varp/var_level_connectivities/0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 647ff04f..441b1e1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning][]. ### Maintenance - Enhanced {doc}`tutorials/getting_started` ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell +- Move from zarr<3 to zarr>3 ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell ### Fixed @@ -21,7 +22,9 @@ and this project adheres to [Semantic Versioning][]. - `EHRData` drops the `.R` field in favor of using `.layers` for any 3D data arrays ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell - `EHRData`'s shape property will always return a 3 dimensional shape. If an `EHRData` object has flat arrays only, the third dimension will be 1. ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell - The following functions now take a `layer` argument: {func}`~ehrdata.io.read_csv`, {func}`~ehrdata.io.from_pandas`, {func}`~ehrdata.io.to_pandas`, {func}`~ehrdata.io.omop.setup_variables`, {func}`~ehrdata.io.omop.setup_interval_variables`, {func}`~ehrdata.dt.ehrdata_blobs`, {func}`~ehrdata.dt.physionet2012`. If it is let to its default, `None`, the `.X` field of `EHRData` is used. Since `.X` is 2D in this release, in cases with 3D data, the `layer` argument needs to be used. ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell - +- `~ehrdata.io.write_zarr` now writes an `EHRData` specific store, with `AnnData` in a substore. +- `~ehrdata.io.write_zarr` now writes an `EHRData` specific store, with `AnnData` in a substore. This change allows to use `AnnData`s change to consolidated Zarr metadata. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell +- `~ehrdata.io.read_zarr` is adapted to read the new store format, and can also deal with `AnnData` stores. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell ## [0.0.9] diff --git a/src/ehrdata/core/constants.py b/src/ehrdata/core/constants.py index 18c6a633..3dc91930 100644 --- a/src/ehrdata/core/constants.py +++ b/src/ehrdata/core/constants.py @@ -11,6 +11,8 @@ DEFAULT_TEM_LAYER_NAME = "tem_data" +EHRDATA_ZARR_ENCODING_VERSION = "0.0.1" + # Missing values # -------------- # These values if encountered as strings are considered to represent missing values in the data diff --git a/src/ehrdata/io/zarr.py b/src/ehrdata/io/zarr.py index be8df7b9..505451b3 100644 --- a/src/ehrdata/io/zarr.py +++ b/src/ehrdata/io/zarr.py @@ -9,6 +9,7 @@ import ehrdata as ed from ehrdata._logger import logger +from ehrdata.core.constants import EHRDATA_ZARR_ENCODING_VERSION from ehrdata.io._array_casting import _cast_arrays_dtype_to_float_or_str_if_nonnumeric_object, _cast_variables_to_float if TYPE_CHECKING: @@ -25,6 +26,8 @@ def read_zarr( ) -> EHRData: """Read a zarr store into an :class:`~ehrdata.EHRData` object. + Can also read :class:`~anndata.AnnData` Zarr stores. In this case, a default `.tem` field is created in the `ehrdata object`. + Args: filename: The filename, or a Zarr storage class. harmonize_missing_values: Whether to call `ehrdata.harmonize_missing_values` on all detected layers. @@ -49,7 +52,11 @@ def read_zarr( f = filename if isinstance(filename, zarr.Group) else zarr.open(filename, mode="r") - if "ehrdata" in f.attrs: + if "encoding-type" not in f.attrs: + err = "The zarr store does not contain an encoding-type attribute." + raise ValueError(err) + + if f.attrs["encoding-type"] == "ehrdata": if "anndata" in f: dictionary_for_init = { k: ad.io.read_elem(f["anndata"][k]) for k, v in dict(f["anndata"]).items() if not k.startswith("raw.") @@ -61,33 +68,28 @@ def read_zarr( dictionary_for_init["tem"] = ad.io.read_elem(f["tem"]) else: warnings.warn("The zarr store does not contain the 'tem' group.", stacklevel=2) - else: - warnings.warn( - "The zarr store does not contain an ehrdata attribute. This is might not be a valid ehrdata Zarr store, and the store might not be readable or be wrongly interpeted.", - stacklevel=2, - ) - if "anndata" in f.attrs: - warnings.warn( - "The zarr store is an AnnData store, which can be read but might not support all ehrdata features.", - stacklevel=2, - ) + elif f.attrs["encoding-type"] == "anndata": dictionary_for_init = {k: ad.io.read_elem(f[k]) for k, v in dict(f).items() if not k.startswith("raw.")} - edata = EHRData(**dictionary_for_init) + else: + err = f"Unkown encoding-type '{f.attrs['encoding-type']}'." + raise ValueError(err) + + edata = EHRData(**dictionary_for_init) - if harmonize_missing_values: - ed.harmonize_missing_values(edata) - logger.info("Harmonizing missing values of X") + if harmonize_missing_values: + ed.harmonize_missing_values(edata) + logger.info("Harmonizing missing values of X") - for key in edata.layers: - ed.harmonize_missing_values(edata, layer=key) - logger.info(f"Harmonizing missing values of layer {key}") + for key in edata.layers: + ed.harmonize_missing_values(edata, layer=key) + logger.info(f"Harmonizing missing values of layer {key}") - if cast_variables_to_float: - _cast_variables_to_float(edata) + if cast_variables_to_float: + _cast_variables_to_float(edata) - return edata + return edata def write_zarr( @@ -119,8 +121,8 @@ def write_zarr( # TODO: add test that checks these fields # TODO: ensure this is "canonical" and what anndata is doing store = zarr.open(filename, mode="w") - store.attrs["ehrdata_version"] = ed.__version__ - store.attrs["ehrdata_type"] = "ehrdata" + store.attrs["encoding-version"] = EHRDATA_ZARR_ENCODING_VERSION + store.attrs["encoding-type"] = "ehrdata" # while adata.write_zarr supports convert_strings_to_categoricals, ad.io.write_elem does not # so we need to convert the strings to categoricals ourselves diff --git a/tests/data/toy_zarr/adata_basic.zarr/.zmetadata b/tests/data/toy_zarr/adata_basic.zarr/.zmetadata new file mode 100644 index 00000000..653be232 --- /dev/null +++ b/tests/data/toy_zarr/adata_basic.zarr/.zmetadata @@ -0,0 +1,382 @@ +{ + "metadata": { + ".zattrs": { + "encoding-type": "anndata", + "encoding-version": "0.1.0" + }, + ".zgroup": { + "zarr_format": 2 + }, + "X/.zarray": { + "chunks": [ + 5, + 4 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": "9 literal 0 HcmV?d00001 diff --git a/tests/data/toy_zarr/adata_basic.zarr/varp/var_level_connectivities/0.0 b/tests/data/toy_zarr/adata_basic.zarr/varp/var_level_connectivities/0.0 index 961195027b141cea377b3b2b8f79261451b5eeca..f156439c41c4dfd687ebeeb613f865f80b2fb280 100644 GIT binary patch literal 144 zcmV;B0B`>S0W%1I0001h0001x0002}xaj@+YVSV{Bt-(x5uv|rr=D2?K=i+v(P>aG zQ{=z7kDPA>Ezv)J8yqo=-Pb=K-%A*+S0W%1I0001h0001x0000;SPkzk!a=`)L3|yBS?RxAG=c+toz%bfvN&n! z_3ytQmCS>VQOG}TQ1P-bYxKYB(}nOQQSv{-mzhRL%Kkt2zU9ZpdHFy6RN>e#E7?ET yhRpXf=;}YM8dxO(DC@uN7HKA;srSE2G1$y0FWA3wj~!vzedRwRvz5&tnc_dR^hino diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/.zattrs new file mode 100644 index 00000000..42cb0339 --- /dev/null +++ b/tests/data/toy_zarr/edata_basic_with_tem.zarr/.zattrs @@ -0,0 +1,4 @@ +{ + "encoding-type": "ehrdata", + "encoding-version": "0.0.1" +} diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/.zattrs new file mode 100644 index 00000000..da73d032 --- /dev/null +++ b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/.zattrs @@ -0,0 +1,4 @@ +{ + "encoding-type": "anndata", + "encoding-version": "0.1.0" +} diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/.zgroup b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/.zgroup similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/.zgroup rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/.zgroup diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/X/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/X/.zarray similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/X/.zarray rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/X/.zarray diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/X/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/X/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/X/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/X/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/X/0.0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/X/0.0 similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/X/0.0 rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/X/0.0 diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/obs/.zgroup b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/.zgroup similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/obs/.zgroup rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/.zgroup diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/other_layer/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/other_layer/.zarray similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/other_layer/.zarray rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/other_layer/.zarray diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/other_layer/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/other_layer/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/other_layer/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/other_layer/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/other_layer/0.0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/other_layer/0.0 similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/other_layer/0.0 rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/other_layer/0.0 diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/tem_layer/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/tem_data/.zarray similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/tem_layer/.zarray rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/tem_data/.zarray diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/tem_layer/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/tem_data/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/tem_layer/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/tem_data/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/tem_layer/0.0.0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/tem_data/0.0.0 similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/layers/tem_layer/0.0.0 rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/layers/tem_data/0.0.0 diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/obs/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/obs/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/obs/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/obs/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/obsm/.zgroup b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/obs/.zgroup similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/obsm/.zgroup rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/obs/.zgroup diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/obs/_index/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/obs/_index/.zarray similarity index 74% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/obs/_index/.zarray rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/obs/_index/.zarray index 1a8c68ba..db861cb5 100644 --- a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/obs/_index/.zarray +++ b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/obs/_index/.zarray @@ -9,9 +9,13 @@ "id": "blosc", "shuffle": 1 }, - "dtype": "9 literal 0 HcmV?d00001 diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/uns/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/uns/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/var/.zgroup b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/.zgroup similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/var/.zgroup rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/.zgroup diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/uns/information/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/information/.zarray similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/uns/information/.zarray rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/information/.zarray diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/var/variables/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/information/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/var/variables/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/information/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/uns/information/0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/information/0 similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/uns/information/0 rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/uns/information/0 diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/var/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/var/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/varm/.zgroup b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/.zgroup similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/varm/.zgroup rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/.zgroup diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/var/variables/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/_index/.zarray similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/var/variables/.zarray rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/_index/.zarray diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/uns/information/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/_index/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/uns/information/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/_index/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/_index/0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/_index/0 new file mode 100644 index 0000000000000000000000000000000000000000..ed5952e5a254d29ada33f16dea0bca29535265a7 GIT binary patch literal 40 hcmZQ#G-i}wU|;}Y4IpL#Vn!e~01<{D!U#kd0{|K#0bT$A literal 0 HcmV?d00001 diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/var/variables/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/variables/.zarray similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/var/variables/.zarray rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/variables/.zarray diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/var/variables/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/variables/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/var/variables/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/variables/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/var/variables/0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/variables/0 similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/var/variables/0 rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/var/variables/0 diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/varm/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/varm/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/varp/.zgroup b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/.zgroup similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/varp/.zgroup rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/.zgroup diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/varm/var_level_representation/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/var_level_representation/.zarray similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/varm/var_level_representation/.zarray rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/var_level_representation/.zarray diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/obsp/obs_level_connectivities/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/var_level_representation/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/obsp/obs_level_connectivities/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/var_level_representation/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/varm/var_level_representation/0.0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/var_level_representation/0.0 similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/varm/var_level_representation/0.0 rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varm/var_level_representation/0.0 diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/varp/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/varp/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/.zattrs diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/X/.zgroup b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/.zgroup similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/X/.zgroup rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/.zgroup diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/varp/var_level_connectivities/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/var_level_connectivities/.zarray similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/varp/var_level_connectivities/.zarray rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/var_level_connectivities/.zarray diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/var/_index/.zattrs b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/var_level_connectivities/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/var/_index/.zattrs rename to tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/var_level_connectivities/.zattrs diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/var_level_connectivities/0.0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/anndata/varp/var_level_connectivities/0.0 new file mode 100644 index 0000000000000000000000000000000000000000..ff45b4b7637490e9396250ffff6d473b7024298c GIT binary patch literal 144 zcmV;B0B`>S0W%1I0001h0001x0000S1`{$3R_wobwT4d1;>|x83onUnRo1_OjqwH) zp6$QzTCoeW3Fg1{|0bQzPV_&Gaz1qJX57Ehxyfj`bOFE^SLFrr=)}Ksf)a=xi_$+* yuw92(4e&pboleJ+;n+V5)A0<1<>$YE1a?MMQ0Bh`Fz~rIgUG+OA()xvru9Exu|(+r literal 0 HcmV?d00001 diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/obs/_index/0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/obs/_index/0 deleted file mode 100644 index f680bc4e6934b8b6151f00f53907f7abe38fd6ea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 56 fcmZQ#H0ID?U|;}Y3m}F8Mks>`N;5-g7AOq>Cn*5Y diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/tem/_index/.zarray b/tests/data/toy_zarr/edata_basic_with_tem.zarr/tem/_index/.zarray index 94d08c46..4f883bac 100644 --- a/tests/data/toy_zarr/edata_basic_with_tem.zarr/tem/_index/.zarray +++ b/tests/data/toy_zarr/edata_basic_with_tem.zarr/tem/_index/.zarray @@ -9,9 +9,13 @@ "id": "blosc", "shuffle": 1 }, - "dtype": "`N;5-g036`}wEzGB diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/varp/var_level_connectivities/0.0 b/tests/data/toy_zarr/edata_basic_with_tem.zarr/varp/var_level_connectivities/0.0 deleted file mode 100644 index 54222f89da1548b8f538f165a7767b52a706f7f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 144 zcmV;B0B`>S0W%1I0001h0001x0000{xLoyvwbVc7Dkvt>;ORe89+mPo#?!wJ>9xGa zHN8J(_HU0$#`M3ehJQOr+0j25WRVp93FAL=Ioxh8P~pD{sHlPS%H+Sb1Eq>OvuVGu yCDHZB9sIxT)Q;Lgfswxgu-JCyfXu%u5X)~fS>!)C5nZ}~9 literal 0 HcmV?d00001 diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/uns/.zattrs b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/uns/.zattrs rename to tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/.zattrs diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varm/.zgroup b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/.zgroup similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/varm/.zgroup rename to tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/.zgroup diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/uns/information/.zarray b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/information/.zarray similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/uns/information/.zarray rename to tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/information/.zarray diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/information/.zattrs b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/information/.zattrs new file mode 100644 index 00000000..ed07997c --- /dev/null +++ b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/information/.zattrs @@ -0,0 +1,4 @@ +{ + "encoding-type": "string-array", + "encoding-version": "0.2.0" +} diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/uns/information/0 b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/information/0 similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/uns/information/0 rename to tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/uns/information/0 diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/var/.zattrs b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/.zattrs similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/var/.zattrs rename to tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/.zattrs diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varp/.zgroup b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/.zgroup similarity index 100% rename from tests/data/toy_zarr/edata_sparse_with_tem.zarr/varp/.zgroup rename to tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/.zgroup diff --git a/tests/data/toy_zarr/edata_basic_with_tem.zarr/var/_index/.zarray b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/_index/.zarray similarity index 74% rename from tests/data/toy_zarr/edata_basic_with_tem.zarr/var/_index/.zarray rename to tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/_index/.zarray index 34562363..95612f63 100644 --- a/tests/data/toy_zarr/edata_basic_with_tem.zarr/var/_index/.zarray +++ b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/anndata/var/_index/.zarray @@ -9,9 +9,13 @@ "id": "blosc", "shuffle": 1 }, - "dtype": "S0W%1I0001h0001x0000S1`{$3R_wobwT4d1;>|x83onUnRo1_OjqwH) zp6$QzTCoeW3Fg1{|0bQzPV_&Gaz1qJX57Ehxyfj`bOFE^SLFrr=)}Ksf)a=xi_$+* yuw92(4e&pboleJ+;n+V5)A0<1<>$YE1a?MMQ0Bh`Fz~rIgUG+OA()xvru9Exu|(+r literal 0 HcmV?d00001 diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/obs/_index/0 b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/obs/_index/0 deleted file mode 100644 index f680bc4e6934b8b6151f00f53907f7abe38fd6ea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 56 fcmZQ#H0ID?U|;}Y3m}F8Mks>`N;5-g7AOq>Cn*5Y diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/obsp/obs_level_connectivities/.zattrs b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/obsp/obs_level_connectivities/.zattrs deleted file mode 100644 index ecdb6018..00000000 --- a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/obsp/obs_level_connectivities/.zattrs +++ /dev/null @@ -1,4 +0,0 @@ -{ - "encoding-type": "array", - "encoding-version": "0.2.0" -} diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/tem/_index/.zarray b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/tem/_index/.zarray index 94d08c46..4f883bac 100644 --- a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/tem/_index/.zarray +++ b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/tem/_index/.zarray @@ -9,9 +9,13 @@ "id": "blosc", "shuffle": 1 }, - "dtype": "`N;5-g036`}wEzGB diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varm/var_level_representation/.zattrs b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varm/var_level_representation/.zattrs deleted file mode 100644 index ecdb6018..00000000 --- a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varm/var_level_representation/.zattrs +++ /dev/null @@ -1,4 +0,0 @@ -{ - "encoding-type": "array", - "encoding-version": "0.2.0" -} diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varp/var_level_connectivities/.zattrs b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varp/var_level_connectivities/.zattrs deleted file mode 100644 index ecdb6018..00000000 --- a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varp/var_level_connectivities/.zattrs +++ /dev/null @@ -1,4 +0,0 @@ -{ - "encoding-type": "array", - "encoding-version": "0.2.0" -} diff --git a/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varp/var_level_connectivities/0.0 b/tests/data/toy_zarr/edata_sparse_with_tem.zarr/varp/var_level_connectivities/0.0 deleted file mode 100644 index 54222f89da1548b8f538f165a7767b52a706f7f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 144 zcmV;B0B`>S0W%1I0001h0001x0000{xLoyvwbVc7Dkvt>;ORe89+mPo#?!wJ>9xGa zHN8J(_HU0$#`M3ehJQOr+0j25WRVp93FAL=Ioxh8P~pD{sHlPS%H+Sb1Eq>OvuVGu yCDHZB9sIxT)Q;Lgfswxgu-JCyfXu%u5X)~fS>!)C5nZ}~ Date: Sun, 2 Nov 2025 12:33:57 +0100 Subject: [PATCH 06/25] expand tests zarr; enable nullable string used for categorical names; disable chunks arg in write_zarr --- .github/workflows/test.yaml | 7 ++- pyproject.toml | 5 ++- src/ehrdata/__init__.py | 5 +++ src/ehrdata/io/zarr.py | 22 +--------- tests/conftest.py | 44 +++++++++++++++++++ tests/io/test_zarr.py | 88 ++++++++++++------------------------- 6 files changed, 90 insertions(+), 81 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 640a7c43..76683b09 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -74,7 +74,12 @@ jobs: - name: create hatch environment run: uvx hatch env create ${{ matrix.env.name }} - name: Show installed packages - run: uvx hatch run ${{ matrix.env.name }}:pip list + run: | + hatch run hatch-test.py3.13:python - <<'PYCODE' + import importlib.metadata as m + for dist in sorted(m.distributions(), key=lambda d: d.metadata["Name"].lower()): + print(f"{dist.metadata['Name']}=={dist.version}") + PYCODE - name: run tests using hatch env: MPLBACKEND: agg diff --git a/pyproject.toml b/pyproject.toml index 23d4e724..c05e5bef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,9 @@ dependencies = [ "xarray", "zarr>=3", ] +optional-dependencies.dask = [ + "dask", +] optional-dependencies.dev = [ "pre-commit", "twine", @@ -61,7 +64,7 @@ optional-dependencies.lamin = [ ] optional-dependencies.test = [ "coverage", - "ehrdata[lamin,torch,vitessce]", + "ehrdata[dask,lamin,torch,vitessce]", "pytest", "pytest-cov", ] diff --git a/src/ehrdata/__init__.py b/src/ehrdata/__init__.py index 32a6b882..5d5b46c8 100644 --- a/src/ehrdata/__init__.py +++ b/src/ehrdata/__init__.py @@ -16,3 +16,8 @@ ] __version__ = version("ehrdata") + +import anndata as ad + +# Opt to use this newer feature of anndata https://github.com/scverse/anndata/blob/6a6bde151eeb231eebac20b66e6002b88052e8db/src/anndata/_io/specs/methods.py#L1151 +ad.settings.allow_write_nullable_strings = True diff --git a/src/ehrdata/io/zarr.py b/src/ehrdata/io/zarr.py index 505451b3..a08fbdcf 100644 --- a/src/ehrdata/io/zarr.py +++ b/src/ehrdata/io/zarr.py @@ -96,7 +96,7 @@ def write_zarr( edata: EHRData, filename: str | Path, *, - chunks: bool | int | tuple[int, ...] | None = None, + # chunks: bool | int | tuple[int, ...] | None = (1000, 1000), blocked by https://github.com/scverse/anndata/issues/2193 convert_strings_to_categoricals: bool = True, ) -> None: """Write :class:`~ehrdata.EHRData` objects to disk. @@ -107,7 +107,6 @@ def write_zarr( Args: edata: Central data object. filename: Name of the output file, can also be prefixed with relative or absolute path to save the file to. - chunks: Chunk shape, passed to :meth:`zarr.Group.create_array` for `Zarr` version 3. convert_strings_to_categoricals: Convert columns of `str` dtype in `.obs` and `.var` and `.tem` to `categorical` dtype. Examples: @@ -118,39 +117,22 @@ def write_zarr( filename = Path(filename) edata = _cast_arrays_dtype_to_float_or_str_if_nonnumeric_object(edata) - # TODO: add test that checks these fields - # TODO: ensure this is "canonical" and what anndata is doing store = zarr.open(filename, mode="w") store.attrs["encoding-version"] = EHRDATA_ZARR_ENCODING_VERSION store.attrs["encoding-type"] = "ehrdata" - # while adata.write_zarr supports convert_strings_to_categoricals, ad.io.write_elem does not - # so we need to convert the strings to categoricals ourselves - # TODO: figure out what anndata is doing - # map all columns of edata.obs to categorical dtype if they are of str dtype - # def _convert_to_categorical(df: pd.DataFrame) -> pd.DataFrame: - # for column in df.columns: - # if df[column].dtype == "str": - # df[column] = df[column].astype("category") - # return df - adata = ad.AnnData(edata) - # TODO: test if convert_strings_to_categoricals: - # inplace conversion adata.strings_to_categoricals(adata.obs) adata.strings_to_categoricals(adata.var) adata.strings_to_categoricals(edata.tem) - # adata.obs = edata.obs if not convert_strings_to_categoricals else _convert_to_categorical(edata.obs) - # adata.var = edata.var if not convert_strings_to_categoricals else _convert_to_categorical(edata.var) - # adata.tem = edata.tem if not convert_strings_to_categoricals else _convert_to_categorical(edata.tem) ad.io.write_elem( store, "anndata", adata, # this will store everything but the .tem field - dataset_kwargs={"chunks": chunks}, + # dataset_kwargs={"chunks": chunks}, # blocked by https://github.com/scverse/anndata/issues/2193 ) ad.io.write_elem(store, "tem", edata.tem) diff --git a/tests/conftest.py b/tests/conftest.py index 0ebac26e..5771ba8f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -110,6 +110,19 @@ def obs_31(): return pd.DataFrame({"obs_col_1": [1, 2, 3]}, index=["obs1", "obs2", "obs3"]) +@pytest.fixture +def obs_32(): + df = pd.DataFrame( + { + "obs_col_1": [1, 2, 3], + "obs_col_2": ["a", "a", "c"], + }, + index=["obs1", "obs2", "obs3"], + ) + df = df.astype({"obs_col_2": "string"}) + return df + + @pytest.fixture def var_21(): return pd.DataFrame({"var_col_1": [1, 2]}, index=["var1", "var2"]) @@ -120,6 +133,19 @@ def var_31(): return pd.DataFrame({"var_col_1": [1, 2, 3]}, index=["var1", "var2", "var3"]) +@pytest.fixture +def var_32(): + df = pd.DataFrame( + { + "var_col_1": [1, 2, 3], + "var_col_2": ["x", "x", "z"], + }, + index=["var1", "var2", "var3"], + ) + df = df.astype({"var_col_2": "string"}) + return df + + @pytest.fixture def tem_11(): return pd.DataFrame({"tem_col_1": [1]}, index=["t1"]) @@ -135,11 +161,29 @@ def tem_31(): return pd.DataFrame({"tem_col_1": [1, 2, 3]}, index=["t1", "t2", "t3"]) +@pytest.fixture +def tem_32(): + df = pd.DataFrame( + { + "tem_col_1": [1, 2, 3], + "tem_col_2": ["l", "l", "n"], + }, + index=["t1", "t2", "t3"], + ) + df = df.astype({"tem_col_2": "string"}) + return df + + @pytest.fixture def edata_333(X_numpy_33, X_numpy_333, obs_31, var_31, tem_31): return EHRData(X=X_numpy_33, layers={DEFAULT_TEM_LAYER_NAME: X_numpy_333}, obs=obs_31, var=var_31, tem=tem_31) +@pytest.fixture +def edata_333_larger_obs_var_tem(X_numpy_33, X_numpy_333, obs_32, var_32, tem_32): + return EHRData(X=X_numpy_33, layers={DEFAULT_TEM_LAYER_NAME: X_numpy_333}, obs=obs_32, var=var_32, tem=tem_32) + + @pytest.fixture def edata_330(X_numpy_33, obs_31, var_31): return EHRData(X=X_numpy_33, obs=obs_31, var=var_31) diff --git a/tests/io/test_zarr.py b/tests/io/test_zarr.py index 4715a667..b01d1f27 100644 --- a/tests/io/test_zarr.py +++ b/tests/io/test_zarr.py @@ -1,4 +1,3 @@ -import anndata as ad import numpy as np import pandas as pd import pytest @@ -28,7 +27,7 @@ def test_read_anndata_zarr_basic(harmonize_missing_values, cast_variables_to_flo store = zarr.open(TEST_PATH_ZARR / "adata_basic.zarr") - # verify the test file is an anndata zarr store + # check the store is an anndata zarr store assert store.attrs["encoding-type"] == "anndata" _assert_shape_matches(edata, (5, 4, 1)) @@ -49,7 +48,7 @@ def test_read_zarr_basic_with_tem(harmonize_missing_values, cast_variables_to_fl assert "timestep" in edata.tem.columns assert all(edata.tem["timestep"].values == ["t1", "t2"]) - # verify the test file is an anndata zarr store + # check the store is an ehrdata zarr store store = zarr.open(TEST_PATH_ZARR / "edata_basic_with_tem.zarr") assert store.attrs["encoding-type"] == "ehrdata" assert store.attrs["encoding-version"] == EHRDATA_ZARR_ENCODING_VERSION @@ -72,64 +71,22 @@ def test_read_zarr_sparse_with_tem(harmonize_missing_values, cast_variables_to_f assert issparse(edata.X) assert issparse(edata.layers["other_layer"]) - # verify the test file is an anndata zarr store + # check the store is an ehrdata zarr store store = zarr.open(TEST_PATH_ZARR / "edata_sparse_with_tem.zarr") assert store.attrs["encoding-type"] == "ehrdata" assert store.attrs["encoding-version"] == EHRDATA_ZARR_ENCODING_VERSION -@pytest.mark.parametrize("edata_name", ["edata_333", "edata_basic_with_tem_full", "edata_nonnumeric_missing_330"]) -def test_write_zarr_basic(edata_name, request, tmp_path): - edata = request.getfixturevalue(edata_name) - store_path = tmp_path / f"{edata_name}.zarr" - - write_zarr(edata, store_path) - - created_group = zarr.open(store_path, mode="r") - assert set(dict(created_group).keys()) == { - "anndata", - "tem", - } - - assert set(dict(created_group["anndata"]).keys()) == { - "X", - "obs", - "var", - "obsm", - "varm", - "layers", - "obsp", - "varp", - "raw", - "uns", - } - - assert np.array_equal(ad.io.read_elem(created_group["anndata"]["X"]).astype(str), edata.X.astype(str)) - - pd.testing.assert_frame_equal(ad.io.read_elem(created_group["anndata"]["obs"]), edata.obs) - pd.testing.assert_frame_equal(ad.io.read_elem(created_group["anndata"]["var"]), edata.var) - pd.testing.assert_frame_equal(ad.io.read_elem(created_group["tem"]), edata.tem) - for key in edata.obsm: - assert key in ad.io.read_elem(created_group["anndata"]["obsm"]) - assert np.array_equal(ad.io.read_elem(created_group["anndata"]["obsm"][key]), edata.obsm[key]) - for key in edata.varm: - assert key in ad.io.read_elem(created_group["anndata"]["varm"]) - assert np.array_equal(ad.io.read_elem(created_group["anndata"]["varm"][key]), edata.varm[key]) - for key in edata.obsp: - assert key in ad.io.read_elem(created_group["anndata"]["obsp"]) - assert np.array_equal(ad.io.read_elem(created_group["anndata"]["obsp"][key]), edata.obsp[key]) - for key in edata.varp: - assert key in ad.io.read_elem(created_group["anndata"]["varp"]) - assert np.array_equal(ad.io.read_elem(created_group["anndata"]["varp"][key]), edata.varp[key]) - for key in edata.uns: - assert key in ad.io.read_elem(created_group["anndata"]["uns"]) - - store = zarr.open(TEST_PATH_ZARR / "edata_sparse_with_tem.zarr") - assert store.attrs["encoding-type"] == "ehrdata" - assert store.attrs["encoding-version"] == EHRDATA_ZARR_ENCODING_VERSION - - -@pytest.mark.parametrize("edata_name", ["edata_333", "edata_basic_with_tem_full", "edata_nonnumeric_missing_330"]) +@pytest.mark.parametrize( + "edata_name", + [ + "edata_330", + "edata_333", + "edata_333_larger_obs_var_tem", + "edata_basic_with_tem_full", + "edata_nonnumeric_missing_330", + ], +) def test_write_read_zarr_basic(edata_name, request, tmp_path): edata = request.getfixturevalue(edata_name) store_path = tmp_path / f"{edata_name}.zarr" @@ -143,9 +100,9 @@ def test_write_read_zarr_basic(edata_name, request, tmp_path): for key in edata.layers: _assert_dtype_object_array_with_missing_values_equal(edata.layers[key], edata_read.layers[key]) - pd.testing.assert_frame_equal(edata.obs, edata_read.obs) - pd.testing.assert_frame_equal(edata.var, edata_read.var) - pd.testing.assert_frame_equal(edata.tem, edata_read.tem) + pd.testing.assert_frame_equal(edata.obs.iloc[:, :1], edata_read.obs.iloc[:, :1]) + pd.testing.assert_frame_equal(edata.var.iloc[:, :1], edata_read.var.iloc[:, :1]) + pd.testing.assert_frame_equal(edata.tem.iloc[:, :1], edata_read.tem.iloc[:, :1]) for key in edata.obsm: assert key in edata_read.obsm assert np.array_equal(edata.obsm[key], edata_read.obsm[key]) @@ -161,3 +118,16 @@ def test_write_read_zarr_basic(edata_name, request, tmp_path): for key in edata.uns: assert key in edata_read.uns assert np.array_equal(edata.uns[key], edata_read.uns[key]) + + # check the test file is an ehrdata zarr store + store = zarr.open(store_path) + assert store.attrs["encoding-type"] == "ehrdata" + assert store.attrs["encoding-version"] == EHRDATA_ZARR_ENCODING_VERSION + + # check success of convert_strings_to_categoricals + if "obs_col_2" in edata_read.obs.columns: + assert edata_read.obs["obs_col_2"].dtype == "category" + if "var_col_2" in edata_read.var.columns: + assert edata_read.var["var_col_2"].dtype == "category" + if "tem_col_2" in edata_read.tem.columns: + assert edata_read.tem["tem_col_2"].dtype == "category" From 7dbc66291c728ecb8c2596728f274f3b282e30f3 Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 12:35:12 +0100 Subject: [PATCH 07/25] cleaner changelog --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 441b1e1b..c0b6044c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,6 @@ and this project adheres to [Semantic Versioning][]. - `EHRData` drops the `.R` field in favor of using `.layers` for any 3D data arrays ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell - `EHRData`'s shape property will always return a 3 dimensional shape. If an `EHRData` object has flat arrays only, the third dimension will be 1. ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell - The following functions now take a `layer` argument: {func}`~ehrdata.io.read_csv`, {func}`~ehrdata.io.from_pandas`, {func}`~ehrdata.io.to_pandas`, {func}`~ehrdata.io.omop.setup_variables`, {func}`~ehrdata.io.omop.setup_interval_variables`, {func}`~ehrdata.dt.ehrdata_blobs`, {func}`~ehrdata.dt.physionet2012`. If it is let to its default, `None`, the `.X` field of `EHRData` is used. Since `.X` is 2D in this release, in cases with 3D data, the `layer` argument needs to be used. ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell -- `~ehrdata.io.write_zarr` now writes an `EHRData` specific store, with `AnnData` in a substore. - `~ehrdata.io.write_zarr` now writes an `EHRData` specific store, with `AnnData` in a substore. This change allows to use `AnnData`s change to consolidated Zarr metadata. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell - `~ehrdata.io.read_zarr` is adapted to read the new store format, and can also deal with `AnnData` stores. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell From a3d6a33af4579e093dd3695e2071948b5a5dc897 Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 12:40:36 +0100 Subject: [PATCH 08/25] fix test workflow package display? --- .github/workflows/test.yaml | 2 +- src/ehrdata/io/zarr.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 76683b09..c2376c80 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -75,7 +75,7 @@ jobs: run: uvx hatch env create ${{ matrix.env.name }} - name: Show installed packages run: | - hatch run hatch-test.py3.13:python - <<'PYCODE' + uvx hatch run hatch-test.py3.13:python - <<'PYCODE' import importlib.metadata as m for dist in sorted(m.distributions(), key=lambda d: d.metadata["Name"].lower()): print(f"{dist.metadata['Name']}=={dist.version}") diff --git a/src/ehrdata/io/zarr.py b/src/ehrdata/io/zarr.py index a08fbdcf..5406cfc0 100644 --- a/src/ehrdata/io/zarr.py +++ b/src/ehrdata/io/zarr.py @@ -43,10 +43,6 @@ def read_zarr( """ from ehrdata import EHRData - # TODO: check that anndata can be read - # TODO: check for backwards compatibility and announce clear version of when it stops - # TODO: check that ehrdata can be read - if isinstance(filename, Path): filename = str(filename) From 7cb4931f54618ce41aa239108b5ff7aa7638bc58 Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 12:50:03 +0100 Subject: [PATCH 09/25] fix test workflow package display? --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c2376c80..f49b906e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -74,6 +74,7 @@ jobs: - name: create hatch environment run: uvx hatch env create ${{ matrix.env.name }} - name: Show installed packages + if: matrix.env.python == '3.13' run: | uvx hatch run hatch-test.py3.13:python - <<'PYCODE' import importlib.metadata as m From 64db485961830baf44c230ec8fec621955f124c1 Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 13:09:39 +0100 Subject: [PATCH 10/25] rtd build with minimal vitessce depndency? --- pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c05e5bef..f6822dcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ optional-dependencies.dev = [ ] optional-dependencies.doc = [ "docutils>=0.8,!=0.18.*,!=0.19.*", - "ehrdata[lamin,torch,vitessce]", + "ehrdata[lamin,torch]", "ipykernel", "ipython", "myst-nb>=1.1", @@ -54,6 +54,7 @@ optional-dependencies.doc = [ "sphinx-tabs", "sphinxcontrib-bibtex>=1", "sphinxext-opengraph", + "vitessce", # vitessce[all] imports problematic dependencies ] optional-dependencies.ehrapy = [ "ehrapy", @@ -85,7 +86,9 @@ installer = "uv" features = [ "dev" ] [tool.hatch.envs.docs] + features = [ "doc" ] + scripts.build = "sphinx-build -M html docs docs/_build {args}" scripts.open = "python -m webbrowser -t docs/_build/html/index.html" scripts.clean = "git clean -fdX -- {args:docs}" From 36a9bbd718f5149ef57dcd9a83b69d10cb04136d Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 13:17:58 +0100 Subject: [PATCH 11/25] no vitessce check? --- pyproject.toml | 2 +- src/ehrdata/integrations/vitessce/_config.py | 218 +++++++++---------- 2 files changed, 110 insertions(+), 110 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f6822dcf..a159c908 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ optional-dependencies.doc = [ "sphinx-tabs", "sphinxcontrib-bibtex>=1", "sphinxext-opengraph", - "vitessce", # vitessce[all] imports problematic dependencies + # "vitessce", # vitessce[all] imports problematic dependencies ] optional-dependencies.ehrapy = [ "ehrapy", diff --git a/src/ehrdata/integrations/vitessce/_config.py b/src/ehrdata/integrations/vitessce/_config.py index 7826dc64..8f42534f 100644 --- a/src/ehrdata/integrations/vitessce/_config.py +++ b/src/ehrdata/integrations/vitessce/_config.py @@ -1,109 +1,109 @@ -from __future__ import annotations - -from functools import reduce -from operator import or_, truediv -from pathlib import Path -from types import MappingProxyType -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from collections.abc import Mapping - - from lamindb import Artifact - from vitessce import VitessceConfig - from zarr.storage import Store - - -def gen_config( - path: Path | None = None, - *, - store: Path | Store | None = None, - url: str | None = None, - artifact: Artifact | None = None, - # arguments not about how the store goes in: - name: str | None = None, - obs_sets: Mapping[str, str] = MappingProxyType({"obs/gender_concept_id": "Gender Concept ID"}), - obs_embeddings: Mapping[str, str] = MappingProxyType({"obsm/X_pca": "PCA"}), -) -> VitessceConfig: - """Generate a VitessceConfig for EHRData. - - Args: - path: Path to the data's Zarr store directory. - store: The data's Zarr store or a path to it. - url: URL pointing to the data's remote Zarr store. - artifact: Lamin artifact representing the data. - name: Name of the dataset. If None, derived from path. - obs_sets: Mapping of observation set paths to names, e.g. - {"obs/some_annotation": "My cool annotation"} - obs_embeddings: Mapping of observation embedding paths to names, e.g. - {"obsm/X_pca": "PCA"} - - Returns: - A :doc:`Vitessce ` configuration object. - Call :meth:`~vitessce.config.VitessceConfig.widget` on it to display it. - """ - obs_type = "person" - feature_type = "variable" - - if name is None: - if artifact is not None: - name = artifact.description - elif path is not None: - name = path.stem - else: - msg = "`name` needs to be specified or derived from `path` or `artifact`." - raise ValueError(msg) - - coordination = { - "obsType": obs_type, - "featureType": feature_type, - } - - from vitessce import AnnDataWrapper, VitessceConfig - from vitessce import Component as cm - - wrapper = AnnDataWrapper( - adata_path=path, - adata_url=url, - # vitessce is old and doesn't deal with proper Paths - adata_store=str(store) if isinstance(store, Path) else store, - adata_artifact=artifact, - obs_set_paths=list(obs_sets.keys()), - obs_set_names=list(obs_sets.values()), - obs_embedding_paths=list(obs_embeddings.keys()), - obs_embedding_names=list(obs_embeddings.values()), - obs_feature_matrix_path="X", - coordination_values=coordination, - ) - - vc = VitessceConfig(schema_version="1.0.15", name=name) - dataset = vc.add_dataset(name=name).add_object(wrapper) - - views = ( - ( - vc.add_view(cm.OBS_SETS, dataset=dataset), - vc.add_view(cm.OBS_SET_SIZES, dataset=dataset), - vc.add_view(cm.OBS_SET_FEATURE_VALUE_DISTRIBUTION, dataset=dataset), - ), - ( - vc.add_view(cm.FEATURE_LIST, dataset=dataset), - vc.add_view(cm.SCATTERPLOT, dataset=dataset, mapping="PCA"), - vc.add_view(cm.FEATURE_VALUE_HISTOGRAM, dataset=dataset), - ), - ( - vc.add_view(cm.DESCRIPTION, dataset=dataset), - vc.add_view(cm.STATUS, dataset=dataset), - vc.add_view(cm.HEATMAP, dataset=dataset), - ), - ) - - vc.link_views( - [view for row in views for view in row], - list(coordination.keys()), - list(coordination.values()), - ) - - # (a / b / c) | (d / e / f) | ... - vc.layout(reduce(or_, (reduce(truediv, row) for row in views))) - - return vc +# from __future__ import annotations + +# from functools import reduce +# from operator import or_, truediv +# from pathlib import Path +# from types import MappingProxyType +# from typing import TYPE_CHECKING + +# if TYPE_CHECKING: +# from collections.abc import Mapping + +# from lamindb import Artifact +# from vitessce import VitessceConfig +# from zarr.storage import Store + + +# def gen_config( +# path: Path | None = None, +# *, +# store: Path | Store | None = None, +# url: str | None = None, +# artifact: Artifact | None = None, +# # arguments not about how the store goes in: +# name: str | None = None, +# obs_sets: Mapping[str, str] = MappingProxyType({"obs/gender_concept_id": "Gender Concept ID"}), +# obs_embeddings: Mapping[str, str] = MappingProxyType({"obsm/X_pca": "PCA"}), +# ) -> VitessceConfig: +# """Generate a VitessceConfig for EHRData. + +# Args: +# path: Path to the data's Zarr store directory. +# store: The data's Zarr store or a path to it. +# url: URL pointing to the data's remote Zarr store. +# artifact: Lamin artifact representing the data. +# name: Name of the dataset. If None, derived from path. +# obs_sets: Mapping of observation set paths to names, e.g. +# {"obs/some_annotation": "My cool annotation"} +# obs_embeddings: Mapping of observation embedding paths to names, e.g. +# {"obsm/X_pca": "PCA"} + +# Returns: +# A :doc:`Vitessce ` configuration object. +# Call :meth:`~vitessce.config.VitessceConfig.widget` on it to display it. +# """ +# obs_type = "person" +# feature_type = "variable" + +# if name is None: +# if artifact is not None: +# name = artifact.description +# elif path is not None: +# name = path.stem +# else: +# msg = "`name` needs to be specified or derived from `path` or `artifact`." +# raise ValueError(msg) + +# coordination = { +# "obsType": obs_type, +# "featureType": feature_type, +# } + +# from vitessce import AnnDataWrapper, VitessceConfig +# from vitessce import Component as cm + +# wrapper = AnnDataWrapper( +# adata_path=path, +# adata_url=url, +# # vitessce is old and doesn't deal with proper Paths +# adata_store=str(store) if isinstance(store, Path) else store, +# adata_artifact=artifact, +# obs_set_paths=list(obs_sets.keys()), +# obs_set_names=list(obs_sets.values()), +# obs_embedding_paths=list(obs_embeddings.keys()), +# obs_embedding_names=list(obs_embeddings.values()), +# obs_feature_matrix_path="X", +# coordination_values=coordination, +# ) + +# vc = VitessceConfig(schema_version="1.0.15", name=name) +# dataset = vc.add_dataset(name=name).add_object(wrapper) + +# views = ( +# ( +# vc.add_view(cm.OBS_SETS, dataset=dataset), +# vc.add_view(cm.OBS_SET_SIZES, dataset=dataset), +# vc.add_view(cm.OBS_SET_FEATURE_VALUE_DISTRIBUTION, dataset=dataset), +# ), +# ( +# vc.add_view(cm.FEATURE_LIST, dataset=dataset), +# vc.add_view(cm.SCATTERPLOT, dataset=dataset, mapping="PCA"), +# vc.add_view(cm.FEATURE_VALUE_HISTOGRAM, dataset=dataset), +# ), +# ( +# vc.add_view(cm.DESCRIPTION, dataset=dataset), +# vc.add_view(cm.STATUS, dataset=dataset), +# vc.add_view(cm.HEATMAP, dataset=dataset), +# ), +# ) + +# vc.link_views( +# [view for row in views for view in row], +# list(coordination.keys()), +# list(coordination.values()), +# ) + +# # (a / b / c) | (d / e / f) | ... +# vc.layout(reduce(or_, (reduce(truediv, row) for row in views))) + +# return vc From 087bbe4f87f2a1aa588aae2bec3f83fc5241494d Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 13:50:06 +0100 Subject: [PATCH 12/25] check docs --- .readthedocs.yaml | 5 + pyproject.toml | 2 +- src/ehrdata/integrations/vitessce/_config.py | 218 +++++++++---------- 3 files changed, 115 insertions(+), 110 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1b8283c2..b167c042 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -10,5 +10,10 @@ build: - asdf global uv latest build: html: + - uvx hatch run hatch-test.py3.13:python - <<'PYCODE' + import importlib.metadata as m + for dist in sorted(m.distributions(), key=lambda d: d.metadata["Name"].lower()): + print(f"{dist.metadata['Name']}=={dist.version}") + PYCODE - uvx hatch run docs:build - mv docs/_build $READTHEDOCS_OUTPUT diff --git a/pyproject.toml b/pyproject.toml index a159c908..f6822dcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ optional-dependencies.doc = [ "sphinx-tabs", "sphinxcontrib-bibtex>=1", "sphinxext-opengraph", - # "vitessce", # vitessce[all] imports problematic dependencies + "vitessce", # vitessce[all] imports problematic dependencies ] optional-dependencies.ehrapy = [ "ehrapy", diff --git a/src/ehrdata/integrations/vitessce/_config.py b/src/ehrdata/integrations/vitessce/_config.py index 8f42534f..7826dc64 100644 --- a/src/ehrdata/integrations/vitessce/_config.py +++ b/src/ehrdata/integrations/vitessce/_config.py @@ -1,109 +1,109 @@ -# from __future__ import annotations - -# from functools import reduce -# from operator import or_, truediv -# from pathlib import Path -# from types import MappingProxyType -# from typing import TYPE_CHECKING - -# if TYPE_CHECKING: -# from collections.abc import Mapping - -# from lamindb import Artifact -# from vitessce import VitessceConfig -# from zarr.storage import Store - - -# def gen_config( -# path: Path | None = None, -# *, -# store: Path | Store | None = None, -# url: str | None = None, -# artifact: Artifact | None = None, -# # arguments not about how the store goes in: -# name: str | None = None, -# obs_sets: Mapping[str, str] = MappingProxyType({"obs/gender_concept_id": "Gender Concept ID"}), -# obs_embeddings: Mapping[str, str] = MappingProxyType({"obsm/X_pca": "PCA"}), -# ) -> VitessceConfig: -# """Generate a VitessceConfig for EHRData. - -# Args: -# path: Path to the data's Zarr store directory. -# store: The data's Zarr store or a path to it. -# url: URL pointing to the data's remote Zarr store. -# artifact: Lamin artifact representing the data. -# name: Name of the dataset. If None, derived from path. -# obs_sets: Mapping of observation set paths to names, e.g. -# {"obs/some_annotation": "My cool annotation"} -# obs_embeddings: Mapping of observation embedding paths to names, e.g. -# {"obsm/X_pca": "PCA"} - -# Returns: -# A :doc:`Vitessce ` configuration object. -# Call :meth:`~vitessce.config.VitessceConfig.widget` on it to display it. -# """ -# obs_type = "person" -# feature_type = "variable" - -# if name is None: -# if artifact is not None: -# name = artifact.description -# elif path is not None: -# name = path.stem -# else: -# msg = "`name` needs to be specified or derived from `path` or `artifact`." -# raise ValueError(msg) - -# coordination = { -# "obsType": obs_type, -# "featureType": feature_type, -# } - -# from vitessce import AnnDataWrapper, VitessceConfig -# from vitessce import Component as cm - -# wrapper = AnnDataWrapper( -# adata_path=path, -# adata_url=url, -# # vitessce is old and doesn't deal with proper Paths -# adata_store=str(store) if isinstance(store, Path) else store, -# adata_artifact=artifact, -# obs_set_paths=list(obs_sets.keys()), -# obs_set_names=list(obs_sets.values()), -# obs_embedding_paths=list(obs_embeddings.keys()), -# obs_embedding_names=list(obs_embeddings.values()), -# obs_feature_matrix_path="X", -# coordination_values=coordination, -# ) - -# vc = VitessceConfig(schema_version="1.0.15", name=name) -# dataset = vc.add_dataset(name=name).add_object(wrapper) - -# views = ( -# ( -# vc.add_view(cm.OBS_SETS, dataset=dataset), -# vc.add_view(cm.OBS_SET_SIZES, dataset=dataset), -# vc.add_view(cm.OBS_SET_FEATURE_VALUE_DISTRIBUTION, dataset=dataset), -# ), -# ( -# vc.add_view(cm.FEATURE_LIST, dataset=dataset), -# vc.add_view(cm.SCATTERPLOT, dataset=dataset, mapping="PCA"), -# vc.add_view(cm.FEATURE_VALUE_HISTOGRAM, dataset=dataset), -# ), -# ( -# vc.add_view(cm.DESCRIPTION, dataset=dataset), -# vc.add_view(cm.STATUS, dataset=dataset), -# vc.add_view(cm.HEATMAP, dataset=dataset), -# ), -# ) - -# vc.link_views( -# [view for row in views for view in row], -# list(coordination.keys()), -# list(coordination.values()), -# ) - -# # (a / b / c) | (d / e / f) | ... -# vc.layout(reduce(or_, (reduce(truediv, row) for row in views))) - -# return vc +from __future__ import annotations + +from functools import reduce +from operator import or_, truediv +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Mapping + + from lamindb import Artifact + from vitessce import VitessceConfig + from zarr.storage import Store + + +def gen_config( + path: Path | None = None, + *, + store: Path | Store | None = None, + url: str | None = None, + artifact: Artifact | None = None, + # arguments not about how the store goes in: + name: str | None = None, + obs_sets: Mapping[str, str] = MappingProxyType({"obs/gender_concept_id": "Gender Concept ID"}), + obs_embeddings: Mapping[str, str] = MappingProxyType({"obsm/X_pca": "PCA"}), +) -> VitessceConfig: + """Generate a VitessceConfig for EHRData. + + Args: + path: Path to the data's Zarr store directory. + store: The data's Zarr store or a path to it. + url: URL pointing to the data's remote Zarr store. + artifact: Lamin artifact representing the data. + name: Name of the dataset. If None, derived from path. + obs_sets: Mapping of observation set paths to names, e.g. + {"obs/some_annotation": "My cool annotation"} + obs_embeddings: Mapping of observation embedding paths to names, e.g. + {"obsm/X_pca": "PCA"} + + Returns: + A :doc:`Vitessce ` configuration object. + Call :meth:`~vitessce.config.VitessceConfig.widget` on it to display it. + """ + obs_type = "person" + feature_type = "variable" + + if name is None: + if artifact is not None: + name = artifact.description + elif path is not None: + name = path.stem + else: + msg = "`name` needs to be specified or derived from `path` or `artifact`." + raise ValueError(msg) + + coordination = { + "obsType": obs_type, + "featureType": feature_type, + } + + from vitessce import AnnDataWrapper, VitessceConfig + from vitessce import Component as cm + + wrapper = AnnDataWrapper( + adata_path=path, + adata_url=url, + # vitessce is old and doesn't deal with proper Paths + adata_store=str(store) if isinstance(store, Path) else store, + adata_artifact=artifact, + obs_set_paths=list(obs_sets.keys()), + obs_set_names=list(obs_sets.values()), + obs_embedding_paths=list(obs_embeddings.keys()), + obs_embedding_names=list(obs_embeddings.values()), + obs_feature_matrix_path="X", + coordination_values=coordination, + ) + + vc = VitessceConfig(schema_version="1.0.15", name=name) + dataset = vc.add_dataset(name=name).add_object(wrapper) + + views = ( + ( + vc.add_view(cm.OBS_SETS, dataset=dataset), + vc.add_view(cm.OBS_SET_SIZES, dataset=dataset), + vc.add_view(cm.OBS_SET_FEATURE_VALUE_DISTRIBUTION, dataset=dataset), + ), + ( + vc.add_view(cm.FEATURE_LIST, dataset=dataset), + vc.add_view(cm.SCATTERPLOT, dataset=dataset, mapping="PCA"), + vc.add_view(cm.FEATURE_VALUE_HISTOGRAM, dataset=dataset), + ), + ( + vc.add_view(cm.DESCRIPTION, dataset=dataset), + vc.add_view(cm.STATUS, dataset=dataset), + vc.add_view(cm.HEATMAP, dataset=dataset), + ), + ) + + vc.link_views( + [view for row in views for view in row], + list(coordination.keys()), + list(coordination.values()), + ) + + # (a / b / c) | (d / e / f) | ... + vc.layout(reduce(or_, (reduce(truediv, row) for row in views))) + + return vc From 2cf790c7813aa3aab00084e732b91970e32585b5 Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 13:52:50 +0100 Subject: [PATCH 13/25] check --- .readthedocs.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index b167c042..8e03ffc0 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,12 +8,12 @@ build: - asdf plugin add uv - asdf install uv latest - asdf global uv latest + - uvx hatch run hatch-test.py3.13:python - <<'PYCODE' + import importlib.metadata as m + for dist in sorted(m.distributions(), key=lambda d: d.metadata["Name"].lower()): + print(f"{dist.metadata['Name']}=={dist.version}") + PYCODE build: html: - - uvx hatch run hatch-test.py3.13:python - <<'PYCODE' - import importlib.metadata as m - for dist in sorted(m.distributions(), key=lambda d: d.metadata["Name"].lower()): - print(f"{dist.metadata['Name']}=={dist.version}") - PYCODE - uvx hatch run docs:build - mv docs/_build $READTHEDOCS_OUTPUT From a02022cb2ee5e660ac678995dc3851fa7ec0d8ed Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 13:53:30 +0100 Subject: [PATCH 14/25] check --- .readthedocs.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8e03ffc0..cc18b68b 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,11 +8,12 @@ build: - asdf plugin add uv - asdf install uv latest - asdf global uv latest - - uvx hatch run hatch-test.py3.13:python - <<'PYCODE' - import importlib.metadata as m - for dist in sorted(m.distributions(), key=lambda d: d.metadata["Name"].lower()): - print(f"{dist.metadata['Name']}=={dist.version}") - PYCODE + - run: | + uvx hatch run hatch-test.py3.13:python - <<'PYCODE' + import importlib.metadata as m + for dist in sorted(m.distributions(), key=lambda d: d.metadata["Name"].lower()): + print(f"{dist.metadata['Name']}=={dist.version}") + PYCODE build: html: - uvx hatch run docs:build From b7d2e6cba031e0028dc73827f51e8f108f458c5c Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 13:54:28 +0100 Subject: [PATCH 15/25] check --- .readthedocs.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index cc18b68b..cd8b6ffe 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,12 +8,7 @@ build: - asdf plugin add uv - asdf install uv latest - asdf global uv latest - - run: | - uvx hatch run hatch-test.py3.13:python - <<'PYCODE' - import importlib.metadata as m - for dist in sorted(m.distributions(), key=lambda d: d.metadata["Name"].lower()): - print(f"{dist.metadata['Name']}=={dist.version}") - PYCODE + - uvx hatch run hatch-test.py3.13:python -c "import importlib.metadata as m; [print(f'{d.metadata[\"Name\"]}=={d.version}') for d in sorted(m.distributions(), key=lambda d: d.metadata['Name'].lower())]" build: html: - uvx hatch run docs:build From c96f3f52eea00b6b2c2332ee18f0dddf722eeafb Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 14:13:16 +0100 Subject: [PATCH 16/25] revert --- .readthedocs.yaml | 1 - docs/conf.py | 2 +- src/ehrdata/core/ehrdata.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index cd8b6ffe..1b8283c2 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,7 +8,6 @@ build: - asdf plugin add uv - asdf install uv latest - asdf global uv latest - - uvx hatch run hatch-test.py3.13:python -c "import importlib.metadata as m; [print(f'{d.metadata[\"Name\"]}=={d.version}') for d in sorted(m.distributions(), key=lambda d: d.metadata['Name'].lower())]" build: html: - uvx hatch run docs:build diff --git a/docs/conf.py b/docs/conf.py index b8cd6cc1..69ea4b1f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -105,7 +105,7 @@ html_static_path = ["_static"] html_css_files = ["css/custom.css"] html_title = project_name -html_logo = "_static/tutorial_images/logo.png" +html_logo = "_static/tutorial_images/ehrdata_logo.png" html_theme_options = { "repository_url": repository_url, "use_repository_button": True, diff --git a/src/ehrdata/core/ehrdata.py b/src/ehrdata/core/ehrdata.py index 5be3e80f..ccc0fc3b 100644 --- a/src/ehrdata/core/ehrdata.py +++ b/src/ehrdata/core/ehrdata.py @@ -171,7 +171,7 @@ def _get_layers_3d_dim(layers: Mapping[str, Any] | None) -> int: class EHRData(AnnData): """Model two and three dimensional electronic health record data. - .. figure:: ../../_static/tutorial_images/logo.png + .. figure:: ../../_static/tutorial_images/ehrdata_logo.png :width: 260px :align: right :class: dark-light From 34ab1cf3a1ca92e9c02c8bdde5fb5f0decd67b73 Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 14:22:57 +0100 Subject: [PATCH 17/25] check only lamin hidden --- src/ehrdata/integrations/vitessce/_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ehrdata/integrations/vitessce/_config.py b/src/ehrdata/integrations/vitessce/_config.py index 7826dc64..ed8713d7 100644 --- a/src/ehrdata/integrations/vitessce/_config.py +++ b/src/ehrdata/integrations/vitessce/_config.py @@ -4,12 +4,12 @@ from operator import or_, truediv from pathlib import Path from types import MappingProxyType -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from collections.abc import Mapping - from lamindb import Artifact + # from lamindb import Artifact from vitessce import VitessceConfig from zarr.storage import Store @@ -19,7 +19,7 @@ def gen_config( *, store: Path | Store | None = None, url: str | None = None, - artifact: Artifact | None = None, + artifact: Any | None = None, # arguments not about how the store goes in: name: str | None = None, obs_sets: Mapping[str, str] = MappingProxyType({"obs/gender_concept_id": "Gender Concept ID"}), From e52787a1d77de1c311af949b0fc8c21530e700ad Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 14:28:08 +0100 Subject: [PATCH 18/25] no vitessce in docstring --- pyproject.toml | 3 +-- src/ehrdata/integrations/vitessce/_config.py | 11 +++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f6822dcf..e9269bf3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ optional-dependencies.dev = [ ] optional-dependencies.doc = [ "docutils>=0.8,!=0.18.*,!=0.19.*", - "ehrdata[lamin,torch]", + "ehrdata[lamin,torch]", # if vitessce is included, rtd build fails with ImportError: cannot import name '_PosixFlavour' from 'pathlib' (/home/docs/.asdf/installs/python/3.13.3/lib/python3.13/pathlib/__init__.py) "ipykernel", "ipython", "myst-nb>=1.1", @@ -54,7 +54,6 @@ optional-dependencies.doc = [ "sphinx-tabs", "sphinxcontrib-bibtex>=1", "sphinxext-opengraph", - "vitessce", # vitessce[all] imports problematic dependencies ] optional-dependencies.ehrapy = [ "ehrapy", diff --git a/src/ehrdata/integrations/vitessce/_config.py b/src/ehrdata/integrations/vitessce/_config.py index ed8713d7..b23fb853 100644 --- a/src/ehrdata/integrations/vitessce/_config.py +++ b/src/ehrdata/integrations/vitessce/_config.py @@ -9,8 +9,7 @@ if TYPE_CHECKING: from collections.abc import Mapping - # from lamindb import Artifact - from vitessce import VitessceConfig + from lamindb import Artifact from zarr.storage import Store @@ -19,12 +18,12 @@ def gen_config( *, store: Path | Store | None = None, url: str | None = None, - artifact: Any | None = None, + artifact: Artifact | None = None, # arguments not about how the store goes in: name: str | None = None, obs_sets: Mapping[str, str] = MappingProxyType({"obs/gender_concept_id": "Gender Concept ID"}), obs_embeddings: Mapping[str, str] = MappingProxyType({"obsm/X_pca": "PCA"}), -) -> VitessceConfig: +) -> Any: """Generate a VitessceConfig for EHRData. Args: @@ -39,8 +38,8 @@ def gen_config( {"obsm/X_pca": "PCA"} Returns: - A :doc:`Vitessce ` configuration object. - Call :meth:`~vitessce.config.VitessceConfig.widget` on it to display it. + A `Vitessce` configuration object. + Call `vitessce.config.VitessceConfig.widget` on it to display it. """ obs_type = "person" feature_type = "variable" From 796254c2d180f6baf3c884110f8550e50571c7b2 Mon Sep 17 00:00:00 2001 From: eroell Date: Sun, 2 Nov 2025 15:05:38 +0100 Subject: [PATCH 19/25] updates --- CHANGELOG.md | 6 +++--- pyproject.toml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0b6044c..c17307c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning][]. ### Maintenance - Enhanced {doc}`tutorials/getting_started` ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell -- Move from zarr<3 to zarr>3 ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell +- Move from zarr<3 to zarr>=3 ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell ### Fixed @@ -22,8 +22,8 @@ and this project adheres to [Semantic Versioning][]. - `EHRData` drops the `.R` field in favor of using `.layers` for any 3D data arrays ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell - `EHRData`'s shape property will always return a 3 dimensional shape. If an `EHRData` object has flat arrays only, the third dimension will be 1. ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell - The following functions now take a `layer` argument: {func}`~ehrdata.io.read_csv`, {func}`~ehrdata.io.from_pandas`, {func}`~ehrdata.io.to_pandas`, {func}`~ehrdata.io.omop.setup_variables`, {func}`~ehrdata.io.omop.setup_interval_variables`, {func}`~ehrdata.dt.ehrdata_blobs`, {func}`~ehrdata.dt.physionet2012`. If it is let to its default, `None`, the `.X` field of `EHRData` is used. Since `.X` is 2D in this release, in cases with 3D data, the `layer` argument needs to be used. ([#184](https://github.com/theislab/ehrdata/pull/184)) @eroell -- `~ehrdata.io.write_zarr` now writes an `EHRData` specific store, with `AnnData` in a substore. This change allows to use `AnnData`s change to consolidated Zarr metadata. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell -- `~ehrdata.io.read_zarr` is adapted to read the new store format, and can also deal with `AnnData` stores. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell +- {func}`~ehrdata.io.write_zarr` now writes an `EHRData` specific store encoding, with `anndata` as a substore. This change allows to use `AnnData` with its change to consolidated Zarr metadata, and better isolates `AnnData`'s io. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell +- {func}`~ehrdata.io.read_zarr` is adapted to read the new store encoding, and can also deal with `AnnData` stores. ([#185](https://github.com/theislab/ehrdata/pull/185)) @eroell ## [0.0.9] diff --git a/pyproject.toml b/pyproject.toml index e9269bf3..16282690 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ optional-dependencies.dev = [ ] optional-dependencies.doc = [ "docutils>=0.8,!=0.18.*,!=0.19.*", - "ehrdata[lamin,torch]", # if vitessce is included, rtd build fails with ImportError: cannot import name '_PosixFlavour' from 'pathlib' (/home/docs/.asdf/installs/python/3.13.3/lib/python3.13/pathlib/__init__.py) + "ehrdata[lamin,torch]", # do not include vitessce[all] in doc: https://github.com/keller-mark/esbuild-py/issues/19 "ipykernel", "ipython", "myst-nb>=1.1", @@ -54,6 +54,7 @@ optional-dependencies.doc = [ "sphinx-tabs", "sphinxcontrib-bibtex>=1", "sphinxext-opengraph", + # "vitessce", # await vitessce accepting zarr>=3 to use this and intersphinx links. if vitessce is included, rtd build fails with ImportError: cannot import name '_PosixFlavour' from 'pathlib' (/home/docs/.asdf/installs/python/3.13.3/lib/python3.13/pathlib/__init__.py) ] optional-dependencies.ehrapy = [ "ehrapy", From 549a11b1147306f5fca4de2230049ebe0eaaca55 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Mon, 3 Nov 2025 11:17:18 +0100 Subject: [PATCH 20/25] Fix docs --- .gitignore | 12 ++++++------ .readthedocs.yaml | 2 +- docs/conf.py | 20 +++++++++++++------- pyproject.toml | 4 ++-- src/ehrdata/core/ehrdata.py | 2 +- src/ehrdata/integrations/vitessce/_config.py | 13 +++++++------ 6 files changed, 30 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 63faed52..99b24a24 100644 --- a/.gitignore +++ b/.gitignore @@ -21,12 +21,12 @@ __pycache__/ /docs/generated/ /docs/_build/ /docs/generated/ -docs/api/ehrdata -docs/api/data -docs/api/tools -docs/api/io -docs/api/plot -!docs/api/api.md +/docs/api/ehrdata.*.rst +/docs/api/data +/docs/api/tools +/docs/api/io +/docs/api/plot +!/docs/api/api.md ehrapy_data test.ipynb diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 1b8283c2..f12e003d 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -10,5 +10,5 @@ build: - asdf global uv latest build: html: - - uvx hatch run docs:build + - uvx hatch run docs:build -W # do not remove -W, fix warnings. - mv docs/_build $READTHEDOCS_OUTPUT diff --git a/docs/conf.py b/docs/conf.py index 69ea4b1f..615bd1e5 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -81,16 +81,18 @@ } intersphinx_mapping = { - "python": ("https://docs.python.org/3", None), "anndata": ("https://anndata.readthedocs.io/en/stable", None), + "fsspec": ("https://filesystem-spec.readthedocs.io/en/stable", None), "h5py": ("https://docs.h5py.org/en/latest", None), + "lamin": ("https://docs.lamin.ai", None), "numpy": ("https://numpy.org/doc/stable", None), - "torch": ("https://pytorch.org/docs/main/", None), - "scipy": ("https://docs.scipy.org/doc/scipy", None), "pandas": ("https://pandas.pydata.org/docs", None), - "zarr": ("https://zarr.readthedocs.io/en/stable", None), + "python": ("https://docs.python.org/3", None), + "scanpy": ("https://scanpy.readthedocs.io/en/stable", None), + "scipy": ("https://docs.scipy.org/doc/scipy", None), + "torch": ("https://docs.pytorch.org/docs/main", None), "vitessce": ("https://python-docs.vitessce.io", None), - "lamin": ("https://docs.lamin.ai", None), + "zarr": ("https://zarr.readthedocs.io/en/stable", None), } # List of patterns, relative to source directory, that match files and @@ -127,6 +129,10 @@ nitpick_ignore = [ ("py:class", "pathlib._local.Path"), ("py:class", "types.EllipsisType"), + # TODO: remove once https://github.com/sphinx-doc/sphinx/pull/13508 is released + ("py:class", "ehrdata._types.TypeAliasType"), + # typing.Union fails in tutorials/tutorial_time_series_with_pypots + ("py:data", "typing.Union"), # https://github.com/duckdb/duckdb-web/issues/3806 ("py:class", "duckdb.duckdb.DuckDBPyConnection"), ("py:class", "_duckdb.DuckDBPyConnection"), @@ -136,18 +142,18 @@ ("py:class", "awkward.highlevel.Array"), ("py:class", "h5py._hl.dataset.Dataset"), ("py:class", "zarr.core.Array"), + ("py:class", "zarr.core.buffer.core.Buffer"), ("py:class", "ehrdata._compat.ZappyArray"), ("py:class", "dask.array.core.Array"), ("py:class", "anndata.compat.CupyArray"), ("py:class", "anndata.compat.CupySparseMatrix"), ("py:class", "sparse.numba_backend._coo.core.COO"), ("py:class", "sparse._coo.core.COO"), - ("py:data", "typing.Union"), # typing.Union fails in tutorials/tutorial_time_series_with_pypots ] # Redirect broken parameter annotation classes qualname_overrides = { - "zarr._storage.store.Store": "zarr.storage.MemoryStore", + "zarr.storage._common.StorePath": "zarr.storage.StorePath", "zarr.core.group.Group": "zarr.group.Group", "lnschema_core.models.Artifact": "lamindb.Artifact", } diff --git a/pyproject.toml b/pyproject.toml index 16282690..2010c1a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,13 +54,13 @@ optional-dependencies.doc = [ "sphinx-tabs", "sphinxcontrib-bibtex>=1", "sphinxext-opengraph", - # "vitessce", # await vitessce accepting zarr>=3 to use this and intersphinx links. if vitessce is included, rtd build fails with ImportError: cannot import name '_PosixFlavour' from 'pathlib' (/home/docs/.asdf/installs/python/3.13.3/lib/python3.13/pathlib/__init__.py) + "vitessce", ] optional-dependencies.ehrapy = [ "ehrapy", ] optional-dependencies.lamin = [ - "lamindb", + "lamindb>=1.1", #"omop", # Not needed and causing an issue in the readthedocs build https://github.com/laminlabs/lamindb/issues/3081 ] optional-dependencies.test = [ diff --git a/src/ehrdata/core/ehrdata.py b/src/ehrdata/core/ehrdata.py index ccc0fc3b..c7478d4b 100644 --- a/src/ehrdata/core/ehrdata.py +++ b/src/ehrdata/core/ehrdata.py @@ -171,7 +171,7 @@ def _get_layers_3d_dim(layers: Mapping[str, Any] | None) -> int: class EHRData(AnnData): """Model two and three dimensional electronic health record data. - .. figure:: ../../_static/tutorial_images/ehrdata_logo.png + .. figure:: /_static/tutorial_images/ehrdata_logo.png :width: 260px :align: right :class: dark-light diff --git a/src/ehrdata/integrations/vitessce/_config.py b/src/ehrdata/integrations/vitessce/_config.py index b23fb853..f90b3c07 100644 --- a/src/ehrdata/integrations/vitessce/_config.py +++ b/src/ehrdata/integrations/vitessce/_config.py @@ -4,26 +4,27 @@ from operator import or_, truediv from pathlib import Path from types import MappingProxyType -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING if TYPE_CHECKING: from collections.abc import Mapping from lamindb import Artifact - from zarr.storage import Store + from vitessce import VitessceConfig + from zarr.storage import StoreLike def gen_config( path: Path | None = None, *, - store: Path | Store | None = None, + store: Path | StoreLike | None = None, url: str | None = None, artifact: Artifact | None = None, # arguments not about how the store goes in: name: str | None = None, obs_sets: Mapping[str, str] = MappingProxyType({"obs/gender_concept_id": "Gender Concept ID"}), obs_embeddings: Mapping[str, str] = MappingProxyType({"obsm/X_pca": "PCA"}), -) -> Any: +) -> VitessceConfig: """Generate a VitessceConfig for EHRData. Args: @@ -38,8 +39,8 @@ def gen_config( {"obsm/X_pca": "PCA"} Returns: - A `Vitessce` configuration object. - Call `vitessce.config.VitessceConfig.widget` on it to display it. + A :doc:`Vitessce ` configuration object. + Call :meth:`~vitessce.config.VitessceConfig.widget` on it to display it. """ obs_type = "person" feature_type = "variable" From 6281b264838e810c0d71fdfa93590536459ae975 Mon Sep 17 00:00:00 2001 From: Phil Schaf Date: Mon, 3 Nov 2025 11:20:45 +0100 Subject: [PATCH 21/25] fmt --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2010c1a4..2ec40c07 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,9 +86,7 @@ installer = "uv" features = [ "dev" ] [tool.hatch.envs.docs] - features = [ "doc" ] - scripts.build = "sphinx-build -M html docs docs/_build {args}" scripts.open = "python -m webbrowser -t docs/_build/html/index.html" scripts.clean = "git clean -fdX -- {args:docs}" From 86fd966ce0275e0395d9f3b0a3db79fe3de32f53 Mon Sep 17 00:00:00 2001 From: eroell Date: Mon, 3 Nov 2025 14:03:40 +0100 Subject: [PATCH 22/25] allow write nullabe strings decorator --- .github/workflows/test.yaml | 6 +----- src/ehrdata/__init__.py | 5 ----- src/ehrdata/io/zarr.py | 12 ++++++++++++ 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f49b906e..4f6e4b70 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -76,11 +76,7 @@ jobs: - name: Show installed packages if: matrix.env.python == '3.13' run: | - uvx hatch run hatch-test.py3.13:python - <<'PYCODE' - import importlib.metadata as m - for dist in sorted(m.distributions(), key=lambda d: d.metadata["Name"].lower()): - print(f"{dist.metadata['Name']}=={dist.version}") - PYCODE + uvx hatch run hatch-test.py3.13:uv pip freeze - name: run tests using hatch env: MPLBACKEND: agg diff --git a/src/ehrdata/__init__.py b/src/ehrdata/__init__.py index 5d5b46c8..32a6b882 100644 --- a/src/ehrdata/__init__.py +++ b/src/ehrdata/__init__.py @@ -16,8 +16,3 @@ ] __version__ = version("ehrdata") - -import anndata as ad - -# Opt to use this newer feature of anndata https://github.com/scverse/anndata/blob/6a6bde151eeb231eebac20b66e6002b88052e8db/src/anndata/_io/specs/methods.py#L1151 -ad.settings.allow_write_nullable_strings = True diff --git a/src/ehrdata/io/zarr.py b/src/ehrdata/io/zarr.py index 5406cfc0..3429e81a 100644 --- a/src/ehrdata/io/zarr.py +++ b/src/ehrdata/io/zarr.py @@ -1,6 +1,7 @@ from __future__ import annotations import warnings +from functools import wraps from pathlib import Path from typing import TYPE_CHECKING @@ -13,6 +14,7 @@ from ehrdata.io._array_casting import _cast_arrays_dtype_to_float_or_str_if_nonnumeric_object, _cast_variables_to_float if TYPE_CHECKING: + from collections.abc import Callable from os import PathLike from ehrdata import EHRData @@ -88,6 +90,16 @@ def read_zarr( return edata +def _allow_write_nullable_strings[T, **P](f: Callable[P, T]) -> Callable[P, T]: + @wraps(f) + def wrapped(*args: P.args, **kwargs: P.kwargs): + with ad.settings.override(allow_write_nullable_strings=True): + return f(*args, **kwargs) + + return wrapped + + +@_allow_write_nullable_strings def write_zarr( edata: EHRData, filename: str | Path, From 606ad91beb48feeebca68befa1b2c8abe3f26143 Mon Sep 17 00:00:00 2001 From: eroell Date: Mon, 3 Nov 2025 16:10:13 +0100 Subject: [PATCH 23/25] write zarr with auto sharding and chunking --- src/ehrdata/io/zarr.py | 42 ++++++++++++++++++++--------- tests/conftest.py | 8 +++--- tests/io/test_zarr.py | 61 ++++++++++++++++++++++++++++++------------ 3 files changed, 79 insertions(+), 32 deletions(-) diff --git a/src/ehrdata/io/zarr.py b/src/ehrdata/io/zarr.py index 3429e81a..580d5de0 100644 --- a/src/ehrdata/io/zarr.py +++ b/src/ehrdata/io/zarr.py @@ -3,7 +3,7 @@ import warnings from functools import wraps from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import anndata as ad import zarr @@ -14,7 +14,7 @@ from ehrdata.io._array_casting import _cast_arrays_dtype_to_float_or_str_if_nonnumeric_object, _cast_variables_to_float if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Mapping from os import PathLike from ehrdata import EHRData @@ -104,7 +104,6 @@ def write_zarr( edata: EHRData, filename: str | Path, *, - # chunks: bool | int | tuple[int, ...] | None = (1000, 1000), blocked by https://github.com/scverse/anndata/issues/2193 convert_strings_to_categoricals: bool = True, ) -> None: """Write :class:`~ehrdata.EHRData` objects to disk. @@ -125,9 +124,7 @@ def write_zarr( filename = Path(filename) edata = _cast_arrays_dtype_to_float_or_str_if_nonnumeric_object(edata) - store = zarr.open(filename, mode="w") - store.attrs["encoding-version"] = EHRDATA_ZARR_ENCODING_VERSION - store.attrs["encoding-type"] = "ehrdata" + store = zarr.open_group(filename, mode="a", use_consolidated=False, zarr_version=3) adata = ad.AnnData(edata) @@ -136,11 +133,32 @@ def write_zarr( adata.strings_to_categoricals(adata.var) adata.strings_to_categoricals(edata.tem) - ad.io.write_elem( - store, - "anndata", - adata, # this will store everything but the .tem field - # dataset_kwargs={"chunks": chunks}, # blocked by https://github.com/scverse/anndata/issues/2193 - ) + anndata_group = store.create_group("anndata") + + def write_sharded(group: zarr.Group, adata: ad.AnnData): + def callback( + func: ad.experimental.Write, + g: zarr.Group, + k: str, + elem: ad.typing.RWAble, + dataset_kwargs: Mapping[str, Any], + iospec: ad.experimental.IOSpec, + ): + if iospec.encoding_type in {"array"} and not isinstance(elem, list): + dataset_kwargs = { + "shards": tuple(int(2 ** (16 / len(elem.shape))) for _ in elem.shape), + **dataset_kwargs, + } + dataset_kwargs["chunks"] = tuple(i // 2 for i in dataset_kwargs["shards"]) + elif iospec.encoding_type in {"csr_matrix", "csc_matrix"}: + dataset_kwargs = {"shards": (2**16,), "chunks": (2**8,), **dataset_kwargs} + func(g, k, elem, dataset_kwargs=dataset_kwargs) + + return ad.experimental.write_dispatched(group, "/", adata, callback=callback) + + write_sharded(anndata_group, adata) ad.io.write_elem(store, "tem", edata.tem) + + store.attrs["encoding-version"] = EHRDATA_ZARR_ENCODING_VERSION + store.attrs["encoding-type"] = "ehrdata" diff --git a/tests/conftest.py b/tests/conftest.py index 5771ba8f..9b247dfb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -252,15 +252,17 @@ def edata_nonnumeric_missing_330(obs_31, var_31): def edata_basic_with_tem_full(): edata_basic_with_tem_dict = { "X": np.ones((5, 4)), - "obs": pd.DataFrame({"survival": [1, 2, 3, 4, 5]}), - "var": pd.DataFrame({"variables": ["var_1", "var_2", "var_3", "var_4"]}), + "obs": pd.DataFrame({"survival": [1, 2, 3, 4, 5]}, index=["obs1", "obs2", "obs3", "obs4", "obs5"]), + "var": pd.DataFrame( + {"variables": ["var_1", "var_2", "var_3", "var_4"]}, index=["var1", "var2", "var3", "var4"] + ), "obsm": {"obs_level_representation": np.ones((5, 2))}, "varm": {"var_level_representation": np.ones((4, 2))}, "layers": {DEFAULT_TEM_LAYER_NAME: np.ones((5, 4)), "other_layer": np.ones((5, 4))}, "obsp": {"obs_level_connectivities": np.ones((5, 5))}, "varp": {"var_level_connectivities": np.random.randn(4, 4)}, "uns": {"information": ["info1"]}, - "tem": pd.DataFrame({"timestep": ["t1"]}), + "tem": pd.DataFrame({"timestep": ["t1"]}, index=["t1"]), } return EHRData(**edata_basic_with_tem_dict) diff --git a/tests/io/test_zarr.py b/tests/io/test_zarr.py index b01d1f27..0298b22a 100644 --- a/tests/io/test_zarr.py +++ b/tests/io/test_zarr.py @@ -1,3 +1,4 @@ +import anndata as ad import numpy as np import pandas as pd import pytest @@ -10,6 +11,7 @@ _assert_shape_matches, ) +from ehrdata import EHRData from ehrdata.core.constants import EHRDATA_ZARR_ENCODING_VERSION from ehrdata.io import read_zarr, write_zarr @@ -77,6 +79,47 @@ def test_read_zarr_sparse_with_tem(harmonize_missing_values, cast_variables_to_f assert store.attrs["encoding-version"] == EHRDATA_ZARR_ENCODING_VERSION +def _check_anndata_part_equal(edata: EHRData, edata_read: EHRData | ad.AnnData): + pd.testing.assert_frame_equal(edata.obs.iloc[:, :1], edata_read.obs.iloc[:, :1]) + pd.testing.assert_frame_equal(edata.var.iloc[:, :1], edata_read.var.iloc[:, :1]) + + for key in edata.obsm: + assert key in edata_read.obsm + assert np.array_equal(edata.obsm[key], edata_read.obsm[key]) + for key in edata.varm: + assert key in edata_read.varm + assert np.array_equal(edata.varm[key], edata_read.varm[key]) + for key in edata.obsp: + assert key in edata_read.obsp + assert np.array_equal(edata.obsp[key], edata_read.obsp[key]) + for key in edata.varp: + assert key in edata_read.varp + assert np.array_equal(edata.varp[key], edata_read.varp[key]) + for key in edata.uns: + assert key in edata_read.uns + assert np.array_equal(edata.uns[key], edata_read.uns[key]) + + +@pytest.mark.parametrize( + "edata_name", + [ + "edata_330", + "edata_333", + "edata_333_larger_obs_var_tem", + "edata_basic_with_tem_full", + "edata_nonnumeric_missing_330", + ], +) +def test_write_read_zarr_anndata_part(edata_name, request, tmp_path): + # this test uses ad.io.read_zarr as a sanity check to ensure the anndata subgroup is written properly + edata = request.getfixturevalue(edata_name) + store_path = tmp_path / f"{edata_name}.zarr" + + write_zarr(edata.copy(), store_path) + adata = ad.io.read_zarr(store_path / "anndata") + _check_anndata_part_equal(edata, adata) + + @pytest.mark.parametrize( "edata_name", [ @@ -100,24 +143,8 @@ def test_write_read_zarr_basic(edata_name, request, tmp_path): for key in edata.layers: _assert_dtype_object_array_with_missing_values_equal(edata.layers[key], edata_read.layers[key]) - pd.testing.assert_frame_equal(edata.obs.iloc[:, :1], edata_read.obs.iloc[:, :1]) - pd.testing.assert_frame_equal(edata.var.iloc[:, :1], edata_read.var.iloc[:, :1]) + _check_anndata_part_equal(edata, edata_read) pd.testing.assert_frame_equal(edata.tem.iloc[:, :1], edata_read.tem.iloc[:, :1]) - for key in edata.obsm: - assert key in edata_read.obsm - assert np.array_equal(edata.obsm[key], edata_read.obsm[key]) - for key in edata.varm: - assert key in edata_read.varm - assert np.array_equal(edata.varm[key], edata_read.varm[key]) - for key in edata.obsp: - assert key in edata_read.obsp - assert np.array_equal(edata.obsp[key], edata_read.obsp[key]) - for key in edata.varp: - assert key in edata_read.varp - assert np.array_equal(edata.varp[key], edata_read.varp[key]) - for key in edata.uns: - assert key in edata_read.uns - assert np.array_equal(edata.uns[key], edata_read.uns[key]) # check the test file is an ehrdata zarr store store = zarr.open(store_path) From f1af318ea6429a8567ca167bb7da1d93f55fe7f3 Mon Sep 17 00:00:00 2001 From: eroell Date: Tue, 4 Nov 2025 10:56:42 +0100 Subject: [PATCH 24/25] write_zarr with arg chunks='auto'|'ehrdata_auto' --- src/ehrdata/io/zarr.py | 21 ++++++++--- tests/conftest.py | 21 +++++++++++ tests/io/test_zarr.py | 85 +++++++++++++++++++----------------------- 3 files changed, 75 insertions(+), 52 deletions(-) diff --git a/src/ehrdata/io/zarr.py b/src/ehrdata/io/zarr.py index 580d5de0..9c292eb3 100644 --- a/src/ehrdata/io/zarr.py +++ b/src/ehrdata/io/zarr.py @@ -3,7 +3,7 @@ import warnings from functools import wraps from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal import anndata as ad import zarr @@ -104,6 +104,7 @@ def write_zarr( edata: EHRData, filename: str | Path, *, + chunks: Literal["auto" | "ehrdata_auto"] = "auto", convert_strings_to_categoricals: bool = True, ) -> None: """Write :class:`~ehrdata.EHRData` objects to disk. @@ -114,6 +115,7 @@ def write_zarr( Args: edata: Central data object. filename: Name of the output file, can also be prefixed with relative or absolute path to save the file to. + chunks: Specify strategy of how data should be chunked. For simplicity, currently only 2 options are available: "auto" will write the data with :func:`~anndata.io.write_elem`'s default settings. `chunks='ehrdata_auto'` will write the data chunked (and sharded) based on a heuristic that loosely speaking writes slightly smaller chunks. convert_strings_to_categoricals: Convert columns of `str` dtype in `.obs` and `.var` and `.tem` to `categorical` dtype. Examples: @@ -124,7 +126,7 @@ def write_zarr( filename = Path(filename) edata = _cast_arrays_dtype_to_float_or_str_if_nonnumeric_object(edata) - store = zarr.open_group(filename, mode="a", use_consolidated=False, zarr_version=3) + store = zarr.open_group(filename, mode="a", use_consolidated=False, zarr_format=3) adata = ad.AnnData(edata) @@ -133,8 +135,8 @@ def write_zarr( adata.strings_to_categoricals(adata.var) adata.strings_to_categoricals(edata.tem) - anndata_group = store.create_group("anndata") - + # write_sharded this is a slightly modified version from https://anndata.readthedocs.io/en/stable/tutorials/zarr-v3.html + # write_sharded is intended as a future blueprint of implementing better chunking defaults for ehrdata based based on real usecases def write_sharded(group: zarr.Group, adata: ad.AnnData): def callback( func: ad.experimental.Write, @@ -156,7 +158,16 @@ def callback( return ad.experimental.write_dispatched(group, "/", adata, callback=callback) - write_sharded(anndata_group, adata) + if chunks == "auto": + ad.io.write_elem(store, "anndata", adata) + elif chunks == "ehrdata_auto": + anndata_group = store.create_group("anndata") + write_sharded(anndata_group, adata) + else: + err = ( + f"chunks={chunks} is not implemented. Currently, only chunks='auto' and chunks='ehrdata_auto' is supported." + ) + raise NotImplementedError(err) ad.io.write_elem(store, "tem", edata.tem) diff --git a/tests/conftest.py b/tests/conftest.py index 9b247dfb..0e5c8c51 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -317,3 +317,24 @@ def _assert_io_read(edata: EHRData): TEST_DATA_PATH = Path(__file__).parent / "data" + + +def _check_aligned_anndata_parts_equal(edata: EHRData, edata_read: EHRData | ad.AnnData): + pd.testing.assert_frame_equal(edata.obs.iloc[:, :1], edata_read.obs.iloc[:, :1]) + pd.testing.assert_frame_equal(edata.var.iloc[:, :1], edata_read.var.iloc[:, :1]) + + for key in edata.obsm: + assert key in edata_read.obsm + assert np.array_equal(edata.obsm[key], edata_read.obsm[key]) + for key in edata.varm: + assert key in edata_read.varm + assert np.array_equal(edata.varm[key], edata_read.varm[key]) + for key in edata.obsp: + assert key in edata_read.obsp + assert np.array_equal(edata.obsp[key], edata_read.obsp[key]) + for key in edata.varp: + assert key in edata_read.varp + assert np.array_equal(edata.varp[key], edata_read.varp[key]) + for key in edata.uns: + assert key in edata_read.uns + assert np.array_equal(edata.uns[key], edata_read.uns[key]) diff --git a/tests/io/test_zarr.py b/tests/io/test_zarr.py index 0298b22a..0e77b6ff 100644 --- a/tests/io/test_zarr.py +++ b/tests/io/test_zarr.py @@ -1,5 +1,4 @@ import anndata as ad -import numpy as np import pandas as pd import pytest import zarr @@ -9,9 +8,9 @@ _assert_dtype_object_array_with_missing_values_equal, _assert_io_read, _assert_shape_matches, + _check_aligned_anndata_parts_equal, ) -from ehrdata import EHRData from ehrdata.core.constants import EHRDATA_ZARR_ENCODING_VERSION from ehrdata.io import read_zarr, write_zarr @@ -20,7 +19,8 @@ @pytest.mark.parametrize("harmonize_missing_values", [False, True]) @pytest.mark.parametrize("cast_variables_to_float", [False, True]) -def test_read_anndata_zarr_basic(harmonize_missing_values, cast_variables_to_float): +def test_read_zarr_anndata_store_as_ehrdata(harmonize_missing_values, cast_variables_to_float): + # ehrdata should be able to read data from a regular anndata zarr store edata = read_zarr( filename=TEST_PATH_ZARR / "adata_basic.zarr", harmonize_missing_values=harmonize_missing_values, @@ -36,6 +36,28 @@ def test_read_anndata_zarr_basic(harmonize_missing_values, cast_variables_to_flo _assert_io_read(edata) +@pytest.mark.parametrize( + "edata_name", + [ + "edata_330", + "edata_333", + "edata_333_larger_obs_var_tem", + "edata_basic_with_tem_full", + "edata_nonnumeric_missing_330", + ], +) +@pytest.mark.parametrize("chunks", ["auto", "ehrdata_auto"]) +def test_write_zarr_anndata_subgroup_of_ehrdata_store(edata_name, chunks, request, tmp_path): + # ehrdata's write should create a regular anndata group + # this test uses ad.io.read_zarr as a sanity check to ensure the anndata subgroup is written properly + edata = request.getfixturevalue(edata_name) + store_path = tmp_path / f"{edata_name}.zarr" + + write_zarr(edata.copy(), store_path, chunks=chunks) + adata = ad.io.read_zarr(store_path / "anndata") + _check_aligned_anndata_parts_equal(edata, adata) + + @pytest.mark.parametrize("harmonize_missing_values", [False, True]) @pytest.mark.parametrize("cast_variables_to_float", [False, True]) def test_read_zarr_basic_with_tem(harmonize_missing_values, cast_variables_to_float): @@ -79,27 +101,6 @@ def test_read_zarr_sparse_with_tem(harmonize_missing_values, cast_variables_to_f assert store.attrs["encoding-version"] == EHRDATA_ZARR_ENCODING_VERSION -def _check_anndata_part_equal(edata: EHRData, edata_read: EHRData | ad.AnnData): - pd.testing.assert_frame_equal(edata.obs.iloc[:, :1], edata_read.obs.iloc[:, :1]) - pd.testing.assert_frame_equal(edata.var.iloc[:, :1], edata_read.var.iloc[:, :1]) - - for key in edata.obsm: - assert key in edata_read.obsm - assert np.array_equal(edata.obsm[key], edata_read.obsm[key]) - for key in edata.varm: - assert key in edata_read.varm - assert np.array_equal(edata.varm[key], edata_read.varm[key]) - for key in edata.obsp: - assert key in edata_read.obsp - assert np.array_equal(edata.obsp[key], edata_read.obsp[key]) - for key in edata.varp: - assert key in edata_read.varp - assert np.array_equal(edata.varp[key], edata_read.varp[key]) - for key in edata.uns: - assert key in edata_read.uns - assert np.array_equal(edata.uns[key], edata_read.uns[key]) - - @pytest.mark.parametrize( "edata_name", [ @@ -110,31 +111,12 @@ def _check_anndata_part_equal(edata: EHRData, edata_read: EHRData | ad.AnnData): "edata_nonnumeric_missing_330", ], ) -def test_write_read_zarr_anndata_part(edata_name, request, tmp_path): - # this test uses ad.io.read_zarr as a sanity check to ensure the anndata subgroup is written properly +@pytest.mark.parametrize("chunks", ["auto", "ehrdata_auto"]) +def test_write_read_zarr_basic(edata_name, chunks, request, tmp_path): edata = request.getfixturevalue(edata_name) store_path = tmp_path / f"{edata_name}.zarr" - write_zarr(edata.copy(), store_path) - adata = ad.io.read_zarr(store_path / "anndata") - _check_anndata_part_equal(edata, adata) - - -@pytest.mark.parametrize( - "edata_name", - [ - "edata_330", - "edata_333", - "edata_333_larger_obs_var_tem", - "edata_basic_with_tem_full", - "edata_nonnumeric_missing_330", - ], -) -def test_write_read_zarr_basic(edata_name, request, tmp_path): - edata = request.getfixturevalue(edata_name) - store_path = tmp_path / f"{edata_name}.zarr" - - write_zarr(edata.copy(), store_path) + write_zarr(edata.copy(), store_path, chunks=chunks) edata_read = read_zarr(store_path) assert edata.shape == edata_read.shape @@ -143,7 +125,7 @@ def test_write_read_zarr_basic(edata_name, request, tmp_path): for key in edata.layers: _assert_dtype_object_array_with_missing_values_equal(edata.layers[key], edata_read.layers[key]) - _check_anndata_part_equal(edata, edata_read) + _check_aligned_anndata_parts_equal(edata, edata_read) pd.testing.assert_frame_equal(edata.tem.iloc[:, :1], edata_read.tem.iloc[:, :1]) # check the test file is an ehrdata zarr store @@ -158,3 +140,12 @@ def test_write_read_zarr_basic(edata_name, request, tmp_path): assert edata_read.var["var_col_2"].dtype == "category" if "tem_col_2" in edata_read.tem.columns: assert edata_read.tem["tem_col_2"].dtype == "category" + + +def test_write_zarr_chunks_error(edata_333, tmp_path): + with pytest.raises(NotImplementedError): + write_zarr(edata_333, tmp_path / "test.zarr", chunks=None) + with pytest.raises(NotImplementedError): + write_zarr(edata_333, tmp_path / "test.zarr", chunks=1000) + with pytest.raises(NotImplementedError): + write_zarr(edata_333, tmp_path / "test.zarr", chunks="foobar") From 2162ad1bffcb9bc84420f13af323b51ebc7a21d8 Mon Sep 17 00:00:00 2001 From: eroell Date: Tue, 4 Nov 2025 11:16:35 +0100 Subject: [PATCH 25/25] cleaner docstring --- src/ehrdata/io/zarr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ehrdata/io/zarr.py b/src/ehrdata/io/zarr.py index 9c292eb3..a4a869e8 100644 --- a/src/ehrdata/io/zarr.py +++ b/src/ehrdata/io/zarr.py @@ -115,7 +115,7 @@ def write_zarr( Args: edata: Central data object. filename: Name of the output file, can also be prefixed with relative or absolute path to save the file to. - chunks: Specify strategy of how data should be chunked. For simplicity, currently only 2 options are available: "auto" will write the data with :func:`~anndata.io.write_elem`'s default settings. `chunks='ehrdata_auto'` will write the data chunked (and sharded) based on a heuristic that loosely speaking writes slightly smaller chunks. + chunks: Specify strategy of how data should be chunked. For simplicity, currently only 2 options are available: `"auto"` will write the data with :func:`~anndata.io.write_elem`'s default settings. `"ehrdata_auto"` will write the data chunked (and sharded) based on a heuristic that loosely speaking writes slightly smaller chunks. convert_strings_to_categoricals: Convert columns of `str` dtype in `.obs` and `.var` and `.tem` to `categorical` dtype. Examples: