From 5092b99e4ec8587e1e9f9f18eedb8602019534bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96yk=C3=BC=20S=C3=BCoglu?= Date: Fri, 18 Sep 2026 09:37:09 +0200 Subject: [PATCH 1/5] for sprase.COO harmonize_missing_values assigns NaNs to 0s, infer_feature_types fixed to support sparse.COO, tests --- src/ehrdata/_feature_types.py | 164 +++++++++++++++++++++++++++++++--- tests/io/test_h5ed.py | 2 +- tests/io/test_zarr.py | 2 +- tests/test_feature_types.py | 121 +++++++++++++++++++++++++ 4 files changed, 277 insertions(+), 12 deletions(-) diff --git a/src/ehrdata/_feature_types.py b/src/ehrdata/_feature_types.py index 289e266d..9e61e124 100644 --- a/src/ehrdata/_feature_types.py +++ b/src/ehrdata/_feature_types.py @@ -1,11 +1,12 @@ from __future__ import annotations import warnings -from functools import wraps +from functools import singledispatch, wraps from typing import TYPE_CHECKING, Literal import numpy as np import pandas as pd +import sparse from dateutil.parser import isoparse # type: ignore from fast_array_utils.conv import to_dense from rich import print @@ -70,6 +71,52 @@ def _detect_feature_type( return NUMERIC_TAG, False # type: ignore +def _detect_feature_types_sparse_coo( + X: sparse.COO, + *, + binary_as: Literal["categorical", "numeric"] = "categorical", +) -> list[tuple[Literal["categorical", "numeric"], bool]]: + """Detect the feature type of every variable (axis 1) of a numeric `sparse.COO` array, without densifying. + + A `sparse.COO` array is always numeric, so classification only needs to decide, per variable, whether its values + are exactly `{0, 1}` (binary, treated like `_detect_feature_type` treats a binary column) or not + """ + n_vars = X.shape[1] + entries_per_var = X.size // n_vars + + order = np.argsort(X.coords[1], kind="stable") + sorted_vars = X.coords[1][order] + sorted_data = X.data[order] + starts = np.searchsorted(sorted_vars, np.arange(n_vars)) + ends = np.searchsorted(sorted_vars, np.arange(n_vars), side="right") + + fill_is_nan = np.issubdtype(X.dtype, np.floating) and np.isnan(X.fill_value) + + results = [] + for j in range(n_vars): + stored = sorted_data[starts[j] : ends[j]] + if np.issubdtype(X.dtype, np.floating): + stored = stored[~np.isnan(stored)] + values = set(np.unique(stored).tolist()) + + has_implicit_entries = (ends[j] - starts[j]) < entries_per_var + if has_implicit_entries and not fill_is_nan: + values.add(X.fill_value) + + if not values: + err_msg = ( + f"Feature at position {j} contains only NaN values. Please drop this feature to infer the feature type." + ) + raise ValueError(err_msg) + + if values == {0, 1}: + results.append((NUMERIC_TAG, False) if binary_as == "numeric" else (CATEGORICAL_TAG, True)) + else: + results.append((NUMERIC_TAG, False)) + + return results + + def infer_feature_types( edata: EHRData, *, @@ -120,16 +167,20 @@ def infer_feature_types( stacklevel=2, ) - if issparse(X): - X = to_dense(X) + if isinstance(X, sparse.COO): + sparse_results = _detect_feature_types_sparse_coo(X, binary_as=binary_as) + else: + sparse_results = None + if issparse(X): + X = to_dense(X) - if X.ndim == 3: - n_obs, n_vars, n_t = X.shape - X = X.transpose(0, 2, 1).reshape(n_obs * n_t, n_vars) + if X.ndim == 3: + n_obs, n_vars, n_t = X.shape + X = X.transpose(0, 2, 1).reshape(n_obs * n_t, n_vars) - df = pd.DataFrame(X.reshape(-1, edata.shape[1]), columns=edata.var_names) + df = pd.DataFrame(X.reshape(-1, edata.shape[1]), columns=edata.var_names) - for feature in edata.var_names: + for i, feature in enumerate(edata.var_names): if ( FEATURE_TYPE_KEY in edata.var and edata.var[FEATURE_TYPE_KEY][feature] is not None @@ -137,7 +188,10 @@ def infer_feature_types( ): feature_types[feature] = edata.var[FEATURE_TYPE_KEY][feature] else: - feature_types[feature], raise_warning = _detect_feature_type(df[feature], binary_as=binary_as) + if sparse_results is not None: + feature_types[feature], raise_warning = sparse_results[i] + else: + feature_types[feature], raise_warning = _detect_feature_type(df[feature], binary_as=binary_as) if raise_warning: uncertain_features.append(feature) @@ -291,21 +345,97 @@ def replace_feature_types( edata.var.loc[features, FEATURE_TYPE_KEY] = corrected_type +@singledispatch +def _harmonize_missing_values_numeric(X, *, var_names: pd.Index, vars: Iterable[str] | None) -> tuple[object, bool]: + """Treat the implicit zero fill value of a numeric array as missing, if that array type is ambiguous about it. + + Returns `(array, changed)`. The default (dense arrays, `scipy.sparse`, ...) is a no-op: `0` is + unambiguous for those, or (for `scipy.sparse`) there is no lossless way to swap the fill value. + """ + return X, False + + +@_harmonize_missing_values_numeric.register(sparse.COO) +def _(X: sparse.COO, *, var_names: pd.Index, vars: Iterable[str] | None) -> tuple[sparse.COO, bool]: + """Swap a numeric :class:`sparse.COO` array's zero fill value to `np.nan`, keeping `vars` columns real. + + Variables are always axis 1 of `X` (both for 2D and 3D EHRData arrays). Columns in `vars` are + excluded from the zero-as-missing treatment: their previously-implicit zeros are materialized as + explicit stored `0` entries so they are not swept up by the new `np.nan` fill value; all other + columns stay sparse. A boolean array, or one whose fill value is already something other than + `0` (a deliberate existing encoding), is left untouched. + """ + if np.issubdtype(X.dtype, np.bool_) or X.fill_value != 0: + return X, False + + excluded = ( + np.asarray([var_names.get_loc(v) for v in vars], dtype=np.intp) + if vars is not None + else np.array([], dtype=np.intp) + ) + + var_coord = X.coords[1] + is_excluded = np.isin(var_coord, excluded) if excluded.size else np.zeros(var_coord.shape, dtype=bool) + + data = X.data.astype(np.float64, copy=True) + data[(data == 0) & ~is_excluded] = np.nan + coords = X.coords + + if excluded.size: + # Materialize every position of the excluded columns that isn't already stored explicitly, + # since those positions would otherwise silently read back as the new np.nan fill value. + excluded_shape = list(X.shape) + excluded_shape[1] = excluded.size + stored = np.zeros(excluded_shape, dtype=bool) + + pos_in_excluded = np.full(X.shape[1], -1, dtype=np.intp) + pos_in_excluded[excluded] = np.arange(excluded.size) + + if is_excluded.any(): + index = ( + coords[0][is_excluded], + pos_in_excluded[var_coord[is_excluded]], + *(coords[ax][is_excluded] for ax in range(2, X.ndim)), + ) + stored[index] = True + + missing = np.argwhere(~stored) + if missing.size: + missing = missing.T.copy() + missing[1] = excluded[missing[1]] + coords = np.concatenate([coords, missing], axis=1) + data = np.concatenate([data, np.zeros(missing.shape[1], dtype=np.float64)]) + + return sparse.COO(coords, data, shape=X.shape, fill_value=np.nan), True + + def harmonize_missing_values( edata: EHRData, *, layer: str | None = None, missing_values: Iterable[str] | None = ["nan", "np.nan", "", "pd.NA"], + vars: Iterable[str] | None = None, copy: bool = False, ) -> EHRData | None: """Harmonize missing values in the :class:`~ehrdata.EHRData` object. This function will replace strings that are considered to represent missing values with `np.nan`. + For a numeric :class:`sparse.COO` layer, the implicit fill value `0` is ambiguous between "not + measured" and "measured as zero". This function treats it as missing for every variable except + those listed in `vars`. + Columns not in `vars` stay sparse; columns in `vars` keep `0` as a real value, + which requires materializing their previously-implicit zeros as explicit stored entries so they + don't pick up the new `np.nan` fill value. Boolean layers (sparse or dense) are also + left untouched, as `0`/`False` is not ambiguous for them. + Args: edata: Data object. layer: The layer to use from the :class:`~ehrdata.EHRData` object. If `None`, the `X` layer is used. missing_values: The strings that are considered to represent missing values and should be replaced with np.nan + vars: For a sparse :class:`sparse.COO` layer only: variable names whose `0` values represent a + real, measured zero rather than a missing value, and so are excluded from the + zero-as-missing treatment described above. copy: Whether to return a copy of the :class:`~ehrdata.EHRData` object with the missing values replaced. Examples: @@ -320,7 +450,21 @@ def harmonize_missing_values( # note that every scipy sparse array is of a numeric dtype and will enter this if block # further sparse.COO, while being allowed in theory to be str dtype, is only allowed numeric dtypes under binsparse specification which we follow closely if np.issubdtype(X.dtype, np.number) or np.issubdtype(X.dtype, np.bool_): - logger.debug(f"ed.harmonize_missing_values does not affect numeric layer {'X' if layer is None else layer}.") + harmonized, changed = _harmonize_missing_values_numeric(X, var_names=edata.var_names, vars=vars) + + if changed: + logger.debug( + f"ed.harmonize_missing_values treats the implicit zero fill value as missing in sparse.COO layer {'X' if layer is None else layer}." + ) + if layer is None: + edata.X = harmonized + else: + edata.layers[layer] = harmonized + else: + logger.debug( + f"ed.harmonize_missing_values does not affect numeric layer {'X' if layer is None else layer}." + ) + return edata if copy else None df = pd.DataFrame(X.reshape(-1, edata.shape[1]), columns=edata.var_names) diff --git a/tests/io/test_h5ed.py b/tests/io/test_h5ed.py index 7c490fd0..897d454e 100644 --- a/tests/io/test_h5ed.py +++ b/tests/io/test_h5ed.py @@ -233,7 +233,7 @@ def test_write_read_h5ed_sparse_coo_3d(slot, tmp_path): assert {"indices_0", "indices_1", "indices_2", "values", "fill_value"} <= set(group.keys()) assert group["indices_0"].dtype == np.int64 # indices are always written as int64 on disk - edata_read = read_h5ed(path) + edata_read = read_h5ed(path, harmonize_missing_values=False) restored = edata_read.X if slot == "X" else edata_read.layers["tem_data"] assert isinstance(restored, sparse.COO) assert np.array_equal(restored.todense(), dense) diff --git a/tests/io/test_zarr.py b/tests/io/test_zarr.py index 470c4638..0c64f73c 100644 --- a/tests/io/test_zarr.py +++ b/tests/io/test_zarr.py @@ -252,7 +252,7 @@ def test_write_read_zarr_sparse_coo_3d(slot, chunks, tmp_path): assert {"indices_0", "indices_1", "indices_2", "values", "fill_value"} <= set(group.keys()) assert group["indices_0"].dtype == np.int64 # indices are always written as int64 on disk - edata_read = read_zarr(path) + edata_read = read_zarr(path, harmonize_missing_values=False) restored = edata_read.X if slot == "X" else edata_read.layers["tem_data"] assert isinstance(restored, sparse.COO) assert np.array_equal(restored.todense(), dense) diff --git a/tests/test_feature_types.py b/tests/test_feature_types.py index fc2a672f..74a713dd 100644 --- a/tests/test_feature_types.py +++ b/tests/test_feature_types.py @@ -1,6 +1,8 @@ import numpy as np import pandas as pd import pytest +import sparse +from scipy.sparse import csr_matrix from ehrdata import EHRData, feature_type_overview, harmonize_missing_values, infer_feature_types, replace_feature_types from ehrdata._logger import logger @@ -57,6 +59,84 @@ def test_harmonize_missing_values_3D(sample_dataset, request): assert missing_value_string not in edata.layers[DEFAULT_TEM_LAYER_NAME].flatten() +def test_harmonize_missing_values_sparse_coo(): + dense = np.array([[1.0, 0.0, 2.0], [0.0, 0.0, 3.0]]) + X = sparse.COO.from_numpy(dense) + edata = EHRData(X=X) + + harmonize_missing_values(edata) + + assert isinstance(edata.X, sparse.COO) + assert edata.X.nnz == X.nnz # stays sparse + assert np.isnan(edata.X.fill_value) + np.testing.assert_array_equal(edata.X.todense(), [[1.0, np.nan, 2.0], [np.nan, np.nan, 3.0]]) + + +def test_harmonize_missing_values_sparse_coo_3d(): + dense = np.array([[[1.0, 0.0], [0.0, 2.0], [3.0, 0.0]], [[0.0, 0.0], [4.0, 0.0], [0.0, 5.0]]]) + X = sparse.COO.from_numpy(dense) + edata = EHRData(layers={DEFAULT_TEM_LAYER_NAME: X}) + + harmonize_missing_values(edata, layer=DEFAULT_TEM_LAYER_NAME) + + result = edata.layers[DEFAULT_TEM_LAYER_NAME] + assert isinstance(result, sparse.COO) + assert result.nnz == X.nnz + assert np.isnan(result.fill_value) + + +def test_harmonize_missing_values_sparse_coo_vars_excluded(): + dense = np.array([[1.0, 0.0, 2.0], [0.0, 0.0, 3.0], [4.0, 0.0, 0.0]]) + X = sparse.COO.from_numpy(dense) + edata = EHRData(X=X, var=pd.DataFrame(index=["v0", "v1", "v2"])) + + harmonize_missing_values(edata, vars=["v1"]) + + result = edata.X + assert isinstance(result, sparse.COO) + assert np.isnan(result.fill_value) + np.testing.assert_array_equal( + result.todense(), + [[1.0, 0.0, 2.0], [np.nan, 0.0, 3.0], [4.0, 0.0, np.nan]], + ) + + +def test_harmonize_missing_values_sparse_coo_3d_vars_excluded(): + dense = np.array([[[1.0, 0.0], [0.0, 2.0], [3.0, 0.0]], [[0.0, 0.0], [4.0, 0.0], [0.0, 5.0]]]) + X = sparse.COO.from_numpy(dense) + edata = EHRData(layers={DEFAULT_TEM_LAYER_NAME: X}, var=pd.DataFrame(index=["v0", "v1", "v2"])) + + harmonize_missing_values(edata, layer=DEFAULT_TEM_LAYER_NAME, vars=["v1"]) + + result = edata.layers[DEFAULT_TEM_LAYER_NAME] + assert isinstance(result, sparse.COO) + assert np.isnan(result.fill_value) + np.testing.assert_array_equal( + result.todense(), + [[[1.0, np.nan], [0.0, 2.0], [3.0, np.nan]], [[np.nan, np.nan], [4.0, 0.0], [np.nan, 5.0]]], + ) + + +def test_harmonize_missing_values_sparse_coo_unknown_var_raises(): + dense = np.array([[1.0, 0.0], [0.0, 2.0]]) + X = sparse.COO.from_numpy(dense) + edata = EHRData(X=X, var=pd.DataFrame(index=["v0", "v1"])) + + with pytest.raises(KeyError): + harmonize_missing_values(edata, vars=["bogus"]) + + +def test_harmonize_missing_values_scipy_sparse_unaffected(): + dense = np.array([[1.0, 0.0, 2.0], [0.0, 0.0, 3.0]]) + X = csr_matrix(dense) + edata = EHRData(X=X) + + harmonize_missing_values(edata) + + assert isinstance(edata.X, csr_matrix) + np.testing.assert_array_equal(edata.X.toarray(), dense) + + @pytest.mark.parametrize( "sample_dataset", [ @@ -125,6 +205,47 @@ def test_feature_type_inference_float_encoded_binary(binary_as, expected): assert edata.var["feature_type"]["binary_feature"] == expected +def test_feature_type_inference_sparse_coo(): + dense = np.array([[1.0, 2.0, 0.0], [4.0, 0.0, 6.0], [0.0, 5.0, 7.0], [8.0, 0.0, 0.0]]) + X = sparse.COO.from_numpy(dense) + edata = EHRData(X=X, var=pd.DataFrame(index=["v0", "v1", "v2"])) + + infer_feature_types(edata, output=None) + + assert list(edata.var["feature_type"]) == ["numeric", "numeric", "numeric"] + assert isinstance(edata.X, sparse.COO) + assert edata.X.nnz == X.nnz # inference must not densify or otherwise mutate the stored array + + +def test_feature_type_inference_sparse_coo_binary(): + dense = np.array([[0.0, 5.0], [1.0, 0.0], [0.0, 3.0], [1.0, 0.0]]) + X = sparse.COO.from_numpy(dense) + edata = EHRData(X=X, var=pd.DataFrame(index=["bin", "num"])) + + infer_feature_types(edata, output=None) + + assert list(edata.var["feature_type"]) == ["categorical", "numeric"] + + +def test_feature_type_inference_sparse_coo_binary_as_numeric(): + dense = np.array([[0.0, 5.0], [1.0, 0.0], [0.0, 3.0], [1.0, 0.0]]) + X = sparse.COO.from_numpy(dense) + edata = EHRData(X=X, var=pd.DataFrame(index=["bin", "num"])) + + infer_feature_types(edata, binary_as="numeric", output=None) + + assert list(edata.var["feature_type"]) == ["numeric", "numeric"] + + +def test_feature_type_inference_sparse_coo_all_nan_raises(): + dense = np.array([[1.0, np.nan], [2.0, np.nan], [3.0, np.nan]]) + X = sparse.COO.from_numpy(dense) + edata = EHRData(X=X, var=pd.DataFrame(index=["v0", "v1"])) + + with pytest.raises(ValueError, match="only NaN"): + infer_feature_types(edata, output=None) + + @pytest.mark.parametrize( "sample_dataset", [ From 9a91f1458ff38f3f934cee123edd282d414b5e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96yk=C3=BC=20S=C3=BCoglu?= Date: Fri, 18 Sep 2026 10:01:13 +0200 Subject: [PATCH 2/5] added sparse.coo to nitpick_ignore --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 62365bd8..2c1a9b62 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -159,6 +159,7 @@ ("py:class", "anndata.compat.CupySparseMatrix"), ("py:class", "sparse.numba_backend._coo.core.COO"), ("py:class", "sparse._coo.core.COO"), + ("py:class", "sparse.COO"), ("py:class", "pandas.core.arrays.base.ExtensionArray"), ] From c536041af48a7112c07e9d885cc18b93ffeeda20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96yk=C3=BC=20S=C3=BCoglu?= Date: Wed, 23 Sep 2026 13:22:15 +0200 Subject: [PATCH 3/5] parametrized tests, infer_feature_types classifies every column of sparse arrays wo densfiying, harmonize_missing_values accepts only numeric sparse.COO --- src/ehrdata/_feature_types.py | 160 +++++++++++++++++++++++++--------- tests/test_feature_types.py | 88 +++++++++++++++---- 2 files changed, 189 insertions(+), 59 deletions(-) diff --git a/src/ehrdata/_feature_types.py b/src/ehrdata/_feature_types.py index 9e61e124..9317f4aa 100644 --- a/src/ehrdata/_feature_types.py +++ b/src/ehrdata/_feature_types.py @@ -6,6 +6,7 @@ import numpy as np import pandas as pd +import scipy.sparse as sp import sparse from dateutil.parser import isoparse # type: ignore from fast_array_utils.conv import to_dense @@ -21,6 +22,9 @@ from ehrdata import EHRData +# sparse array types +_FAST_SPARSE_TYPES = (sparse.COO, sp.csr_array, sp.csr_matrix, sp.csc_array, sp.csc_matrix) + def _detect_feature_type( col: pd.Series, @@ -71,16 +75,63 @@ def _detect_feature_type( return NUMERIC_TAG, False # type: ignore -def _detect_feature_types_sparse_coo( - X: sparse.COO, +def _classify_sparse_binary_columns( + var_coord: np.ndarray, + data: np.ndarray, + n_vars: int, + entries_per_var: int, *, - binary_as: Literal["categorical", "numeric"] = "categorical", + fill_value, + fill_is_nan: bool, +) -> np.ndarray: + """Per variable, decides if its values (explicit + implicit) are exactly `{0, 1}`.""" + is_float = np.issubdtype(data.dtype, np.floating) + nan_mask = np.isnan(data) if is_float else np.zeros(data.shape, dtype=bool) + + has_zero = np.zeros(n_vars, dtype=bool) + has_one = np.zeros(n_vars, dtype=bool) + has_other = np.zeros(n_vars, dtype=bool) + stored_count = np.zeros(n_vars, dtype=np.int64) + non_nan_count = np.zeros(n_vars, dtype=np.int64) + + np.logical_or.at(has_zero, var_coord, (data == 0) & ~nan_mask) + np.logical_or.at(has_one, var_coord, (data == 1) & ~nan_mask) + np.logical_or.at(has_other, var_coord, ~nan_mask & (data != 0) & (data != 1)) + np.add.at(stored_count, var_coord, 1) + np.add.at(non_nan_count, var_coord, ~nan_mask) + + has_implicit = stored_count < entries_per_var + if not fill_is_nan: + non_nan_count += has_implicit * (entries_per_var - stored_count) + if fill_value == 0: + has_zero |= has_implicit + elif fill_value == 1: + has_one |= has_implicit + else: + has_other |= has_implicit + + all_nan = np.nonzero(non_nan_count == 0)[0] + if all_nan.size: + err_msg = f"Feature at position {all_nan[0]} contains only NaN values. Please drop this feature to infer the feature type." + raise ValueError(err_msg) + + return has_zero & has_one & ~has_other + + +def _binary_mask_to_results( + is_binary: np.ndarray, binary_as: Literal["categorical", "numeric"] ) -> list[tuple[Literal["categorical", "numeric"], bool]]: - """Detect the feature type of every variable (axis 1) of a numeric `sparse.COO` array, without densifying. + binary_result = (NUMERIC_TAG, False) if binary_as == "numeric" else (CATEGORICAL_TAG, True) + numeric_result = (NUMERIC_TAG, False) + return [binary_result if b else numeric_result for b in is_binary] - A `sparse.COO` array is always numeric, so classification only needs to decide, per variable, whether its values - are exactly `{0, 1}` (binary, treated like `_detect_feature_type` treats a binary column) or not - """ + +def _detect_feature_types_sparse_coo_non_numeric( + X: sparse.COO, + *, + binary_as: Literal["categorical", "numeric"] = "categorical", +) -> list[tuple[Literal["date", "categorical", "numeric"], bool]]: + """Detect the feature type of every variable of a non-numeric `sparse.COO` array.""" n_vars = X.shape[1] entries_per_var = X.size // n_vars @@ -90,33 +141,60 @@ def _detect_feature_types_sparse_coo( starts = np.searchsorted(sorted_vars, np.arange(n_vars)) ends = np.searchsorted(sorted_vars, np.arange(n_vars), side="right") - fill_is_nan = np.issubdtype(X.dtype, np.floating) and np.isnan(X.fill_value) - results = [] for j in range(n_vars): - stored = sorted_data[starts[j] : ends[j]] - if np.issubdtype(X.dtype, np.floating): - stored = stored[~np.isnan(stored)] - values = set(np.unique(stored).tolist()) - - has_implicit_entries = (ends[j] - starts[j]) < entries_per_var - if has_implicit_entries and not fill_is_nan: - values.add(X.fill_value) - - if not values: - err_msg = ( - f"Feature at position {j} contains only NaN values. Please drop this feature to infer the feature type." - ) - raise ValueError(err_msg) - - if values == {0, 1}: - results.append((NUMERIC_TAG, False) if binary_as == "numeric" else (CATEGORICAL_TAG, True)) + if ends[j] - starts[j] < entries_per_var: + # rare: a genuine implicit fill value among non-numeric data, densify just this column + col = pd.Series(X[:, j].todense().reshape(-1)) else: - results.append((NUMERIC_TAG, False)) + col = pd.Series(sorted_data[starts[j] : ends[j]]) + results.append(_detect_feature_type(col, binary_as=binary_as)) return results +@singledispatch +def _detect_feature_types_sparse(X, *, binary_as: Literal["categorical", "numeric"] = "categorical"): + """Detect the feature type of every variable (axis 1) of a sparse array, without densifying.""" + msg = f"_detect_feature_types_sparse does not support array type {type(X)}." + raise NotImplementedError(msg) + + +@_detect_feature_types_sparse.register(sparse.COO) +def _(X: sparse.COO, *, binary_as: Literal["categorical", "numeric"] = "categorical"): + # sparse.COO isn't restricted to numeric dtypes in memory (only ehrdata's on-disk binsparse + # writer enforces that); route non-numeric arrays through the dedicated per-column path. + if not (np.issubdtype(X.dtype, np.number) or np.issubdtype(X.dtype, np.bool_)): + return _detect_feature_types_sparse_coo_non_numeric(X, binary_as=binary_as) + + n_vars = X.shape[1] + entries_per_var = X.size // n_vars # rows * time steps (for 3D) per variable + fill_is_nan = np.issubdtype(X.dtype, np.floating) and np.isnan(X.fill_value) + is_binary = _classify_sparse_binary_columns( + X.coords[1], X.data, n_vars, entries_per_var, fill_value=X.fill_value, fill_is_nan=fill_is_nan + ) + return _binary_mask_to_results(is_binary, binary_as) + + +@_detect_feature_types_sparse.register(sp.csr_array) +@_detect_feature_types_sparse.register(sp.csr_matrix) +def _(X, *, binary_as: Literal["categorical", "numeric"] = "categorical"): + # CSR stores each entry's column index in .indices + n_vars = X.shape[1] + is_binary = _classify_sparse_binary_columns(X.indices, X.data, n_vars, X.shape[0], fill_value=0, fill_is_nan=False) + return _binary_mask_to_results(is_binary, binary_as) + + +@_detect_feature_types_sparse.register(sp.csc_array) +@_detect_feature_types_sparse.register(sp.csc_matrix) +def _(X, *, binary_as: Literal["categorical", "numeric"] = "categorical"): + # CSC stores entries grouped by column via .indptr + n_vars = X.shape[1] + var_coord = np.repeat(np.arange(n_vars), np.diff(X.indptr)) + is_binary = _classify_sparse_binary_columns(var_coord, X.data, n_vars, X.shape[0], fill_value=0, fill_is_nan=False) + return _binary_mask_to_results(is_binary, binary_as) + + def infer_feature_types( edata: EHRData, *, @@ -167,12 +245,12 @@ def infer_feature_types( stacklevel=2, ) - if isinstance(X, sparse.COO): - sparse_results = _detect_feature_types_sparse_coo(X, binary_as=binary_as) + # The non-densifying fast path applies to the array types registered on _detect_feature_types_sparse + # (sparse.COO handles numeric and non-numeric dtypes internally; scipy.sparse is always numeric). + if isinstance(X, _FAST_SPARSE_TYPES): + sparse_results = _detect_feature_types_sparse(X, binary_as=binary_as) else: sparse_results = None - if issparse(X): - X = to_dense(X) if X.ndim == 3: n_obs, n_vars, n_t = X.shape @@ -357,14 +435,7 @@ def _harmonize_missing_values_numeric(X, *, var_names: pd.Index, vars: Iterable[ @_harmonize_missing_values_numeric.register(sparse.COO) def _(X: sparse.COO, *, var_names: pd.Index, vars: Iterable[str] | None) -> tuple[sparse.COO, bool]: - """Swap a numeric :class:`sparse.COO` array's zero fill value to `np.nan`, keeping `vars` columns real. - - Variables are always axis 1 of `X` (both for 2D and 3D EHRData arrays). Columns in `vars` are - excluded from the zero-as-missing treatment: their previously-implicit zeros are materialized as - explicit stored `0` entries so they are not swept up by the new `np.nan` fill value; all other - columns stay sparse. A boolean array, or one whose fill value is already something other than - `0` (a deliberate existing encoding), is left untouched. - """ + """Swap a numeric :class:`sparse.COO` array's zero fill value to `np.nan`, keeping `vars` columns real.""" if np.issubdtype(X.dtype, np.bool_) or X.fill_value != 0: return X, False @@ -447,8 +518,15 @@ def harmonize_missing_values( edata = edata.copy() X = edata.X if layer is None else edata.layers[layer] - # note that every scipy sparse array is of a numeric dtype and will enter this if block - # further sparse.COO, while being allowed in theory to be str dtype, is only allowed numeric dtypes under binsparse specification which we follow closely + # reject non numeric sparse.COO explicitly rather than silently densifying + if isinstance(X, sparse.COO) and not (np.issubdtype(X.dtype, np.number) or np.issubdtype(X.dtype, np.bool_)): + err_msg = ( + f"ed.harmonize_missing_values can only be used on a numeric sparse.COO layer " + f"({'X' if layer is None else layer!r} has dtype {X.dtype})." + ) + raise NotImplementedError(err_msg) + + # every scipy sparse array is of a numeric dtype and will enter this if block if np.issubdtype(X.dtype, np.number) or np.issubdtype(X.dtype, np.bool_): harmonized, changed = _harmonize_missing_values_numeric(X, var_names=edata.var_names, vars=vars) diff --git a/tests/test_feature_types.py b/tests/test_feature_types.py index 74a713dd..36005bf4 100644 --- a/tests/test_feature_types.py +++ b/tests/test_feature_types.py @@ -1,11 +1,13 @@ import numpy as np import pandas as pd import pytest +import scipy.sparse as sp import sparse from scipy.sparse import csr_matrix from ehrdata import EHRData, feature_type_overview, harmonize_missing_values, infer_feature_types, replace_feature_types from ehrdata._logger import logger +from ehrdata._types import ARRAY_TYPES_NUMERIC from ehrdata.core.constants import DEFAULT_TEM_LAYER_NAME, MISSING_VALUES @@ -205,6 +207,45 @@ def test_feature_type_inference_float_encoded_binary(binary_as, expected): assert edata.var["feature_type"]["binary_feature"] == expected +@pytest.mark.parametrize("array_type", ARRAY_TYPES_NUMERIC) +def test_feature_type_inference_array_types_consistent(array_type): + """infer_feature_types must classify the same numeric data identically regardless of array type.""" + dense = np.array( + [ + [1.0, 0.0, 2.5, np.nan], + [0.0, 1.0, 3.5, 4.0], + [1.0, 0.0, 0.0, 5.0], + [0.0, 1.0, 4.5, 6.0], + ] + ) + edata = EHRData( + X=array_type(dense), var=pd.DataFrame(index=["binary", "binary2", "numeric", "numeric_with_missing"]) + ) + + infer_feature_types(edata, output=None) + + assert list(edata.var["feature_type"]) == ["categorical", "categorical", "numeric", "numeric"] + + +@pytest.mark.parametrize("array_type", ARRAY_TYPES_NUMERIC) +def test_feature_type_inference_array_types_binary_as_numeric_consistent(array_type): + dense = np.array([[0.0, 5.0], [1.0, 0.0], [0.0, 3.0], [1.0, 0.0]]) + edata = EHRData(X=array_type(dense), var=pd.DataFrame(index=["binary", "numeric"])) + + infer_feature_types(edata, binary_as="numeric", output=None) + + assert list(edata.var["feature_type"]) == ["numeric", "numeric"] + + +@pytest.mark.parametrize("array_type", ARRAY_TYPES_NUMERIC) +def test_feature_type_inference_array_types_all_nan_raises_consistently(array_type): + dense = np.array([[1.0, np.nan], [2.0, np.nan], [3.0, np.nan]]) + edata = EHRData(X=array_type(dense), var=pd.DataFrame(index=["v0", "v1"])) + + with pytest.raises(ValueError, match="only NaN"): + infer_feature_types(edata, output=None) + + def test_feature_type_inference_sparse_coo(): dense = np.array([[1.0, 2.0, 0.0], [4.0, 0.0, 6.0], [0.0, 5.0, 7.0], [8.0, 0.0, 0.0]]) X = sparse.COO.from_numpy(dense) @@ -217,33 +258,44 @@ def test_feature_type_inference_sparse_coo(): assert edata.X.nnz == X.nnz # inference must not densify or otherwise mutate the stored array -def test_feature_type_inference_sparse_coo_binary(): - dense = np.array([[0.0, 5.0], [1.0, 0.0], [0.0, 3.0], [1.0, 0.0]]) - X = sparse.COO.from_numpy(dense) - edata = EHRData(X=X, var=pd.DataFrame(index=["bin", "num"])) +def test_feature_type_inference_sparse_coo_non_numeric(): + # a string/object sparse.COO must be classified correctly instead of + # being silently treated as numeric + arr = np.array([["a", "b"], ["c", "d"], ["a", "b"]], dtype=object) + X = sparse.COO.from_numpy(arr) + assert X.nnz == arr.size + edata = EHRData(X=X, var=pd.DataFrame(index=["v0", "v1"])) infer_feature_types(edata, output=None) - assert list(edata.var["feature_type"]) == ["categorical", "numeric"] + assert list(edata.var["feature_type"]) == ["categorical", "categorical"] -def test_feature_type_inference_sparse_coo_binary_as_numeric(): - dense = np.array([[0.0, 5.0], [1.0, 0.0], [0.0, 3.0], [1.0, 0.0]]) - X = sparse.COO.from_numpy(dense) - edata = EHRData(X=X, var=pd.DataFrame(index=["bin", "num"])) +def test_feature_type_inference_sparse_coo_non_numeric_with_implicit_fill(): + # a non-numeric sparse.COO deliberately constructed with a custom fill_value + coords = np.array([[0, 1], [0, 1]]) + data = np.array(["x", "y"], dtype=object) + X = sparse.COO(coords, data, shape=(3, 2), fill_value="z") + assert X.nnz < 3 * 2 # genuine implicit entries present# - infer_feature_types(edata, binary_as="numeric", output=None) + edata = EHRData(X=X, var=pd.DataFrame(index=["v0", "v1"])) + infer_feature_types(edata, output=None) - assert list(edata.var["feature_type"]) == ["numeric", "numeric"] + dense_edata = EHRData(X=X.todense(), var=pd.DataFrame(index=["v0", "v1"])) + infer_feature_types(dense_edata, output=None) + assert list(edata.var["feature_type"]) == list(dense_edata.var["feature_type"]) -def test_feature_type_inference_sparse_coo_all_nan_raises(): - dense = np.array([[1.0, np.nan], [2.0, np.nan], [3.0, np.nan]]) - X = sparse.COO.from_numpy(dense) - edata = EHRData(X=X, var=pd.DataFrame(index=["v0", "v1"])) - with pytest.raises(ValueError, match="only NaN"): - infer_feature_types(edata, output=None) +@pytest.mark.parametrize("ctor", [sp.csr_matrix, sp.csc_matrix]) +def test_feature_type_inference_scipy_sparse_matrix_variants(ctor): + """The fast path is registered for both the array and matrix scipy.sparse variants.""" + dense = np.array([[0.0, 5.0], [1.0, 0.0], [0.0, 3.0], [1.0, 0.0]]) + edata = EHRData(X=ctor(dense), var=pd.DataFrame(index=["binary", "numeric"])) + + infer_feature_types(edata, output=None) + + assert list(edata.var["feature_type"]) == ["categorical", "numeric"] @pytest.mark.parametrize( @@ -259,7 +311,7 @@ def test_feature_type_overview_vanilla(sample_dataset, request, capsys): edata = EHRData(X=data.values, var=pd.DataFrame(data.columns)) feature_type_overview(edata) assert ( - " Detected feature types for EHRData object with 4 obs and 11 vars\n╠══ 📅 Date features\n╠══ 📐 Numerical features\n║ ╠══ 0\n║ ╠══ 1\n║ ╠══ 2\n║ ╠══ 3\n║ ╚══ 4\n╚══ 🗂️ Categorical features\n ╠══ 10 (2 categories)\n ╠══ 5 (4 categories)\n ╠══ 6 (3 categories)\n ╠══ 7 (2 categories)\n ╠══ 8 (2 categories)\n ╚══ 9 (2 categories)" + " Detected feature types for EHRData object with 4 obs and 11 vars\n╠══ 📅 Date features\n╠══ 📐 Numerical features\nâ•‘ ╠══ 0\nâ•‘ ╠══ 1\nâ•‘ ╠══ 2\nâ•‘ ╠══ 3\nâ•‘ ╚══ 4\n╚══ 🗂️ Categorical features\n ╠══ 10 (2 categories)\n ╠══ 5 (4 categories)\n ╠══ 6 (3 categories)\n ╠══ 7 (2 categories)\n ╠══ 8 (2 categories)\n ╚══ 9 (2 categories)" in capsys.readouterr().out ) From a3b7464114427ff5f7f4fda4d7c13b4ed7ae8d93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96yk=C3=BC=20S=C3=BCoglu?= Date: Tue, 29 Sep 2026 09:52:30 +0200 Subject: [PATCH 4/5] harmonize missing values singledispatched for scipy.sparse as well, harmonize_on_read catches the NotImplementedError, test adjusted --- src/ehrdata/_feature_types.py | 44 +++++++++++++++++++++++------------ tests/test_feature_types.py | 8 +++---- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/ehrdata/_feature_types.py b/src/ehrdata/_feature_types.py index 9317f4aa..106cf90f 100644 --- a/src/ehrdata/_feature_types.py +++ b/src/ehrdata/_feature_types.py @@ -15,6 +15,7 @@ from scipy.sparse import issparse from ehrdata._logger import logger +from ehrdata._types import CSArray, CSMatrix from ehrdata.core.constants import CATEGORICAL_TAG, DATE_TAG, FEATURE_TYPE_KEY, MISSING_VALUES, NUMERIC_TAG if TYPE_CHECKING: @@ -23,7 +24,7 @@ from ehrdata import EHRData # sparse array types -_FAST_SPARSE_TYPES = (sparse.COO, sp.csr_array, sp.csr_matrix, sp.csc_array, sp.csc_matrix) +_FAST_SPARSE_TYPES = (sparse.COO, CSMatrix, CSArray) def _detect_feature_type( @@ -425,11 +426,7 @@ def replace_feature_types( @singledispatch def _harmonize_missing_values_numeric(X, *, var_names: pd.Index, vars: Iterable[str] | None) -> tuple[object, bool]: - """Treat the implicit zero fill value of a numeric array as missing, if that array type is ambiguous about it. - - Returns `(array, changed)`. The default (dense arrays, `scipy.sparse`, ...) is a no-op: `0` is - unambiguous for those, or (for `scipy.sparse`) there is no lossless way to swap the fill value. - """ + """Treat the implicit zero fill value of a numeric array as missing, if that array type is ambiguous about it.""" return X, False @@ -480,6 +477,18 @@ def _(X: sparse.COO, *, var_names: pd.Index, vars: Iterable[str] | None) -> tupl return sparse.COO(coords, data, shape=X.shape, fill_value=np.nan), True +@_harmonize_missing_values_numeric.register(sp.csr_array) +@_harmonize_missing_values_numeric.register(sp.csr_matrix) +@_harmonize_missing_values_numeric.register(sp.csc_array) +@_harmonize_missing_values_numeric.register(sp.csc_matrix) +def _(X, *, var_names: pd.Index, vars: Iterable[str] | None) -> tuple[object, bool]: + err_msg = ( + f"ed.harmonize_missing_values cannot treat the implicit zero fill value as missing for " + f"array type {type(X)}. Please convert to a dense array or sparse.COO first." + ) + raise NotImplementedError(err_msg) + + def harmonize_missing_values( edata: EHRData, *, @@ -557,12 +566,17 @@ def harmonize_missing_values( def _harmonize_on_read(edata: EHRData) -> None: - if edata.X is not None: - harmonize_missing_values(edata) - logger.info("Harmonizing missing values of X") - - for key in edata.layers: - if key is None: # anndata 0.13: the unified `.X` slot, already harmonized above - continue - harmonize_missing_values(edata, layer=key) - logger.info(f"Harmonizing missing values of layer {key}") + layers = [None] if edata.X is not None else [] + # anndata 0.13's unified `.X` slot shows up as key None in edata.layers too; skip the duplicate + layers += [key for key in edata.layers if key is not None] + + for layer in layers: + label = "X" if layer is None else f"layer {layer}" + try: + harmonize_missing_values(edata, layer=layer) + logger.info(f"Harmonizing missing values of {label}") + except NotImplementedError: + # scipy.sparse (CSR/CSC) can't be harmonized without densifying + logger.debug( + f"Skipping missing-value harmonization of {label}: not supported for scipy.sparse without densifying." + ) diff --git a/tests/test_feature_types.py b/tests/test_feature_types.py index 36005bf4..918f0ace 100644 --- a/tests/test_feature_types.py +++ b/tests/test_feature_types.py @@ -128,15 +128,13 @@ def test_harmonize_missing_values_sparse_coo_unknown_var_raises(): harmonize_missing_values(edata, vars=["bogus"]) -def test_harmonize_missing_values_scipy_sparse_unaffected(): +def test_harmonize_missing_values_scipy_sparse_raises(): dense = np.array([[1.0, 0.0, 2.0], [0.0, 0.0, 3.0]]) X = csr_matrix(dense) edata = EHRData(X=X) - harmonize_missing_values(edata) - - assert isinstance(edata.X, csr_matrix) - np.testing.assert_array_equal(edata.X.toarray(), dense) + with pytest.raises(NotImplementedError, match="cannot treat the implicit zero fill value as missing"): + harmonize_missing_values(edata) @pytest.mark.parametrize( From 86dbd986d2062dbc2209b611faf1ed24e134930e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96yk=C3=BC=20S=C3=BCoglu?= Date: Tue, 29 Sep 2026 10:16:03 +0200 Subject: [PATCH 5/5] revert test_feature_type_overview_vanilla change --- tests/test_feature_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_feature_types.py b/tests/test_feature_types.py index 918f0ace..ded78f2a 100644 --- a/tests/test_feature_types.py +++ b/tests/test_feature_types.py @@ -309,7 +309,7 @@ def test_feature_type_overview_vanilla(sample_dataset, request, capsys): edata = EHRData(X=data.values, var=pd.DataFrame(data.columns)) feature_type_overview(edata) assert ( - " Detected feature types for EHRData object with 4 obs and 11 vars\n╠══ 📅 Date features\n╠══ 📐 Numerical features\nâ•‘ ╠══ 0\nâ•‘ ╠══ 1\nâ•‘ ╠══ 2\nâ•‘ ╠══ 3\nâ•‘ ╚══ 4\n╚══ 🗂️ Categorical features\n ╠══ 10 (2 categories)\n ╠══ 5 (4 categories)\n ╠══ 6 (3 categories)\n ╠══ 7 (2 categories)\n ╠══ 8 (2 categories)\n ╚══ 9 (2 categories)" + " Detected feature types for EHRData object with 4 obs and 11 vars\n╠══ 📅 Date features\n╠══ 📐 Numerical features\n║ ╠══ 0\n║ ╠══ 1\n║ ╠══ 2\n║ ╠══ 3\n║ ╚══ 4\n╚══ 🗂️ Categorical features\n ╠══ 10 (2 categories)\n ╠══ 5 (4 categories)\n ╠══ 6 (3 categories)\n ╠══ 7 (2 categories)\n ╠══ 8 (2 categories)\n ╚══ 9 (2 categories)" in capsys.readouterr().out )