diff --git a/biolearn/data_library.py b/biolearn/data_library.py index b0c62f9..a3a3ec9 100644 --- a/biolearn/data_library.py +++ b/biolearn/data_library.py @@ -1029,7 +1029,15 @@ def _check_keys_exist(self, data, keys: list[str]): def _create_metadata(self, metadata_query_url): response = requests.get(metadata_query_url) - json_data = response.json() + response.raise_for_status() + try: + json_data = response.json() + except ValueError as e: + # NCBI serves an HTML CAPTCHA page when it blocks automated requests + raise ValueError( + f"GEO metadata query did not return JSON: {metadata_query_url}. " + "NCBI may be blocking automated requests." + ) from e smaples = [item for item in json_data["GeoMetaData"]] return self._convert_to_metadata_df(self.metadata_keys, smaples) diff --git a/biolearn/model.py b/biolearn/model.py index e4edf3a..30d34c9 100644 --- a/biolearn/model.py +++ b/biolearn/model.py @@ -1082,6 +1082,7 @@ def preprocess(df): "model": { "type": "HurdleAPIModel", "use_production": True, + "sites_file": "Hurdle_CpGs.csv", "default_imputation": "none", }, "usage": { @@ -1995,12 +1996,14 @@ class HurdleAPIModel: DEFAULT_TIMEOUT = 30 # seconds MAX_RETRIES = 3 RETRY_DELAY = 1 # seconds + DEFAULT_SITES_FILE = "Hurdle_CpGs.csv" def __init__( self, api_key: Optional[str] = None, use_production: bool = False, timeout: Optional[int] = None, + sites_file: str = DEFAULT_SITES_FILE, **details: Any, ) -> None: self.details = details @@ -2023,7 +2026,7 @@ def __init__( # Load required CpG sites try: - cpg_file = get_data_file("Hurdle_CpGs.csv") + cpg_file = get_data_file(sites_file) self.required_cpgs = pd.read_csv(cpg_file, encoding="ISO-8859-1")[ "ProbeID" ].tolist() @@ -2040,6 +2043,7 @@ def from_definition( model_def = clock_definition["model"] return cls( use_production=model_def.get("use_production", False), + sites_file=model_def.get("sites_file", cls.DEFAULT_SITES_FILE), **{k: v for k, v in clock_definition.items() if k != "model"}, ) diff --git a/biolearn/test/test_autoscan_geo_matrix_parser.py b/biolearn/test/test_autoscan_geo_matrix_parser.py index 207a9ad..1e5b30b 100644 --- a/biolearn/test/test_autoscan_geo_matrix_parser.py +++ b/biolearn/test/test_autoscan_geo_matrix_parser.py @@ -1,6 +1,12 @@ +import pandas as pd import pytest +from unittest.mock import Mock, patch -from biolearn.data_library import DataLibrary, NoMatrixDataError +from biolearn.data_library import ( + AutoScanGeoMatrixParser, + DataLibrary, + NoMatrixDataError, +) from biolearn.util import get_data_file @@ -28,5 +34,29 @@ def test_series_has_no_matrix_data_error(): library_file=get_data_file("geo_autoscan_library.yaml") ) - with pytest.raises(NoMatrixDataError): - library.get("GSE121633").load() + # geo2r now answers scripted requests with a CAPTCHA page, so stub the + # metadata call and exercise only the matrix check under test + stub_metadata = pd.DataFrame({"id": ["GSE121633"]}) + with patch.object( + AutoScanGeoMatrixParser, "_create_metadata", return_value=stub_metadata + ): + with pytest.raises(NoMatrixDataError): + library.get("GSE121633").load() + + +def test_metadata_query_non_json_response_error(): + parser = AutoScanGeoMatrixParser( + { + "matrix_file": "unused", + "metadata_keys_parse": {}, + "metadata_query": "unused", + } + ) + html_response = Mock() + html_response.json.side_effect = ValueError("Expecting value") + + with patch( + "biolearn.data_library.requests.get", return_value=html_response + ): + with pytest.raises(ValueError, match="did not return JSON"): + parser._create_metadata("https://example.org/geo2r") diff --git a/biolearn/test/test_hurdle_model.py b/biolearn/test/test_hurdle_model.py index 4b3d51a..589d0db 100644 --- a/biolearn/test/test_hurdle_model.py +++ b/biolearn/test/test_hurdle_model.py @@ -10,6 +10,7 @@ from biolearn.model import HurdleAPIModel from biolearn.model_gallery import ModelGallery +from biolearn.util import get_data_file class TestHurdleAPIModel: @@ -156,6 +157,31 @@ def test_model_gallery_integration(self): model = gallery.get("HurdleInflammAge") assert isinstance(model, HurdleAPIModel) + def test_sites_loaded_from_definition_sites_file(self): + """Test that required sites come from the definition's sites_file.""" + definition = ModelGallery().model_definitions["HurdleInflammAge"] + sites_file = definition["model"]["sites_file"] + expected = pd.read_csv( + get_data_file(sites_file), encoding="ISO-8859-1" + )["ProbeID"].tolist() + + with patch.dict(os.environ, {"HURDLE_API_KEY": "test_key"}): + model = HurdleAPIModel.from_definition(definition) + + assert len(expected) > 0 + assert model.methylation_sites() == expected + + def test_missing_sites_file_falls_back_to_all_sites(self): + """Test that an unknown sites_file warns and leaves sites unset.""" + with pytest.warns( + UserWarning, match="Could not load Hurdle CpG sites" + ): + model = HurdleAPIModel( + api_key="test_key", sites_file="does_not_exist.csv" + ) + + assert model.methylation_sites() == [] + @patch("builtins.input", return_value="yes") def test_consent_only_asked_once(self, mock_input): """Test that consent is only requested once.""" diff --git a/biolearn/test/test_model.py b/biolearn/test/test_model.py index 7f04b6e..493198a 100644 --- a/biolearn/test/test_model.py +++ b/biolearn/test/test_model.py @@ -1,3 +1,4 @@ +import os import pytest from math import isclose import pandas as pd @@ -5,6 +6,7 @@ import numpy as np from biolearn import model from biolearn.util import ( + get_data_file, get_test_data_file, load_test_data_file, ) @@ -146,6 +148,22 @@ def test_dunedin_pace_normalization(): ), "Dataframes are not equal within the given tolerance." +@pytest.mark.parametrize( + "model_name, model_entry", + [ + (name, entry) + for name, entry in model.model_definitions.items() + if "sites_file" in entry["model"] + ], +) +def test_sites_file_exists(model_name, model_entry): + # The docs link to this file, so it must ship with the package + sites_file = model_entry["model"]["sites_file"] + assert os.path.isfile( + get_data_file(sites_file) + ), f"{model_name} sites_file '{sites_file}' not found in biolearn/data" + + # Run the test if __name__ == "__main__": pytest.main([__file__]) diff --git a/doc/generate.py b/doc/generate.py index f892e85..5596b57 100644 --- a/doc/generate.py +++ b/doc/generate.py @@ -4,6 +4,8 @@ import csv import os +DATA_URL = "https://github.com/bio-learn/biolearn/blob/master/biolearn/data" + def generate_models_csv(models, output_file="generated/model_table.csv"): header = ["Name", "Year", "Species", "Tissue", "Predicts", "Source", "Coefficients"] @@ -21,7 +23,14 @@ def generate_models_csv(models, output_file="generated/model_table.csv"): else: source_link = f"`paper <{details['source']}>`_" model_file = details.get('model', {}).get('file') - coefficients_link = f"`coefficients file `_" if model_file else "N/A" + # Models without a coefficients file (e.g. API models) may list their required sites instead + sites_file = details.get('model', {}).get('sites_file') + if model_file: + coefficients_link = f"`coefficients file <{DATA_URL}/{model_file}>`_" + elif sites_file: + coefficients_link = f"`required sites <{DATA_URL}/{sites_file}>`_" + else: + coefficients_link = "N/A" row = [ name,