fix(docs): link required sites for models without a coefficients file

biolearn/data_library.py

@@ -1029,7 +1029,15 @@ def _check_keys_exist(self, data, keys: list[str]):
 
     def _create_metadata(self, metadata_query_url):
         response = requests.get(metadata_query_url)
-        json_data = response.json()
+        response.raise_for_status()
+        try:
+            json_data = response.json()
+        except ValueError as e:
+            # NCBI serves an HTML CAPTCHA page when it blocks automated requests
+            raise ValueError(
+                f"GEO metadata query did not return JSON: {metadata_query_url}. "
+                "NCBI may be blocking automated requests."
+            ) from e
         smaples = [item for item in json_data["GeoMetaData"]]
         return self._convert_to_metadata_df(self.metadata_keys, smaples)
 

biolearn/model.py

@@ -1082,6 +1082,7 @@ def preprocess(df):
         "model": {
             "type": "HurdleAPIModel",
             "use_production": True,
+            "sites_file": "Hurdle_CpGs.csv",
             "default_imputation": "none",
         },
         "usage": {
@@ -1995,12 +1996,14 @@ class HurdleAPIModel:
     DEFAULT_TIMEOUT = 30  # seconds
     MAX_RETRIES = 3
     RETRY_DELAY = 1  # seconds
+    DEFAULT_SITES_FILE = "Hurdle_CpGs.csv"
 
     def __init__(
         self,
         api_key: Optional[str] = None,
         use_production: bool = False,
         timeout: Optional[int] = None,
+        sites_file: str = DEFAULT_SITES_FILE,
         **details: Any,
     ) -> None:
         self.details = details
@@ -2023,7 +2026,7 @@ def __init__(
 
         # Load required CpG sites
         try:
-            cpg_file = get_data_file("Hurdle_CpGs.csv")
+            cpg_file = get_data_file(sites_file)
             self.required_cpgs = pd.read_csv(cpg_file, encoding="ISO-8859-1")[
                 "ProbeID"
             ].tolist()
@@ -2040,6 +2043,7 @@ def from_definition(
         model_def = clock_definition["model"]
         return cls(
             use_production=model_def.get("use_production", False),
+            sites_file=model_def.get("sites_file", cls.DEFAULT_SITES_FILE),
             **{k: v for k, v in clock_definition.items() if k != "model"},
         )
 

biolearn/test/test_autoscan_geo_matrix_parser.py

@@ -1,6 +1,12 @@
+import pandas as pd
 import pytest
+from unittest.mock import Mock, patch
 
-from biolearn.data_library import DataLibrary, NoMatrixDataError
+from biolearn.data_library import (
+    AutoScanGeoMatrixParser,
+    DataLibrary,
+    NoMatrixDataError,
+)
 from biolearn.util import get_data_file
 
 
@@ -28,5 +34,29 @@ def test_series_has_no_matrix_data_error():
         library_file=get_data_file("geo_autoscan_library.yaml")
     )
 
-    with pytest.raises(NoMatrixDataError):
-        library.get("GSE121633").load()
+    # geo2r now answers scripted requests with a CAPTCHA page, so stub the
+    # metadata call and exercise only the matrix check under test
+    stub_metadata = pd.DataFrame({"id": ["GSE121633"]})
+    with patch.object(
+        AutoScanGeoMatrixParser, "_create_metadata", return_value=stub_metadata
+    ):
+        with pytest.raises(NoMatrixDataError):
+            library.get("GSE121633").load()
+
+
+def test_metadata_query_non_json_response_error():
+    parser = AutoScanGeoMatrixParser(
+        {
+            "matrix_file": "unused",
+            "metadata_keys_parse": {},
+            "metadata_query": "unused",
+        }
+    )
+    html_response = Mock()
+    html_response.json.side_effect = ValueError("Expecting value")
+
+    with patch(
+        "biolearn.data_library.requests.get", return_value=html_response
+    ):
+        with pytest.raises(ValueError, match="did not return JSON"):
+            parser._create_metadata("https://example.org/geo2r")

biolearn/test/test_hurdle_model.py

@@ -10,6 +10,7 @@
 
 from biolearn.model import HurdleAPIModel
 from biolearn.model_gallery import ModelGallery
+from biolearn.util import get_data_file
 
 
 class TestHurdleAPIModel:
@@ -156,6 +157,31 @@ def test_model_gallery_integration(self):
             model = gallery.get("HurdleInflammAge")
             assert isinstance(model, HurdleAPIModel)
 
+    def test_sites_loaded_from_definition_sites_file(self):
+        """Test that required sites come from the definition's sites_file."""
+        definition = ModelGallery().model_definitions["HurdleInflammAge"]
+        sites_file = definition["model"]["sites_file"]
+        expected = pd.read_csv(
+            get_data_file(sites_file), encoding="ISO-8859-1"
+        )["ProbeID"].tolist()
+
+        with patch.dict(os.environ, {"HURDLE_API_KEY": "test_key"}):
+            model = HurdleAPIModel.from_definition(definition)
+
+        assert len(expected) > 0
+        assert model.methylation_sites() == expected
+
+    def test_missing_sites_file_falls_back_to_all_sites(self):
+        """Test that an unknown sites_file warns and leaves sites unset."""
+        with pytest.warns(
+            UserWarning, match="Could not load Hurdle CpG sites"
+        ):
+            model = HurdleAPIModel(
+                api_key="test_key", sites_file="does_not_exist.csv"
+            )
+
+        assert model.methylation_sites() == []
+
     @patch("builtins.input", return_value="yes")
     def test_consent_only_asked_once(self, mock_input):
         """Test that consent is only requested once."""

biolearn/test/test_model.py

@@ -1,10 +1,12 @@
+import os
 import pytest
 from math import isclose
 import pandas as pd
 from collections import defaultdict
 import numpy as np
 from biolearn import model
 from biolearn.util import (
+    get_data_file,
     get_test_data_file,
     load_test_data_file,
 )
@@ -146,6 +148,22 @@ def test_dunedin_pace_normalization():
     ), "Dataframes are not equal within the given tolerance."
 
 
+@pytest.mark.parametrize(
+    "model_name, model_entry",
+    [
+        (name, entry)
+        for name, entry in model.model_definitions.items()
+        if "sites_file" in entry["model"]
+    ],
+)
+def test_sites_file_exists(model_name, model_entry):
+    # The docs link to this file, so it must ship with the package
+    sites_file = model_entry["model"]["sites_file"]
+    assert os.path.isfile(
+        get_data_file(sites_file)
+    ), f"{model_name} sites_file '{sites_file}' not found in biolearn/data"
+
+
 # Run the test
 if __name__ == "__main__":
     pytest.main([__file__])

doc/generate.py

@@ -4,6 +4,8 @@
 import csv
 import os
 
+DATA_URL = "https://github.com/bio-learn/biolearn/blob/master/biolearn/data"
+
 def generate_models_csv(models, output_file="generated/model_table.csv"):
     header = ["Name", "Year", "Species", "Tissue", "Predicts", "Source", "Coefficients"]
 
@@ -21,7 +23,14 @@ def generate_models_csv(models, output_file="generated/model_table.csv"):
             else:
                 source_link = f"`paper <{details['source']}>`_"
             model_file = details.get('model', {}).get('file')
-            coefficients_link = f"`coefficients file <https://github.com/bio-learn/biolearn/blob/master/biolearn/data/{model_file}>`_" if model_file else "N/A"
+            # Models without a coefficients file (e.g. API models) may list their required sites instead
+            sites_file = details.get('model', {}).get('sites_file')
+            if model_file:
+                coefficients_link = f"`coefficients file <{DATA_URL}/{model_file}>`_"
+            elif sites_file:
+                coefficients_link = f"`required sites <{DATA_URL}/{sites_file}>`_"
+            else:
+                coefficients_link = "N/A"
 
             row = [
                 name,