diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 59020d4..e37ed34 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,7 +38,15 @@ jobs: strategy: fail-fast: true matrix: - python-version: ["3.10", "3.11", "3.12"] + include: + - python-version: "3.10" + pandas: "pandas>=2.2.2,<3" + - python-version: "3.11" + pandas: "pandas>=2.2.2,<3" + - python-version: "3.11" + pandas: "pandas>=3,<4" + - python-version: "3.12" + pandas: "pandas>=3,<4" steps: - uses: actions/checkout@v7 - name: Set up Python ${{ matrix.python-version }} @@ -74,10 +82,14 @@ jobs: echo `which netMHCpan` && netMHCpan -h echo `which netMHCcons` && netMHCcons -h - name: Install dependencies + env: + PANDAS_REQUIREMENT: ${{ matrix.pandas }} run: | python -m pip install --upgrade pip python -m pip install pytest pytest-cov ruff - bash scripts/pip_install.sh -e . + bash scripts/pip_install.sh -e . "$PANDAS_REQUIREMENT" + python -m pip check + python -m pip show pandas pyensembl gtfparse - name: Resolve the installed default Ensembl release id: ensembl-release run: | @@ -166,7 +178,7 @@ jobs: - name: Save coverage for the upload job uses: actions/upload-artifact@v7 with: - name: coverage-python-${{ matrix.python-version }} + name: coverage-python-${{ matrix.python-version }}-${{ strategy.job-index }} path: .coverage include-hidden-files: true if-no-files-found: error diff --git a/CHANGELOG.md b/CHANGELOG.md index 33b2c16..229f551 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ between published tags; older pre-5.0 notes are retained below. For current interfaces, see the [consumer guide](docs/consumer-guide.md). +## 5.94.2 + +- Require gtfparse 3.0.2+ and PyEnsembl 2.24.1+, allowing the current pandas 3 + annotation stack alongside Varcode (#484). Retain pandas 2.2.2+ for Python + 3.10 and explicitly test both pandas 2 and 3 on Python 3.11. +- Select pandas output in both GTF expression readers and verify their + transcript FPKM values against the same StringTie fixture. + ## 5.94.1 - Apply explicit Cufflinks HIDATA expression replacements under pandas diff --git a/requirements.txt b/requirements.txt index 67b36f6..b2414f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ numpy>=2.0.0,<3.0.0 -pandas>=2.0.0 +pandas>=2.2.2,<4.0.0 mhctools>=3.45.1 varcode>=4.18.0 -gtfparse>=2.8.1,<3.0.0 +gtfparse>=3.0.2,<4.0.0 mhcgnomes>=3.4.0 -pyensembl>=2.3.0,<3.0.0 +pyensembl>=2.24.1,<3.0.0 # PEP 440 versions and validated package names for release preflight packaging>=23.2 argcomplete>=1.10 diff --git a/tests/test_load_stringtie_gtf_fpkm.py b/tests/test_load_stringtie_gtf_fpkm.py index d88e35c..aeaab6c 100644 --- a/tests/test_load_stringtie_gtf_fpkm.py +++ b/tests/test_load_stringtie_gtf_fpkm.py @@ -1,11 +1,15 @@ -from topiary.rna import load_transcript_fpkm_dict_from_gtf +"""Both GTF readers preserve known abundances with the pandas parser backend.""" + +import pytest from .common import eq_ from .data import data_path +from .test_twin_conformance import GTF_FPKM_TWINS -def test_load_stringtie_gtf_transcripts(): - transcript_fpkms = load_transcript_fpkm_dict_from_gtf( +@pytest.mark.parametrize("name,reader", GTF_FPKM_TWINS, ids=[name for name, _ in GTF_FPKM_TWINS]) +def test_load_stringtie_gtf_transcripts(name, reader): + transcript_fpkms = reader( data_path("B16-StringTie-chr1-subset.gtf") ) transcript_ids = set(transcript_fpkms.keys()) @@ -18,27 +22,3 @@ def test_load_stringtie_gtf_transcripts(): eq_(expected_transcript_ids, transcript_ids) for transcript_id, fpkm in expected_fpkms_dict.items(): eq_(fpkm, transcript_fpkms[transcript_id]) - - -def test_load_expression_reads_a_gtf_as_pandas(): - """The other GTF door, which had no test and was broken. - - ``topiary.rna.gtf`` and ``expression_loader._load_gtf`` both read - GTFs, and only the first was covered. gtfparse 2.x returns polars by - default; ``_load_gtf`` treats the result as pandas, so every GTF - read through ``load_expression`` (and through the CLI's - ``--transcript-expression``) raised "expected 17 values when - selecting columns by boolean mask" from polars, while this file's - other test kept passing. - """ - import pandas as pd - - from topiary.rna.expression_loader import load_expression - - df = load_expression(data_path("B16-StringTie-chr1-subset.gtf")) - - assert isinstance(df, pd.DataFrame) - assert not df.empty - # Transcript rows only, carrying the abundance column. - assert "reference_id" in df.columns - assert "TPM" in df.columns or "FPKM" in df.columns diff --git a/tests/test_twin_conformance.py b/tests/test_twin_conformance.py index bfd0d4a..9263521 100644 --- a/tests/test_twin_conformance.py +++ b/tests/test_twin_conformance.py @@ -41,6 +41,8 @@ attach_rna_evidence, ) from topiary.rna import load_cufflinks_dataframe, load_cufflinks_dict, load_cufflinks_fpkm_dict +from topiary.rna import load_transcript_fpkm_dict_from_gtf +from topiary.rna.expression_loader import load_expression from topiary import ( APPROXIMATED, MEASURED, CachedPredictor, ProteinFragment, TopiaryPredictor, from_predictions, fragments_from_variants, read_fragments, read_pvacseq, write_fragments, unique_fragments, @@ -125,6 +127,14 @@ def calls(self, arg, value): ) +# Both public StringTie readers must retain the same transcript FPKM values. +GTF_FPKM_TWINS = ( + ("values", load_transcript_fpkm_dict_from_gtf), + ("dataframe", lambda path: load_expression( + path, val_cols="FPKM").set_index("reference_id")["FPKM"].to_dict()), +) + + # Drive both input forms through the same hypothesis validation battery. ISOVAR_HYPOTHESIS_INPUT_TWINS = ( ("mapping", lambda export, path: read_isovar_hypotheses(export, tag="hypotheses")), diff --git a/topiary/__init__.py b/topiary/__init__.py index ce36236..c8998eb 100644 --- a/topiary/__init__.py +++ b/topiary/__init__.py @@ -208,7 +208,7 @@ encode_amino_acids, ) -__version__ = "5.94.1" +__version__ = "5.94.2" __all__ = [ "normalize_isovar_rna_support", diff --git a/topiary/rna/expression_loader.py b/topiary/rna/expression_loader.py index 6ba7988..d11b836 100644 --- a/topiary/rna/expression_loader.py +++ b/topiary/rna/expression_loader.py @@ -139,11 +139,8 @@ def _load_gtf(filepath, id_col=None, val_cols=None): """Load expression data from a StringTie GTF file.""" import gtfparse - # result_type is required, not cosmetic: gtfparse 2.x defaults to - # returning polars, and everything below treats df as pandas. Without - # it, reading any GTF raises "expected 17 values when selecting - # columns by boolean mask" from polars on the next line. The keyword - # exists only from gtfparse 2.7, which requirements.txt now floors. + # Keep the backend explicit: filtering and numeric conversion below + # use pandas, which is also gtfparse 3's default result type. df = gtfparse.read_gtf(filepath, result_type="pandas") # Filter to transcript features only if "feature" in df.columns: diff --git a/topiary/rna/gtf.py b/topiary/rna/gtf.py index 6907ef7..d22b20e 100644 --- a/topiary/rna/gtf.py +++ b/topiary/rna/gtf.py @@ -41,7 +41,8 @@ def load_transcript_fpkm_dict_from_gtf( quantification of abundance. Returns a dictionary mapping Ensembl IDs of transcripts to FPKM values. """ - df = gtfparse.read_gtf(gtf_path, column_converters={fpkm_column_name: float}) + df = gtfparse.read_gtf( + gtf_path, column_converters={fpkm_column_name: float}, result_type="pandas") transcript_ids = _get_gtf_column(transcript_id_column_name, gtf_path, df) fpkm_values = _get_gtf_column(fpkm_column_name, gtf_path, df) features = _get_gtf_column(feature_column_name, gtf_path, df)