Remove required Polars dependency and default to pandas (3.0.0)

.github/workflows/tests.yml

@@ -4,6 +4,10 @@
 name: Tests
 on: [push, pull_request]
 
+env:
+  # Reruns share run_id; each attempt needs its own parallel coverage report.
+  COVERALLS_SERVICE_NUMBER: ${{ github.run_id }}-${{ github.run_attempt }}
+
 jobs:
   packaging:
     runs-on: ubuntu-latest
@@ -40,13 +44,43 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
-        polars-version: ["latest"]
         include:
+          - python-version: "3.9"
+            pandas-version: "minimum"
+            pyarrow-version: "minimum"
+            polars-version: "none"
+          - python-version: "3.10"
+            pandas-version: "latest"
+            pyarrow-version: "latest"
+            polars-version: "none"
+          - python-version: "3.11"
+            pandas-version: "latest"
+            pyarrow-version: "latest"
+            polars-version: "none"
+          - python-version: "3.11"
+            pandas-version: "latest"
+            pyarrow-version: "minimum"
+            polars-version: "none"
+          - python-version: "3.11"
+            pandas-version: "2.3.3"
+            pyarrow-version: "latest"
+            polars-version: "none"
           - python-version: "3.11"
+            pandas-version: "latest"
+            pyarrow-version: "latest"
+            polars-version: "minimum"
+          - python-version: "3.11"
+            pandas-version: "latest"
+            pyarrow-version: "latest"
             polars-version: "1.31.0"
           - python-version: "3.11"
+            pandas-version: "latest"
+            pyarrow-version: "latest"
             polars-version: "1.32.0"
+          - python-version: "3.11"
+            pandas-version: "latest"
+            pyarrow-version: "latest"
+            polars-version: "latest"
 
     steps:
       - name: Checkout repository
@@ -61,9 +95,36 @@ jobs:
           python -m pip install ruff pytest pytest-cov coveralls
           pip install -r requirements.txt
           pip install .
-      - name: Install compatibility Polars version
-        if: matrix.polars-version != 'latest'
-        run: python -m pip install 'polars==${{ matrix.polars-version }}'
+      - name: Install compatibility dependency versions
+        env:
+          PANDAS_VERSION: ${{ matrix.pandas-version }}
+          PYARROW_VERSION: ${{ matrix.pyarrow-version }}
+          POLARS_VERSION: ${{ matrix.polars-version }}
+        run: |
+          python - <<'PY'
+          import os
+          import subprocess
+          import sys
+          from importlib.metadata import requires
+          from packaging.requirements import Requirement
+
+          requirements = [Requirement(text) for text in requires("gtfparse")]
+          pins = []
+          for name in ("pandas", "pyarrow", "polars"):
+              version = os.environ[f"{name.upper()}_VERSION"]
+              if version == "none":
+                  continue
+              if version == "minimum":
+                  requirement = next(r for r in requirements if r.name == name)
+                  version = next(s.version for s in requirement.specifier if s.operator == ">=")
+              pins.append(name if version == "latest" else f"{name}=={version}")
+          subprocess.run([sys.executable, "-m", "pip", "install", *pins], check=True)
+          PY
+      - name: Verify base installation has no Polars
+        if: matrix.polars-version == 'none'
+        run: python -c "from importlib.util import find_spec; assert find_spec('polars') is None"
+      - name: Check installed dependency requirements
+        run: python -m pip check
       - name: Run default linting script
         run: |
           ./lint.sh
@@ -74,7 +135,7 @@ jobs:
         if: matrix.python-version == '3.11'
         uses: coverallsapp/github-action@v2.2.3
         with:
-          flag-name: polars-${{ matrix.polars-version }}
+          flag-name: pandas-${{ matrix.pandas-version }}-arrow-${{ matrix.pyarrow-version }}-polars-${{ matrix.polars-version }}
           parallel: true
 
   coverage:

MANIFEST.in

@@ -1,3 +1,4 @@
 include lint.sh test.sh
 recursive-include tests *.py
 recursive-include tests/data *.gtf *.gtf.gz
+recursive-include benchmarks *.py *.md *.json

README.md

@@ -8,6 +8,41 @@ gtfparse
 ========
 Parsing tools for GTF (gene transfer format) files.
 
+## Version 3 migration
+
+`read_gtf`, `parse_gtf`, and `parse_gtf_and_expand_attributes` now return pandas
+DataFrames by default. GTF reading uses PyArrow; attribute processing and writing
+use pandas. Polars is no longer a required dependency and is imported only for
+explicit Polars conversion. Ordinary installations need only pandas and PyArrow.
+
+```sh
+pip install gtfparse
+# Optional, for applications that still request Polars frames:
+pip install 'gtfparse[polars]'
+```
+
+```python
+from gtfparse import read_gtf, write_gtf
+
+df = read_gtf("gene_annotations.gtf")  # pandas.DataFrame
+write_gtf(df, "gene_annotations.gtf.gz")
+
+# Requires the optional extra:
+polars_df = read_gtf("gene_annotations.gtf", result_type="polars")
+```
+
+Existing pandas and dictionary callers retain their output shape, fixed-column
+dtypes, attribute handling, filters, converters and progress callbacks. Applications
+that used the previous default Polars output should add `result_type="polars"`
+and install the extra, or use pandas methods. `write_gtf` accepts either frame
+type. The legacy `parse_with_polars_lazy` helper remains an optional adapter:
+it reads the file eagerly and returns a Polars LazyFrame, as before.
+
+Empty or malformed GTF fields now raise `gtfparse.ParsingError`, including wrong
+field counts and invalid numeric fields. Whole comment lines are ignored;
+literal `#` inside attributes is preserved. Text and binary streams stay open
+after parsing, and gzip is detected from its contents.
+
 ## Reporting parsing progress
 
 Pass `progress_callback(stage, completed, total)` to `read_gtf` to connect
@@ -17,7 +52,7 @@ does not display a progress bar or add a progress-library dependency.
 
 | Stage | Reports |
 | --- | --- |
-| `read` | `(0, None)` before Polars loads/filters the file; `(rows, rows)` after it succeeds |
+| `read` | `(0, None)` before loading/filtering the file; `(rows, rows)` after it succeeds |
 | `attributes` | `(0, rows)`, then every 10,000 rows, then `(rows, rows)` |
 | `convert` | `(0, None)` before output conversion/transforms; `(rows, rows)` after success |
 
@@ -26,8 +61,7 @@ percentages. `None` means the total is unknown: loading and conversion expose
 only stage boundaries, so use an indeterminate indicator during those steps.
 Attribute expansion is omitted when `expand_attribute_column=False`. A stage
 that fails does not emit completion. An empty filtered result is supported;
-its attributes stage emits a single `(0, 0)` event. Empty input files retain
-the existing Polars `NoDataError` behavior.
+its attributes stage emits a single `(0, 0)` event. Empty input files raise `gtfparse.ParsingError`.
 
 For example, with the optional `tqdm` package installed:
 

benchmarks/README.md

@@ -0,0 +1,110 @@
+# GTF reader backend comparison
+
+Version 3 uses PyArrow for fixed-column parsing and pandas for attribute
+processing and writing. Polars is an optional output adapter. This directory
+contains the measurement harness and the evidence for that decision.
+
+## Reproduction
+
+The recorded runs use Python 3.12.6, NumPy 2.5.3, PyArrow 25.0.1, Polars 1.44.2,
+and pandas 2.3.3 or 3.0.6. Install the checkout with `pip install '.[dev,polars]'`
+in isolated environments. Run one fresh process at a time with four threads:
+
+```sh
+POLARS_MAX_THREADS=4 python benchmarks/compare_readers.py sample.gtf --backend production
+POLARS_MAX_THREADS=4 python benchmarks/compare_readers.py sample.gtf --backend pandas
+POLARS_MAX_THREADS=4 python benchmarks/compare_readers.py sample.gtf --backend arrow
+POLARS_MAX_THREADS=4 python benchmarks/compare_readers.py sample.gtf --backend arrow-extension
+```
+
+For the old production baseline, extract commit `3e0912c` (gtfparse 2.9.1) into
+a separate directory and use the same harness with `--library-root /path/to/baseline`.
+Repeat each combination three times, interleaving readers and reversing their
+order for the second repetition. Imports and validation are outside wall/CPU
+timing. Peak RSS is the process high-water mark, including imports, captured
+before checksumming or constructing a reference frame. OS caches are not flushed.
+The shared workstation and short runs limit precise throughput claims.
+
+Each invocation emits JSON with wall/CPU time, stage timings, pre-validation
+peak RSS, versions, dimensions, column order and a content checksum. Hashing
+normalizes equivalent string/categorical representations and ordinary numeric
+representations. The corpus coordinates are below float64's exact-integer limit;
+separate regression tests verify exact parsing of larger int64 coordinates.
+Use `--verify` for DataFrame equality against production on small inputs.
+`--features exon`, `--usecols seqname start end gene_id transcript_id`, and
+`--result-type polars` exercise selective reads and optional conversion.
+
+## Corpus and equivalence
+
+- [Ensembl release 114, human GRCh38](https://ftp.ensembl.org/pub/release-114/gtf/homo_sapiens/Homo_sapiens.GRCh38.114.gtf.gz)
+- [GENCODE v48, human primary assembly](https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_48/gencode.v48.primary_assembly.annotation.gtf.gz)
+
+The repeated sweep uses the first 250,000 annotation rows of each source with
+its original header, in plain and gzip form. The full GENCODE gzip file provides
+a separate scaling check on 4,119,244 rows. Original GTFs are not committed.
+Input byte sizes, SHA-256 hashes, source URLs and selection rules are in
+[results.json](results.json), alongside all 122 retained measurement records,
+three-run medians/ranges and stage medians. Discarded production measurements
+from before the final filtering changes are excluded.
+
+All measured readers produce matching hashes, including the full-file pair.
+The five real repository fixtures also pass 50 old/new API comparisons and
+30 prototype comparisons on each pandas version (160 checks). The API checks
+include raw attributes, filtering, selection, aliases, version casts, biotype
+inference and converters; separate regressions cover streams, nullable fields,
+comments, malformed inputs and progress/cancellation semantics.
+
+## Production results
+
+The table shows three-run medians for equivalent pandas output. Complete load
+time includes fixed-column reading, attribute expansion, assembly and transforms.
+
+| pandas | Input (250k rows) | Old seconds | New seconds | Old peak MiB | New peak MiB |
+| --- | --- | ---: | ---: | ---: | ---: |
+| 2.3.3 | ensembl-250k.gtf | 2.51 | 2.75 | 560 | 614 |
+| 2.3.3 | ensembl-250k.gtf.gz | 2.54 | 2.79 | 561 | 613 |
+| 2.3.3 | gencode-250k.gtf | 2.08 | 2.42 | 567 | 581 |
+| 2.3.3 | gencode-250k.gtf.gz | 2.16 | 2.42 | 568 | 581 |
+| 3.0.6 | ensembl-250k.gtf | 2.28 | 2.69 | 552 | 601 |
+| 3.0.6 | ensembl-250k.gtf.gz | 2.27 | 2.69 | 553 | 603 |
+| 3.0.6 | gencode-250k.gtf | 2.01 | 2.41 | 560 | 597 |
+| 3.0.6 | gencode-250k.gtf.gz | 2.03 | 2.44 | 560 | 592 |
+
+Across these subsets, the new reader takes 1.10–1.20x the old total time and
+1.03–1.10x the peak RSS. The single full GENCODE check with pandas 2.3.3 takes
+38.88s / 6189 MiB for the baseline and 45.40s / 5348 MiB for version 3: matching
+content, 1.17x time, and lower peak memory. This pair verifies scaling and
+correctness; one observation per reader does not establish stable throughput.
+
+The complete-load result meets #69's proposed 1.5x total-time threshold. The
+callback `read` stages are different contracts for timing: version 3 includes
+stream preprocessing and Arrow-to-pandas conversion; the old reader constructs
+Polars frames and converts to pandas later. The reported read-stage times do
+not establish the proposed 3x raw-CSV threshold. This migration is justified by
+the measured complete-load cost and the user's preference to remove the required
+Polars dependency, rather than a claim that Arrow is faster at raw CSV parsing.
+Python attribute expansion remains the largest stage (roughly 1.7–1.8s on these
+subsets). Eliminating it or changing its semantics is outside this change.
+
+## Candidate evaluation and limits
+
+The pandas C prototype is slower end to end on this corpus. The direct Arrow
+and Arrow-extension prototypes have similar complete-load costs; extension
+scalars introduce iteration/conversion overhead, and nullable dictionary casts
+need an object bridge on pandas 2.3.3. These costs are included in the results.
+The conventional Arrow-to-pandas path avoids that bridge and preserves the
+established fixed-column and attribute output types.
+
+The three prototypes are measurement tools. They support the paths, gzip,
+filters, column selection and result types used here, but not the entire public
+API. They omit buffers, custom converters, aliases, biotype inference, quote
+cleanup, raw mode and complete progress/failure behavior. The pandas prototype
+skips leading comments; the Arrow prototypes skip comment rows only on a field
+count mismatch. The production reader implements whole-line comment filtering,
+BOM and stream handling, exact field counts, declared numeric types and filtering
+before pandas conversion so discarded null coordinates do not change dtypes.
+
+Pandas reader behavior is documented in
+[read_csv](https://pandas.pydata.org/docs/reference/api/pandas.read_csv.html),
+and Arrow's typed CSV options in its
+[CSV guide](https://arrow.apache.org/docs/python/csv.html).