Add GTF parsing progress and fix attribute data loss (2.9.0)

README.md

@@ -8,6 +8,53 @@ gtfparse
 ========
 Parsing tools for GTF (gene transfer format) files.
 
+## Reporting parsing progress
+
+Pass `progress_callback(stage, completed, total)` to `read_gtf` to connect
+parsing to your own progress UI. The callback runs synchronously; exceptions
+from it stop parsing and propagate to the caller. Without a callback, gtfparse
+does not display a progress bar or add a progress-library dependency.
+
+| Stage | Reports |
+| --- | --- |
+| `read` | `(0, None)` before Polars loads/filters the file; `(rows, rows)` after it succeeds |
+| `attributes` | `(0, rows)`, then every 10,000 rows, then `(rows, rows)` |
+| `convert` | `(0, None)` before output conversion/transforms; `(rows, rows)` after success |
+
+Counts refer to rows retained after `features` filtering, not bytes or overall
+percentages. `None` means the total is unknown: loading and conversion expose
+only stage boundaries, so use an indeterminate indicator during those steps.
+Attribute expansion is omitted when `expand_attribute_column=False`. A stage
+that fails does not emit completion. An empty filtered result is supported;
+its attributes stage emits a single `(0, 0)` event. Empty input files retain
+the existing Polars `NoDataError` behavior.
+
+For example, with the optional `tqdm` package installed:
+
+```python
+from gtfparse import read_gtf
+from tqdm.auto import tqdm
+
+with tqdm(unit="rows") as bar:
+    def show_progress(stage, completed, total):
+        if stage != bar.desc:
+            bar.total = total
+            bar.reset()
+            bar.set_description_str(stage)
+        bar.total = total
+        bar.update(completed - bar.n)
+        bar.refresh()
+
+    df = read_gtf("gene_annotations.gtf", progress_callback=show_progress)
+```
+
+`parse_gtf` reports the `read` stage, `parse_gtf_and_expand_attributes` reports
+`read` and `attributes`, `parse_gtf_pandas` reports `read` and `convert`, and
+`expand_attribute_strings` reports only `attributes` using the same callback.
+
+Quoted attribute values preserve spaces, semicolons, and apostrophes. Raw
+attributes (`expand_attribute_column=False`) retain their original quote marks.
+
 # Example usage
 
 ## Parsing all rows of a GTF file into a Pandas DataFrame
@@ -41,5 +88,3 @@ gene_fpkms = {
     if feature == "gene"
 }
 ```
-
-

gtfparse/__init__.py

@@ -24,7 +24,7 @@
 )
 from .write_gtf import write_gtf
 
-__version__ = "2.8.1"
+__version__ = "2.9.0"
 
 __all__ = [
     "GENCODE_BIOTYPE_ALIASES",

gtfparse/attribute_parsing.py

@@ -11,25 +11,37 @@
 # limitations under the License.
 
 import logging
+import re
 from collections import OrderedDict
 from sys import intern
 
 logger = logging.getLogger(__name__)
 
+# Quoted values may contain semicolons. Keep this compatible with both Python's
+# re module and Polars' regex engine so the low-level split column agrees with
+# attribute expansion. Unquoted values and single quotes are accepted as well.
+ATTRIBUTE_PATTERN = r"""[^\s;]+[ \t]+(?:"[^"]*"|'[^']*'|[^;]+)"""
+_ATTRIBUTE_PAIRS = re.compile(ATTRIBUTE_PATTERN)
 
-def expand_attribute_strings(attribute_strings, quote_char="'", missing_value="", usecols=None):
+
+def expand_attribute_strings(
+    attribute_strings, quote_char="'", missing_value="", usecols=None, *, progress_callback=None
+):
     """
     The last column of GTF has a variable number of key value pairs
     of the format: "key1 value1; key2 value2;"
     Parse these into a dictionary mapping each key onto a list of values,
-    where the value is None for any row where the key was missing.
+    using missing_value for any row where the key was missing.
 
     Parameters
     ----------
-    attribute_strings : list of str
+    attribute_strings : sequence of str or sequences of str
+        Raw attribute fields or already separated key/value pairs per row.
+        None represents a row with no attributes.
 
     quote_char : str
-        Quote character to remove from values
+        Additional surrounding quote character to remove from values.
+        Standard double quotes are always recognized.
 
     missing_value : any
         If an attribute is missing from a row, give it this value.
@@ -38,6 +50,11 @@ def expand_attribute_strings(attribute_strings, quote_char="'", missing_value=""
         If not None, then only expand columns included in this set,
         otherwise use all columns.
 
+    progress_callback : callable, optional
+        Called as ``callback("attributes", completed_rows, total_rows)`` at
+        the start, every 10,000 rows, and at completion. An empty input emits
+        one ``("attributes", 0, 0)`` event. Callback exceptions propagate.
+
     Returns OrderedDict of column->value list mappings, in the order they
     appeared in the attribute strings.
     """
@@ -46,29 +63,23 @@ def expand_attribute_strings(attribute_strings, quote_char="'", missing_value=""
     extra_columns = {}
     column_order = []
 
-    #
-    # SOME NOTES ABOUT THE BIZARRE STRING INTERNING GOING ON BELOW
-    #
     # While parsing millions of repeated strings (e.g. "gene_id" and "TP53"),
     # we can save a lot of memory by making sure there's only one string
-    # object per unique string. The canonical way to do this is using
-    # the 'intern' function. One problem is that Py2 won't let you intern
-    # unicode objects, so to get around this we call intern(str(...)).
-    #
-    # It also turns out to be faster to check interned strings ourselves
-    # using a local dictionary, hence the two dictionaries below
-    # and pair of try/except blocks in the loop.
+    # object per unique column name. Cache interned names locally as well.
     column_interned_strings = {}
 
+    if progress_callback is not None:
+        progress_callback("attributes", 0, n)
+
     for i, kv_strings in enumerate(attribute_strings):
-        if type(kv_strings) is str:
-            kv_strings = kv_strings.split(";")
+        if isinstance(kv_strings, str):
+            kv_strings = _ATTRIBUTE_PAIRS.findall(kv_strings)
+        elif kv_strings is None:
+            kv_strings = ()
         for kv in kv_strings:
-            # We're slicing the first two elements out of split() because
-            # Ensembl release 79 added values like:
-            #   transcript_support_level "1 (assigned to previous version 5)";
-            # ...which gets mangled by splitting on spaces.
-            parts = kv.strip().split(" ", 2)[:2]
+            # Split once: values such as transcript_support_level may include
+            # spaces, and whitespace between a key and its value may vary.
+            parts = kv.strip().split(None, 1)
 
             if len(parts) != 2:
                 continue
@@ -78,14 +89,14 @@ def expand_attribute_strings(attribute_strings, quote_char="'", missing_value=""
             try:
                 column_name = column_interned_strings[column_name]
             except KeyError:
-                column_name = intern(str(column_name))
+                column_name = intern(column_name)
                 column_interned_strings[column_name] = column_name
 
             if usecols is not None and column_name not in usecols:
                 continue
 
-            if value[0] == quote_char:
-                value = value.replace(quote_char, "")
+            if len(value) >= 2 and value[0] in ('"', quote_char) and value[-1] == value[0]:
+                value = value[1:-1]
 
             try:
                 column = extra_columns[column_name]
@@ -102,5 +113,9 @@ def expand_attribute_strings(attribute_strings, quote_char="'", missing_value=""
                 extra_columns[column_name] = column
                 column_order.append(column_name)
 
+        completed = i + 1
+        if progress_callback is not None and (completed % 10_000 == 0 or completed == n):
+            progress_callback("attributes", completed, n)
+
     logger.info("Extracted GTF attributes: %s", column_order)
     return OrderedDict((column_name, extra_columns[column_name]) for column_name in column_order)