Trim CSV blank runs without repeatedly shifting the row list

packages/markitdown/src/markitdown/converters/_csv_converter.py

@@ -30,17 +30,27 @@ def _escape_table_cell(value: str) -> str:
 
 def _trim_outer_blank_rows(rows: list[list[str]]) -> None:
     """Remove empty rows from the beginning and end, and immediately after the header. This operation is performed in-place."""
-    # Pop empty rows from the beginning
-    while len(rows) > 0 and not rows[0]:
-        rows.pop(0)
-
-    # Pop empty rows after the header
-    while len(rows) > 1 and not rows[1]:
-        rows.pop(1)
-
-    # Pop empty rows from the end
-    while len(rows) > 0 and not rows[-1]:
-        rows.pop(-1)
+    start = 0
+    while start < len(rows) and not rows[start]:
+        start += 1
+
+    if start == len(rows):
+        rows.clear()
+        return
+
+    header_index = start
+    start += 1
+    while start < len(rows) and not rows[start]:
+        start += 1
+
+    end = len(rows)
+    while end > start and not rows[end - 1]:
+        end -= 1
+
+    # Remove each blank run at once, rather than shifting the list per row.
+    del rows[end:]
+    del rows[header_index + 1 : start]
+    del rows[:header_index]
 
 
 class CsvConverter(DocumentConverter):

packages/markitdown/tests/test_csv_blank_runs.py

@@ -0,0 +1,35 @@
+"""CSV conversion preserves table contents when trimming long blank runs."""
+
+import io
+
+import pytest
+
+from markitdown import MarkItDown, StreamInfo
+
+
+@pytest.mark.parametrize("position", ["leading", "after_header", "trailing", "all"])
+def test_csv_long_blank_runs(position: str) -> None:
+    blank = b"\n" * 100_000
+    header = b"name,value\n"
+    # An internal blank row and a wider data row must survive trimming.
+    data = b"Alice,1\n\nBob,2,extra\n"
+    content = {
+        "leading": blank + header + data,
+        "after_header": header + blank + data,
+        "trailing": header + data + blank,
+        "all": blank,
+    }[position]
+
+    result = MarkItDown(enable_plugins=False).convert_stream(
+        io.BytesIO(content),
+        stream_info=StreamInfo(extension=".csv", charset="utf-8"),
+    )
+
+    expected = (
+        "| name | value |  |\n"
+        "| --- | --- | --- |\n"
+        "| Alice | 1 |  |\n"
+        "|  |  |  |\n"
+        "| Bob | 2 | extra |"
+    )
+    assert result.markdown == ("" if position == "all" else expected)