diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 82fd29747..1aee0ffb8 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -30,17 +30,27 @@ def _escape_table_cell(value: str) -> str: def _trim_outer_blank_rows(rows: list[list[str]]) -> None: """Remove empty rows from the beginning and end, and immediately after the header. This operation is performed in-place.""" - # Pop empty rows from the beginning - while len(rows) > 0 and not rows[0]: - rows.pop(0) - - # Pop empty rows after the header - while len(rows) > 1 and not rows[1]: - rows.pop(1) - - # Pop empty rows from the end - while len(rows) > 0 and not rows[-1]: - rows.pop(-1) + start = 0 + while start < len(rows) and not rows[start]: + start += 1 + + if start == len(rows): + rows.clear() + return + + header_index = start + start += 1 + while start < len(rows) and not rows[start]: + start += 1 + + end = len(rows) + while end > start and not rows[end - 1]: + end -= 1 + + # Remove each blank run at once, rather than shifting the list per row. + del rows[end:] + del rows[header_index + 1 : start] + del rows[:header_index] class CsvConverter(DocumentConverter): diff --git a/packages/markitdown/tests/test_csv_blank_runs.py b/packages/markitdown/tests/test_csv_blank_runs.py new file mode 100644 index 000000000..357af591e --- /dev/null +++ b/packages/markitdown/tests/test_csv_blank_runs.py @@ -0,0 +1,35 @@ +"""CSV conversion preserves table contents when trimming long blank runs.""" + +import io + +import pytest + +from markitdown import MarkItDown, StreamInfo + + +@pytest.mark.parametrize("position", ["leading", "after_header", "trailing", "all"]) +def test_csv_long_blank_runs(position: str) -> None: + blank = b"\n" * 100_000 + header = b"name,value\n" + # An internal blank row and a wider data row must survive trimming. + data = b"Alice,1\n\nBob,2,extra\n" + content = { + "leading": blank + header + data, + "after_header": header + blank + data, + "trailing": header + data + blank, + "all": blank, + }[position] + + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(content), + stream_info=StreamInfo(extension=".csv", charset="utf-8"), + ) + + expected = ( + "| name | value | |\n" + "| --- | --- | --- |\n" + "| Alice | 1 | |\n" + "| | | |\n" + "| Bob | 2 | extra |" + ) + assert result.markdown == ("" if position == "all" else expected)