From 66e2149a6aeea8086f9fb67af2167ab1a242db5c Mon Sep 17 00:00:00 2001 From: Adam Fourney Date: Fri, 11 Sep 2026 12:17:52 -0700 Subject: [PATCH] Improve efficiency of escaping pipes in CSVs --- .../src/markitdown/converters/_csv_converter.py | 3 ++- packages/markitdown/tests/test_module_misc.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 1aee0ffb8..331b0712a 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -15,7 +15,8 @@ # Matches a pipe together with the (possibly empty) run of backslashes in front # of it, so that run can be doubled before the pipe is escaped. -_PIPE_ESCAPE_RE = re.compile(r"(\\*)\|") +# The lookbehind avoids retrying from each position inside a backslash run. +_PIPE_ESCAPE_RE = re.compile(r"(? str: diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 84035c4b4..bc0312460 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -1800,6 +1800,20 @@ def test_csv_backslash_without_a_pipe_is_left_alone() -> None: assert r"| Widget | C:\temp\file.txt |" in result +@pytest.mark.parametrize( + "suffix,escaped_suffix", + [("", ""), ("|", r"\|"), ("x|", r"x\|")], +) +def test_csv_long_backslash_runs(suffix: str, escaped_suffix: str) -> None: + backslashes = "\\" * 65_536 + value = backslashes + suffix + expected = backslashes * (2 if suffix == "|" else 1) + escaped_suffix + + result = _convert_csv(f"{value}\n{value}\n".encode("utf-8"), charset="utf-8") + + assert result == f"| {expected} |\n| --- |\n| {expected} |" + + # --------------------------------------------------------------------------- # Regression test for issue #1960: # exiftool_path pointing to a nonexistent binary used to leak a raw