Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions packages/markitdown/src/markitdown/converters/_markdownify.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,21 @@

_PERCENT_ENCODED_OCTET = re.compile(r"%[0-9A-Fa-f]{2}")

# Matches a pipe together with the (possibly empty) run of backslashes in
# front of it, anchored so the run starts after a non-backslash (or at the
# start of the string). The run therefore holds *every* backslash in front
# of the pipe, which makes the substitution parity-correct: a pipe preceded
# by an even run (including zero) is unescaped and gains one backslash,
# while an already-escaped pipe (odd run) matches nothing and is left alone.
# Idempotency matters because some converters (e.g. XLSX via _escape_sheet,
# PPTX via html-escaped cells) pre-escape pipes before the HTML reaches us.
_TABLE_PIPE_RE = re.compile(r"(?<!\\)((?:\\\\)*)\|")


def _escape_table_cell_pipes(text: str) -> str:
"""Escape unescaped pipes so cell text cannot split a Markdown table."""
return _TABLE_PIPE_RE.sub(r"\1\\|", text)


def _quote_path_preserving_percent_encoded_octets(path: str) -> str:
"""Quote a URL path while preserving existing %HH byte encodings."""
Expand Down Expand Up @@ -172,5 +187,19 @@ def convert_strike(self, el: Any, text: str, *args, **kwargs) -> str:
"""Obsolete <strike> is still in the wild; treat it like <s>/<del>."""
return self.convert_s(el, text, *args, **kwargs) # type: ignore

def convert_td(self, el: Any, text: str, parent_tags: Any) -> str:
"""Same as usual, but escape pipes so cell text cannot split the table."""
colspan = 1
if "colspan" in el.attrs and el["colspan"].isdigit():
colspan = max(1, min(1000, int(el["colspan"])))
return " " + _escape_table_cell_pipes(text.strip().replace("\n", " ")) + " |" * colspan

def convert_th(self, el: Any, text: str, parent_tags: Any) -> str:
"""Same as usual, but escape pipes so header text cannot split the table."""
colspan = 1
if "colspan" in el.attrs and el["colspan"].isdigit():
colspan = max(1, min(1000, int(el["colspan"])))
return " " + _escape_table_cell_pipes(text.strip().replace("\n", " ")) + " |" * colspan

def convert_soup(self, soup: Any) -> str:
return super().convert_soup(soup) # type: ignore
46 changes: 46 additions & 0 deletions packages/markitdown/tests/test_html_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,52 @@ def test_html_href_does_not_quote_query_or_fragment() -> None:
assert f"[example]({expected_href})" in markdown


def test_html_table_pipe_in_cell_is_escaped() -> None:
html = (
"<table><thead><tr><th>Name</th><th>Note</th></tr></thead>"
"<tbody><tr><td>Alice</td><td>Has a | pipe</td></tr></tbody></table>"
)

markdown = _convert_html(html)

assert "| Alice | Has a \\| pipe |" in markdown


def test_html_table_pipe_in_header_is_escaped() -> None:
html = (
"<table><thead><tr><th>A|B</th><th>C</th></tr></thead>"
"<tbody><tr><td>1</td><td>2</td></tr></tbody></table>"
)

markdown = _convert_html(html)

assert "| A\\|B | C |" in markdown


def test_html_table_already_escaped_pipe_is_not_doubled() -> None:
html = (
"<table><thead><tr><th>H1</th><th>H2</th></tr></thead>"
"<tbody><tr><td>a\\|b</td><td>c</td></tr></tbody></table>"
)

markdown = _convert_html(html)

assert "| a\\|b | c |" in markdown
assert "\\\\|" not in markdown


def test_html_table_pipe_in_inline_code_is_escaped() -> None:
html = (
"<table><tr><th>Value</th><th>Control</th></tr>"
"<tr><td><code>a|b</code></td><td>end</td></tr></table>"
)

markdown = _convert_html(html)

assert "`a\\|b`" in markdown
assert "| end |" in markdown


def test_img_prefers_data_src_over_placeholder_data_uri() -> None:
placeholder = (
"data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBTAA7"
Expand Down