From de4b8957e57cbf197f676a4e40a7e06063c2384e Mon Sep 17 00:00:00 2001 From: Adam Fourney Date: Sat, 12 Sep 2026 12:58:30 -0700 Subject: [PATCH 1/3] Preserve whitepace underlines --- .../tests/test_docx_converter.py | 50 ++++++++++++++++--- .../src/markitdown/converters/_markdownify.py | 3 ++ .../markitdown/tests/test_html_converter.py | 29 +++++++++++ packages/markitdown/tests/test_module_misc.py | 41 ++++++++++++--- 4 files changed, 111 insertions(+), 12 deletions(-) diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py index f9df439c1..dd0eaf138 100644 --- a/packages/markitdown-ocr/tests/test_docx_converter.py +++ b/packages/markitdown-ocr/tests/test_docx_converter.py @@ -281,15 +281,19 @@ def test_docx_styles_with_redundant_default_namespace( # --------------------------------------------------------------------------- -def _underlined_docx(tmp_path: Path) -> Path: +def _underlined_docx( + tmp_path: Path, + *, + paragraph_xml: str = ( + "plain " + 'underlined' + ), +) -> Path: docx_file = tmp_path / "underlined.docx" - document_xml = """ + document_xml = f""" - - plain - underlined - + {paragraph_xml} """ @@ -333,6 +337,40 @@ def test_docx_underlined_text_is_preserved_with_ocr( assert "plain underlined" in md +@pytest.mark.parametrize("use_ocr", [False, True]) +@pytest.mark.parametrize( + ("run_xml", "expected"), + [ + (' ', "First Last"), + ("", "First Last"), + (" ", "First\u00a0Last"), + ("", "First \nLast"), + ], +) +def test_docx_underlined_whitespace_is_preserved( + tmp_path: Path, + svc: MockOCRService, + use_ocr: bool, + run_xml: str, + expected: str, +) -> None: + path = _underlined_docx( + tmp_path, + paragraph_xml=( + "First" + f'{run_xml}' + "Last" + ), + ) + converter = DocxConverterWithOCR() + with path.open("rb") as stream: + result = converter.convert( + stream, StreamInfo(extension=".docx"), ocr_service=svc if use_ocr else None + ) + + assert result.markdown == expected + + # --------------------------------------------------------------------------- # ZIP local file header casing mismatch # --------------------------------------------------------------------------- diff --git a/packages/markitdown/src/markitdown/converters/_markdownify.py b/packages/markitdown/src/markitdown/converters/_markdownify.py index e0bbcaf2a..ed3414486 100644 --- a/packages/markitdown/src/markitdown/converters/_markdownify.py +++ b/packages/markitdown/src/markitdown/converters/_markdownify.py @@ -163,6 +163,9 @@ def convert_u( convert_as_inline: Optional[bool] = False, **kwargs, ) -> str: + if not text.strip(): + return text + prefix, suffix, text = markdownify.chomp(text) # type: ignore if not text: return "" diff --git a/packages/markitdown/tests/test_html_converter.py b/packages/markitdown/tests/test_html_converter.py index e56435196..3c5d8976a 100644 --- a/packages/markitdown/tests/test_html_converter.py +++ b/packages/markitdown/tests/test_html_converter.py @@ -1,6 +1,10 @@ import io +import pytest +from bs4 import BeautifulSoup + from markitdown import MarkItDown +from markitdown.converters._markdownify import _CustomMarkdownify def _convert_html(html: str, **kwargs) -> str: @@ -12,6 +16,31 @@ def _convert_html(html: str, **kwargs) -> str: return result.markdown +@pytest.mark.parametrize( + "whitespace", ["", " ", " ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "] +) +def test_underline_preserves_whitespace_verbatim(whitespace: str) -> None: + element = BeautifulSoup("", "html.parser").u + + assert _CustomMarkdownify().convert_u(element, whitespace) == whitespace + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + ("", "FirstLast"), + (" ", "First Last"), + ("\t", "First Last"), + (" ", "First\u00a0Last"), + ("
", "First\nLast"), + ("word", "FirstwordLast"), + (" word ", "First word Last"), + ], +) +def test_html_underlined_content_is_preserved(content: str, expected: str) -> None: + assert _convert_html(f"

First{content}Last

") == expected + + def test_preserves_non_utf8_percent_encoded_href_path() -> None: href = "https://abc.com/hist/" "%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8" html = f'example' diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 18690d188..434cd3c5b 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -387,15 +387,20 @@ def test_docx_comments() -> None: validate_strings(result, DOCX_COMMENT_TEST_STRINGS) -def _write_underlined_docx(path, embedded_style_map: Optional[str] = None) -> str: +def _write_underlined_docx( + path, + embedded_style_map: Optional[str] = None, + *, + paragraph_xml: str = ( + "plain " + 'underlined' + ), +) -> str: """Write a minimal .docx holding one underlined run, and return its path.""" - document_xml = """ + document_xml = f""" - - plain - underlined - + {paragraph_xml} """ @@ -440,6 +445,30 @@ def test_docx_underlined_text_is_preserved(tmp_path) -> None: assert "plain underlined" in result.markdown +@pytest.mark.parametrize( + ("run_xml", "expected"), + [ + (' ', "First Last"), + ("", "First Last"), + (" ", "First\u00a0Last"), + ("", "First\nLast"), + ], +) +def test_docx_underlined_whitespace_is_preserved( + tmp_path, run_xml: str, expected: str +) -> None: + docx_file = _write_underlined_docx( + tmp_path / "underlined_whitespace.docx", + paragraph_xml=( + "First" + f'{run_xml}' + "Last" + ), + ) + + assert MarkItDown().convert(docx_file).markdown == expected + + def test_docx_embedded_style_map_overrides_underline_default(tmp_path) -> None: # A style map embedded in the document takes precedence over the default # "u => u" mapping that preserves underlines. From 89a2ff9933c6d5458976b300a1113e4edb3ec4a2 Mon Sep 17 00:00:00 2001 From: afourney Date: Sat, 12 Sep 2026 13:15:47 -0700 Subject: [PATCH 2/3] Fix whitespace handling in test_docx_converter Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- packages/markitdown-ocr/tests/test_docx_converter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py index dd0eaf138..6ea505957 100644 --- a/packages/markitdown-ocr/tests/test_docx_converter.py +++ b/packages/markitdown-ocr/tests/test_docx_converter.py @@ -344,7 +344,7 @@ def test_docx_underlined_text_is_preserved_with_ocr( (' ', "First Last"), ("", "First Last"), (" ", "First\u00a0Last"), - ("", "First \nLast"), + ("", "First\nLast"), ], ) def test_docx_underlined_whitespace_is_preserved( From b6244a01240bd50c818af9665b6243116272fc7b Mon Sep 17 00:00:00 2001 From: Adam Fourney Date: Sat, 12 Sep 2026 13:22:04 -0700 Subject: [PATCH 3/3] Repair failing test. --- packages/markitdown-ocr/tests/test_docx_converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py index 6ea505957..8988c0334 100644 --- a/packages/markitdown-ocr/tests/test_docx_converter.py +++ b/packages/markitdown-ocr/tests/test_docx_converter.py @@ -344,7 +344,8 @@ def test_docx_underlined_text_is_preserved_with_ocr( (' ', "First Last"), ("", "First Last"), (" ", "First\u00a0Last"), - ("", "First\nLast"), + # Direct conversion keeps the two-space hard break; the dispatcher strips it. + ("", "First \nLast"), ], ) def test_docx_underlined_whitespace_is_preserved(