Preserve whitespace underlines

packages/markitdown-ocr/tests/test_docx_converter.py

@@ -281,15 +281,19 @@ def test_docx_styles_with_redundant_default_namespace(
 # ---------------------------------------------------------------------------
 
 
-def _underlined_docx(tmp_path: Path) -> Path:
+def _underlined_docx(
+    tmp_path: Path,
+    *,
+    paragraph_xml: str = (
+        "<w:r><w:t>plain </w:t></w:r>"
+        '<w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>'
+    ),
+) -> Path:
     docx_file = tmp_path / "underlined.docx"
-    document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+    document_xml = f"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
   <w:body>
-    <w:p>
-      <w:r><w:t>plain </w:t></w:r>
-      <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>
-    </w:p>
+    <w:p>{paragraph_xml}</w:p>
   </w:body>
 </w:document>"""
 
@@ -333,6 +337,41 @@ def test_docx_underlined_text_is_preserved_with_ocr(
     assert "plain <u>underlined</u>" in md
 
 
+@pytest.mark.parametrize("use_ocr", [False, True])
+@pytest.mark.parametrize(
+    ("run_xml", "expected"),
+    [
+        ('<w:t xml:space="preserve"> </w:t>', "First Last"),
+        ("<w:tab/>", "First Last"),
+        ("<w:t>&#160;</w:t>", "First\u00a0Last"),
+        # Direct conversion keeps the two-space hard break; the dispatcher strips it.
+        ("<w:br/>", "First  \nLast"),
+    ],
+)
+def test_docx_underlined_whitespace_is_preserved(
+    tmp_path: Path,
+    svc: MockOCRService,
+    use_ocr: bool,
+    run_xml: str,
+    expected: str,
+) -> None:
+    path = _underlined_docx(
+        tmp_path,
+        paragraph_xml=(
+            "<w:r><w:t>First</w:t></w:r>"
+            f'<w:r><w:rPr><w:u w:val="single"/></w:rPr>{run_xml}</w:r>'
+            "<w:r><w:t>Last</w:t></w:r>"
+        ),
+    )
+    converter = DocxConverterWithOCR()
+    with path.open("rb") as stream:
+        result = converter.convert(
+            stream, StreamInfo(extension=".docx"), ocr_service=svc if use_ocr else None
+        )
+
+    assert result.markdown == expected
+
+
 # ---------------------------------------------------------------------------
 # ZIP local file header casing mismatch
 # ---------------------------------------------------------------------------

packages/markitdown/src/markitdown/converters/_markdownify.py

@@ -163,6 +163,9 @@ def convert_u(
         convert_as_inline: Optional[bool] = False,
         **kwargs,
     ) -> str:
+        if not text.strip():
+            return text
+
         prefix, suffix, text = markdownify.chomp(text)  # type: ignore
         if not text:
             return ""

packages/markitdown/tests/test_html_converter.py

@@ -1,6 +1,10 @@
 import io
 
+import pytest
+from bs4 import BeautifulSoup
+
 from markitdown import MarkItDown
+from markitdown.converters._markdownify import _CustomMarkdownify
 
 
 def _convert_html(html: str, **kwargs) -> str:
@@ -12,6 +16,31 @@ def _convert_html(html: str, **kwargs) -> str:
     return result.markdown
 
 
+@pytest.mark.parametrize(
+    "whitespace", ["", " ", "  ", "\t", "\n", "\r\n", "\u00a0", " \t\n\u00a0 "]
+)
+def test_underline_preserves_whitespace_verbatim(whitespace: str) -> None:
+    element = BeautifulSoup("<u></u>", "html.parser").u
+
+    assert _CustomMarkdownify().convert_u(element, whitespace) == whitespace
+
+
+@pytest.mark.parametrize(
+    ("content", "expected"),
+    [
+        ("", "FirstLast"),
+        (" ", "First Last"),
+        ("\t", "First Last"),
+        ("&#160;", "First\u00a0Last"),
+        ("<br>", "First\nLast"),
+        ("word", "First<u>word</u>Last"),
+        (" word ", "First <u>word</u> Last"),
+    ],
+)
+def test_html_underlined_content_is_preserved(content: str, expected: str) -> None:
+    assert _convert_html(f"<p>First<u>{content}</u>Last</p>") == expected
+
+
 def test_preserves_non_utf8_percent_encoded_href_path() -> None:
     href = "https://abc.com/hist/" "%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8"
     html = f'<a href="{href}">example</a>'

packages/markitdown/tests/test_module_misc.py

@@ -387,15 +387,20 @@ def test_docx_comments() -> None:
     validate_strings(result, DOCX_COMMENT_TEST_STRINGS)
 
 
-def _write_underlined_docx(path, embedded_style_map: Optional[str] = None) -> str:
+def _write_underlined_docx(
+    path,
+    embedded_style_map: Optional[str] = None,
+    *,
+    paragraph_xml: str = (
+        "<w:r><w:t>plain </w:t></w:r>"
+        '<w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>'
+    ),
+) -> str:
     """Write a minimal .docx holding one underlined run, and return its path."""
-    document_xml = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+    document_xml = f"""<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
   <w:body>
-    <w:p>
-      <w:r><w:t>plain </w:t></w:r>
-      <w:r><w:rPr><w:u w:val="single"/></w:rPr><w:t>underlined</w:t></w:r>
-    </w:p>
+    <w:p>{paragraph_xml}</w:p>
   </w:body>
 </w:document>"""
 
@@ -440,6 +445,30 @@ def test_docx_underlined_text_is_preserved(tmp_path) -> None:
     assert "plain <u>underlined</u>" in result.markdown
 
 
+@pytest.mark.parametrize(
+    ("run_xml", "expected"),
+    [
+        ('<w:t xml:space="preserve"> </w:t>', "First Last"),
+        ("<w:tab/>", "First Last"),
+        ("<w:t>&#160;</w:t>", "First\u00a0Last"),
+        ("<w:br/>", "First\nLast"),
+    ],
+)
+def test_docx_underlined_whitespace_is_preserved(
+    tmp_path, run_xml: str, expected: str
+) -> None:
+    docx_file = _write_underlined_docx(
+        tmp_path / "underlined_whitespace.docx",
+        paragraph_xml=(
+            "<w:r><w:t>First</w:t></w:r>"
+            f'<w:r><w:rPr><w:u w:val="single"/></w:rPr>{run_xml}</w:r>'
+            "<w:r><w:t>Last</w:t></w:r>"
+        ),
+    )
+
+    assert MarkItDown().convert(docx_file).markdown == expected
+
+
 def test_docx_embedded_style_map_overrides_underline_default(tmp_path) -> None:
     # A style map embedded in the document takes precedence over the default
     # "u => u" mapping that preserves underlines.