diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py index 2538206ded..f9df439c13 100644 --- a/packages/markitdown-ocr/tests/test_docx_converter.py +++ b/packages/markitdown-ocr/tests/test_docx_converter.py @@ -226,6 +226,56 @@ def test_docx_no_ocr_service_no_tags() -> None: assert "[End OCR]*" not in md +@pytest.mark.parametrize("double_strike", [False, True]) +@pytest.mark.parametrize("use_ocr", [False, True]) +def test_docx_styles_with_redundant_default_namespace( + svc: MockOCRService, use_ocr: bool, double_strike: bool +) -> None: + path = TEST_DATA_DIR / "docx_image_middle.docx" + if not path.exists(): + pytest.skip(f"Test file not found: {path}") + original = path.read_bytes() + fixture = io.BytesIO() + namespace = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + declaration = f'xmlns:w="{namespace}"'.encode("utf-8") + with zipfile.ZipFile(io.BytesIO(original)) as source, zipfile.ZipFile( + fixture, "w" + ) as target: + for item in source.infolist(): + content = source.read(item) + if item.filename == "word/styles.xml": + if double_strike: + assert content.count(b"") == 1 + content = content.replace( + b"", + b'' + b'' + b'' + b"", + 1, + ) + assert content.count(declaration) == 1 + content = content.replace( + declaration, declaration + f' xmlns="{namespace}"'.encode(), 1 + ) + target.writestr(item, content) + + converter = DocxConverterWithOCR() + service = svc if use_ocr else None + expected = converter.convert( + io.BytesIO(original), StreamInfo(extension=".docx"), ocr_service=service + ).markdown + fixture.seek(0) + actual = converter.convert( + fixture, StreamInfo(extension=".docx"), ocr_service=service + ).markdown + + assert "# Introduction" in actual + assert actual == expected + if use_ocr: + assert _MOCK_TEXT in actual + + # --------------------------------------------------------------------------- # Underlined runs survive both the OCR and the non-OCR mammoth paths # --------------------------------------------------------------------------- diff --git a/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py b/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py index 46847a2a34..c6dc303414 100644 --- a/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py +++ b/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py @@ -216,15 +216,35 @@ def _pre_process_styles(content: bytes) -> bytes: to ``paragraph``, so the attribute is filled in rather than dropping the style, which would discard its formatting (a heading would be emitted as plain body text). A style with no ``w:styleId`` cannot be referenced by the - document body, so it is removed. + document body, so it is removed. Double strikethrough is normalized to + single strikethrough in the same namespace-aware pass. + + Match elements and attributes by namespace URI, preserving their qualified + names when repairing the XML. Return the original bytes if no repair is needed. """ - soup = BeautifulSoup(content, features="xml") - for tag in soup.find_all("w:style"): - if not tag.has_attr("w:styleId"): - tag.decompose() - elif not tag.has_attr("w:type"): - tag["w:type"] = "paragraph" - return str(soup).encode() + from lxml import etree + + namespace = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + parser = etree.XMLParser(resolve_entities=False, no_network=True) + root = etree.fromstring(content, parser=parser) + changed = False + + for style in root.findall(namespace + "style"): + if namespace + "styleId" not in style.attrib: + root.remove(style) + changed = True + elif namespace + "type" not in style.attrib: + style.set(namespace + "type", "paragraph") + changed = True + + for strike in root.iter(namespace + "dstrike"): + strike.tag = namespace + "strike" + changed = True + + if not changed: + return content + + return etree.tostring(root.getroottree(), encoding="utf-8", xml_declaration=True) def pre_process_docx(input_docx: BinaryIO) -> BinaryIO: @@ -250,7 +270,7 @@ def pre_process_docx(input_docx: BinaryIO) -> BinaryIO: "word/document.xml": (_pre_process_strike, _pre_process_math), "word/footnotes.xml": (_pre_process_strike, _pre_process_math), "word/endnotes.xml": (_pre_process_strike, _pre_process_math), - "word/styles.xml": (_pre_process_strike, _pre_process_styles), + "word/styles.xml": (_pre_process_styles,), } with zipfile.ZipFile(input_docx, mode="r") as zip_input: files = {name: zip_input.read(name) for name in zip_input.namelist()} diff --git a/packages/markitdown/tests/test_docx_styles.py b/packages/markitdown/tests/test_docx_styles.py new file mode 100644 index 0000000000..218fe03c5d --- /dev/null +++ b/packages/markitdown/tests/test_docx_styles.py @@ -0,0 +1,152 @@ +"""DOCX stylesheet repair must preserve namespace-qualified attributes.""" + +import io +from pathlib import Path +import re +import zipfile + +from lxml import etree +import pytest + +from markitdown import MarkItDown, StreamInfo +from markitdown.converter_utils.docx.pre_process import _pre_process_styles + + +WORD_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +W = f"{{{WORD_NAMESPACE}}}" +TEST_DOCX = Path(__file__).parent / "test_files" / "test.docx" + + +@pytest.mark.parametrize("double_strike", [False, True]) +@pytest.mark.parametrize("missing_type", [False, True]) +def test_docx_styles_with_redundant_default_namespace( + missing_type: bool, double_strike: bool +) -> None: + markitdown = MarkItDown() + expected = markitdown.convert(TEST_DOCX).markdown + fixture = io.BytesIO() + declaration = f'xmlns:w="{WORD_NAMESPACE}"'.encode("utf-8") + + with zipfile.ZipFile(TEST_DOCX) as source, zipfile.ZipFile(fixture, "w") as target: + for item in source.infolist(): + content = source.read(item) + if item.filename == "word/styles.xml": + if double_strike: + assert content.count(b"") == 1 + content = content.replace( + b"", + b'' + b'' + b'' + b"", + 1, + ) + assert content.count(declaration) == 1 + content = content.replace( + declaration, + declaration + f' xmlns="{WORD_NAMESPACE}"'.encode("utf-8"), + 1, + ) + if missing_type: + content, count = re.subn( + rb' bytes: + prefix = request.param + return f""" +<{prefix}:styles xmlns:{prefix}="{WORD_NAMESPACE}" xmlns="{WORD_NAMESPACE}" + xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" + xmlns:w14="http://schemas.microsoft.com/office/word/2010/wordml" + mc:Ignorable="w14"> + <{prefix}:style {prefix}:type="paragraph" {prefix}:styleId="Normal"> + <{prefix}:name {prefix}:val="Normal"/> + +""".encode( + "utf-8" + ) + + +def test_valid_styles_are_returned_unchanged(styles_xml: bytes) -> None: + assert _pre_process_styles(styles_xml) == styles_xml + + +def test_missing_type_repair_preserves_namespaces(styles_xml: bytes) -> None: + malformed, count = re.subn(rb'\s+(?:w|word):type="paragraph"', b"", styles_xml) + assert count == 1 + original = etree.fromstring(styles_xml) + + repaired = etree.fromstring(_pre_process_styles(malformed)) + + assert repaired.nsmap == original.nsmap + assert repaired.attrib == original.attrib + style = repaired.find(W + "style") + assert style is not None + assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"} + name = style.find(W + "name") + assert name is not None + assert name.attrib == {W + "val": "Normal"} + + +def test_styles_without_ids_are_removed(styles_xml: bytes) -> None: + malformed, count = re.subn(rb'\s+(?:w|word):styleId="Normal"', b"", styles_xml) + assert count == 1 + + repaired = etree.fromstring(_pre_process_styles(malformed)) + + assert repaired.findall(W + "style") == [] + + +@pytest.mark.parametrize("value", [None, "0", "1"]) +@pytest.mark.parametrize("missing_type", [False, True]) +def test_style_strike_repair_preserves_namespaces( + styles_xml: bytes, value: str | None, missing_type: bool +) -> None: + original = etree.fromstring(styles_xml) + style = original.find(W + "style") + assert style is not None + if missing_type: + del style.attrib[W + "type"] + properties = etree.SubElement(style, W + "rPr") + strike = etree.SubElement(properties, W + "dstrike") + if value is not None: + strike.set(W + "val", value) + etree.SubElement(properties, W + "b") + + repaired = etree.fromstring(_pre_process_styles(etree.tostring(original))) + + assert repaired.nsmap == original.nsmap + assert repaired.attrib == original.attrib + style = repaired.find(W + "style") + assert style is not None + assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"} + assert repaired.find(".//" + W + "dstrike") is None + strike = style.find(f"{W}rPr/{W}strike") + assert strike is not None + assert strike.attrib == ({} if value is None else {W + "val": value}) + assert style.find(f"{W}rPr/{W}b") is not None + + +def test_other_namespace_dstrike_is_unchanged(styles_xml: bytes) -> None: + original = etree.fromstring(styles_xml) + etree.SubElement(original, "{urn:extension}dstrike") + content = etree.tostring(original) + + assert _pre_process_styles(content) == content