From db95c44d663974254a7b39cba7e91eb727c0cd82 Mon Sep 17 00:00:00 2001 From: Adam Fourney Date: Fri, 11 Sep 2026 14:28:45 -0700 Subject: [PATCH 1/3] fix: avoid splitting UTF-8 characters during charset detection --- .../markitdown/src/markitdown/_markitdown.py | 32 ++++- .../markitdown/tests/test_charset_sample.py | 120 ++++++++++++++++++ 2 files changed, 150 insertions(+), 2 deletions(-) create mode 100644 packages/markitdown/tests/test_charset_sample.py diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index fa67aad26f..888d1e6260 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -70,6 +70,34 @@ def _get_content_disposition_filename(content_disposition: str) -> Optional[str] return extended_filename or fallback_filename +def _read_charset_sample(file_stream: BinaryIO) -> bytes: + """Read a 64 KiB sample, completing a trailing split UTF-8 character.""" + sample = file_stream.read(65536) + if len(sample) < 65536: + return sample + + decoder = codecs.getincrementaldecoder("utf-8")(errors="strict") + try: + decoder.decode(sample, final=False) + if not decoder.getstate()[0]: + return sample + + suffix = b"" + for _ in range(3): + chunk = file_stream.read(1) + if not chunk: + break + suffix += chunk + decoder.decode(chunk, final=False) + if not decoder.getstate()[0]: + return sample + suffix + except UnicodeDecodeError: + # Leave invalid UTF-8 unchanged for the existing charset detector. + pass + + return sample + + # Lower priority values are tried first. PRIORITY_SPECIFIC_FILE_FORMAT = ( 0.0 # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia @@ -743,9 +771,9 @@ def _get_stream_info_guesses( # If it's text, also guess the charset charset = None if result.prediction.output.is_text: - # Read the first 64k to guess the charset + # Complete a split UTF-8 character at the sample boundary. file_stream.seek(cur_pos) - stream_page = file_stream.read(65536) + stream_page = _read_charset_sample(file_stream) charset_result = charset_normalizer.from_bytes(stream_page).best() if charset_result is not None: diff --git a/packages/markitdown/tests/test_charset_sample.py b/packages/markitdown/tests/test_charset_sample.py new file mode 100644 index 0000000000..9b3036bd56 --- /dev/null +++ b/packages/markitdown/tests/test_charset_sample.py @@ -0,0 +1,120 @@ +"""Charset sampling must not mistake a split UTF-8 character for another encoding.""" + +import io + +import pytest + +from markitdown import MarkItDown, StreamInfo +from markitdown._markitdown import _read_charset_sample + + +_SAMPLE_SIZE = 65536 +_SPLIT_CHARACTERS = [ + (character, split) + for character in ("\u00e9", "\u65e5", "\U0001f600") + for split in range(1, len(character.encode("utf-8"))) +] + + +def _split_utf8_json(character: str, split: int) -> bytes: + prefix = '{"name":"r\u00e9sum\u00e9","notes":"'.encode("utf-8") + return ( + prefix + + b"a" * (_SAMPLE_SIZE - split - len(prefix)) + + (character + '"}').encode("utf-8") + ) + + +@pytest.fixture(scope="module") +def markitdown() -> MarkItDown: + return MarkItDown() + + +@pytest.mark.parametrize("character,split", _SPLIT_CHARACTERS) +def test_charset_sample_completes_only_the_split_character( + character: str, split: int +) -> None: + data = _split_utf8_json(character, split) + stream = io.BytesIO(data) + expected_size = _SAMPLE_SIZE + len(character.encode("utf-8")) - split + + sample = _read_charset_sample(stream) + + assert sample == data[:expected_size] + assert stream.tell() == expected_size + assert expected_size <= _SAMPLE_SIZE + 3 + sample.decode("utf-8") + + +@pytest.mark.parametrize("character,split", _SPLIT_CHARACTERS) +def test_split_utf8_json_preserves_content( + markitdown: MarkItDown, character: str, split: int +) -> None: + data = _split_utf8_json(character, split) + stream = io.BytesIO(data) + + guesses = markitdown._get_stream_info_guesses(stream, StreamInfo()) + assert guesses[0].charset == "utf-8" + assert stream.tell() == 0 + + result = markitdown.convert_stream(stream) + + assert result.markdown == data.decode("utf-8") + + +@pytest.mark.parametrize( + "sample,tail", + [ + (b"", b""), + (b"ordinary text", b""), + (b"short incomplete \xc3", b""), + (b"a" * _SAMPLE_SIZE, b"\xc3\xa9"), + (b"a" * (_SAMPLE_SIZE - 2) + b"\xc3\xa9", b"\xf0\x9f\x98\x80"), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xc3", b""), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xf0", b"\x9f"), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xc3", b"x"), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xe0", b"\x80\x80"), + (b"a" * (_SAMPLE_SIZE - 1) + b"\xed", b"\xa0\x80"), + (b"\xff" + b"a" * (_SAMPLE_SIZE - 2) + b"\xc3", b"\xa9"), + ], + ids=[ + "empty", + "short", + "short-incomplete", + "ascii", + "complete-utf8", + "incomplete-at-eof", + "incomplete-after-lookahead", + "invalid-continuation", + "overlong", + "surrogate", + "non-utf8-prefix", + ], +) +def test_other_charset_samples_are_unchanged(sample: bytes, tail: bytes) -> None: + stream = io.BytesIO(sample + tail) + + assert _read_charset_sample(stream) == sample + assert stream.tell() <= _SAMPLE_SIZE + 3 + + +def test_charset_guesses_restore_nonzero_stream_position( + markitdown: MarkItDown, +) -> None: + stream = io.BytesIO(b"prefix" + _split_utf8_json("\U0001f600", 1)) + stream.seek(len(b"prefix")) + + markitdown._get_stream_info_guesses(stream, StreamInfo()) + + assert stream.tell() == len(b"prefix") + + +def test_explicit_charset_still_takes_precedence(markitdown: MarkItDown) -> None: + data = _split_utf8_json("\u00e9", 1) + + result = markitdown.convert_stream( + io.BytesIO(data), + stream_info=StreamInfo(extension=".json", charset="cp1252"), + ) + + assert result.markdown == data.decode("cp1252") From 16389ca2ef7cecb84eabc31a4d6ccbde3b6b9f26 Mon Sep 17 00:00:00 2001 From: Adam Fourney Date: Fri, 11 Sep 2026 14:50:28 -0700 Subject: [PATCH 2/3] fix(docx): preserve namespaces when repairing stylesheets --- .../tests/test_docx_converter.py | 39 +++++++ .../converter_utils/docx/pre_process.py | 29 +++-- packages/markitdown/tests/test_docx_styles.py | 101 ++++++++++++++++++ 3 files changed, 162 insertions(+), 7 deletions(-) create mode 100644 packages/markitdown/tests/test_docx_styles.py diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py index 2538206ded..81e81d2051 100644 --- a/packages/markitdown-ocr/tests/test_docx_converter.py +++ b/packages/markitdown-ocr/tests/test_docx_converter.py @@ -226,6 +226,45 @@ def test_docx_no_ocr_service_no_tags() -> None: assert "[End OCR]*" not in md +@pytest.mark.parametrize("use_ocr", [False, True]) +def test_docx_styles_with_redundant_default_namespace( + svc: MockOCRService, use_ocr: bool +) -> None: + path = TEST_DATA_DIR / "docx_image_middle.docx" + if not path.exists(): + pytest.skip(f"Test file not found: {path}") + original = path.read_bytes() + fixture = io.BytesIO() + namespace = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + declaration = f'xmlns:w="{namespace}"'.encode("utf-8") + with zipfile.ZipFile(io.BytesIO(original)) as source, zipfile.ZipFile( + fixture, "w" + ) as target: + for item in source.infolist(): + content = source.read(item) + if item.filename == "word/styles.xml": + assert content.count(declaration) == 1 + content = content.replace( + declaration, declaration + f' xmlns="{namespace}"'.encode(), 1 + ) + target.writestr(item, content) + + converter = DocxConverterWithOCR() + service = svc if use_ocr else None + expected = converter.convert( + io.BytesIO(original), StreamInfo(extension=".docx"), ocr_service=service + ).markdown + fixture.seek(0) + actual = converter.convert( + fixture, StreamInfo(extension=".docx"), ocr_service=service + ).markdown + + assert "# Introduction" in actual + assert actual == expected + if use_ocr: + assert _MOCK_TEXT in actual + + # --------------------------------------------------------------------------- # Underlined runs survive both the OCR and the non-OCR mammoth paths # --------------------------------------------------------------------------- diff --git a/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py b/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py index 46847a2a34..360f117d00 100644 --- a/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py +++ b/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py @@ -217,14 +217,29 @@ def _pre_process_styles(content: bytes) -> bytes: style, which would discard its formatting (a heading would be emitted as plain body text). A style with no ``w:styleId`` cannot be referenced by the document body, so it is removed. + + Match elements and attributes by namespace URI, preserving their qualified + names when repairing the XML. Return the original bytes if no repair is needed. """ - soup = BeautifulSoup(content, features="xml") - for tag in soup.find_all("w:style"): - if not tag.has_attr("w:styleId"): - tag.decompose() - elif not tag.has_attr("w:type"): - tag["w:type"] = "paragraph" - return str(soup).encode() + from lxml import etree + + namespace = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + parser = etree.XMLParser(resolve_entities=False, no_network=True) + root = etree.fromstring(content, parser=parser) + changed = False + + for style in root.findall(namespace + "style"): + if namespace + "styleId" not in style.attrib: + root.remove(style) + changed = True + elif namespace + "type" not in style.attrib: + style.set(namespace + "type", "paragraph") + changed = True + + if not changed: + return content + + return etree.tostring(root.getroottree(), encoding="utf-8", xml_declaration=True) def pre_process_docx(input_docx: BinaryIO) -> BinaryIO: diff --git a/packages/markitdown/tests/test_docx_styles.py b/packages/markitdown/tests/test_docx_styles.py new file mode 100644 index 0000000000..c43dd6fbb4 --- /dev/null +++ b/packages/markitdown/tests/test_docx_styles.py @@ -0,0 +1,101 @@ +"""DOCX stylesheet repair must preserve namespace-qualified attributes.""" + +import io +from pathlib import Path +import re +import zipfile + +from lxml import etree +import pytest + +from markitdown import MarkItDown, StreamInfo +from markitdown.converter_utils.docx.pre_process import _pre_process_styles + + +WORD_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" +W = f"{{{WORD_NAMESPACE}}}" +TEST_DOCX = Path(__file__).parent / "test_files" / "test.docx" + + +@pytest.mark.parametrize("missing_type", [False, True]) +def test_docx_styles_with_redundant_default_namespace(missing_type: bool) -> None: + markitdown = MarkItDown() + expected = markitdown.convert(TEST_DOCX).markdown + fixture = io.BytesIO() + declaration = f'xmlns:w="{WORD_NAMESPACE}"'.encode("utf-8") + + with zipfile.ZipFile(TEST_DOCX) as source, zipfile.ZipFile(fixture, "w") as target: + for item in source.infolist(): + content = source.read(item) + if item.filename == "word/styles.xml": + assert content.count(declaration) == 1 + content = content.replace( + declaration, + declaration + f' xmlns="{WORD_NAMESPACE}"'.encode("utf-8"), + 1, + ) + if missing_type: + content, count = re.subn( + rb' bytes: + prefix = request.param + return f""" +<{prefix}:styles xmlns:{prefix}="{WORD_NAMESPACE}" xmlns="{WORD_NAMESPACE}" + xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" + xmlns:w14="http://schemas.microsoft.com/office/word/2010/wordml" + mc:Ignorable="w14"> + <{prefix}:style {prefix}:type="paragraph" {prefix}:styleId="Normal"> + <{prefix}:name {prefix}:val="Normal"/> + +""".encode( + "utf-8" + ) + + +def test_valid_styles_are_returned_unchanged(styles_xml: bytes) -> None: + assert _pre_process_styles(styles_xml) == styles_xml + + +def test_missing_type_repair_preserves_namespaces(styles_xml: bytes) -> None: + malformed, count = re.subn(rb'\s+(?:w|word):type="paragraph"', b"", styles_xml) + assert count == 1 + original = etree.fromstring(styles_xml) + + repaired = etree.fromstring(_pre_process_styles(malformed)) + + assert repaired.nsmap == original.nsmap + assert repaired.attrib == original.attrib + style = repaired.find(W + "style") + assert style is not None + assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"} + name = style.find(W + "name") + assert name is not None + assert name.attrib == {W + "val": "Normal"} + + +def test_styles_without_ids_are_removed(styles_xml: bytes) -> None: + malformed, count = re.subn(rb'\s+(?:w|word):styleId="Normal"', b"", styles_xml) + assert count == 1 + + repaired = etree.fromstring(_pre_process_styles(malformed)) + + assert repaired.findall(W + "style") == [] From 1ac833f60ca7133f66440bd0f5179d45e7eb6f58 Mon Sep 17 00:00:00 2001 From: Adam Fourney Date: Fri, 11 Sep 2026 14:58:26 -0700 Subject: [PATCH 3/3] Fixed d:strike as well --- .../tests/test_docx_converter.py | 13 ++++- .../converter_utils/docx/pre_process.py | 9 +++- packages/markitdown/tests/test_docx_styles.py | 53 ++++++++++++++++++- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/packages/markitdown-ocr/tests/test_docx_converter.py b/packages/markitdown-ocr/tests/test_docx_converter.py index 81e81d2051..f9df439c13 100644 --- a/packages/markitdown-ocr/tests/test_docx_converter.py +++ b/packages/markitdown-ocr/tests/test_docx_converter.py @@ -226,9 +226,10 @@ def test_docx_no_ocr_service_no_tags() -> None: assert "[End OCR]*" not in md +@pytest.mark.parametrize("double_strike", [False, True]) @pytest.mark.parametrize("use_ocr", [False, True]) def test_docx_styles_with_redundant_default_namespace( - svc: MockOCRService, use_ocr: bool + svc: MockOCRService, use_ocr: bool, double_strike: bool ) -> None: path = TEST_DATA_DIR / "docx_image_middle.docx" if not path.exists(): @@ -243,6 +244,16 @@ def test_docx_styles_with_redundant_default_namespace( for item in source.infolist(): content = source.read(item) if item.filename == "word/styles.xml": + if double_strike: + assert content.count(b"") == 1 + content = content.replace( + b"", + b'' + b'' + b'' + b"", + 1, + ) assert content.count(declaration) == 1 content = content.replace( declaration, declaration + f' xmlns="{namespace}"'.encode(), 1 diff --git a/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py b/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py index 360f117d00..c6dc303414 100644 --- a/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py +++ b/packages/markitdown/src/markitdown/converter_utils/docx/pre_process.py @@ -216,7 +216,8 @@ def _pre_process_styles(content: bytes) -> bytes: to ``paragraph``, so the attribute is filled in rather than dropping the style, which would discard its formatting (a heading would be emitted as plain body text). A style with no ``w:styleId`` cannot be referenced by the - document body, so it is removed. + document body, so it is removed. Double strikethrough is normalized to + single strikethrough in the same namespace-aware pass. Match elements and attributes by namespace URI, preserving their qualified names when repairing the XML. Return the original bytes if no repair is needed. @@ -236,6 +237,10 @@ def _pre_process_styles(content: bytes) -> bytes: style.set(namespace + "type", "paragraph") changed = True + for strike in root.iter(namespace + "dstrike"): + strike.tag = namespace + "strike" + changed = True + if not changed: return content @@ -265,7 +270,7 @@ def pre_process_docx(input_docx: BinaryIO) -> BinaryIO: "word/document.xml": (_pre_process_strike, _pre_process_math), "word/footnotes.xml": (_pre_process_strike, _pre_process_math), "word/endnotes.xml": (_pre_process_strike, _pre_process_math), - "word/styles.xml": (_pre_process_strike, _pre_process_styles), + "word/styles.xml": (_pre_process_styles,), } with zipfile.ZipFile(input_docx, mode="r") as zip_input: files = {name: zip_input.read(name) for name in zip_input.namelist()} diff --git a/packages/markitdown/tests/test_docx_styles.py b/packages/markitdown/tests/test_docx_styles.py index c43dd6fbb4..218fe03c5d 100644 --- a/packages/markitdown/tests/test_docx_styles.py +++ b/packages/markitdown/tests/test_docx_styles.py @@ -17,8 +17,11 @@ TEST_DOCX = Path(__file__).parent / "test_files" / "test.docx" +@pytest.mark.parametrize("double_strike", [False, True]) @pytest.mark.parametrize("missing_type", [False, True]) -def test_docx_styles_with_redundant_default_namespace(missing_type: bool) -> None: +def test_docx_styles_with_redundant_default_namespace( + missing_type: bool, double_strike: bool +) -> None: markitdown = MarkItDown() expected = markitdown.convert(TEST_DOCX).markdown fixture = io.BytesIO() @@ -28,6 +31,16 @@ def test_docx_styles_with_redundant_default_namespace(missing_type: bool) -> Non for item in source.infolist(): content = source.read(item) if item.filename == "word/styles.xml": + if double_strike: + assert content.count(b"") == 1 + content = content.replace( + b"", + b'' + b'' + b'' + b"", + 1, + ) assert content.count(declaration) == 1 content = content.replace( declaration, @@ -99,3 +112,41 @@ def test_styles_without_ids_are_removed(styles_xml: bytes) -> None: repaired = etree.fromstring(_pre_process_styles(malformed)) assert repaired.findall(W + "style") == [] + + +@pytest.mark.parametrize("value", [None, "0", "1"]) +@pytest.mark.parametrize("missing_type", [False, True]) +def test_style_strike_repair_preserves_namespaces( + styles_xml: bytes, value: str | None, missing_type: bool +) -> None: + original = etree.fromstring(styles_xml) + style = original.find(W + "style") + assert style is not None + if missing_type: + del style.attrib[W + "type"] + properties = etree.SubElement(style, W + "rPr") + strike = etree.SubElement(properties, W + "dstrike") + if value is not None: + strike.set(W + "val", value) + etree.SubElement(properties, W + "b") + + repaired = etree.fromstring(_pre_process_styles(etree.tostring(original))) + + assert repaired.nsmap == original.nsmap + assert repaired.attrib == original.attrib + style = repaired.find(W + "style") + assert style is not None + assert style.attrib == {W + "type": "paragraph", W + "styleId": "Normal"} + assert repaired.find(".//" + W + "dstrike") is None + strike = style.find(f"{W}rPr/{W}strike") + assert strike is not None + assert strike.attrib == ({} if value is None else {W + "val": value}) + assert style.find(f"{W}rPr/{W}b") is not None + + +def test_other_namespace_dstrike_is_unchanged(styles_xml: bytes) -> None: + original = etree.fromstring(styles_xml) + etree.SubElement(original, "{urn:extension}dstrike") + content = etree.tostring(original) + + assert _pre_process_styles(content) == content