From 48c84c8aea51e35a6b6368a237ffb3a44caf92a1 Mon Sep 17 00:00:00 2001 From: Adam Fourney Date: Fri, 11 Sep 2026 11:00:33 -0700 Subject: [PATCH] Fix ANSI Outlook MSG decoding for Japanese code pages and padded strings --- .../converters/_outlook_msg_converter.py | 37 ++++- .../markitdown/tests/test_outlook_msg_ansi.py | 146 +++++++++++++++++- 2 files changed, 173 insertions(+), 10 deletions(-) diff --git a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py index 79d7656e5..e3668651a 100644 --- a/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py +++ b/packages/markitdown/src/markitdown/converters/_outlook_msg_converter.py @@ -1,4 +1,5 @@ import codecs +import re import struct import sys from typing import Any, Dict, Union, BinaryIO @@ -62,8 +63,8 @@ 10079: "mac_iceland", 10081: "mac_turkish", 50220: "iso2022_jp", - 50221: "iso2022_jp", - 50222: "iso2022_jp", + 50221: "iso2022_jp_ext", # Supports ESC ( I for halfwidth Katakana. + 50222: "iso2022_jp_ext", # SO/SI also need normalization before decoding. 50225: "iso2022_kr", 51932: "euc_jp", 51936: "gb2312", @@ -277,11 +278,16 @@ def _get_ansi_stream_data( except Exception: return None + # Some writers include trailing NUL terminators or padding. Remove them + # before charset detection as well as decoding; str.strip() keeps NULs. + data = data.rstrip(b"\x00") if not data: return None if encoding is not None: try: + if encoding == "iso2022_jp_ext": + return self._decode_iso2022_jp(data).strip() return data.decode(encoding).strip() except (UnicodeDecodeError, LookupError): pass # The declared code page does not fit; fall back to detection @@ -291,6 +297,33 @@ def _get_ansi_stream_data( return str(detected).strip() return data.decode("utf-8", errors="ignore").strip() + def _decode_iso2022_jp(self, data: bytes) -> str: + """Decode Katakana in Windows code pages 50221 and 50222. + + Python's extended codec supports 50221's ESC ( I designation, but leaves + 50222's SO/SI controls untouched. Translate those controls to equivalent + designations, retaining the preceding mode so SI can restore it even + when it is JIS Roman or double-byte Japanese rather than ASCII. + """ + mode = b"\x1b(B" + + def replace_control(match: re.Match[bytes]) -> bytes: + nonlocal mode + control = match.group() + if control == b"\x0e": # SO: temporarily select halfwidth Katakana. + return b"\x1b(I" + if control == b"\x0f": # SI: restore the preceding designation. + return mode + mode = control + return control + + normalized = re.sub( + rb"\x1b(?:\([BJI]|\$[@B]|\$\(D)|[\x0e\x0f]", + replace_control, + data, + ) + return normalized.decode("iso2022_jp_ext") + def _get_stream_data(self, msg: Any, stream_path: str) -> Union[str, None]: """Helper to safely extract and decode stream data from the MSG file.""" assert olefile is not None diff --git a/packages/markitdown/tests/test_outlook_msg_ansi.py b/packages/markitdown/tests/test_outlook_msg_ansi.py index 57b4db15d..0c3972332 100644 --- a/packages/markitdown/tests/test_outlook_msg_ansi.py +++ b/packages/markitdown/tests/test_outlook_msg_ansi.py @@ -7,8 +7,9 @@ from unittest.mock import patch import olefile +import pytest -from markitdown import MarkItDown +from markitdown import DocumentConverterResult, MarkItDown from markitdown._stream_info import StreamInfo from markitdown.converters._outlook_msg_converter import OutlookMsgConverter @@ -104,15 +105,146 @@ def close(self): return _FakeOleFileIO -def _convert(streams: dict) -> str: +def _convert_result(streams: dict) -> DocumentConverterResult: with patch.object(olefile, "OleFileIO", _fake_olefile(streams)): - return ( - OutlookMsgConverter() - .convert(io.BytesIO(b""), StreamInfo(extension=".msg")) - .markdown + return OutlookMsgConverter().convert( + io.BytesIO(b""), StreamInfo(extension=".msg") ) +def _convert(streams: dict) -> str: + return _convert_result(streams).markdown + + +@pytest.mark.parametrize("codepage", [1252, 0, 99999, 20127]) +@pytest.mark.parametrize("terminator", [b"\x00", b"\x00\x00"]) +def test_ansi_terminators_are_removed(codepage: int, terminator: bytes) -> None: + """Strip terminators with declared, detected, and contradicted code pages.""" + streams = _ansi_streams(codepage=codepage) + for path in streams: + if path.endswith("001E"): + streams[path] = b" \t" + streams[path] + b"\r\n " + terminator + + result = _convert_result(streams) + + assert result.title == SUBJECT + assert result.markdown == ( + f"# Email Message\n\n**From:** {SENDER}\n**To:** {RECIPIENT}\n" + f"**Subject:** {SUBJECT}\n\n## Content\n\n{BODY}" + ) + + +@pytest.mark.parametrize("codepage", [1252, 0]) +@pytest.mark.parametrize("value", [b"", b"\x00", b"\x00\x00"]) +def test_empty_ansi_properties_are_omitted(codepage: int, value: bytes) -> None: + streams = _ansi_streams(codepage=codepage) + for path in streams: + if path.endswith("001E"): + streams[path] = value + + result = _convert_result(streams) + + assert not result.title + assert result.markdown == "# Email Message\n\n\n## Content" + + +def test_ansi_terminators_are_removed_without_a_detected_charset() -> None: + streams = _ansi_streams(encoding="utf-8", codepage=0) + for path in streams: + streams[path] += b"\x00" + with patch("markitdown.converters._outlook_msg_converter.from_bytes") as detect: + detect.return_value.best.return_value = None + result = _convert_result(streams) + + assert result.title == SUBJECT + assert result.markdown.endswith(BODY) + assert "\x00" not in result.markdown + + +@pytest.mark.parametrize( + "codepage, value, expected", + [ + (50220, b"\x1b$BF|K\\8l\x1b(B", "日本語"), + (50221, b"\x1b(I6@6E\x1b(B", "ｶﾀｶﾅ"), + (50221, b"\x1b$BF|K\\\x1b(I6@6E\x1b(B ABC", "日本ｶﾀｶﾅ ABC"), + (50222, b"Plain ASCII", "Plain ASCII"), + (50222, b"\x0e6@6E\x0f", "ｶﾀｶﾅ"), + (50222, b"ABC \x0e6@6E\x0f XYZ", "ABC ｶﾀｶﾅ XYZ"), + # SI must restore the preceding designation, which may not be ASCII. + (50222, b"\x1b$BF|\x0e6@6E\x0fK\\\x1b(B", "日ｶﾀｶﾅ本"), + (50222, b"\x1b(J\\\x0e6@6E\x0f~\x1b(B", "¥ｶﾀｶﾅ‾"), + (50222, b"\x0e6@\x0f/\x0e6E\x0f", "ｶﾀ/ｶﾅ"), + (50222, b"\x0e6@6E", "ｶﾀｶﾅ"), + ], +) +def test_japanese_codepages_decode_without_detection( + codepage: int, value: bytes, expected: str +) -> None: + """Use literal wire bytes so an incorrect encoder cannot mask a decoder bug.""" + streams = { + f"__substg1.0_{tag}001E": value + b"\x00" + for tag in (SENDER_TAG, RECIPIENT_TAG, SUBJECT_TAG, BODY_TAG) + } + streams["__properties_version1.0"] = _properties_stream( + {PR_MESSAGE_CODEPAGE: codepage} + ) + with patch( + "markitdown.converters._outlook_msg_converter.from_bytes", + side_effect=AssertionError("The declared Japanese code page must be honored"), + ): + result = _convert_result(streams) + + assert result.title == expected + assert result.markdown == ( + f"# Email Message\n\n**From:** {expected}\n**To:** {expected}\n" + f"**Subject:** {expected}\n\n## Content\n\n{expected}" + ) + + +@pytest.mark.parametrize( + "codepage, body", + [(50221, b"\x1b(I6@6E\x1b(B"), (50222, b"\x0e6@6E\x0f")], +) +def test_japanese_internet_codepage_decodes_body(codepage: int, body: bytes) -> None: + streams = { + "__substg1.0_0037001E": AMBIGUOUS_LATIN.encode("cp1252") + b"\x00", + "__substg1.0_1000001E": body + b"\x00", + "__properties_version1.0": _properties_stream( + {PR_MESSAGE_CODEPAGE: 1252, PR_INTERNET_CPID: codepage} + ), + } + with patch( + "markitdown.converters._outlook_msg_converter.from_bytes", + side_effect=AssertionError("The declared code pages must be honored"), + ): + result = _convert_result(streams) + + assert result.title == AMBIGUOUS_LATIN + assert result.markdown == ( + f"# Email Message\n\n**Subject:** {AMBIGUOUS_LATIN}\n\n## Content\n\nｶﾀｶﾅ" + ) + + +@pytest.mark.parametrize( + "codepage, value", + [(50221, b"\x1b(I~\x1b(B"), (50222, b"\x0e~\x0f")], +) +def test_invalid_japanese_bytes_fall_back_to_detection( + codepage: int, value: bytes +) -> None: + """Invalid Katakana must still reach detection with the original bytes.""" + streams = { + "__substg1.0_0037001E": value + b"\x00", + "__properties_version1.0": _properties_stream({PR_MESSAGE_CODEPAGE: codepage}), + } + with patch("markitdown.converters._outlook_msg_converter.from_bytes") as detect: + detect.return_value.best.return_value = "Recovered subject" + result = _convert_result(streams) + + detect.assert_called_once_with(value) + assert result.title == "Recovered subject" + + def test_ansi_message_keeps_headers_and_body() -> None: """A non-Unicode .msg must convert like its Unicode counterpart.""" markdown = _convert(_ansi_streams()) @@ -259,6 +391,4 @@ def test_real_unicode_fixture_still_converts() -> None: if __name__ == "__main__": - import pytest - pytest.main([__file__, "-v"])