diff --git a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py index 6a26560a9..e6a28560e 100644 --- a/packages/markitdown/src/markitdown/converters/_plain_text_converter.py +++ b/packages/markitdown/src/markitdown/converters/_plain_text_converter.py @@ -52,9 +52,17 @@ def convert( stream_info: StreamInfo, **kwargs: Any, # Options to pass to the converter ) -> DocumentConverterResult: + raw_content = file_stream.read() if stream_info.charset: - text_content = file_stream.read().decode(stream_info.charset) + try: + text_content = raw_content.decode(stream_info.charset) + except UnicodeDecodeError: + # The charset was guessed from a small sample of the content + # (e.g. the first 4k bytes) and may not hold for the whole + # file. Fall back to detecting from the full content. + text_content = str(from_bytes(raw_content).best()) else: + text_content = str(from_bytes(raw_content).best()) data = file_stream.read() detected = from_bytes(data).best() text_content = ( diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index 18690d188..a2b73dd43 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -817,6 +817,16 @@ def test_input_as_strings() -> None: assert "# Test" in result.text_content +def test_plain_text_charset_guessed_from_prefix() -> None: + # Charset is guessed from only the first 4096 bytes of the stream. If + # that prefix happens to be pure ASCII but a multi-byte UTF-8 character + # (e.g. an em dash) appears later, the charset would previously be + # mis-detected as "ascii" and decoding the full content would raise + # UnicodeDecodeError. See PlainTextConverter.convert. + markitdown = MarkItDown() + input_data = ("a" * 4096 + "em dash — end").encode("utf-8") + result = markitdown.convert_stream(io.BytesIO(input_data), file_extension=".txt") + assert "em dash — end" in result.text_content def _mock_response(content_disposition: str) -> MagicMock: response = MagicMock() response.headers = {"content-disposition": content_disposition}