diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index 888d1e6260..b6e03339a4 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -5,6 +5,7 @@ import shutil import traceback import io +import csv from dataclasses import dataclass from email.message import Message from email.utils import collapse_rfc2231_value @@ -98,6 +99,18 @@ def _read_charset_sample(file_stream: BinaryIO) -> bytes: return sample +def _looks_like_delimited_text(data: bytes) -> bool: + detected = charset_normalizer.from_bytes(data).best() + text = str(detected) if detected is not None else data.decode("utf-8", "ignore") + + try: + dialect = csv.Sniffer().sniff(text.lstrip("\ufeff"), delimiters=",;\t|") + except csv.Error: + return False + + return dialect.delimiter in {",", ";", "\t", "|"} + + # Lower priority values are tried first. PRIORITY_SPECIFIC_FILE_FORMAT = ( 0.0 # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia @@ -778,24 +791,38 @@ def _get_stream_info_guesses( if charset_result is not None: charset = self._normalize_charset(charset_result.encoding) + else: + stream_page = b"" + + output_mimetype = result.prediction.output.mime_type + output_extensions = result.prediction.output.extensions + if ( + base_guess.mimetype is None + and base_guess.extension is None + and result.prediction.output.is_text + and output_mimetype == "text/csv" + and not _looks_like_delimited_text(stream_page) + ): + output_mimetype = "text/plain" + output_extensions = ["txt"] # Normalize the first extension listed guessed_extension = None - if len(result.prediction.output.extensions) > 0: - guessed_extension = "." + result.prediction.output.extensions[0] + if len(output_extensions) > 0: + guessed_extension = "." + output_extensions[0] # Determine if the guess is compatible with the base guess compatible = True if ( base_guess.mimetype is not None - and base_guess.mimetype != result.prediction.output.mime_type + and base_guess.mimetype != output_mimetype ): compatible = False if ( base_guess.extension is not None and base_guess.extension.lstrip(".") - not in result.prediction.output.extensions + not in output_extensions ): compatible = False @@ -810,7 +837,7 @@ def _get_stream_info_guesses( guesses.append( StreamInfo( mimetype=base_guess.mimetype - or result.prediction.output.mime_type, + or output_mimetype, extension=base_guess.extension or guessed_extension, charset=base_guess.charset or charset, filename=base_guess.filename, @@ -823,7 +850,7 @@ def _get_stream_info_guesses( guesses.append(enhanced_guess) guesses.append( StreamInfo( - mimetype=result.prediction.output.mime_type, + mimetype=output_mimetype, extension=guessed_extension, charset=charset, filename=base_guess.filename, diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py index 331b0712a3..3c4bf4c990 100644 --- a/packages/markitdown/src/markitdown/converters/_csv_converter.py +++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py @@ -54,6 +54,29 @@ def _trim_outer_blank_rows(rows: list[list[str]]) -> None: del rows[:header_index] +def _looks_like_delimited_csv(file_stream: BinaryIO) -> bool: + """Return true when extensionless text has a real CSV-style delimiter.""" + cur_pos = file_stream.tell() + try: + data = file_stream.read(8192) + finally: + file_stream.seek(cur_pos) + + if not data: + return False + + detected = from_bytes(data).best() + content = str(detected) if detected is not None else data.decode("utf-8", "ignore") + sample = content.lstrip("\ufeff") + + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|") + except csv.Error: + return False + + return dialect.delimiter in {",", ";", "\t", "|"} + + class CsvConverter(DocumentConverter): """ Converts CSV files to Markdown tables. @@ -74,7 +97,7 @@ def accepts( return True for prefix in ACCEPTED_MIME_TYPE_PREFIXES: if mimetype.startswith(prefix): - return True + return _looks_like_delimited_csv(file_stream) return False def convert( diff --git a/packages/markitdown/tests/test_csv_detection.py b/packages/markitdown/tests/test_csv_detection.py new file mode 100644 index 0000000000..e1ec0a18d6 --- /dev/null +++ b/packages/markitdown/tests/test_csv_detection.py @@ -0,0 +1,26 @@ +import io + +from markitdown import MarkItDown + + +def test_extensionless_log_text_is_not_rendered_as_csv_table() -> None: + content = ( + "2026-09-12 18:00:01 INFO boot ok\n" + "2026-09-12 18:00:02 ERROR disk full\n" + ) + + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(content.encode("utf-8")) + ) + + assert result.markdown == content + + +def test_extensionless_comma_csv_is_still_rendered_as_table() -> None: + content = "name,age\nAlice,30\n" + + result = MarkItDown(enable_plugins=False).convert_stream( + io.BytesIO(content.encode("utf-8")) + ) + + assert result.markdown == "| name | age |\n| --- | --- |\n| Alice | 30 |"