Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 33 additions & 6 deletions packages/markitdown/src/markitdown/_markitdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import shutil
import traceback
import io
import csv
from dataclasses import dataclass
from email.message import Message
from email.utils import collapse_rfc2231_value
Expand Down Expand Up @@ -98,6 +99,18 @@ def _read_charset_sample(file_stream: BinaryIO) -> bytes:
return sample


def _looks_like_delimited_text(data: bytes) -> bool:
detected = charset_normalizer.from_bytes(data).best()
text = str(detected) if detected is not None else data.decode("utf-8", "ignore")

try:
dialect = csv.Sniffer().sniff(text.lstrip("\ufeff"), delimiters=",;\t|")
except csv.Error:
return False

return dialect.delimiter in {",", ";", "\t", "|"}


# Lower priority values are tried first.
PRIORITY_SPECIFIC_FILE_FORMAT = (
0.0 # e.g., .docx, .pdf, .xlsx, Or specific pages, e.g., wikipedia
Expand Down Expand Up @@ -778,24 +791,38 @@ def _get_stream_info_guesses(

if charset_result is not None:
charset = self._normalize_charset(charset_result.encoding)
else:
stream_page = b""

output_mimetype = result.prediction.output.mime_type
output_extensions = result.prediction.output.extensions
if (
base_guess.mimetype is None
and base_guess.extension is None
and result.prediction.output.is_text
and output_mimetype == "text/csv"
and not _looks_like_delimited_text(stream_page)
):
output_mimetype = "text/plain"
output_extensions = ["txt"]

# Normalize the first extension listed
guessed_extension = None
if len(result.prediction.output.extensions) > 0:
guessed_extension = "." + result.prediction.output.extensions[0]
if len(output_extensions) > 0:
guessed_extension = "." + output_extensions[0]

# Determine if the guess is compatible with the base guess
compatible = True
if (
base_guess.mimetype is not None
and base_guess.mimetype != result.prediction.output.mime_type
and base_guess.mimetype != output_mimetype
):
compatible = False

if (
base_guess.extension is not None
and base_guess.extension.lstrip(".")
not in result.prediction.output.extensions
not in output_extensions
):
compatible = False

Expand All @@ -810,7 +837,7 @@ def _get_stream_info_guesses(
guesses.append(
StreamInfo(
mimetype=base_guess.mimetype
or result.prediction.output.mime_type,
or output_mimetype,
extension=base_guess.extension or guessed_extension,
charset=base_guess.charset or charset,
filename=base_guess.filename,
Expand All @@ -823,7 +850,7 @@ def _get_stream_info_guesses(
guesses.append(enhanced_guess)
guesses.append(
StreamInfo(
mimetype=result.prediction.output.mime_type,
mimetype=output_mimetype,
extension=guessed_extension,
charset=charset,
filename=base_guess.filename,
Expand Down
25 changes: 24 additions & 1 deletion packages/markitdown/src/markitdown/converters/_csv_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,29 @@ def _trim_outer_blank_rows(rows: list[list[str]]) -> None:
del rows[:header_index]


def _looks_like_delimited_csv(file_stream: BinaryIO) -> bool:
"""Return true when extensionless text has a real CSV-style delimiter."""
cur_pos = file_stream.tell()
try:
data = file_stream.read(8192)
finally:
file_stream.seek(cur_pos)

if not data:
return False

detected = from_bytes(data).best()
content = str(detected) if detected is not None else data.decode("utf-8", "ignore")
sample = content.lstrip("\ufeff")

try:
dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|")
except csv.Error:
return False

return dialect.delimiter in {",", ";", "\t", "|"}


class CsvConverter(DocumentConverter):
"""
Converts CSV files to Markdown tables.
Expand All @@ -74,7 +97,7 @@ def accepts(
return True
for prefix in ACCEPTED_MIME_TYPE_PREFIXES:
if mimetype.startswith(prefix):
return True
return _looks_like_delimited_csv(file_stream)
return False

def convert(
Expand Down
26 changes: 26 additions & 0 deletions packages/markitdown/tests/test_csv_detection.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import io

from markitdown import MarkItDown


def test_extensionless_log_text_is_not_rendered_as_csv_table() -> None:
content = (
"2026-09-12 18:00:01 INFO boot ok\n"
"2026-09-12 18:00:02 ERROR disk full\n"
)

result = MarkItDown(enable_plugins=False).convert_stream(
io.BytesIO(content.encode("utf-8"))
)

assert result.markdown == content


def test_extensionless_comma_csv_is_still_rendered_as_table() -> None:
content = "name,age\nAlice,30\n"

result = MarkItDown(enable_plugins=False).convert_stream(
io.BytesIO(content.encode("utf-8"))
)

assert result.markdown == "| name | age |\n| --- | --- |\n| Alice | 30 |"