diff --git a/packages/markitdown/src/markitdown/converters/_html_converter.py b/packages/markitdown/src/markitdown/converters/_html_converter.py
index 029b27f57..463e2045b 100644
--- a/packages/markitdown/src/markitdown/converters/_html_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_html_converter.py
@@ -53,18 +53,33 @@ def convert(
encoding = "utf-8" if stream_info.charset is None else stream_info.charset
soup = BeautifulSoup(file_stream, "html.parser", from_encoding=encoding)
+ # Capture the title before any element is removed from the soup
+ title = None if soup.title is None else soup.title.string
+
# Remove javascript and style blocks
for script in soup(["script", "style"]):
script.extract()
- # Print only the main content
- body_elm = soup.find("body")
+ # A browser moves stray content into and keeps document
+ # metadata in , but the html.parser tree leaves elements where
+ # they were written. Converting only the element would drop
+ # content placed before or after it, while converting the whole
+ # document would leak text (e.g. the ) into the
+ # markdown. Remove the metadata containers instead, and convert
+ # everything that remains.
+ for head in soup(["head"]):
+ head.extract()
+
+ # HTML5 allows the <head> tags to be omitted, so a document <title>
+ # can sit outside any <head> element. A <title> inside SVG or MathML
+ # describes a graphic and is content, so those are kept.
+ for title_elm in soup.find_all("title"):
+ if title_elm.find_parent(["svg", "math"]) is None:
+ title_elm.extract()
+
webpage_text = ""
try:
- if body_elm:
- webpage_text = _CustomMarkdownify(**kwargs).convert_soup(body_elm)
- else:
- webpage_text = _CustomMarkdownify(**kwargs).convert_soup(soup)
+ webpage_text = _CustomMarkdownify(**kwargs).convert_soup(soup)
except RecursionError:
if strict:
raise
@@ -77,8 +92,7 @@ def convert(
"(RecursionError). Falling back to plain-text extraction.",
stacklevel=2,
)
- target = body_elm if body_elm else soup
- webpage_text = target.get_text("\n", strip=True)
+ webpage_text = soup.get_text("\n", strip=True)
assert isinstance(webpage_text, str)
@@ -87,7 +101,7 @@ def convert(
return DocumentConverterResult(
markdown=webpage_text,
- title=None if soup.title is None else soup.title.string,
+ title=title,
)
def convert_string(
diff --git a/packages/markitdown/tests/test_html_head_content.py b/packages/markitdown/tests/test_html_head_content.py
new file mode 100644
index 000000000..d6821bb2c
--- /dev/null
+++ b/packages/markitdown/tests/test_html_head_content.py
@@ -0,0 +1,55 @@
+import io
+
+from markitdown import MarkItDown
+
+
+def _convert_html(html: str):
+ return MarkItDown().convert_stream(
+ io.BytesIO(html.encode("utf-8")),
+ file_extension=".html",
+ )
+
+
+def test_title_does_not_leak_into_markdown_when_body_tag_is_absent() -> None:
+ # HTML5 allows <html>, <head> and <body> tags to be omitted, and many
+ # hand-written pages do. Nothing here moves <title> out of the document
+ # tree the way a browser would, so its text must not appear in the body.
+ result = _convert_html(
+ "<!DOCTYPE html><title>Page TitleHello body
"
+ )
+
+ assert result.markdown == "Hello body"
+ assert result.title == "Page Title"
+
+
+def test_head_element_does_not_leak_into_markdown_when_body_tag_is_absent() -> None:
+ result = _convert_html(
+ "THeading
"
+ )
+
+ assert result.markdown == "# Heading"
+ assert result.title == "T"
+
+
+def test_content_outside_body_element_is_not_dropped() -> None:
+ # Content before or after is still rendered by browsers, so it
+ # must survive conversion instead of being silently discarded.
+ result = _convert_html(
+ "Intro outside body
Main
"
+ "Footer outside
"
+ )
+
+ assert "Intro outside body" in result.markdown
+ assert "Main" in result.markdown
+ assert "Footer outside" in result.markdown
+
+
+def test_svg_title_is_preserved() -> None:
+ # inside <svg> describes a graphic and is content, unlike a
+ # document <title>, so it must survive the metadata removal.
+ result = _convert_html(
+ "<body><svg><title>Square"
+ "text
"
+ )
+
+ assert "Square" in result.markdown