From 5717e3b8ceaf26d9c3414df8e15abcda06df1f9c Mon Sep 17 00:00:00 2001 From: Manohar Paturi <186662190+ManoharPaturi@users.noreply.github.com> Date: Sun, 13 Sep 2026 21:25:07 +0530 Subject: [PATCH] fix(html): keep head metadata out of markdown without dropping stray content A browser relocates stray content into and keeps document metadata in , but BeautifulSoup's html.parser leaves elements where they were written. The converter converted only the element when one existed, which silently dropped content placed before or after it, and converted the whole document when no existed, which leaked (and other <head>) text into the markdown body. Remove the metadata containers (<head>, and a document <title> that is not inside SVG/MathML) instead of selecting the <body> element, and convert everything that remains. The document title is now captured before removal. --- .../markitdown/converters/_html_converter.py | 32 ++++++++--- .../tests/test_html_head_content.py | 55 +++++++++++++++++++ 2 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 packages/markitdown/tests/test_html_head_content.py diff --git a/packages/markitdown/src/markitdown/converters/_html_converter.py b/packages/markitdown/src/markitdown/converters/_html_converter.py index 029b27f57b..463e2045b6 100644 --- a/packages/markitdown/src/markitdown/converters/_html_converter.py +++ b/packages/markitdown/src/markitdown/converters/_html_converter.py @@ -53,18 +53,33 @@ def convert( encoding = "utf-8" if stream_info.charset is None else stream_info.charset soup = BeautifulSoup(file_stream, "html.parser", from_encoding=encoding) + # Capture the title before any element is removed from the soup + title = None if soup.title is None else soup.title.string + # Remove javascript and style blocks for script in soup(["script", "style"]): script.extract() - # Print only the main content - body_elm = soup.find("body") + # A browser moves stray content into <body> and keeps document + # metadata in <head>, but the html.parser tree leaves elements where + # they were written. Converting only the <body> element would drop + # content placed before or after it, while converting the whole + # document would leak <head> text (e.g. the <title>) into the + # markdown. Remove the metadata containers instead, and convert + # everything that remains. + for head in soup(["head"]): + head.extract() + + # HTML5 allows the <head> tags to be omitted, so a document <title> + # can sit outside any <head> element. A <title> inside SVG or MathML + # describes a graphic and is content, so those are kept. + for title_elm in soup.find_all("title"): + if title_elm.find_parent(["svg", "math"]) is None: + title_elm.extract() + webpage_text = "" try: - if body_elm: - webpage_text = _CustomMarkdownify(**kwargs).convert_soup(body_elm) - else: - webpage_text = _CustomMarkdownify(**kwargs).convert_soup(soup) + webpage_text = _CustomMarkdownify(**kwargs).convert_soup(soup) except RecursionError: if strict: raise @@ -77,8 +92,7 @@ def convert( "(RecursionError). Falling back to plain-text extraction.", stacklevel=2, ) - target = body_elm if body_elm else soup - webpage_text = target.get_text("\n", strip=True) + webpage_text = soup.get_text("\n", strip=True) assert isinstance(webpage_text, str) @@ -87,7 +101,7 @@ def convert( return DocumentConverterResult( markdown=webpage_text, - title=None if soup.title is None else soup.title.string, + title=title, ) def convert_string( diff --git a/packages/markitdown/tests/test_html_head_content.py b/packages/markitdown/tests/test_html_head_content.py new file mode 100644 index 0000000000..d6821bb2ca --- /dev/null +++ b/packages/markitdown/tests/test_html_head_content.py @@ -0,0 +1,55 @@ +import io + +from markitdown import MarkItDown + + +def _convert_html(html: str): + return MarkItDown().convert_stream( + io.BytesIO(html.encode("utf-8")), + file_extension=".html", + ) + + +def test_title_does_not_leak_into_markdown_when_body_tag_is_absent() -> None: + # HTML5 allows <html>, <head> and <body> tags to be omitted, and many + # hand-written pages do. Nothing here moves <title> out of the document + # tree the way a browser would, so its text must not appear in the body. + result = _convert_html( + "<!DOCTYPE html><title>Page Title

Hello body

" + ) + + assert result.markdown == "Hello body" + assert result.title == "Page Title" + + +def test_head_element_does_not_leak_into_markdown_when_body_tag_is_absent() -> None: + result = _convert_html( + "T

Heading

" + ) + + assert result.markdown == "# Heading" + assert result.title == "T" + + +def test_content_outside_body_element_is_not_dropped() -> None: + # Content before or after is still rendered by browsers, so it + # must survive conversion instead of being silently discarded. + result = _convert_html( + "

Intro outside body

Main

" + "

Footer outside

" + ) + + assert "Intro outside body" in result.markdown + assert "Main" in result.markdown + assert "Footer outside" in result.markdown + + +def test_svg_title_is_preserved() -> None: + # inside <svg> describes a graphic and is content, unlike a + # document <title>, so it must survive the metadata removal. + result = _convert_html( + "<body><svg><title>Square" + "

text

" + ) + + assert "Square" in result.markdown