Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 23 additions & 9 deletions packages/markitdown/src/markitdown/converters/_html_converter.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,18 +53,33 @@ def convert(
encoding = "utf-8" if stream_info.charset is None else stream_info.charset
soup = BeautifulSoup(file_stream, "html.parser", from_encoding=encoding)

# Capture the title before any element is removed from the soup
title = None if soup.title is None else soup.title.string

# Remove javascript and style blocks
for script in soup(["script", "style"]):
script.extract()

# Print only the main content
body_elm = soup.find("body")
# A browser moves stray content into <body> and keeps document
# metadata in <head>, but the html.parser tree leaves elements where
# they were written. Converting only the <body> element would drop
# content placed before or after it, while converting the whole
# document would leak <head> text (e.g. the <title>) into the
# markdown. Remove the metadata containers instead, and convert
# everything that remains.
for head in soup(["head"]):
head.extract()

# HTML5 allows the <head> tags to be omitted, so a document <title>
# can sit outside any <head> element. A <title> inside SVG or MathML
# describes a graphic and is content, so those are kept.
for title_elm in soup.find_all("title"):
if title_elm.find_parent(["svg", "math"]) is None:
title_elm.extract()

webpage_text = ""
try:
if body_elm:
webpage_text = _CustomMarkdownify(**kwargs).convert_soup(body_elm)
else:
webpage_text = _CustomMarkdownify(**kwargs).convert_soup(soup)
webpage_text = _CustomMarkdownify(**kwargs).convert_soup(soup)
except RecursionError:
if strict:
raise
Expand All @@ -77,8 +92,7 @@ def convert(
"(RecursionError). Falling back to plain-text extraction.",
stacklevel=2,
)
target = body_elm if body_elm else soup
webpage_text = target.get_text("\n", strip=True)
webpage_text = soup.get_text("\n", strip=True)

assert isinstance(webpage_text, str)

Expand All @@ -87,7 +101,7 @@ def convert(

return DocumentConverterResult(
markdown=webpage_text,
title=None if soup.title is None else soup.title.string,
title=title,
)

def convert_string(
Expand Down
55 changes: 55 additions & 0 deletions packages/markitdown/tests/test_html_head_content.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import io

from markitdown import MarkItDown


def _convert_html(html: str):
return MarkItDown().convert_stream(
io.BytesIO(html.encode("utf-8")),
file_extension=".html",
)


def test_title_does_not_leak_into_markdown_when_body_tag_is_absent() -> None:
# HTML5 allows <html>, <head> and <body> tags to be omitted, and many
# hand-written pages do. Nothing here moves <title> out of the document
# tree the way a browser would, so its text must not appear in the body.
result = _convert_html(
"<!DOCTYPE html><title>Page Title</title><p>Hello body</p>"
)

assert result.markdown == "Hello body"
assert result.title == "Page Title"


def test_head_element_does_not_leak_into_markdown_when_body_tag_is_absent() -> None:
result = _convert_html(
"<html><head><title>T</title></head><h1>Heading</h1></html>"
)

assert result.markdown == "# Heading"
assert result.title == "T"


def test_content_outside_body_element_is_not_dropped() -> None:
# Content before or after <body> is still rendered by browsers, so it
# must survive conversion instead of being silently discarded.
result = _convert_html(
"<html><p>Intro outside body</p><body><p>Main</p></body>"
"<p>Footer outside</p></html>"
)

assert "Intro outside body" in result.markdown
assert "Main" in result.markdown
assert "Footer outside" in result.markdown


def test_svg_title_is_preserved() -> None:
# <title> inside <svg> describes a graphic and is content, unlike a
# document <title>, so it must survive the metadata removal.
result = _convert_html(
"<body><svg><title>Square</title><rect width='4' height='4'/></svg>"
"<p>text</p></body>"
)

assert "Square" in result.markdown