Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

from .._base_converter import DocumentConverter, DocumentConverterResult
from .._stream_info import StreamInfo
from ._html_converter import HtmlConverter

# Optional YouTube transcription support
try:
Expand Down Expand Up @@ -35,7 +36,7 @@


class YouTubeConverter(DocumentConverter):
"""Handle YouTube specially, focusing on the video title, description, and transcript."""
"""Extract YouTube metadata and transcripts, or fall back to HTML."""

def _get_video_id(self, url: str) -> Union[str, None]:
"""Extract a YouTube video ID from supported URL formats."""
Expand Down Expand Up @@ -92,6 +93,7 @@ def convert(
**kwargs: Any, # Options to pass to the converter
) -> DocumentConverterResult:
# Parse the stream
start_position = file_stream.tell()
encoding = "utf-8" if stream_info.charset is None else stream_info.charset
soup = bs4.BeautifulSoup(file_stream, "html.parser", from_encoding=encoding)

Expand Down Expand Up @@ -134,7 +136,7 @@ def convert(
pass

# Start preparing the page
webpage_text = "# YouTube\n"
webpage_text = ""

title = self._get(metadata, ["title", "og:title", "name"]) or ""

Expand Down Expand Up @@ -203,8 +205,12 @@ def convert(
if transcript_text:
webpage_text += f"\n### Transcript\n{transcript_text}\n"

if not webpage_text:
Comment thread
afourney marked this conversation as resolved.
file_stream.seek(start_position)
return HtmlConverter().convert(file_stream, stream_info, **kwargs)

return DocumentConverterResult(
markdown=webpage_text,
markdown="# YouTube\n" + webpage_text,
title=title,
)

Expand Down
12 changes: 6 additions & 6 deletions packages/markitdown/tests/test_module_misc.py
Original file line number Diff line number Diff line change
Expand Up @@ -1339,7 +1339,7 @@ def test_pptx_chart_with_title_text_frame() -> None:


def test_youtube_converter_missing_title_metadata() -> None:
"""Test that YouTubeConverter converts streams with and without title metadata without raising AssertionError."""
"""Missing titles fall back to HTML when no video content is extracted."""
from unittest.mock import patch
from markitdown.converters._youtube_converter import YouTubeConverter

Expand All @@ -1358,24 +1358,24 @@ def test_youtube_converter_missing_title_metadata() -> None:
html_content_no_title = b"<html><head></head><body>Video Content</body></html>"
stream_no_title = io.BytesIO(html_content_no_title)
result_no_title = converter.convert(stream_no_title, stream_info)
assert result_no_title.title == ""
assert "# YouTube" in result_no_title.markdown
assert result_no_title.title is None
assert result_no_title.markdown == "Video Content"

# Case 2: Stream with an empty <title> tag
html_content_empty_title = (
b"<html><head><title></title></head><body>Video Content</body></html>"
)
stream_empty_title = io.BytesIO(html_content_empty_title)
result_empty_title = converter.convert(stream_empty_title, stream_info)
assert result_empty_title.title == ""
assert "# YouTube" in result_empty_title.markdown
assert result_empty_title.title is None
assert result_empty_title.markdown == "Video Content"

# Case 3: Stream whose title is only available from the <title> tag
html_content_title_tag = b"<html><head><title>Fallback Title</title></head><body>Video Content</body></html>"
stream_title_tag = io.BytesIO(html_content_title_tag)
result_title_tag = converter.convert(stream_title_tag, stream_info)
assert result_title_tag.title == "Fallback Title"
assert "# YouTube" in result_title_tag.markdown
assert result_title_tag.markdown == "# YouTube\n\n## Fallback Title\n"


def test_zip_duplicate_filenames_preserve_each_entry() -> None:
Expand Down
168 changes: 168 additions & 0 deletions packages/markitdown/tests/test_youtube_converter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
import io
from types import SimpleNamespace
from unittest.mock import MagicMock

import pytest

from markitdown import MarkItDown, StreamInfo
from markitdown.converters import HtmlConverter, YouTubeConverter
import markitdown.converters._youtube_converter as youtube_module


@pytest.fixture
def transcript_api(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
api = MagicMock()
api.return_value.list.return_value = []
api.return_value.fetch.return_value = []
monkeypatch.setattr(youtube_module, "YouTubeTranscriptApi", api, raising=False)
monkeypatch.setattr(youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", False)
return api


@pytest.mark.parametrize("head", ["", "<title></title>"])
@pytest.mark.parametrize("use_dispatcher", [False, True])
@pytest.mark.parametrize("transcript_capable", [False, True])
def test_youtube_empty_extraction_preserves_html(
head: str,
use_dispatcher: bool,
transcript_capable: bool,
transcript_api: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(
youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", transcript_capable
)
html = (
f"<html><head>{head}"
"<script>do_not_render_script()</script>"
"<style>.do_not_render_style { color: red; }</style></head><body>"
"<h1>Saved transcript</h1>"
"<table><tr><th>Time</th><th>Text</th></tr>"
"<tr><td>00:01</td><td>caf\u00e9</td></tr></table></body></html>"
).encode("windows-1252")
prefix = b"<p>Ignored stream prefix</p>"
stream = io.BytesIO(prefix + html)
stream.seek(len(prefix))
stream_info = StreamInfo(
mimetype="text/html",
extension=".html",
charset="windows-1252",
url="https://www.youtube.com/watch?v=12345",
)

if use_dispatcher:
result = MarkItDown().convert_stream(
stream, stream_info=stream_info, heading_style="ATX_CLOSED"
)
else:
result = YouTubeConverter().convert(
stream, stream_info, heading_style="ATX_CLOSED"
)

expected = HtmlConverter().convert(
io.BytesIO(html), stream_info, heading_style="ATX_CLOSED"
)
assert result.markdown == expected.markdown
assert result.title == expected.title
assert "# Saved transcript #" in result.markdown
assert "| 00:01 | caf\u00e9 |" in result.markdown
assert "Ignored stream prefix" not in result.markdown
assert "do_not_render" not in result.markdown
if transcript_capable:
transcript_api.return_value.fetch.assert_called_once()
else:
transcript_api.assert_not_called()


@pytest.mark.parametrize(
("head", "title", "content"),
[
("<title>Video title</title>", "Video title", "\n## Video title\n"),
(
'<meta property="og:title" content="Video title">',
"Video title",
"\n## Video title\n",
),
(
'<meta itemprop="name" content="Video title">',
"Video title",
"\n## Video title\n",
),
(
'<meta name="description" content="Video description">',
"",
"\n### Description\nVideo description\n",
),
(
'<meta property="og:description" content="Video description">',
"",
"\n### Description\nVideo description\n",
),
(
'<script>var ytInitialData = {"attributedDescriptionBodyText": '
'{"content": "Video description"}};</script>',
"",
"\n### Description\nVideo description\n",
),
(
'<meta itemprop="interactionCount" content="42">'
'<meta name="keywords" content="example">'
'<meta itemprop="duration" content="PT1M">',
"",
"\n### Video Metadata\n"
"- **Views:** 42\n- **Keywords:** example\n- **Runtime:** PT1M\n\n",
),
],
)
def test_youtube_preserves_metadata_output(
head: str, title: str, content: str, transcript_api: MagicMock
) -> None:
html = f"<html><head>{head}</head><body>Unused HTML body</body></html>"
result = YouTubeConverter().convert(
io.BytesIO(html.encode("utf-8")),
StreamInfo(
mimetype="text/html",
charset="utf-8",
url="https://www.youtube.com/watch?v=12345",
),
)

assert result.title == title
assert result.markdown == "# YouTube\n" + content
transcript_api.assert_not_called()


@pytest.mark.parametrize("title", ["", "Video title"])
def test_youtube_preserves_library_transcript(
title: str,
transcript_api: MagicMock,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", True)
transcript_api.return_value.list.return_value = [
SimpleNamespace(language_code="fr")
]
transcript_api.return_value.fetch.return_value = [
SimpleNamespace(text="Bonjour"),
SimpleNamespace(text="tout le monde"),
]
head = f"<title>{title}</title>" if title else ""
html = f"<html><head>{head}</head><body>Unused HTML body</body></html>"

result = YouTubeConverter().convert(
io.BytesIO(html.encode("utf-8")),
StreamInfo(
mimetype="text/html",
charset="utf-8",
url="https://www.youtube.com/watch?v=12345",
),
youtube_transcript_languages=["fr"],
)

title_section = f"\n## {title}\n" if title else ""
assert result.title == title
assert result.markdown == (
f"# YouTube\n{title_section}\n### Transcript\nBonjour tout le monde\n"
)
transcript_api.return_value.list.assert_called_once_with("12345")
transcript_api.return_value.fetch.assert_called_once_with("12345", languages=["fr"])