diff --git a/packages/markitdown/src/markitdown/converters/_youtube_converter.py b/packages/markitdown/src/markitdown/converters/_youtube_converter.py index c3779743c6..70e8457bc1 100644 --- a/packages/markitdown/src/markitdown/converters/_youtube_converter.py +++ b/packages/markitdown/src/markitdown/converters/_youtube_converter.py @@ -7,6 +7,7 @@ from .._base_converter import DocumentConverter, DocumentConverterResult from .._stream_info import StreamInfo +from ._html_converter import HtmlConverter # Optional YouTube transcription support try: @@ -35,7 +36,7 @@ class YouTubeConverter(DocumentConverter): - """Handle YouTube specially, focusing on the video title, description, and transcript.""" + """Extract YouTube metadata and transcripts, or fall back to HTML.""" def _get_video_id(self, url: str) -> Union[str, None]: """Extract a YouTube video ID from supported URL formats.""" @@ -92,6 +93,7 @@ def convert( **kwargs: Any, # Options to pass to the converter ) -> DocumentConverterResult: # Parse the stream + start_position = file_stream.tell() encoding = "utf-8" if stream_info.charset is None else stream_info.charset soup = bs4.BeautifulSoup(file_stream, "html.parser", from_encoding=encoding) @@ -134,7 +136,7 @@ def convert( pass # Start preparing the page - webpage_text = "# YouTube\n" + webpage_text = "" title = self._get(metadata, ["title", "og:title", "name"]) or "" @@ -203,8 +205,12 @@ def convert( if transcript_text: webpage_text += f"\n### Transcript\n{transcript_text}\n" + if not webpage_text: + file_stream.seek(start_position) + return HtmlConverter().convert(file_stream, stream_info, **kwargs) + return DocumentConverterResult( - markdown=webpage_text, + markdown="# YouTube\n" + webpage_text, title=title, ) diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py index bc0312460f..18690d1889 100644 --- a/packages/markitdown/tests/test_module_misc.py +++ b/packages/markitdown/tests/test_module_misc.py @@ -1339,7 +1339,7 @@ def test_pptx_chart_with_title_text_frame() -> None: def test_youtube_converter_missing_title_metadata() -> None: - """Test that YouTubeConverter converts streams with and without title metadata without raising AssertionError.""" + """Missing titles fall back to HTML when no video content is extracted.""" from unittest.mock import patch from markitdown.converters._youtube_converter import YouTubeConverter @@ -1358,8 +1358,8 @@ def test_youtube_converter_missing_title_metadata() -> None: html_content_no_title = b"Video Content" stream_no_title = io.BytesIO(html_content_no_title) result_no_title = converter.convert(stream_no_title, stream_info) - assert result_no_title.title == "" - assert "# YouTube" in result_no_title.markdown + assert result_no_title.title is None + assert result_no_title.markdown == "Video Content" # Case 2: Stream with an empty tag html_content_empty_title = ( @@ -1367,15 +1367,15 @@ def test_youtube_converter_missing_title_metadata() -> None: ) stream_empty_title = io.BytesIO(html_content_empty_title) result_empty_title = converter.convert(stream_empty_title, stream_info) - assert result_empty_title.title == "" - assert "# YouTube" in result_empty_title.markdown + assert result_empty_title.title is None + assert result_empty_title.markdown == "Video Content" # Case 3: Stream whose title is only available from the <title> tag html_content_title_tag = b"<html><head><title>Fallback TitleVideo Content" stream_title_tag = io.BytesIO(html_content_title_tag) result_title_tag = converter.convert(stream_title_tag, stream_info) assert result_title_tag.title == "Fallback Title" - assert "# YouTube" in result_title_tag.markdown + assert result_title_tag.markdown == "# YouTube\n\n## Fallback Title\n" def test_zip_duplicate_filenames_preserve_each_entry() -> None: diff --git a/packages/markitdown/tests/test_youtube_converter.py b/packages/markitdown/tests/test_youtube_converter.py new file mode 100644 index 0000000000..4293f620f2 --- /dev/null +++ b/packages/markitdown/tests/test_youtube_converter.py @@ -0,0 +1,168 @@ +import io +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from markitdown import MarkItDown, StreamInfo +from markitdown.converters import HtmlConverter, YouTubeConverter +import markitdown.converters._youtube_converter as youtube_module + + +@pytest.fixture +def transcript_api(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + api = MagicMock() + api.return_value.list.return_value = [] + api.return_value.fetch.return_value = [] + monkeypatch.setattr(youtube_module, "YouTubeTranscriptApi", api, raising=False) + monkeypatch.setattr(youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", False) + return api + + +@pytest.mark.parametrize("head", ["", ""]) +@pytest.mark.parametrize("use_dispatcher", [False, True]) +@pytest.mark.parametrize("transcript_capable", [False, True]) +def test_youtube_empty_extraction_preserves_html( + head: str, + use_dispatcher: bool, + transcript_capable: bool, + transcript_api: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", transcript_capable + ) + html = ( + f"{head}" + "" + "" + "

Saved transcript

" + "" + "
TimeText
00:01caf\u00e9
" + ).encode("windows-1252") + prefix = b"

Ignored stream prefix

" + stream = io.BytesIO(prefix + html) + stream.seek(len(prefix)) + stream_info = StreamInfo( + mimetype="text/html", + extension=".html", + charset="windows-1252", + url="https://www.youtube.com/watch?v=12345", + ) + + if use_dispatcher: + result = MarkItDown().convert_stream( + stream, stream_info=stream_info, heading_style="ATX_CLOSED" + ) + else: + result = YouTubeConverter().convert( + stream, stream_info, heading_style="ATX_CLOSED" + ) + + expected = HtmlConverter().convert( + io.BytesIO(html), stream_info, heading_style="ATX_CLOSED" + ) + assert result.markdown == expected.markdown + assert result.title == expected.title + assert "# Saved transcript #" in result.markdown + assert "| 00:01 | caf\u00e9 |" in result.markdown + assert "Ignored stream prefix" not in result.markdown + assert "do_not_render" not in result.markdown + if transcript_capable: + transcript_api.return_value.fetch.assert_called_once() + else: + transcript_api.assert_not_called() + + +@pytest.mark.parametrize( + ("head", "title", "content"), + [ + ("Video title", "Video title", "\n## Video title\n"), + ( + '', + "Video title", + "\n## Video title\n", + ), + ( + '', + "Video title", + "\n## Video title\n", + ), + ( + '', + "", + "\n### Description\nVideo description\n", + ), + ( + '', + "", + "\n### Description\nVideo description\n", + ), + ( + '', + "", + "\n### Description\nVideo description\n", + ), + ( + '' + '' + '', + "", + "\n### Video Metadata\n" + "- **Views:** 42\n- **Keywords:** example\n- **Runtime:** PT1M\n\n", + ), + ], +) +def test_youtube_preserves_metadata_output( + head: str, title: str, content: str, transcript_api: MagicMock +) -> None: + html = f"{head}Unused HTML body" + result = YouTubeConverter().convert( + io.BytesIO(html.encode("utf-8")), + StreamInfo( + mimetype="text/html", + charset="utf-8", + url="https://www.youtube.com/watch?v=12345", + ), + ) + + assert result.title == title + assert result.markdown == "# YouTube\n" + content + transcript_api.assert_not_called() + + +@pytest.mark.parametrize("title", ["", "Video title"]) +def test_youtube_preserves_library_transcript( + title: str, + transcript_api: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(youtube_module, "IS_YOUTUBE_TRANSCRIPT_CAPABLE", True) + transcript_api.return_value.list.return_value = [ + SimpleNamespace(language_code="fr") + ] + transcript_api.return_value.fetch.return_value = [ + SimpleNamespace(text="Bonjour"), + SimpleNamespace(text="tout le monde"), + ] + head = f"{title}" if title else "" + html = f"{head}Unused HTML body" + + result = YouTubeConverter().convert( + io.BytesIO(html.encode("utf-8")), + StreamInfo( + mimetype="text/html", + charset="utf-8", + url="https://www.youtube.com/watch?v=12345", + ), + youtube_transcript_languages=["fr"], + ) + + title_section = f"\n## {title}\n" if title else "" + assert result.title == title + assert result.markdown == ( + f"# YouTube\n{title_section}\n### Transcript\nBonjour tout le monde\n" + ) + transcript_api.return_value.list.assert_called_once_with("12345") + transcript_api.return_value.fetch.assert_called_once_with("12345", languages=["fr"])