From 38c6da16f0295e08dd0f011730dceb4a49e0d0cb Mon Sep 17 00:00:00 2001 From: mattv8 Date: Thu, 6 Aug 2026 11:21:03 -0600 Subject: [PATCH 1/2] feat: replace document parsing with anydoc --- docker/Dockerfile | 2 +- docker/Dockerfile.runtime | 2 + pyproject.toml | 12 +- ragtime/core/document_conversion.py | 75 ++++++ ragtime/core/file_constants.py | 56 +++-- ragtime/indexer/document_parser.py | 378 ++-------------------------- runtime/worker/service.py | 29 ++- tests/test_document_extensions.py | 57 +++++ tests/test_document_parser.py | 238 ++++++++++++++++++ tests/test_runtime_pdf_read.py | 66 +++++ 10 files changed, 504 insertions(+), 411 deletions(-) create mode 100644 ragtime/core/document_conversion.py create mode 100644 tests/test_document_extensions.py create mode 100644 tests/test_document_parser.py diff --git a/docker/Dockerfile b/docker/Dockerfile index bebf0e73..0c7df326 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -107,7 +107,7 @@ RUN --mount=type=cache,id=ragtime-pip-cache,target=/root/.cache/pip,sharing=lock else \ pip install -r /tmp/requirements.app.txt; \ fi && \ - python -c "import chonkie, pandas" + python -c "import anydoc, chonkie, pandas; assert callable(anydoc.to_markdown_bytes); assert anydoc.format_from_extension('csv') is not None" # ============================================================================= # Stage 3: Python CI base diff --git a/docker/Dockerfile.runtime b/docker/Dockerfile.runtime index 6352b6c9..f782e4b0 100644 --- a/docker/Dockerfile.runtime +++ b/docker/Dockerfile.runtime @@ -67,6 +67,7 @@ RUN --mount=type=cache,id=ragtime-pip-cache,target=/root/.cache/pip,sharing=lock pip install --upgrade pip && \ python /tmp/install_deps_from_pyproject.py /runtime/pyproject.toml runtime /tmp/requirements.runtime.txt && \ pip install -r /tmp/requirements.runtime.txt && \ + python -c "import anydoc; assert callable(anydoc.to_markdown_bytes); assert anydoc.format_from_extension('csv') is not None" && \ pip install poetry pipenv uv && \ curl -fsSL https://bun.sh/install | bash @@ -74,6 +75,7 @@ COPY runtime/ /runtime/runtime/ RUN mkdir -p /runtime/ragtime/core COPY ragtime/__init__.py /runtime/ragtime/__init__.py COPY ragtime/core/__init__.py /runtime/ragtime/core/__init__.py +COPY ragtime/core/document_conversion.py /runtime/ragtime/core/document_conversion.py COPY ragtime/core/file_constants.py /runtime/ragtime/core/file_constants.py COPY docker/entrypoint.runtime.sh /entrypoint.runtime.sh diff --git a/pyproject.toml b/pyproject.toml index afc50db9..b7612265 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,16 +44,12 @@ app = [ "python-jose[cryptography]>=3.3.0,<4.0.0", "webauthn>=2.0.0,<3.0.0", "slowapi>=0.1.9,<1.0.0", - "pypdf>=4.0.0,<5.0.0", + "firecrawl-anydoc==0.1.6", "python-docx>=1.1.0,<2.0.0", "openpyxl>=3.1.0,<4.0.0", "pandas>=2.2.0,<3.0.0", - "python-pptx>=0.6.21,<1.0.0", - "odfpy>=1.4.1,<2.0.0", "xlrd>=2.0.1,<3.0.0", "chonkie[code]>=1.3.1,<2.0.0", - "striprtf>=0.0.26,<1.0.0", - "ebooklib>=0.18,<1.0.0", "beautifulsoup4>=4.12.0,<5.0.0", "lxml>=5.0.0,<6.0.0", "extract-msg>=0.48.0,<1.0.0", @@ -72,7 +68,7 @@ runtime = [ "pydantic>=2.5.0,<3.0.0", "httpx>=0.26.0,<1.0.0", "mcp>=1.0.0,<2.0.0", - "pypdf>=4.0.0,<5.0.0", + "firecrawl-anydoc==0.1.6", ] test = [ "mypy>=1.15.0,<2.0.0", @@ -166,10 +162,6 @@ warn_unused_ignores = false [[tool.mypy.overrides]] module = [ - "ebooklib", - "odf", - "odf.*", - "pptx", "pytesseract", ] ignore_missing_imports = true diff --git a/ragtime/core/document_conversion.py b/ragtime/core/document_conversion.py new file mode 100644 index 00000000..e2fc12bb --- /dev/null +++ b/ragtime/core/document_conversion.py @@ -0,0 +1,75 @@ +"""Shared AnyDoc conversion adapter.""" + +import os +import threading +from dataclasses import dataclass +from enum import Enum + + +class DocumentConversionFailure(str, Enum): + UNSUPPORTED = "unsupported" + MALFORMED = "malformed" + ENCRYPTED = "encrypted" + RESOURCE_LIMIT = "resource_limit" + MISSING_PART = "missing_part" + DEPENDENCY = "dependency" + UNEXPECTED = "unexpected" + + +@dataclass(frozen=True) +class DocumentConversionResult: + text: str + failure: DocumentConversionFailure | None = None + detail: str | None = None + + +_MAX_CONCURRENT_CONVERSIONS = max(1, min(8, os.cpu_count() or 1)) +_CONVERSION_SEMAPHORE = threading.Semaphore(_MAX_CONCURRENT_CONVERSIONS) + + +def convert_document_bytes(content: bytes, suffix: str) -> DocumentConversionResult: + try: + import anydoc + except ImportError as exc: + return DocumentConversionResult( + text="", + failure=DocumentConversionFailure.DEPENDENCY, + detail=str(exc), + ) + + try: + detected_format = anydoc.format_from_bytes(content) + normalized_format = detected_format or anydoc.format_from_extension(suffix) + if not normalized_format: + return DocumentConversionResult( + text="", + failure=DocumentConversionFailure.UNSUPPORTED, + detail=f"Unsupported document format for {suffix or 'content'}", + ) + + with _CONVERSION_SEMAPHORE: + markdown = anydoc.to_markdown_bytes(content, normalized_format) + except ImportError as exc: + return DocumentConversionResult( + text="", + failure=DocumentConversionFailure.DEPENDENCY, + detail=str(exc), + ) + except anydoc.UnsupportedError as exc: + return DocumentConversionResult(text="", failure=DocumentConversionFailure.UNSUPPORTED, detail=str(exc)) + except anydoc.MalformedError as exc: + return DocumentConversionResult(text="", failure=DocumentConversionFailure.MALFORMED, detail=str(exc)) + except anydoc.EncryptedError as exc: + return DocumentConversionResult(text="", failure=DocumentConversionFailure.ENCRYPTED, detail=str(exc)) + except anydoc.ResourceLimitError as exc: + return DocumentConversionResult(text="", failure=DocumentConversionFailure.RESOURCE_LIMIT, detail=str(exc)) + except anydoc.MissingPartError as exc: + return DocumentConversionResult(text="", failure=DocumentConversionFailure.MISSING_PART, detail=str(exc)) + except Exception as exc: + return DocumentConversionResult(text="", failure=DocumentConversionFailure.UNEXPECTED, detail=str(exc)) + + if isinstance(markdown, bytes): + text = markdown.decode("utf-8", errors="replace") + else: + text = str(markdown) + return DocumentConversionResult(text=text) diff --git a/ragtime/core/file_constants.py b/ragtime/core/file_constants.py index 3073d50d..fc3d2242 100644 --- a/ragtime/core/file_constants.py +++ b/ragtime/core/file_constants.py @@ -108,27 +108,36 @@ # ============================================================================= # PARSEABLE DOCUMENT EXTENSIONS # ============================================================================= -# Documents that require special parsers (PDF, Office, OpenDocument, etc.). -# Both the filesystem indexer and git/upload indexer can parse these using -# document_parser.py extractors. -PARSEABLE_DOCUMENT_EXTENSIONS: set[str] = { - # Office documents +# AnyDoc README-supported document formats. Keep this as the canonical +# AnyDoc taxonomy so parser-facing sets can reuse it without local drift. +ANYDOC_DOCUMENT_EXTENSIONS: set[str] = { ".pdf", ".doc", ".docx", - ".xls", - ".xlsx", + ".docm", ".ppt", + ".pps", + ".pot", ".pptx", - # OpenDocument + ".pptm", + ".ppsx", + ".ppsm", + ".xls", + ".xlsx", + ".xlsm", + ".xlsb", ".odt", ".ods", ".odp", - # Rich text ".rtf", - # Ebooks ".epub", - # Email + ".csv", +} + +# Documents that require special parsers (PDF, Office, OpenDocument, etc.). +# Both the filesystem indexer and git/upload indexer can parse these using +# document_parser.py extractors. +PARSEABLE_DOCUMENT_EXTENSIONS: set[str] = ANYDOC_DOCUMENT_EXTENSIONS | { ".eml", ".msg", } @@ -180,22 +189,7 @@ ".txt", ".md", ".rst", - ".csv", - # Office documents - parsed by document_parser.py - ".pdf", - ".doc", - ".docx", - ".xls", - ".xlsx", - ".ppt", - ".pptx", - # OpenDocument formats - ".odt", - ".ods", - ".odp", - ".rtf", - # Ebooks - ".epub", + *ANYDOC_DOCUMENT_EXTENSIONS, # Email ".eml", ".msg", @@ -485,10 +479,18 @@ ".pdf": None, ".doc": None, ".docx": None, + ".docm": None, ".xls": None, ".xlsx": None, + ".xlsm": None, + ".xlsb": None, ".ppt": None, ".pptx": None, + ".pps": None, + ".pot": None, + ".pptm": None, + ".ppsx": None, + ".ppsm": None, ".rtf": None, # OpenDocument ".odt": None, diff --git a/ragtime/indexer/document_parser.py b/ragtime/indexer/document_parser.py index 6f4957a0..33ad880c 100644 --- a/ragtime/indexer/document_parser.py +++ b/ragtime/indexer/document_parser.py @@ -2,17 +2,11 @@ Document Content Parser Extracts text content from various document formats: -- PDF (.pdf) -- Word (.docx, .doc) -- Excel (.xlsx, .xls) -- PowerPoint (.pptx) -- OpenDocument (.odt, .ods, .odp) -- RTF (.rtf) -- EPUB (.epub) +- AnyDoc-supported documents (PDF, Office, OpenDocument, RTF, EPUB, CSV) - Email (.eml, .msg) - HTML (.html, .htm) - with tag stripping - Images with OCR (.png, .jpg, .jpeg, .tiff, .bmp, .gif, .webp) -- Plain text (.txt, .md, .rst, .json, .xml, .csv) +- Plain text (.txt, .md, .rst, .json, .xml) - Code files (.py, .js, .ts, etc.) OCR Modes: @@ -30,11 +24,8 @@ from pathlib import Path from typing import Any, Literal, Optional -from ragtime.core.file_constants import ( - DOCUMENT_EXTENSIONS, - OCR_EXTENSIONS, - RAW_CAMERA_EXTENSIONS, -) +from ragtime.core.document_conversion import convert_document_bytes +from ragtime.core.file_constants import ANYDOC_DOCUMENT_EXTENSIONS, DOCUMENT_EXTENSIONS, OCR_EXTENSIONS, RAW_CAMERA_EXTENSIONS from ragtime.core.logging import get_logger from ragtime.core.vision_models import ( VisionOcrResult, @@ -93,28 +84,8 @@ def extract_text_from_file( # Route to appropriate parser try: - if suffix == ".pdf": - return _extract_pdf(content) - elif suffix == ".docx": - return _extract_docx(content) - elif suffix == ".doc": - return _extract_doc_legacy(file_path, content) - elif suffix == ".xlsx": - return _extract_xlsx(content) - elif suffix == ".xls": - return _extract_xls(content) - elif suffix == ".pptx": - return _extract_pptx(content) - elif suffix == ".odt": - return _extract_odt(content) - elif suffix == ".ods": - return _extract_ods(content) - elif suffix == ".odp": - return _extract_odp(content) - elif suffix == ".rtf": - return _extract_rtf(content) - elif suffix == ".epub": - return _extract_epub(content) + if suffix in ANYDOC_DOCUMENT_EXTENSIONS: + return _extract_anydoc(content, suffix, file_path) elif suffix == ".eml": return _extract_eml(content) elif suffix == ".msg": @@ -282,28 +253,8 @@ async def extract_text_from_file_async( return "" # All other file types run in thread pool to avoid blocking event loop - if suffix == ".pdf": - return await asyncio.to_thread(_extract_pdf, content) - elif suffix == ".docx": - return await asyncio.to_thread(_extract_docx, content) - elif suffix == ".doc": - return await asyncio.to_thread(_extract_doc_legacy, file_path, content) - elif suffix == ".xlsx": - return await asyncio.to_thread(_extract_xlsx, content) - elif suffix == ".xls": - return await asyncio.to_thread(_extract_xls, content) - elif suffix == ".pptx": - return await asyncio.to_thread(_extract_pptx, content) - elif suffix == ".odt": - return await asyncio.to_thread(_extract_odt, content) - elif suffix == ".ods": - return await asyncio.to_thread(_extract_ods, content) - elif suffix == ".odp": - return await asyncio.to_thread(_extract_odp, content) - elif suffix == ".rtf": - return await asyncio.to_thread(_extract_rtf, content) - elif suffix == ".epub": - return await asyncio.to_thread(_extract_epub, content) + if suffix in ANYDOC_DOCUMENT_EXTENSIONS: + return await asyncio.to_thread(_extract_anydoc, content, suffix, file_path) elif suffix == ".eml": return await asyncio.to_thread(_extract_eml, content) elif suffix == ".msg": @@ -319,260 +270,14 @@ async def extract_text_from_file_async( return "" -def _extract_pdf(content: bytes) -> str: - """Extract text from PDF file.""" - try: - from pypdf import PdfReader - except ImportError: - logger.warning("pypdf not installed, cannot extract PDF content") - return "" - - try: - reader = PdfReader(io.BytesIO(content)) - text_parts = [] - for page in reader.pages: - text = page.extract_text() - if text: - text_parts.append(text) - return "\n\n".join(text_parts) - except Exception as e: - logger.warning(f"PDF extraction error: {e}") - return "" - - -def _extract_docx(content: bytes) -> str: - """Extract text from Word DOCX file.""" - try: - from docx import Document - except ImportError: - logger.warning("python-docx not installed, cannot extract DOCX content") - return "" - - try: - doc = Document(io.BytesIO(content)) - text_parts = [] - - # Extract paragraphs - for para in doc.paragraphs: - if para.text.strip(): - text_parts.append(para.text) - - # Extract tables - for table in doc.tables: - for row in table.rows: - row_text = " | ".join(cell.text.strip() for cell in row.cells if cell.text.strip()) - if row_text: - text_parts.append(row_text) - - return "\n\n".join(text_parts) - except Exception as e: - logger.warning(f"DOCX extraction error: {e}") - return "" - - -def _extract_doc_legacy(file_path: Path, content: bytes) -> str: - """Extract text from legacy Word DOC file.""" - # Legacy .doc files are more complex - # Try antiword if available, otherwise skip - try: - result = subprocess.run(["antiword", str(file_path)], capture_output=True, text=True, timeout=30) - if result.returncode == 0: - return result.stdout - except FileNotFoundError: - logger.debug("antiword not installed, cannot extract legacy DOC content") - except Exception as e: - logger.warning(f"DOC extraction error: {e}") - - return "" - - -def _extract_xlsx(content: bytes) -> str: - """Extract text from Excel XLSX file.""" - try: - from openpyxl import load_workbook - except ImportError: - logger.warning("openpyxl not installed, cannot extract XLSX content") - return "" - - try: - wb = load_workbook(io.BytesIO(content), read_only=True, data_only=True) - text_parts = [] - - for sheet_name in wb.sheetnames: - sheet = wb[sheet_name] - text_parts.append(f"## Sheet: {sheet_name}") - - for row in sheet.iter_rows(): - row_values = [] - for cell in row: - if cell.value is not None: - row_values.append(str(cell.value)) - if row_values: - text_parts.append(" | ".join(row_values)) - - wb.close() - return "\n".join(text_parts) - except Exception as e: - logger.warning(f"XLSX extraction error: {e}") - return "" - - -def _extract_xls(content: bytes) -> str: - """Extract text from legacy Excel XLS file.""" - try: - import xlrd - except ImportError: - logger.warning("xlrd not installed, cannot extract XLS content") - return "" - - try: - wb = xlrd.open_workbook(file_contents=content) - text_parts = [] - - for sheet in wb.sheets(): - text_parts.append(f"## Sheet: {sheet.name}") - - for row_idx in range(sheet.nrows): - row_values = [] - for col_idx in range(sheet.ncols): - cell = sheet.cell(row_idx, col_idx) - if cell.value: - row_values.append(str(cell.value)) - if row_values: - text_parts.append(" | ".join(row_values)) - - return "\n".join(text_parts) - except Exception as e: - logger.warning(f"XLS extraction error: {e}") - return "" - - -def _extract_pptx(content: bytes) -> str: - """Extract text from PowerPoint PPTX file.""" - try: - from pptx import Presentation - except ImportError: - logger.warning("python-pptx not installed, cannot extract PPTX content") - return "" - - try: - prs = Presentation(io.BytesIO(content)) - text_parts = [] - - for slide_num, slide in enumerate(prs.slides, 1): - slide_texts = [] - for shape in slide.shapes: - if hasattr(shape, "text") and shape.text.strip(): - slide_texts.append(shape.text) - - if slide_texts: - text_parts.append(f"## Slide {slide_num}") - text_parts.extend(slide_texts) - - return "\n\n".join(text_parts) - except Exception as e: - logger.warning(f"PPTX extraction error: {e}") - return "" - - -def _extract_odt(content: bytes) -> str: - """Extract text from OpenDocument Text file.""" - try: - from odf import text as odf_text - from odf.opendocument import load - except ImportError: - logger.warning("odfpy not installed, cannot extract ODT content") - return "" - - try: - doc = load(io.BytesIO(content)) - text_parts = [] - - for para in doc.getElementsByType(odf_text.P): - text = "" - for node in para.childNodes: - if hasattr(node, "data"): - text += node.data - if text.strip(): - text_parts.append(text) - - return "\n\n".join(text_parts) - except Exception as e: - logger.warning(f"ODT extraction error: {e}") - return "" - - -def _extract_ods(content: bytes) -> str: - """Extract text from OpenDocument Spreadsheet file.""" - try: - from odf import table as odf_table - from odf import text as odf_text - from odf.opendocument import load - except ImportError: - logger.warning("odfpy not installed, cannot extract ODS content") - return "" - - try: - doc = load(io.BytesIO(content)) - text_parts = [] - - for sheet in doc.getElementsByType(odf_table.Table): - sheet_name = sheet.getAttribute("name") or "Sheet" - text_parts.append(f"## Sheet: {sheet_name}") - - for row in sheet.getElementsByType(odf_table.TableRow): - row_values = [] - for cell in row.getElementsByType(odf_table.TableCell): - cell_text = "" - for p in cell.getElementsByType(odf_text.P): - for node in p.childNodes: - if hasattr(node, "data"): - cell_text += node.data - if cell_text: - row_values.append(cell_text) - if row_values: - text_parts.append(" | ".join(row_values)) - - return "\n".join(text_parts) - except Exception as e: - logger.warning(f"ODS extraction error: {e}") - return "" - - -def _extract_odp(content: bytes) -> str: - """Extract text from OpenDocument Presentation file.""" - try: - from odf import draw as odf_draw - from odf import text as odf_text - from odf.opendocument import load - except ImportError: - logger.warning("odfpy not installed, cannot extract ODP content") - return "" - - try: - doc = load(io.BytesIO(content)) - text_parts = [] - - for page_num, page in enumerate(doc.getElementsByType(odf_draw.Page), 1): - page_texts = [] - for frame in page.getElementsByType(odf_draw.Frame): - for text_box in frame.getElementsByType(odf_draw.TextBox): - for p in text_box.getElementsByType(odf_text.P): - text = "" - for node in p.childNodes: - if hasattr(node, "data"): - text += node.data - if text.strip(): - page_texts.append(text) - - if page_texts: - text_parts.append(f"## Slide {page_num}") - text_parts.extend(page_texts) - - return "\n\n".join(text_parts) - except Exception as e: - logger.warning(f"ODP extraction error: {e}") +def _extract_anydoc(content: bytes, suffix: str, file_path: Path) -> str: + """Extract text from AnyDoc-supported formats.""" + result = convert_document_bytes(content, suffix) + if result.failure is not None: + detail = f": {result.detail}" if result.detail else "" + logger.warning(f"AnyDoc extraction failed for {file_path.name} ({result.failure.value}){detail}") return "" + return result.text def _extract_text(content: bytes) -> str: @@ -588,55 +293,6 @@ def _extract_text(content: bytes) -> str: return content.decode("utf-8", errors="replace") -def _extract_rtf(content: bytes) -> str: - """Extract text from RTF file.""" - try: - from striprtf.striprtf import rtf_to_text - except ImportError: - logger.warning("striprtf not installed, cannot extract RTF content") - return "" - - try: - # Decode RTF content - text = content.decode("utf-8", errors="replace") - return rtf_to_text(text) - except Exception as e: - logger.warning(f"RTF extraction error: {e}") - return "" - - -def _extract_epub(content: bytes) -> str: - """Extract text from EPUB ebook file.""" - try: - import ebooklib - from ebooklib import epub - except ImportError: - logger.warning("ebooklib not installed, cannot extract EPUB content") - return "" - - try: - from bs4 import BeautifulSoup - except ImportError: - logger.warning("beautifulsoup4 not installed, cannot extract EPUB content") - return "" - - try: - book = epub.read_epub(io.BytesIO(content)) - text_parts = [] - - for item in book.get_items(): - if item.get_type() == ebooklib.ITEM_DOCUMENT: - soup = BeautifulSoup(item.get_content(), "html.parser") - text = soup.get_text(separator="\n", strip=True) - if text: - text_parts.append(text) - - return "\n\n".join(text_parts) - except Exception as e: - logger.warning(f"EPUB extraction error: {e}") - return "" - - def _extract_eml(content: bytes) -> str: """Extract text from email EML file.""" try: @@ -804,8 +460,8 @@ async def _extract_image_vision_ocr( vision_base_url: str, vision_model: str, provider: str = "ollama", - api_key: str | None = None, - source_format: str | None = None, + api_key: Optional[str] = None, + source_format: Optional[str] = None, timeout: float = 60.0, ) -> str: """ diff --git a/runtime/worker/service.py b/runtime/worker/service.py index b4072004..3faf5765 100644 --- a/runtime/worker/service.py +++ b/runtime/worker/service.py @@ -4,7 +4,6 @@ import base64 import hashlib import html -import io import json import logging import os @@ -3211,17 +3210,23 @@ def _build_pdf_query_matches( @staticmethod def _extract_pdf_text(content: bytes) -> str: try: - from pypdf import PdfReader - except ImportError: - raise RuntimeError("Runtime PDF extraction dependency pypdf is not installed") - - reader = PdfReader(io.BytesIO(content)) - text_parts: list[str] = [] - for page in reader.pages: - page_text = page.extract_text() - if page_text: - text_parts.append(page_text) - return "\n\n".join(text_parts) + from ragtime.core.document_conversion import convert_document_bytes + except ImportError as exc: + raise RuntimeError("Runtime document conversion adapter is not available") from exc + + result = convert_document_bytes(content, ".pdf") + failure = getattr(result, "failure", None) + if failure is None: + return str(getattr(result, "text", "") or "") + + failure_name = getattr(failure, "name", None) or str(failure) + if failure_name == "UNSUPPORTED": + return "" + + detail = str(getattr(result, "detail", "") or "").strip() + if detail: + raise RuntimeError(detail) + raise RuntimeError(f"Document conversion failed ({failure_name})") async def read_pdf( self, diff --git a/tests/test_document_extensions.py b/tests/test_document_extensions.py new file mode 100644 index 00000000..21ccc29f --- /dev/null +++ b/tests/test_document_extensions.py @@ -0,0 +1,57 @@ +import unittest + +from ragtime.core.file_constants import ( + ANYDOC_DOCUMENT_EXTENSIONS, + DOCUMENT_EXTENSIONS, + LANG_MAPPING, + NEVER_SUGGEST_EXCLUDE_EXTENSIONS, + PARSEABLE_DOCUMENT_EXTENSIONS, +) + +EXPECTED_ANYDOC_DOCUMENT_EXTENSIONS = { + ".csv", + ".doc", + ".docx", + ".docm", + ".epub", + ".odp", + ".ods", + ".odt", + ".pdf", + ".pot", + ".pps", + ".ppsm", + ".ppsx", + ".ppt", + ".pptm", + ".pptx", + ".rtf", + ".xls", + ".xlsb", + ".xlsm", + ".xlsx", +} + +EXPECTED_EMAIL_DOCUMENT_EXTENSIONS = {".eml", ".msg"} + +EXPECTED_PARSEABLE_DOCUMENT_EXTENSIONS = EXPECTED_ANYDOC_DOCUMENT_EXTENSIONS | EXPECTED_EMAIL_DOCUMENT_EXTENSIONS + + +class DocumentExtensionTests(unittest.TestCase): + def test_anydoc_document_extensions_match_anydoc_taxonomy(self) -> None: + self.assertSetEqual(ANYDOC_DOCUMENT_EXTENSIONS, EXPECTED_ANYDOC_DOCUMENT_EXTENSIONS) + + def test_parseable_document_extensions_add_email_formats_to_anydoc_taxonomy(self) -> None: + self.assertSetEqual(PARSEABLE_DOCUMENT_EXTENSIONS, EXPECTED_PARSEABLE_DOCUMENT_EXTENSIONS) + + def test_parseable_document_extensions_are_never_suggested_for_exclusion(self) -> None: + self.assertTrue(PARSEABLE_DOCUMENT_EXTENSIONS <= NEVER_SUGGEST_EXCLUDE_EXTENSIONS) + + def test_parseable_document_extensions_are_known_document_extensions(self) -> None: + self.assertTrue(PARSEABLE_DOCUMENT_EXTENSIONS <= DOCUMENT_EXTENSIONS) + + def test_parseable_document_extensions_route_to_plain_text_chunking(self) -> None: + for extension in EXPECTED_PARSEABLE_DOCUMENT_EXTENSIONS: + with self.subTest(extension=extension): + self.assertIn(extension, DOCUMENT_EXTENSIONS) + self.assertIsNone(LANG_MAPPING.get(extension)) diff --git a/tests/test_document_parser.py b/tests/test_document_parser.py new file mode 100644 index 00000000..6ae23acf --- /dev/null +++ b/tests/test_document_parser.py @@ -0,0 +1,238 @@ +import importlib.util +import sys +import types +import unittest +from pathlib import Path +from unittest import mock + + +def _load_document_parser_module(): + module_name = "tests._document_parser_under_test" + module_path = Path(__file__).resolve().parents[1] / "ragtime/indexer/document_parser.py" + spec = importlib.util.spec_from_file_location(module_name, module_path) + if spec is None or spec.loader is None: + raise AssertionError("Failed to load document_parser module for tests") + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + + fake_file_constants = types.SimpleNamespace( + ANYDOC_DOCUMENT_EXTENSIONS={".pdf", ".docx", ".csv", ".docm", ".xlsm", ".xlsb", ".pps", ".pot", ".pptm", ".ppsx", ".ppsm"}, + DOCUMENT_EXTENSIONS={".txt", ".pdf", ".docx", ".eml"}, + OCR_EXTENSIONS={".png", ".jpg"}, + RAW_CAMERA_EXTENSIONS=set(), + ) + fake_vision_models = types.SimpleNamespace( + VisionOcrResult=object, + extract_text_with_vision=mock.AsyncMock(return_value=""), + extract_text_with_vision_structured=mock.AsyncMock(return_value=None), + ) + fake_logger = mock.Mock() + fake_logging_module = types.SimpleNamespace(get_logger=mock.Mock(return_value=fake_logger)) + + with mock.patch.dict( + sys.modules, + { + "ragtime.core.file_constants": fake_file_constants, + "ragtime.core.logging": fake_logging_module, + "ragtime.core.vision_models": fake_vision_models, + }, + clear=False, + ): + spec.loader.exec_module(module) + return module + + +class DocumentConversionTests(unittest.TestCase): + def _fake_anydoc(self, **overrides): + class ConvertError(Exception): + pass + + class UnsupportedError(ConvertError): + pass + + class MalformedError(ConvertError): + pass + + class EncryptedError(ConvertError): + pass + + class ResourceLimitError(ConvertError): + pass + + class MissingPartError(ConvertError): + pass + + module = types.SimpleNamespace( + ConvertError=ConvertError, + UnsupportedError=UnsupportedError, + MalformedError=MalformedError, + EncryptedError=EncryptedError, + ResourceLimitError=ResourceLimitError, + MissingPartError=MissingPartError, + format_from_bytes=mock.Mock(return_value="pdf"), + format_from_extension=mock.Mock(return_value=None), + to_markdown_bytes=mock.Mock(return_value=b"converted text"), + ) + for name, value in overrides.items(): + setattr(module, name, value) + return module + + def test_convert_document_bytes_prefers_detected_content_type(self) -> None: + from ragtime.core.document_conversion import convert_document_bytes + + fake_anydoc = self._fake_anydoc( + format_from_bytes=mock.Mock(return_value="pdf"), + format_from_extension=mock.Mock(return_value="docx"), + ) + + with mock.patch.dict(sys.modules, {"anydoc": fake_anydoc}, clear=False): + result = convert_document_bytes(b"%PDF-1.7", ".docx") + + self.assertEqual(result.text, "converted text") + self.assertIsNone(result.failure) + fake_anydoc.format_from_bytes.assert_called_once_with(b"%PDF-1.7") + fake_anydoc.format_from_extension.assert_not_called() + fake_anydoc.to_markdown_bytes.assert_called_once_with(b"%PDF-1.7", "pdf") + + def test_convert_document_bytes_falls_back_to_extension(self) -> None: + from ragtime.core.document_conversion import convert_document_bytes + + fake_anydoc = self._fake_anydoc( + format_from_bytes=mock.Mock(return_value=None), + format_from_extension=mock.Mock(return_value="csv"), + to_markdown_bytes=mock.Mock(return_value=b"a,b\n1,2\n"), + ) + + with mock.patch.dict(sys.modules, {"anydoc": fake_anydoc}, clear=False): + result = convert_document_bytes(b"a,b\n1,2\n", ".csv") + + self.assertEqual(result.text, "a,b\n1,2\n") + self.assertIsNone(result.failure) + fake_anydoc.format_from_bytes.assert_called_once_with(b"a,b\n1,2\n") + fake_anydoc.format_from_extension.assert_called_once_with(".csv") + fake_anydoc.to_markdown_bytes.assert_called_once_with(b"a,b\n1,2\n", "csv") + + def test_convert_document_bytes_returns_typed_failures(self) -> None: + from ragtime.core.document_conversion import DocumentConversionFailure, convert_document_bytes + + cases = [ + ("UnsupportedError", DocumentConversionFailure.UNSUPPORTED), + ("MalformedError", DocumentConversionFailure.MALFORMED), + ("EncryptedError", DocumentConversionFailure.ENCRYPTED), + ("ResourceLimitError", DocumentConversionFailure.RESOURCE_LIMIT), + ("MissingPartError", DocumentConversionFailure.MISSING_PART), + ("RuntimeError", DocumentConversionFailure.UNEXPECTED), + ] + + for error_name, expected in cases: + with self.subTest(error_name=error_name): + fake_anydoc = self._fake_anydoc() + error_type = getattr(fake_anydoc, error_name, type(error_name, (Exception,), {})) + fake_anydoc.to_markdown_bytes = mock.Mock(side_effect=error_type("boom")) + + with mock.patch.dict(sys.modules, {"anydoc": fake_anydoc}, clear=False): + result = convert_document_bytes(b"data", ".pdf") + + self.assertEqual(result.failure, expected) + self.assertEqual(result.detail, "boom") + self.assertEqual(result.text, "") + + def test_convert_document_bytes_reports_dependency_failures(self) -> None: + from ragtime.core.document_conversion import DocumentConversionFailure, convert_document_bytes + + original_import = __import__ + + def fake_import(name, *args, **kwargs): + if name == "anydoc": + raise ImportError("missing anydoc") + return original_import(name, *args, **kwargs) + + with mock.patch("builtins.__import__", side_effect=fake_import): + result = convert_document_bytes(b"data", ".pdf") + + self.assertEqual(result.failure, DocumentConversionFailure.DEPENDENCY) + self.assertEqual(result.detail, "missing anydoc") + self.assertEqual(result.text, "") + + +class DocumentParserRoutingTests(unittest.IsolatedAsyncioTestCase): + def test_extract_text_from_file_uses_document_conversion_adapter(self) -> None: + document_parser = _load_document_parser_module() + + with mock.patch.object( + document_parser, + "convert_document_bytes", + return_value=types.SimpleNamespace(text="converted pdf", failure=None, detail=None), + ) as convert_mock: + result = document_parser.extract_text_from_file(Path("example.pdf"), content=b"pdf bytes") + + self.assertEqual(result, "converted pdf") + convert_mock.assert_called_once_with(b"pdf bytes", ".pdf") + + async def test_extract_text_from_file_async_uses_document_conversion_adapter(self) -> None: + document_parser = _load_document_parser_module() + + with mock.patch.object( + document_parser, + "convert_document_bytes", + return_value=types.SimpleNamespace(text="converted docx", failure=None, detail=None), + ) as convert_mock: + result = await document_parser.extract_text_from_file_async(Path("example.docx"), content=b"docx bytes") + + self.assertEqual(result, "converted docx") + convert_mock.assert_called_once_with(b"docx bytes", ".docx") + + def test_extract_text_from_file_routes_csv_through_document_conversion_adapter(self) -> None: + document_parser = _load_document_parser_module() + + with mock.patch.object( + document_parser, + "convert_document_bytes", + return_value=types.SimpleNamespace(text="converted csv", failure=None, detail=None), + ) as convert_mock: + result = document_parser.extract_text_from_file(Path("table.csv"), content=b"a,b\n1,2\n") + + self.assertEqual(result, "converted csv") + convert_mock.assert_called_once_with(b"a,b\n1,2\n", ".csv") + + def test_extract_text_from_file_keeps_eml_routing(self) -> None: + document_parser = _load_document_parser_module() + + with ( + mock.patch.object( + document_parser, + "convert_document_bytes", + return_value=types.SimpleNamespace(text="wrong", failure=None, detail=None), + ) as convert_mock, + mock.patch.object( + document_parser, + "_extract_eml", + return_value="mail body", + ) as eml_mock, + ): + result = document_parser.extract_text_from_file(Path("message.eml"), content=b"mail bytes") + + self.assertEqual(result, "mail body") + eml_mock.assert_called_once_with(b"mail bytes") + convert_mock.assert_not_called() + + def test_extract_text_from_file_keeps_msg_routing(self) -> None: + document_parser = _load_document_parser_module() + + with ( + mock.patch.object( + document_parser, + "convert_document_bytes", + return_value=types.SimpleNamespace(text="wrong", failure=None, detail=None), + ) as convert_mock, + mock.patch.object( + document_parser, + "_extract_msg", + return_value="msg body", + ) as msg_mock, + ): + result = document_parser.extract_text_from_file(Path("message.msg"), content=b"msg bytes") + + self.assertEqual(result, "msg body") + msg_mock.assert_called_once_with(b"msg bytes") + convert_mock.assert_not_called() diff --git a/tests/test_runtime_pdf_read.py b/tests/test_runtime_pdf_read.py index 5817947d..c80d6fb0 100644 --- a/tests/test_runtime_pdf_read.py +++ b/tests/test_runtime_pdf_read.py @@ -1,4 +1,7 @@ +import sys import unittest +from enum import Enum +from types import SimpleNamespace from unittest.mock import patch import httpx @@ -22,7 +25,29 @@ def _patch_async_client(transport: httpx.MockTransport): ) +class _FakeConversionFailure(Enum): + UNSUPPORTED = "unsupported" + OTHER = "other" + + +def _patch_document_converter(*, text: str = "", failure: _FakeConversionFailure | None = None, detail: str | None = None): + module = SimpleNamespace( + convert_document_bytes=lambda content, suffix: SimpleNamespace( + text=text, + failure=failure, + detail=detail, + ) + ) + return patch.dict(sys.modules, {"ragtime.core.document_conversion": module}) + + class RuntimePdfReadTests(unittest.IsolatedAsyncioTestCase): + def test_extract_pdf_text_uses_document_converter_markdown(self): + with _patch_document_converter(text="# Title\n\nConverted body"): + text = WorkerService._extract_pdf_text(b"%PDF-1.4\nconverted") + + self.assertEqual(text, "# Title\n\nConverted body") + async def test_extracts_requested_range(self): service = WorkerService() @@ -125,6 +150,47 @@ def handler(request: httpx.Request) -> httpx.Response: self.assertIn("Access Denied", result.body_preview) self.assertIn("denied", result.error.lower()) + async def test_returns_empty_when_document_converter_reports_unsupported(self): + service = WorkerService() + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "application/pdf"}, + content=b"%PDF-1.4\nunsupported pdf bytes", + ) + + transport = httpx.MockTransport(handler) + with _patch_async_client(transport), _patch_document_converter(failure=_FakeConversionFailure.UNSUPPORTED): + result = await service.read_pdf(RuntimePdfReadRequest(url="https://example.com/image-only.pdf")) + + self.assertEqual(result.status, "empty") + self.assertEqual(result.failure_mode, "empty_pdf_text") + + async def test_returns_pdf_extract_error_for_document_converter_failures(self): + service = WorkerService() + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "application/pdf"}, + content=b"%PDF-1.4\nproblem pdf bytes", + ) + + transport = httpx.MockTransport(handler) + with ( + _patch_async_client(transport), + _patch_document_converter( + failure=_FakeConversionFailure.OTHER, + detail="OCR backend unavailable", + ), + ): + result = await service.read_pdf(RuntimePdfReadRequest(url="https://example.com/problem.pdf")) + + self.assertEqual(result.status, "error") + self.assertEqual(result.failure_mode, "pdf_extract_error") + self.assertIn("OCR backend unavailable", result.error) + if __name__ == "__main__": unittest.main() From 04d3803d6b86dd723531fddd4e05ea5482e02997 Mon Sep 17 00:00:00 2001 From: mattv8 Date: Thu, 3 Sep 2026 10:55:15 -0600 Subject: [PATCH 2/2] feat: bump anydoc to 0.2.4 and address parsing review nits - Bump firecrawl-anydoc 0.1.6 -> 0.2.4 in app and runtime extras; map the new NeedsOcrError to a typed NEEDS_OCR conversion failure and treat it like UNSUPPORTED (empty text) in runtime PDF reads, matching the old pypdf behavior for scanned/image-only PDFs - Remove the dead pandas dependency (zero imports repo-wide) from pyproject and the Dockerfile import asserts; dev image assert now verifies anydoc instead - Prefill all AnyDoc-supported document extensions in the filesystem indexer default include patterns (both ToolWizard defaults) - Deny-list the new binary office variants (.docm, .pptm, .pps, .ppsx, .ppsm, .pot, .rtf, .epub) in the userspace prefetch heuristic; .csv stays prefetchable since it is text-editable - Revert Optional[str] style churn in document_parser to pre-PR unions - Cover NEEDS_OCR in document parser and runtime PDF read tests --- docker/Dockerfile | 2 +- docker/Dockerfile.dev | 2 +- pyproject.toml | 5 ++- ragtime/core/document_conversion.py | 3 ++ .../frontend/src/components/ToolWizard.tsx | 34 +++++++++++++++++++ .../frontend/src/utils/userspacePrefetch.ts | 8 +++++ ragtime/indexer/document_parser.py | 4 +-- runtime/worker/service.py | 2 +- tests/test_document_parser.py | 5 +++ tests/test_runtime_pdf_read.py | 18 ++++++++++ 10 files changed, 75 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 0c7df326..2555bb1b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -107,7 +107,7 @@ RUN --mount=type=cache,id=ragtime-pip-cache,target=/root/.cache/pip,sharing=lock else \ pip install -r /tmp/requirements.app.txt; \ fi && \ - python -c "import anydoc, chonkie, pandas; assert callable(anydoc.to_markdown_bytes); assert anydoc.format_from_extension('csv') is not None" + python -c "import anydoc, chonkie; assert callable(anydoc.to_markdown_bytes); assert anydoc.format_from_extension('csv') is not None" # ============================================================================= # Stage 3: Python CI base diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev index 0e4e6404..8ef3680a 100644 --- a/docker/Dockerfile.dev +++ b/docker/Dockerfile.dev @@ -52,7 +52,7 @@ RUN pip install --no-cache-dir --upgrade pip && \ python /tmp/install_deps_from_pyproject.py /ragtime/pyproject.toml app /tmp/requirements.app.txt && \ python /tmp/install_deps_from_pyproject.py /ragtime/pyproject.toml test /tmp/requirements.test.txt && \ pip install --no-cache-dir -r /tmp/requirements.app.txt -r /tmp/requirements.test.txt && \ - python -c "import chonkie, pandas" + python -c "import anydoc, chonkie" # Copy frontend package.json for npm install layer caching COPY ragtime/frontend/package.json ragtime/frontend/package-lock.json* /ragtime/ragtime/frontend/ diff --git a/pyproject.toml b/pyproject.toml index b7612265..86d9f829 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,10 +44,9 @@ app = [ "python-jose[cryptography]>=3.3.0,<4.0.0", "webauthn>=2.0.0,<3.0.0", "slowapi>=0.1.9,<1.0.0", - "firecrawl-anydoc==0.1.6", + "firecrawl-anydoc==0.2.4", "python-docx>=1.1.0,<2.0.0", "openpyxl>=3.1.0,<4.0.0", - "pandas>=2.2.0,<3.0.0", "xlrd>=2.0.1,<3.0.0", "chonkie[code]>=1.3.1,<2.0.0", "beautifulsoup4>=4.12.0,<5.0.0", @@ -68,7 +67,7 @@ runtime = [ "pydantic>=2.5.0,<3.0.0", "httpx>=0.26.0,<1.0.0", "mcp>=1.0.0,<2.0.0", - "firecrawl-anydoc==0.1.6", + "firecrawl-anydoc==0.2.4", ] test = [ "mypy>=1.15.0,<2.0.0", diff --git a/ragtime/core/document_conversion.py b/ragtime/core/document_conversion.py index e2fc12bb..6ed65f5e 100644 --- a/ragtime/core/document_conversion.py +++ b/ragtime/core/document_conversion.py @@ -8,6 +8,7 @@ class DocumentConversionFailure(str, Enum): UNSUPPORTED = "unsupported" + NEEDS_OCR = "needs_ocr" MALFORMED = "malformed" ENCRYPTED = "encrypted" RESOURCE_LIMIT = "resource_limit" @@ -57,6 +58,8 @@ def convert_document_bytes(content: bytes, suffix: str) -> DocumentConversionRes ) except anydoc.UnsupportedError as exc: return DocumentConversionResult(text="", failure=DocumentConversionFailure.UNSUPPORTED, detail=str(exc)) + except anydoc.NeedsOcrError as exc: + return DocumentConversionResult(text="", failure=DocumentConversionFailure.NEEDS_OCR, detail=str(exc)) except anydoc.MalformedError as exc: return DocumentConversionResult(text="", failure=DocumentConversionFailure.MALFORMED, detail=str(exc)) except anydoc.EncryptedError as exc: diff --git a/ragtime/frontend/src/components/ToolWizard.tsx b/ragtime/frontend/src/components/ToolWizard.tsx index d1cf9719..ccf9b2b9 100644 --- a/ragtime/frontend/src/components/ToolWizard.tsx +++ b/ragtime/frontend/src/components/ToolWizard.tsx @@ -2556,9 +2556,26 @@ export function ToolWizard({ '**/*.txt', '**/*.md', '**/*.pdf', + '**/*.doc', '**/*.docx', + '**/*.docm', + '**/*.xls', '**/*.xlsx', + '**/*.xlsm', + '**/*.xlsb', + '**/*.ppt', '**/*.pptx', + '**/*.pptm', + '**/*.pps', + '**/*.ppsx', + '**/*.ppsm', + '**/*.pot', + '**/*.odt', + '**/*.ods', + '**/*.odp', + '**/*.rtf', + '**/*.epub', + '**/*.csv', '**/*.py', '**/*.json', '**/*.png', @@ -2608,9 +2625,26 @@ export function ToolWizard({ '**/*.txt', '**/*.md', '**/*.pdf', + '**/*.doc', '**/*.docx', + '**/*.docm', + '**/*.xls', '**/*.xlsx', + '**/*.xlsm', + '**/*.xlsb', + '**/*.ppt', '**/*.pptx', + '**/*.pptm', + '**/*.pps', + '**/*.ppsx', + '**/*.ppsm', + '**/*.pot', + '**/*.odt', + '**/*.ods', + '**/*.odp', + '**/*.rtf', + '**/*.epub', + '**/*.csv', '**/*.py', '**/*.json', '**/*.png', diff --git a/ragtime/frontend/src/utils/userspacePrefetch.ts b/ragtime/frontend/src/utils/userspacePrefetch.ts index 83ffe22f..0964b14c 100644 --- a/ragtime/frontend/src/utils/userspacePrefetch.ts +++ b/ragtime/frontend/src/utils/userspacePrefetch.ts @@ -32,8 +32,14 @@ const NON_PREFETCHABLE_USER_SPACE_FILE_EXTENSIONS = [ '.pdf', '.doc', '.docx', + '.docm', '.ppt', '.pptx', + '.pptm', + '.pps', + '.ppsx', + '.ppsm', + '.pot', '.xls', '.xlsx', '.xlsm', @@ -41,6 +47,8 @@ const NON_PREFETCHABLE_USER_SPACE_FILE_EXTENSIONS = [ '.odt', '.ods', '.odp', + '.rtf', + '.epub', '.msg', // binary/tabular data formats '.parquet', diff --git a/ragtime/indexer/document_parser.py b/ragtime/indexer/document_parser.py index 33ad880c..a78fb237 100644 --- a/ragtime/indexer/document_parser.py +++ b/ragtime/indexer/document_parser.py @@ -460,8 +460,8 @@ async def _extract_image_vision_ocr( vision_base_url: str, vision_model: str, provider: str = "ollama", - api_key: Optional[str] = None, - source_format: Optional[str] = None, + api_key: str | None = None, + source_format: str | None = None, timeout: float = 60.0, ) -> str: """ diff --git a/runtime/worker/service.py b/runtime/worker/service.py index 3faf5765..f6b0ed7b 100644 --- a/runtime/worker/service.py +++ b/runtime/worker/service.py @@ -3220,7 +3220,7 @@ def _extract_pdf_text(content: bytes) -> str: return str(getattr(result, "text", "") or "") failure_name = getattr(failure, "name", None) or str(failure) - if failure_name == "UNSUPPORTED": + if failure_name in ("UNSUPPORTED", "NEEDS_OCR"): return "" detail = str(getattr(result, "detail", "") or "").strip() diff --git a/tests/test_document_parser.py b/tests/test_document_parser.py index 6ae23acf..0c8b7690 100644 --- a/tests/test_document_parser.py +++ b/tests/test_document_parser.py @@ -50,6 +50,9 @@ class ConvertError(Exception): class UnsupportedError(ConvertError): pass + class NeedsOcrError(ConvertError): + pass + class MalformedError(ConvertError): pass @@ -65,6 +68,7 @@ class MissingPartError(ConvertError): module = types.SimpleNamespace( ConvertError=ConvertError, UnsupportedError=UnsupportedError, + NeedsOcrError=NeedsOcrError, MalformedError=MalformedError, EncryptedError=EncryptedError, ResourceLimitError=ResourceLimitError, @@ -117,6 +121,7 @@ def test_convert_document_bytes_returns_typed_failures(self) -> None: cases = [ ("UnsupportedError", DocumentConversionFailure.UNSUPPORTED), + ("NeedsOcrError", DocumentConversionFailure.NEEDS_OCR), ("MalformedError", DocumentConversionFailure.MALFORMED), ("EncryptedError", DocumentConversionFailure.ENCRYPTED), ("ResourceLimitError", DocumentConversionFailure.RESOURCE_LIMIT), diff --git a/tests/test_runtime_pdf_read.py b/tests/test_runtime_pdf_read.py index c80d6fb0..308d2868 100644 --- a/tests/test_runtime_pdf_read.py +++ b/tests/test_runtime_pdf_read.py @@ -27,6 +27,7 @@ def _patch_async_client(transport: httpx.MockTransport): class _FakeConversionFailure(Enum): UNSUPPORTED = "unsupported" + NEEDS_OCR = "needs_ocr" OTHER = "other" @@ -167,6 +168,23 @@ def handler(request: httpx.Request) -> httpx.Response: self.assertEqual(result.status, "empty") self.assertEqual(result.failure_mode, "empty_pdf_text") + async def test_returns_empty_when_document_converter_reports_needs_ocr(self): + service = WorkerService() + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"content-type": "application/pdf"}, + content=b"%PDF-1.4\nimage only pdf bytes", + ) + + transport = httpx.MockTransport(handler) + with _patch_async_client(transport), _patch_document_converter(failure=_FakeConversionFailure.NEEDS_OCR): + result = await service.read_pdf(RuntimePdfReadRequest(url="https://example.com/scanned.pdf")) + + self.assertEqual(result.status, "empty") + self.assertEqual(result.failure_mode, "empty_pdf_text") + async def test_returns_pdf_extract_error_for_document_converter_failures(self): service = WorkerService()