From 3619ff51b3eb79bbc5c96f781470573fb975e691 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:36:01 +0000 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Sentinel:=20[CRITIC?= =?UTF-8?q?AL]=20=ED=8C=8C=EC=9D=BC=20=ED=8C=8C=EC=8B=B1=20=EC=A4=91=20?= =?UTF-8?q?=EB=B0=9C=EC=83=9D=ED=95=98=EB=8A=94=20=EC=B2=98=EB=A6=AC?= =?UTF-8?q?=EB=90=98=EC=A7=80=20=EC=95=8A=EC=9D=80=20=EC=98=88=EC=99=B8?= =?UTF-8?q?=EB=A1=9C=20=EC=9D=B8=ED=95=9C=20DoS=20=EC=B7=A8=EC=95=BD?= =?UTF-8?q?=EC=A0=90=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .jules/sentinel.md | 5 +++++ src/newsdom_api/main.py | 3 ++- tests/test_parse_endpoint.py | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/.jules/sentinel.md b/.jules/sentinel.md index 2b5d819c..98d6f30a 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -90,3 +90,8 @@ **Vulnerability:** The `_safe_upload_filename` function used `filename.replace`, `PurePosixPath`, and `re.sub` on unbounded client input, making it vulnerable to ReDoS or CPU/memory exhaustion (DoS) when fed extremely long strings. **Learning:** Even fast standard library functions like `PurePosixPath` and string replacements can cause significant lag when chained on strings in the megabytes. String processing operations should always bound their inputs first if the input is untrusted and can be arbitrarily large. **Prevention:** Cap the length of client-provided filename strings early by slicing them (e.g. `filename = filename[-512:]`) before doing more complex string parsing or regex replacements, especially when only the basename suffix is relevant. + +## 2025-05-18 - [CRITICAL] Prevent DoS via Unhandled PdfReader Exceptions +**Vulnerability:** 악의적인 페이로드 업로드 시 `PdfReader`에서 발생하는 `TypeError` 또는 `MemoryError` 등 처리되지 않은 예외로 인해 500 상태 코드 및 서버 리소스 소진 유발 가능성 발견. +**Learning:** `PdfReader`는 손상되거나 특수하게 조작된 PDF 파일 파싱 시 다양한 형태의 내장 예외(built-in exceptions)를 던질 수 있으며, 이를 특정 예외로만 잡을 경우 예상치 못한 시스템 장애(DoS)로 이어질 수 있음. +**Prevention:** `_validate_pdf_structure` 내부에서 `Exception`을 포괄적으로 잡아내어 415 상태 코드로 안전하게 처리(fail securely)하고, 로거(logger)를 통해 예외 정보를 남겨 추적성을 유지함. diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index f61aafc2..9f782304 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -193,7 +193,8 @@ def _validate_pdf_structure(file_path: Path) -> None: reader = PdfReader(file_path, strict=True) if len(reader.pages) < 1: raise ValueError("PDF has no pages") - except (PdfReadError, RecursionError, ValueError, OverflowError): + except Exception as exc: + LOGGER.error("Unhandled exception during PDF structure validation", exc_info=exc) raise HTTPException( status_code=415, detail=UNSUPPORTED_MEDIA_DETAIL, diff --git a/tests/test_parse_endpoint.py b/tests/test_parse_endpoint.py index 1491ada0..684d94aa 100644 --- a/tests/test_parse_endpoint.py +++ b/tests/test_parse_endpoint.py @@ -555,3 +555,18 @@ def spy_unlink(self, missing_ok=False): # We should have unlinked exactly one file, which should be in the temp directory assert len(unlinked_paths) == 1 assert "tmp" in unlinked_paths[0].lower() or "temp" in unlinked_paths[0].lower() + +def test_parse_endpoint_catches_unhandled_pdfreader_exceptions(monkeypatch): + def reject_pdf(*_args, **_kwargs): + raise TypeError("malformed dictionary") + + monkeypatch.setattr("newsdom_api.main.PdfReader", reject_pdf) + + client = TestClient(app) + response = client.post( + "/parse", + headers={"Authorization": "Bearer development-bypass-token"}, + files={"file": ("fixture.pdf", b"%PDF-1.4\n%%EOF", "application/pdf")}, + ) + assert response.status_code == 415 + assert response.json()["detail"] == "Unsupported Media Type" From c131f586102a544809e86ca567d4c791d1a556a9 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Mon, 31 Aug 2026 22:57:58 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Sentinel:=20[CRITIC?= =?UTF-8?q?AL]=20=ED=8C=8C=EC=9D=BC=20=ED=8C=8C=EC=8B=B1=20=EC=A4=91=20?= =?UTF-8?q?=EB=B0=9C=EC=83=9D=ED=95=98=EB=8A=94=20=EC=B2=98=EB=A6=AC?= =?UTF-8?q?=EB=90=98=EC=A7=80=20=EC=95=8A=EC=9D=80=20=EC=98=88=EC=99=B8?= =?UTF-8?q?=EB=A1=9C=20=EC=9D=B8=ED=95=9C=20DoS=20=EC=B7=A8=EC=95=BD?= =?UTF-8?q?=EC=A0=90=20=EC=88=98=EC=A0=95=20=EB=B0=8F=20=ED=8C=8C=EC=8B=B1?= =?UTF-8?q?=20=EC=A0=95=EA=B7=9C=EC=8B=9D=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fix.patch | 7 + fix2.patch | 9 + src/newsdom_api/mineru_runner.py | 2 +- src/newsdom_api/mineru_runner.py.orig | 343 ++++++++++++++++++++++++++ 4 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 fix.patch create mode 100644 fix2.patch create mode 100644 src/newsdom_api/mineru_runner.py.orig diff --git a/fix.patch b/fix.patch new file mode 100644 index 00000000..2b42991a --- /dev/null +++ b/fix.patch @@ -0,0 +1,7 @@ +--- src/newsdom_api/mineru_runner.py ++++ src/newsdom_api/mineru_runner.py +@@ -16,7 +16,7 @@ + # ⚡ Bolt: Use a pre-compiled regex to push pattern matching to C, + # avoiding the Python-level overhead of `any()` and generator comprehensions +-_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<>\\n\\r]") ++_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<>\\n\\r]") diff --git a/fix2.patch b/fix2.patch new file mode 100644 index 00000000..7eaed1c1 --- /dev/null +++ b/fix2.patch @@ -0,0 +1,9 @@ +--- src/newsdom_api/mineru_runner.py ++++ src/newsdom_api/mineru_runner.py +@@ -17,7 +17,7 @@ + # ⚡ Bolt: Use a pre-compiled regex to push pattern matching to C, + # avoiding the Python-level overhead of `any()` and generator comprehensions +-_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<>\\n\\r]") ++_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<\>\n\r]") + + # MinerU 3.4.4's public CLI defaults to ``ch``. That model covers Chinese, diff --git a/src/newsdom_api/mineru_runner.py b/src/newsdom_api/mineru_runner.py index 45e74fe3..22f589dc 100644 --- a/src/newsdom_api/mineru_runner.py +++ b/src/newsdom_api/mineru_runner.py @@ -16,7 +16,7 @@ # ⚡ Bolt: Use a pre-compiled regex to push pattern matching to C, # avoiding the Python-level overhead of `any()` and generator comprehensions -_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<>\n\r]") +_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<\>\n\r]") # MinerU 3.4.4's public CLI defaults to ``ch``. That model covers Chinese, # English, Japanese, Traditional Chinese, and Latin; callers can select another diff --git a/src/newsdom_api/mineru_runner.py.orig b/src/newsdom_api/mineru_runner.py.orig new file mode 100644 index 00000000..0aee338d --- /dev/null +++ b/src/newsdom_api/mineru_runner.py.orig @@ -0,0 +1,343 @@ +"""Invoke MinerU as an external parser and collect its structured outputs.""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import tempfile +from functools import lru_cache +from pathlib import Path +from typing import Any + +from .errors import MineruIncompleteOutputError, MineruRuntimeUnavailableError + +# ⚡ Bolt: Use a pre-compiled regex to push pattern matching to C, +# avoiding the Python-level overhead of `any()` and generator comprehensions +_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<>\\n\\r]") + +# MinerU 3.4.4's public CLI defaults to ``ch``. That model covers Chinese, +# English, Japanese, Traditional Chinese, and Latin; callers can select another +# supported script family through the request parameter. +DEFAULT_LANGUAGE = "ch" +DEFAULT_MODE = "auto" + +# Parsing modes understood by MinerU's ``-m`` flag. ``auto`` picks txt vs. ocr +# per document, ``ocr`` forces optical recognition, ``txt`` extracts the +# embedded text layer only. +VALID_MODES = frozenset({"auto", "ocr", "txt"}) + +# Public language keys accepted by MinerU 3.4.4's CLI. Keeping this contract +# locally lets the API return a client-visible 422 instead of a downstream 503. +VALID_LANGUAGES = frozenset( + { + "ch", + "ch_server", + "korean", + "ta", + "te", + "ka", + "th", + "el", + "arabic", + "east_slavic", + "cyrillic", + "devanagari", + } +) + +# Compatibility aliases published by MinerU. Aliases are canonicalized before +# subprocess execution so behavior does not depend on a particular CLI wrapper. +_LANGUAGE_ALIASES = { + "en": "ch", + "japan": "ch", + "chinese_cht": "ch", + "latin": "ch", + "ar": "arabic", + "fa": "arabic", + "ug": "arabic", + "ur": "arabic", + "ps": "arabic", + "ku": "arabic", + "sd": "arabic", + "bal": "arabic", + "ru": "east_slavic", + "be": "east_slavic", + "uk": "east_slavic", + "hi": "devanagari", + "mr": "devanagari", + "ne": "devanagari", + "bh": "devanagari", + "mai": "devanagari", + "ang": "devanagari", + "bho": "devanagari", + "mah": "devanagari", + "sck": "devanagari", + "new": "devanagari", + "gom": "devanagari", + "sa": "devanagari", + "bgc": "devanagari", + "rs_cyrillic": "cyrillic", + "bg": "cyrillic", + "mn": "cyrillic", + "abq": "cyrillic", + "ady": "cyrillic", + "kbd": "cyrillic", + "ava": "cyrillic", + "dar": "cyrillic", + "inh": "cyrillic", + "che": "cyrillic", + "lbe": "cyrillic", + "lez": "cyrillic", + "tab": "cyrillic", + "kk": "cyrillic", + "ky": "cyrillic", + "tg": "cyrillic", + "mk": "cyrillic", + "tt": "cyrillic", + "cv": "cyrillic", + "ba": "cyrillic", + "mhr": "cyrillic", + "mo": "cyrillic", + "udm": "cyrillic", + "kv": "cyrillic", + "os": "cyrillic", + "bua": "cyrillic", + "xal": "cyrillic", + "tyv": "cyrillic", + "sah": "cyrillic", + "kaa": "cyrillic", +} + +# Method subdirectories MinerU may create beneath the output directory. +_KNOWN_METHOD_DIRS = ("auto", "ocr", "txt") + + +def normalize_mode(mode: str) -> str: + """Validate and normalize a MinerU parsing mode. + + Returns the lower-cased mode when it is one of ``auto``/``ocr``/``txt`` and + raises :class:`ValueError` otherwise so callers can surface a client error. + """ + + normalized = str(mode).strip().lower() + if normalized not in VALID_MODES: + raise ValueError(f"Unsupported MinerU mode: {mode!r}") + return normalized + + +def normalize_language(language: str) -> str: + """Validate and normalize a MinerU language code. + + Returns the canonical MinerU 3.4.4 public language key and raises + :class:`ValueError` otherwise so callers receive a client error instead of + a downstream runtime failure. + """ + + normalized = str(language).strip().lower() + canonical = _LANGUAGE_ALIASES.get(normalized, normalized) + if canonical not in VALID_LANGUAGES: + raise ValueError(f"Unsupported MinerU language: {language!r}") + return canonical + + +def _mineru_command_arg(value: str | Path, *, label: str) -> str: + """Validate a path or executable string before passing it to MinerU argv.""" + + value_str = str(value) + if _UNSAFE_CHARS_PATTERN.search(value_str): + raise ValueError(f"Unsafe {label} for MinerU command") + if value_str.startswith("-"): + raise ValueError(f"Unsafe {label} for MinerU command") + return value_str + + +def build_mineru_command( + input_pdf: Path, + output_dir: Path, + mineru_bin: str = "mineru", + *, + language: str = DEFAULT_LANGUAGE, + mode: str = DEFAULT_MODE, +) -> list[str]: + """Build the MinerU CLI command for pipeline execution. + + ``language`` maps to MinerU's ``-l`` flag and ``mode`` to ``-m``. Both are + validated so callers cannot inject arbitrary argv values; the defaults are + aligned with MinerU 3.4.4 (``ch``/``auto``). + """ + + validated_mode = normalize_mode(mode) + validated_language = normalize_language(language) + + return [ + _mineru_command_arg(mineru_bin, label="MinerU executable"), + "-p", + _mineru_command_arg(input_pdf, label="input PDF path"), + "-o", + _mineru_command_arg(output_dir, label="output directory path"), + "-b", + "pipeline", + "-m", + validated_mode, + "-l", + validated_language, + ] + + +@lru_cache +def _cached_which(cmd: str) -> str | None: + """Cache shutil.which to avoid redundant filesystem lookups.""" + return shutil.which(cmd) + + +def _resolve_mineru_bin() -> str: + """Resolve the MinerU executable path for this process. + + The shutil.which result is cached, but NEWSDOM_MINERU_BIN is evaluated + on every call to allow runtime overrides. + """ + + configured = os.environ.get("NEWSDOM_MINERU_BIN") + if configured: + return configured + found = _cached_which("mineru") + if not found: + raise MineruRuntimeUnavailableError( + stderr=( + "Could not find 'mineru' executable. " + "Ensure it is installed and on the PATH, or set NEWSDOM_MINERU_BIN." + ) + ) + return found + + +def mineru_runtime_available() -> bool: + """Return whether the configured MinerU executable resolves for this process.""" + + try: + executable = _resolve_mineru_bin() + except MineruRuntimeUnavailableError: + return False + if os.path.sep in executable or (os.path.altsep and os.path.altsep in executable): + path = Path(executable) + return path.is_file() and os.access(path, os.X_OK) + return _cached_which(executable) is not None + + +def _find_output_dir(base_output_dir: Path, method: str = DEFAULT_MODE) -> Path: + """Locate the parse-method output directory created by MinerU. + + MinerU writes results under ``///`` where the method + subdirectory reflects the parsing mode. The requested ``method`` is tried + first, then the other known method directories, so ``auto`` runs that + resolve to a concrete txt/ocr layout are still discovered. + """ + + search_order = [method, *(m for m in _KNOWN_METHOD_DIRS if m != method)] + for candidate_method in search_order: + try: + return next(base_output_dir.glob(f"*/{candidate_method}")) + except StopIteration: + continue + raise FileNotFoundError("MinerU output directory was not produced") + + +def _execute_mineru(cmd: list[str]) -> subprocess.CompletedProcess[str]: + """Execute the MinerU command and handle runtime errors.""" + try: + return subprocess.run( + cmd, check=True, capture_output=True, text=True, timeout=300, shell=False + ) + except subprocess.TimeoutExpired as exc: + stdout_str = ( + exc.stdout.decode("utf-8", "replace") + if isinstance(exc.stdout, bytes) + else exc.stdout + ) + raise MineruRuntimeUnavailableError( + returncode=-1, + stdout=stdout_str or "", + stderr="OCR processing timed out after 5 minutes", + ) from exc + except subprocess.CalledProcessError as exc: + raise MineruRuntimeUnavailableError( + returncode=exc.returncode, + stdout=exc.output, + stderr=exc.stderr, + ) from exc + except FileNotFoundError as exc: + raise MineruRuntimeUnavailableError() from exc + + +def _read_mineru_json(path: Path, *, artifact: str) -> Any: + """Read a MinerU JSON artifact with safe, differentiated failure messages.""" + try: + return json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise MineruIncompleteOutputError(f"{artifact} JSON was malformed") from exc + except (OSError, UnicodeDecodeError) as exc: + raise MineruIncompleteOutputError(f"{artifact} JSON could not be read") from exc + + +def _parse_mineru_output( + output_dir: Path, input_pdf: Path, method: str = DEFAULT_MODE +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + """Parse the JSON outputs generated by MinerU.""" + try: + ocr_dir = _find_output_dir(output_dir, method) + content_path = ocr_dir / f"{input_pdf.stem}_content_list.json" + if not content_path.exists(): + try: + content_path = next(ocr_dir.glob("*_content_list.json")) + except StopIteration: + raise FileNotFoundError("MinerU content list JSON was not produced") + try: + model_path = next(ocr_dir.glob("*_model.json")) + except StopIteration: + raise FileNotFoundError("MinerU model JSON was not produced") + except FileNotFoundError as exc: + raise MineruIncompleteOutputError() from exc + content_list = _read_mineru_json(content_path, artifact="content list") + model = _read_mineru_json(model_path, artifact="model") + + return content_list, model + + +def run_mineru( + input_pdf: Path, + *, + language: str = DEFAULT_LANGUAGE, + mode: str = DEFAULT_MODE, +) -> dict[str, Any]: + """Run MinerU on a PDF and return parsed JSON artifacts plus raw process output. + + ``language`` and ``mode`` are forwarded to the MinerU CLI (validated by + :func:`build_mineru_command`). PATH lookups for the default MinerU + executable are cached, while NEWSDOM_MINERU_BIN is evaluated on each call to + allow runtime overrides. + """ + + resolved_mode = normalize_mode(mode) + mineru_bin = _resolve_mineru_bin() + with tempfile.TemporaryDirectory(prefix="newsdom-mineru-") as tempdir: + output_dir = Path(tempdir) + cmd = build_mineru_command( + input_pdf, + output_dir, + mineru_bin=mineru_bin, + language=language, + mode=resolved_mode, + ) + + completed = _execute_mineru(cmd) + content_list, model = _parse_mineru_output(output_dir, input_pdf, resolved_mode) + + return { + "content_list": content_list, + "model": model, + "stdout": completed.stdout, + "stderr": completed.stderr, + } From a6ef917f55cb0afe981d8e7ad6325578c1f75a1a Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Tue, 1 Sep 2026 00:40:26 +0000 Subject: [PATCH 3/3] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20Sentinel:=20[CRITIC?= =?UTF-8?q?AL]=20=ED=8C=8C=EC=9D=BC=20=ED=8C=8C=EC=8B=B1=20=EC=A4=91=20?= =?UTF-8?q?=EB=B0=9C=EC=83=9D=ED=95=98=EB=8A=94=20=EC=B2=98=EB=A6=AC?= =?UTF-8?q?=EB=90=98=EC=A7=80=20=EC=95=8A=EC=9D=80=20=EC=98=88=EC=99=B8?= =?UTF-8?q?=EB=A1=9C=20=EC=9D=B8=ED=95=9C=20DoS=20=EC=B7=A8=EC=95=BD?= =?UTF-8?q?=EC=A0=90=20=EC=88=98=EC=A0=95=20=EB=B0=8F=20=ED=8C=8C=EC=8B=B1?= =?UTF-8?q?=20=EC=A0=95=EA=B7=9C=EC=8B=9D=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .jules/sentinel.md | 5 + fix.patch | 7 - fix2.patch | 9 - src/newsdom_api/mineru_runner.py.orig | 343 -------------------------- 4 files changed, 5 insertions(+), 359 deletions(-) delete mode 100644 fix.patch delete mode 100644 fix2.patch delete mode 100644 src/newsdom_api/mineru_runner.py.orig diff --git a/.jules/sentinel.md b/.jules/sentinel.md index 98d6f30a..d105a58b 100644 --- a/.jules/sentinel.md +++ b/.jules/sentinel.md @@ -95,3 +95,8 @@ **Vulnerability:** 악의적인 페이로드 업로드 시 `PdfReader`에서 발생하는 `TypeError` 또는 `MemoryError` 등 처리되지 않은 예외로 인해 500 상태 코드 및 서버 리소스 소진 유발 가능성 발견. **Learning:** `PdfReader`는 손상되거나 특수하게 조작된 PDF 파일 파싱 시 다양한 형태의 내장 예외(built-in exceptions)를 던질 수 있으며, 이를 특정 예외로만 잡을 경우 예상치 못한 시스템 장애(DoS)로 이어질 수 있음. **Prevention:** `_validate_pdf_structure` 내부에서 `Exception`을 포괄적으로 잡아내어 415 상태 코드로 안전하게 처리(fail securely)하고, 로거(logger)를 통해 예외 정보를 남겨 추적성을 유지함. + +## 2026-08-31 - Fix Regex Escape Sequence for Newlines in MinerU Argv Blocklist +**Vulnerability:** 셸 인젝션 방지 정규식 `_UNSAFE_CHARS_PATTERN`에서 개행 문자를 필터링하기 위해 사용된 `\n`과 `\r`가 정규식 내에서 이스케이프되지 않고 `\n`과 `\r`라는 문자 그 자체로 인식되어, 개행 문자를 통한 인젝션 방지가 제대로 동작하지 않는 버그 발견. +**Learning:** Python 정규식(`re.compile()`)에서 raw string(`r""`)을 사용할 때 `\n` 등 특수 제어 문자를 문자 클래스(`[]`) 내에 작성하더라도 의도한 대로 이스케이프되지 않는 경우가 발생할 수 있음. 명시적인 백슬래시 이스케이프(`\n`, `\r`)를 제공하거나 raw string을 쓰지 않아야 함. +**Prevention:** `[\0&;|`$<>\n\r]`를 올바르게 이스케이프하기 위해 정규식 내 제어 문자에 대해 안전한 이스케이프 처리를 유지해야 함. diff --git a/fix.patch b/fix.patch deleted file mode 100644 index 2b42991a..00000000 --- a/fix.patch +++ /dev/null @@ -1,7 +0,0 @@ ---- src/newsdom_api/mineru_runner.py -+++ src/newsdom_api/mineru_runner.py -@@ -16,7 +16,7 @@ - # ⚡ Bolt: Use a pre-compiled regex to push pattern matching to C, - # avoiding the Python-level overhead of `any()` and generator comprehensions --_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<>\\n\\r]") -+_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<>\\n\\r]") diff --git a/fix2.patch b/fix2.patch deleted file mode 100644 index 7eaed1c1..00000000 --- a/fix2.patch +++ /dev/null @@ -1,9 +0,0 @@ ---- src/newsdom_api/mineru_runner.py -+++ src/newsdom_api/mineru_runner.py -@@ -17,7 +17,7 @@ - # ⚡ Bolt: Use a pre-compiled regex to push pattern matching to C, - # avoiding the Python-level overhead of `any()` and generator comprehensions --_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<>\\n\\r]") -+_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<\>\n\r]") - - # MinerU 3.4.4's public CLI defaults to ``ch``. That model covers Chinese, diff --git a/src/newsdom_api/mineru_runner.py.orig b/src/newsdom_api/mineru_runner.py.orig deleted file mode 100644 index 0aee338d..00000000 --- a/src/newsdom_api/mineru_runner.py.orig +++ /dev/null @@ -1,343 +0,0 @@ -"""Invoke MinerU as an external parser and collect its structured outputs.""" - -from __future__ import annotations - -import json -import os -import re -import shutil -import subprocess -import tempfile -from functools import lru_cache -from pathlib import Path -from typing import Any - -from .errors import MineruIncompleteOutputError, MineruRuntimeUnavailableError - -# ⚡ Bolt: Use a pre-compiled regex to push pattern matching to C, -# avoiding the Python-level overhead of `any()` and generator comprehensions -_UNSAFE_CHARS_PATTERN = re.compile(r"[\0&;|`$<>\\n\\r]") - -# MinerU 3.4.4's public CLI defaults to ``ch``. That model covers Chinese, -# English, Japanese, Traditional Chinese, and Latin; callers can select another -# supported script family through the request parameter. -DEFAULT_LANGUAGE = "ch" -DEFAULT_MODE = "auto" - -# Parsing modes understood by MinerU's ``-m`` flag. ``auto`` picks txt vs. ocr -# per document, ``ocr`` forces optical recognition, ``txt`` extracts the -# embedded text layer only. -VALID_MODES = frozenset({"auto", "ocr", "txt"}) - -# Public language keys accepted by MinerU 3.4.4's CLI. Keeping this contract -# locally lets the API return a client-visible 422 instead of a downstream 503. -VALID_LANGUAGES = frozenset( - { - "ch", - "ch_server", - "korean", - "ta", - "te", - "ka", - "th", - "el", - "arabic", - "east_slavic", - "cyrillic", - "devanagari", - } -) - -# Compatibility aliases published by MinerU. Aliases are canonicalized before -# subprocess execution so behavior does not depend on a particular CLI wrapper. -_LANGUAGE_ALIASES = { - "en": "ch", - "japan": "ch", - "chinese_cht": "ch", - "latin": "ch", - "ar": "arabic", - "fa": "arabic", - "ug": "arabic", - "ur": "arabic", - "ps": "arabic", - "ku": "arabic", - "sd": "arabic", - "bal": "arabic", - "ru": "east_slavic", - "be": "east_slavic", - "uk": "east_slavic", - "hi": "devanagari", - "mr": "devanagari", - "ne": "devanagari", - "bh": "devanagari", - "mai": "devanagari", - "ang": "devanagari", - "bho": "devanagari", - "mah": "devanagari", - "sck": "devanagari", - "new": "devanagari", - "gom": "devanagari", - "sa": "devanagari", - "bgc": "devanagari", - "rs_cyrillic": "cyrillic", - "bg": "cyrillic", - "mn": "cyrillic", - "abq": "cyrillic", - "ady": "cyrillic", - "kbd": "cyrillic", - "ava": "cyrillic", - "dar": "cyrillic", - "inh": "cyrillic", - "che": "cyrillic", - "lbe": "cyrillic", - "lez": "cyrillic", - "tab": "cyrillic", - "kk": "cyrillic", - "ky": "cyrillic", - "tg": "cyrillic", - "mk": "cyrillic", - "tt": "cyrillic", - "cv": "cyrillic", - "ba": "cyrillic", - "mhr": "cyrillic", - "mo": "cyrillic", - "udm": "cyrillic", - "kv": "cyrillic", - "os": "cyrillic", - "bua": "cyrillic", - "xal": "cyrillic", - "tyv": "cyrillic", - "sah": "cyrillic", - "kaa": "cyrillic", -} - -# Method subdirectories MinerU may create beneath the output directory. -_KNOWN_METHOD_DIRS = ("auto", "ocr", "txt") - - -def normalize_mode(mode: str) -> str: - """Validate and normalize a MinerU parsing mode. - - Returns the lower-cased mode when it is one of ``auto``/``ocr``/``txt`` and - raises :class:`ValueError` otherwise so callers can surface a client error. - """ - - normalized = str(mode).strip().lower() - if normalized not in VALID_MODES: - raise ValueError(f"Unsupported MinerU mode: {mode!r}") - return normalized - - -def normalize_language(language: str) -> str: - """Validate and normalize a MinerU language code. - - Returns the canonical MinerU 3.4.4 public language key and raises - :class:`ValueError` otherwise so callers receive a client error instead of - a downstream runtime failure. - """ - - normalized = str(language).strip().lower() - canonical = _LANGUAGE_ALIASES.get(normalized, normalized) - if canonical not in VALID_LANGUAGES: - raise ValueError(f"Unsupported MinerU language: {language!r}") - return canonical - - -def _mineru_command_arg(value: str | Path, *, label: str) -> str: - """Validate a path or executable string before passing it to MinerU argv.""" - - value_str = str(value) - if _UNSAFE_CHARS_PATTERN.search(value_str): - raise ValueError(f"Unsafe {label} for MinerU command") - if value_str.startswith("-"): - raise ValueError(f"Unsafe {label} for MinerU command") - return value_str - - -def build_mineru_command( - input_pdf: Path, - output_dir: Path, - mineru_bin: str = "mineru", - *, - language: str = DEFAULT_LANGUAGE, - mode: str = DEFAULT_MODE, -) -> list[str]: - """Build the MinerU CLI command for pipeline execution. - - ``language`` maps to MinerU's ``-l`` flag and ``mode`` to ``-m``. Both are - validated so callers cannot inject arbitrary argv values; the defaults are - aligned with MinerU 3.4.4 (``ch``/``auto``). - """ - - validated_mode = normalize_mode(mode) - validated_language = normalize_language(language) - - return [ - _mineru_command_arg(mineru_bin, label="MinerU executable"), - "-p", - _mineru_command_arg(input_pdf, label="input PDF path"), - "-o", - _mineru_command_arg(output_dir, label="output directory path"), - "-b", - "pipeline", - "-m", - validated_mode, - "-l", - validated_language, - ] - - -@lru_cache -def _cached_which(cmd: str) -> str | None: - """Cache shutil.which to avoid redundant filesystem lookups.""" - return shutil.which(cmd) - - -def _resolve_mineru_bin() -> str: - """Resolve the MinerU executable path for this process. - - The shutil.which result is cached, but NEWSDOM_MINERU_BIN is evaluated - on every call to allow runtime overrides. - """ - - configured = os.environ.get("NEWSDOM_MINERU_BIN") - if configured: - return configured - found = _cached_which("mineru") - if not found: - raise MineruRuntimeUnavailableError( - stderr=( - "Could not find 'mineru' executable. " - "Ensure it is installed and on the PATH, or set NEWSDOM_MINERU_BIN." - ) - ) - return found - - -def mineru_runtime_available() -> bool: - """Return whether the configured MinerU executable resolves for this process.""" - - try: - executable = _resolve_mineru_bin() - except MineruRuntimeUnavailableError: - return False - if os.path.sep in executable or (os.path.altsep and os.path.altsep in executable): - path = Path(executable) - return path.is_file() and os.access(path, os.X_OK) - return _cached_which(executable) is not None - - -def _find_output_dir(base_output_dir: Path, method: str = DEFAULT_MODE) -> Path: - """Locate the parse-method output directory created by MinerU. - - MinerU writes results under ``///`` where the method - subdirectory reflects the parsing mode. The requested ``method`` is tried - first, then the other known method directories, so ``auto`` runs that - resolve to a concrete txt/ocr layout are still discovered. - """ - - search_order = [method, *(m for m in _KNOWN_METHOD_DIRS if m != method)] - for candidate_method in search_order: - try: - return next(base_output_dir.glob(f"*/{candidate_method}")) - except StopIteration: - continue - raise FileNotFoundError("MinerU output directory was not produced") - - -def _execute_mineru(cmd: list[str]) -> subprocess.CompletedProcess[str]: - """Execute the MinerU command and handle runtime errors.""" - try: - return subprocess.run( - cmd, check=True, capture_output=True, text=True, timeout=300, shell=False - ) - except subprocess.TimeoutExpired as exc: - stdout_str = ( - exc.stdout.decode("utf-8", "replace") - if isinstance(exc.stdout, bytes) - else exc.stdout - ) - raise MineruRuntimeUnavailableError( - returncode=-1, - stdout=stdout_str or "", - stderr="OCR processing timed out after 5 minutes", - ) from exc - except subprocess.CalledProcessError as exc: - raise MineruRuntimeUnavailableError( - returncode=exc.returncode, - stdout=exc.output, - stderr=exc.stderr, - ) from exc - except FileNotFoundError as exc: - raise MineruRuntimeUnavailableError() from exc - - -def _read_mineru_json(path: Path, *, artifact: str) -> Any: - """Read a MinerU JSON artifact with safe, differentiated failure messages.""" - try: - return json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise MineruIncompleteOutputError(f"{artifact} JSON was malformed") from exc - except (OSError, UnicodeDecodeError) as exc: - raise MineruIncompleteOutputError(f"{artifact} JSON could not be read") from exc - - -def _parse_mineru_output( - output_dir: Path, input_pdf: Path, method: str = DEFAULT_MODE -) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: - """Parse the JSON outputs generated by MinerU.""" - try: - ocr_dir = _find_output_dir(output_dir, method) - content_path = ocr_dir / f"{input_pdf.stem}_content_list.json" - if not content_path.exists(): - try: - content_path = next(ocr_dir.glob("*_content_list.json")) - except StopIteration: - raise FileNotFoundError("MinerU content list JSON was not produced") - try: - model_path = next(ocr_dir.glob("*_model.json")) - except StopIteration: - raise FileNotFoundError("MinerU model JSON was not produced") - except FileNotFoundError as exc: - raise MineruIncompleteOutputError() from exc - content_list = _read_mineru_json(content_path, artifact="content list") - model = _read_mineru_json(model_path, artifact="model") - - return content_list, model - - -def run_mineru( - input_pdf: Path, - *, - language: str = DEFAULT_LANGUAGE, - mode: str = DEFAULT_MODE, -) -> dict[str, Any]: - """Run MinerU on a PDF and return parsed JSON artifacts plus raw process output. - - ``language`` and ``mode`` are forwarded to the MinerU CLI (validated by - :func:`build_mineru_command`). PATH lookups for the default MinerU - executable are cached, while NEWSDOM_MINERU_BIN is evaluated on each call to - allow runtime overrides. - """ - - resolved_mode = normalize_mode(mode) - mineru_bin = _resolve_mineru_bin() - with tempfile.TemporaryDirectory(prefix="newsdom-mineru-") as tempdir: - output_dir = Path(tempdir) - cmd = build_mineru_command( - input_pdf, - output_dir, - mineru_bin=mineru_bin, - language=language, - mode=resolved_mode, - ) - - completed = _execute_mineru(cmd) - content_list, model = _parse_mineru_output(output_dir, input_pdf, resolved_mode) - - return { - "content_list": content_list, - "model": model, - "stdout": completed.stdout, - "stderr": completed.stderr, - }