diff --git a/CHANGELOG.md b/CHANGELOG.md index 2398ea5c..71a07ea4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > acceptance are aligned for an actual 0.3.0 publication. ### Changed +- Align `/parse` with Naruon's signed PDF DOM transport at a bounded 64 MiB. + The endpoint still validates authentication before multipart parsing and + rejects the first byte above the ceiling with `413`. + - `/parse`를 언어 선택형 파서로 일반화: MinerU `-l japan`/`-m ocr` 하드코딩을 제거하고 optional form 필드 `language`(MinerU 3.4.4 공식 기본 `ch`, 공개 언어군/alias 검증)와 `mode`(`auto`/`ocr`/`txt`, 기본 `auto`)로 파라미터화. `mode=auto`는 born-digital PDF가 강제 OCR을 건너뛰도록 함. 기존 입력 `language=japan&mode=ocr`는 공식 규약대로 `ch`/`ocr`로 정규화됨. - OpenAPI 제목/설명, README, `ArticleNode.headline` 문서를 일반 문서용 (section heading) 표현으로 재구성하여 특정 언어/신문 가정을 소비자에게 노출하지 않도록 함. 응답 스키마 필드는 하위 호환을 위해 변경하지 않음. diff --git a/docs/adr/0003-bounded-pdf-upload-transport.md b/docs/adr/0003-bounded-pdf-upload-transport.md new file mode 100644 index 00000000..0a426bff --- /dev/null +++ b/docs/adr/0003-bounded-pdf-upload-transport.md @@ -0,0 +1,44 @@ +# ADR-0003: Bounded PDF upload transport + +**Status:** Accepted +**Date:** 2026-08-21 +**Decision owner:** NewsDOM maintainers +**Scope:** Authenticated `POST /parse` upload boundary +**Figma File ID:** N/A — sidecar API contract; no visual surface. + +## Context + +Naruon's direct PDF DOM upload contract is bounded at 64 MiB, but the owning +NewsDOM sidecar still rejected the same customer PDF above 20 MiB. That +cross-service mismatch made the equivalent email and manual workflows behave +differently and caused a customer-visible failure after the request crossed a +service boundary. + +## Decision + +Set `MAX_PARSE_UPLOAD_BYTES` to 64 MiB. Keep bearer authentication before +multipart body parsing, the streaming first-byte-over-limit check, PDF +signature validation, temporary-file cleanup, and the `413 Payload Too Large` +response unchanged. + +Naruon remains the consumer-side owner of its signed persistence boundary. This +ADR only changes the sidecar's transport ceiling; parser runtime, concurrency, +storage quotas, and deployment capacity remain separate controls. + +## Consequences + +- Customers can use the same bounded 64 MiB expectation for direct and sidecar + PDF DOM ingestion. +- A larger valid upload can reach the parser, so deployment capacity and parser + timeout controls remain mandatory. +- No unbounded body read is introduced; the endpoint continues to stop on the + first byte above the limit. + +## References (APA 7th) + +Internet Engineering Task Force. (2022). *HTTP semantics (RFC 9110).* RFC + Editor. https://www.rfc-editor.org/rfc/rfc9110 + +National Institute of Standards and Technology. (2025). *Secure software + development framework (SSDF) version 1.2* (NIST Special Publication 800-218 + Rev. 1, Initial Public Draft). https://doi.org/10.6028/NIST.SP.800-218r1.ipd diff --git a/docs/adr/README.md b/docs/adr/README.md index 6efe928f..32eee56c 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -7,6 +7,8 @@ This directory contains Architecture Decision Records (ADRs) for the `newsdom-ap | ADR | Title | Status | Date | | --------------------------------------------------------- | --------------------------------------------- | -------- | ---------- | | [0001](0001-defer-openssf-best-practices-enrollment.md) | Defer OpenSSF Best Practices Enrollment | Accepted | 2026-04-24 | +| [0002](0002-single-maintainer-review-exception.md) | Single-maintainer protected-branch review exception | Accepted | 2026-04-24 | +| [0003](0003-bounded-pdf-upload-transport.md) | Bounded PDF upload transport | Accepted | 2026-08-21 | ## ADR Status diff --git a/docs/doctoring/bounded-pdf-upload-transport.md b/docs/doctoring/bounded-pdf-upload-transport.md new file mode 100644 index 00000000..72b53818 --- /dev/null +++ b/docs/doctoring/bounded-pdf-upload-transport.md @@ -0,0 +1,21 @@ +# Doctoring record: bounded PDF upload transport + +**Observed gap:** The live NewsDOM sidecar accepted only 20 MiB while the +Naruon direct PDF DOM contract allowed 64 MiB, so equivalent customer workflows +were inconsistent. + +**Correction:** `MAX_PARSE_UPLOAD_BYTES` and its boundary tests now use 64 MiB. +Authentication remains checked before multipart parsing, and streaming input +still fails closed at the first byte over the bound. + +**Evidence:** `tests/test_parse_endpoint.py` covers the 64 MiB contract and the +unknown-size streaming over-limit path. Full coverage and exact-head hosted +checks remain required before merge. + +**References (APA 7th):** + +- Internet Engineering Task Force. (2022). *HTTP semantics (RFC 9110).* RFC + Editor. https://www.rfc-editor.org/rfc/rfc9110 +- National Institute of Standards and Technology. (2025). *Secure software + development framework (SSDF) version 1.2* (NIST Special Publication 800-218 + Rev. 1, Initial Public Draft). https://doi.org/10.6028/NIST.SP.800-218r1.ipd diff --git a/manual/api-reference.md b/manual/api-reference.md index 7ceec967..f8bcf5a4 100644 --- a/manual/api-reference.md +++ b/manual/api-reference.md @@ -32,6 +32,10 @@ FastAPI는 OpenAPI 기반의 대화형 API 문서를 자동으로 생성합니 #### 요청 매개변수 (Request Body) - **`file`** (`UploadFile`, 필수): 변환할 PDF 바이너리 파일 데이터 (`multipart/form-data`) +PDF 파일은 최대 64 MiB까지 허용됩니다. 한도를 초과하면 서버가 임시 파일을 +MinerU에 전달하지 않고 `413 Payload Too Large`를 반환하므로, 고객은 PDF를 +분할한 뒤 각 파일을 다시 업로드해야 합니다. + #### cURL 테스트 예제 ```bash diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index f61aafc2..960a8637 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -41,7 +41,9 @@ from .schemas import HealthResponse, ParseResponse, ReadinessResponse from .service import parse_pdf -MAX_PARSE_UPLOAD_BYTES = 20 * 1024 * 1024 +# Keep the sidecar transport ceiling aligned with Naruon's direct PDF DOM +# upload. The streaming read still rejects the first byte above this bound. +MAX_PARSE_UPLOAD_BYTES = 64 * 1024 * 1024 MAX_AUTHORIZATION_HEADER_BYTES = MAX_BEARER_HEADER_BYTES UNSUPPORTED_MEDIA_DETAIL = "Unsupported Media Type" PAYLOAD_TOO_LARGE_DETAIL = "Payload Too Large" diff --git a/tests/test_parse_endpoint.py b/tests/test_parse_endpoint.py index 1491ada0..89fce227 100644 --- a/tests/test_parse_endpoint.py +++ b/tests/test_parse_endpoint.py @@ -32,17 +32,35 @@ class _ReadTrackingUpload: filename = "fixture.pdf" size = 10 * 1024 * 1024 - def __init__(self, payload: bytes): + def __init__(self, payload: bytes | int): self._payload = payload self._offset = 0 self.read_sizes: list[int] = [] + self.bytes_returned = 0 async def read(self, size: int = -1) -> bytes: self.read_sizes.append(size) - if size < 0: - size = len(self._payload) - self._offset - chunk = self._payload[self._offset : self._offset + size] + if isinstance(self._payload, int): + remaining = self._payload - self._offset + if remaining <= 0: + return b"" + if size < 0: + size = remaining + count = min(size, remaining) + prefix = b"%PDF-" + chunk = b"" + if self._offset < len(prefix): + prefix_count = min(count, len(prefix) - self._offset) + chunk = prefix[self._offset : self._offset + prefix_count] + count -= prefix_count + if count: + chunk += b"x" * count + else: + if size < 0: + size = len(self._payload) - self._offset + chunk = self._payload[self._offset : self._offset + size] self._offset += len(chunk) + self.bytes_returned += len(chunk) return chunk @@ -358,6 +376,29 @@ def fake_parse_pdf_bytes(file_path, filename, **kwargs): assert response.json()["detail"] == "Payload Too Large" +def test_parse_endpoint_budget_matches_naruon_transport_contract(): + """Keep the sidecar upload ceiling aligned with Naruon's PDF transport.""" + assert MAX_PARSE_UPLOAD_BYTES == 64 * 1024 * 1024 + + +@pytest.mark.asyncio +async def test_parse_endpoint_accepts_exact_upload_budget(monkeypatch): + """Accept a valid streamed PDF whose final byte is exactly at the limit.""" + monkeypatch.setattr("newsdom_api.main._validate_pdf_structure", lambda _: None) + monkeypatch.setattr( + "newsdom_api.main.parse_pdf", + lambda file_path, filename, **kwargs: {"document_id": "fixture", "pages": []}, + ) + + upload = _ReadTrackingUpload(MAX_PARSE_UPLOAD_BYTES) + upload.size = MAX_PARSE_UPLOAD_BYTES + + result = await parse(upload) + + assert result == {"document_id": "fixture", "pages": []} + assert upload.bytes_returned == MAX_PARSE_UPLOAD_BYTES + + @pytest.mark.asyncio async def test_parse_endpoint_rejects_large_file_without_size_metadata(): upload = _ReadTrackingUpload(b"%PDF-" + (b"x" * MAX_PARSE_UPLOAD_BYTES)) diff --git a/tests/test_parse_upload_budget_contract.py b/tests/test_parse_upload_budget_contract.py new file mode 100644 index 00000000..3665eaeb --- /dev/null +++ b/tests/test_parse_upload_budget_contract.py @@ -0,0 +1,49 @@ +"""Exact streaming-boundary regressions for the public PDF upload budget.""" + +import pytest +from fastapi import HTTPException + +from newsdom_api.main import MAX_PARSE_UPLOAD_BYTES, parse + + +class _VirtualPdfUpload: + """Stream a bounded synthetic PDF without allocating the full payload.""" + + content_type = "application/pdf" + filename = "fixture.pdf" + size = None + + def __init__(self, total_bytes: int) -> None: + self.total_bytes = total_bytes + self.bytes_returned = 0 + + async def read(self, size: int = -1) -> bytes: + """Return at most ``size`` bytes while preserving a valid PDF prefix.""" + remaining = self.total_bytes - self.bytes_returned + if remaining <= 0: + return b"" + count = remaining if size < 0 else min(size, remaining) + prefix = b"%PDF-" + start = self.bytes_returned + chunk = b"" + if start < len(prefix): + prefix_count = min(count, len(prefix) - start) + chunk = prefix[start : start + prefix_count] + count -= prefix_count + if count: + chunk += b"x" * count + self.bytes_returned += len(chunk) + return chunk + + +@pytest.mark.asyncio +async def test_streaming_upload_rejects_exact_first_byte_over_budget() -> None: + """Reject after consuming exactly the first byte beyond the 64 MiB ceiling.""" + upload = _VirtualPdfUpload(MAX_PARSE_UPLOAD_BYTES + 1) + + with pytest.raises(HTTPException) as exc_info: + await parse(upload) + + assert exc_info.value.status_code == 413 + assert exc_info.value.detail == "Payload Too Large" + assert upload.bytes_returned == MAX_PARSE_UPLOAD_BYTES + 1