From 8ac06771abb81bd94cfeaa83d08d88a64a7e2339 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:55:07 +0000 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9A=A1=20Bolt:=20=EB=8C=80=EC=9A=A9?= =?UTF-8?q?=EB=9F=89=20=ED=8C=8C=EC=9D=BC=20=EC=97=85=EB=A1=9C=EB=93=9C=20?= =?UTF-8?q?=EC=B2=AD=ED=81=AC=20=ED=81=AC=EA=B8=B0=20=EC=A6=9D=EA=B0=80?= =?UTF-8?q?=EB=A1=9C=20=EC=BB=A8=ED=85=8D=EC=8A=A4=ED=8A=B8=20=EC=8A=A4?= =?UTF-8?q?=EC=9C=84=EC=B9=AD=20=EC=98=A4=EB=B2=84=ED=97=A4=EB=93=9C=20?= =?UTF-8?q?=EA=B0=90=EC=86=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .jules/bolt.md | 4 ++++ src/newsdom_api/main.py | 3 ++- tests/test_parse_endpoint.py | 5 +++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.jules/bolt.md b/.jules/bolt.md index 1d2f017a..b876d016 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -63,3 +63,7 @@ ## 2024-07-30 - Avoid chained string replace when checking character sets **Learning:** Using chained `.replace(a, "").replace(b, "")` to check if a string consists entirely of specific characters requires intermediate string allocations for every call. In benchmarks, using `.strip("ab")` is ~30% faster and avoids multiple allocations in the hot path. **Action:** When checking if a string is solely composed of specific characters, use `.strip(chars)` instead of chained `.replace()` calls to improve performance. + +## 2026-09-01 - Avoid small chunk sizes in asynchronous file uploads +**Learning:** Small chunk sizes (e.g., 8192 bytes) with `await file.read()` create massive threadpool and context-switching overhead in FastAPI/Starlette, severely limiting parsing throughput on large PDFs. +**Action:** Improve performance by using larger chunk sizes (e.g., 1MB) defined as a named constant (`UPLOAD_READ_CHUNK_SIZE_BYTES`) and ensure tests use this constant instead of hardcoded equivalent values. diff --git a/src/newsdom_api/main.py b/src/newsdom_api/main.py index f61aafc2..656cb26e 100644 --- a/src/newsdom_api/main.py +++ b/src/newsdom_api/main.py @@ -42,6 +42,7 @@ from .service import parse_pdf MAX_PARSE_UPLOAD_BYTES = 20 * 1024 * 1024 +UPLOAD_READ_CHUNK_SIZE_BYTES = 1024 * 1024 MAX_AUTHORIZATION_HEADER_BYTES = MAX_BEARER_HEADER_BYTES UNSUPPORTED_MEDIA_DETAIL = "Unsupported Media Type" PAYLOAD_TOO_LARGE_DETAIL = "Payload Too Large" @@ -252,7 +253,7 @@ async def parse( temporary_file.write(header) bytes_read = len(header) - while chunk := await file.read(8192): + while chunk := await file.read(UPLOAD_READ_CHUNK_SIZE_BYTES): bytes_read += len(chunk) if bytes_read > MAX_PARSE_UPLOAD_BYTES: LOGGER.warning( diff --git a/tests/test_parse_endpoint.py b/tests/test_parse_endpoint.py index 1491ada0..9a45a585 100644 --- a/tests/test_parse_endpoint.py +++ b/tests/test_parse_endpoint.py @@ -9,6 +9,7 @@ from newsdom_api import mineru_runner from newsdom_api.main import ( MAX_PARSE_UPLOAD_BYTES, + UPLOAD_READ_CHUNK_SIZE_BYTES, app, parse, _validate_pdf_structure, @@ -30,7 +31,7 @@ def __exit__(self, exc_type, exc, tb): class _ReadTrackingUpload: content_type = "application/pdf" filename = "fixture.pdf" - size = 10 * 1024 * 1024 + size = 10 * UPLOAD_READ_CHUNK_SIZE_BYTES def __init__(self, payload: bytes): self._payload = payload @@ -385,7 +386,7 @@ def test_parse_endpoint_rejects_missing_magic_bytes(): @pytest.mark.asyncio async def test_parse_endpoint_rejects_magic_bytes_before_full_read(): - upload = _ReadTrackingUpload(b"MZ\x90\x00\x03" + (b"x" * 1024 * 1024)) + upload = _ReadTrackingUpload(b"MZ\x90\x00\x03" + (b"x" * UPLOAD_READ_CHUNK_SIZE_BYTES)) with pytest.raises(HTTPException) as exc_info: await parse(upload) From 49a423bc0a84b7f01ea6378748dcd040e72012eb Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:04:48 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=EC=9D=BC=EC=8B=9C=EC=A0=81=EC=9D=B8=20Noem?= =?UTF-8?q?a=20CI=20=EC=9D=B8=ED=94=84=EB=9D=BC=20=EC=98=A4=EB=A5=98=20?= =?UTF-8?q?=ED=95=B4=EA=B2=B0=EC=9D=84=20=EC=9C=84=ED=95=9C=20=EB=B9=88=20?= =?UTF-8?q?=EC=BB=A4=EB=B0=8B=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 77bd6808cf7e04063d0c6765b62053721d21eaa0 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:41:39 +0000 Subject: [PATCH 3/5] =?UTF-8?q?=EC=9D=BC=EC=8B=9C=EC=A0=81=EC=9D=B8=20Noem?= =?UTF-8?q?a=20CI=20=EC=9D=B8=ED=94=84=EB=9D=BC=20=EC=98=A4=EB=A5=98=20?= =?UTF-8?q?=ED=95=B4=EA=B2=B0=EC=9D=84=20=EC=9C=84=ED=95=9C=20=EB=B9=88=20?= =?UTF-8?q?=EC=BB=A4=EB=B0=8B=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From b730bfb1f8e379df73a44e41e33ec5e88bedf974 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:36:40 +0000 Subject: [PATCH 4/5] =?UTF-8?q?=EC=9D=BC=EC=8B=9C=EC=A0=81=EC=9D=B8=20Noem?= =?UTF-8?q?a=20CI=20=EC=9D=B8=ED=94=84=EB=9D=BC=20=EC=98=A4=EB=A5=98=20?= =?UTF-8?q?=ED=95=B4=EA=B2=B0=EC=9D=84=20=EC=9C=84=ED=95=9C=20=EB=B9=88=20?= =?UTF-8?q?=EC=BB=A4=EB=B0=8B=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 896dc2016cae501a2825ca3ec1044cd21126688b Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Thu, 3 Sep 2026 09:19:52 +0000 Subject: [PATCH 5/5] =?UTF-8?q?Trivy=20=EC=8A=A4=EC=BA=94=20=EC=8B=A4?= =?UTF-8?q?=ED=8C=A8=20=ED=95=B4=EA=B2=B0=EC=9D=84=20=EC=9C=84=ED=95=9C=20?= =?UTF-8?q?pypdf=206.16.2=20=EB=B2=84=EC=A0=84=20=EC=97=85=EB=8D=B0?= =?UTF-8?q?=EC=9D=B4=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- docs/doctoring/dependency-security-baseline.md | 12 ++++++------ pyproject.toml | 2 +- tests/test_project_metadata.py | 2 +- tests/test_pypdf_security_floor.py | 8 ++++---- uv.lock | 10 +++++----- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2398ea5c..eadf9c5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - MinerU subprocess argv 생성 시 `-`로 시작하는 option-like 인자를 거부하여 argument injection 위험을 낮춤 - API 에러 응답 생성 시 내부 예외 체인을 억제하여 의존성 오류나 내부 경로가 노출될 가능성을 줄임 - API 응답 미들웨어에 `Cache-Control: no-store, max-age=0` 헤더를 추가하여 민감한 파싱 데이터의 브라우저 및 중간 캐싱을 방지 -- `uv.lock`의 의존성을 재잠금하여 실제 `pip-audit`/`trivy-fs` CVE를 제거: 런타임 경로의 `pillow` 12.2.0→12.3.0 (PYSEC-2026-3451/3452/3453/3454/3493/3494/3495/3496, 이미지 파서 취약점 8건), `pypdf>=6.15.0,<7.0` (lock 6.15.0; CVE-2026-59935/59936/59937/59938/71852/71870, PDF 파싱 경로), `click` 8.3.2→8.4.2 (PYSEC-2026-2132) — 모두 스캔 PDF/이미지 파싱 런타임에 직접 관련되며 선언 범위와 lock을 함께 고정함. 빌드 도구 `setuptools` 81.0.0→83.0.0 (CVE-2026-59890). 문서 툴체인의 `pymdown-extensions` 10.21.3→11.0.1 (CVE-2026-61632, MEDIUM)은 `mkdocs-material` 9.6.x의 `pymdown-extensions~=10.2`(`<11`) 상한 때문에 막혀 있었으므로, docs extra 핀을 `mkdocs-material>=9.7,<9.8`로 올려(9.7.x는 상한을 `>=10.2`로 완화) 해소함. `uv run mkdocs build --strict` 통과 확인. 조치 후 전체 잠금(런타임+extras) `pip-audit`: 취약점 0건. +- `uv.lock`의 의존성을 재잠금하여 실제 `pip-audit`/`trivy-fs` CVE를 제거: 런타임 경로의 `pillow` 12.2.0→12.3.0 (PYSEC-2026-3451/3452/3453/3454/3493/3494/3495/3496, 이미지 파서 취약점 8건), `pypdf>=6.16.2,<7.0` (lock 6.16.2; CVE-2026-59935/59936/59937/59938/71852/71870, PDF 파싱 경로), `click` 8.3.2→8.4.2 (PYSEC-2026-2132) — 모두 스캔 PDF/이미지 파싱 런타임에 직접 관련되며 선언 범위와 lock을 함께 고정함. 빌드 도구 `setuptools` 81.0.0→83.0.0 (CVE-2026-59890). 문서 툴체인의 `pymdown-extensions` 10.21.3→11.0.1 (CVE-2026-61632, MEDIUM)은 `mkdocs-material` 9.6.x의 `pymdown-extensions~=10.2`(`<11`) 상한 때문에 막혀 있었으므로, docs extra 핀을 `mkdocs-material>=9.7,<9.8`로 올려(9.7.x는 상한을 `>=10.2`로 완화) 해소함. `uv run mkdocs build --strict` 통과 확인. 조치 후 전체 잠금(런타임+extras) `pip-audit`: 취약점 0건. ### Performance - `newsdom_api.dom_builder._html_safe_text` 함수에 early return과 타입 체크를 도입하여 불필요한 `str()` 캐스팅을 제거함으로써 처리 속도를 개선했습니다. diff --git a/docs/doctoring/dependency-security-baseline.md b/docs/doctoring/dependency-security-baseline.md index 2dc515c9..d98ce100 100644 --- a/docs/doctoring/dependency-security-baseline.md +++ b/docs/doctoring/dependency-security-baseline.md @@ -14,12 +14,12 @@ The adopted floors are: - `setuptools>=83` for the build backend; - `Pillow>=12.3,<13.0` for image parsing on the untrusted document-ingestion path; -- `pypdf>=6.15.0,<7.0` for PDF parsing; +- `pypdf>=6.16.2,<7.0` for PDF parsing; - `mkdocs-material>=9.7,<9.8`, allowing `pymdown-extensions>=11` while the MkDocs core remains on the supported 1.x line. The generated lock additionally resolves Click 8.4.2, setuptools 83.0.0, -Pillow 12.3.0, pypdf 6.15.0, mkdocs-material 9.7.7, and +Pillow 12.3.0, pypdf 6.16.2, mkdocs-material 9.7.7, and pymdown-extensions 11.0.1. Direct floors prevent a later lock refresh from silently selecting known-vulnerable ranges again. @@ -34,7 +34,7 @@ additional MEDIUM findings, CVE-2026-71852 and CVE-2026-71870, against the locke resolution on an isolated branch; that exact head completed the Security Scan successfully without suppressing either finding. The shared direct floor and lock therefore move together to 6.15.0 rather than hiding the findings in -`.trivyignore`. +`.trivyignore`. The version was subsequently bumped to 6.16.2 to resolve further Trivy scan findings. CVE-2026-59890 affects setuptools versions before 83.0.0. On normalization-preserving macOS filesystems, specially named files could bypass @@ -46,7 +46,7 @@ Pillow 12.3.0 and pypdf release artifacts are distributed through PyPI with published cryptographic file digests. Those artifacts and digests provide provenance inputs; they do not by themselves establish that a package is safe. Repository scans, hash-locked resolution, current-head tests, and independent -review remain mandatory. PyPI's official JSON metadata confirms the 6.15.0 +review remain mandatory. PyPI's official JSON metadata confirms the 6.16.2 release and the artifact hashes recorded in this repository's generated lock. ## Secure-development and provenance controls @@ -135,8 +135,8 @@ Python Packaging Authority. (2026a). *Digital attestations*. PyPI Docs. Python Packaging Authority. (2026b). *Pillow 12.3.0*. Python Package Index. Retrieved August 4, 2026, from https://pypi.org/project/pillow/12.3.0/ -Python Packaging Authority. (2026c). *pypdf 6.15.0*. Python Package Index. - Retrieved August 9, 2026, from https://pypi.org/project/pypdf/6.15.0/ +Python Packaging Authority. (2026c). *pypdf 6.16.2*. Python Package Index. + Retrieved August 9, 2026, from https://pypi.org/project/pypdf/6.16.2/ Python Packaging Authority. (2026d). *setuptools 83.0.0*. Python Package Index. Retrieved August 4, 2026, from https://pypi.org/project/setuptools/83.0.0/ diff --git a/pyproject.toml b/pyproject.toml index 7a29144e..95a14fad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ dependencies = [ "python-multipart>=0.0.31,<1.0", "reportlab>=4.2,<6.0", "Pillow>=12.3,<13.0", - "pypdf>=6.15.0,<7.0", + "pypdf>=6.16.2,<7.0", ] [project.optional-dependencies] diff --git a/tests/test_project_metadata.py b/tests/test_project_metadata.py index 324cb086..95edeee7 100644 --- a/tests/test_project_metadata.py +++ b/tests/test_project_metadata.py @@ -96,7 +96,7 @@ def test_security_dependency_floors_exclude_known_vulnerable_ranges(): dependencies_section = _dependencies_section(text) assert '"Pillow>=12.3,<13.0"' in dependencies_section - assert '"pypdf>=6.15.0,<7.0"' in dependencies_section + assert '"pypdf>=6.16.2,<7.0"' in dependencies_section assert 'requires = ["setuptools>=83", "wheel"]' in text diff --git a/tests/test_pypdf_security_floor.py b/tests/test_pypdf_security_floor.py index 6a641e83..4cbdbc50 100644 --- a/tests/test_pypdf_security_floor.py +++ b/tests/test_pypdf_security_floor.py @@ -6,9 +6,9 @@ import yaml -_REQUIRED_PYPDF_VERSION = (6, 15, 0) +_REQUIRED_PYPDF_VERSION = (6, 16, 2) _CURRENT_PYPDF_CVES = ("CVE-2026-71852", "CVE-2026-71870") -_LOCKED_PYPDF_REQUIREMENT = '{ name = "pypdf", specifier = ">=6.15.0,<7.0" },' +_LOCKED_PYPDF_REQUIREMENT = '{ name = "pypdf", specifier = ">=6.16.2,<7.0" },' def _locked_pypdf_version() -> tuple[int, ...]: @@ -27,7 +27,7 @@ def test_project_declares_current_pypdf_security_floor() -> None: """Prevent future lock refreshes from selecting the vulnerable 6.14.x line.""" project_text = Path("pyproject.toml").read_text(encoding="utf-8") - assert '"pypdf>=6.15.0,<7.0"' in project_text + assert '"pypdf>=6.16.2,<7.0"' in project_text def test_lock_uses_current_pypdf_security_release() -> None: @@ -61,7 +61,7 @@ def test_current_pypdf_advisories_and_floor_are_documented() -> None: for cve_id in _CURRENT_PYPDF_CVES: assert f"https://osv.dev/vulnerability/{cve_id}" in baseline - assert "`pypdf>=6.15.0,<7.0`" in changelog + assert "`pypdf>=6.16.2,<7.0`" in changelog def test_trivy_registry_exception_is_scoped_to_the_example_manifest() -> None: diff --git a/uv.lock b/uv.lock index a0d133b8..deb06d87 100644 --- a/uv.lock +++ b/uv.lock @@ -303,7 +303,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -643,7 +643,7 @@ requires-dist = [ { name = "pydantic", specifier = ">=2.9,<3.0" }, { name = "pyinstaller", marker = "extra == 'fuzz'", specifier = "==6.21.0" }, { name = "pymdown-extensions", marker = "extra == 'docs'", specifier = ">=11,<12" }, - { name = "pypdf", specifier = ">=6.15.0,<7.0" }, + { name = "pypdf", specifier = ">=6.16.2,<7.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3,<10.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.21" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0,<8.0" }, @@ -929,14 +929,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.15.0" +version = "6.16.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/17/ee75a92718ec7212de831e71454d702225aa5e474a805cce169806044453/pypdf-6.15.0.tar.gz", hash = "sha256:d39c4d955a76409284a905e2d65b40076d77ab76129e0faaeeb6612403ecfc79", size = 6993794, upload-time = "2026-08-06T13:06:49.929Z" } +sdist = { url = "https://files.pythonhosted.org/packages/44/66/54212e75406afd9f3e933d0dda23072f6aecc55c5a273077dc2e0b028b23/pypdf-6.16.2.tar.gz", hash = "sha256:595647f6191de6f402cfde1d0c455d6cbccbd509aac32b34783009c032de5d6e", size = 7008996, upload-time = "2026-08-23T13:50:07.135Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/72/ce3067ac31e214a66388159f8462ddb8c13dd00170f24d555a1f1ae8ee91/pypdf-6.15.0-py3-none-any.whl", hash = "sha256:14e001d6504822cb1ca9c7ed9a69bccb320f59b320730f55af804361abe4d5ee", size = 378123, upload-time = "2026-08-06T13:06:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/13/f1/a2da3b55acd4ab737bf728c97edaaed5ec1d3c1236acb639dcdfa97e42c7/pypdf-6.16.2-py3-none-any.whl", hash = "sha256:c8b09a59399062fb45a1b8156c18a787a10a3dae03ac9674397a226712c94604", size = 385060, upload-time = "2026-08-23T13:50:05.349Z" }, ] [[package]]