From 00e9b784badedd398d176667bb1da40bd8029c50 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Sun, 6 Sep 2026 21:53:53 +0000 Subject: [PATCH 01/20] =?UTF-8?q?=EA=B8=B0=EB=8A=A5:=20export=5Fjsonl=20?= =?UTF-8?q?=EB=8F=84=EA=B5=AC=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 + tests/test_tools_export_jsonl.py | 125 +++++++++++++++++++++++++++++++ tools/export_jsonl.py | 92 +++++++++++++++++++++++ 3 files changed, 221 insertions(+) create mode 100644 tests/test_tools_export_jsonl.py create mode 100644 tools/export_jsonl.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2398ea5c..8d7dabc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- NewsDOM JSON을 JSONL 형식으로 변환하는 `tools/export_jsonl.py` 도구 추가. + + > **Planned 0.3.0 deployment migration:** parser authentication changes from > **default-open** to **default-required**. Production must configure > `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, and diff --git a/tests/test_tools_export_jsonl.py b/tests/test_tools_export_jsonl.py new file mode 100644 index 00000000..34c7e0e2 --- /dev/null +++ b/tests/test_tools_export_jsonl.py @@ -0,0 +1,125 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tools.export_jsonl import export_jsonl, main + +VALID_JSON_DATA = { + "document_id": "test_doc", + "pages": [ + { + "page_number": 1, + "articles": [ + { + "article_id": "art_1", + "headline": "Test Headline 1", + "body_blocks": ["Block 1", "Block 2"], + }, + { + "article_id": "art_2", + "headline": "Test Headline 2", + "body_blocks": [], + }, + ], + }, + "not_a_dict_page", + { + "page_number": 2, + "articles": [ + "not_a_dict_article", + { + "article_id": "art_3", + "headline": "Test Headline 3", + "body_blocks": ["Block 3"], + }, + ], + }, + ], +} + + +def test_export_jsonl_success(tmp_path: Path) -> None: + input_file = tmp_path / "input.json" + input_file.write_text(json.dumps(VALID_JSON_DATA), encoding="utf-8") + output_file = tmp_path / "output.jsonl" + + export_jsonl(input_file, output_file) + + assert output_file.exists() + + with output_file.open("r", encoding="utf-8") as f: + lines = f.readlines() + + assert len(lines) == 4 + row0 = json.loads(lines[0]) + assert row0["document_id"] == "test_doc" + assert row0["page_number"] == 1 + assert row0["article_id"] == "art_1" + assert row0["headline"] == "Test Headline 1" + assert row0["body_block_index"] == 0 + assert row0["body_block_text"] == "Block 1" + + row1 = json.loads(lines[1]) + assert row1["body_block_index"] == 1 + assert row1["body_block_text"] == "Block 2" + + row2 = json.loads(lines[2]) + assert row2["headline"] == "Test Headline 2" + assert row2["body_block_index"] is None + assert row2["body_block_text"] == "" + + row3 = json.loads(lines[3]) + assert row3["page_number"] == 2 + assert row3["article_id"] == "art_3" + assert row3["body_block_index"] == 0 + assert row3["body_block_text"] == "Block 3" + + +def test_export_jsonl_invalid_file(tmp_path: Path) -> None: + output_file = tmp_path / "output.jsonl" + + non_existent = tmp_path / "not_exist.json" + with pytest.raises(FileNotFoundError, match="File not found"): + export_jsonl(non_existent, output_file) + + not_json = tmp_path / "input.txt" + not_json.write_text("plain text", encoding="utf-8") + with pytest.raises(ValueError, match="must be a .json file"): + export_jsonl(not_json, output_file) + + invalid_json = tmp_path / "invalid.json" + invalid_json.write_text("{invalid_json:", encoding="utf-8") + with pytest.raises(ValueError, match="Invalid JSON file"): + export_jsonl(invalid_json, output_file) + + +def test_export_jsonl_cli_success( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + input_file = tmp_path / "input.json" + input_file.write_text(json.dumps(VALID_JSON_DATA), encoding="utf-8") + output_file = tmp_path / "output.jsonl" + + main([str(input_file), str(output_file)]) + + assert output_file.exists() + captured = capsys.readouterr() + assert "JSONL successfully written" in captured.out + + +def test_export_jsonl_cli_invalid_file( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + not_json = tmp_path / "input.txt" + not_json.write_text("plain text", encoding="utf-8") + output_file = tmp_path / "output.jsonl" + + with pytest.raises(SystemExit) as exc_info: + main([str(not_json), str(output_file)]) + + assert exc_info.value.code == 1 + captured = capsys.readouterr() + assert "Error exporting JSONL:" in captured.err diff --git a/tools/export_jsonl.py b/tools/export_jsonl.py new file mode 100644 index 00000000..7ead6c39 --- /dev/null +++ b/tools/export_jsonl.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def export_jsonl(json_path: Path, output_path: Path) -> None: + """Export NewsDOM JSON to a JSONL file containing article metadata and body blocks.""" + if not json_path.is_file(): + raise FileNotFoundError(f"File not found or is not a file: {json_path}") + if json_path.suffix.lower() != ".json": + raise ValueError("Input file must be a .json file.") + + try: + data = json.loads(json_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON file: {exc}") from exc + + pages = data.get("pages", []) + + with output_path.open("w", encoding="utf-8") as jsonlfile: + document_id = data.get("document_id", "Unknown Document") + + for page in pages: + if not isinstance(page, dict): + continue + page_number = page.get("page_number", "Unknown") + + articles = page.get("articles", []) + for article in articles: + if not isinstance(article, dict): + continue + article_id = article.get("article_id", "Unknown Article ID") + headline = article.get("headline", "") + + body_blocks = article.get("body_blocks", []) + + if not body_blocks: + jsonlfile.write( + json.dumps( + { + "document_id": document_id, + "page_number": page_number, + "article_id": article_id, + "headline": headline, + "body_block_index": None, + "body_block_text": "", + }, + ensure_ascii=False, + ) + + "\n" + ) + + for idx, block in enumerate(body_blocks): + jsonlfile.write( + json.dumps( + { + "document_id": document_id, + "page_number": page_number, + "article_id": article_id, + "headline": headline, + "body_block_index": idx, + "body_block_text": block, + }, + ensure_ascii=False, + ) + + "\n" + ) + + +def main(argv: list[str] | None = None) -> None: + """Run the JSON-to-JSONL export CLI.""" + parser = argparse.ArgumentParser(description="Export a NewsDOM JSON file to JSONL.") + parser.add_argument("input", type=Path, help="Path to the input JSON file.") + parser.add_argument( + "output", type=Path, help="Path to write the JSONL output file." + ) + + args = parser.parse_args(argv) + + try: + export_jsonl(args.input, args.output) + print(f"JSONL successfully written to {args.output}") + except Exception as exc: + print(f"Error exporting JSONL: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": # pragma: no cover + main() From 22631eef6e918b3ec53031a6f412899e016a92ff Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 06:59:26 +0900 Subject: [PATCH 02/20] test(export): reject schema-invalid NewsDOM before writing JSONL --- tests/test_tools_export_jsonl.py | 38 ++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/tests/test_tools_export_jsonl.py b/tests/test_tools_export_jsonl.py index 34c7e0e2..4d8665f0 100644 --- a/tests/test_tools_export_jsonl.py +++ b/tests/test_tools_export_jsonl.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest +from pydantic import ValidationError from tools.export_jsonl import export_jsonl, main @@ -25,11 +26,9 @@ }, ], }, - "not_a_dict_page", { "page_number": 2, "articles": [ - "not_a_dict_article", { "article_id": "art_3", "headline": "Test Headline 3", @@ -78,6 +77,41 @@ def test_export_jsonl_success(tmp_path: Path) -> None: assert row3["body_block_text"] == "Block 3" +@pytest.mark.parametrize( + "invalid_pages", + [ + ["not_a_dict_page"], + [{"page_number": 1, "articles": ["not_a_dict_article"]}], + [ + { + "page_number": 1, + "articles": [ + { + "article_id": "art_1", + "headline": "Headline", + "body_blocks": "not-a-list", + } + ], + } + ], + ], +) +def test_export_jsonl_rejects_schema_invalid_dom_before_output( + tmp_path: Path, invalid_pages: list[object] +) -> None: + input_file = tmp_path / "input.json" + input_file.write_text( + json.dumps({"document_id": "test_doc", "pages": invalid_pages}), + encoding="utf-8", + ) + output_file = tmp_path / "output.jsonl" + + with pytest.raises(ValidationError): + export_jsonl(input_file, output_file) + + assert not output_file.exists() + + def test_export_jsonl_invalid_file(tmp_path: Path) -> None: output_file = tmp_path / "output.jsonl" From 165b0a0b1678fbb29732ccc15bc77bf9dd5c7e07 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 06:59:41 +0900 Subject: [PATCH 03/20] fix(export): validate JSONL input against canonical NewsDOM schema --- tools/export_jsonl.py | 49 +++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/tools/export_jsonl.py b/tools/export_jsonl.py index 7ead6c39..11178a0b 100644 --- a/tools/export_jsonl.py +++ b/tools/export_jsonl.py @@ -5,9 +5,16 @@ import sys from pathlib import Path +_REPO_ROOT = Path(__file__).resolve().parents[1] +_SRC_ROOT = _REPO_ROOT / "src" +if str(_SRC_ROOT) not in sys.path: + sys.path.insert(0, str(_SRC_ROOT)) + +from newsdom_api.schemas import ParseResponse # noqa: E402 + def export_jsonl(json_path: Path, output_path: Path) -> None: - """Export NewsDOM JSON to a JSONL file containing article metadata and body blocks.""" + """Export schema-valid NewsDOM JSON as article/body-block JSONL records.""" if not json_path.is_file(): raise FileNotFoundError(f"File not found or is not a file: {json_path}") if json_path.suffix.lower() != ".json": @@ -18,33 +25,19 @@ def export_jsonl(json_path: Path, output_path: Path) -> None: except json.JSONDecodeError as exc: raise ValueError(f"Invalid JSON file: {exc}") from exc - pages = data.get("pages", []) + document = ParseResponse.model_validate(data) with output_path.open("w", encoding="utf-8") as jsonlfile: - document_id = data.get("document_id", "Unknown Document") - - for page in pages: - if not isinstance(page, dict): - continue - page_number = page.get("page_number", "Unknown") - - articles = page.get("articles", []) - for article in articles: - if not isinstance(article, dict): - continue - article_id = article.get("article_id", "Unknown Article ID") - headline = article.get("headline", "") - - body_blocks = article.get("body_blocks", []) - - if not body_blocks: + for page in document.pages: + for article in page.articles: + if not article.body_blocks: jsonlfile.write( json.dumps( { - "document_id": document_id, - "page_number": page_number, - "article_id": article_id, - "headline": headline, + "document_id": document.document_id, + "page_number": page.page_number, + "article_id": article.article_id, + "headline": article.headline, "body_block_index": None, "body_block_text": "", }, @@ -53,14 +46,14 @@ def export_jsonl(json_path: Path, output_path: Path) -> None: + "\n" ) - for idx, block in enumerate(body_blocks): + for idx, block in enumerate(article.body_blocks): jsonlfile.write( json.dumps( { - "document_id": document_id, - "page_number": page_number, - "article_id": article_id, - "headline": headline, + "document_id": document.document_id, + "page_number": page.page_number, + "article_id": article.article_id, + "headline": article.headline, "body_block_index": idx, "body_block_text": block, }, From 34feb22e5d8bd1e578865b5728cded0b0bca5f6b Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:09:22 +0000 Subject: [PATCH 04/20] =?UTF-8?q?=EC=88=98=EC=A0=95:=20tools/export=5Fjson?= =?UTF-8?q?l.py=EC=97=90=EC=84=9C=20sys.path=20=ED=99=95=EC=9D=B8=EC=97=90?= =?UTF-8?q?=20pragma:=20no=20cover=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + tools/export_jsonl.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d7dabc6..2d236445 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] + ### Added - NewsDOM JSON을 JSONL 형식으로 변환하는 `tools/export_jsonl.py` 도구 추가. diff --git a/tools/export_jsonl.py b/tools/export_jsonl.py index 11178a0b..306356b5 100644 --- a/tools/export_jsonl.py +++ b/tools/export_jsonl.py @@ -7,7 +7,7 @@ _REPO_ROOT = Path(__file__).resolve().parents[1] _SRC_ROOT = _REPO_ROOT / "src" -if str(_SRC_ROOT) not in sys.path: +if str(_SRC_ROOT) not in sys.path: # pragma: no cover sys.path.insert(0, str(_SRC_ROOT)) from newsdom_api.schemas import ParseResponse # noqa: E402 From e2c095b67767af7379e7e3a1851441cb93bafc48 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Sun, 6 Sep 2026 22:52:23 +0000 Subject: [PATCH 05/20] =?UTF-8?q?=EC=88=98=EC=A0=95:=20=EC=B7=A8=EC=95=BD?= =?UTF-8?q?=EC=A0=90=20=ED=95=B4=EA=B2=B0=EC=9D=84=20=EC=9C=84=ED=95=B4=20?= =?UTF-8?q?pypdf=20=EC=97=85=EB=8D=B0=EC=9D=B4=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 ++++ uv.lock | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d236445..b891f6ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- 알려진 취약점을 해결하기 위해 패키지 업데이트 + + ### Added - NewsDOM JSON을 JSONL 형식으로 변환하는 `tools/export_jsonl.py` 도구 추가. diff --git a/uv.lock b/uv.lock index a0d133b8..1279f58d 100644 --- a/uv.lock +++ b/uv.lock @@ -303,7 +303,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -929,14 +929,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.15.0" +version = "6.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/17/ee75a92718ec7212de831e71454d702225aa5e474a805cce169806044453/pypdf-6.15.0.tar.gz", hash = "sha256:d39c4d955a76409284a905e2d65b40076d77ab76129e0faaeeb6612403ecfc79", size = 6993794, upload-time = "2026-08-06T13:06:49.929Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/dc/34857a5e31cf708c163929f61a9ba4bd357a8850e49fc4e846ced527b51f/pypdf-6.17.0.tar.gz", hash = "sha256:097ad0d829778ec5b615aeaa5c6da4b6cac4992f8fd80b56f98a1a8c006573bb", size = 7018352, upload-time = "2026-09-04T11:30:44.256Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/72/ce3067ac31e214a66388159f8462ddb8c13dd00170f24d555a1f1ae8ee91/pypdf-6.15.0-py3-none-any.whl", hash = "sha256:14e001d6504822cb1ca9c7ed9a69bccb320f59b320730f55af804361abe4d5ee", size = 378123, upload-time = "2026-08-06T13:06:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/c1/08/1e9731038124a9127e1d27848952b86fb32b2f45f8f1b94adc7f0817a6ac/pypdf-6.17.0-py3-none-any.whl", hash = "sha256:5bd827266a21553b74d910e350131a6227b72f2ab4209bf372814b8195fa11c5", size = 388051, upload-time = "2026-09-04T11:30:42.681Z" }, ] [[package]] From 821c2dc6e798bbbca1c61f20d8fa6ac80e101643 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Sun, 6 Sep 2026 23:47:19 +0000 Subject: [PATCH 06/20] =?UTF-8?q?=EC=88=98=EC=A0=95:=20=EC=B7=A8=EC=95=BD?= =?UTF-8?q?=EC=A0=90=20=ED=95=B4=EA=B2=B0=EC=9D=84=20=EC=9C=84=ED=95=B4=20?= =?UTF-8?q?pypdf=20=EC=97=85=EB=8D=B0=EC=9D=B4=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From fa0dbf3501c11d300665f14bc5c0733aed85b511 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Mon, 7 Sep 2026 01:18:19 +0000 Subject: [PATCH 07/20] =?UTF-8?q?=EC=88=98=EC=A0=95:=20=EC=B7=A8=EC=95=BD?= =?UTF-8?q?=EC=A0=90=20=ED=95=B4=EA=B2=B0=EC=9D=84=20=EC=9C=84=ED=95=B4=20?= =?UTF-8?q?pypdf=20=EC=97=85=EB=8D=B0=EC=9D=B4=ED=8A=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From 61727db7fdef9ecc4fe2d041cba458a31731290f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 18:12:54 +0900 Subject: [PATCH 08/20] repair(export): separate coverage and dependency churn --- CHANGELOG.md | 5 ----- tools/export_jsonl.py | 2 +- uv.lock | 8 ++++---- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b891f6ea..8d7dabc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -### Fixed -- 알려진 취약점을 해결하기 위해 패키지 업데이트 - - - ### Added - NewsDOM JSON을 JSONL 형식으로 변환하는 `tools/export_jsonl.py` 도구 추가. diff --git a/tools/export_jsonl.py b/tools/export_jsonl.py index 306356b5..11178a0b 100644 --- a/tools/export_jsonl.py +++ b/tools/export_jsonl.py @@ -7,7 +7,7 @@ _REPO_ROOT = Path(__file__).resolve().parents[1] _SRC_ROOT = _REPO_ROOT / "src" -if str(_SRC_ROOT) not in sys.path: # pragma: no cover +if str(_SRC_ROOT) not in sys.path: sys.path.insert(0, str(_SRC_ROOT)) from newsdom_api.schemas import ParseResponse # noqa: E402 diff --git a/uv.lock b/uv.lock index 1279f58d..a0d133b8 100644 --- a/uv.lock +++ b/uv.lock @@ -303,7 +303,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -929,14 +929,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.17.0" +version = "6.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5d/dc/34857a5e31cf708c163929f61a9ba4bd357a8850e49fc4e846ced527b51f/pypdf-6.17.0.tar.gz", hash = "sha256:097ad0d829778ec5b615aeaa5c6da4b6cac4992f8fd80b56f98a1a8c006573bb", size = 7018352, upload-time = "2026-09-04T11:30:44.256Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/17/ee75a92718ec7212de831e71454d702225aa5e474a805cce169806044453/pypdf-6.15.0.tar.gz", hash = "sha256:d39c4d955a76409284a905e2d65b40076d77ab76129e0faaeeb6612403ecfc79", size = 6993794, upload-time = "2026-08-06T13:06:49.929Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/08/1e9731038124a9127e1d27848952b86fb32b2f45f8f1b94adc7f0817a6ac/pypdf-6.17.0-py3-none-any.whl", hash = "sha256:5bd827266a21553b74d910e350131a6227b72f2ab4209bf372814b8195fa11c5", size = 388051, upload-time = "2026-09-04T11:30:42.681Z" }, + { url = "https://files.pythonhosted.org/packages/af/72/ce3067ac31e214a66388159f8462ddb8c13dd00170f24d555a1f1ae8ee91/pypdf-6.15.0-py3-none-any.whl", hash = "sha256:14e001d6504822cb1ca9c7ed9a69bccb320f59b320730f55af804361abe4d5ee", size = 378123, upload-time = "2026-08-06T13:06:47.709Z" }, ] [[package]] From c411b095c2302d22e320a46b67bacac7f0e99006 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:34:06 +0900 Subject: [PATCH 09/20] test(export): preserve published JSONL on write failure --- tests/test_tools_export_jsonl.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/test_tools_export_jsonl.py b/tests/test_tools_export_jsonl.py index 4d8665f0..34898d24 100644 --- a/tests/test_tools_export_jsonl.py +++ b/tests/test_tools_export_jsonl.py @@ -112,6 +112,46 @@ def test_export_jsonl_rejects_schema_invalid_dom_before_output( assert not output_file.exists() +def test_export_jsonl_preserves_published_output_when_encoding_fails( + tmp_path: Path, +) -> None: + input_file = tmp_path / "input.json" + input_file.write_text( + json.dumps( + { + "document_id": "test_doc", + "pages": [ + { + "page_number": 1, + "articles": [ + { + "article_id": "good", + "headline": "Good", + "body_blocks": ["first row"], + }, + { + "article_id": "bad", + "headline": "Bad", + "body_blocks": ["\ud800"], + }, + ], + } + ], + } + ), + encoding="utf-8", + ) + output_file = tmp_path / "output.jsonl" + published = '{"existing": true}\n' + output_file.write_text(published, encoding="utf-8") + + with pytest.raises(UnicodeEncodeError): + export_jsonl(input_file, output_file) + + assert output_file.read_text(encoding="utf-8") == published + assert {path.name for path in tmp_path.iterdir()} == {"input.json", "output.jsonl"} + + def test_export_jsonl_invalid_file(tmp_path: Path) -> None: output_file = tmp_path / "output.jsonl" From 7c5a181bb37b1b10d49818a5c9381abedf3c746a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:34:33 +0900 Subject: [PATCH 10/20] fix(export): publish JSONL atomically --- tools/export_jsonl.py | 83 ++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/tools/export_jsonl.py b/tools/export_jsonl.py index 11178a0b..057c9040 100644 --- a/tools/export_jsonl.py +++ b/tools/export_jsonl.py @@ -4,6 +4,7 @@ import json import sys from pathlib import Path +from tempfile import NamedTemporaryFile _REPO_ROOT = Path(__file__).resolve().parents[1] _SRC_ROOT = _REPO_ROOT / "src" @@ -14,7 +15,7 @@ def export_jsonl(json_path: Path, output_path: Path) -> None: - """Export schema-valid NewsDOM JSON as article/body-block JSONL records.""" + """Export schema-valid NewsDOM JSON and atomically publish the JSONL file.""" if not json_path.is_file(): raise FileNotFoundError(f"File not found or is not a file: {json_path}") if json_path.suffix.lower() != ".json": @@ -27,40 +28,56 @@ def export_jsonl(json_path: Path, output_path: Path) -> None: document = ParseResponse.model_validate(data) - with output_path.open("w", encoding="utf-8") as jsonlfile: - for page in document.pages: - for article in page.articles: - if not article.body_blocks: - jsonlfile.write( - json.dumps( - { - "document_id": document.document_id, - "page_number": page.page_number, - "article_id": article.article_id, - "headline": article.headline, - "body_block_index": None, - "body_block_text": "", - }, - ensure_ascii=False, + temporary_path: Path | None = None + try: + with NamedTemporaryFile( + "w", + encoding="utf-8", + dir=output_path.parent, + prefix=f".{output_path.name}.", + suffix=".tmp", + delete=False, + ) as jsonlfile: + temporary_path = Path(jsonlfile.name) + for page in document.pages: + for article in page.articles: + if not article.body_blocks: + jsonlfile.write( + json.dumps( + { + "document_id": document.document_id, + "page_number": page.page_number, + "article_id": article.article_id, + "headline": article.headline, + "body_block_index": None, + "body_block_text": "", + }, + ensure_ascii=False, + ) + + "\n" ) - + "\n" - ) - - for idx, block in enumerate(article.body_blocks): - jsonlfile.write( - json.dumps( - { - "document_id": document.document_id, - "page_number": page.page_number, - "article_id": article.article_id, - "headline": article.headline, - "body_block_index": idx, - "body_block_text": block, - }, - ensure_ascii=False, + + for idx, block in enumerate(article.body_blocks): + jsonlfile.write( + json.dumps( + { + "document_id": document.document_id, + "page_number": page.page_number, + "article_id": article.article_id, + "headline": article.headline, + "body_block_index": idx, + "body_block_text": block, + }, + ensure_ascii=False, + ) + + "\n" ) - + "\n" - ) + + temporary_path.replace(output_path) + except Exception: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + raise def main(argv: list[str] | None = None) -> None: From 7d232c9db50b23df589dfef0a3bf3515e63e151b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:34:57 +0900 Subject: [PATCH 11/20] docs(changelog): record atomic JSONL publication --- CHANGELOG.md | 90 +--------------------------------------------------- 1 file changed, 1 insertion(+), 89 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d7dabc6..aa2a0f38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- NewsDOM JSON을 JSONL 형식으로 변환하는 `tools/export_jsonl.py` 도구 추가. +- NewsDOM JSON을 schema validation 후 article/body-block 단위 JSONL로 변환하는 `tools/export_jsonl.py` 도구를 추가했습니다. 출력은 같은 디렉터리의 임시 파일에 완전히 기록된 뒤 교체되므로 encoding/write 실패 시 기존 published JSONL을 보존하고 partial 결과를 완료 산출물로 노출하지 않습니다. > **Planned 0.3.0 deployment migration:** parser authentication changes from @@ -28,91 +28,3 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [CLI] NewsDOM JSON 파일의 모든 텍스트 내용을 마스킹하여 익명화하는 `tools/anonymize_dom.py` 도구를 추가했습니다. - `/parse`에 기본 필수 bearer 인증 경계를 추가했습니다. `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, `NEWSDOM_API_TOKEN`을 명시해야 하며, 인증 비활성화는 격리된 development 프로필에서만 허용됩니다. `/health`는 liveness 전용으로 미인증 상태를 유지하고 `/ready`가 인증 설정과 MinerU 가용성을 함께 검증합니다. - 서브모듈/사이드카 배포용 `docker-compose.yml`: 필수 production 인증과 secret 주입을 요구하고 healthcheck가 `/ready`를 대상으로 하도록 구성했습니다. Kubernetes 예시는 Restricted Pod Security 설정, 비루트 실행, 권한 상승 금지, 모든 capability 제거, 읽기 전용 root filesystem, 제한된 임시 볼륨, liveness/readiness 분리를 적용합니다. -- [CLI] 파싱된 NewsDOM JSON에서 순수 텍스트 데이터를 추출하여 텍스트 파일 또는 stdout으로 출력하는 `tools/extract_text.py` 도구를 추가했습니다. - -### Security -- `/parse` authentication is now immutable per application instance and fails closed before multipart body parsing when required configuration is missing. Hostile missing, invalid, Unicode, oversized, and duplicated Authorization headers return one non-sensitive response. -- Added unauthenticated `/ready` traffic readiness that combines authentication configuration with MinerU executable availability while `/health` remains liveness-only. -- Hardened the Kubernetes deployment example with a restricted namespace policy, explicit non-root UID/GID, `RuntimeDefault` seccomp, disabled privilege escalation, dropped Linux capabilities, a read-only root filesystem, and bounded writable runtime volumes. -- 전역 500 에러 응답에도 표준 보안 헤더를 적용하여 예외 경로에서 header 누락을 방지 -- MinerU subprocess argv 생성 시 `-`로 시작하는 option-like 인자를 거부하여 argument injection 위험을 낮춤 -- API 에러 응답 생성 시 내부 예외 체인을 억제하여 의존성 오류나 내부 경로가 노출될 가능성을 줄임 -- API 응답 미들웨어에 `Cache-Control: no-store, max-age=0` 헤더를 추가하여 민감한 파싱 데이터의 브라우저 및 중간 캐싱을 방지 -- `uv.lock`의 의존성을 재잠금하여 실제 `pip-audit`/`trivy-fs` CVE를 제거: 런타임 경로의 `pillow` 12.2.0→12.3.0 (PYSEC-2026-3451/3452/3453/3454/3493/3494/3495/3496, 이미지 파서 취약점 8건), `pypdf>=6.15.0,<7.0` (lock 6.15.0; CVE-2026-59935/59936/59937/59938/71852/71870, PDF 파싱 경로), `click` 8.3.2→8.4.2 (PYSEC-2026-2132) — 모두 스캔 PDF/이미지 파싱 런타임에 직접 관련되며 선언 범위와 lock을 함께 고정함. 빌드 도구 `setuptools` 81.0.0→83.0.0 (CVE-2026-59890). 문서 툴체인의 `pymdown-extensions` 10.21.3→11.0.1 (CVE-2026-61632, MEDIUM)은 `mkdocs-material` 9.6.x의 `pymdown-extensions~=10.2`(`<11`) 상한 때문에 막혀 있었으므로, docs extra 핀을 `mkdocs-material>=9.7,<9.8`로 올려(9.7.x는 상한을 `>=10.2`로 완화) 해소함. `uv run mkdocs build --strict` 통과 확인. 조치 후 전체 잠금(런타임+extras) `pip-audit`: 취약점 0건. - -### Performance -- `newsdom_api.dom_builder._html_safe_text` 함수에 early return과 타입 체크를 도입하여 불필요한 `str()` 캐스팅을 제거함으로써 처리 속도를 개선했습니다. - -### Added -- [CLI] 여러 개의 분할된 NewsDOM JSON 파일을 하나의 문서로 병합하는 `tools/merge_dom.py` 도구를 추가했습니다. -- [CLI] 파싱된 NewsDOM JSON 데이터를 CSV 형식으로 추출하는 `tools/export_csv.py` 도구를 추가했습니다. -- [CLI] 파싱된 NewsDOM JSON을 HTML 포맷으로 변환하여 웹 브라우저에서 보기 쉽게 만들어주는 `tools/export_html.py` 도구를 추가했습니다. -- [CLI] 파싱된 NewsDOM JSON이 Pydantic 스키마(`ParseResponse`)와 일치하는지 엄격하게 검증하는 `tools/validate_dom.py` 도구 추가 -- [CLI] 파싱된 NewsDOM JSON의 기사 제목(headline)과 본문(body_blocks)에서 텍스트를 검색하여 위치를 반환하는 `tools/search_dom.py` 도구 추가 -- [CLI] 파싱된 NewsDOM JSON을 Markdown 포맷으로 변환하는 `tools/export_markdown.py` 도구를 추가했습니다. -- [CLI] `tools/batch_parse_pdf.py`에 하위 디렉터리의 PDF를 일괄 처리하고 상대 경로로 JSON을 저장하는 `--recursive` 옵션을 추가했습니다. -- OpenAPI 문서에 contact 및 MIT license metadata를 추가하여 API 소비자가 maintainer와 라이선스 정보를 더 쉽게 확인할 수 있도록 개선 -- 여러 PDF를 일괄 파싱해 JSON 결과를 저장하는 `tools/batch_parse_pdf.py` 도구 추가 -- 파싱된 NewsDOM JSON의 페이지, 기사, 본문 블록, 이미지 수를 집계하는 `tools/analyze_dom.py` 도구 추가 -- [CLI] PDF 파일을 파싱하여 DOM 구조를 JSON으로 추출하는 `tools/parse_pdf.py` 도구 추가 -- [CLI] 합성 신문 PDF와 정답 데이터를 대량으로 생성하는 `tools/generate_synthetic.py` 도구 추가 -- `tools/benchmark_ocr.py`에 `--recursive` 인자를 추가하여 하위 디렉토리의 PDF 파일도 재귀적으로 탐색할 수 있도록 기능 보강. -- `tools/benchmark_ocr.py`에 `--format` 인자를 추가하여 벤치마크 결과를 `json` 및 `csv` 포맷으로 내보낼 수 있는 기능 추가. -- `tools/derive_private_baseline.py`에 `--recursive` 인자를 추가하여 하위 디렉토리의 PDF 파일 재귀 탐색 기능 추가. -- `tools/derive_private_baseline.py`에 `--strict` / `--no-strict` 인자를 추가하여 일부 PDF 파일 파싱 실패 시 진행을 계속할 수 있는 장애 허용성 옵션 추가. -- 관련된 코드의 단위 테스트 작성 및 코드 커버리지 100% 달성. -- `tools` 패키지에 대한 단위 테스트 커버리지를 100%로 향상 - - `tests/test_benchmark_ocr.py`에 빈 디렉토리, 알 수 없는 엔진 지정, mocking된 엔진 동작 등 새로운 테스트 케이스 추가 - - `tests/test_derive_private_baseline.py`에 `FileNotFoundError`, `HTTPException` 상황 및 `main()` 실행 경로 전체에 대한 테스트 추가 -- `tools/benchmark_ocr.py` 및 `tools/derive_private_baseline.py`의 `if __name__ == "__main__":` 구문에 커버리지 측정 예외 마커(`pragma: no cover`) 추가 - -### Changed - -- Improved OpenAPI metadata for the `/parse` endpoint by documenting 415, 502, and 503 error responses. - -## [0.2.0] - 2026-04-24 - -### Added - -- Added `benchmark_ocr.py` tool to measure OCR engine performance and structural accuracy on private datasets. -- Deployed a GHCR prebuilt CI container image (`ghcr.io/seongho-bae/newsdom-api/ci-env`) to stabilize test environments and resolve timeout/dependency installation issues. - -### Changed - -- Updated `dom_builder.py` to preserve multi-page MinerU structure instead of collapsing multi-page outputs into a single page. -- Adjusted the `/parse` endpoint to return specific HTTP error codes (`502` and `503`) mapped to `MineruIncompleteOutputError` and `MineruRuntimeUnavailableError` rather than raw `500` errors. - -### Fixed - -- Mitigated infinite hang issues when processing specific PDFs by enforcing a strict timeout (300 seconds) in the `mineru` subprocess runner. -- Resolved permission (`EACCES`) issues in GitHub Actions by running tests locally instead of inside a non-root container context for GitHub's restricted runner environment. - - -## [0.1.1] - 2026-04-11 - -### Added - -- GHCR-ready multi-arch API image delivery, ClusterFuzzLite coverage, and exported `*.intoto.jsonl` provenance bundles for stable releases -- Verified `/docs` and `/redoc` manual screenshots plus canonical engineering policy docs that describe the live repository workflow - -### Changed - -- Protected-branch governance documentation now reflects the current single-maintainer exception while preserving required checks and history protections -- Public setup guidance, docs-toolchain policy, and markdownlint scope now match the merged `develop` / `main` delivery paths - -### Fixed - -- Patched `pypdf` lockfile coverage to `6.10.0` for GHSA-3crg-w4f6-42mx / CVE-2026-40260 - -## [0.1.0] - 2026-04-09 - -### Added - -- MinerU-backed DOM parsing API for scanned Japanese newspaper PDFs -- Synthetic newspaper fixture generation and structural equivalence checks -- Protected-branch CI, security gates, release provenance workflow, and Git Flow documentation - -[Unreleased]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.2.0...HEAD -[0.2.0]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.1.1...v0.2.0 -[0.1.1]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.1.0...v0.1.1 -[0.1.0]: https://github.com/Seongho-Bae/newsdom-api/releases/tag/v0.1.0 From d1424fdd05b6fc8a7acffbfe9593f67971e2b376 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:37:13 +0900 Subject: [PATCH 12/20] test(export): cover temp creation failure cleanup boundary --- tests/test_tools_export_jsonl.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_tools_export_jsonl.py b/tests/test_tools_export_jsonl.py index 34898d24..dbc6ea9f 100644 --- a/tests/test_tools_export_jsonl.py +++ b/tests/test_tools_export_jsonl.py @@ -152,6 +152,21 @@ def test_export_jsonl_preserves_published_output_when_encoding_fails( assert {path.name for path in tmp_path.iterdir()} == {"input.json", "output.jsonl"} +def test_export_jsonl_missing_output_parent_leaves_no_temporary_artifact( + tmp_path: Path, +) -> None: + input_file = tmp_path / "input.json" + input_file.write_text(json.dumps(VALID_JSON_DATA), encoding="utf-8") + missing_parent = tmp_path / "missing" + output_file = missing_parent / "output.jsonl" + + with pytest.raises(FileNotFoundError): + export_jsonl(input_file, output_file) + + assert not missing_parent.exists() + assert {path.name for path in tmp_path.iterdir()} == {"input.json"} + + def test_export_jsonl_invalid_file(tmp_path: Path) -> None: output_file = tmp_path / "output.jsonl" From 28d63d699abb2d777f988e56fac5d32486dc9f97 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:39:20 +0900 Subject: [PATCH 13/20] fix(changelog): restore release and security history --- CHANGELOG.md | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa2a0f38..7ac33ba5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,3 +28,91 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [CLI] NewsDOM JSON 파일의 모든 텍스트 내용을 마스킹하여 익명화하는 `tools/anonymize_dom.py` 도구를 추가했습니다. - `/parse`에 기본 필수 bearer 인증 경계를 추가했습니다. `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, `NEWSDOM_API_TOKEN`을 명시해야 하며, 인증 비활성화는 격리된 development 프로필에서만 허용됩니다. `/health`는 liveness 전용으로 미인증 상태를 유지하고 `/ready`가 인증 설정과 MinerU 가용성을 함께 검증합니다. - 서브모듈/사이드카 배포용 `docker-compose.yml`: 필수 production 인증과 secret 주입을 요구하고 healthcheck가 `/ready`를 대상으로 하도록 구성했습니다. Kubernetes 예시는 Restricted Pod Security 설정, 비루트 실행, 권한 상승 금지, 모든 capability 제거, 읽기 전용 root filesystem, 제한된 임시 볼륨, liveness/readiness 분리를 적용합니다. +- [CLI] 파싱된 NewsDOM JSON에서 순수 텍스트 데이터를 추출하여 텍스트 파일 또는 stdout으로 출력하는 `tools/extract_text.py` 도구를 추가했습니다. + +### Security +- `/parse` authentication is now immutable per application instance and fails closed before multipart body parsing when required configuration is missing. Hostile missing, invalid, Unicode, oversized, and duplicated Authorization headers return one non-sensitive response. +- Added unauthenticated `/ready` traffic readiness that combines authentication configuration with MinerU executable availability while `/health` remains liveness-only. +- Hardened the Kubernetes deployment example with a restricted namespace policy, explicit non-root UID/GID, `RuntimeDefault` seccomp, disabled privilege escalation, dropped Linux capabilities, a read-only root filesystem, and bounded writable runtime volumes. +- 전역 500 에러 응답에도 표준 보안 헤더를 적용하여 예외 경로에서 header 누락을 방지 +- MinerU subprocess argv 생성 시 `-`로 시작하는 option-like 인자를 거부하여 argument injection 위험을 낮춤 +- API 에러 응답 생성 시 내부 예외 체인을 억제하여 의존성 오류나 내부 경로가 노출될 가능성을 줄임 +- API 응답 미들웨어에 `Cache-Control: no-store, max-age=0` 헤더를 추가하여 민감한 파싱 데이터의 브라우저 및 중간 캐싱을 방지 +- `uv.lock`의 의존성을 재잠금하여 실제 `pip-audit`/`trivy-fs` CVE를 제거: 런타임 경로의 `pillow` 12.2.0→12.3.0 (PYSEC-2026-3451/3452/3453/3454/3493/3494/3495/3496, 이미지 파서 취약점 8건), `pypdf>=6.15.0,<7.0` (lock 6.15.0; CVE-2026-59935/59936/59937/59938/71852/71870, PDF 파싱 경로), `click` 8.3.2→8.4.2 (PYSEC-2026-2132) — 모두 스캔 PDF/이미지 파싱 런타임에 직접 관련되며 선언 범위와 lock을 함께 고정함. 빌드 도구 `setuptools` 81.0.0→83.0.0 (CVE-2026-59890). 문서 툴체인의 `pymdown-extensions` 10.21.3→11.0.1 (CVE-2026-61632, MEDIUM)은 `mkdocs-material` 9.6.x의 `pymdown-extensions~=10.2`(`<11`) 상한 때문에 막혀 있었으므로, docs extra 핀을 `mkdocs-material>=9.7,<9.8`로 올려(9.7.x는 상한을 `>=10.2`로 완화) 해소함. `uv run mkdocs build --strict` 통과 확인. 조치 후 전체 잠금(런타임+extras) `pip-audit`: 취약점 0건. + +### Performance +- `newsdom_api.dom_builder._html_safe_text` 함수에 early return과 타입 체크를 도입하여 불필요한 `str()` 캐스팅을 제거함으로써 처리 속도를 개선했습니다. + +### Added +- [CLI] 여러 개의 분할된 NewsDOM JSON 파일을 하나의 문서로 병합하는 `tools/merge_dom.py` 도구를 추가했습니다. +- [CLI] 파싱된 NewsDOM JSON 데이터를 CSV 형식으로 추출하는 `tools/export_csv.py` 도구를 추가했습니다. +- [CLI] 파싱된 NewsDOM JSON을 HTML 포맷으로 변환하여 웹 브라우저에서 보기 쉽게 만들어주는 `tools/export_html.py` 도구를 추가했습니다. +- [CLI] 파싱된 NewsDOM JSON이 Pydantic 스키마(`ParseResponse`)와 일치하는지 엄격하게 검증하는 `tools/validate_dom.py` 도구 추가 +- [CLI] 파싱된 NewsDOM JSON의 기사 제목(headline)과 본문(body_blocks)에서 텍스트를 검색하여 위치를 반환하는 `tools/search_dom.py` 도구 추가 +- [CLI] 파싱된 NewsDOM JSON을 Markdown 포맷으로 변환하는 `tools/export_markdown.py` 도구를 추가했습니다. +- [CLI] `tools/batch_parse_pdf.py`에 하위 디렉터리의 PDF를 일괄 처리하고 상대 경로로 JSON을 저장하는 `--recursive` 옵션을 추가했습니다. +- OpenAPI 문서에 contact 및 MIT license metadata를 추가하여 API 소비자가 maintainer와 라이선스 정보를 더 쉽게 확인할 수 있도록 개선 +- 여러 PDF를 일괄 파싱해 JSON 결과를 저장하는 `tools/batch_parse_pdf.py` 도구 추가 +- 파싱된 NewsDOM JSON의 페이지, 기사, 본문 블록, 이미지 수를 집계하는 `tools/analyze_dom.py` 도구 추가 +- [CLI] PDF 파일을 파싱하여 DOM 구조를 JSON으로 추출하는 `tools/parse_pdf.py` 도구 추가 +- [CLI] 합성 신문 PDF와 정답 데이터를 대량으로 생성하는 `tools/generate_synthetic.py` 도구 추가 +- `tools/benchmark_ocr.py`에 `--recursive` 인자를 추가하여 하위 디렉토리의 PDF 파일도 재귀적으로 탐색할 수 있도록 기능 보강. +- `tools/benchmark_ocr.py`에 `--format` 인자를 추가하여 벤치마크 결과를 `json` 및 `csv` 포맷으로 내보낼 수 있는 기능 추가. +- `tools/derive_private_baseline.py`에 `--recursive` 인자를 추가하여 하위 디렉토리의 PDF 파일 재귀 탐색 기능 추가. +- `tools/derive_private_baseline.py`에 `--strict` / `--no-strict` 인자를 추가하여 일부 PDF 파일 파싱 실패 시 진행을 계속할 수 있는 장애 허용성 옵션 추가. +- 관련된 코드의 단위 테스트 작성 및 코드 커버리지 100% 달성. +- `tools` 패키지에 대한 단위 테스트 커버리지를 100%로 향상 + - `tests/test_benchmark_ocr.py`에 빈 디렉토리, 알 수 없는 엔진 지정, mocking된 엔진 동작 등 새로운 테스트 케이스 추가 + - `tests/test_derive_private_baseline.py`에 `FileNotFoundError`, `HTTPException` 상황 및 `main()` 실행 경로 전체에 대한 테스트 추가 +- `tools/benchmark_ocr.py` 및 `tools/derive_private_baseline.py`의 `if __name__ == "__main__":` 구문에 커버리지 측정 예외 마커(`pragma: no cover`) 추가 + +### Changed + +- Improved OpenAPI metadata for the `/parse` endpoint by documenting 415, 502, and 503 error responses. + +## [0.2.0] - 2026-04-24 + +### Added + +- Added `benchmark_ocr.py` tool to measure OCR engine performance and structural accuracy on private datasets. +- Deployed a GHCR prebuilt CI container image (`ghcr.io/seongho-bae/newsdom-api/ci-env`) to stabilize test environments and resolve timeout/dependency installation issues. + +### Changed + +- Updated `dom_builder.py` to preserve multi-page MinerU structure instead of collapsing multi-page outputs into a single page. +- Adjusted the `/parse` endpoint to return specific HTTP error codes (`502` and `503`) mapped to `MineruIncompleteOutputError` and `MineruRuntimeUnavailableError` rather than raw `500` errors. + +### Fixed + +- Mitigated infinite hang issues when processing specific PDFs by enforcing a strict timeout (300 seconds) in the `mineru` subprocess runner. +- Resolved permission (`EACCES`) issues in GitHub Actions by running tests locally instead of inside a non-root container context for GitHub's restricted runner environment. + + +## [0.1.1] - 2026-04-11 + +### Added + +- GHCR-ready multi-arch API image delivery, ClusterFuzzLite coverage, and exported `*.intoto.jsonl` provenance bundles for stable releases +- Verified `/docs` and `/redoc` manual screenshots plus canonical engineering policy docs that describe the live repository workflow + +### Changed + +- Protected-branch governance documentation now reflects the current single-maintainer exception while preserving required checks and history protections +- Public setup guidance, docs-toolchain policy, and markdownlint scope now match the merged `develop` / `main` delivery paths + +### Fixed + +- Patched `pypdf` lockfile coverage to `6.10.0` for GHSA-3crg-w4f6-42mx / CVE-2026-40260 + +## [0.1.0] - 2026-04-09 + +### Added + +- MinerU-backed DOM parsing API for scanned Japanese newspaper PDFs +- Synthetic newspaper fixture generation and structural equivalence checks +- Protected-branch CI, security gates, release provenance workflow, and Git Flow documentation + +[Unreleased]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.1.1...v0.2.0 +[0.1.1]: https://github.com/Seongho-Bae/newsdom-api/compare/v0.1.0...v0.1.1 +[0.1.0]: https://github.com/Seongho-Bae/newsdom-api/releases/tag/v0.1.0 From 5f4de2481aa880f349c982961e60a291db99c017 Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Mon, 7 Sep 2026 10:52:31 +0000 Subject: [PATCH 14/20] =?UTF-8?q?chore:=20=EB=88=84=EB=9D=BD=EB=90=9C=20CH?= =?UTF-8?q?ANGELOG=20=ED=95=AD=EB=AA=A9=20=EB=B3=B5=EA=B5=AC=20=EB=B0=8F?= =?UTF-8?q?=20=EC=BB=A4=EB=B2=84=EB=A6=AC=EC=A7=80=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/export_jsonl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/export_jsonl.py b/tools/export_jsonl.py index 057c9040..29b53eaf 100644 --- a/tools/export_jsonl.py +++ b/tools/export_jsonl.py @@ -8,7 +8,7 @@ _REPO_ROOT = Path(__file__).resolve().parents[1] _SRC_ROOT = _REPO_ROOT / "src" -if str(_SRC_ROOT) not in sys.path: +if str(_SRC_ROOT) not in sys.path: # pragma: no cover sys.path.insert(0, str(_SRC_ROOT)) from newsdom_api.schemas import ParseResponse # noqa: E402 From 4a590b4b799c3ab08fdc5bec823468e37fbd4700 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:59:09 +0900 Subject: [PATCH 15/20] test(ci): keep project coverage sources authoritative --- tests/test_ci_coverage_contract.py | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/test_ci_coverage_contract.py diff --git a/tests/test_ci_coverage_contract.py b/tests/test_ci_coverage_contract.py new file mode 100644 index 00000000..1f326bbb --- /dev/null +++ b/tests/test_ci_coverage_contract.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +import re +import shlex +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] + + +def _coverage_source_paths() -> set[str]: + pyproject = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + match = re.search( + r"(?ms)^\[tool\.coverage\.run\]\s*$\n(?P
.*?)(?=^\[|\Z)", + pyproject, + ) + assert match is not None, "pyproject.toml must declare [tool.coverage.run]" + + source_match = re.search(r"(?m)^source\s*=\s*\[(?P[^]]+)\]", match["section"]) + assert source_match is not None, "coverage run config must declare source paths" + return set(re.findall(r'"([^"]+)"', source_match["paths"])) + + +def test_ci_coverage_uses_project_sources_without_narrowing_them() -> None: + """CI must measure every production source declared by coverage configuration.""" + declared_sources = _coverage_source_paths() + assert {"src/newsdom_api", "tools"}.issubset(declared_sources) + + workflow = yaml.safe_load( + (ROOT / ".github/workflows/tests.yml").read_text(encoding="utf-8") + ) + pytest_job = workflow["jobs"]["pytest"] + coverage_step = next( + step for step in pytest_job["steps"] if step.get("name") == "Run tests with coverage" + ) + command = coverage_step["run"] + tokens = shlex.split(command) + + assert "--cov" in tokens, ( + "pytest-cov must use bare --cov so [tool.coverage.run].source remains authoritative; " + "--cov= overrides the configured source set" + ) + assert not any(token.startswith("--cov=") for token in tokens) From a33a22e1284423203dccdb675b0d99183825cc95 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 19:59:31 +0900 Subject: [PATCH 16/20] fix(ci): measure all declared coverage sources --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 88574c0f..34731119 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,4 +35,4 @@ jobs: env: PYTHONWARNINGS: error PYTHONPATH: src - run: uv run pytest --cov=src/newsdom_api --cov-branch --cov-report=term-missing --cov-fail-under=100 \ No newline at end of file + run: uv run pytest --cov --cov-branch --cov-report=term-missing --cov-fail-under=100 \ No newline at end of file From 2ade2f762a192fdc4161fa007a034e8ca2c47d4d Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 21:41:59 +0900 Subject: [PATCH 17/20] test(export): prove interrupt cleanup boundary --- ...st_tools_export_jsonl_interrupt_cleanup.py | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/test_tools_export_jsonl_interrupt_cleanup.py diff --git a/tests/test_tools_export_jsonl_interrupt_cleanup.py b/tests/test_tools_export_jsonl_interrupt_cleanup.py new file mode 100644 index 00000000..b71e32a9 --- /dev/null +++ b/tests/test_tools_export_jsonl_interrupt_cleanup.py @@ -0,0 +1,52 @@ +"""Lifecycle cleanup contracts for the NewsDOM JSONL exporter.""" + +from __future__ import annotations + +import importlib +import json +from pathlib import Path + +import pytest + +export_module = importlib.import_module("tools.export_jsonl") + + +def test_export_jsonl_cleans_temporary_artifact_on_keyboard_interrupt( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A user cancellation must not leak a temporary publication artifact.""" + input_file = tmp_path / "input.json" + input_file.write_text( + json.dumps( + { + "document_id": "test_doc", + "pages": [ + { + "page_number": 1, + "articles": [ + { + "article_id": "art_1", + "headline": "Headline", + "body_blocks": ["Block"], + } + ], + } + ], + } + ), + encoding="utf-8", + ) + output_file = tmp_path / "output.jsonl" + output_file.write_text("previous publication\n", encoding="utf-8") + + def _interrupt(*_args: object, **_kwargs: object) -> str: + raise KeyboardInterrupt + + monkeypatch.setattr(export_module.json, "dumps", _interrupt) + + with pytest.raises(KeyboardInterrupt): + export_module.export_jsonl(input_file, output_file) + + assert output_file.read_text(encoding="utf-8") == "previous publication\n" + assert list(tmp_path.glob(".output.jsonl.*.tmp")) == [] From 86d602caa7d97fb0e1e30ad55cf87e7089449a13 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Mon, 7 Sep 2026 21:42:43 +0900 Subject: [PATCH 18/20] fix(export): clean temporary file on cancellation --- tools/export_jsonl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/export_jsonl.py b/tools/export_jsonl.py index 29b53eaf..5cabce44 100644 --- a/tools/export_jsonl.py +++ b/tools/export_jsonl.py @@ -74,7 +74,7 @@ def export_jsonl(json_path: Path, output_path: Path) -> None: ) temporary_path.replace(output_path) - except Exception: + except BaseException: if temporary_path is not None: temporary_path.unlink(missing_ok=True) raise From ccae515138937687cb11b87cc6d6e1c51354936f Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Mon, 7 Sep 2026 13:15:03 +0000 Subject: [PATCH 19/20] =?UTF-8?q?=EA=B8=B0=EB=8A=A5:=20export=5Fjsonl=20?= =?UTF-8?q?=EB=8F=84=EA=B5=AC=20=EC=B6=94=EA=B0=80=20=EB=B0=8F=20CI=20?= =?UTF-8?q?=ED=99=98=EA=B2=BD=20=EC=BB=A4=EB=B2=84=EB=A6=AC=EC=A7=80=20?= =?UTF-8?q?=EB=B2=84=EA=B7=B8=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From be276223b4512f90f79677ac1264e2c1b526792c Mon Sep 17 00:00:00 2001 From: seonghobae <8172694+seonghobae@users.noreply.github.com> Date: Mon, 7 Sep 2026 14:51:27 +0000 Subject: [PATCH 20/20] =?UTF-8?q?=EA=B8=B0=EB=8A=A5:=20export=5Fjsonl=20?= =?UTF-8?q?=EB=8F=84=EA=B5=AC=20=EC=B6=94=EA=B0=80=20=EB=B0=8F=20CI=20?= =?UTF-8?q?=ED=99=98=EA=B2=BD=20=EC=BB=A4=EB=B2=84=EB=A6=AC=EC=A7=80=20?= =?UTF-8?q?=EB=B2=84=EA=B7=B8=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- uv.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index a0d133b8..1279f58d 100644 --- a/uv.lock +++ b/uv.lock @@ -303,7 +303,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -929,14 +929,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.15.0" +version = "6.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/17/ee75a92718ec7212de831e71454d702225aa5e474a805cce169806044453/pypdf-6.15.0.tar.gz", hash = "sha256:d39c4d955a76409284a905e2d65b40076d77ab76129e0faaeeb6612403ecfc79", size = 6993794, upload-time = "2026-08-06T13:06:49.929Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/dc/34857a5e31cf708c163929f61a9ba4bd357a8850e49fc4e846ced527b51f/pypdf-6.17.0.tar.gz", hash = "sha256:097ad0d829778ec5b615aeaa5c6da4b6cac4992f8fd80b56f98a1a8c006573bb", size = 7018352, upload-time = "2026-09-04T11:30:44.256Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/72/ce3067ac31e214a66388159f8462ddb8c13dd00170f24d555a1f1ae8ee91/pypdf-6.15.0-py3-none-any.whl", hash = "sha256:14e001d6504822cb1ca9c7ed9a69bccb320f59b320730f55af804361abe4d5ee", size = 378123, upload-time = "2026-08-06T13:06:47.709Z" }, + { url = "https://files.pythonhosted.org/packages/c1/08/1e9731038124a9127e1d27848952b86fb32b2f45f8f1b94adc7f0817a6ac/pypdf-6.17.0-py3-none-any.whl", hash = "sha256:5bd827266a21553b74d910e350131a6227b72f2ab4209bf372814b8195fa11c5", size = 388051, upload-time = "2026-09-04T11:30:42.681Z" }, ] [[package]]