diff --git a/CHANGELOG.md b/CHANGELOG.md index 2398ea5c..a11e2225 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > image placeholder until package, OpenAPI, image, provenance, and release > acceptance are aligned for an actual 0.3.0 publication. +### Added +- [CLI] 파싱된 NewsDOM JSON 데이터를 JSONL 형식으로 추출하여 LLM 학습 및 RAG 구축 등에 활용할 수 있도록 지원하는 `tools/export_jsonl.py` 도구를 추가했습니다. + ### Changed - `/parse`를 언어 선택형 파서로 일반화: MinerU `-l japan`/`-m ocr` 하드코딩을 제거하고 optional form 필드 `language`(MinerU 3.4.4 공식 기본 `ch`, 공개 언어군/alias 검증)와 `mode`(`auto`/`ocr`/`txt`, 기본 `auto`)로 파라미터화. `mode=auto`는 born-digital PDF가 강제 OCR을 건너뛰도록 함. 기존 입력 `language=japan&mode=ocr`는 공식 규약대로 `ch`/`ocr`로 정규화됨. - OpenAPI 제목/설명, README, `ArticleNode.headline` 문서를 일반 문서용 (section heading) 표현으로 재구성하여 특정 언어/신문 가정을 소비자에게 노출하지 않도록 함. 응답 스키마 필드는 하위 호환을 위해 변경하지 않음. diff --git a/tests/test_tools_export_jsonl.py b/tests/test_tools_export_jsonl.py new file mode 100644 index 00000000..1c5c40be --- /dev/null +++ b/tests/test_tools_export_jsonl.py @@ -0,0 +1,86 @@ +import json +from pathlib import Path +from unittest.mock import patch +import pytest + +from tools.export_jsonl import export_jsonl, main + + +def test_export_jsonl_success(tmp_path: Path) -> None: + input_file = tmp_path / "input.json" + output_file = tmp_path / "output.jsonl" + + data = { + "document_id": "doc123", + "pages": [ + { + "page_number": 1, + "articles": [ + { + "article_id": "art1", + "headline": "Head", + "body_blocks": ["Block 1", "Block 2"] + }, + "invalid_article_type" + ] + }, + "invalid_page_type" + ] + } + input_file.write_text(json.dumps(data), encoding="utf-8") + + export_jsonl(input_file, output_file) + + assert output_file.exists() + lines = output_file.read_text(encoding="utf-8").splitlines() + assert len(lines) == 1 + + record = json.loads(lines[0]) + assert record["document_id"] == "doc123" + assert record["page_number"] == 1 + assert record["article_id"] == "art1" + assert record["headline"] == "Head" + assert record["body"] == "Block 1\nBlock 2" + + +def test_export_jsonl_file_not_found(tmp_path: Path) -> None: + input_file = tmp_path / "not_found.json" + output_file = tmp_path / "output.jsonl" + with pytest.raises(FileNotFoundError, match="File not found"): + export_jsonl(input_file, output_file) + + +def test_export_jsonl_wrong_extension(tmp_path: Path) -> None: + input_file = tmp_path / "input.txt" + input_file.write_text("{}", encoding="utf-8") + output_file = tmp_path / "output.jsonl" + with pytest.raises(ValueError, match="Input file must be a .json file."): + export_jsonl(input_file, output_file) + + +def test_export_jsonl_invalid_json(tmp_path: Path) -> None: + input_file = tmp_path / "input.json" + input_file.write_text("{invalid", encoding="utf-8") + output_file = tmp_path / "output.jsonl" + with pytest.raises(ValueError, match="Invalid JSON file"): + export_jsonl(input_file, output_file) + + +@patch("sys.argv", ["export_jsonl.py", "input.json", "output.jsonl"]) +@patch("tools.export_jsonl.export_jsonl") +def test_main_success(mock_export, capsys) -> None: + main() + mock_export.assert_called_once() + captured = capsys.readouterr() + assert "JSONL successfully written" in captured.out + + +@patch("sys.argv", ["export_jsonl.py", "input.txt", "output.jsonl"]) +@patch("tools.export_jsonl.export_jsonl") +def test_main_error(mock_export, capsys) -> None: + mock_export.side_effect = ValueError("Test error") + with pytest.raises(SystemExit) as excinfo: + main() + assert excinfo.value.code == 1 + captured = capsys.readouterr() + assert "Error exporting JSONL: Test error" in captured.err diff --git a/tools/export_jsonl.py b/tools/export_jsonl.py new file mode 100644 index 00000000..93ee5c3e --- /dev/null +++ b/tools/export_jsonl.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +def export_jsonl(json_path: Path, output_path: Path) -> None: + """Export NewsDOM JSON to a JSONL file containing article metadata and body.""" + if not json_path.is_file(): + raise FileNotFoundError(f"File not found or is not a file: {json_path}") + if json_path.suffix.lower() != ".json": + raise ValueError("Input file must be a .json file.") + + try: + data = json.loads(json_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSON file: {exc}") from exc + + pages = data.get("pages", []) + document_id = data.get("document_id", "Unknown Document") + + with output_path.open("w", encoding="utf-8") as jsonlfile: + for page in pages: + if not isinstance(page, dict): + continue + page_number = page.get("page_number", "Unknown") + + articles = page.get("articles", []) + for article in articles: + if not isinstance(article, dict): + continue + + article_id = article.get("article_id", "Unknown Article ID") + headline = article.get("headline", "") + body_blocks = article.get("body_blocks", []) + + record = { + "document_id": document_id, + "page_number": page_number, + "article_id": article_id, + "headline": headline, + "body": "\n".join(body_blocks) + } + jsonlfile.write(json.dumps(record, ensure_ascii=False) + "\n") + + +def main(argv: list[str] | None = None) -> None: + """Run the JSON-to-JSONL export CLI.""" + parser = argparse.ArgumentParser(description="Export a NewsDOM JSON file to JSONL.") + parser.add_argument("input", type=Path, help="Path to the input JSON file.") + parser.add_argument("output", type=Path, help="Path to write the JSONL output file.") + + args = parser.parse_args(argv) + + try: + export_jsonl(args.input, args.output) + print(f"JSONL successfully written to {args.output}") + except Exception as exc: + print(f"Error exporting JSONL: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": # pragma: no cover + main()