✨ Feature: JSONL 내보내기 도구 추가

CHANGELOG.md

@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - OpenAPI 제목/설명, README, `ArticleNode.headline` 문서를 일반 문서용 (section heading) 표현으로 재구성하여 특정 언어/신문 가정을 소비자에게 노출하지 않도록 함. 응답 스키마 필드는 하위 호환을 위해 변경하지 않음.
 
 ### Added
+- `tools/export_jsonl.py` 도구 추가: NewsDOM JSON 데이터를 언어 모델 학습에 유용한 JSONL 형식으로 변환하는 기능 제공.
 - [CLI] 단일 NewsDOM JSON 파일을 페이지 단위로 분리하는 `tools/split_dom.py` 도구를 추가했습니다.
 - [CLI] NewsDOM JSON 파일의 모든 텍스트 내용을 마스킹하여 익명화하는 `tools/anonymize_dom.py` 도구를 추가했습니다.
 - `/parse`에 기본 필수 bearer 인증 경계를 추가했습니다. `NEWSDOM_AUTH_MODE=required`, `NEWSDOM_RUNTIME_PROFILE=production`, `NEWSDOM_API_TOKEN`을 명시해야 하며, 인증 비활성화는 격리된 development 프로필에서만 허용됩니다. `/health`는 liveness 전용으로 미인증 상태를 유지하고 `/ready`가 인증 설정과 MinerU 가용성을 함께 검증합니다.

tests/test_tools_export_jsonl.py

@@ -0,0 +1,90 @@
+import json
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from tools.export_jsonl import export_jsonl, main
+
+
+def test_export_jsonl_success(tmp_path: Path):
+    input_file = tmp_path / "test.json"
+    output_file = tmp_path / "test.jsonl"
+
+    input_data = """
+    {
+        "document_id": "doc1",
+        "pages": [
+            {
+                "page_number": 1,
+                "articles": [
+                    {
+                        "article_id": "art1",
+                        "headline": "Title 1",
+                        "body_blocks": ["Block 1", "Block 2"]
+                    },
+                    "invalid_article"
+                ]
+            },
+            "invalid_page"
+        ]
+    }
+    """
+    input_file.write_text(input_data, encoding="utf-8")
+
+    export_jsonl(input_file, output_file)
+
+    assert output_file.exists()
+    lines = output_file.read_text(encoding="utf-8").strip().split("\n")
+    assert len(lines) == 1
+
+    record = json.loads(lines[0])
+    assert record["document_id"] == "doc1"
+    assert record["page_number"] == 1
+    assert record["article_id"] == "art1"
+    assert record["headline"] == "Title 1"
+    assert record["body_text"] == "Block 1\nBlock 2"
+
+
+def test_export_jsonl_file_not_found(tmp_path: Path):
+    with pytest.raises(FileNotFoundError, match="File not found"):
+        export_jsonl(tmp_path / "missing.json", tmp_path / "out.jsonl")
+
+
+def test_export_jsonl_invalid_extension(tmp_path: Path):
+    input_file = tmp_path / "test.txt"
+    input_file.write_text("{}", encoding="utf-8")
+    with pytest.raises(ValueError, match="Input file must be a .json file."):
+        export_jsonl(input_file, tmp_path / "out.jsonl")
+
+
+def test_export_jsonl_invalid_json(tmp_path: Path):
+    input_file = tmp_path / "test.json"
+    input_file.write_text("{invalid}", encoding="utf-8")
+    with pytest.raises(ValueError, match="Invalid JSON file"):
+        export_jsonl(input_file, tmp_path / "out.jsonl")
+
+
+@patch("sys.argv", ["export_jsonl.py", "in.json", "out.jsonl"])
+def test_main_success(tmp_path: Path, monkeypatch, capsys):
+    input_file = tmp_path / "in.json"
+    input_file.write_text('{"document_id": "doc"}', encoding="utf-8")
+
+    monkeypatch.chdir(tmp_path)
+
+    try:
+        main()
+    except SystemExit:
+        pass
+
+    captured = capsys.readouterr()
+    assert "JSONL successfully written to out.jsonl" in captured.out
+
+
+@patch("sys.argv", ["export_jsonl.py", "missing.json", "out.jsonl"])
+def test_main_failure(capsys):
+    with pytest.raises(SystemExit) as exc_info:
+        main()
+    assert exc_info.value.code == 1
+    captured = capsys.readouterr()
+    assert "Error exporting JSONL:" in captured.err

tools/export_jsonl.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+
+def export_jsonl(json_path: Path, output_path: Path) -> None:
+    """Export NewsDOM JSON to a JSONL file where each line is an article."""
+    if not json_path.is_file():
+        raise FileNotFoundError(f"File not found or is not a file: {json_path}")
+    if json_path.suffix.lower() != ".json":
+        raise ValueError("Input file must be a .json file.")
+
+    try:
+        data = json.loads(json_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        raise ValueError(f"Invalid JSON file: {exc}") from exc
+
+    pages = data.get("pages", [])
+    document_id = data.get("document_id", "Unknown Document")
+
+    with output_path.open("w", encoding="utf-8") as jsonl_file:
+        for page in pages:
+            if not isinstance(page, dict):
+                continue
+            page_number = page.get("page_number", "Unknown")
+
+            articles = page.get("articles", [])
+            for article in articles:
+                if not isinstance(article, dict):
+                    continue
+
+                record = {
+                    "document_id": document_id,
+                    "page_number": page_number,
+                    "article_id": article.get("article_id", "Unknown Article ID"),
+                    "headline": article.get("headline", ""),
+                    "body_text": "\n".join(article.get("body_blocks", [])),
+                }
+
+                jsonl_file.write(json.dumps(record, ensure_ascii=False) + "\n")
+
+
+def main(argv: list[str] | None = None) -> None:
+    """Run the JSON-to-JSONL export CLI."""
+    parser = argparse.ArgumentParser(description="Export a NewsDOM JSON file to JSONL (one article per line).")
+    parser.add_argument("input", type=Path, help="Path to the input JSON file.")
+    parser.add_argument("output", type=Path, help="Path to write the JSONL output file.")
+
+    args = parser.parse_args(argv)
+
+    try:
+        export_jsonl(args.input, args.output)
+        print(f"JSONL successfully written to {args.output}")
+    except Exception as exc:
+        print(f"Error exporting JSONL: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()