From 84662ac7cf359455c59d37b54f201133558e9097 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:25:12 +0900 Subject: [PATCH 01/11] test(hwpx): specify ordered section recognition contract --- backend/tests/test_hwpx_recognition.py | 174 +++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 backend/tests/test_hwpx_recognition.py diff --git a/backend/tests/test_hwpx_recognition.py b/backend/tests/test_hwpx_recognition.py new file mode 100644 index 000000000..e140bbb14 --- /dev/null +++ b/backend/tests/test_hwpx_recognition.py @@ -0,0 +1,174 @@ +"""Executable contracts for safe HWPX section-text recognition.""" + +from __future__ import annotations + +import hashlib +import io +import zipfile + +import pytest + +from services import hwpx_recognition as recognition + + +_OPF_NS = "http://www.idpf.org/2007/opf/" +_HP_NS = "http://www.owpml.org/owpml/2021/paragraph" +_HS_NS = "http://www.owpml.org/owpml/2021/section" + + +def _section_xml(*paragraphs: str) -> str: + """Return a minimal standards-shaped HWPX section XML document.""" + rendered = "".join( + f'{text}' + for text in paragraphs + ) + return ( + f'' + f"{rendered}" + ) + + +def _content_hpf( + *, + manifest: tuple[tuple[str, str], ...], + spine: tuple[str, ...], +) -> str: + """Return a minimal OPF manifest/spine used by an HWPX package.""" + manifest_xml = "".join( + f'' + for item_id, href in manifest + ) + spine_xml = "".join(f'' for item_id in spine) + return ( + f'{manifest_xml}' + f"{spine_xml}" + ) + + +def _hwpx_package( + *, + sections: dict[str, str], + spine: tuple[str, ...], + manifest_hrefs: dict[str, str] | None = None, +) -> bytes: + """Build a small HWPX package with explicit manifest and spine order.""" + hrefs = manifest_hrefs or { + section_id: f"Contents/{section_id}.xml" for section_id in sections + } + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("mimetype", b"application/hwp+zip") + archive.writestr( + "Contents/content.hpf", + _content_hpf( + manifest=tuple((section_id, hrefs[section_id]) for section_id in sections), + spine=spine, + ), + ) + for section_id, xml in sections.items(): + archive.writestr(f"Contents/{section_id}.xml", xml) + return buffer.getvalue() + + +def test_recognize_hwpx_follows_spine_and_preserves_paragraph_provenance() -> None: + """Read sections in OPF spine order and retain semantic source positions.""" + payload = _hwpx_package( + sections={ + "section0": _section_xml("첫 번째 구역", "둘째 문단"), + "section1": _section_xml("두 번째 구역"), + }, + spine=("section1", "section0"), + ) + + records = recognition.recognize_hwpx_package( + payload, + filename="proposal.hwpx", + source_kind="attachment", + source_record_uid="attachment-42", + ) + + assert records.parse_text == "두 번째 구역\n\n첫 번째 구역\n\n둘째 문단" + assert records.parse_result.source_content_hash == hashlib.sha256(payload).hexdigest() + assert [segment.safe_text_content for segment in records.parse_result.segments] == [ + "두 번째 구역", + "첫 번째 구역", + "둘째 문단", + ] + assert [segment.segment_path for segment in records.parse_result.segments] == [ + "/document[1]/section[1]/paragraph[1]", + "/document[1]/section[2]/paragraph[1]", + "/document[1]/section[2]/paragraph[2]", + ] + + +def test_recognize_hwpx_rejects_manifest_path_traversal() -> None: + """Never resolve an OPF manifest href outside the HWPX package root.""" + payload = _hwpx_package( + sections={"section0": _section_xml("safe")}, + spine=("section0",), + manifest_hrefs={"section0": "../section0.xml"}, + ) + + with pytest.raises(ValueError, match="unsafe manifest href"): + recognition.recognize_hwpx_package( + payload, + filename="unsafe.hwpx", + source_kind="attachment", + source_record_uid="attachment-43", + ) + + +def test_recognize_hwpx_bounds_expanded_xml_before_reading( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Reject a ZIP expansion budget violation before XML parsing.""" + payload = _hwpx_package( + sections={"section0": _section_xml("x" * 256)}, + spine=("section0",), + ) + monkeypatch.setattr(recognition, "MAX_HWPX_XML_MEMBER_BYTES", 64) + + with pytest.raises(ValueError, match="XML member exceeds"): + recognition.recognize_hwpx_package( + payload, + filename="oversized.hwpx", + source_kind="attachment", + source_record_uid="attachment-44", + ) + + +def test_recognize_hwpx_rejects_unsafe_xml_entities() -> None: + """Defused XML parsing must reject entity-bearing section payloads.""" + dangerous_section = ( + ']>' + f'' + "&exfil;" + ) + payload = _hwpx_package( + sections={"section0": dangerous_section}, + spine=("section0",), + ) + + with pytest.raises(ValueError, match="unsafe XML"): + recognition.recognize_hwpx_package( + payload, + filename="entity.hwpx", + source_kind="attachment", + source_record_uid="attachment-45", + ) + + +def test_recognize_hwpx_requires_spine_referenced_section() -> None: + """Fail closed when OPF spine metadata cannot resolve a section member.""" + payload = _hwpx_package( + sections={"section0": _section_xml("safe")}, + spine=("missing-section",), + ) + + with pytest.raises(ValueError, match="spine item"): + recognition.recognize_hwpx_package( + payload, + filename="missing.hwpx", + source_kind="attachment", + source_record_uid="attachment-46", + ) From 66d3fd336c1cfaf37691e238c8ac3481b7eb2d56 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:26:48 +0900 Subject: [PATCH 02/11] feat(hwpx): parse ordered section text safely --- backend/services/hwpx_recognition.py | 289 +++++++++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 backend/services/hwpx_recognition.py diff --git a/backend/services/hwpx_recognition.py b/backend/services/hwpx_recognition.py new file mode 100644 index 000000000..e7c5a6e85 --- /dev/null +++ b/backend/services/hwpx_recognition.py @@ -0,0 +1,289 @@ +"""Safely recognize HWPX section text into Naruon's content graph. + +The importer retains HWPX bytes for deferred processing. This module performs the +next deterministic worker-side boundary: it revalidates the ZIP package, +follows the OPF spine order from ``Contents/content.hpf``, parses section XML +with defused XML semantics, and emits paragraph-level provenance without +extracting files, executing active content, or fetching external resources. +""" + +from __future__ import annotations + +import hashlib +import io +import re +import zipfile +from dataclasses import dataclass +from pathlib import PurePosixPath +from xml.etree.ElementTree import ParseError + +from defusedxml import ElementTree as DefusedElementTree +from defusedxml.common import DefusedXmlException + +from services.content_graph import ParseResult, PdfDomSection, parse_pdf_dom + +HWPX_PARSE_CONTENT_TYPE = "application/hwp+zip" +HWPX_PARSED_STATUS = "hwpx_xml_package_parsed" +HWPX_FAILED_STATUS = "hwpx_xml_package_failed" +MAX_HWPX_XML_MEMBER_BYTES = 4 * 1024 * 1024 +MAX_HWPX_TOTAL_XML_BYTES = 16 * 1024 * 1024 +MAX_HWPX_PACKAGE_ENTRIES = 4_096 +MAX_HWPX_MEMBER_NAME_BYTES = 1 * 1024 * 1024 + +_HWPX_MIMETYPE = b"application/hwp+zip" +_CONTENT_HPF_PATH = "Contents/content.hpf" +_SECTION_PATH_RE = re.compile(r"^Contents/section[0-9]+\.xml$") + + +@dataclass(frozen=True, slots=True) +class HwpxRecognitionRecords: + """Carry recognized HWPX text, graph records, and bounded parse counts.""" + + parse_text: str + parse_result: ParseResult + section_count: int + paragraph_count: int + + +def _local_name(tag: str) -> str: + """Return the local XML name without trusting a particular namespace URI.""" + + return tag.rsplit("}", 1)[-1] + + +def _safe_member_name(name: str) -> str: + """Validate a ZIP member as one normalized package-internal POSIX path.""" + + if not name or "\\" in name or "\x00" in name: + raise ValueError("HWPX package contains an unsafe member path") + path = PurePosixPath(name) + if path.is_absolute() or any(part in {"", ".", ".."} for part in path.parts): + raise ValueError("HWPX package contains an unsafe member path") + return path.as_posix() + + +def _package_entries(archive: zipfile.ZipFile) -> dict[str, zipfile.ZipInfo]: + """Return unique, unencrypted HWPX members within metadata budgets.""" + + entries = archive.infolist() + if not 0 < len(entries) <= MAX_HWPX_PACKAGE_ENTRIES: + raise ValueError("HWPX package exceeds the member-count limit") + + result: dict[str, zipfile.ZipInfo] = {} + aggregate_name_bytes = 0 + for entry in entries: + name = _safe_member_name(entry.filename) + aggregate_name_bytes += len(name.encode("utf-8", errors="surrogatepass")) + if aggregate_name_bytes > MAX_HWPX_MEMBER_NAME_BYTES: + raise ValueError("HWPX package exceeds the member-name metadata limit") + if name in result: + raise ValueError("HWPX package contains duplicate member paths") + if entry.flag_bits & 0x1: + raise ValueError("HWPX package contains encrypted members") + result[name] = entry + return result + + +def _read_member( + archive: zipfile.ZipFile, + entry: zipfile.ZipInfo, + *, + label: str, +) -> bytes: + """Read one already-selected XML member within its expansion budget.""" + + if entry.is_dir() or entry.file_size > MAX_HWPX_XML_MEMBER_BYTES: + raise ValueError(f"HWPX {label} XML member exceeds the expansion limit") + payload = archive.read(entry) + if len(payload) != entry.file_size or len(payload) > MAX_HWPX_XML_MEMBER_BYTES: + raise ValueError(f"HWPX {label} XML member exceeds the expansion limit") + return payload + + +def _parse_xml(payload: bytes, *, label: str): + """Parse one bounded XML member with entity/DTD defenses enabled.""" + + try: + return DefusedElementTree.fromstring(payload) + except (DefusedXmlException, ParseError, ValueError) as exc: + raise ValueError(f"HWPX {label} contains unsafe XML") from exc + + +def _resolve_manifest_href(href: str) -> str: + """Resolve an OPF item href only to the standard HWPX section namespace.""" + + value = href.strip() + if not value or "\\" in value or "\x00" in value or "?" in value or "#" in value: + raise ValueError("HWPX content.hpf contains an unsafe manifest href") + raw_path = PurePosixPath(value) + if raw_path.is_absolute() or any(part in {"", ".", ".."} for part in raw_path.parts): + raise ValueError("HWPX content.hpf contains an unsafe manifest href") + + if raw_path.parts and raw_path.parts[0] == "Contents": + resolved = raw_path.as_posix() + else: + resolved = (PurePosixPath("Contents") / raw_path).as_posix() + if not _SECTION_PATH_RE.fullmatch(resolved): + raise ValueError("HWPX content.hpf contains a non-section spine target") + return resolved + + +def _section_paths_from_spine(content_hpf_root) -> tuple[str, ...]: + """Resolve section member paths using OPF manifest identity and spine order.""" + + manifest_element = next( + (child for child in content_hpf_root if _local_name(child.tag) == "manifest"), + None, + ) + spine_element = next( + (child for child in content_hpf_root if _local_name(child.tag) == "spine"), + None, + ) + if manifest_element is None or spine_element is None: + raise ValueError("HWPX content.hpf is missing manifest or spine metadata") + + manifest: dict[str, str] = {} + for item in manifest_element: + if _local_name(item.tag) != "item": + continue + item_id = (item.get("id") or "").strip() + href = (item.get("href") or "").strip() + if not item_id or not href or item_id in manifest: + raise ValueError("HWPX content.hpf contains ambiguous manifest identity") + manifest[item_id] = href + + section_paths: list[str] = [] + for itemref in spine_element: + if _local_name(itemref.tag) != "itemref": + continue + item_id = (itemref.get("idref") or "").strip() + if not item_id or item_id not in manifest: + raise ValueError("HWPX spine item cannot be resolved through the manifest") + section_paths.append(_resolve_manifest_href(manifest[item_id])) + + if not section_paths or len(section_paths) != len(set(section_paths)): + raise ValueError("HWPX spine must reference unique section members") + return tuple(section_paths) + + +def _paragraph_text(paragraph) -> str: + """Extract text controls from one paragraph without duplicating nested paragraphs.""" + + parts: list[str] = [] + + def visit(element, *, is_root: bool = False) -> None: + for child in element: + local_name = _local_name(child.tag) + if local_name == "p" and not is_root: + continue + if local_name == "t": + parts.append("".join(child.itertext())) + elif local_name in {"lineBreak", "br"}: + parts.append("\n") + elif local_name == "tab": + parts.append("\t") + else: + visit(child) + + visit(paragraph, is_root=True) + return "".join(parts).strip() + + +def _section_paragraphs(section_root) -> tuple[str, ...]: + """Return non-empty paragraphs in document order from one HWPX section.""" + + paragraphs: list[str] = [] + for element in section_root.iter(): + if _local_name(element.tag) != "p": + continue + text = _paragraph_text(element) + if text: + paragraphs.append(text) + return tuple(paragraphs) + + +def recognize_hwpx_package( + payload: bytes, + *, + filename: str, + source_kind: str, + source_record_uid: str, +) -> HwpxRecognitionRecords: + """Recognize bounded HWPX section text and paragraph provenance. + + ``Contents/content.hpf`` is the ordering authority: manifest IDs resolve + package paths and spine references define reading order. Each selected XML + member is bounded before decompression and parsed with ``defusedxml``. + Images, OLE objects, external resources, macros, and non-section package + members are intentionally not interpreted by this slice. + """ + + if not isinstance(payload, bytes) or not payload.startswith(b"PK"): + raise ValueError("Pending attachment payload is not a HWPX package") + + try: + archive = zipfile.ZipFile(io.BytesIO(payload)) + except (zipfile.BadZipFile, ValueError) as exc: + raise ValueError("Pending attachment payload is not a HWPX package") from exc + + with archive: + entries = _package_entries(archive) + mimetype_entry = entries.get("mimetype") + content_hpf_entry = entries.get(_CONTENT_HPF_PATH) + if mimetype_entry is None or content_hpf_entry is None: + raise ValueError("HWPX package is missing required identity metadata") + if mimetype_entry.is_dir() or mimetype_entry.file_size > 128: + raise ValueError("HWPX package has an invalid mimetype member") + if archive.read(mimetype_entry) != _HWPX_MIMETYPE: + raise ValueError("HWPX package has an invalid mimetype member") + + content_hpf_payload = _read_member( + archive, + content_hpf_entry, + label="content.hpf", + ) + content_hpf_root = _parse_xml(content_hpf_payload, label="content.hpf") + section_paths = _section_paths_from_spine(content_hpf_root) + + selected_entries: list[zipfile.ZipInfo] = [] + expanded_total = len(content_hpf_payload) + for section_path in section_paths: + entry = entries.get(section_path) + if entry is None: + raise ValueError("HWPX spine section member is missing from the package") + if entry.file_size > MAX_HWPX_XML_MEMBER_BYTES: + raise ValueError("HWPX section XML member exceeds the expansion limit") + expanded_total += entry.file_size + if expanded_total > MAX_HWPX_TOTAL_XML_BYTES: + raise ValueError("HWPX selected XML exceeds the total expansion limit") + selected_entries.append(entry) + + sections: list[PdfDomSection] = [] + paragraph_count = 0 + for entry in selected_entries: + section_payload = _read_member(archive, entry, label="section") + section_root = _parse_xml(section_payload, label="section") + paragraphs = _section_paragraphs(section_root) + paragraph_count += len(paragraphs) + sections.append(PdfDomSection(heading="", paragraphs=paragraphs)) + + source_content_hash = hashlib.sha256(payload).hexdigest() + parse_result = parse_pdf_dom( + source_kind=source_kind, + source_record_uid=source_record_uid, + sections=sections, + source_content_hash=source_content_hash, + display_name=filename, + content_type=HWPX_PARSE_CONTENT_TYPE, + ) + parse_text = "\n\n".join( + segment.safe_text_content + for segment in parse_result.segments + if segment.segment_kind == "paragraph" + ) + return HwpxRecognitionRecords( + parse_text=parse_text, + parse_result=parse_result, + section_count=len(sections), + paragraph_count=paragraph_count, + ) From ef8e990f2a88c861bd0f9135861e040a30aff8cc Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:40:23 +0900 Subject: [PATCH 03/11] test(hwpx): specify deferred worker handoff --- backend/tests/test_hwpx_worker.py | 175 ++++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 backend/tests/test_hwpx_worker.py diff --git a/backend/tests/test_hwpx_worker.py b/backend/tests/test_hwpx_worker.py new file mode 100644 index 000000000..7acb38697 --- /dev/null +++ b/backend/tests/test_hwpx_worker.py @@ -0,0 +1,175 @@ +"""Contract tests for deferred HWPX background recognition. + +The attachment importer deliberately stores validated HWPX bytes as a pending +base64 payload. These tests require the existing recognition worker to consume +that pending state locally, without requiring a NewsDOM provider, while keeping +failure states explicit and bounded. +""" + +from __future__ import annotations + +import base64 +import io +import zipfile + +import pytest +from sqlalchemy.dialects import postgresql + +from db.models import Attachment, Email +from services.newsdom_worker import ( + RESULT_FAILED, + RESULT_RECOGNIZED, + NewsdomRecognitionWorker, + process_pending_attachment, +) + +HWPX_PENDING_STATUS = "hwpx_xml_package_pending" +HWPX_PARSED_STATUS = "hwpx_xml_package_parsed" +HWPX_FAILED_STATUS = "hwpx_xml_package_failed" + + +def _hwpx_payload(*, include_section: bool = True) -> bytes: + """Build one minimal standards-shaped HWPX package for worker tests.""" + + package = io.BytesIO() + with zipfile.ZipFile(package, "w", compression=zipfile.ZIP_STORED) as archive: + archive.writestr("mimetype", b"application/hwp+zip") + archive.writestr( + "Contents/content.hpf", + b""" + + + + + + + + +""", + ) + if include_section: + archive.writestr( + "Contents/section0.xml", + b""" + + Quarterly decision record + Approve the next action. + +""", + ) + return package.getvalue() + + +def _pending_hwpx_attachment(payload: bytes) -> Attachment: + """Create an in-memory HWPX attachment with its owning email relationship.""" + + email = Email() + email.organization_id = "org-hwpx" + attachment = Attachment( + id=73, + filename="decision.hwpx", + content=base64.b64encode(payload).decode("ascii"), + content_type="application/hwp+zip", + parse_content_type="application/hwp+zip", + parser_key="hwpx", + parse_status=HWPX_PENDING_STATUS, + ) + email.attachments.append(attachment) + return attachment + + +async def _must_not_resolve_provider(*_args, **_kwargs): + """Fail when deterministic HWPX recognition tries to resolve NewsDOM.""" + + raise AssertionError("HWPX recognition must not require a NewsDOM provider") + + +async def _must_not_call_newsdom(**_kwargs): + """Fail when deterministic HWPX recognition reaches the NewsDOM sidecar.""" + + raise AssertionError("HWPX recognition must not call the NewsDOM sidecar") + + +@pytest.mark.asyncio +async def test_pending_hwpx_attachment_is_recognized_without_provider() -> None: + """A pending HWPX package becomes searchable text plus graph provenance.""" + + attachment = _pending_hwpx_attachment(_hwpx_payload()) + + result = await process_pending_attachment( + session=object(), + attachment=attachment, + config_resolver=_must_not_resolve_provider, + request_fn=_must_not_call_newsdom, + ) + + assert result == RESULT_RECOGNIZED + assert attachment.parse_status == HWPX_PARSED_STATUS + assert attachment.parse_error_code is None + assert attachment.parse_content_type == "application/hwp+zip" + assert attachment.parser_key == "hwpx" + assert attachment.content == ( + "Quarterly decision record\n\nApprove the next action." + ) + assert [segment.safe_text_content for segment in attachment.content_segments] == [ + "Quarterly decision record", + "Approve the next action.", + ] + assert attachment.content_nodes + + +@pytest.mark.asyncio +async def test_pending_hwpx_attachment_revalidates_retained_bytes() -> None: + """Tampered retained bytes fail closed before any provider or XML work.""" + + attachment = _pending_hwpx_attachment(b"not-a-zip") + + result = await process_pending_attachment( + session=object(), + attachment=attachment, + config_resolver=_must_not_resolve_provider, + request_fn=_must_not_call_newsdom, + ) + + assert result == RESULT_FAILED + assert attachment.parse_status == HWPX_FAILED_STATUS + assert attachment.parse_error_code == "invalid_pending_payload" + assert attachment.content_nodes == [] + assert attachment.content_segments == [] + + +@pytest.mark.asyncio +async def test_pending_hwpx_attachment_records_recognizer_failure() -> None: + """A valid HWPX identity with a broken spine never masquerades as parsed.""" + + attachment = _pending_hwpx_attachment(_hwpx_payload(include_section=False)) + + result = await process_pending_attachment( + session=object(), + attachment=attachment, + config_resolver=_must_not_resolve_provider, + request_fn=_must_not_call_newsdom, + ) + + assert result == RESULT_FAILED + assert attachment.parse_status == HWPX_FAILED_STATUS + assert attachment.parse_error_code == "recognition_failed" + assert attachment.content_nodes == [] + assert attachment.content_segments == [] + + +def test_worker_selects_pdf_and_hwpx_pending_attachments() -> None: + """The bounded sweep must include both deferred attachment families.""" + + worker = NewsdomRecognitionWorker(batch_limit=7) + statement = worker._pending_attachment_statement(None) + sql = str( + statement.compile( + dialect=postgresql.dialect(), + compile_kwargs={"literal_binds": True}, + ) + ) + + assert "pdf_dom_recognition_pending" in sql + assert HWPX_PENDING_STATUS in sql + assert "LIMIT 7" in sql From 944a5303b814171b1f12553d9fe45d75a416440c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:42:10 +0900 Subject: [PATCH 04/11] feat(hwpx): process pending packages in worker --- backend/services/newsdom_worker.py | 145 ++++++++++++++++++++++++----- 1 file changed, 122 insertions(+), 23 deletions(-) diff --git a/backend/services/newsdom_worker.py b/backend/services/newsdom_worker.py index 26e4cffbb..64f49afc4 100644 --- a/backend/services/newsdom_worker.py +++ b/backend/services/newsdom_worker.py @@ -1,13 +1,13 @@ -"""Background worker glue for NewsDOM PDF DOM recognition. +"""Background worker glue for deferred PDF and HWPX recognition. -Attachments and workspace documents whose PDF recognition was deferred at -import time are processed here: the sidecar is called (via -:mod:`services.newsdom_pdf_recognition`) and the returned tree is landed into -the persisted attachment/document content and, for attachments, the -``content_nodes`` / ``content_segments`` graph. +PDF attachments and workspace documents whose NewsDOM recognition was deferred +at import time are processed through the configured sidecar. HWPX attachments +are processed locally through the bounded OWPML recognizer. Both paths land +recognized attachment text and provenance into ``content_nodes`` / +``content_segments`` without allowing a pending payload to masquerade as parsed. The apply functions are deliberately session-free so they can be unit tested -with in-memory model instances and a mocked NewsDOM client. +with in-memory model instances and injected adapters. """ from __future__ import annotations @@ -31,6 +31,13 @@ from db.session import AsyncSessionLocal from services.attachment_parser import decode_deferred_attachment_payload from services.content_graph import ParseResult +from services.hwpx_recognition import ( + HWPX_FAILED_STATUS, + HWPX_PARSED_STATUS, + HWPX_PARSE_CONTENT_TYPE, + HwpxRecognitionRecords, + recognize_hwpx_package, +) from services.newsdom_client import ( NewsdomConfigurationError, NewsdomRequestError, @@ -52,6 +59,9 @@ logger = logging.getLogger(__name__) _sysrand = random.SystemRandom() +HWPX_PENDING_STATUS = "hwpx_xml_package_pending" +HWPX_PARSER_KEY = "hwpx" + # How the worker resolves a runtime config for an organization. Injectable so # the per-item processors are unit-testable without a database. ConfigResolver = Callable[ @@ -123,6 +133,26 @@ def apply_recognition_to_attachment( ) +def apply_hwpx_recognition_to_attachment( + *, + email: Email, + attachment: Attachment, + records: HwpxRecognitionRecords, +) -> None: + """Land recognized HWPX text and provenance on one attachment.""" + + attachment.content = records.parse_text + attachment.parse_content_type = HWPX_PARSE_CONTENT_TYPE + attachment.parser_key = HWPX_PARSER_KEY + attachment.parse_status = HWPX_PARSED_STATUS + attachment.parse_error_code = None + _append_parse_result_to_attachment( + email=email, + attachment=attachment, + parse_result=records.parse_result, + ) + + def apply_recognition_to_document( *, document: Document, @@ -157,6 +187,29 @@ async def recognize_attachment_pdf( return records +def recognize_attachment_hwpx( + *, + email: Email, + attachment: Attachment, + hwpx_bytes: bytes, + source_record_uid: str, +) -> HwpxRecognitionRecords: + """Recognize a HWPX package and land its ordered text and graph.""" + + records = recognize_hwpx_package( + hwpx_bytes, + filename=attachment.filename or "attachment.hwpx", + source_kind="attachment", + source_record_uid=source_record_uid, + ) + apply_hwpx_recognition_to_attachment( + email=email, + attachment=attachment, + records=records, + ) + return records + + async def recognize_document_pdf( *, document: Document, @@ -198,6 +251,14 @@ async def recognize_document_pdf( } +def _attachment_failed_status(attachment: Attachment) -> str: + """Return the parser-family-specific visible failure status.""" + + if attachment.parse_status == HWPX_PENDING_STATUS: + return HWPX_FAILED_STATUS + return PDF_DOM_RECOGNITION_FAILED_STATUS + + async def process_pending_attachment( *, session: AsyncSession, @@ -205,18 +266,54 @@ async def process_pending_attachment( config_resolver: ConfigResolver = resolve_newsdom_config_from_db, request_fn: ParseRequestFn = request_pdf_dom, ) -> str: - """Recognize one pending attachment PDF, or record a safe outcome. + """Recognize one pending PDF or HWPX attachment, or record a safe outcome. - Returns ``RESULT_RECOGNIZED`` on success, ``RESULT_PENDING`` when no active - provider is configured yet (left pending to retry later), or - ``RESULT_FAILED`` when the payload or the sidecar response is unusable (a - visible failure status is recorded - never a false ``parsed``). + HWPX recognition is deterministic and local, so it never resolves a + NewsDOM provider. PDF recognition retains the provider-aware retry behavior: + ``RESULT_PENDING`` means no active provider is usable yet. Invalid retained + bytes or recognition failures are always recorded explicitly and never + reported as parsed. """ email = attachment.email if email is None: - attachment.parse_status = PDF_DOM_RECOGNITION_FAILED_STATUS + attachment.parse_status = _attachment_failed_status(attachment) attachment.parse_error_code = "orphan_attachment" return RESULT_FAILED + + if attachment.parse_status == HWPX_PENDING_STATUS: + try: + hwpx_bytes = decode_deferred_attachment_payload( + attachment.content, + attachment.parse_content_type or HWPX_PARSE_CONTENT_TYPE, + ) + except ValueError as exc: + attachment.parse_status = HWPX_FAILED_STATUS + attachment.parse_error_code = "invalid_pending_payload" + logger.warning( + "HWPX attachment %s rejected before recognition: %s", + getattr(attachment, "id", "?"), + exc, + ) + return RESULT_FAILED + + try: + recognize_attachment_hwpx( + email=email, + attachment=attachment, + hwpx_bytes=hwpx_bytes, + source_record_uid=f"attachment-{attachment.id}", + ) + except ValueError as exc: + attachment.parse_status = HWPX_FAILED_STATUS + attachment.parse_error_code = "recognition_failed" + logger.warning( + "HWPX attachment %s recognition failed: %s", + getattr(attachment, "id", "?"), + exc, + ) + return RESULT_FAILED + return RESULT_RECOGNIZED + try: pdf_bytes = decode_deferred_attachment_payload(attachment.content) except ValueError as exc: @@ -365,13 +462,14 @@ async def _release_sweep_lease(session: AsyncSession) -> None: class NewsdomRecognitionWorker: - """Periodically recognize pending PDF attachments and workspace documents. + """Periodically recognize pending PDF/HWPX attachments and PDF documents. Mirrors :class:`ReplySlaScheduler`: a jittered periodic loop, a PostgreSQL advisory-lock lease so only one replica sweeps per cycle, and per-item - error isolation. Items whose organization has no active NewsDOM provider are - left pending (they recognize once a provider is configured); unusable - payloads/responses are marked failed rather than parsed. + error isolation. HWPX work is deterministic/local. PDF items whose + organization has no active NewsDOM provider remain pending until a provider + is configured; unusable payloads/responses are marked failed rather than + parsed. """ def __init__( @@ -444,8 +542,7 @@ async def _sweep(self) -> None: lease = await _try_acquire_sweep_lease(session) if lease is False: logger.debug( - "NewsDOM recognition sweep skipped: another replica holds " - "the lease." + "Recognition sweep skipped: another replica holds the lease." ) return try: @@ -471,22 +568,24 @@ async def _sweep_attachments(self, session: AsyncSession) -> None: await session.commit() if result != RESULT_PENDING: logger.info( - "NewsDOM attachment %s recognition result: %s", + "Attachment %s recognition result: %s", attachment.id, result, ) except Exception: await session.rollback() logger.error( - "NewsDOM attachment %s recognition raised.", + "Attachment %s recognition raised.", getattr(attachment, "id", "?"), exc_info=True, ) def _pending_attachment_statement(self, after_id: int | None): - """Build the next deterministic attachment batch query.""" + """Build the next deterministic PDF/HWPX attachment batch query.""" statement = select(Attachment).where( - Attachment.parse_status == PDF_DOM_RECOGNITION_PENDING_STATUS + Attachment.parse_status.in_( + (PDF_DOM_RECOGNITION_PENDING_STATUS, HWPX_PENDING_STATUS) + ) ) if after_id is not None: statement = statement.where(Attachment.id > after_id) From fdf157dd675f8ca91a248f801c2c6e4c75e0732f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:43:48 +0900 Subject: [PATCH 05/11] test(hwpx): cover orphan failure status --- backend/tests/test_hwpx_worker.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/backend/tests/test_hwpx_worker.py b/backend/tests/test_hwpx_worker.py index 7acb38697..3ff4a64a0 100644 --- a/backend/tests/test_hwpx_worker.py +++ b/backend/tests/test_hwpx_worker.py @@ -158,6 +158,32 @@ async def test_pending_hwpx_attachment_records_recognizer_failure() -> None: assert attachment.content_segments == [] +@pytest.mark.asyncio +async def test_orphan_pending_hwpx_attachment_uses_hwpx_failure_status() -> None: + """An orphan HWPX row fails visibly without being mislabeled as PDF.""" + + attachment = Attachment( + id=74, + filename="orphan.hwpx", + content=base64.b64encode(_hwpx_payload()).decode("ascii"), + content_type="application/hwp+zip", + parse_content_type="application/hwp+zip", + parser_key="hwpx", + parse_status=HWPX_PENDING_STATUS, + ) + + result = await process_pending_attachment( + session=object(), + attachment=attachment, + config_resolver=_must_not_resolve_provider, + request_fn=_must_not_call_newsdom, + ) + + assert result == RESULT_FAILED + assert attachment.parse_status == HWPX_FAILED_STATUS + assert attachment.parse_error_code == "orphan_attachment" + + def test_worker_selects_pdf_and_hwpx_pending_attachments() -> None: """The bounded sweep must include both deferred attachment families.""" From fbc7b4637e31ddf421af4aa5d2d94ae24e970e9f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:45:50 +0900 Subject: [PATCH 06/11] test(hwpx): cover canonical MIME fallback --- backend/tests/test_hwpx_worker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/tests/test_hwpx_worker.py b/backend/tests/test_hwpx_worker.py index 3ff4a64a0..324e8f2b3 100644 --- a/backend/tests/test_hwpx_worker.py +++ b/backend/tests/test_hwpx_worker.py @@ -95,6 +95,7 @@ async def test_pending_hwpx_attachment_is_recognized_without_provider() -> None: """A pending HWPX package becomes searchable text plus graph provenance.""" attachment = _pending_hwpx_attachment(_hwpx_payload()) + attachment.parse_content_type = "" result = await process_pending_attachment( session=object(), From b66d9334677a4184c3793ee66ce20eb7696a97c2 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 19:47:10 +0900 Subject: [PATCH 07/11] docs(hwpx): trace worker recognition boundary --- .../hwp-hwpx-attachment-recognition.md | 174 ++++++++++++++---- 1 file changed, 141 insertions(+), 33 deletions(-) diff --git a/docs/doctoring/hwp-hwpx-attachment-recognition.md b/docs/doctoring/hwp-hwpx-attachment-recognition.md index ebdd660a5..0d3f2eed7 100644 --- a/docs/doctoring/hwp-hwpx-attachment-recognition.md +++ b/docs/doctoring/hwp-hwpx-attachment-recognition.md @@ -1,5 +1,14 @@ # HWP and HWPX attachment-recognition boundary +## Status + +This document separates protected-`develop` truth from active pull-request work. +The HWP/HWPX deferred-import contract is owned by active parent PR #1353. The +ordered HWPX section recognizer and its production worker handoff are owned by +stacked active PR #1373. Neither capability is shipped from protected `develop` +until its exact integrated head passes the live review, CI, security, coverage, +provenance, and branch-governance gates and is merged. + ## Decision Naruon recognizes HWPX and HWP attachments before OCR, XML extraction, or LLM @@ -8,10 +17,15 @@ classifies the parser family, applies bounded signature checks, retains exact source bytes as a base64 deferred-recognition payload, and records a stable pending or rejection status. -This keeps email import deterministic and evidence-preserving while later -sandboxed workers perform heavier extraction. +The HWPX worker then performs a deterministic local recognition stage rather +than sending package bytes to an external model or NewsDOM provider. It repeats +package validation, resolves body sections through `Contents/content.hpf`, +parses only bounded selected XML with entity/DTD defenses, and lands ordered +paragraph text plus source-bound content-graph provenance. This keeps email +import deterministic and evidence-preserving while making recognized Korean +enterprise document text searchable without model dependence. -## Shipped boundary +## Active deferred-import boundary — PR #1353 - `.hwpx` and `.owpml` files with generic binary MIME types are resolved to the HWPX parser family. @@ -26,8 +40,8 @@ sandboxed workers perform heavier extraction. members, unsupported ZIP structures, malformed ZIP metadata, and exceeded limits fail closed as `invalid_hwpx_payload`. - Import does not decompress document sections, extract files, execute active - content, or fetch external resources. Later workers must repeat path, - compression, XML, resource, and expansion-ratio validation before extraction. + content, or fetch external resources. Later workers repeat path, XML, + encryption, resource, and expansion validation before extraction. - `.hwp` files with generic binary MIME types are resolved to the HWP parser family. - HWP bytes must carry both the OLE Compound File container signature and the @@ -41,27 +55,82 @@ sandboxed workers perform heavier extraction. - Invalid HWPX/HWP/PDF payloads fail closed and are not retained as deferred parser inputs. +## Active HWPX semantic-recognition boundary — PR #1373 + +The current bounded vertical slice consumes only `hwpx_xml_package_pending` +email attachments. It reuses Naruon's existing leased background-recognition +worker instead of creating another scheduler or service authority. + +1. The worker decodes the retained base64 bytes with the HWPX content-family + validator. Corrupted or family-mismatched retained payloads become + `hwpx_xml_package_failed` with `invalid_pending_payload` before XML parsing. +2. `mimetype` and `Contents/content.hpf` are mandatory. The worker rejects + duplicate paths, encrypted entries, traversal paths, unsafe manifest hrefs, + unresolved or repeated spine targets, and XML that violates the defused + parser boundary. +3. Manifest identity resolves each spine item and the spine defines section + reading order. Only `Contents/sectionN.xml` targets are accepted in this + slice. +4. Each selected XML member is bounded before decompression/read, and the + selected `content.hpf` plus section XML expansion total is bounded. +5. Paragraphs are emitted in section/document order into the existing content + graph. The graph UIDs remain bound to SHA-256 of the exact original HWPX + source bytes rather than a lossy text reserialization. +6. Recognition succeeds without a NewsDOM/provider configuration. Provider + resolution and network calls remain PDF-only behavior. +7. Orphan attachments and parser failures remain visible failure states; no + error path is reported as parsed. +8. The production sweep selects PDF and HWPX pending rows together with the + existing bounded, cursor-based, leased batch semantics so an HWPX attachment + cannot remain permanently invisible to the running worker. + +This slice deliberately reuses the existing `PdfDomSection`/`parse_pdf_dom` +graph construction primitive as a format-neutral document→section→paragraph +builder. That reuse does not assert that HWPX is PDF or transfer PDF parsing +semantics; the recognizer supplies HWPX-derived ordered sections and the exact +HWPX source hash. + +## Standards confirmation + +The Korean national standards registry identifies KS X 6101, *Open +Word-Processor Markup Language (OWPML) document structure*, and records its +latest confirmation/revision date as 2024-10-30. Hancom's current format +material states that HWPX follows KS X 6101/OWPML and is a ZIP-packaged XML +format designed for machine-readable document content. Hancom's HWPX parsing +guidance also maps body XML to the OWPML body/section/paragraph schemas. The +worker therefore treats package and XML structure as deterministic document +evidence, not as an LLM interpretation target. + +No psychometric/statistical or model-orchestration claim is introduced by this +slice, so peer-reviewed model evidence is not a gating dependency here. The +material authority is the current national standard plus the format owner's +primary technical documentation. + ## HWPX resource bounds -| Boundary | Current import limit | Purpose | -| --- | ---: | --- | -| Complete deferred source | 20 MiB | Prevent oversized payload retention. | -| ZIP entry count | 4,096 | Bound member-object and traversal work. | -| Central-directory bytes | 4 MiB | Bound metadata parsing before member materialization. | -| Aggregate decoded member-name bytes | 1 MiB | Prevent path/name metadata amplification. | -| `mimetype` uncompressed bytes | 128 bytes | Keep signature validation deterministic and non-expansive. | +| Boundary | Current bound | Applied at | Purpose | +| --- | ---: | --- | --- | +| Complete deferred source | 20 MiB | import + deferred decode | Prevent oversized payload retention. | +| ZIP entry count | 4,096 | import + worker | Bound member-object and traversal work. | +| Central-directory bytes | 4 MiB | import | Bound metadata parsing before member materialization. | +| Aggregate decoded member-name bytes | 1 MiB | import + worker | Prevent path/name metadata amplification. | +| `mimetype` uncompressed bytes | 128 bytes | import + worker | Keep signature validation deterministic and non-expansive. | +| One selected XML member | 4 MiB | worker | Bound decompression and parser memory per selected member. | +| Selected XML total | 16 MiB | worker | Bound aggregate semantic-recognition expansion. | -These are admission limits, not statements about the maximum document Hancom -Office can create. An operator may change them only with reviewed capacity and -security evidence. The recognition step deliberately rejects ZIP64 or multi-disk -packages rather than widening a low-cost email-import boundary. +These are product admission/worker limits, not statements about the maximum +document Hancom Office can create. An operator may change them only with +reviewed capacity and security evidence. The import boundary deliberately +rejects ZIP64 or multi-disk packages rather than widening low-cost email import. -## Test-first repair evidence +## Test-first evidence + +### Import hardening — parent PR #1353 The initial HWPX slice accepted a ZIP by member names alone. A generic ZIP could -therefore imitate `mimetype`, `version.xml`, and section paths without carrying -the HWPX signature, while a small source file could still devote most of its -bytes to a very large central directory. +therefore imitate HWPX paths without carrying the HWPX signature, while a small +source file could still devote most of its bytes to a very large central +directory. Commit `4b51240eb8521459ef622e49bd463a1a6d783288` added failing public-boundary regressions for wrong and duplicate `mimetype` members, entry count, @@ -78,35 +147,68 @@ FileHeader identity marker as a second admission signal; commit `c8837fb00d74bd4ddc3152e0fe793e71f9e1f41f` aligned the positive fixture with that real contract. +### Ordered recognition and worker handoff — PR #1373 + +- `84662ac7cf359455c59d37b54f201133558e9097` specifies OPF-spine ordering, + path/provenance, package traversal, XML expansion, unsafe-XML, and unresolved + spine behavior before the recognizer exists. +- `66d3fd336c1cfaf37691e238c8ac3481b7eb2d56` implements the bounded HWPX + recognizer. +- `ef8e990f2a88c861bd0f9135861e040a30aff8cc` specifies the production worker + handoff: local recognition without a provider, retained-byte revalidation, + failure visibility, content-graph landing, and pending-row selection. +- `944a5303b814171b1f12553d9fe45d75a416440c` wires the HWPX pending state into + the existing leased recognition worker and preserves the PDF path. +- `fdf157dd675f8ca91a248f801c2c6e4c75e0732f` and + `fbc7b4637e31ddf421af4aa5d2d94ae24e970e9f` cover parser-family orphan status + and canonical MIME fallback branches. + Hosted exact-head CI, security, coverage, and review evidence remains -authoritative for merge. +authoritative for merge; the commit lineage above is implementation +traceability, not a substitute for current-head gates. + +## Requirement traceability + +| Requirement | Production owner | Test evidence | Maturity | +| --- | --- | --- | --- | +| Bounded HWPX admission and exact source retention | `backend/services/attachment_parser.py` | `test_attachment_parser*.py` | Active parent PR #1353 | +| OPF manifest/spine ordered HWPX paragraph recognition | `backend/services/hwpx_recognition.py` | `test_hwpx_recognition.py` | Active stacked PR #1373 | +| Deferred HWPX worker selection and local execution | `backend/services/newsdom_worker.py` | `test_hwpx_worker.py` | Active stacked PR #1373 | +| Parsed attachment text + graph provenance | shared content-graph landing path | HWPX worker + recognizer tests | Active stacked PR #1373 | +| Binary HWP conversion | future sandboxed converter | none yet | Planned / out of this slice | +| HWPX tables, images, layout fidelity | future bounded recognizers | none yet | Planned / out of this slice | +| Protected-`develop` shipped HWP/HWPX recognition | protected branch | integrated release gates | Not yet shipped | ## Status codes -| Parser family | Pending status | Rejection status | +| Parser family | Pending status | Parsed/failed state | | --- | --- | --- | -| PDF | `pdf_dom_recognition_pending` | `invalid_pdf_payload` | -| HWPX | `hwpx_xml_package_pending` | `invalid_hwpx_payload` | -| HWP | `hwp_conversion_pending` | `invalid_hwp_payload` | +| PDF | `pdf_dom_recognition_pending` | existing NewsDOM parsed/failed states | +| HWPX | `hwpx_xml_package_pending` | `hwpx_xml_package_parsed` / `hwpx_xml_package_failed` | +| HWP | `hwp_conversion_pending` | converter not implemented in this slice | ## Out of scope -This slice does not implement semantic HWPX section extraction, embedded image -recognition, table reconstruction, HWP binary conversion, OCR, or LLM/VLM -interpretation. Those belong to a later worker-backed pipeline from the -evidence-based workspace epic. +This slice does not reconstruct HWPX tables, images, charts, layout, styles, or +embedded objects; convert binary HWP; perform OCR; fetch external resources; or +call LLM/VLM providers. Those require separately bounded workers and acceptance +evidence. The worker is not a general ZIP/XML extraction service. ## Safety and buyer value Korean enterprise mailboxes often carry HWP and HWPX evidence. Treating those attachments as opaque unsupported binaries breaks context synthesis, search -coverage, and auditability. Treating them as text or passing them directly to an -LLM is also unsafe. This slice gives the product an auditable middle state: the -source bytes are preserved, the file family is explicit, and follow-on workers -can proceed without losing provenance. +coverage, and auditability. Treating them as unbounded XML/ZIP input or passing +source bytes directly to an LLM is also unsafe. The active vertical slice gives +the product an auditable path from retained source bytes to ordered searchable +paragraphs and exact-source provenance while keeping failure states explicit. ## References +Korean Agency for Technology and Standards. (2024, October 30). *KS X 6101: +Open Word-Processor Markup Language (OWPML) document structure*. e-Nara Standard +Certification. https://www.standard.go.kr/KSCI/standardIntro/getStandardSearchView.do?ksNo=KSX6101&menuId=503&tmprKsNo=KSX6101&topMenuId=502 + Hancom Inc. (n.d.). *HWP binary format and HWPML document format*. Hancom Support. https://www.hancom.com/support/downloadCenter/hwpOwpml @@ -119,5 +221,11 @@ https://tech.hancom.com/%ED%95%9C-%EA%B8%80-%EB%AC%B8%EC%84%9C-%ED%8C%8C%EC%9D%B Hancom Tech. (2025b, February 26). *HWPX format structure*. https://tech.hancom.com/hwpxformat/ +Hancom Tech. (2025c). *Parsing HWPX format with Python (Part 1)*. +https://tech.hancom.com/python-hwpx-parsing-1/ + +Hancom Tech. (2025d). *Parsing HWPX format with Python (Part 2)*. +https://tech.hancom.com/python-hwpx-parsing-2/ + PKWARE, Inc. (2024). *APPNOTE.TXT: .ZIP file format specification*. https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT From 595ae602f35d5ec75cb43d4696d0b141a6ae9874 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 18:11:32 +0000 Subject: [PATCH 08/11] test(hwpx): align pending fixtures with parent identity Parent #1353 admission requires version.xml before a HWPX payload can remain pending. Worker fixtures omitted that member, so deferred revalidation failed closed before recognition. Co-authored-by: Seongho Bae --- backend/tests/test_hwpx_recognition.py | 1 + backend/tests/test_hwpx_worker.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/backend/tests/test_hwpx_recognition.py b/backend/tests/test_hwpx_recognition.py index e140bbb14..38c51ddad 100644 --- a/backend/tests/test_hwpx_recognition.py +++ b/backend/tests/test_hwpx_recognition.py @@ -58,6 +58,7 @@ def _hwpx_package( buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive: archive.writestr("mimetype", b"application/hwp+zip") + archive.writestr("version.xml", b'') archive.writestr( "Contents/content.hpf", _content_hpf( diff --git a/backend/tests/test_hwpx_worker.py b/backend/tests/test_hwpx_worker.py index 324e8f2b3..b13f89533 100644 --- a/backend/tests/test_hwpx_worker.py +++ b/backend/tests/test_hwpx_worker.py @@ -34,6 +34,8 @@ def _hwpx_payload(*, include_section: bool = True) -> bytes: package = io.BytesIO() with zipfile.ZipFile(package, "w", compression=zipfile.ZIP_STORED) as archive: archive.writestr("mimetype", b"application/hwp+zip") + # Parent #1353 admission requires version.xml before a payload can be pending. + archive.writestr("version.xml", b'') archive.writestr( "Contents/content.hpf", b""" From 32099709bafcee19fb32c385bbe89e0df15fe102 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 19 Aug 2026 16:55:32 +0900 Subject: [PATCH 09/11] fix(hwpx): harden deferred recognition provenance --- backend/services/hwpx_recognition.py | 33 +++++++--- backend/services/newsdom_worker.py | 19 +++++- backend/tests/test_hwpx_recognition.py | 88 +++++++++++++++++++++++++- backend/tests/test_hwpx_worker.py | 26 +++++++- 4 files changed, 151 insertions(+), 15 deletions(-) diff --git a/backend/services/hwpx_recognition.py b/backend/services/hwpx_recognition.py index e7c5a6e85..e221af830 100644 --- a/backend/services/hwpx_recognition.py +++ b/backend/services/hwpx_recognition.py @@ -89,13 +89,17 @@ def _read_member( entry: zipfile.ZipInfo, *, label: str, + max_bytes: int = MAX_HWPX_XML_MEMBER_BYTES, ) -> bytes: """Read one already-selected XML member within its expansion budget.""" - if entry.is_dir() or entry.file_size > MAX_HWPX_XML_MEMBER_BYTES: + if entry.is_dir() or entry.file_size > max_bytes: raise ValueError(f"HWPX {label} XML member exceeds the expansion limit") - payload = archive.read(entry) - if len(payload) != entry.file_size or len(payload) > MAX_HWPX_XML_MEMBER_BYTES: + try: + payload = archive.read(entry) + except (NotImplementedError, zipfile.BadZipFile) as exc: + raise ValueError(f"HWPX {label} XML member could not be read") from exc + if len(payload) != entry.file_size or len(payload) > max_bytes: raise ValueError(f"HWPX {label} XML member exceeds the expansion limit") return payload @@ -171,10 +175,10 @@ def _paragraph_text(paragraph) -> str: parts: list[str] = [] - def visit(element, *, is_root: bool = False) -> None: + def visit(element) -> None: for child in element: local_name = _local_name(child.tag) - if local_name == "p" and not is_root: + if local_name == "p": continue if local_name == "t": parts.append("".join(child.itertext())) @@ -185,7 +189,7 @@ def visit(element, *, is_root: bool = False) -> None: else: visit(child) - visit(paragraph, is_root=True) + visit(paragraph) return "".join(parts).strip() @@ -229,12 +233,21 @@ def recognize_hwpx_package( with archive: entries = _package_entries(archive) mimetype_entry = entries.get("mimetype") + version_entry = entries.get("version.xml") content_hpf_entry = entries.get(_CONTENT_HPF_PATH) - if mimetype_entry is None or content_hpf_entry is None: + if ( + mimetype_entry is None + or version_entry is None + or content_hpf_entry is None + ): raise ValueError("HWPX package is missing required identity metadata") - if mimetype_entry.is_dir() or mimetype_entry.file_size > 128: - raise ValueError("HWPX package has an invalid mimetype member") - if archive.read(mimetype_entry) != _HWPX_MIMETYPE: + mimetype_payload = _read_member( + archive, + mimetype_entry, + label="mimetype", + max_bytes=128, + ) + if mimetype_payload != _HWPX_MIMETYPE: raise ValueError("HWPX package has an invalid mimetype member") content_hpf_payload = _read_member( diff --git a/backend/services/newsdom_worker.py b/backend/services/newsdom_worker.py index 64f49afc4..32a60861a 100644 --- a/backend/services/newsdom_worker.py +++ b/backend/services/newsdom_worker.py @@ -13,6 +13,7 @@ from __future__ import annotations import asyncio +import hashlib import logging import random from collections.abc import Awaitable, Callable @@ -69,6 +70,20 @@ ] +def _attachment_source_record_uid(email: Email, attachment: Attachment) -> str: + """Return opaque provenance derived from stable message and source data.""" + identity = "\x00".join( + ( + email.message_id or "", + email.thread_id or "", + attachment.filename or "", + attachment.content or "", + ) + ) + digest = hashlib.sha256(identity.encode("utf-8", errors="surrogatepass")) + return f"attachment:{digest.hexdigest()[:32]}" + + def _append_parse_result_to_attachment( *, email: Email, @@ -301,7 +316,7 @@ async def process_pending_attachment( email=email, attachment=attachment, hwpx_bytes=hwpx_bytes, - source_record_uid=f"attachment-{attachment.id}", + source_record_uid=_attachment_source_record_uid(email, attachment), ) except ValueError as exc: attachment.parse_status = HWPX_FAILED_STATUS @@ -343,7 +358,7 @@ async def process_pending_attachment( attachment=attachment, pdf_bytes=pdf_bytes, config=config, - source_record_uid=f"attachment-{attachment.id}", + source_record_uid=_attachment_source_record_uid(email, attachment), request_fn=request_fn, ) except NewsdomConfigurationError as exc: diff --git a/backend/tests/test_hwpx_recognition.py b/backend/tests/test_hwpx_recognition.py index 38c51ddad..976f1b7ba 100644 --- a/backend/tests/test_hwpx_recognition.py +++ b/backend/tests/test_hwpx_recognition.py @@ -50,6 +50,7 @@ def _hwpx_package( sections: dict[str, str], spine: tuple[str, ...], manifest_hrefs: dict[str, str] | None = None, + include_version: bool = True, ) -> bytes: """Build a small HWPX package with explicit manifest and spine order.""" hrefs = manifest_hrefs or { @@ -58,7 +59,8 @@ def _hwpx_package( buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w", compression=zipfile.ZIP_DEFLATED) as archive: archive.writestr("mimetype", b"application/hwp+zip") - archive.writestr("version.xml", b'') + if include_version: + archive.writestr("version.xml", b'') archive.writestr( "Contents/content.hpf", _content_hpf( @@ -101,6 +103,90 @@ def test_recognize_hwpx_follows_spine_and_preserves_paragraph_provenance() -> No "/document[1]/section[2]/paragraph[2]", ] + changed_records = recognition.recognize_hwpx_package( + _hwpx_package( + sections={"section0": _section_xml("변경된 문서")}, + spine=("section0",), + ), + filename="proposal.hwpx", + source_kind="attachment", + source_record_uid="attachment-42", + ) + assert changed_records.parse_result.nodes[0].content_node_uid != ( + records.parse_result.nodes[0].content_node_uid + ) + assert changed_records.parse_result.segments[0].content_segment_uid != ( + records.parse_result.segments[0].content_segment_uid + ) + + +def test_recognize_hwpx_rejects_missing_version_member() -> None: + """Revalidate the required version member before section recognition.""" + payload = _hwpx_package( + sections={"section0": _section_xml("safe")}, + spine=("section0",), + include_version=False, + ) + + with pytest.raises(ValueError, match="missing required identity"): + recognition.recognize_hwpx_package( + payload, + filename="missing-version.hwpx", + source_kind="attachment", + source_record_uid="attachment-47", + ) + + +@pytest.mark.parametrize("member_name", ("mimetype", "Contents/content.hpf")) +def test_recognize_hwpx_normalizes_zip_read_failures( + monkeypatch: pytest.MonkeyPatch, + member_name: str, +) -> None: + """Convert ZIP read implementation failures into bounded parse errors.""" + payload = _hwpx_package( + sections={"section0": _section_xml("safe")}, + spine=("section0",), + ) + original_read = zipfile.ZipFile.read + + def broken_read(archive, member, *args, **kwargs): + if getattr(member, "filename", member) == member_name: + raise zipfile.BadZipFile("simulated read failure") + return original_read(archive, member, *args, **kwargs) + + monkeypatch.setattr(zipfile.ZipFile, "read", broken_read) + + with pytest.raises(ValueError, match="could not be read"): + recognition.recognize_hwpx_package( + payload, + filename="read-failure.hwpx", + source_kind="attachment", + source_record_uid="attachment-48", + ) + + +def test_paragraph_text_does_not_duplicate_nested_paragraphs() -> None: + """Nested paragraph nodes are skipped by the containing paragraph.""" + nested_section = ( + f'' + "nested" + "" + ) + payload = _hwpx_package( + sections={"section0": nested_section}, + spine=("section0",), + ) + + records = recognition.recognize_hwpx_package( + payload, + filename="nested.hwpx", + source_kind="attachment", + source_record_uid="attachment-49", + ) + + assert records.parse_text == "nested" + assert records.paragraph_count == 1 + def test_recognize_hwpx_rejects_manifest_path_traversal() -> None: """Never resolve an OPF manifest href outside the HWPX package root.""" diff --git a/backend/tests/test_hwpx_worker.py b/backend/tests/test_hwpx_worker.py index b13f89533..3a93fccd7 100644 --- a/backend/tests/test_hwpx_worker.py +++ b/backend/tests/test_hwpx_worker.py @@ -28,14 +28,17 @@ HWPX_FAILED_STATUS = "hwpx_xml_package_failed" -def _hwpx_payload(*, include_section: bool = True) -> bytes: +def _hwpx_payload( + *, include_section: bool = True, include_version: bool = True +) -> bytes: """Build one minimal standards-shaped HWPX package for worker tests.""" package = io.BytesIO() with zipfile.ZipFile(package, "w", compression=zipfile.ZIP_STORED) as archive: archive.writestr("mimetype", b"application/hwp+zip") # Parent #1353 admission requires version.xml before a payload can be pending. - archive.writestr("version.xml", b'') + if include_version: + archive.writestr("version.xml", b'') archive.writestr( "Contents/content.hpf", b""" @@ -119,6 +122,8 @@ async def test_pending_hwpx_attachment_is_recognized_without_provider() -> None: "Approve the next action.", ] assert attachment.content_nodes + assert attachment.content_nodes[0].source_record_uid.startswith("attachment:") + assert attachment.content_nodes[0].source_record_uid != "attachment-73" @pytest.mark.asyncio @@ -161,6 +166,23 @@ async def test_pending_hwpx_attachment_records_recognizer_failure() -> None: assert attachment.content_segments == [] +@pytest.mark.asyncio +async def test_pending_hwpx_attachment_revalidates_version_member() -> None: + """A retained package missing version.xml cannot become parsed.""" + attachment = _pending_hwpx_attachment(_hwpx_payload(include_version=False)) + + result = await process_pending_attachment( + session=object(), + attachment=attachment, + config_resolver=_must_not_resolve_provider, + request_fn=_must_not_call_newsdom, + ) + + assert result == RESULT_FAILED + assert attachment.parse_status == HWPX_FAILED_STATUS + assert attachment.parse_error_code == "invalid_pending_payload" + + @pytest.mark.asyncio async def test_orphan_pending_hwpx_attachment_uses_hwpx_failure_status() -> None: """An orphan HWPX row fails visibly without being mislabeled as PDF.""" From cb42955543a3f82c6003db459624d52b39551606 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 5 Sep 2026 14:41:27 +0900 Subject: [PATCH 10/11] fix(hwpx): preserve strict worker provenance contract --- backend/services/hwpx_recognition.py | 3 +++ backend/services/newsdom_worker.py | 24 ------------------------ backend/tests/test_newsdom_worker.py | 16 ++++++++++++---- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/backend/services/hwpx_recognition.py b/backend/services/hwpx_recognition.py index e221af830..a5b6292b9 100644 --- a/backend/services/hwpx_recognition.py +++ b/backend/services/hwpx_recognition.py @@ -280,6 +280,9 @@ def recognize_hwpx_package( paragraph_count += len(paragraphs) sections.append(PdfDomSection(heading="", paragraphs=paragraphs)) + if paragraph_count == 0: + raise ValueError("HWPX package contains no readable paragraph text") + source_content_hash = hashlib.sha256(payload).hexdigest() parse_result = parse_pdf_dom( source_kind=source_kind, diff --git a/backend/services/newsdom_worker.py b/backend/services/newsdom_worker.py index 502f9d8d5..951df16f9 100644 --- a/backend/services/newsdom_worker.py +++ b/backend/services/newsdom_worker.py @@ -32,7 +32,6 @@ from db.session import AsyncSessionLocal from services.attachment_parser import ( HWP_CONVERSION_PENDING_STATUS, - HWPX_XML_PACKAGE_PENDING_STATUS, decode_deferred_attachment_payload, ) from services.content_graph import ParseResult @@ -353,29 +352,6 @@ async def process_pending_attachment( ) return RESULT_FAILED - if attachment.parse_status == HWPX_XML_PACKAGE_PENDING_STATUS: - try: - records = recognize_hwpx( - hwpx_bytes=deferred_bytes, - source_record_uid=f"attachment-{attachment.id}", - display_name=attachment.filename or "", - ) - except HwpxRecognitionError as exc: - attachment.parse_status = PDF_DOM_RECOGNITION_FAILED_STATUS - attachment.parse_error_code = "hwpx_recognition_failed" - logger.warning( - "HWPX attachment %s recognition failed: %s", - getattr(attachment, "id", "?"), - exc, - ) - return RESULT_FAILED - apply_hwpx_recognition_to_attachment( - email=email, - attachment=attachment, - records=records, - ) - return RESULT_RECOGNIZED - pdf_bytes = deferred_bytes config = await config_resolver(session, email.organization_id) diff --git a/backend/tests/test_newsdom_worker.py b/backend/tests/test_newsdom_worker.py index fd5bcb18b..f904d37df 100644 --- a/backend/tests/test_newsdom_worker.py +++ b/backend/tests/test_newsdom_worker.py @@ -29,6 +29,8 @@ RESULT_FAILED = newsdom_worker_module.RESULT_FAILED RESULT_PENDING = newsdom_worker_module.RESULT_PENDING RESULT_RECOGNIZED = newsdom_worker_module.RESULT_RECOGNIZED +HWPX_FAILED_STATUS = newsdom_worker_module.HWPX_FAILED_STATUS +HWPX_PARSED_STATUS = newsdom_worker_module.HWPX_PARSED_STATUS process_pending_attachment = newsdom_worker_module.process_pending_attachment process_pending_document = newsdom_worker_module.process_pending_document @@ -101,7 +103,13 @@ def _hwpx_payload(*, include_text: bool = True) -> bytes: compress_type=zipfile.ZIP_STORED, ) archive.writestr("version.xml", "") - archive.writestr("Contents/content.hpf", "") + archive.writestr( + "Contents/content.hpf", + """ + + + """, + ) text = ( "보안 정책" "원본 근거를 보존합니다." @@ -416,7 +424,7 @@ async def test_hwpx_pending_attachment_is_extracted_and_graph_landed(): ) assert result == RESULT_RECOGNIZED - assert attachment.parse_status == "parsed" + assert attachment.parse_status == HWPX_PARSED_STATUS assert attachment.parser_key == "hwpx" assert attachment.content == "보안 정책\n\n원본 근거를 보존합니다." assert len(attachment.content_nodes) == 4 @@ -455,8 +463,8 @@ async def test_hwpx_pending_attachment_records_safe_failure_for_empty_text(): result = await process_pending_attachment(session=object(), attachment=attachment) assert result == RESULT_FAILED - assert attachment.parse_status == PDF_DOM_RECOGNITION_FAILED_STATUS - assert attachment.parse_error_code == "hwpx_recognition_failed" + assert attachment.parse_status == HWPX_FAILED_STATUS + assert attachment.parse_error_code == "recognition_failed" @pytest.mark.asyncio From 2915d3f9bb16a6cc6bb01f2ae0f418874a599592 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 6 Sep 2026 10:00:44 +0900 Subject: [PATCH 11/11] docs(hwpx): inherit admission-worker alignment --- docs/doctoring/hwp-hwpx-attachment-recognition.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/doctoring/hwp-hwpx-attachment-recognition.md b/docs/doctoring/hwp-hwpx-attachment-recognition.md index 22061fa91..e7d9aa489 100644 --- a/docs/doctoring/hwp-hwpx-attachment-recognition.md +++ b/docs/doctoring/hwp-hwpx-attachment-recognition.md @@ -32,13 +32,15 @@ enterprise document text searchable without model dependence. - HWPX content types are recognized as deferred OWPML XML packages. - HWPX bytes must be a bounded single-disk ZIP package with one unambiguous `mimetype` member whose exact content is `application/hwp+zip`, a `version.xml` - member, and either package-manifest or section evidence. + member, and at least one canonical `Contents/sectionN.xml` member. A package + manifest by itself is not sufficient admission evidence because the worker + cannot materialize a sectionless package. - The importer validates the end-of-central-directory entry count and directory size before Python materializes ZIP members, then bounds aggregate member-name bytes and the tiny `mimetype` payload before reading it. - Duplicate `mimetype` members, wrong signature text, encrypted signature - members, unsupported ZIP structures, malformed ZIP metadata, and exceeded - limits fail closed as `invalid_hwpx_payload`. + members, unsupported ZIP structures, malformed ZIP metadata, sectionless + packages, and exceeded limits fail closed as `invalid_hwpx_payload`. - Import does not decompress document sections, extract files, execute active content, or fetch external resources. Later workers repeat path, XML, encryption, resource, and expansion validation before extraction. @@ -142,6 +144,12 @@ central-directory bytes, aggregate name bytes, and signature-member bytes. Commit `b737ae83c94ee8a5aaf9c22a8239056e26ffe029` then implemented the bounded end-of-central-directory preflight and exact signature validation. +A later review found that import admission still accepted a manifest-only HWPX +package that the worker must reject because no canonical section exists. RED +commit `44a268b988f9a3092368bd774a26582647e319a9` adds that public-boundary +regression; causal fix `4281904b438ac50c2d6c40d14207119c383227a8` requires +section evidence before deferred queue admission. + The initial HWP slice likewise admitted any OLE Compound File if the caller supplied an HWP extension or media type. Commit `d97281ce7f452a10b0a5c76718d37d126958a4ae` added regressions proving that an