feat(tools): harden contact and text measurement contracts

backend/api/tools.py

@@ -18,6 +18,7 @@
     _resolve_global_addresses,
 )
 from services.llm_provider_urls import build_pinned_https_async_client
+from services.text_structure_statistics import measure_text_structure
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel, Field
 
@@ -387,23 +388,36 @@ def _parameter_matches_type(value: Any, expected_type: str) -> bool:
     tone_analyzer_handler,
 )
 
-async def text_analyzer_handler(params: Dict[str, Any]) -> Dict[str, int]:
-    text = params.get("text", "")
-    char_count = len(text)
-    char_count_no_spaces = len(
-        text.replace(" ", "").replace("\n", "").replace("\r", "").replace("\t", "")
-    )
+async def text_analyzer_handler(params: Dict[str, Any]) -> Dict[str, Any]:
+    """Return descriptive text counts while preserving documented legacy aliases."""
+    statistics = measure_text_structure(params.get("text", ""))
+    legacy_aliases = {
+        "char_count": "character_count",
+        "char_count_no_spaces": "non_whitespace_character_count",
+        "word_count": "whitespace_token_count",
+    }
     return {
-        "char_count": char_count,
-        "char_count_no_spaces": char_count_no_spaces,
-        "word_count": len(text.split()),
+        "character_count": statistics.character_count,
+        "non_whitespace_character_count": statistics.non_whitespace_character_count,
+        "whitespace_token_count": statistics.whitespace_token_count,
+        "terminal_punctuation_run_count": statistics.terminal_punctuation_run_count,
+        "segmentation_contract": statistics.segmentation_contract,
+        "legacy_aliases": legacy_aliases,
+        "char_count": statistics.character_count,
+        "char_count_no_spaces": statistics.non_whitespace_character_count,
+        "word_count": statistics.whitespace_token_count,
     }
 
 registry.register(
     ToolInfo(
         code="text_analyzer",
         name="텍스트 분석기 (Text Analyzer)",
-        description="텍스트의 글자 수, 단어 수, 공백 제외 글자 수를 분석합니다.",
+        description=(
+            "텍스트의 문자 수, Unicode 공백 제외 문자 수, 공백 구분 토큰 수, "
+            "종결 문장부호 연속 구간 수를 계산합니다. 기존 char_count, "
+            "char_count_no_spaces, word_count는 호환 별칭이며 단어·문장 수를 "
+            "뜻하지 않습니다."
+        ),
         category="유틸리티",
         parameters={"text": "string"},
     ),

backend/services/contact_information_extractor.py

@@ -0,0 +1,101 @@
+"""Pure, bounded contact-information extraction for explicitly supplied text."""
+
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+MAX_CONTACT_INPUT_CHARS = 100_000
+_EMAIL_ATOM = r"A-Za-z0-9!#$%&'*+/=?^_`{|}~"
+_ASCII_EMAIL_PATTERN = re.compile(
+    rf"(?<![{_EMAIL_ATOM}.-])"
+    rf"[{_EMAIL_ATOM}-]+(?:\.[{_EMAIL_ATOM}-]+)*@"
+    rf"(?:[A-Za-z0-9](?:[A-Za-z0-9-]{{0,61}}[A-Za-z0-9])?\.)+"
+    r"[A-Za-z]{2,63}(?![A-Za-z0-9-])"
+)
+_UNICODE_EMAIL_PATTERN = re.compile(
+    rf"(?<![\w{_EMAIL_ATOM}.-])"
+    rf"[\w{_EMAIL_ATOM}-]+(?:\.[\w{_EMAIL_ATOM}-]+)*@"
+    r"(?:[^\W_](?:(?:[^\W_]|-){0,61}[^\W_])?\.)+"
+    r"[^\W_]{2,63}(?![\w-])"
+)
+_PHONE_PATTERN = re.compile(
+    r"(?<!\d)(?:(?:\+82[ .-]?10|010)[ .-]?\d{3,4}[ .-]?\d{4}"
+    r"|\d{2,3}-\d{3,4}-\d{4}"
+    r"|(?:\+?1[ .-]?)?(?:\(\d{3}\)|\d{3})[ .-]?\d{3}[ .-]?\d{4})(?!\d)"
+)
+
+
+@dataclass(frozen=True)
+class ContactInformationMatches:
+    """Contact-like values found in source order without persistence or normalization."""
+
+    email_addresses: tuple[str, ...]
+    phone_numbers: tuple[str, ...]
+
+
+def _domain_identity(domain_part: str) -> tuple[tuple[str, str], ...]:
+    """Compare ASCII DNS labels case-insensitively and Unicode U-labels exactly."""
+    return tuple(
+        ("ascii", label.casefold()) if label.isascii() else ("unicode", label)
+        for label in domain_part.split(".")
+    )
+
+
+def _mailbox_identity(
+    value: str,
+) -> tuple[str, tuple[tuple[str, str], ...]]:
+    """Return exact local-part plus loss-avoiding domain identity for one mailbox."""
+    local_part, separator, domain_part = value.rpartition("@")
+    if not separator:
+        return value, ()
+    return local_part, _domain_identity(domain_part)
+
+
+def _deduplicate_matches(
+    matches: Iterable[re.Match[str]], *, preserve_mailbox_local_part: bool = False
+) -> tuple[str, ...]:
+    """Return unique match values in source order under the requested identity rule."""
+    ordered_matches = sorted(matches, key=lambda match: (match.start(), match.end()))
+    values: list[str] = []
+    seen: set[object] = set()
+    for match in ordered_matches:
+        value = match.group(0)
+        identity: object
+        if preserve_mailbox_local_part:
+            identity = _mailbox_identity(value)
+        else:
+            identity = value.casefold()
+        if identity in seen:
+            continue
+        seen.add(identity)
+        values.append(value)
+    return tuple(values)
+
+
+def extract_contact_information(text: str) -> ContactInformationMatches:
+    """Extract bounded email/phone patterns from caller-supplied text with no side effects.
+
+    Mailbox local-parts are preserved and compared exactly. ASCII DNS/A-label domain
+    labels are compared case-insensitively, while non-ASCII IDNA U-labels are compared
+    exactly so irreversible Unicode case folding cannot collapse distinct domain names.
+    The function does not perform IDNA A-label/U-label conversion, normalization, or
+    deliverability validation. It does not log, persist, index, or transmit the input or
+    extracted PII. Callers remain responsible for authorization, purpose limitation,
+    retention, and downstream disclosure. Pattern matching is intentionally bounded and
+    is not a claim that every international email address or telephone numbering plan is
+    recognized.
+    """
+    if len(text) > MAX_CONTACT_INPUT_CHARS:
+        raise ValueError(
+            f"Contact text must not exceed {MAX_CONTACT_INPUT_CHARS} characters"
+        )
+
+    email_matches = list(_ASCII_EMAIL_PATTERN.finditer(text))
+    email_matches.extend(_UNICODE_EMAIL_PATTERN.finditer(text))
+    phone_matches = _PHONE_PATTERN.finditer(text)
+    return ContactInformationMatches(
+        email_addresses=_deduplicate_matches(
+            email_matches, preserve_mailbox_local_part=True
+        ),
+        phone_numbers=_deduplicate_matches(phone_matches),
+    )

backend/services/text_structure_statistics.py

@@ -0,0 +1,47 @@
+"""Transparent text-structure counts without an inferred readability scale."""
+
+import re
+from dataclasses import dataclass
+
+MAX_TEXT_STRUCTURE_INPUT_CHARS = 100_000
+_TERMINAL_PUNCTUATION_RUN_PATTERN = re.compile(r"[.!?。！？．]+")
+SEGMENTATION_CONTRACT = "whitespace-and-terminal-punctuation-runs-v2"
+
+
+@dataclass(frozen=True)
+class TextStructureStatistics:
+    """Descriptive counts whose segmentation rule is explicit in the result."""
+
+    character_count: int
+    non_whitespace_character_count: int
+    whitespace_token_count: int
+    terminal_punctuation_run_count: int
+    segmentation_contract: str = SEGMENTATION_CONTRACT
+
+
+def measure_text_structure(text: str) -> TextStructureStatistics:
+    """Measure source text without presenting punctuation runs as sentence counts.
+
+    Tokens are whitespace-delimited and terminal punctuation is counted as contiguous
+    runs. The latter deliberately does not claim sentence segmentation: periods inside
+    decimals, hostnames, abbreviations, and similar text are still punctuation runs.
+    These rules therefore expose transparent source statistics rather than a
+    locale-invariant readability or sentence construct, particularly for CJK and other
+    scripts whose lexical segmentation is not represented by spaces.
+    """
+    if len(text) > MAX_TEXT_STRUCTURE_INPUT_CHARS:
+        raise ValueError(
+            "Text structure input must not exceed "
+            f"{MAX_TEXT_STRUCTURE_INPUT_CHARS} characters"
+        )
+
+    return TextStructureStatistics(
+        character_count=len(text),
+        non_whitespace_character_count=sum(
+            1 for character in text if not character.isspace()
+        ),
+        whitespace_token_count=len(text.split()),
+        terminal_punctuation_run_count=len(
+            _TERMINAL_PUNCTUATION_RUN_PATTERN.findall(text)
+        ),
+    )

backend/tests/test_text_analysis_services.py

@@ -0,0 +1,98 @@
+import pytest
+
+from services.contact_information_extractor import extract_contact_information
+from services.text_structure_statistics import measure_text_structure
+
+
+def test_contact_information_preserves_mailbox_identity_and_source_order() -> None:
+    result = extract_contact_information(
+        "Primary: Ada.Example@example.com, same Ada.Example@EXAMPLE.com, "
+        "distinct ada.example@example.com, mobile +82 10-1234-5678, "
+        "desk (415) 555-0123."
+    )
+
+    assert result.email_addresses == (
+        "Ada.Example@example.com",
+        "ada.example@example.com",
+    )
+    assert result.phone_numbers == ("+82 10-1234-5678", "(415) 555-0123")
+
+
+def test_contact_information_preserves_unicode_local_part_case() -> None:
+    result = extract_contact_information("문의: Üser@예시.한국 / üser@예시.한국")
+
+    assert result.email_addresses == ("Üser@예시.한국", "üser@예시.한국")
+    assert result.phone_numbers == ()
+
+
+def test_contact_information_does_not_casefold_distinct_unicode_domain_labels() -> None:
+    result = extract_contact_information(
+        "IDNA: Ada@faß.de, distinct Ada@fass.de, "
+        "ASCII-label case Ada@EXAMPLE.한국 / Ada@example.한국"
+    )
+
+    assert result.email_addresses == (
+        "Ada@faß.de",
+        "Ada@fass.de",
+        "Ada@EXAMPLE.한국",
+    )
+
+
+def test_contact_information_supports_unicode_email_without_normalizing_output() -> None:
+    result = extract_contact_information("문의: 사용자@예시.한국")
+
+    assert result.email_addresses == ("사용자@예시.한국",)
+    assert result.phone_numbers == ()
+
+
+def test_contact_information_rejects_incidental_long_numbers() -> None:
+    result = extract_contact_information(
+        "invoice 2026090512345678 and account 12345678901234567890"
+    )
+
+    assert result.phone_numbers == ()
+
+
+def test_contact_information_rejects_oversized_input() -> None:
+    with pytest.raises(ValueError, match="must not exceed 100000 characters"):
+        extract_contact_information("x" * 100_001)
+
+
+def test_text_structure_statistics_are_descriptive_not_readability_scores() -> None:
+    result = measure_text_structure("One short sentence. Two words!")
+
+    assert result.character_count == 30
+    assert result.non_whitespace_character_count == 26
+    assert result.whitespace_token_count == 5
+    assert result.terminal_punctuation_run_count == 2
+    assert result.segmentation_contract == "whitespace-and-terminal-punctuation-runs-v2"
+    assert not hasattr(result, "sentence_boundary_count")
+    assert not hasattr(result, "readability_score")
+
+
+def test_text_structure_statistics_do_not_label_punctuation_runs_as_sentences() -> None:
+    result = measure_text_structure("Version 3.14... https://example.com/a.")
+
+    assert result.terminal_punctuation_run_count == 4
+    assert not hasattr(result, "sentence_boundary_count")
+
+
+def test_text_structure_statistics_keep_cjk_contract_explicit() -> None:
+    result = measure_text_structure("첫 문장입니다. 次の文です。")
+
+    assert result.whitespace_token_count == 3
+    assert result.terminal_punctuation_run_count == 2
+
+
+def test_text_structure_statistics_handle_empty_input() -> None:
+    result = measure_text_structure("")
+
+    assert result.character_count == 0
+    assert result.non_whitespace_character_count == 0
+    assert result.whitespace_token_count == 0
+    assert result.terminal_punctuation_run_count == 0
+
+
+def test_text_structure_statistics_reject_oversized_input() -> None:
+    with pytest.raises(ValueError, match="must not exceed 100000 characters"):
+        measure_text_structure("x" * 100_001)

backend/tests/test_text_analyzer_measurement_contract.py

@@ -0,0 +1,35 @@
+import pytest
+
+from api.tools import registry, text_analyzer_handler
+
+
+@pytest.mark.asyncio
+async def test_text_analyzer_exposes_descriptive_measurement_contract() -> None:
+    result = await text_analyzer_handler(
+        {"text": "A\u00a0B 3.14... https://example.com/a."}
+    )
+
+    assert result["character_count"] == 34
+    assert result["non_whitespace_character_count"] == 31
+    assert result["whitespace_token_count"] == 4
+    assert result["terminal_punctuation_run_count"] == 4
+    assert result["segmentation_contract"] == (
+        "whitespace-and-terminal-punctuation-runs-v2"
+    )
+    assert result["legacy_aliases"] == {
+        "char_count": "character_count",
+        "char_count_no_spaces": "non_whitespace_character_count",
+        "word_count": "whitespace_token_count",
+    }
+    assert result["char_count"] == result["character_count"]
+    assert result["char_count_no_spaces"] == result["non_whitespace_character_count"]
+    assert result["word_count"] == result["whitespace_token_count"]
+
+
+def test_text_analyzer_catalog_discloses_legacy_alias_semantics() -> None:
+    tool = registry.get("text_analyzer")
+
+    assert tool is not None
+    assert "공백 구분 토큰 수" in tool.description
+    assert "호환 별칭" in tool.description
+    assert "단어·문장 수를 뜻하지 않습니다" in tool.description

docs/doctoring/contact-information-mailbox-identity.md

@@ -0,0 +1,56 @@
+# Contact-information mailbox identity
+
+## Problem
+
+`contact_information_extractor` originally deduplicated every extracted value with `casefold()`. That can collapse distinct email mailboxes whose local-parts differ only by case. RFC 5321 requires SMTP implementations to preserve mailbox local-part case, while ASCII DNS labels use case-insensitive comparison. RFC 6531 extends mailbox syntax for SMTPUTF8 without replacing the RFC 5321 local-part rule.
+
+The first repair therefore changed mailbox identity to exact local-part plus a case-insensitive domain. A follow-up review found that the domain rule was still too broad for internationalized domain labels: Python `casefold()` maps some Unicode strings irreversibly, including German sharp S (`ß`) to `ss`. IDNA2008 does not define U-label equivalence by arbitrary Unicode case folding. RFC 5891 requires A-labels to compare as case-insensitive ASCII and U-labels to compare as-is, without case folding or other intermediate steps; RFC 5894 explicitly calls out the irreversible `ß` → `ss` mapping as a reason IDNA2008 moved away from the IDNA2003 folding model.
+
+That matters for extraction because `Ada@faß.de` and `Ada@fass.de` must not be silently collapsed merely because a generic Unicode fold produces the same string. The defect is data loss, not a presentation difference.
+
+## Decision
+
+Naruon keeps the extracted mailbox representation unchanged and uses a loss-avoiding identity rule:
+
+- local-part: exact comparison;
+- ASCII domain labels, including A-label/LDH representations: case-insensitive comparison;
+- non-ASCII U-labels: exact comparison, with no Unicode case folding.
+
+The comparison is label-by-label so `Ada@EXAMPLE.한국` and `Ada@example.한국` deduplicate on the ASCII label while the identical `한국` U-label is preserved exactly. `Ada@faß.de` and `Ada@fass.de` remain distinct.
+
+This pure extractor deliberately does not perform IDNA A-label/U-label conversion, NFC normalization, provider-specific alias canonicalization, MX/deliverability lookup, or account-directory resolution. That means semantically equivalent A-label and U-label spellings may remain as separate extracted representations. For a non-normalizing extraction boundary, retaining a duplicate is preferable to deleting a potentially distinct mailbox. Any future canonicalization must use an explicit validated IDNA profile and its own migration evidence rather than generic Unicode folding.
+
+The extraction boundary remains purpose-limited: it operates only on caller-supplied text, does not log, persist, index, or transmit the input or extracted PII, and leaves authorization, retention, and downstream disclosure to the caller.
+
+## Alternatives rejected
+
+Case-folding the complete mailbox was rejected because it can merge local-parts that SMTP requires implementations to preserve. Case-folding the complete Unicode domain was also rejected because IDNA2008 U-label comparison is exact and generic folding is not reversible.
+
+Treating every domain label as exact was rejected because ASCII DNS labels are case-insensitive and would emit avoidable duplicates such as `Ada@example.com` and `Ada@EXAMPLE.com`. Automatically converting between A-label and U-label forms was deferred because this service currently performs bounded pattern extraction, not IDNA validation or normalization; adding conversion here would silently broaden the contract.
+
+Provider-specific rules such as local-part lowercasing, dot removal, plus-tag stripping, or account-directory lookup were rejected because they are not portable mailbox identity rules and would introduce external provider semantics into a pure service.
+
+## Executable traceability
+
+- Local-part RED: `1bf449a078580cc6545f862c07a8d68bf968b005` requires domain-only ASCII case changes to deduplicate while preserving local-part case.
+- Local-part fix: `fd629039b1d4a4b36d8ab5cdb1bdbc8e4e787c1b` introduces exact local-part comparison.
+- IDNA U-label RED: `2d6da9a86c4d4a9a231cfd96884482e5d5bd9b1d` requires `faß.de` and `fass.de` to remain distinct while preserving case-insensitive comparison of an ASCII label in a mixed internationalized domain.
+- IDNA U-label fix: `f7b9974ba0e267d45dae82e3626f6a33a7e16d21` compares ASCII domain labels case-insensitively and non-ASCII labels exactly.
+- Regression owner: `backend/tests/test_text_analysis_services.py`.
+- Production owner: `backend/services/contact_information_extractor.py`.
+
+These commits establish RED-before-fix source provenance. Current-head pytest, repository checks, independent review, protected merge, and release remain separate evidence gates.
+
+## References
+
+Housley, R. (2024). *Internationalization updates to RFC 5280* (RFC 9549). RFC Editor. https://doi.org/10.17487/RFC9549
+
+Klensin, J. (2008). *Simple Mail Transfer Protocol* (RFC 5321). RFC Editor. https://doi.org/10.17487/RFC5321
+
+Klensin, J. C. (2010a). *Internationalized domain names for applications (IDNA): Definitions and document framework* (RFC 5890). RFC Editor. https://doi.org/10.17487/RFC5890
+
+Klensin, J. C. (2010b). *Internationalized domain names in applications (IDNA): Protocol* (RFC 5891). RFC Editor. https://doi.org/10.17487/RFC5891
+
+Klensin, J. C. (2010c). *Internationalized domain names for applications (IDNA): Background, explanation, and rationale* (RFC 5894). RFC Editor. https://doi.org/10.17487/RFC5894
+
+Yao, J., & Mao, W. (2012). *SMTP extension for internationalized email* (RFC 6531). RFC Editor. https://doi.org/10.17487/RFC6531