From ed8e51f980fd40405a6b473b59e0535a54ee6f13 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 15 Sep 2026 12:11:18 +0900 Subject: [PATCH] feat(quality): require the prose a Korean customer reads to be Korean The gate checked everything about a report except the language it was written in. Measured on `main`: a report whose section titles, summaries, opportunities, cautions, actions, executive summary and practical skills were all replaced with English passed `validate_report` with zero issues. Only the disclaimer and the relationships section were incidentally protected, because those two checks happen to search for Korean words. The drift path is concrete rather than hypothetical. `generate`'s schema-repair turn appends the pydantic error text and the full JSON Schema, both English, to the conversation and then asks for the complete answer again. `_reader_prose` collects exactly the strings `render_html` and `render_pdf` put in front of a customer, each paired with its document path so a violation names the field. Internal values are absent by construction: evidence notes, the fingerprint, the model identity, prompt versions and quality notes. So is `subject_name`, because a customer whose name is written in Latin script is not a defect in prose this product wrote. The check is per string rather than a whole-document ratio, so one section that drifted is caught instead of being averaged away by seven that did not. Two fixtures head their sections with the dictionary key, which the HTML renderer emits as an `

` and a reader would see as a heading reading "natal". Both now carry Korean titles, which is what a real report contains. The change sits well away from the hunks `refactor/orchestrator-free-runtime` has in `tests/test_analysis.py`, at lines 7, 15, 49 and 88. Co-Authored-By: Claude Opus 5 --- CHANGELOG.md | 2 + src/four_pillars/quality.py | 46 ++++++++++++++++ tests/test_analysis.py | 15 +++++- tests/test_quality.py | 14 ++++- tests/test_report_language_contract.py | 73 ++++++++++++++++++++++++++ 5 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 tests/test_report_language_contract.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9feef31..69f91b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format follows Keep a Changelog, and release numbers follow Semantic Version ### Added +- The report quality gate now requires the prose a customer reads to contain Korean. Every other customer-visible property was checked before publication while the language was not, so a report whose body had drifted entirely into English passed with zero issues. The schema-repair turn appends an English pydantic error and the full English JSON Schema to the conversation, which is a known way to pull a model's output language across. The subject's own name is excluded, so a customer whose name is written in Latin script is never a violation. + - Independent KASI/NAOJ 2026 golden fixtures for all twelve month-changing solar terms, enforcing a two-minute timing budget and five-minute year/month pillar transition checks without network or test-only ephemeris dependencies. - Offline authority-fixture governance that detects missing evidence, provenance, tolerance, traceability, and calculation-version contracts in the hourly product-gap audit. diff --git a/src/four_pillars/quality.py b/src/four_pillars/quality.py index 918d460..48b7851 100644 --- a/src/four_pillars/quality.py +++ b/src/four_pillars/quality.py @@ -62,6 +62,43 @@ def _all_text(report: ReportDocument) -> str: return json.dumps(payload, ensure_ascii=False) +HANGUL = re.compile(r"[\uac00-\ud7a3]") +"""Match one precomposed Hangul syllable. + +The subject's own name is excluded from the language check, so a customer whose +name is written in Latin script is never treated as a violation. Every other +string below is prose this product wrote and a Korean reader has to read. +""" + + +def _reader_prose(report: ReportDocument) -> list[tuple[str, str]]: + """Return every reader-visible string with the document path that produced it. + + The set is exactly what ``render_html`` and ``render_pdf`` put in front of a + customer. Internal values such as evidence notes, the fingerprint, the model + identity, prompt versions, and quality notes are deliberately absent, as is + ``subject_name``. + """ + prose: list[tuple[str, str]] = [ + ("title", report.title), + ("executive_summary", report.executive_summary), + ("disclaimer", report.disclaimer), + ] + for key, section in report.sections.items(): + prose.append((f"sections.{key}.title", section.title)) + prose.append((f"sections.{key}.summary", section.summary)) + for field in ("opportunities", "cautions", "actions"): + for index, value in enumerate(getattr(section, field)): + prose.append((f"sections.{key}.{field}[{index}]", value)) + for index, skill in enumerate(report.practical_skills): + prose.append((f"practical_skills[{index}].name", skill.name)) + prose.append((f"practical_skills[{index}].purpose", skill.purpose)) + prose.append((f"practical_skills[{index}].when_to_use", skill.when_to_use)) + for step, value in enumerate(skill.steps): + prose.append((f"practical_skills[{index}].steps[{step}]", value)) + return prose + + def validate_report( report: ReportDocument, expected_fingerprint: str, @@ -105,6 +142,15 @@ def validate_report( "sections.relationships", ) ) + for path, value in _reader_prose(report): + if HANGUL.search(value) is None: + issues.append( + QualityIssue( + "foreign_language", + "한국어 독자에게 전달되는 문장에 한글이 없습니다.", + path, + ) + ) text = _all_text(report) if allowed_pillars is not None: for mentioned in sorted(set(PILLAR_PATTERN.findall(text)) - allowed_pillars): diff --git a/tests/test_analysis.py b/tests/test_analysis.py index e6ea297..55f01b2 100644 --- a/tests/test_analysis.py +++ b/tests/test_analysis.py @@ -18,6 +18,17 @@ REQUIRED = ("natal", "daewoon", "annual", "monthly", "work", "money", "relationships", "daily_rhythm") +# A real report heads each section in Korean; the identifier is the dictionary key. +TITLES_KO = { + "natal": "타고난 기질", + "daewoon": "대운의 흐름", + "annual": "올해의 세운", + "monthly": "이달의 월운", + "work": "일과 역할", + "money": "돈과 자원", + "relationships": "가까운 관계", + "daily_rhythm": "하루의 리듬", +} def section(title: str) -> ReportSection: @@ -67,7 +78,7 @@ async def generate(self, *, system_prompt, user_payload, response_model, **kwarg summary = "시키는 대로 책임지는 사람입니다." if self.invalid_synthesis else "혜지 님은 책임 범위와 지원 조건을 확인합니다." return SynthesisDraft( executive_summary=summary, - sections={key: section(key) for key in REQUIRED}, + sections={key: section(TITLES_KO[key]) for key in REQUIRED}, disclaimer="이 보고서는 전통 명리학의 상징 자료입니다. 의학·법률·재정 판단은 실제 정보와 전문가 의견을 우선합니다.", ), trace if response_model is ReportDocument: @@ -77,7 +88,7 @@ async def generate(self, *, system_prompt, user_payload, response_model, **kwarg title="최혜지 사주 보고서", executive_summary="혜지 님은 책임 범위와 지원 조건을 확인합니다.", calculation_fingerprint=fingerprint, - sections={key: section(key) for key in REQUIRED}, + sections={key: section(TITLES_KO[key]) for key in REQUIRED}, practical_skills=[ PracticalSkill( name="주간 검토", diff --git a/tests/test_quality.py b/tests/test_quality.py index d644785..0e51005 100644 --- a/tests/test_quality.py +++ b/tests/test_quality.py @@ -18,10 +18,22 @@ ) +SECTION_TITLES_KO = { + "natal": "타고난 기질", + "daewoon": "대운의 흐름", + "annual": "올해의 세운", + "monthly": "이달의 월운", + "work": "일과 역할", + "money": "돈과 자원", + "relationships": "가까운 관계", + "daily_rhythm": "하루의 리듬", +} + + def _section(key: str) -> ReportSection: summary = "가까운 관계에서는 신뢰와 협력을 구체적인 약속으로 키울 수 있습니다." if key == "relationships" else "계산 근거를 생활의 조건부 판단 기준으로 설명합니다." return ReportSection( - title=key, + title=SECTION_TITLES_KO[key], summary=summary, opportunities=["현실적인 조건을 확인하면 안정적인 성과 가능성이 있습니다."], cautions=["미래 사건을 단정하지 말고 실제 자료와 대화를 먼저 확인합니다."], diff --git a/tests/test_report_language_contract.py b/tests/test_report_language_contract.py new file mode 100644 index 0000000..c08c117 --- /dev/null +++ b/tests/test_report_language_contract.py @@ -0,0 +1,73 @@ +"""Require the prose a Korean customer reads to actually be Korean. + +Every other customer-visible property is checked before a report is published: +the required sections exist, each has an opportunity, a caution and an action, +no ungrounded 간지 appears, forbidden and vague copy is rejected, and the +disclaimer carries its required terms. Nothing checked the language. + +That is not theoretical. The schema-repair turn in `nim.py` appends the pydantic +error text and the full JSON Schema, both English, to the conversation and then +asks for the complete answer again, which is a known way to pull a model's +output language across. +""" + +from __future__ import annotations + +import pytest +from test_quality import FINGERPRINT, valid_report + +from four_pillars.quality import validate_report + +CODE = "foreign_language" + + +def _english_body(report): + """Replace the reader-visible prose with English, leaving identifiers alone.""" + for key, section in report.sections.items(): + section.title = key.replace("_", " ").title() + section.summary = "This section explains the calculation as a judgement aid." + section.opportunities = ["Confirming conditions makes a stable outcome possible."] + section.cautions = ["Do not treat future events as settled."] + section.actions = ["Record fact, impact, alternative, request."] + report.executive_summary = "A self-review document from traditional symbolism." + for skill in report.practical_skills: + skill.name = "Weekly review" + skill.purpose = "Detect overload early." + skill.steps = ["Write next week's three key events."] + skill.when_to_use = "When commitments rise together." + return report + + +def test_an_english_report_is_rejected() -> None: + """The whole body in English must not reach a Korean customer unchallenged.""" + codes = [issue.code for issue in validate_report(_english_body(valid_report()), FINGERPRINT)] + + assert CODE in codes + + +def test_the_korean_fixture_still_passes() -> None: + """A report written in Korean gains nothing from this check.""" + assert validate_report(valid_report(), FINGERPRINT) == [] + + +@pytest.mark.parametrize( + "field", + ["title", "summary", "opportunities", "cautions", "actions"], +) +def test_one_english_field_in_one_section_is_enough_to_fail(field: str) -> None: + """A per-report ratio would miss this; the check is per reader-visible string.""" + report = valid_report() + english = "This single field drifted out of Korean." + section = report.sections["work"] + setattr(section, field, english if field in {"title", "summary"} else [english]) + + assert CODE in [issue.code for issue in validate_report(report, FINGERPRINT)] + + +def test_a_latin_script_customer_name_is_not_a_violation() -> None: + """The subject's own name is theirs, not prose this product wrote.""" + report = valid_report() + report.subject_name = "Alex Marchetti" + report.title = f"{report.subject_name} 사주 보고서" + + assert CODE not in [issue.code for issue in validate_report(report, FINGERPRINT)]