diff --git a/backend/services/ui_localization_policy.py b/backend/services/ui_localization_policy.py new file mode 100644 index 000000000..a0821ae6f --- /dev/null +++ b/backend/services/ui_localization_policy.py @@ -0,0 +1,565 @@ +"""Pure policy primitives for Naruon's UI Localization Catalog. + +This module intentionally owns no persistence, HTTP routing, browser runtime, or +machine-translation behavior. It defines the fail-closed identities and locale +selection rules that later boundaries consume after their canonical owner +prerequisites are integrated. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from string import Formatter +from typing import Literal, cast + +SupportedLocaleCode = Literal["ko", "en", "ja", "zh", "vi", "es", "de", "fr"] +LocaleSelectionSource = Literal[ + "persisted_preference", + "session_preference", + "accept_language", + "product_default", +] +UiLocalizationValidationCode = Literal[ + "ui_locale_unsupported", + "ui_locale_input_invalid", + "ui_accept_language_invalid", + "ui_screen_key_invalid", + "ui_message_key_invalid", + "ui_translation_input_invalid", + "ui_placeholder_schema_invalid", + "ui_translation_placeholder_mismatch", +] + +SUPPORTED_LOCALE_CODES: tuple[SupportedLocaleCode, ...] = ( + "ko", + "en", + "ja", + "zh", + "vi", + "es", + "de", + "fr", +) +DEFAULT_LOCALE_CODE: SupportedLocaleCode = "ko" +_SUPPORTED_LOCALE_SET = frozenset(SUPPORTED_LOCALE_CODES) +_MAX_LOCALE_TAG_CHARS = 128 +_MAX_ACCEPT_LANGUAGE_CHARS = 8192 +_MAX_ACCEPT_LANGUAGE_MEMBERS = 64 +_MAX_ACCEPT_LANGUAGE_EMPTY_MEMBERS = 32 +_MAX_SCREEN_KEY_CHARS = 128 +_MAX_MESSAGE_KEY_CHARS = 128 +_MAX_TRANSLATION_MESSAGE_CHARS = 16_384 +_MAX_PLACEHOLDER_NAME_CHARS = 64 +_MAX_PLACEHOLDER_SCHEMA_ITEMS = 32 +_LANGTAG_PATTERN = re.compile( + r"^(?:[A-Za-z]{2,3}(?:-[A-Za-z]{3}){0,3}|[A-Za-z]{4}|[A-Za-z]{5,8})" + r"(?:-[A-Za-z]{4})?" + r"(?:-(?:[A-Za-z]{2}|[0-9]{3}))?" + r"(?:-(?:[A-Za-z0-9]{5,8}|[0-9][A-Za-z0-9]{3}))*" + r"(?:-[0-9A-WY-Za-wy-z](?:-[A-Za-z0-9]{2,8})+)*" + r"(?:-[xX](?:-[A-Za-z0-9]{1,8})+)?$", + flags=re.ASCII | re.IGNORECASE, +) +_PRIVATEUSE_TAG_PATTERN = re.compile( + r"^[xX](?:-[A-Za-z0-9]{1,8})+$", + flags=re.ASCII, +) +_GRANDFATHERED_TAGS = frozenset( + tag.lower() + for tag in ( + "en-GB-oed", + "i-ami", + "i-bnn", + "i-default", + "i-enochian", + "i-hak", + "i-klingon", + "i-lux", + "i-mingo", + "i-navajo", + "i-pwn", + "i-tao", + "i-tay", + "i-tsu", + "sgn-BE-FR", + "sgn-BE-NL", + "sgn-CH-DE", + "art-lojban", + "cel-gaulish", + "no-bok", + "no-nyn", + "zh-guoyu", + "zh-hakka", + "zh-min", + "zh-min-nan", + "zh-xiang", + ) +) +_SCREEN_KEY_PATTERN = re.compile( + r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+$", + flags=re.ASCII, +) +_MESSAGE_KEY_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$", flags=re.ASCII) +_PLACEHOLDER_NAME_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$", flags=re.ASCII) +_ACCEPT_LANGUAGE_ITEM_PATTERN = re.compile( + r"^(?P\*|[A-Za-z]{1,8}(?:-[A-Za-z0-9]{1,8})*)" + r"(?:[ \t]*;[ \t]*[qQ]=(?P0(?:\.\d{0,3})?|1(?:\.0{0,3})?))?$", + flags=re.ASCII, +) + + +class UiLocalizationValidationError(ValueError): + """Stable typed validation failure emitted by the localization policy.""" + + def __init__(self, error_code: UiLocalizationValidationCode, message: str) -> None: + """Create a validation failure with a stable machine-readable code.""" + super().__init__(message) + self.error_code = error_code + + +@dataclass(frozen=True, slots=True) +class UiLocaleSelection: + """Resolved locale and the authority tier that selected it.""" + + locale_code: SupportedLocaleCode + selection_source: LocaleSelectionSource + + +def _reject_control_characters( + value: str, + *, + error_code: UiLocalizationValidationCode, + message: str, + allow_horizontal_tab: bool = False, +) -> None: + """Reject C0/DEL controls while preserving the caller's boundary code.""" + for character in value: + code_point = ord(character) + if code_point == 0x7F or ( + code_point < 0x20 and not (allow_horizontal_tab and character == "\t") + ): + raise UiLocalizationValidationError(error_code, message) + + +def _has_repeated_extension_singleton(locale_tag: str) -> bool: + """Return whether a structural langtag repeats an extension singleton before private use.""" + seen_singletons: set[str] = set() + for subtag in locale_tag.split("-")[1:]: + normalized = subtag.lower() + if normalized == "x": + break + if len(subtag) != 1: + continue + if normalized in seen_singletons: + return True + seen_singletons.add(normalized) + return False + + +def _has_multiple_extlang_subtags(locale_tag: str) -> bool: + """Return whether a structural langtag occupies a reserved second extlang position.""" + subtags = locale_tag.split("-") + if len(subtags[0]) not in (2, 3): + return False + extlang_count = 0 + for subtag in subtags[1:4]: + if len(subtag) != 3 or not subtag.isalpha(): + break + extlang_count += 1 + return extlang_count > 1 + + +def _has_repeated_variant_subtag(locale_tag: str) -> bool: + """Return whether a structural langtag repeats a variant before extensions/private use.""" + subtags = locale_tag.split("-") + index = 1 + primary_language = subtags[0] + + if len(primary_language) in (2, 3): + extlang_count = 0 + while ( + index < len(subtags) + and extlang_count < 3 + and len(subtags[index]) == 3 + and subtags[index].isalpha() + ): + index += 1 + extlang_count += 1 + + if index < len(subtags) and len(subtags[index]) == 4 and subtags[index].isalpha(): + index += 1 + + if index < len(subtags) and ( + (len(subtags[index]) == 2 and subtags[index].isalpha()) + or (len(subtags[index]) == 3 and subtags[index].isdigit()) + ): + index += 1 + + seen_variants: set[str] = set() + while index < len(subtags): + subtag = subtags[index] + if len(subtag) == 1: + break + is_variant = (5 <= len(subtag) <= 8 and subtag.isalnum()) or ( + len(subtag) == 4 and subtag[0].isdigit() and subtag[1:].isalnum() + ) + if not is_variant: + break + normalized = subtag.lower() + if normalized in seen_variants: + return True + seen_variants.add(normalized) + index += 1 + return False + + +def normalize_supported_locale(locale_tag: str) -> SupportedLocaleCode: + """Normalize a supported language tag to its release-level product language.""" + if not isinstance(locale_tag, str): + raise UiLocalizationValidationError( + "ui_locale_input_invalid", + "locale input must be a string", + ) + if len(locale_tag) > _MAX_LOCALE_TAG_CHARS: + raise UiLocalizationValidationError( + "ui_locale_input_invalid", + "locale input exceeds the 128-character product limit", + ) + _reject_control_characters( + locale_tag, + error_code="ui_locale_input_invalid", + message="locale input contains forbidden control characters", + ) + candidate = locale_tag + candidate_lower = candidate.lower() + is_grandfathered = candidate_lower in _GRANDFATHERED_TAGS + is_structural_langtag = _LANGTAG_PATTERN.fullmatch(candidate) is not None + if ( + is_structural_langtag + and not is_grandfathered + and _has_multiple_extlang_subtags(candidate) + ): + raise UiLocalizationValidationError( + "ui_locale_input_invalid", + "locale input contains more than one RFC 5646 extended language subtag", + ) + if ( + is_structural_langtag + and not is_grandfathered + and _has_repeated_extension_singleton(candidate) + ): + raise UiLocalizationValidationError( + "ui_locale_input_invalid", + "locale input repeats an RFC 5646 extension singleton", + ) + if ( + is_structural_langtag + and not is_grandfathered + and _has_repeated_variant_subtag(candidate) + ): + raise UiLocalizationValidationError( + "ui_locale_input_invalid", + "locale input repeats an RFC 5646 variant subtag", + ) + if not candidate or not ( + is_structural_langtag + or _PRIVATEUSE_TAG_PATTERN.fullmatch(candidate) + or is_grandfathered + ): + raise UiLocalizationValidationError( + "ui_locale_input_invalid", + "locale input is not a well-formed bounded RFC 5646 language tag", + ) + primary_language = candidate.split("-", 1)[0].lower() + if primary_language not in _SUPPORTED_LOCALE_SET: + raise UiLocalizationValidationError( + "ui_locale_unsupported", + "locale is not supported by this product release", + ) + return cast(SupportedLocaleCode, primary_language) + + +def _accept_language_preferences( + header_value: str, + product_default: SupportedLocaleCode, +) -> tuple[SupportedLocaleCode, ...]: + """Return supported preferences using weighted RFC 4647 lookup ordering.""" + if not isinstance(header_value, str): + raise UiLocalizationValidationError( + "ui_accept_language_invalid", + "Accept-Language must be a string", + ) + if len(header_value) > _MAX_ACCEPT_LANGUAGE_CHARS: + raise UiLocalizationValidationError( + "ui_accept_language_invalid", + "Accept-Language exceeds the 8192-character product limit", + ) + _reject_control_characters( + header_value, + error_code="ui_accept_language_invalid", + message="Accept-Language contains forbidden control characters", + allow_horizontal_tab=True, + ) + candidate = header_value.strip(" \t") + if not candidate: + return () + + explicit_best: dict[SupportedLocaleCode, tuple[float, int]] = {} + concrete_priorities: list[tuple[float, int]] = [] + wildcard_best: tuple[float, int] | None = None + non_empty_members = 0 + empty_members = 0 + + for position, raw_item in enumerate(candidate.split(",")): + item = raw_item.strip(" \t") + if not item: + empty_members += 1 + if empty_members > _MAX_ACCEPT_LANGUAGE_EMPTY_MEMBERS: + raise UiLocalizationValidationError( + "ui_accept_language_invalid", + "Accept-Language contains too many empty list members", + ) + continue + non_empty_members += 1 + if non_empty_members > _MAX_ACCEPT_LANGUAGE_MEMBERS: + raise UiLocalizationValidationError( + "ui_accept_language_invalid", + "Accept-Language contains too many language ranges", + ) + match = _ACCEPT_LANGUAGE_ITEM_PATTERN.fullmatch(item) + if match is None: + raise UiLocalizationValidationError( + "ui_accept_language_invalid", + "Accept-Language contains an invalid language range or quality value", + ) + quality = float(match.group("quality") or "1") + language_range = match.group("range") + if language_range == "*": + if wildcard_best is None or quality > wildcard_best[0]: + wildcard_best = (quality, position) + continue + + if quality > 0: + concrete_priorities.append((quality, position)) + primary_language = language_range.split("-", 1)[0].lower() + if primary_language not in _SUPPORTED_LOCALE_SET: + continue + locale_code = cast(SupportedLocaleCode, primary_language) + current = explicit_best.get(locale_code) + if current is None or quality > current[0]: + explicit_best[locale_code] = (quality, position) + + ranked: list[tuple[float, int, SupportedLocaleCode | None]] = [ + (quality, position, locale_code) + for locale_code, (quality, position) in explicit_best.items() + if quality > 0 + ] + if wildcard_best is not None and wildcard_best[0] > 0: + ranked.append((wildcard_best[0], wildcard_best[1], None)) + ranked.sort(key=lambda item: (-item[0], item[1])) + + excluded_locales = { + locale_code + for locale_code, (quality, _) in explicit_best.items() + if quality == 0 + } + ordered: list[SupportedLocaleCode] = [] + for quality, position, locale_code in ranked: + if locale_code is None: + wildcard_priority = (-quality, position) + if any( + (-concrete_quality, concrete_position) > wildcard_priority + for concrete_quality, concrete_position in concrete_priorities + ): + continue + if not excluded_locales: + break + wildcard_candidates = (product_default, *SUPPORTED_LOCALE_CODES) + for wildcard_candidate in wildcard_candidates: + if ( + wildcard_candidate not in excluded_locales + and wildcard_candidate not in ordered + ): + ordered.append(wildcard_candidate) + break + break + ordered.append(locale_code) + return tuple(ordered) + + +def select_ui_locale( + *, + persisted_preference: str | None = None, + session_preference: str | None = None, + accept_language: str | None = None, + product_default: SupportedLocaleCode = DEFAULT_LOCALE_CODE, +) -> UiLocaleSelection: + """Resolve one supported locale using the product's explicit authority order.""" + if not isinstance(product_default, str) or product_default not in _SUPPORTED_LOCALE_SET: + raise UiLocalizationValidationError( + "ui_locale_unsupported", + "product default locale must be supported", + ) + if persisted_preference is not None: + return UiLocaleSelection( + normalize_supported_locale(persisted_preference), + "persisted_preference", + ) + if session_preference is not None: + return UiLocaleSelection( + normalize_supported_locale(session_preference), + "session_preference", + ) + if accept_language is not None: + preferences = _accept_language_preferences(accept_language, product_default) + if preferences: + return UiLocaleSelection(preferences[0], "accept_language") + return UiLocaleSelection(product_default, "product_default") + + +def validate_screen_key(screen_key: str) -> str: + """Return a normalized screen identity or fail closed on invalid input.""" + if ( + not isinstance(screen_key, str) + or len(screen_key) > _MAX_SCREEN_KEY_CHARS + or not _SCREEN_KEY_PATTERN.fullmatch(screen_key) + ): + raise UiLocalizationValidationError( + "ui_screen_key_invalid", + "screen_key must be a dotted lowercase product identity up to 128 characters", + ) + return screen_key + + +def validate_message_key(message_key: str) -> str: + """Return a normalized message identity or fail closed on invalid input.""" + if ( + not isinstance(message_key, str) + or len(message_key) > _MAX_MESSAGE_KEY_CHARS + or not _MESSAGE_KEY_PATTERN.fullmatch(message_key) + ): + raise UiLocalizationValidationError( + "ui_message_key_invalid", + "message_key must be lowercase snake_case up to 128 characters", + ) + return message_key + + +def _validate_translation_text(message_text: str) -> None: + """Reject text that cannot cross the catalog's bounded UTF-8/PostgreSQL boundary.""" + if not isinstance(message_text, str): + raise UiLocalizationValidationError( + "ui_translation_input_invalid", + "translated message must be a string", + ) + if len(message_text) > _MAX_TRANSLATION_MESSAGE_CHARS: + raise UiLocalizationValidationError( + "ui_translation_input_invalid", + "translated message exceeds the 16384-character product limit", + ) + if "\x00" in message_text or any( + 0xD800 <= ord(character) <= 0xDFFF for character in message_text + ): + raise UiLocalizationValidationError( + "ui_translation_input_invalid", + "translated message must contain persistable Unicode scalar text without NUL", + ) + + +def _reject_placeholder_operators(message_text: str) -> None: + """Reject formatter operators that Formatter.parse normalizes away when empty.""" + index = 0 + while index < len(message_text): + character = message_text[index] + if character == "{": + if index + 1 < len(message_text) and message_text[index + 1] == "{": + index += 2 + continue + closing_index = message_text.find("}", index + 1) + if closing_index == -1: + return + field_source = message_text[index + 1 : closing_index] + if ":" in field_source or "!" in field_source: + raise UiLocalizationValidationError( + "ui_placeholder_schema_invalid", + "placeholders must be simple named fields without conversion or format specifiers", + ) + index = closing_index + 1 + continue + if character == "}" and index + 1 < len(message_text) and message_text[index + 1] == "}": + index += 2 + continue + index += 1 + + +def extract_placeholder_names(message_text: str) -> tuple[str, ...]: + """Extract bounded simple named interpolation fields without evaluating the message.""" + _validate_translation_text(message_text) + _reject_placeholder_operators(message_text) + names: list[str] = [] + seen: set[str] = set() + try: + parsed = Formatter().parse(message_text) + for _, field_name, format_spec, conversion in parsed: + if field_name is None: + continue + if ( + len(field_name) > _MAX_PLACEHOLDER_NAME_CHARS + or not _PLACEHOLDER_NAME_PATTERN.fullmatch(field_name) + or format_spec + or conversion is not None + ): + raise UiLocalizationValidationError( + "ui_placeholder_schema_invalid", + "placeholders must be bounded simple named fields without conversion or format specifiers", + ) + if field_name not in seen: + if len(names) >= _MAX_PLACEHOLDER_SCHEMA_ITEMS: + raise UiLocalizationValidationError( + "ui_placeholder_schema_invalid", + "translated message contains too many unique placeholders", + ) + seen.add(field_name) + names.append(field_name) + except ValueError as error: + raise UiLocalizationValidationError( + "ui_placeholder_schema_invalid", + "message contains malformed placeholder syntax", + ) from error + return tuple(names) + + +def validate_translation_placeholders( + translated_message: str, + placeholder_schema: tuple[str, ...] | list[str], +) -> tuple[str, ...]: + """Require translated text to preserve one bounded versioned placeholder schema.""" + if not isinstance(placeholder_schema, (tuple, list)): + raise UiLocalizationValidationError( + "ui_placeholder_schema_invalid", + "placeholder schema must be a tuple or list of names", + ) + if len(placeholder_schema) > _MAX_PLACEHOLDER_SCHEMA_ITEMS: + raise UiLocalizationValidationError( + "ui_placeholder_schema_invalid", + "placeholder schema exceeds the 32-name product limit", + ) + schema = tuple(placeholder_schema) + if any( + not isinstance(name, str) + or len(name) > _MAX_PLACEHOLDER_NAME_CHARS + or not _PLACEHOLDER_NAME_PATTERN.fullmatch(name) + for name in schema + ) or len(schema) != len(set(schema)): + raise UiLocalizationValidationError( + "ui_placeholder_schema_invalid", + "placeholder schema must contain unique lowercase snake_case names up to 64 characters", + ) + actual = extract_placeholder_names(translated_message) + if set(actual) != set(schema): + raise UiLocalizationValidationError( + "ui_translation_placeholder_mismatch", + "translated message placeholders do not match the versioned schema", + ) + return actual \ No newline at end of file diff --git a/backend/tests/test_ui_localization_bcp47_extension_singletons.py b/backend/tests/test_ui_localization_bcp47_extension_singletons.py new file mode 100644 index 000000000..49d286db7 --- /dev/null +++ b/backend/tests/test_ui_localization_bcp47_extension_singletons.py @@ -0,0 +1,21 @@ +"""RFC 5646 extension-singleton regressions for explicit locale preferences.""" + +import pytest + +from services.ui_localization_policy import ( + UiLocalizationValidationError, + normalize_supported_locale, +) + + +@pytest.mark.parametrize("locale_tag", ("en-a-bbb-a-ccc", "EN-A-bbb-a-ccc")) +def test_explicit_locale_rejects_repeated_extension_singleton(locale_tag): + """RFC 5646 forbids repeating an extension singleton outside private use.""" + with pytest.raises(UiLocalizationValidationError) as caught: + normalize_supported_locale(locale_tag) + assert caught.value.error_code == "ui_locale_input_invalid" + + +def test_private_use_may_repeat_extension_letter_after_x(): + """Singleton-shaped private-use data is not a repeated extension singleton.""" + assert normalize_supported_locale("en-a-bbb-x-a-ccc") == "en" diff --git a/backend/tests/test_ui_localization_bcp47_well_formedness.py b/backend/tests/test_ui_localization_bcp47_well_formedness.py new file mode 100644 index 000000000..52bcedb54 --- /dev/null +++ b/backend/tests/test_ui_localization_bcp47_well_formedness.py @@ -0,0 +1,79 @@ +"""BCP 47 well-formedness regressions for explicit locale preferences.""" + +import pytest + +from services.ui_localization_policy import ( + UiLocalizationValidationError, + normalize_supported_locale, +) + + +@pytest.mark.parametrize("locale_tag", ("en-x", "en-u")) +def test_explicit_locale_rejects_terminal_extension_or_private_use_singletons(locale_tag): + """A supported primary language does not make a malformed RFC 5646 suffix valid.""" + with pytest.raises(UiLocalizationValidationError) as caught: + normalize_supported_locale(locale_tag) + assert caught.value.error_code == "ui_locale_input_invalid" + + +@pytest.mark.parametrize("locale_tag", (" en-US", "en-US ", " en-US ")) +def test_explicit_locale_rejects_surrounding_spaces(locale_tag): + """RFC 5646 language-tag syntax does not permit SP around an explicit locale value.""" + with pytest.raises(UiLocalizationValidationError) as caught: + normalize_supported_locale(locale_tag) + assert caught.value.error_code == "ui_locale_input_invalid" + + +@pytest.mark.parametrize("locale_tag", ("en-x-private", "en-u-ca-gregory")) +def test_explicit_locale_accepts_well_formed_private_use_and_extension_suffixes(locale_tag): + """Valid RFC 5646 suffixes remain compatible with release-level locale normalization.""" + assert normalize_supported_locale(locale_tag) == "en" + + +@pytest.mark.parametrize( + ("locale_tag", "expected_locale"), + ( + ("en-GB-oed", "en"), + ("zh-min", "zh"), + ("zh-min-nan", "zh"), + ), +) +def test_explicit_locale_accepts_supported_primary_grandfathered_language_tags( + locale_tag, + expected_locale, +): + """RFC 5646 grandfathered tags remain Language-Tag values even outside langtag ABNF.""" + assert normalize_supported_locale(locale_tag) == expected_locale + + +@pytest.mark.parametrize("locale_tag", ("x-private", "i-klingon", "sgn-BE-FR")) +def test_well_formed_language_tags_without_supported_primary_are_unsupported(locale_tag): + """Well-formed private-use/irregular tags are unsupported, not syntactically invalid.""" + with pytest.raises(UiLocalizationValidationError) as caught: + normalize_supported_locale(locale_tag) + assert caught.value.error_code == "ui_locale_unsupported" + + +@pytest.mark.parametrize("locale_tag", ("en-1901-1901", "en-oxendict-OXENDICT")) +def test_explicit_locale_rejects_duplicate_variant_subtags(locale_tag): + """RFC 5646 validity forbids duplicate variant subtags case-insensitively.""" + with pytest.raises(UiLocalizationValidationError) as caught: + normalize_supported_locale(locale_tag) + assert caught.value.error_code == "ui_locale_input_invalid" + + +def test_duplicate_variant_spelling_inside_private_use_is_not_a_variant_duplicate(): + """Private-use subtags do not participate in the duplicate-variant validity rule.""" + assert normalize_supported_locale("en-1901-x-1901") == "en" + + +def test_explicit_locale_rejects_more_than_one_extlang_subtag(): + """RFC 5646 reserves second and third extlang positions as permanently invalid.""" + with pytest.raises(UiLocalizationValidationError) as caught: + normalize_supported_locale("zh-cmn-yue") + assert caught.value.error_code == "ui_locale_input_invalid" + + +def test_explicit_locale_accepts_one_extlang_with_following_structural_subtags(): + """One extlang remains valid before script and region subtags.""" + assert normalize_supported_locale("zh-cmn-Hans-CN") == "zh" diff --git a/backend/tests/test_ui_localization_content_resource_bounds.py b/backend/tests/test_ui_localization_content_resource_bounds.py new file mode 100644 index 000000000..4a75c67e5 --- /dev/null +++ b/backend/tests/test_ui_localization_content_resource_bounds.py @@ -0,0 +1,67 @@ +"""Resource-bound regressions for translation text and placeholder metadata.""" + +import pytest + +from services.ui_localization_policy import ( + UiLocalizationValidationError, + extract_placeholder_names, + validate_translation_placeholders, +) + + +def assert_error(code, callable_): + """Assert one stable localization validation code from a failing call.""" + with pytest.raises(UiLocalizationValidationError) as caught: + callable_() + assert caught.value.error_code == code + + +def test_translation_text_character_budget_is_bounded_before_placeholder_work(): + """UI copy accepts 16,384 characters and rejects the next character without truncation.""" + assert extract_placeholder_names("가" * 16_384) == () + assert_error( + "ui_translation_input_invalid", + lambda: extract_placeholder_names("가" * 16_385), + ) + + +def test_placeholder_name_character_budget_is_bounded(): + """Versioned placeholder identities accept 64 characters and reject the 65th.""" + accepted_name = "a" + ("b" * 63) + rejected_name = accepted_name + "c" + + assert validate_translation_placeholders( + "{" + accepted_name + "}", + (accepted_name,), + ) == (accepted_name,) + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders( + "{" + rejected_name + "}", + (rejected_name,), + ), + ) + + +def test_placeholder_schema_item_budget_is_bounded_before_collection_copy(): + """A catalog message accepts 32 unique placeholders and rejects the 33rd.""" + accepted_schema = tuple(f"field_{index}" for index in range(32)) + accepted_message = " ".join(f"{{{name}}}" for name in accepted_schema) + rejected_schema = (*accepted_schema, "field_32") + rejected_message = accepted_message + " {field_32}" + + assert validate_translation_placeholders( + accepted_message, + accepted_schema, + ) == accepted_schema + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders( + rejected_message, + rejected_schema, + ), + ) + assert_error( + "ui_placeholder_schema_invalid", + lambda: extract_placeholder_names(rejected_message), + ) diff --git a/backend/tests/test_ui_localization_http_whitespace.py b/backend/tests/test_ui_localization_http_whitespace.py new file mode 100644 index 000000000..396f3ac28 --- /dev/null +++ b/backend/tests/test_ui_localization_http_whitespace.py @@ -0,0 +1,125 @@ +"""HTTP/locale whitespace regressions for the UI Localization Catalog policy.""" + +import pytest + +from services.ui_localization_policy import ( + UiLocalizationValidationError, + normalize_supported_locale, + select_ui_locale, +) + + +def assert_error(code, callable_): + """Assert a stable localization validation code from one failing call.""" + with pytest.raises(UiLocalizationValidationError) as caught: + callable_() + assert caught.value.error_code == code + + +@pytest.mark.parametrize( + "locale_tag", + ( + "\ten-US", + "en-US\v", + "\fen-US", + "\u00a0en-US", + ), +) +def test_explicit_locale_rejects_control_and_unicode_boundary_whitespace(locale_tag): + """HTAB, VT, FF, and NBSP cannot surround an explicit locale identity.""" + assert_error("ui_locale_input_invalid", lambda: normalize_supported_locale(locale_tag)) + + +@pytest.mark.parametrize( + "header_value", + ( + "\ven-US", + "en-US\f", + "en-US\v;q=0.8", + "en-US;\fq=0.8", + "\u00a0en-US", + ), +) +def test_accept_language_rejects_non_http_whitespace(header_value): + """VT, FF, and Unicode whitespace are not RFC 9110 OWS and fail closed.""" + assert_error( + "ui_accept_language_invalid", + lambda: select_ui_locale(accept_language=header_value), + ) + + +def test_accept_language_allows_only_sp_and_htab_as_ows(): + """RFC 9110 OWS remains accepted around list members and q-weights.""" + selected = select_ui_locale(accept_language="\ten-US\t,\tfr ;\tq=0.5\t") + assert (selected.locale_code, selected.selection_source) == ("en", "accept_language") + + +def test_accept_language_quality_parameter_name_is_case_insensitive(): + """RFC 9110 defines the q parameter name as case-insensitive.""" + selected = select_ui_locale(accept_language="fr;Q=0.4, en;q=0.8") + assert (selected.locale_code, selected.selection_source) == ("en", "accept_language") + + selected = select_ui_locale(accept_language="fr;Q=0.9, en;q=0.8") + assert (selected.locale_code, selected.selection_source) == ("fr", "accept_language") + + +def test_accept_language_ignores_reasonable_empty_list_elements(): + """RFC 9110 recipients ignore empty members introduced while list values are merged.""" + selected = select_ui_locale(accept_language=", fr;q=0.4, , en;q=0.8,") + assert (selected.locale_code, selected.selection_source) == ("en", "accept_language") + + selected = select_ui_locale(accept_language=",") + assert (selected.locale_code, selected.selection_source) == ("ko", "product_default") + + +def test_accept_language_bounds_empty_list_elements_against_dos(): + """Only a bounded reasonable number of empty RFC 9110 list members is ignored.""" + accepted = ",".join([""] * 32 + ["en"]) + selected = select_ui_locale(accept_language=accepted) + assert (selected.locale_code, selected.selection_source) == ("en", "accept_language") + + excessive = ",".join([""] * 33 + ["en"]) + assert_error( + "ui_accept_language_invalid", + lambda: select_ui_locale(accept_language=excessive), + ) + + +def test_locale_and_accept_language_have_explicit_resource_bounds(): + """Valid basic ranges cannot create unbounded parser work at product boundaries.""" + locale_at_limit = "en-" + "-".join(f"var{index:05d}" for index in range(14)) + assert len(locale_at_limit) == 128 + assert normalize_supported_locale(locale_at_limit) == "en" + + locale_over_limit = locale_at_limit + "-abcdefgh" + assert_error( + "ui_locale_input_invalid", + lambda: normalize_supported_locale(locale_over_limit), + ) + + accepted_members = ",".join(["pt"] * 64) + selected = select_ui_locale(accept_language=accepted_members) + assert (selected.locale_code, selected.selection_source) == ("ko", "product_default") + + excessive_members = ",".join(["pt"] * 65) + assert_error( + "ui_accept_language_invalid", + lambda: select_ui_locale(accept_language=excessive_members), + ) + + header_at_limit = "en-" + "-".join(["abcdefgh"] * 910) + assert len(header_at_limit) == 8192 + selected = select_ui_locale(accept_language=header_at_limit) + assert (selected.locale_code, selected.selection_source) == ("en", "accept_language") + + header_over_limit = header_at_limit + "-abcdefgh" + assert_error( + "ui_accept_language_invalid", + lambda: select_ui_locale(accept_language=header_over_limit), + ) + + +def test_wildcard_before_unsupported_range_does_not_claim_negotiation_source(): + """RFC 4647 lookup skips a wildcard when any later concrete range remains to try.""" + selected = select_ui_locale(accept_language="*;q=0.9, pt-BR;q=0.8") + assert (selected.locale_code, selected.selection_source) == ("ko", "product_default") diff --git a/backend/tests/test_ui_localization_identity_bounds.py b/backend/tests/test_ui_localization_identity_bounds.py new file mode 100644 index 000000000..02bfad941 --- /dev/null +++ b/backend/tests/test_ui_localization_identity_bounds.py @@ -0,0 +1,41 @@ +"""Resource-bound regressions for UI Localization Catalog identities.""" + +import pytest + +from services.ui_localization_policy import ( + UiLocalizationValidationError, + validate_message_key, + validate_screen_key, +) + + +def assert_error(code, callable_): + """Assert a stable localization validation code from one failing call.""" + with pytest.raises(UiLocalizationValidationError) as caught: + callable_() + assert caught.value.error_code == code + + +def test_screen_key_has_explicit_product_length_bound(): + """A syntactically valid screen identity cannot create unbounded catalog/index work.""" + screen_key_at_limit = "a." + ("b" * 126) + assert len(screen_key_at_limit) == 128 + assert validate_screen_key(screen_key_at_limit) == screen_key_at_limit + + screen_key_over_limit = screen_key_at_limit + "b" + assert_error( + "ui_screen_key_invalid", + lambda: validate_screen_key(screen_key_over_limit), + ) + + +def test_message_key_has_explicit_product_length_bound(): + """A syntactically valid message identity cannot grow without a product ceiling.""" + message_key_at_limit = "m" * 128 + assert validate_message_key(message_key_at_limit) == message_key_at_limit + + message_key_over_limit = message_key_at_limit + "m" + assert_error( + "ui_message_key_invalid", + lambda: validate_message_key(message_key_over_limit), + ) diff --git a/backend/tests/test_ui_localization_policy.py b/backend/tests/test_ui_localization_policy.py new file mode 100644 index 000000000..1f4a897d9 --- /dev/null +++ b/backend/tests/test_ui_localization_policy.py @@ -0,0 +1,251 @@ +"""Regression coverage for the UI Localization Catalog domain policy.""" + +import pytest + +from services.ui_localization_policy import ( + SUPPORTED_LOCALE_CODES, + UiLocalizationValidationError, + extract_placeholder_names, + normalize_supported_locale, + select_ui_locale, + validate_message_key, + validate_screen_key, + validate_translation_placeholders, +) + + +def assert_error(code, callable_): + """Assert a stable localization validation code from one failing call.""" + with pytest.raises(UiLocalizationValidationError) as caught: + callable_() + assert caught.value.error_code == code + + +def test_supported_release_locales_are_fixed(): + """The first catalog release is exactly the eight product languages.""" + assert SUPPORTED_LOCALE_CODES == ("ko", "en", "ja", "zh", "vi", "es", "de", "fr") + + +def test_locale_normalization_preserves_supported_primary_language(): + """Supported regional and script variants resolve to the release language.""" + assert normalize_supported_locale("ko-KR") == "ko" + assert normalize_supported_locale("zh-Hant-TW") == "zh" + assert normalize_supported_locale("FR-fr") == "fr" + + +def test_explicit_unsupported_or_invalid_locale_fails_closed(): + """Explicit preferences never silently turn into a different locale.""" + assert_error("ui_locale_unsupported", lambda: normalize_supported_locale("pt-BR")) + assert_error("ui_locale_input_invalid", lambda: normalize_supported_locale("en_US")) + assert_error( + "ui_locale_input_invalid", + lambda: normalize_supported_locale("en\r\nX-Test: bad"), + ) + + +@pytest.mark.parametrize("locale_tag", ("\r\nen-US", "en-US\r\n")) +def test_locale_normalization_rejects_edge_crlf_before_trimming(locale_tag): + """CR/LF at either edge is rejected rather than normalized away by whitespace trimming.""" + assert_error("ui_locale_input_invalid", lambda: normalize_supported_locale(locale_tag)) + + +def test_selection_precedence_is_persisted_session_accept_language_default(): + """Locale authority follows the catalog contract in descending precedence.""" + selected = select_ui_locale( + persisted_preference="de-DE", + session_preference="fr-FR", + accept_language="es;q=1.0", + ) + assert (selected.locale_code, selected.selection_source) == ("de", "persisted_preference") + + selected = select_ui_locale(session_preference="fr-FR", accept_language="es;q=1.0") + assert (selected.locale_code, selected.selection_source) == ("fr", "session_preference") + + selected = select_ui_locale(accept_language="es-MX;q=0.7, ja-JP;q=0.9") + assert (selected.locale_code, selected.selection_source) == ("ja", "accept_language") + + selected = select_ui_locale(accept_language="pt-BR, it;q=0.7") + assert (selected.locale_code, selected.selection_source) == ("ko", "product_default") + + +def test_accept_language_duplicate_quality_is_not_first_occurrence_biased(): + """A later duplicate with higher quality outranks lower-quality peers.""" + selected = select_ui_locale( + accept_language="fr;q=0.7, fr-FR;q=0.9, fr-CA;q=0.6, en;q=0.8" + ) + assert selected.locale_code == "fr" + + +def test_accept_language_zero_quality_and_wildcard_follow_lookup_order(): + """Wildcard lookup defers to concrete ranges and never revives a q=0 locale.""" + selected = select_ui_locale(accept_language="ko;q=0, *;q=0.8, en;q=0.7") + assert selected.locale_code == "en" + selected = select_ui_locale(accept_language="ko;q=0, *;q=0.8") + assert (selected.locale_code, selected.selection_source) == ("en", "accept_language") + selected = select_ui_locale(accept_language="fr;q=0, *;q=0") + assert selected.locale_code == "ko" + + +def test_wildcard_lookup_skips_to_later_concrete_range(): + """RFC 4647 lookup skips wildcard and attributes wildcard-only fallback to product default.""" + selected = select_ui_locale(accept_language="en;q=0.9, *;q=0.8, fr;q=0.7") + assert selected.locale_code == "en" + selected = select_ui_locale(accept_language="*;q=0.7, *;q=0.9, *;q=0.5, fr;q=0.8") + assert selected.locale_code == "fr" + selected = select_ui_locale(accept_language="*;q=0.9") + assert (selected.locale_code, selected.selection_source) == ("ko", "product_default") + selected = select_ui_locale(accept_language="*;q=0.9", product_default="fr") + assert (selected.locale_code, selected.selection_source) == ("fr", "product_default") + + +def test_malformed_accept_language_fails_closed(): + """Malformed quality values and header injection do not fall through to defaults.""" + assert_error( + "ui_accept_language_invalid", + lambda: select_ui_locale(accept_language="en;q=1.2"), + ) + assert_error( + "ui_accept_language_invalid", + lambda: select_ui_locale(accept_language="en\r\nX-Test: bad"), + ) + + +@pytest.mark.parametrize("header_value", ("\r\nen-US", "en-US\r\n")) +def test_accept_language_rejects_edge_crlf_before_trimming(header_value): + """Header-edge CR/LF cannot disappear before the injection guard runs.""" + assert_error( + "ui_accept_language_invalid", + lambda: select_ui_locale(accept_language=header_value), + ) + + +def test_non_string_locale_inputs_fail_closed(): + """Non-string boundary values produce typed validation failures.""" + assert_error("ui_locale_input_invalid", lambda: normalize_supported_locale(None)) + assert_error("ui_accept_language_invalid", lambda: select_ui_locale(accept_language=42)) + + +def test_empty_accept_language_and_no_header_use_default(): + """An absent effective negotiation result uses the configured product default.""" + selected = select_ui_locale(accept_language=" ") + assert (selected.locale_code, selected.selection_source) == ("ko", "product_default") + selected = select_ui_locale() + assert (selected.locale_code, selected.selection_source) == ("ko", "product_default") + + +def test_custom_supported_default_and_invalid_default(): + """Deployments may choose another supported default but never an unsupported one.""" + selected = select_ui_locale(accept_language="it", product_default="fr") + assert selected.locale_code == "fr" + assert_error( + "ui_locale_unsupported", + lambda: select_ui_locale(product_default="pt"), + ) + + +def test_screen_and_message_identities_are_bounded(): + """Catalog identifiers stay within stable product-key syntax.""" + assert validate_screen_key("settings.identity") == "settings.identity" + assert validate_message_key("oidc_login_pending") == "oidc_login_pending" + assert_error("ui_screen_key_invalid", lambda: validate_screen_key("settings")) + assert_error("ui_screen_key_invalid", lambda: validate_screen_key("../settings.identity")) + assert_error("ui_message_key_invalid", lambda: validate_message_key("OIDC.Login")) + + +def test_non_string_and_invalid_identity_inputs_fail_closed(): + """Non-string and non-snake-case identities are rejected before persistence.""" + assert_error("ui_screen_key_invalid", lambda: validate_screen_key(None)) + assert_error("ui_message_key_invalid", lambda: validate_message_key(None)) + assert_error("ui_message_key_invalid", lambda: validate_message_key("oidc-login")) + + +def test_placeholder_extraction_is_literal_and_ordered(): + """Interpolation fields are inspected without evaluating format expressions.""" + assert extract_placeholder_names("{account_name}님, {count}건 / {account_name}") == ( + "account_name", + "count", + ) + assert extract_placeholder_names("{{literal}}") == () + + +def test_placeholder_schema_rejects_missing_extra_and_formatter_features(): + """Translations preserve the exact schema and reject formatter field traversal.""" + assert validate_translation_placeholders( + "{count}개의 알림이 {account_name}님에게 있습니다.", + ("account_name", "count"), + ) == ("count", "account_name") + assert_error( + "ui_translation_placeholder_mismatch", + lambda: validate_translation_placeholders( + "{account_name}님", + ("account_name", "count"), + ), + ) + assert_error( + "ui_translation_placeholder_mismatch", + lambda: validate_translation_placeholders("{account_name}님 {extra}", ("account_name",)), + ) + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders("{account_name.value}", ("account_name",)), + ) + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders("{count:03d}", ("count",)), + ) + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders("{account_name!r}", ("account_name",)), + ) + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders("{account_name:}", ("account_name",)), + ) + + +def test_placeholder_input_and_schema_must_be_bounded(): + """Malformed text, duplicates, and invalid schema names fail before publication.""" + assert_error("ui_translation_input_invalid", lambda: extract_placeholder_names(None)) + assert_error("ui_placeholder_schema_invalid", lambda: extract_placeholder_names("{missing")) + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders("{name}", ("name", "name")), + ) + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders("{name}", ("Name",)), + ) + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders("{name}", (None,)), + ) + + +@pytest.mark.parametrize( + "placeholder_schema", + ( + None, + {"name": "ignored-value"}, + ), +) +def test_placeholder_schema_container_type_fails_closed(placeholder_schema): + """Only the declared tuple/list schema boundary is accepted at runtime.""" + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders("{name}", placeholder_schema), + ) + + +@pytest.mark.parametrize( + "placeholder_schema", + ( + (["name"],), + ({"name": "ignored-value"},), + ), +) +def test_placeholder_schema_unhashable_element_fails_closed(placeholder_schema): + """Nested mutable values fail with the catalog error instead of raw TypeError.""" + assert_error( + "ui_placeholder_schema_invalid", + lambda: validate_translation_placeholders("{name}", placeholder_schema), + ) diff --git a/backend/tests/test_ui_localization_product_default.py b/backend/tests/test_ui_localization_product_default.py new file mode 100644 index 000000000..b17a84024 --- /dev/null +++ b/backend/tests/test_ui_localization_product_default.py @@ -0,0 +1,14 @@ +"""Runtime-boundary regression for the UI Localization Catalog product default.""" + +import pytest + +from services.ui_localization_policy import UiLocalizationValidationError, select_ui_locale + + +@pytest.mark.parametrize("product_default", ([], {})) +def test_unhashable_product_default_fails_with_typed_locale_error(product_default): + """Malformed runtime defaults never leak Python hashability errors.""" + with pytest.raises(UiLocalizationValidationError) as caught: + select_ui_locale(product_default=product_default) + + assert caught.value.error_code == "ui_locale_unsupported" diff --git a/backend/tests/test_ui_localization_translation_text.py b/backend/tests/test_ui_localization_translation_text.py new file mode 100644 index 000000000..c3d78cb52 --- /dev/null +++ b/backend/tests/test_ui_localization_translation_text.py @@ -0,0 +1,46 @@ +"""Translation-text boundary regressions for the UI Localization Catalog policy.""" + +import pytest + +from services.ui_localization_policy import ( + UiLocalizationValidationError, + extract_placeholder_names, + validate_translation_placeholders, +) + + +def assert_error(code, callable_): + """Assert a stable localization validation code from one failing call.""" + with pytest.raises(UiLocalizationValidationError) as caught: + callable_() + assert caught.value.error_code == code + + +@pytest.mark.parametrize( + "message_text", + ( + "\x00{name}", + "{name}\x00", + "\ud800{name}", + "{name}\udfff", + ), +) +def test_translation_text_rejects_non_persistable_unicode(message_text): + """NUL and surrogate code points fail at the translation boundary before publication.""" + assert_error( + "ui_translation_input_invalid", + lambda: validate_translation_placeholders(message_text, ("name",)), + ) + + +def test_non_string_translation_text_keeps_translation_boundary_identity(): + """A non-string translation reports the translation boundary, not a schema failure.""" + assert_error("ui_translation_input_invalid", lambda: extract_placeholder_names(None)) + + +def test_translation_text_allows_normal_multiline_unicode(): + """The boundary does not reject ordinary Unicode, newlines, or tabs used by UI copy.""" + assert validate_translation_placeholders( + "안녕하세요, {name}님.\n\t다음 일정이 있습니다.", + ("name",), + ) == ("name",) diff --git a/docs/doctoring/ui-localization-bcp47-registry-independent-validity.md b/docs/doctoring/ui-localization-bcp47-registry-independent-validity.md new file mode 100644 index 000000000..edba6ba2b --- /dev/null +++ b/docs/doctoring/ui-localization-bcp47-registry-independent-validity.md @@ -0,0 +1,85 @@ +# UI localization: BCP 47 registry-independent validity guards + +## Decision + +Naruon's explicit-locale policy separates three boundaries: + +1. structural `Language-Tag` syntax accepted by RFC 5646 ABNF, including private-use-only and grandfathered forms; +2. registry-independent RFC 5646 validity constraints that can be checked without consulting a moving IANA Language Subtag Registry snapshot; +3. Naruon's release support set (`ko`, `en`, `ja`, `zh`, `vi`, `es`, `de`, `fr`). + +The structural `langtag` branch now enforces three registry-independent validity rules in addition to ABNF shape: + +- an extension singleton may appear at most once before the private-use `x` boundary; +- a variant subtag may not repeat case-insensitively before extensions/private use; +- despite the compatibility ABNF reserving up to three extlang positions, only the first extlang position may be occupied. + +Those structural guards apply only to ordinary `langtag` values. RFC 5646's fixed `grandfathered` alternatives are complete registered tags whose apparent subtags cannot be reinterpreted as extlang/variant structure. The fixed regular and irregular grandfathered lists are therefore recognized before registry-independent structural guards are applied. + +These checks intentionally remain narrower than full BCP 47 validity. Registry membership, variant `Prefix` relationships, deprecation, preferred values, canonicalization, and other registry-dated semantics still require a versioned IANA Language Subtag Registry authority and are not inferred here. + +## Findings + +### Repeated extension singleton + +The existing `_LANGTAG_PATTERN` accepted `en-a-bbb-a-ccc`: each `a` independently satisfied the extension-singleton production. RFC 5646 §2.2.6 requires each singleton subtag to appear at most once outside private use and explicitly makes that form invalid, while `en-a-bbb-x-a-ccc` remains valid because the second `a` is private-use data. + +RED `da7e2653a158a259f4d327f1cf017674c7721064` pinned the failure. Causal fix `81e061c000a13261ced87b7d6f71fbce64637821` added the bounded case-insensitive singleton check and stopped evaluation at private-use `x`. + +### Duplicate variant subtag + +A structural ABNF match also accepted duplicate variants such as `en-1901-1901` and case variants such as `en-oxendict-OXENDICT`. RFC 5646 §2.2.9 states that a valid tag contains no duplicate variant subtags. This condition is registry-independent once the structural parser has identified the variant segment. + +RED `c34b45957799575ea7e11c84992fdf84461b783f` requires both duplicate forms to fail with `ui_locale_input_invalid`, while preserving `en-1901-x-1901 -> en` because the second spelling is private-use data. Causal fix `96adc96dd5fab6e31734c2c23257b0f0f40447ee` walks the structural positions after primary/extlang/script/region, compares variant spellings case-insensitively, and stops before extensions/private use. + +### Reserved second and third extlang positions + +RFC 5646 §2.2.2 explains that the `extlang` ABNF retains up to three positions for compatibility, but the second and third positions are permanently reserved: tags occupying them are and will always remain invalid. The RFC 5646 errata record also notes that §2.2.9's compact validity list omitted this condition even though §2.2.2 makes it normative. + +The previous regex therefore accepted `zh-cmn-yue` and normalized it to `zh`. RED `c49c80904ae1c38ee529a4b65e2fdb6612863c47` requires that form to fail with `ui_locale_input_invalid` while preserving the single-extlang form `zh-cmn-Hans-CN -> zh`. Causal fix `983194dd129cc593a4caa3c8531fb669986e4262` rejects structural tags that occupy more than one three-letter extlang position immediately after a two- or three-letter primary language subtag. + +### Grandfathered tags are not ordinary structural subtags + +Fresh code-current review of predecessor `97279b9dfd8d47d3033007d36398ccf7aa9e6943` found a contradiction between the fixed grandfathered-tag regression and production control flow. `backend/tests/test_ui_localization_bcp47_well_formedness.py` already requires `zh-min-nan -> zh`, but `_LANGTAG_PATTERN` also matches that spelling as `zh` plus two apparent three-letter extlang positions. The generic multi-extlang guard therefore rejected the RFC-defined grandfathered tag before the grandfathered alternative could be considered. + +RFC 5646 §2.1 lists `zh-min-nan` in the fixed `regular` grandfathered production and explicitly states that regular grandfathered tags can appear to match `langtag` even though their apparent subtags do not carry the ordinary extlang/variant semantics. This is a fixed standards exception, not a moving IANA-registry lookup. + +The realistic RED already existed on the predecessor exact head: the focused test expected `normalize_supported_locale("zh-min-nan") == "zh"`, while the source raised `ui_locale_input_invalid`. Causal fix `8666ef7bf2451f4e4ca8e93c69d24d5a9fb6436c` replaces the partial/misnamed grandfathered set with RFC 5646's complete fixed regular+irregular list and exempts exact grandfathered values from the ordinary structural duplicate/extlang guards. Unsupported grandfathered tags still fail at the product-support boundary as `ui_locale_unsupported`; no preferred-value rewriting or registry-date semantics are introduced. + +## Product significance + +Naruon ultimately collapses a supported explicit tag to one release language, but persisted/session locale input is still an authority boundary. Accepting an RFC-invalid tag here allows downstream systems to disagree about whether the original identity was admissible, while rejecting an RFC-defined grandfathered tag creates the opposite interoperability failure. The policy therefore recognizes the fixed RFC grandfathered alternatives first, applies stable registry-independent structural invariants only to ordinary `langtag` values, and then applies the eight-locale product-support check. + +## Rejected alternatives + +- **Treat ABNF match as sufficient validity.** Rejected because RFC 5646 explicitly adds validity rules outside the ABNF. +- **Treat regular grandfathered tags as ordinary extlang/variant sequences.** Rejected because RFC 5646 says their meaning is defined by the complete grandfathered registration even when they appear to match `langtag`. +- **Treat every repeated spelling after `x` as a duplicate.** Rejected because all subtags after the private-use singleton are private-use data. +- **Use live IANA lookup for this repair.** Rejected because the grandfathered alternatives are a fixed RFC list and a live registry would introduce a moving external authority without a version/update/reproducibility contract. +- **Canonicalize grandfathered tags to Preferred-Value.** Rejected because that is a registry/canonicalization policy outside this pure admission boundary and would rewrite caller identity. +- **Canonicalize or silently repair invalid caller input.** Rejected because explicit locale preferences are an authority boundary and fail closed. +- **Ban extlang entirely.** Rejected because RFC 5646 still permits one extlang position and Naruon's release-level normalization can preserve that structural compatibility without claiming registry canonicalization. + +## Acceptance and follow-up + +The focused regressions and causal source repairs are necessary but not sufficient for merge. The exact current head still requires repository full-suite/coverage/security execution and qualifying independent post-last-push review. Persistence and HTTP owners must preserve the typed invalid-versus-unsupported boundary without truncation, silent canonical repair, or reinterpretation of fixed grandfathered tags as ordinary subtag structure. + +Full registry validity remains deliberately out of scope until an owner defines a dated IANA Language Subtag Registry version, update cadence, provenance, rollback behavior, and reproducible tests. Variant-prefix recommendations and canonical preferred-value replacement must not be guessed into this pure-policy slice. + +## Traceability + +- Product Gap: #1731 +- Pure policy owner: #1740 +- Source: `backend/services/ui_localization_policy.py` +- Focused regressions: `backend/tests/test_ui_localization_bcp47_extension_singletons.py`, `backend/tests/test_ui_localization_bcp47_well_formedness.py` +- Repeated-singleton RED/fix: `da7e2653a158a259f4d327f1cf017674c7721064` -> `81e061c000a13261ced87b7d6f71fbce64637821` +- Duplicate-variant RED/fix: `c34b45957799575ea7e11c84992fdf84461b783f` -> `96adc96dd5fab6e31734c2c23257b0f0f40447ee` +- Multi-extlang RED/fix: `c49c80904ae1c38ee529a4b65e2fdb6612863c47` -> `983194dd129cc593a4caa3c8531fb669986e4262` +- Grandfathered-structure RED: predecessor `97279b9dfd8d47d3033007d36398ccf7aa9e6943`, `test_explicit_locale_accepts_supported_primary_grandfathered_language_tags` +- Grandfathered-structure fix: `8666ef7bf2451f4e4ca8e93c69d24d5a9fb6436c` + +## References + +Phillips, A., & Davis, M. (2009). *Tags for identifying languages* (BCP 47, RFC 5646). RFC Editor. https://doi.org/10.17487/RFC5646 + +RFC Editor. (2018). *Errata ID 5457: RFC 5646, Section 2.2.9*. https://www.rfc-editor.org/errata/eid5457 diff --git a/docs/doctoring/ui-localization-catalog-domain-policy.md b/docs/doctoring/ui-localization-catalog-domain-policy.md new file mode 100644 index 000000000..a4b77d592 --- /dev/null +++ b/docs/doctoring/ui-localization-catalog-domain-policy.md @@ -0,0 +1,118 @@ +# UI Localization Catalog domain policy + +## Decision status + +This document records the executable pure-domain slice for issue #1731. It remains **Proposed** and deliberately stops before persistence, API delivery, browser caching, catalog publication, Storybook composition, or page migration. + +The canonical workspace/Alembic owner is still #1503. Translation persistence must not create a migration from protected `develop` while that lineage is unintegrated. When #1503 or a verified complete successor reaches protected ancestry, the catalog persistence work must create the next ordinary descendant from the then-current single Alembic head. + +Open work already contains unrelated `ADR-0005` proposals. This slice therefore does not allocate another ADR number before live ADR-owner reconciliation. + +## Bounded context and ubiquitous language + +The **UI Localization Catalog** bounded context owns UI-copy identity and locale selection only. Ontology/concept labels are a separate authority and must not be stored or served through the same resource. + +Current domain objects are: + +- `SupportedLocaleCode`: exactly `ko`, `en`, `ja`, `zh`, `vi`, `es`, `de`, `fr`; +- `UiLocaleSelection`: selected locale plus the authority tier that selected it; +- `screen_key`: dotted lowercase product-screen identity such as `settings.identity`; +- `message_key`: lowercase snake-case message identity within a screen; +- `placeholder_schema`: immutable named interpolation fields a published translation must preserve. + +Locale authority is persisted preference → session preference → weighted `Accept-Language` → product default. Explicit persisted/session values are validated rather than silently replaced. Regional/script variants resolve to one release-level product language; this is intentionally narrower than a general-purpose BCP 47 registry. + +## Locale and HTTP negotiation boundary + +Stable machine-readable validation codes belong to the boundary that rejects the input. + +Explicit persisted/session locale input accepts no surrounding whitespace. C0 controls and DEL are rejected before structural parsing; ASCII SP and non-control Unicode whitespace such as NBSP are rejected because the anchored RFC 5646 language-tag alternatives do not admit them. The explicit-locale path does not trim or rewrite caller identity before validation. `Accept-Language` applies HTTP field semantics instead: C0/DEL controls are rejected except HTAB where RFC 9110 permits it as OWS, and only SP/HTAB are normalized as optional whitespace. Python generic `str.strip()` and regex `\s` are intentionally not protocol authority. + +The quality parameter name is case-insensitive, so `q=` and `Q=` carry identical parameter-name semantics while qvalue syntax remains strict. + +`Accept-Language` is an HTTP list field. RFC 9110 §5.6.1.2 requires recipients to ignore a reasonable number of empty list elements while not tolerating enough to create a denial-of-service mechanism. Naruon therefore ignores at most 32 empty members and rejects the 33rd with `ui_accept_language_invalid`. Empty members never become wildcard/default votes. + +RFC 4647 lookup ordering metadata is preserved even for release-unsupported ranges. For `*;q=0.9, pt-BR;q=0.8`, the later concrete range still exists in the language priority list, so the wildcard does not claim the selection. Portuguese then fails to match the eight release locales and the result reaches `product_default`. + +The product default is also runtime-validated before set membership. Type annotations cannot prevent an arbitrary caller from supplying an unhashable object, so invalid defaults fail with `ui_locale_unsupported` rather than raw Python `TypeError`. + +### Explicit parser resource limits + +The policy now bounds valid as well as malformed negotiation input before expensive normalization/list work: + +- explicit persisted/session locale value: at most **128 characters**; +- raw `Accept-Language` field value: at most **8192 characters**; +- non-empty `Accept-Language` members: at most **64**; +- ignored empty members: at most **32**. + +These are Naruon product limits, not IETF constants. Over-limit values are rejected; they are never truncated. RFC 9110 §5.4 permits implementations to refuse field values larger than they are willing to process, while RFC 4647 §4.4 and RFC 5646 §4.4.1 permit documented language range/tag length limits. RFC 5646 requires limited-buffer protocols to support at least 35 characters; the 128-character explicit-locale budget exceeds that minimum. Outer proxies/HTTP servers may impose stricter limits and are not weakened by this policy. + +The focused resource-bound record is `docs/doctoring/ui-localization-negotiation-resource-bounds.md`. + +## Placeholder invariant + +Translation publication preserves the exact named placeholder schema. Only simple lowercase named fields such as `{account_name}` are accepted. Attribute/index traversal, positional fields, conversion flags, format-specifier syntax, malformed braces, missing placeholders, and extra placeholders fail closed. + +The schema container is itself part of the boundary. Runtime callers may provide only tuple/list collections. A mapping, generator, scalar, `None`, or arbitrary iterable is not silently coerced. Elements are validated as strings matching the placeholder-name grammar before duplicate detection so nested mutable values cannot leak raw hashability errors. + +Python's `Formatter.parse()` normalizes an explicit empty format specifier: `{name}` and `{name:}` both expose an empty parsed `format_spec`. Because Naruon's catalog contract forbids format-specifier syntax entirely, the policy performs a narrow lexical check for real replacement fields containing `:` or `!` before `Formatter.parse()`, while doubled braces remain literals. See `docs/doctoring/ui-localization-placeholder-canonical-syntax.md`. + +This is catalog integrity validation, not a rendering engine. + +## Translation text persistence boundary + +A Python `str` is not sufficient proof that catalog text can cross the UTF-8/PostgreSQL boundary. PostgreSQL character types cannot store U+0000, and Unicode scalar values exclude surrogate code points U+D800–U+DFFF. The policy therefore rejects NUL and isolated surrogate code points before placeholder parsing with `ui_translation_input_invalid`. + +Ordinary Unicode text, combining marks, CJK text, newlines, and horizontal tabs remain valid. The policy does not rewrite user-facing copy. + +## Rejected alternatives + +A browser-wide catalog bundle and per-key network fetches are rejected in favor of future screen-scoped versioned resources. A new translation table/Alembic revision in this PR is rejected because it would create a parallel migration authority ahead of #1503. Locale parsing duplicated inside routes/components is rejected because it would fragment precedence and fallback semantics. + +Generic runtime whitespace, case-sensitive `q`, unconditional rejection of recipient-side empty list members, unbounded empty-member tolerance, truncation of oversized locale/header values, and dependence on reverse-proxy/framework header limits are rejected. The pure policy is callable outside the future HTTP route, so its resource and syntax boundaries must be explicit and deterministic. + +Coercing arbitrary placeholder iterables, running `set(schema)` before element validation, and deferring NUL/surrogate rejection to downstream drivers/codecs are rejected because they leak implementation-specific runtime failures instead of stable product errors. + +LLM translation is outside this slice. Future assisted translation must use a released contextual-orchestrator contract and cannot bypass deterministic publication/completeness/placeholder validation. + +## Verification and repair history + +The focused policy suite covers supported-language identity, precedence, regional/script normalization, weighted `Accept-Language`, duplicate ranges, wildcard/q=0 handling, malformed/control input, screen/message identity, literal braces, placeholder schema mismatch/formatter features, runtime schema-container rejection, unhashable schema elements, translation-text persistence, product-default runtime typing, and explicit parser resource limits. + +This document records the first **fourteen** ordinary-forward RED→repair sequences that established the core catalog policy. Later RFC 5646/RFC 4647 validity, resource-bound, explicit-locale no-rewrite, wildcard-provenance, and fixed-grandfathered repairs are recorded in their focused doctoring files and current #1740 authority rather than being backfilled into this historical sequence count. The first three established wildcard/q=0 basics, boundary-specific control codes, and control rejection before whitespace normalization. Subsequent repairs recorded here are: + +4. RED `8265e9bb08cc407c860f148f1823dd0f4ab6d13d` → fix `9a6c8bb04be237c968b58987ca6ac4efb789a482`: replace generic Python whitespace semantics with boundary-specific SP/HTAB/C0 rules. +5. RED `962e250c54d0da324ec29b21283ab0b02cd55e8c` → fix `b20b5593498c09855e6ef6fb213d9bf6ea764e15`: case-insensitive `q`/`Q` parameter name. +6. RED `e299d29b14943c368e581bcb3d35303a1ba3fb8f` → fix `92a50acef8c7a3fbb38a926994add1641ac6c49d`: tuple/list-only placeholder schema container. +7. RED `3e251cc253d2ccf62ec1a3209ab629c6cffbe149` → fix `dec6c694dcf930ef184b158de04260d8db433bec`: reject NUL/surrogate/non-string translation input. +8. RED `489bfd4fa2d6f3630cf33412f6d910d73c47e110` → fix `244cff4f8d8805f2b3eacf91a75c7da84052bfc2`: validate schema elements before hash-based uniqueness. +9. RED `652137bd5b4fae436af207d8823a4e6e5122b8a0` → fix `f079bb270aadb2748937d5491b718e87b4198834`: runtime product-default type validation before set membership. +10. RED `20a7475c190585a1f94d34bd19c363f97dcc4e81` → fix `eff473a04fd8cca5d76396c36ff0c1dbf3a033f6`: RFC 9110 recipient-side empty-list semantics. +11. RED `7a68f034ef817abdad2c5744d8cbf4d0cf65ce03` → fix `b88aff67897bed9a533d21987d2db2618144b850`: bound ignored empty members at 32. +12. RED `2ebe9758cb915841d86330bbabcd618f3f091d71` → fix `1664d9707d4b5b9366b422f8faf612418ecd7789`: preserve later unsupported concrete ranges for truthful RFC 4647 wildcard control flow and `selection_source`. +13. RED `8b19c7dfe43e3dca3c1e4b8980c0a9998a3aec43` → fix `6ed310e1065940fa14c258b48412490e7b98d751`: distinguish canonical `{name}` from forbidden `{name:}` before parser normalization. +14. RED `978fdbbc6135fa88a8d43e0971ed075ae546f9de` → fix `0fdb5371eef63d6d7abe87ac14b69269c8519ff8`: bound explicit locale length, raw `Accept-Language` length, and non-empty member count before parser work. + +Focused traceability for repair 14 is commit `d25c7a627af720242c9503f8f2b90553c7c94405`. + +This automation environment does not contain an exact repository checkout, so no full-suite/local-container PASS is claimed for the current head. Exact-head hosted CI/security and qualifying independent post-last-push review must settle naturally. No source-neutral wake commit or predecessor receipt transfer is authorized. + +## Next causal work + +After canonical migration ancestry is protected, add versioned catalog aggregates and an ordinary Alembic descendant with immutable published-version semantics, item-level idempotent authoring, and tenant/workspace authorization. Then add a screen-scoped async read endpoint with version/ETag identity and realistic k6/E2E evidence, followed by page composition and KO/EN/JA/ZH/VI/ES/DE/FR Storybook/browser/a11y acceptance covering normal/loading/empty/error/permission states, CJK wrapping/font fallback, text expansion, keyboard/focus/touch, and assistive technology. + +## TRACEABILITY + +Fielding, R. T., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). RFC Editor. https://doi.org/10.17487/RFC9110 + +Phillips, A., & Davis, M. (2006). *Matching of language tags* (RFC 4647, BCP 47). RFC Editor. https://doi.org/10.17487/RFC4647 + +Phillips, A., & Davis, M. (2009). *Tags for identifying languages* (RFC 5646, BCP 47). RFC Editor. https://doi.org/10.17487/RFC5646 + +Python Software Foundation. (2026). *string — Common string operations: Format string syntax* (Python 3.14 documentation). https://docs.python.org/3.14/library/string.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: Character types*. https://www.postgresql.org/docs/18/datatype-character.html + +The Unicode Consortium. (2026). *The Unicode Standard, Version 18.0: Core specification, Chapter 3*. https://www.unicode.org/versions/Unicode18.0.0/core-spec/chapter-3/ + +RFC 9110 defines `Accept-Language` as weighted language ranges, OWS as SP/HTAB, recipient-side list behavior, and implementation-specific field processing limits. RFC 4647 supplies lookup and range-length considerations. RFC 5646 supplies language-tag length guidance and explicitly permits documented refusal above an implementation limit rather than semantic-changing truncation. PostgreSQL/Unicode references define the persistence/scalar-value boundary. Python format-string documentation explains why canonical placeholder source requires a lexical check before parsed field semantics. diff --git a/docs/doctoring/ui-localization-content-resource-bounds.md b/docs/doctoring/ui-localization-content-resource-bounds.md new file mode 100644 index 000000000..4c4475f23 --- /dev/null +++ b/docs/doctoring/ui-localization-content-resource-bounds.md @@ -0,0 +1,78 @@ +# UI localization content resource bounds + +Status: Proposed with PR #1740. This record describes the pure-domain contract only; it does not claim that persistence, HTTP admission, browser rendering, or publication workflows are shipped. + +## Problem + +The localization policy already bounded locale tags, `Accept-Language`, `screen_key`, and `message_key`, but translation content and placeholder metadata remained syntactically constrained and operationally unbounded. A caller could submit an arbitrarily large translated message, a placeholder name of arbitrary length, or a tuple/list containing an arbitrary number of placeholder identities. The policy would then scan the whole message, run formatter parsing, allocate placeholder collections, and copy the complete schema before any downstream persistence or transport boundary had an opportunity to refuse the work. + +That is inconsistent with the catalog's fail-closed resource policy. PostgreSQL `text` being variable-length does not define a safe product message budget, and an outer HTTP/proxy field or body limit cannot be the only guard for code paths that may also be called from offline authoring, tests, migrations, or future publication jobs. + +MITRE CWE-400 describes uncontrolled resource consumption as an architecture/design weakness class and explicitly recommends placing specific limits of scale on protocols and resource-triggering inputs. This record uses that guidance only as resource-design traceability; it does not label this PR as a CVE or assign security severity from CWE-400 alone. + +## RED + +Commit `e1bdccc3b95bc41a8e03d29f3c581457cf586da5` adds focused boundary cases that are false on the predecessor policy: + +- translation text accepts exactly 16,384 Python characters and rejects 16,385 with `ui_translation_input_invalid`; +- one placeholder name accepts exactly 64 characters and rejects 65 with `ui_placeholder_schema_invalid`; +- one message/schema accepts 32 unique placeholder names and rejects the 33rd with `ui_placeholder_schema_invalid`; +- direct placeholder extraction also rejects a 33rd unique placeholder, so callers cannot bypass the versioned-schema guard by invoking the extractor directly. + +These are product limits, not values supplied by PostgreSQL, Unicode, Python, or an IETF specification. + +## Decision + +Causal fix `8cf8880fb86a87af22b9f18afd36f45acfb0e70d` adds three domain ceilings: + +- `_MAX_TRANSLATION_MESSAGE_CHARS = 16_384`; +- `_MAX_PLACEHOLDER_NAME_CHARS = 64`; +- `_MAX_PLACEHOLDER_SCHEMA_ITEMS = 32`. + +The checks are intentionally ordered to bound work before expensive or allocating steps: + +1. translation text type and character count are checked before scanning Unicode scalar validity, lexical placeholder checks, or `Formatter.parse()`; +2. tuple/list schema length is checked before copying it to a tuple; +3. placeholder-name length is checked before accepting the identifier as catalog metadata; +4. direct extraction refuses a new unique placeholder once 32 names are already retained. + +Over-limit input is rejected and never truncated. Truncation could alias distinct catalog content or placeholder identity and would make the immutable publication record dishonest. + +The 16,384-character text ceiling is deliberately generous for a single UI message while making parser work finite. It is a character/code-point boundary in Python rather than a byte budget. Transport, persistence, export, or rendering owners may impose a stricter byte-size limit when their own contract requires it, but they must not silently accept a larger value by bypassing this domain policy. + +## Rejected alternatives + +### Leave content unbounded because the future API can cap request size + +Rejected. The policy is callable outside HTTP, and resource safety would depend on every future caller reproducing the same limit correctly. + +### Rely on PostgreSQL `text` + +Rejected. PostgreSQL describes `text` as variable-length character data; that storage type is not a Naruon semantic or parser-work budget. Letting storage failure or memory pressure define the product limit would also move the failure away from the typed catalog boundary. + +### Truncate text or placeholder metadata + +Rejected. Truncation can change rendered meaning, collapse distinct placeholder identities, or publish a resource different from the reviewed source. + +### Count UTF-8 bytes in this pure policy + +Not selected for this layer. Python parsing cost here is driven primarily by the in-memory string and placeholder structure. A later transport/persistence owner can add an explicit byte budget if required, but that must be a separately tested contract rather than an implicit codec side effect. + +## Downstream contract + +Future persistence/publication and screen-scoped HTTP owners must preserve the 16,384/64/32 limits or enforce stricter documented limits without silent truncation. Publication completeness, immutable resource versions, placeholder equality, and the existing KO/EN/JA/ZH/VI/ES/DE/FR release set remain unchanged. + +Realistic API/k6/E2E acceptance should cover normal payloads, exact-boundary payloads, and over-limit rejection on the final endpoint without sample shrinking or cache-only measurement. This pure-policy PR does not claim those runtime receipts. + +## Traceability + +- RED: `e1bdccc3b95bc41a8e03d29f3c581457cf586da5`. +- Causal source fix: `8cf8880fb86a87af22b9f18afd36f45acfb0e70d`. +- Canonical product Gap: #1731. +- Product/technical Gap ledger owner: #1602. + +## References + +MITRE. (2026). *CWE-400: Uncontrolled resource consumption (Version 4.20)*. Common Weakness Enumeration. https://cwe.mitre.org/data/definitions/400.html + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: Data types*. https://www.postgresql.org/docs/18/datatype.html diff --git a/docs/doctoring/ui-localization-explicit-locale-bcp47-well-formedness.md b/docs/doctoring/ui-localization-explicit-locale-bcp47-well-formedness.md new file mode 100644 index 000000000..8a6772cf8 --- /dev/null +++ b/docs/doctoring/ui-localization-explicit-locale-bcp47-well-formedness.md @@ -0,0 +1,105 @@ +# UI localization explicit-locale BCP 47 well-formedness + +Status: Proposed with PR #1740. This record covers the pure explicit-locale policy only; it does not change the broader RFC 4647 `Accept-Language` language-range grammar. + +## Problem + +`normalize_supported_locale()` originally used the broad pattern `[A-Za-z]{2,8}(?:-[A-Za-z0-9]{1,8})*`. That shape was adequate to recover a primary language but was not the RFC 5646 `Language-Tag` grammar. It accepted malformed explicit preferences such as `en-x` and `en-u`: both have a supported primary language, but RFC 5646 requires private-use `x` to be followed by at least one private-use subtag and an extension singleton such as `u` to be followed by at least one 2-8 character extension subtag. + +Accepting those values as persisted/session locale identities made the explicit preference boundary weaker than its own documentation and blurred two different standards surfaces: RFC 5646 language tags for explicit stored/session identities versus RFC 4647 basic language ranges used inside `Accept-Language`. + +A second standards sweep found the first structural repair incomplete. RFC 5646 defines `Language-Tag = langtag / privateuse / grandfathered`; some grandfathered values are intentionally outside the `langtag` production, while regular grandfathered values may also look structurally like `langtag`. For the current release-primary set, `en-GB-oed`, `zh-min`, and `zh-min-nan` require recognition as complete fixed grandfathered tags before ordinary structural guards reinterpret apparent subtags. Other fixed grandfathered values remain syntactically well formed even when Naruon does not support their primary language. + +A third sweep found a typed-boundary defect left by treating only supported-primary tags as syntactically relevant. RFC 5646 permits a language tag to consist entirely of private-use subtags, and grandfathered tags remain well-formed language tags. Values such as `x-private`, `i-klingon`, and `sgn-BE-FR` are therefore syntactically well-formed but do not map to one of Naruon's eight release primaries. They belong to `ui_locale_unsupported`, not `ui_locale_input_invalid`. + +A later boundary sweep found a separate whitespace-normalization defect. The explicit persisted/session path called `locale_tag.strip(" ")` before RFC 5646 validation, so `" en-US"`, `"en-US "`, and `" en-US "` were silently accepted as `en-US`. RFC 5646 states that whitespace is not permitted in a language tag. That differs from the HTTP `Accept-Language` surface, where HTTP list syntax deliberately permits optional whitespace around field-value list elements. Explicit stored/session locale identity therefore must not borrow HTTP OWS normalization. + +## RED + +Commit `478bd95fdbd2b6601fae1ab70c3513abf0f985fe` adds focused cases that fail on the predecessor policy: + +- `en-x` must fail with `ui_locale_input_invalid`; +- `en-u` must fail with `ui_locale_input_invalid`; +- `en-x-private` remains accepted and normalizes to `en`; +- `en-u-ca-gregory` remains accepted and normalizes to `en`. + +Commit `8d9a0f67a47b26eb1e36a7bc65681470033c6f74` adds the grandfathered-tag regression: + +- `en-GB-oed` normalizes to `en`; +- `zh-min` normalizes to `zh`; +- `zh-min-nan` normalizes to `zh`. + +Commit `d23d4095d1c2ac8542f1ac247b6fee87fcb9485c` adds the syntax-versus-support classification regression: + +- `x-private` is well-formed private-use-only syntax and must fail with `ui_locale_unsupported`; +- `i-klingon` is an irregular grandfathered language tag and must fail with `ui_locale_unsupported`; +- `sgn-BE-FR` is an irregular grandfathered language tag and must fail with `ui_locale_unsupported`. + +Commit `2ff28c208b8fd5a83bee3e179f4d66b31831be43` adds the explicit-boundary whitespace regression: + +- leading SP before `en-US` must fail with `ui_locale_input_invalid`; +- trailing SP after `en-US` must fail with `ui_locale_input_invalid`; +- leading and trailing SP together must fail with `ui_locale_input_invalid`. + +The regressions are intentionally scoped to explicit locale values. They do not tighten RFC 4647 language-range parsing in `Accept-Language`, where the grammar and HTTP OWS boundary are different. + +## Decision + +Causal fix `3a87209306ad1b3dee67cb9e0e9e513dfb7a4f3a` replaces the permissive explicit-locale pattern with an RFC 5646 structural `langtag` pattern covering language/extlang, optional script and region, variants, extensions, and optional private-use suffix. + +Follow-up causal fix `aceef1b282d948411fe9b04b8cd08750fcdacfe8` restored the supported-primary grandfathered forms `zh-min` and `zh-min-nan` alongside `en-GB-oed` without introducing a moving IANA-registry dependency. + +Causal fix `dc077bec9ae47e1421fa678ae43451f65784c410` established separate admission branches for structural `langtag`, private-use-only tags, and the then-explicit fixed grandfathered exceptions before release support is evaluated. Only after a candidate is established as well-formed does the policy inspect the first subtag for membership in `ko/en/ja/zh/vi/es/de/fr`. This preserves the stable semantic distinction between malformed input and well-formed-but-unsupported input. + +Current source repair `8666ef7bf2451f4e4ca8e93c69d24d5a9fb6436c` completes that boundary: both RFC 5646 regular and irregular grandfathered tags are recognized from the RFC's fixed list before ordinary `langtag` structural guards run. This matters for `zh-min-nan`, whose spelling also matches the structural pattern and would otherwise be misread as occupying two extlang positions. Fixed-grandfathered recognition establishes syntax only; unsupported primary languages still fail afterward as `ui_locale_unsupported`. Ordinary non-grandfathered `langtag` values continue through the multi-extlang, repeated-singleton, and duplicate-variant validity guards. + +The complete regular+irregular grandfathered list is fixed by RFC 5646, so this admission exception does not create a moving registry dependency. Registry validity, Prefix relationships, deprecation, Preferred-Value replacement, and canonicalization beyond that fixed RFC list remain outside this pure policy until a dated/versioned IANA-registry contract exists. + +Causal fix `c94c9345008cf6e339a9f90f59afccc18adea39f` removes SP trimming from the explicit locale path. The raw persisted/session value now reaches the RFC 5646 structural checks unchanged, so surrounding SP is rejected as malformed. The `Accept-Language` parser keeps its SP/HTAB handling because that is an HTTP field-value concern rather than a language-tag normalization rule. + +## Rejected alternatives + +### Keep the broad subtag regex because only the primary language is used + +Rejected. Persisted/session preference is itself product data. A malformed suffix should not be accepted simply because the first subtag happens to map to a supported release locale. + +### Reuse the `Accept-Language` language-range regex + +Rejected. RFC 4647 basic language ranges deliberately have a broader structural grammar than RFC 5646 language tags. Sharing the same regex would preserve the original boundary confusion. + +### Trim explicit locale strings before validation + +Rejected. Trimming converts malformed persisted/session identity data into a different valid language tag and hides the rejecting boundary. HTTP OWS handling belongs only to the `Accept-Language` parser; RFC 5646 language-tag syntax itself does not permit whitespace. + +### Validate every subtag against the live IANA Language Subtag Registry + +Not selected for this slice. RFC 5646 defines well-formedness separately from registry validity. A registry-validity guarantee would require a pinned registry version, update/release process, reproducibility evidence, and failure policy; none belongs implicitly inside this bounded pure-domain repair. + +### Treat private-use-only or grandfathered values as malformed because Naruon cannot render them + +Rejected. Support is a product-release question after syntax classification. Collapsing well-formed unsupported values into `ui_locale_input_invalid` would make error semantics depend on the current release set and contradict the RFC 5646 syntax contract. + +## Downstream contract + +Persistence and future authoring/API boundaries must keep explicit locale values on this RFC 5646 well-formedness path and must not substitute the broader `Accept-Language` range grammar. They also must not trim or otherwise rewrite explicit locale identity before validation. A well-formed tag that lacks a supported release primary fails as unsupported; malformed syntax fails as input-invalid. Fixed regular/irregular grandfathered values must be recognized before ordinary structural guards rather than reinterpreted from apparent subtags. A pinned registry-validity requirement may be added later only as an explicit versioned contract. + +## Traceability + +- Seventeenth RED: `478bd95fdbd2b6601fae1ab70c3513abf0f985fe`. +- Seventeenth causal source fix: `3a87209306ad1b3dee67cb9e0e9e513dfb7a4f3a`. +- Eighteenth RED: `8d9a0f67a47b26eb1e36a7bc65681470033c6f74`. +- Eighteenth causal source fix: `aceef1b282d948411fe9b04b8cd08750fcdacfe8`. +- Nineteenth RED: `d23d4095d1c2ac8542f1ac247b6fee87fcb9485c`. +- Nineteenth causal source fix: `dc077bec9ae47e1421fa678ae43451f65784c410`. +- Twenty-fourth RED: `2ff28c208b8fd5a83bee3e179f4d66b31831be43`. +- Twenty-fourth causal source fix: `c94c9345008cf6e339a9f90f59afccc18adea39f`. +- Twenty-fifth pre-existing RED: predecessor exact `97279b9dfd8d47d3033007d36398ccf7aa9e6943`, focused `zh-min-nan` regression. +- Twenty-fifth causal source fix: `8666ef7bf2451f4e4ca8e93c69d24d5a9fb6436c`. +- Canonical product Gap: #1731. +- Product/technical Gap ledger owner: #1602. + +## References + +Phillips, A., & Davis, M. (2009). *Tags for identifying languages* (BCP 47; RFC 5646). RFC Editor. https://doi.org/10.17487/RFC5646 + +Phillips, A., & Davis, M. (2006). *Matching of language tags* (BCP 47; RFC 4647). RFC Editor. https://doi.org/10.17487/RFC4647 diff --git a/docs/doctoring/ui-localization-identity-resource-bounds.md b/docs/doctoring/ui-localization-identity-resource-bounds.md new file mode 100644 index 000000000..f493c4fb0 --- /dev/null +++ b/docs/doctoring/ui-localization-identity-resource-bounds.md @@ -0,0 +1,38 @@ +# UI Localization Catalog identity resource bounds + +## Status and owned boundary + +**Proposed.** This record belongs to #1740's pure UI Localization Catalog policy. It changes only validation of product-owned `screen_key` and `message_key` identities; persistence, Alembic ownership, HTTP routing, browser caching and UI composition remain outside this PR. + +## Verified finding + +The policy described `screen_key` and `message_key` as stable bounded product identities, but the implementation bounded only their character grammar. Any syntactically valid lowercase identifier could therefore grow without limit before later persistence, indexing, cache-key construction and observability boundaries. + +That was a contract mismatch. The future catalog schema uses `(screen_key, message_key)` as durable identity, so accepting unbounded valid identifiers in the pure policy would push resource control downstream and make database/index/API behavior deployment-dependent. + +## RED and causal repair + +RED `c0d802d33bf68ca6a8fbe70e4aaee5726b6d4a65` adds executable boundaries for both identities: + +- a valid 128-character `screen_key` is accepted and the 129th character is rejected with `ui_screen_key_invalid`; +- a valid 128-character `message_key` is accepted and the 129th character is rejected with `ui_message_key_invalid`. + +Causal fix `c40e2aa244249878405896fc1255d8c1df9d9dc5` introduces `_MAX_SCREEN_KEY_CHARS = 128` and `_MAX_MESSAGE_KEY_CHARS = 128` before the existing regex acceptance. Over-limit values fail closed and are never truncated. + +The 128-character limits are Naruon product constraints, not PostgreSQL or IETF constants. They are intentionally aligned with the existing explicit-locale ceiling so product-owned textual identifiers have a deterministic upper bound before persistence and cache/index composition. The future persistence owner may choose narrower physical column/index constraints only if the API/domain contract is updated first; it must not silently truncate these identities. + +## Rejected alternatives + +Relying on PostgreSQL `text`, B-tree implementation limits, reverse proxies, or application-memory pressure is rejected because none of those is the domain contract. Truncation is rejected because `(screen_key, message_key)` is semantic identity: shortening it can alias two different product messages. Bounding only the eventual database column is also rejected because this pure policy is callable before persistence exists. + +## Verification boundary + +The RED and causal fix are committed. This runtime could not obtain an exact repository checkout because outbound Git resolution is unavailable, so no local/container PASS is claimed. Current-head hosted CI/security and qualifying independent post-last-push review remain the acceptance authority. + +Future persistence/API tests must prove that the same 128/128 contract is preserved at write/read boundaries, that no transport or migration silently truncates values, and that indexed screen-scoped lookup remains within the buyer-path performance budget. + +## TRACEABILITY + +PostgreSQL Global Development Group. (2026). *PostgreSQL 18 documentation: Character types*. https://www.postgresql.org/docs/18/datatype-character.html + +Naruon's limits are product architecture decisions layered above PostgreSQL. PostgreSQL `text` does not supply the semantic identity ceiling required by the product; the domain therefore owns and tests the limit explicitly. diff --git a/docs/doctoring/ui-localization-negotiation-resource-bounds.md b/docs/doctoring/ui-localization-negotiation-resource-bounds.md new file mode 100644 index 000000000..7055a08d7 --- /dev/null +++ b/docs/doctoring/ui-localization-negotiation-resource-bounds.md @@ -0,0 +1,56 @@ +# UI localization negotiation resource bounds + +## Status and owned boundary + +This record belongs to #1740's pure UI Localization Catalog policy. It does not add an HTTP route, persistence, browser cache, or a new transport error mapping. The purpose is to keep locale negotiation work bounded before later API owners expose the policy on a request path. + +## Verified finding + +The earlier policy bounded only RFC 9110 empty-list tolerance. Valid non-empty input remained attacker-controlled in three dimensions: + +- an explicit persisted/session locale could contain an arbitrarily long syntactically valid basic language range; +- one `Accept-Language` field value could be arbitrarily long before normalization and comma splitting; +- one bounded-size field could still contain an unnecessarily large number of non-empty language ranges. + +That leaves parser CPU/allocation proportional to unconstrained input. In particular, `candidate.split(",")` materializes the whole list after the raw string has already been accepted, so bounding only ignored empty members is not a complete resource boundary. + +## RED and causal repair + +RED `978fdbbc6135fa88a8d43e0971ed075ae546f9de` adds executable edges for inputs that were previously accepted: + +- explicit locale length 128 characters is accepted and the next 9-character subtag is rejected with `ui_locale_input_invalid`; +- 64 non-empty `Accept-Language` members are accepted, while the 65th fails with `ui_accept_language_invalid`; +- an 8192-character syntactically valid `Accept-Language` range is accepted, while extending it by another 9-character subtag fails with `ui_accept_language_invalid`. + +Causal fix `0fdb5371eef63d6d7abe87ac14b69269c8519ff8` introduces explicit product limits before expensive normalization/list processing: + +- `_MAX_LOCALE_TAG_CHARS = 128`; +- `_MAX_ACCEPT_LANGUAGE_CHARS = 8192`; +- `_MAX_ACCEPT_LANGUAGE_MEMBERS = 64`; +- the existing `_MAX_ACCEPT_LANGUAGE_EMPTY_MEMBERS = 32` remains unchanged. + +The policy rejects over-limit values; it never truncates them. The 128/8192/64 values are Naruon product limits, not IETF constants. They are intentionally explicit so later transport, observability, load-test, and security evidence can verify the same boundary instead of relying on framework defaults. + +## Standards rationale + +RFC 9110 §5.4 does not define a universal field-size limit. It explicitly allows a server to refuse a request field value larger than it is willing to process and requires an appropriate 4xx response rather than silently ignoring an oversized request field. The future HTTP owner must map the stable policy failure consistently with that transport contract; this pure-policy slice does not choose the endpoint/status code. + +RFC 4647 §4.4 allows language-range length restrictions comparable to language-tag restrictions. RFC 5646 §4.4.1 states that implementations may refuse language tags above a specified length, requires such limits and their failure behavior to be documented, and requires limited-buffer protocols to support at least 35 characters. Naruon's 128-character explicit-locale budget exceeds that minimum and rejects rather than truncates. + +The 8192-character `Accept-Language` value budget and 64 non-empty-member budget are application resource limits. They preserve ample interoperability headroom for ordinary preference lists while preventing one localization request from controlling unbounded split, regex, ranking, and duplicate-processing work. Outer HTTP servers/proxies may impose stricter limits; this policy does not weaken those boundaries. + +## Rejected alternatives + +Relying only on reverse-proxy, ASGI-server, or browser header limits is rejected because the pure policy is callable outside the eventual HTTP route and framework limits vary by deployment. Truncation is rejected because it can change language preference semantics. Bounding only empty list members is rejected because valid non-empty ranges and one very long range still leave work attacker-controlled. + +## Verification boundary + +The source and focused regression are committed. This automation environment does not contain an exact repository checkout, so no full-suite/local-container PASS is claimed here. Hosted checks and qualifying independent review must be reacquired on the final exact head after this documentation commit. Future HTTP/k6 evidence should additionally verify that an over-limit request is rejected before buyer-path work and that normal negotiation remains within the product p95 budget. + +## TRACEABILITY + +Fielding, R. T., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). RFC Editor. https://doi.org/10.17487/RFC9110 + +Phillips, A., & Davis, M. (2006). *Matching of language tags* (RFC 4647, BCP 47). RFC Editor. https://doi.org/10.17487/RFC4647 + +Phillips, A., & Davis, M. (2009). *Tags for identifying languages* (RFC 5646, BCP 47). RFC Editor. https://doi.org/10.17487/RFC5646 diff --git a/docs/doctoring/ui-localization-placeholder-canonical-syntax.md b/docs/doctoring/ui-localization-placeholder-canonical-syntax.md new file mode 100644 index 000000000..0cb464359 --- /dev/null +++ b/docs/doctoring/ui-localization-placeholder-canonical-syntax.md @@ -0,0 +1,39 @@ +# UI Localization Catalog: canonical placeholder syntax + +## Decision status + +**Proposed.** This note records the thirteenth ordinary-forward repair in #1740. It changes only the pure placeholder-policy boundary; persistence, API delivery, browser rendering, catalog publication and authoring remain outside this PR. + +## Finding + +The catalog contract already says that published placeholders are simple named fields such as `{account_name}` and that conversion flags and format specifications are forbidden. The implementation used `string.Formatter.parse()` and rejected a field when `format_spec` was truthy or `conversion` was non-null. + +That was insufficient for the source form `{account_name:}`. Python's format-string grammar treats the colon as the delimiter that introduces a `format_spec`, but an empty format specification has the same formatting result as the default. `Formatter.parse()` therefore yields an empty `format_spec` for both `{account_name}` and `{account_name:}`. The previous policy lost the lexical distinction and silently accepted a non-canonical spelling that its own contract said was forbidden. + +This matters even though both forms render the same value. Catalog source is versioned publication data. If alternate lexical spellings are accepted as equivalent after parsing, source identity and validation semantics depend on lossy runtime normalization instead of the product contract. + +## Decision + +The policy now validates the replacement-field source before `Formatter.parse()` normalizes it. Escaped literal braces remain literals. For a real replacement field, the presence of `:` or `!` is rejected with `ui_placeholder_schema_invalid` before the parsed field-name/schema comparison runs. + +The canonical accepted form remains exactly a simple named field such as `{account_name}`. `{account_name:}`, `{count:03d}`, `{account_name!r}`, attribute/index traversal and malformed fields are not alternate spellings of the schema; they are invalid publication input. + +RED `8b19c7dfe43e3dca3c1e4b8980c0a9998a3aec43` adds the empty-format-specifier regression. Causal fix `6ed310e1065940fa14c258b48412490e7b98d751` preserves lexical operator information before the standard-library parser erases the empty-spec distinction. + +## Rejected alternatives + +Treating `{name:}` as equivalent to `{name}` is rejected because the product boundary explicitly forbids format-specifier syntax and published source identity should not depend on an implementation detail of `Formatter.parse()`. + +Replacing the full parser with a bespoke interpolation engine is also rejected. Python's parser remains the authority for brace balancing and field decomposition; the product adds only the narrow lexical check required to preserve the stricter catalog grammar. + +## Verification + +The canonical policy suite now requires `{account_name:}` to fail with `ui_placeholder_schema_invalid`, alongside existing rejection of non-empty format specifications and conversions. Existing escaped-literal behavior remains owned by the same suite. + +No local/container PASS or predecessor hosted receipt is claimed in this runtime. Current-head full-suite, coverage, security and independent review evidence must be reacquired on the exact post-repair head. + +## TRACEABILITY + +Python Software Foundation. (2026). *string — Common string operations: Format string syntax* (Python 3.14 documentation). https://docs.python.org/3.14/library/string.html + +Python 3.14 defines a replacement field as a `field_name` optionally followed by `! conversion` and/or `: format_spec`, and documents that an empty format specification produces the same result as the default formatting behavior. `Formatter.parse()` returns the parsed `format_spec` value rather than preserving whether an empty `:` delimiter was present in the original source. Naruon therefore retains a stricter, source-aware catalog grammar on top of the standard parser rather than equating semantically similar but lexically different publication inputs. diff --git a/docs/doctoring/ui-localization-rfc4647-wildcard-ordering.md b/docs/doctoring/ui-localization-rfc4647-wildcard-ordering.md new file mode 100644 index 000000000..a307d1d15 --- /dev/null +++ b/docs/doctoring/ui-localization-rfc4647-wildcard-ordering.md @@ -0,0 +1,40 @@ +# UI localization RFC 4647 wildcard ordering repair + +## Decision status + +**Proposed.** This note extends the pure UI Localization Catalog policy owned by #1740 and does not create persistence, API, browser, Storybook, ontology-label, or LLM-translation ownership. + +## Verified findings + +The first repair found that the policy retained only supported concrete language ranges before applying RFC 4647 lookup wildcard ordering. That changed the meaning of a valid priority list such as: + +```text +Accept-Language: *;q=0.9, pt-BR;q=0.8 +``` + +RFC 4647 lookup says `*` is skipped when another language range follows in the language priority list. The later `pt-BR` range still exists even though Naruon's current release has no Portuguese resource. Lookup therefore tries the later concrete range, finds no supported resource, and reaches the product default. The prior implementation discarded `pt-BR` before deciding whether the wildcard had a later concrete range, so it incorrectly attributed the selected default to `accept_language` rather than `product_default`. + +A later standards recheck found the symmetric provenance defect when `*` is the only effective positive range. RFC 4647 §3.4 states that `*` does not convey enough information to choose a language tag: when it is the only range, or when no language range follows it, lookup computes the default value. The policy instead materialized the configured product default as an `accept_language` result, so `Accept-Language: *;q=0.9` selected the same locale value but reported the wrong authority tier. The defect also appeared with a non-Korean configured default. + +These are authority/provenance defects, not necessarily selected-locale defects. `UiLocaleSelection.selection_source` is product state used to explain which authority tier selected the locale, so wildcard control flow and unsupported concrete ranges cannot be collapsed into a fabricated header-selected locale. + +## RED and causal repairs + +- RED `2ebe9758cb915841d86330bbabcd618f3f091d71` adds `*;q=0.9, pt-BR;q=0.8` and requires `(ko, product_default)`. +- Causal fix `1664d9707d4b5b9366b422f8faf612418ecd7789` records every positive-quality concrete range's priority before filtering against the eight supported product locales. Wildcard handling then skips the wildcard whenever a later positive concrete range remains in the RFC lookup priority order. Unsupported ranges still never become supported locale candidates. +- RED `deaf07941cfc2b607c2148f434ce1196a3db29d9` pins wildcard-only provenance for both the ordinary Korean default and a custom supported default: `*;q=0.9` must return `(ko, product_default)`, and the same input with `product_default="fr"` must return `(fr, product_default)`. +- Causal fix `1ce7b6ed5883beda305ea2552ae5af00509510c7` stops a wildcard from manufacturing an `accept_language` selection when no supported q=0 exclusion changes the fallback set. The caller therefore computes the configured product default and records `selection_source=product_default`. Existing supported q=0 exclusions remain meaningful: when the header explicitly excludes the configured default, wildcard fallback may still choose the first non-excluded release locale and truthfully remain header-dependent. + +The repair deliberately does not generalize Naruon into a full BCP 47 registry. Release resources remain exactly `ko/en/ja/zh/vi/es/de/fr`; the change preserves protocol ordering and exclusion metadata long enough to compute the correct selection authority. + +## Verification boundary + +The source and focused regressions are committed to the exact #1740 branch. This runtime cannot obtain an exact repository checkout, so no local/container PASS is claimed. Hosted checks and an independent post-last-push review must be reacquired on the final exact head; predecessor receipts do not transfer. + +## TRACEABILITY + +Phillips, A., & Davis, M. (2006). *Matching of language tags* (RFC 4647, BCP 47). RFC Editor. https://doi.org/10.17487/RFC4647 + +Fielding, R. T., Nottingham, M., & Reschke, J. (2022). *HTTP semantics* (RFC 9110). RFC Editor. https://doi.org/10.17487/RFC9110 + +RFC 4647 §3.4 specifies that the special range `*` is skipped when later language ranges remain, and that when `*` is the only range or no range follows it, the default value is computed. RFC 4647 §3.4.1 requires the application to define that defaulting behavior. RFC 9110 supplies the weighted `Accept-Language` language-priority list. Naruon preserves those ordering and defaulting rules while reducing successful matches to its eight release locales.