From 64a00b2e2a8d94a2f765d349fed5a2f1f6e574d3 Mon Sep 17 00:00:00 2001 From: Iurii Rogulia Date: Fri, 18 Sep 2026 14:26:26 +0300 Subject: [PATCH] nbsp: do not insert before the semicolon that ends a character reference polytypo/polytypo#22, spec 1.3.0 (nbsp.md 3.3 step 4). text mode has no markup concept, so fr -- whose narrowBeforePunctuation lists ";" -- inserted U+202F before the ";" that ends a reference: 'Bonjour : oui' and 'Tom & Jerry' stopped being what they were. Only locales that put a space before ";" could reach it, so the defect was invisible in English. N1/N2 now walk left from a ";" over ASCII alphanumerics, optionally one "#", and decline if an "&" closes the walk. Shape, not the named-reference table: declining on '¬aname;' costs nothing, and no runtime carries thousands of entries for it. The walk is bounded at 32, one past the longest named reference. Five tests, two pinning what the guard does not do. Locally 1126 tests pass; the markdown-mode cases and the packaging tests need dependencies this machine does not have and are left to CI. All 1073 canonical fixtures that run here pass with spec 1.3.0 locale data generated in, each idempotent. --- src/polytypo/_engine/rules/nbsp.py | 31 ++++++++++++++++++++++++++++++ tests/test_rules/test_nbsp.py | 22 +++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/polytypo/_engine/rules/nbsp.py b/src/polytypo/_engine/rules/nbsp.py index b792679..948340d 100644 --- a/src/polytypo/_engine/rules/nbsp.py +++ b/src/polytypo/_engine/rules/nbsp.py @@ -17,6 +17,11 @@ SP = 0x20 NBSP = 0xA0 NNBSP = 0x202F +SEMICOLON = 0x3B +AMPERSAND = 0x26 +HASH = 0x23 +# The longest HTML named reference is 31 code points ("CounterClockwiseContourIntegral"). +MAX_CHARACTER_REFERENCE_NAME = 32 NOBREAK = frozenset({NBSP, NNBSP}) OTHER_SPACE = frozenset( { @@ -105,6 +110,27 @@ def scan(cp: list[int], locale_data: dict[str, Any], ctx: RuleContext) -> list[E # ---------------------------------------------------------------- N1 / N2 +def _ends_character_reference(cp: list[int], i: int) -> bool: + """nbsp.md 3.3 step 4: do the code points left of this ";" have the shape of a character + reference? A bounded left walk over ASCII alphanumerics, optionally one "#", then "&". Shape, + not the HTML named-reference table -- declining on "¬aname;" costs nothing, and no runtime + carries thousands of entries for it. The bound is the longest named reference (31) plus one. + """ + j = i - 1 + while j >= 0 and _is_ascii_alphanumeric(cp[j]): + j -= 1 + length = i - 1 - j + if length < 1 or length > MAX_CHARACTER_REFERENCE_NAME: + return False + if j >= 0 and cp[j] == HASH: + j -= 1 + return j >= 0 and cp[j] == AMPERSAND + + +def _is_ascii_alphanumeric(c: int) -> bool: + return 0x30 <= c <= 0x39 or 0x41 <= c <= 0x5A or 0x61 <= c <= 0x7A + + def _before_punctuation_common( cp: list[int], target_set: set[int], @@ -139,6 +165,11 @@ def _before_punctuation_common( # Quote-glyph guard. if (prev in SPACELIKE and _at(cp, i - 2) in openish) or prev in openish: continue + # Character-reference guard (nbsp.md 3.3 step 4, spec 1.3.0). text mode has no markup + # concept, so a locale listing ";" used to insert before the ";" that *ends* a reference + # and `Bonjour : oui` stopped being what it was. + if cp[i] == SEMICOLON and _ends_character_reference(cp, i): + continue left = prev if left == already_correct: continue # already correct diff --git a/tests/test_rules/test_nbsp.py b/tests/test_rules/test_nbsp.py index 713bb5e..ca83c3b 100644 --- a/tests/test_rules/test_nbsp.py +++ b/tests/test_rules/test_nbsp.py @@ -34,3 +34,25 @@ def test_n1_right_context_accepts_span_boundary_marker() -> None: # nbsp.md 3.3 step 2 (spec 1.2.0): the inline marker is in nbsp's CLOSEISH. out = polytypo.html.transform("gel : il", locale="fr") assert out == f"gel{NBSP}: il" + + +def test_n2_character_reference_guard_numeric() -> None: + # nbsp.md 3.3 step 4 (spec 1.3.0): the ";" closes " ", so N2 emits nothing. The colon + # takes nothing either -- step 1's run guard sees a mark to its left. + assert polytypo.transform("Bonjour : oui", locale="fr") == "Bonjour : oui" + + +def test_n2_character_reference_guard_named() -> None: + assert polytypo.transform("Tom & Jerry", locale="fr") == "Tom & Jerry" + + +def test_n2_character_reference_guard_tests_shape_not_the_entity_table() -> None: + assert polytypo.transform("a ¬aname; b", locale="fr") == "a ¬aname; b" + + +def test_n2_ordinary_semicolon_still_binds() -> None: + assert polytypo.transform("Oui ; non", locale="fr") == f"Oui{NNBSP}; non" + + +def test_n2_semicolon_after_digit_run_without_ampersand_binds() -> None: + assert polytypo.transform("Section 4; suite", locale="fr") == f"Section 4{NNBSP}; suite"