From 44b8e3dc32099855a52a020b0581c07677f76bf0 Mon Sep 17 00:00:00 2001 From: Iurii Rogulia Date: Sat, 19 Sep 2026 12:30:32 +0300 Subject: [PATCH] ranges: admit a symbol repeated closed up on both members MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements spec 1.3.0's ranges.md 3.2a and the dashes.md step 8 amendment it forced (polytypo/polytypo@bb9a54e, canonical issue #23), porting polytypo-js@5bf847f. `$15-$20` and `35%-50%` are range candidates now; through 1.2.0 `$15-20` converted and `$15-$20` did not, because the flank next to the dash was the symbol. The discriminator is closed-up versus spaced, per Chicago 9.19 — so `$` and `%` are one rule and `15 kg-20 kg` is out of scope. CLOSED_SYMBOL is a literal code-point set, never a Unicode category test: the currency part is the U+20A0-U+20CF block by its own bounds, because the assigned subset drifts between runtimes built against different UCD releases and block bounds do not. Tests pin both sides — U+20B9/U+20B4, which no other case names, and U+058F/ U+FFE5, currency signs deliberately outside the set that an Sc test would admit. range_flanks() is the whole feature: both sides decided from the original flanks simultaneously, a side's walk taken only if the symbol is matched on the opposite member, None for a non-candidate — which is the signal `dashes` reads to stay out. So `$15-€20` and `15-$20` never change hands, and G4/G5 still compare digit runs and never see a symbol. T1 (spacing_transition_guard_blocks) is the part that could not be skipped. Widening a range member widens what a `dashes` edit elsewhere can disturb: an unamended T1 reads the symbol as the neighbour, stops one code point short of the dash, lets a tight token become spaced, and that U+0020 flips a neighbouring range's guard on the next pass. The reach is now CLOSED-SYMBOL-transparent at two positions per side. Four witnesses in the tests, one per position and side, plus the composed shape the canonical exhaustive sweep could not reach. Python's two-step lookout already read effective neighbours on both branches, which is what spec 1.3.0's step 8 now says in text; a test pins it. Verified beyond the suite: all 1219 runnable text/html canonical fixture cases at spec 1.3.0, across all 18 locales, pass and are idempotent against this build. 1191 tests pass, ruff, ruff format, mypy --strict and the packaging test are clean. Co-Authored-By: Claude Opus 5 --- src/polytypo/_engine/rules/_dash_shared.py | 115 +++++++++++++++++++-- src/polytypo/_engine/rules/dashes.py | 8 +- src/polytypo/_engine/rules/ranges.py | 33 +++--- tests/test_rules/test_ranges.py | 98 ++++++++++++++++++ 4 files changed, 227 insertions(+), 27 deletions(-) diff --git a/src/polytypo/_engine/rules/_dash_shared.py b/src/polytypo/_engine/rules/_dash_shared.py index c552cbc..f59c2d6 100644 --- a/src/polytypo/_engine/rules/_dash_shared.py +++ b/src/polytypo/_engine/rules/_dash_shared.py @@ -21,6 +21,18 @@ JOINER = 0x2060 DASH_OR_INERT = DASH | INERT_DASH +# ranges.md 3.2a CLOSED-SYMBOL (spec 1.3.0): the symbols conventionally written closed up to a +# number. A literal code-point set, never a Unicode category test -- a category makes the verdict +# depend on which Unicode version a runtime was built against, and the five runtimes must agree. +# The currency part is the U+20A0-U+20CF block by its own bounds, not the subset assigned in some +# Unicode version: the assigned subset drifts between releases, block bounds do not. +CLOSED_SYMBOL = ( + frozenset({0x24}) + | frozenset(range(0xA2, 0xA6)) + | frozenset(range(0x20A0, 0x20D0)) + | frozenset({0x25, 0x2030, 0x2031, 0xB0}) +) + def at(cp: list[int], i: int) -> int: if i < 0 or i >= len(cp): @@ -100,7 +112,10 @@ def find_token(cp: list[int], i: int) -> tuple[DashToken | None, int]: left_cp = cp[L] right_cp = cp[R] - if crossed_joiner and not (left_cp in DIGIT and right_cp in DIGIT): + # dashes.md 3.2a: re-entry across a joiner is only ever a bound range `ranges` produced on + # an earlier pass. Spec 1.3.0 reads that condition after the closed-up-symbol walk, so + # `$15-$20` re-enters the same way `1914-1918` does. + if crossed_joiner and range_flanks(cp, L, R) is None: return None, e if left_cp in BREAK or right_cp in BREAK: return None, e @@ -133,6 +148,73 @@ def find_token(cp: list[int], i: int) -> tuple[DashToken | None, int]: return token, e +@dataclass(frozen=True) +class RangeFlanks: + """ranges.md 3.2a: a token's flanks after the closed-up-symbol walk, with the digit runs the + guards and the replacement then read. `left`/`right` are L'/R'; `outer_left`/`outer_right` + are the matched outer symbols' indices, or -1.""" + + left: int + right: int + a: int + b: int + outer_left: int + outer_right: int + + +def range_flanks(cp: list[int], left: int, right: int) -> RangeFlanks | None: + """ranges.md 3.2 and 3.2a -- is this token a range candidate, and where are its digit runs? + None means it is not one, which is the signal that the token belongs to `dashes`. + + Both sides are decided from the ORIGINAL left/right, simultaneously; a side consumes a + closed-up symbol only when the opposite member repeats the same code point. An unmatched + symbol leaves the flank a non-DIGIT, so `$15-\u20ac20` and `15-$20` are not candidates and do + not change hands.""" + n = len(cp) + inner_right = ( + cp[right] + if at(cp, right) in CLOSED_SYMBOL and right + 1 < n and cp[right + 1] in DIGIT + else None + ) + inner_left = ( + cp[left] if at(cp, left) in CLOSED_SYMBOL and left > 0 and cp[left - 1] in DIGIT else None + ) + + left_index = left if inner_left is None else left - 1 + right_index = right if inner_right is None else right + 1 + if at(cp, left_index) not in DIGIT or at(cp, right_index) not in DIGIT: + return None + + a = left_index + while a > 0 and cp[a - 1] in DIGIT: + a -= 1 + b = right_index + while b + 1 < n and cp[b + 1] in DIGIT: + b += 1 + + outer_left = -1 + if inner_right is not None: + outer_left = _walk_across_joiners(cp, a - 1, -1) + if at(cp, outer_left) != inner_right: + return None + outer_right = -1 + if inner_left is not None: + outer_right = _walk_across_joiners(cp, b + 1, 1) + if at(cp, outer_right) != inner_left: + return None + + return RangeFlanks(left_index, right_index, a, b, outer_left, outer_right) + + +def _skip_closed_up_symbol(cp: list[int], from_idx: int, step: int) -> int: + """T1's reach is transparent to one CLOSED-SYMBOL on either end of a digit run (spec 1.3.0): + the run it protects may be a range member carrying an outer symbol.""" + i = _walk_across_joiners(cp, from_idx, step) + if at(cp, i) not in CLOSED_SYMBOL: + return from_idx + return i + step + + def _cluster_has_multiple_dash_runs(cp: list[int], s: int, e: int) -> bool: n = len(cp) cluster_alphabet = DASH | INERT_DASH | DIGIT | frozenset({JOINER}) @@ -163,11 +245,11 @@ def effective_neighbor(cp: list[int], i: int, step: int) -> int: return at(cp, j) -def _two_step_lookout_blocks(cp: list[int], from_idx: int, step: int) -> bool: - """Reading effective neighbors outward from `from_idx` (exclusive) in `step` direction: +def _two_step_lookout_blocks(cp: list[int], start_idx: int, step: int) -> bool: + """Reading effective neighbors outward from `start_idx` (inclusive) in `step` direction: if the first is DASH/INERT-DASH, or the first is space-like and the second (one step further out) is DASH/INERT-DASH, this returns True.""" - first_idx = _walk_across_joiners(cp, from_idx + step, step) + first_idx = _walk_across_joiners(cp, start_idx, step) first = at(cp, first_idx) if first in DASH_OR_INERT: return True @@ -186,18 +268,25 @@ def spacing_transition_guard_blocks(cp: list[int], token: DashToken) -> bool: n = len(cp) L, R = token.left_idx, token.right_idx + # Spec 1.3.0, position p1: step over a CLOSED-SYMBOL sitting between the token and the run. + if at(cp, L) in CLOSED_SYMBOL and L > 0 and cp[L - 1] in DIGIT: + L -= 1 + if at(cp, R) in CLOSED_SYMBOL and n > R + 1 and cp[R + 1] in DIGIT: + R += 1 + if 0 <= L < n and cp[L] in DIGIT: d = L while d > 0 and cp[d - 1] in DIGIT: d -= 1 - if _two_step_lookout_blocks(cp, d, -1): + # Position p2: and over one at the far end of the run. + if _two_step_lookout_blocks(cp, _skip_closed_up_symbol(cp, d - 1, -1), -1): return True if 0 <= R < n and cp[R] in DIGIT: d = R while d < n - 1 and cp[d + 1] in DIGIT: d += 1 - if _two_step_lookout_blocks(cp, d, 1): + if _two_step_lookout_blocks(cp, _skip_closed_up_symbol(cp, d + 1, 1), 1): return True return False @@ -208,9 +297,15 @@ def spacing_transition_guard_blocks(cp: list[int], token: DashToken) -> bool: OPEN_BRACKET = frozenset({0x28, 0x5B, 0x7B}) -def composition_guard_blocks(cp: list[int], token: DashToken) -> bool: +def composition_guard_blocks( + cp: list[int], token: DashToken, flanks: RangeFlanks | None = None +) -> bool: """T2 (dashes.md 3.2 step 9): for a -spaced replacement, decline if cp[R] is in T2_SET or - CLOSE_BRACKET, or cp[L] is in OPEN_BRACKET.""" - if token.right_cp in T2_SET or token.right_cp in CLOSE_BRACKET: + CLOSE_BRACKET, or cp[L] is in OPEN_BRACKET. `ranges` passes its walked flanks, because + ranges.md 3.2a makes cp[L']/cp[R'] what every shared guard sees once a closed-up symbol has + been consumed.""" + left_cp = token.left_cp if flanks is None else at(cp, flanks.left) + right_cp = token.right_cp if flanks is None else at(cp, flanks.right) + if right_cp in T2_SET or right_cp in CLOSE_BRACKET: return True - return token.left_cp in OPEN_BRACKET + return left_cp in OPEN_BRACKET diff --git a/src/polytypo/_engine/rules/dashes.py b/src/polytypo/_engine/rules/dashes.py index cd18821..a099d45 100644 --- a/src/polytypo/_engine/rules/dashes.py +++ b/src/polytypo/_engine/rules/dashes.py @@ -33,8 +33,12 @@ def scan(cp: list[int], locale_data: dict[str, Any], ctx: RuleContext) -> list[E i = next_i if token is None: continue - if token.left_cp in shared.DIGIT and token.right_cp in shared.DIGIT: - continue # digit-flanked: ranges' territory exclusively, never dashes'. + # A range candidate is `ranges`' territory, never `dashes`' -- declined unconditionally, + # whether or not `ranges` is enabled. Since spec 1.3.0 a candidate may carry a matched + # closed-up symbol on a flank (ranges.md 3.2a), which is why this is `range_flanks` + # rather than a digit test on both flanks. + if shared.range_flanks(cp, token.left_idx, token.right_idx) is not None: + continue edit = _try_parenthetical(cp, token, form) if edit is not None: diff --git a/src/polytypo/_engine/rules/ranges.py b/src/polytypo/_engine/rules/ranges.py index 4749df1..6d5379f 100644 --- a/src/polytypo/_engine/rules/ranges.py +++ b/src/polytypo/_engine/rules/ranges.py @@ -35,30 +35,33 @@ def scan(cp: list[int], locale_data: dict[str, Any], ctx: RuleContext) -> list[E i = next_i if token is None: continue - if token.left_cp not in shared.DIGIT or token.right_cp not in shared.DIGIT: - continue # not a range candidate; dashes' concern + # ranges.md 3.2, 3.2a -- a candidate iff both flanks are DIGIT once a matched closed-up + # symbol has been walked over. Anything else is dashes' concern. + flanks = shared.range_flanks(cp, token.left_idx, token.right_idx) + if flanks is None: + continue - edit = _try_range(cp, token, form) + edit = _try_range(cp, token, flanks, form) if edit is not None: edits.append(edit) return edits -def _try_range(cp: list[int], token: shared.DashToken, form: str) -> Edit | None: - L, R = token.left_idx, token.right_idx - - a = L - while a > 0 and cp[a - 1] in shared.DIGIT: - a -= 1 - b = R - while b < len(cp) - 1 and cp[b + 1] in shared.DIGIT: - b += 1 +def _try_range( + cp: list[int], token: shared.DashToken, flanks: shared.RangeFlanks, form: str +) -> Edit | None: + """G1-G5 over the flanks and digit runs ranges.md 3.2a's walk produced. `before`/`after` read + past a matched outer closed-up symbol, so G1-G3 judge the text in front of the whole member + rather than the symbol itself -- which is what declines `US$15-$20` on G1.""" + L, R, a, b = flanks.left, flanks.right, flanks.a, flanks.b lrun = "".join(chr(c) for c in cp[a : L + 1]) rrun = "".join(chr(c) for c in cp[R : b + 1]) - before = shared.effective_neighbor(cp, a - 1, -1) - after = shared.effective_neighbor(cp, b + 1, 1) + before_from = a if flanks.outer_left < 0 else flanks.outer_left + after_from = b if flanks.outer_right < 0 else flanks.outer_right + before = shared.effective_neighbor(cp, before_from - 1, -1) + after = shared.effective_neighbor(cp, after_from + 1, 1) # G1: no letter adjacency. if before != NONE and is_letter(before): @@ -87,7 +90,7 @@ def _try_range(cp: list[int], token: shared.DashToken, form: str) -> Edit | None if spaced: if token.lsp == 0 and token.rsp == 0 and shared.spacing_transition_guard_blocks(cp, token): return None - if shared.composition_guard_blocks(cp, token): + if shared.composition_guard_blocks(cp, token, flanks): return None span_start, span_end = token.span_start, token.span_end diff --git a/tests/test_rules/test_ranges.py b/tests/test_rules/test_ranges.py index c337f51..c31d63d 100644 --- a/tests/test_rules/test_ranges.py +++ b/tests/test_rules/test_ranges.py @@ -29,3 +29,101 @@ def test_g4_directional_one_two_digit_form() -> None: def test_g5_declines_a_decreasing_pair() -> None: assert _t("a 100-1") == "a 100-1" + + +WJ = "⁠" +EN = "–" +EM = "—" + + +def _t_de(text: str) -> str: + return polytypo.transform(text, locale="de-DE", rules={"ranges": True}) + + +class TestClosedUpSymbols: + """ranges.md 3.2a (spec 1.3.0): a symbol repeated closed up on both members.""" + + def test_admits_a_repeated_prefix(self) -> None: + assert _t("$15-$20") == f"$15{WJ}{EN}{WJ}$20" + assert _t_de("€15-€20") == f"€15{WJ}{EN}{WJ}€20" + + def test_admits_a_repeated_suffix(self) -> None: + # The mirror case: the walk moves the LEFT flank and matches the outer symbol after Rrun. + assert _t("35%-50%") == f"35%{WJ}{EN}{WJ}50%" + assert _t("15°-20°") == f"15°{WJ}{EN}{WJ}20°" + assert polytypo.transform("35%-50%", locale="ru", rules={"ranges": True}) == ( + f"35%{WJ}{EM}{WJ}50%" + ) + + def test_requires_the_same_code_point(self) -> None: + # A currency conversion, not a range; and two half-written shapes. + assert _t("$15-€20") == "$15-€20" + assert _t("15-$20") == "15-$20" + assert _t("15%-20") == "15%-20" + + def test_leaves_the_elided_forms_alone(self) -> None: + assert _t("$15-20") == f"$15{WJ}{EN}{WJ}20" + assert _t("15-20%") == f"15{WJ}{EN}{WJ}20%" + + def test_consumes_at_most_one_code_point_per_side(self) -> None: + assert _t("US$15-US$20") == "US$15-US$20" + assert _t("15°C-20°C") == "15°C-20°C" + + def test_block_membership_is_by_bounds_not_by_category(self) -> None: + # U+20B9 and U+20B4 are block members no other test names; U+058F and U+FFE5 are + # currency signs in Unicode's own classification, deliberately outside CLOSED-SYMBOL. + assert _t("₹15-₹20") == f"₹15{WJ}{EN}{WJ}₹20" + assert _t("₴100-₴200") == f"₴100{WJ}{EN}{WJ}₴200" + assert _t("֏15-֏20") == "֏15-֏20" + assert _t("¥15-¥20") == "¥15-¥20" + + def test_decides_both_flanks_simultaneously(self) -> None: + assert _t("%15%-%20%") == f"%15%{WJ}{EN}{WJ}%20%" + + def test_takes_the_spaced_token_away_from_dashes(self) -> None: + # Through spec 1.2.0 `dashes` converted this with DEFAULT options. + assert polytypo.transform("$15 - $20", locale="en-US") == "$15 - $20" + assert _t("$15 - $20") == f"$15{WJ}{EN}{WJ}$20" + # Only a MATCHED symbol moves a token between rules. + assert polytypo.transform("$15 - €20", locale="en-US") == "$15—€20" + + +class TestClosedUpSymbolsAndT1: + """dashes.md 3.2 step 8 (spec 1.3.0). Widening a range member widens what a `dashes` edit + elsewhere can disturb; each witness drifted on the second pass before T1's reach became + CLOSED-SYMBOL-transparent, and each is inert in its all-digit shape.""" + + WITNESSES = ["a—$15-$20", "35%-50%—b", "a--15% - 20%", "$1 - $1--a"] + + def test_every_witness_is_a_fixed_point(self) -> None: + for locale in ("de-DE", "ru", "en-GB", "fi"): + for text in self.WITNESSES: + once = polytypo.transform(text, locale=locale, rules={"ranges": True}) + assert polytypo.transform(once, locale=locale, rules={"ranges": True}) == once + + def test_composes_both_positions_on_one_side(self) -> None: + for text in ("a--$15% - $20%", "$15% - $20%--a", "a--$15% - $20%--b"): + once = _t_de(text) + assert _t_de(once) == once + + def test_symbol_shape_matches_its_all_digit_analogue(self) -> None: + assert _t_de("a—15-20") == "a—15-20" + assert _t_de("a—$15-$20") == "a—$15-$20" + assert _t_de("a--1 - 1") == "a--1 - 1" + assert _t_de("a--$1 - $1") == "a--$1 - $1" + + def test_tight_parenthetical_locale_is_untouched(self) -> None: + # T1 only applies when the chosen form is spaced — the boundary of the cost, and the + # reason the repair is T1 and not the unconditional cluster guard. + assert polytypo.transform("price--$50--drop", locale="en-US") == "price—$50—drop" + assert polytypo.transform("Anstieg--50%--war", locale="de-DE") == "Anstieg--50%--war" + assert polytypo.transform("Anstieg--50--war", locale="de-DE") == "Anstieg--50--war" + + def test_right_branch_reads_effective_neighbours(self) -> None: + # dashes.md 3.2 step 8 disagreed with 3.2b in its own text through spec 1.2.0; no + # implementation ever did. The space ends the cluster, so step 7 does not cover this. + assert _t_de(f"a--15{WJ} - 20") == f"a--15{WJ} - 20" + assert _t_de(f"a--$15{WJ}-{WJ}$20") == f"a--$15{WJ}-{WJ}$20" + + def test_converts_a_hyphen_between_an_existing_joiner_pair(self) -> None: + assert _t(f"$15{WJ}-{WJ}$20") == f"$15{WJ}{EN}{WJ}$20"