Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
115 changes: 105 additions & 10 deletions src/polytypo/_engine/rules/_dash_shared.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,18 @@
JOINER = 0x2060
DASH_OR_INERT = DASH | INERT_DASH

# ranges.md 3.2a CLOSED-SYMBOL (spec 1.3.0): the symbols conventionally written closed up to a
# number. A literal code-point set, never a Unicode category test -- a category makes the verdict
# depend on which Unicode version a runtime was built against, and the five runtimes must agree.
# The currency part is the U+20A0-U+20CF block by its own bounds, not the subset assigned in some
# Unicode version: the assigned subset drifts between releases, block bounds do not.
CLOSED_SYMBOL = (
frozenset({0x24})
| frozenset(range(0xA2, 0xA6))
| frozenset(range(0x20A0, 0x20D0))
| frozenset({0x25, 0x2030, 0x2031, 0xB0})
)


def at(cp: list[int], i: int) -> int:
if i < 0 or i >= len(cp):
Expand Down Expand Up @@ -100,7 +112,10 @@ def find_token(cp: list[int], i: int) -> tuple[DashToken | None, int]:
left_cp = cp[L]
right_cp = cp[R]

if crossed_joiner and not (left_cp in DIGIT and right_cp in DIGIT):
# dashes.md 3.2a: re-entry across a joiner is only ever a bound range `ranges` produced on
# an earlier pass. Spec 1.3.0 reads that condition after the closed-up-symbol walk, so
# `$15<J>-<J>$20` re-enters the same way `1914<J>-<J>1918` does.
if crossed_joiner and range_flanks(cp, L, R) is None:
return None, e
if left_cp in BREAK or right_cp in BREAK:
return None, e
Expand Down Expand Up @@ -133,6 +148,73 @@ def find_token(cp: list[int], i: int) -> tuple[DashToken | None, int]:
return token, e


@dataclass(frozen=True)
class RangeFlanks:
"""ranges.md 3.2a: a token's flanks after the closed-up-symbol walk, with the digit runs the
guards and the replacement then read. `left`/`right` are L'/R'; `outer_left`/`outer_right`
are the matched outer symbols' indices, or -1."""

left: int
right: int
a: int
b: int
outer_left: int
outer_right: int


def range_flanks(cp: list[int], left: int, right: int) -> RangeFlanks | None:
"""ranges.md 3.2 and 3.2a -- is this token a range candidate, and where are its digit runs?
None means it is not one, which is the signal that the token belongs to `dashes`.

Both sides are decided from the ORIGINAL left/right, simultaneously; a side consumes a
closed-up symbol only when the opposite member repeats the same code point. An unmatched
symbol leaves the flank a non-DIGIT, so `$15-\u20ac20` and `15-$20` are not candidates and do
not change hands."""
n = len(cp)
inner_right = (
cp[right]
if at(cp, right) in CLOSED_SYMBOL and right + 1 < n and cp[right + 1] in DIGIT
else None
)
inner_left = (
cp[left] if at(cp, left) in CLOSED_SYMBOL and left > 0 and cp[left - 1] in DIGIT else None
)

left_index = left if inner_left is None else left - 1
right_index = right if inner_right is None else right + 1
if at(cp, left_index) not in DIGIT or at(cp, right_index) not in DIGIT:
return None

a = left_index
while a > 0 and cp[a - 1] in DIGIT:
a -= 1
b = right_index
while b + 1 < n and cp[b + 1] in DIGIT:
b += 1

outer_left = -1
if inner_right is not None:
outer_left = _walk_across_joiners(cp, a - 1, -1)
if at(cp, outer_left) != inner_right:
return None
outer_right = -1
if inner_left is not None:
outer_right = _walk_across_joiners(cp, b + 1, 1)
if at(cp, outer_right) != inner_left:
return None

return RangeFlanks(left_index, right_index, a, b, outer_left, outer_right)


def _skip_closed_up_symbol(cp: list[int], from_idx: int, step: int) -> int:
"""T1's reach is transparent to one CLOSED-SYMBOL on either end of a digit run (spec 1.3.0):
the run it protects may be a range member carrying an outer symbol."""
i = _walk_across_joiners(cp, from_idx, step)
if at(cp, i) not in CLOSED_SYMBOL:
return from_idx
return i + step


def _cluster_has_multiple_dash_runs(cp: list[int], s: int, e: int) -> bool:
n = len(cp)
cluster_alphabet = DASH | INERT_DASH | DIGIT | frozenset({JOINER})
Expand Down Expand Up @@ -163,11 +245,11 @@ def effective_neighbor(cp: list[int], i: int, step: int) -> int:
return at(cp, j)


def _two_step_lookout_blocks(cp: list[int], from_idx: int, step: int) -> bool:
"""Reading effective neighbors outward from `from_idx` (exclusive) in `step` direction:
def _two_step_lookout_blocks(cp: list[int], start_idx: int, step: int) -> bool:
"""Reading effective neighbors outward from `start_idx` (inclusive) in `step` direction:
if the first is DASH/INERT-DASH, or the first is space-like and the second (one step
further out) is DASH/INERT-DASH, this returns True."""
first_idx = _walk_across_joiners(cp, from_idx + step, step)
first_idx = _walk_across_joiners(cp, start_idx, step)
first = at(cp, first_idx)
if first in DASH_OR_INERT:
return True
Expand All @@ -186,18 +268,25 @@ def spacing_transition_guard_blocks(cp: list[int], token: DashToken) -> bool:
n = len(cp)
L, R = token.left_idx, token.right_idx

# Spec 1.3.0, position p1: step over a CLOSED-SYMBOL sitting between the token and the run.
if at(cp, L) in CLOSED_SYMBOL and L > 0 and cp[L - 1] in DIGIT:
L -= 1
if at(cp, R) in CLOSED_SYMBOL and n > R + 1 and cp[R + 1] in DIGIT:
R += 1

if 0 <= L < n and cp[L] in DIGIT:
d = L
while d > 0 and cp[d - 1] in DIGIT:
d -= 1
if _two_step_lookout_blocks(cp, d, -1):
# Position p2: and over one at the far end of the run.
if _two_step_lookout_blocks(cp, _skip_closed_up_symbol(cp, d - 1, -1), -1):
return True

if 0 <= R < n and cp[R] in DIGIT:
d = R
while d < n - 1 and cp[d + 1] in DIGIT:
d += 1
if _two_step_lookout_blocks(cp, d, 1):
if _two_step_lookout_blocks(cp, _skip_closed_up_symbol(cp, d + 1, 1), 1):
return True

return False
Expand All @@ -208,9 +297,15 @@ def spacing_transition_guard_blocks(cp: list[int], token: DashToken) -> bool:
OPEN_BRACKET = frozenset({0x28, 0x5B, 0x7B})


def composition_guard_blocks(cp: list[int], token: DashToken) -> bool:
def composition_guard_blocks(
cp: list[int], token: DashToken, flanks: RangeFlanks | None = None
) -> bool:
"""T2 (dashes.md 3.2 step 9): for a -spaced replacement, decline if cp[R] is in T2_SET or
CLOSE_BRACKET, or cp[L] is in OPEN_BRACKET."""
if token.right_cp in T2_SET or token.right_cp in CLOSE_BRACKET:
CLOSE_BRACKET, or cp[L] is in OPEN_BRACKET. `ranges` passes its walked flanks, because
ranges.md 3.2a makes cp[L']/cp[R'] what every shared guard sees once a closed-up symbol has
been consumed."""
left_cp = token.left_cp if flanks is None else at(cp, flanks.left)
right_cp = token.right_cp if flanks is None else at(cp, flanks.right)
if right_cp in T2_SET or right_cp in CLOSE_BRACKET:
return True
return token.left_cp in OPEN_BRACKET
return left_cp in OPEN_BRACKET
8 changes: 6 additions & 2 deletions src/polytypo/_engine/rules/dashes.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,12 @@ def scan(cp: list[int], locale_data: dict[str, Any], ctx: RuleContext) -> list[E
i = next_i
if token is None:
continue
if token.left_cp in shared.DIGIT and token.right_cp in shared.DIGIT:
continue # digit-flanked: ranges' territory exclusively, never dashes'.
# A range candidate is `ranges`' territory, never `dashes`' -- declined unconditionally,
# whether or not `ranges` is enabled. Since spec 1.3.0 a candidate may carry a matched
# closed-up symbol on a flank (ranges.md 3.2a), which is why this is `range_flanks`
# rather than a digit test on both flanks.
if shared.range_flanks(cp, token.left_idx, token.right_idx) is not None:
continue

edit = _try_parenthetical(cp, token, form)
if edit is not None:
Expand Down
33 changes: 18 additions & 15 deletions src/polytypo/_engine/rules/ranges.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,30 +35,33 @@ def scan(cp: list[int], locale_data: dict[str, Any], ctx: RuleContext) -> list[E
i = next_i
if token is None:
continue
if token.left_cp not in shared.DIGIT or token.right_cp not in shared.DIGIT:
continue # not a range candidate; dashes' concern
# ranges.md 3.2, 3.2a -- a candidate iff both flanks are DIGIT once a matched closed-up
# symbol has been walked over. Anything else is dashes' concern.
flanks = shared.range_flanks(cp, token.left_idx, token.right_idx)
if flanks is None:
continue

edit = _try_range(cp, token, form)
edit = _try_range(cp, token, flanks, form)
if edit is not None:
edits.append(edit)
return edits


def _try_range(cp: list[int], token: shared.DashToken, form: str) -> Edit | None:
L, R = token.left_idx, token.right_idx

a = L
while a > 0 and cp[a - 1] in shared.DIGIT:
a -= 1
b = R
while b < len(cp) - 1 and cp[b + 1] in shared.DIGIT:
b += 1
def _try_range(
cp: list[int], token: shared.DashToken, flanks: shared.RangeFlanks, form: str
) -> Edit | None:
"""G1-G5 over the flanks and digit runs ranges.md 3.2a's walk produced. `before`/`after` read
past a matched outer closed-up symbol, so G1-G3 judge the text in front of the whole member
rather than the symbol itself -- which is what declines `US$15-$20` on G1."""
L, R, a, b = flanks.left, flanks.right, flanks.a, flanks.b

lrun = "".join(chr(c) for c in cp[a : L + 1])
rrun = "".join(chr(c) for c in cp[R : b + 1])

before = shared.effective_neighbor(cp, a - 1, -1)
after = shared.effective_neighbor(cp, b + 1, 1)
before_from = a if flanks.outer_left < 0 else flanks.outer_left
after_from = b if flanks.outer_right < 0 else flanks.outer_right
before = shared.effective_neighbor(cp, before_from - 1, -1)
after = shared.effective_neighbor(cp, after_from + 1, 1)

# G1: no letter adjacency.
if before != NONE and is_letter(before):
Expand Down Expand Up @@ -87,7 +90,7 @@ def _try_range(cp: list[int], token: shared.DashToken, form: str) -> Edit | None
if spaced:
if token.lsp == 0 and token.rsp == 0 and shared.spacing_transition_guard_blocks(cp, token):
return None
if shared.composition_guard_blocks(cp, token):
if shared.composition_guard_blocks(cp, token, flanks):
return None

span_start, span_end = token.span_start, token.span_end
Expand Down
98 changes: 98 additions & 0 deletions tests/test_rules/test_ranges.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,3 +29,101 @@ def test_g4_directional_one_two_digit_form() -> None:

def test_g5_declines_a_decreasing_pair() -> None:
assert _t("a 100-1") == "a 100-1"


WJ = "⁠"
EN = "–"
EM = "—"


def _t_de(text: str) -> str:
return polytypo.transform(text, locale="de-DE", rules={"ranges": True})


class TestClosedUpSymbols:
"""ranges.md 3.2a (spec 1.3.0): a symbol repeated closed up on both members."""

def test_admits_a_repeated_prefix(self) -> None:
assert _t("$15-$20") == f"$15{WJ}{EN}{WJ}$20"
assert _t_de("€15-€20") == f"€15{WJ}{EN}{WJ}€20"

def test_admits_a_repeated_suffix(self) -> None:
# The mirror case: the walk moves the LEFT flank and matches the outer symbol after Rrun.
assert _t("35%-50%") == f"35%{WJ}{EN}{WJ}50%"
assert _t("15°-20°") == f"15°{WJ}{EN}{WJ}20°"
assert polytypo.transform("35%-50%", locale="ru", rules={"ranges": True}) == (
f"35%{WJ}{EM}{WJ}50%"
)

def test_requires_the_same_code_point(self) -> None:
# A currency conversion, not a range; and two half-written shapes.
assert _t("$15-€20") == "$15-€20"
assert _t("15-$20") == "15-$20"
assert _t("15%-20") == "15%-20"

def test_leaves_the_elided_forms_alone(self) -> None:
assert _t("$15-20") == f"$15{WJ}{EN}{WJ}20"
assert _t("15-20%") == f"15{WJ}{EN}{WJ}20%"

def test_consumes_at_most_one_code_point_per_side(self) -> None:
assert _t("US$15-US$20") == "US$15-US$20"
assert _t("15°C-20°C") == "15°C-20°C"

def test_block_membership_is_by_bounds_not_by_category(self) -> None:
# U+20B9 and U+20B4 are block members no other test names; U+058F and U+FFE5 are
# currency signs in Unicode's own classification, deliberately outside CLOSED-SYMBOL.
assert _t("₹15-₹20") == f"₹15{WJ}{EN}{WJ}₹20"
assert _t("₴100-₴200") == f"₴100{WJ}{EN}{WJ}₴200"
assert _t("֏15-֏20") == "֏15-֏20"
assert _t("¥15-¥20") == "¥15-¥20"

def test_decides_both_flanks_simultaneously(self) -> None:
assert _t("%15%-%20%") == f"%15%{WJ}{EN}{WJ}%20%"

def test_takes_the_spaced_token_away_from_dashes(self) -> None:
# Through spec 1.2.0 `dashes` converted this with DEFAULT options.
assert polytypo.transform("$15 - $20", locale="en-US") == "$15 - $20"
assert _t("$15 - $20") == f"$15{WJ}{EN}{WJ}$20"
# Only a MATCHED symbol moves a token between rules.
assert polytypo.transform("$15 - €20", locale="en-US") == "$15—€20"


class TestClosedUpSymbolsAndT1:
"""dashes.md 3.2 step 8 (spec 1.3.0). Widening a range member widens what a `dashes` edit
elsewhere can disturb; each witness drifted on the second pass before T1's reach became
CLOSED-SYMBOL-transparent, and each is inert in its all-digit shape."""

WITNESSES = ["a—$15-$20", "35%-50%—b", "a--15% - 20%", "$1 - $1--a"]

def test_every_witness_is_a_fixed_point(self) -> None:
for locale in ("de-DE", "ru", "en-GB", "fi"):
for text in self.WITNESSES:
once = polytypo.transform(text, locale=locale, rules={"ranges": True})
assert polytypo.transform(once, locale=locale, rules={"ranges": True}) == once

def test_composes_both_positions_on_one_side(self) -> None:
for text in ("a--$15% - $20%", "$15% - $20%--a", "a--$15% - $20%--b"):
once = _t_de(text)
assert _t_de(once) == once

def test_symbol_shape_matches_its_all_digit_analogue(self) -> None:
assert _t_de("a—15-20") == "a—15-20"
assert _t_de("a—$15-$20") == "a—$15-$20"
assert _t_de("a--1 - 1") == "a--1 - 1"
assert _t_de("a--$1 - $1") == "a--$1 - $1"

def test_tight_parenthetical_locale_is_untouched(self) -> None:
# T1 only applies when the chosen form is spaced — the boundary of the cost, and the
# reason the repair is T1 and not the unconditional cluster guard.
assert polytypo.transform("price--$50--drop", locale="en-US") == "price—$50—drop"
assert polytypo.transform("Anstieg--50%--war", locale="de-DE") == "Anstieg--50%--war"
assert polytypo.transform("Anstieg--50--war", locale="de-DE") == "Anstieg--50--war"

def test_right_branch_reads_effective_neighbours(self) -> None:
# dashes.md 3.2 step 8 disagreed with 3.2b in its own text through spec 1.2.0; no
# implementation ever did. The space ends the cluster, so step 7 does not cover this.
assert _t_de(f"a--15{WJ} - 20") == f"a--15{WJ} - 20"
assert _t_de(f"a--$15{WJ}-{WJ}$20") == f"a--$15{WJ}-{WJ}$20"

def test_converts_a_hyphen_between_an_existing_joiner_pair(self) -> None:
assert _t(f"$15{WJ}-{WJ}$20") == f"$15{WJ}{EN}{WJ}$20"
Loading