Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 85 additions & 3 deletions lib/polytypo/engine/rules/dash_shared.rb
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,21 @@ def digit?(cp)
cp >= DIGIT_ZERO && cp <= DIGIT_NINE
end

# ranges.md 3.2a CLOSED-SYMBOL (spec 1.3.0): the symbols conventionally written closed up
# to a number. A literal code-point set, never a Unicode category test -- a category makes
# the verdict depend on which Unicode version a runtime was built against, and the five
# runtimes must agree. The currency part is the U+20A0-U+20CF block by its own bounds, not
# the subset assigned in some Unicode version: the assigned subset drifts between
# releases, block bounds do not.
def closed_up_symbol?(cp)
return true if cp == 0x0024
return true if cp >= 0x00A2 && cp <= 0x00A5
return true if cp >= 0x20A0 && cp <= 0x20CF
return true if cp == 0x0025 || cp == 0x2030 || cp == 0x2031

cp == 0x00B0
end

# BREAK, including Engine::LINE_MARKER: a member of BREAK for every rule everywhere
# (modes.md 3.2).
def break?(cp)
Expand Down Expand Up @@ -207,6 +222,63 @@ def effective_neighbour(cp, from, step)
cp[i]
end

# ranges.md 3.2a: a token's flanks after the closed-up-symbol walk, with the digit runs
# the guards and the replacement then read. `left`/`right` are L\u2032/R\u2032;
# `outer_left`/`outer_right` are the matched outer symbols' indices, or nil.
RangeFlanks = Data.define(:left, :right, :a, :b, :outer_left, :outer_right)

# ranges.md 3.2 and 3.2a -- is this token a range candidate, and where are its digit runs?
# nil means it is not one, which is the signal that the token belongs to `dashes`.
#
# Both sides are decided from the ORIGINAL left/right, simultaneously; a side consumes a
# closed-up symbol only when the opposite member repeats the same code point. An unmatched
# symbol leaves the flank a non-DIGIT, so "$15-\u20ac20" and "15-$20" are not candidates
# and do not change hands.
def range_flanks(cp, left, right)
n = cp.length
inner_right = if right >= 0 && right < n && closed_up_symbol?(cp[right]) &&
right + 1 < n && digit?(cp[right + 1])
cp[right]
end
inner_left = if left.positive? && left < n && closed_up_symbol?(cp[left]) &&
digit?(cp[left - 1])
cp[left]
end

l = inner_left.nil? ? left : left - 1
r = inner_right.nil? ? right : right + 1
return nil if l.negative? || l >= n || r.negative? || r >= n
return nil unless digit?(cp[l]) && digit?(cp[r])

a = l
a -= 1 while a.positive? && digit?(cp[a - 1])
b = r
b += 1 while b + 1 < n && digit?(cp[b + 1])

outer_left = nil
unless inner_right.nil?
outer_left = effective_index(cp, a - 1, -1)
return nil if outer_left.nil? || cp[outer_left] != inner_right
end
outer_right = nil
unless inner_left.nil?
outer_right = effective_index(cp, b + 1, 1)
return nil if outer_right.nil? || cp[outer_right] != inner_left
end

RangeFlanks.new(left: l, right: r, a: a, b: b, outer_left: outer_left,
outer_right: outer_right)
end

# T1's reach is transparent to one CLOSED-SYMBOL on either end of a digit run (spec
# 1.3.0): the run it protects may be a range member carrying an outer symbol.
def skip_closed_up_symbol(cp, from, step)
i = effective_index(cp, from, step)
return from if i.nil? || !closed_up_symbol?(cp[i])

i + step
end

# dashes.md 3.2 step 8 (T1) -- the spacing-transition guard. A tight token must not become
# spaced when doing so would insert a U+0020 between itself and a digit run that has
# another dash on its far side, read through effective neighbours (3.2b). Shared because a
Expand All @@ -215,10 +287,17 @@ def effective_neighbour(cp, from, step)
def spacing_transition_blocked?(cp, left, right)
n = cp.length

# Spec 1.3.0, position p1: step over a CLOSED-SYMBOL between the token and the run.
left -= 1 if left >= 0 && left < n && closed_up_symbol?(cp[left]) && left.positive? &&
digit?(cp[left - 1])
right += 1 if right >= 0 && right < n && closed_up_symbol?(cp[right]) &&
right + 1 < n && digit?(cp[right + 1])

if left >= 0 && left < n && digit?(cp[left])
d = left
d -= 1 while d.positive? && digit?(cp[d - 1])
i1 = effective_index(cp, d - 1, -1)
# Position p2: and over one at the far end of the run.
i1 = effective_index(cp, skip_closed_up_symbol(cp, d - 1, -1), -1)
one = i1.nil? ? Polytypo::Engine::NONE : cp[i1]
two = i1.nil? ? Polytypo::Engine::NONE : effective_neighbour(cp, i1 - 1, -1)
return true if dash_union?(one)
Expand All @@ -228,7 +307,7 @@ def spacing_transition_blocked?(cp, left, right)
if right >= 0 && right < n && digit?(cp[right])
d = right
d += 1 while d + 1 < n && digit?(cp[d + 1])
i1 = effective_index(cp, d + 1, 1)
i1 = effective_index(cp, skip_closed_up_symbol(cp, d + 1, 1), 1)
one = i1.nil? ? Polytypo::Engine::NONE : cp[i1]
two = i1.nil? ? Polytypo::Engine::NONE : effective_neighbour(cp, i1 + 1, 1)
return true if dash_union?(one)
Expand Down Expand Up @@ -309,7 +388,10 @@ def find_tokens(cp)
left_cp = cp[left]
right_cp = cp[right]

next if crossed_joiner && !(digit?(left_cp) && digit?(right_cp))
# dashes.md 3.2a: re-entry across a joiner is only ever a bound range `ranges`
# produced on an earlier pass. Spec 1.3.0 reads that condition after the
# closed-up-symbol walk, so a bound range carrying symbols re-enters the same way.
next if crossed_joiner && range_flanks(cp, left, right).nil?
next if break?(left_cp) || break?(right_cp)

# dashes.md 3.2 step 6 -- isolation guard.
Expand Down
5 changes: 4 additions & 1 deletion lib/polytypo/engine/rules/dashes.rb
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,10 @@ def scan(cp, locale_data, _ctx)
DashShared.find_tokens(cp).each do |token|
# A digit-flanked token is `ranges`' territory, never `dashes`' -- declined
# unconditionally, whether or not `ranges` is enabled (operator decision, spec 0.5.0).
next if DashShared.digit?(token.left_cp) && DashShared.digit?(token.right_cp)
# A range candidate is `ranges`' territory, never `dashes`'. Since spec 1.3.0 a
# candidate may carry a matched closed-up symbol on a flank (ranges.md 3.2a), which
# is why this is range_flanks rather than a digit test on both flanks.
next unless DashShared.range_flanks(cp, token.left, token.right).nil?

# dashes.md 3.4 P5 -- authored en-dash mark-identity veto (spec 0.6.0). A run
# consisting of exactly one U+2013 is declined unconditionally: every locale, tight or
Expand Down
44 changes: 24 additions & 20 deletions lib/polytypo/engine/rules/ranges.rb
Original file line number Diff line number Diff line change
Expand Up @@ -38,17 +38,19 @@ def non_decreasing?(cp, left_start, right_start, length)
true
end

# ranges.md 3.3, G1-G5. left/right are the token's post-joiner-walk flank indices (both
# already known to be DIGIT by the caller).
def guards_pass?(cp, left, right)
n = cp.length
a = left
a -= 1 while a.positive? && DashShared.digit?(cp[a - 1])
b = right
b += 1 while b + 1 < n && DashShared.digit?(cp[b + 1])

before = DashShared.effective_neighbour(cp, a - 1, -1)
after = DashShared.effective_neighbour(cp, b + 1, 1)
# ranges.md 3.2, G1-G5, over the flanks and digit runs 3.2a's walk produced. before/after
# read past a matched outer closed-up symbol, so G1-G3 judge the text in front of the
# whole member rather than the symbol itself -- which is what declines "US$15-$20" on G1.
def guards_pass?(cp, flanks)
left = flanks.left
right = flanks.right
a = flanks.a
b = flanks.b

before_from = flanks.outer_left || a
after_from = flanks.outer_right || b
before = DashShared.effective_neighbour(cp, before_from - 1, -1)
after = DashShared.effective_neighbour(cp, after_from + 1, 1)

# G1 -- no letter adjacency.
return false if Polytypo::Engine::UnicodeUtil.letter?(before)
Expand Down Expand Up @@ -78,26 +80,28 @@ def scan(cp, locale_data, _ctx)
style = locale_data["dash"]["range"]

DashShared.find_tokens(cp).each do |token|
# ranges.md 3.2 -- a range candidate iff both flanks are DIGIT. `ranges` never
# processes any other token shape; that is `dashes`' territory, and `dashes` declines
# a digit-flanked token unconditionally too (operator decision, spec 0.5.0) -- neither
# rule reinterprets the other's shape, whether or not `ranges` is enabled.
next unless DashShared.digit?(token.left_cp) && DashShared.digit?(token.right_cp)
# ranges.md 3.2, 3.2a -- a candidate iff both flanks are DIGIT once a matched
# closed-up symbol has been walked over. Anything else is `dashes`' territory, and
# `dashes` declines a candidate unconditionally too (operator decision, spec 0.5.0).
flanks = DashShared.range_flanks(cp, token.left, token.right)
next if flanks.nil?

next unless guards_pass?(cp, token.left, token.right)
next unless guards_pass?(cp, flanks)

# "none": the locale has no verified range convention, so nothing is substituted --
# not a fallback to dash.parenthetical, nothing (ranges.md 2).
next if style == "none"

if DashShared.spaced_style?(style)
# T1: a tight token may not become spaced across a digit run that has a far dash.
# T1/T2 read the walked flanks: ranges.md 3.2a makes cp[L']/cp[R'] what every
# shared guard sees once a closed-up symbol has been consumed.
next if token.lsp.zero? && token.rsp.zero? &&
DashShared.spacing_transition_blocked?(cp, token.left, token.right)
DashShared.spacing_transition_blocked?(cp, flanks.left, flanks.right)

# T2: the emitted U+0020 must not land where `spaces` (order 10) would delete it.
next if DashShared.strip_before_or_close_bracket?(token.right_cp)
next if DashShared.open_bracket?(token.left_cp)
next if DashShared.strip_before_or_close_bracket?(cp[flanks.right])
next if DashShared.open_bracket?(cp[flanks.left])
end

# ranges.md 3.3.1: never make an edit whose entire content is invisible. Try the
Expand Down
115 changes: 115 additions & 0 deletions spec/closed_up_symbol_ranges_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
# frozen_string_literal: true

require "spec_helper"

# spec/rules/ranges.md §3.2a (spec 1.3.0) -- a symbol repeated closed up on both members, and the
# dashes.md §3.2 step 8 amendment it forced.
RSpec.describe "ranges: closed-up symbols" do
wj = "⁠"
en = "–"
em = "—"

def ranges(text, locale = "en-US")
Polytypo.transform(text, locale: locale, rules: { "ranges" => true })
end

describe "§3.2a candidacy" do
it "admits a repeated prefix" do
expect(ranges("$15-$20")).to eq("$15#{wj}#{en}#{wj}$20")
expect(ranges("€15-€20", "de-DE")).to eq("€15#{wj}#{en}#{wj}€20")
end

it "admits a repeated suffix, the mirror case" do
expect(ranges("35%-50%")).to eq("35%#{wj}#{en}#{wj}50%")
expect(ranges("35%-50%", "ru")).to eq("35%#{wj}#{em}#{wj}50%")
expect(ranges("15°-20°")).to eq("15°#{wj}#{en}#{wj}20°")
end

it "requires the same code point on both members" do
expect(ranges("$15-€20")).to eq("$15-€20")
expect(ranges("15-$20")).to eq("15-$20")
expect(ranges("15%-20")).to eq("15%-20")
end

it "leaves the elided forms exactly as they were" do
expect(ranges("$15-20")).to eq("$15#{wj}#{en}#{wj}20")
expect(ranges("15-20%")).to eq("15#{wj}#{en}#{wj}20%")
end

it "consumes at most one code point per side" do
expect(ranges("US$15-US$20")).to eq("US$15-US$20")
expect(ranges("15°C-20°C")).to eq("15°C-20°C")
end

it "decides both flanks from the original indices, simultaneously" do
expect(ranges("%15%-%20%")).to eq("%15%#{wj}#{en}#{wj}%20%")
end

it "converts a hyphen an author typed between an existing joiner pair" do
expect(ranges("$15#{wj}-#{wj}$20")).to eq("$15#{wj}#{en}#{wj}$20")
end
end

describe "CLOSED-SYMBOL is an enumeration, not a category" do
it "admits a block member no other example names" do
expect(ranges("₹15-₹20")).to eq("₹15#{wj}#{en}#{wj}₹20")
expect(ranges("₴100-₴200")).to eq("₴100#{wj}#{en}#{wj}₴200")
end

it "refuses a currency sign outside the set" do
# U+058F and U+FFE5 are Sc in Unicode's own classification and deliberately excluded.
expect(ranges("֏15-֏20")).to eq("֏15-֏20")
expect(ranges("¥15-¥20")).to eq("¥15-¥20")
end
end

describe "dashes.md §3.2 step 8" do
witnesses = ["a—$15-$20", "35%-50%—b", "a--15% - 20%", "$1 - $1--a"]

%w[de-DE ru en-GB fi].each do |locale|
it "is a fixed point on every witness in #{locale}" do
witnesses.each do |text|
once = Polytypo.transform(text, locale: locale, rules: { "ranges" => true })
twice = Polytypo.transform(once, locale: locale, rules: { "ranges" => true })
expect(twice).to eq(once)
end
end
end

it "composes both transparency positions on one side" do
# Eleven tokens — out of reach of the canonical exhaustive sweep.
["a--$15% - $20%", "$15% - $20%--a", "a--$15% - $20%--b"].each do |text|
once = ranges(text, "de-DE")
expect(ranges(once, "de-DE")).to eq(once)
end
end

it "makes the symbol shape behave exactly like its all-digit analogue" do
expect(ranges("a—15-20", "de-DE")).to eq("a—15-20")
expect(ranges("a—$15-$20", "de-DE")).to eq("a—$15-$20")
expect(ranges("a--1 - 1", "de-DE")).to eq("a--1 - 1")
expect(ranges("a--$1 - $1", "de-DE")).to eq("a--$1 - $1")
end

it "leaves a tight-parenthetical locale untouched, the boundary of the cost" do
expect(Polytypo.transform("price--$50--drop", locale: "en-US")).to eq("price—$50—drop")
expect(Polytypo.transform("Anstieg--50%--war", locale: "de-DE")).to eq("Anstieg--50%--war")
expect(Polytypo.transform("Anstieg--50--war", locale: "de-DE")).to eq("Anstieg--50--war")
end

it "reads effective neighbours in T1's right branch, symbol or no symbol" do
expect(ranges("a--15#{wj} - 20", "de-DE")).to eq("a--15#{wj} - 20")
expect(ranges("a--$15#{wj}-#{wj}$20", "de-DE")).to eq("a--$15#{wj}-#{wj}$20")
end
end

describe "the accepted cost, with default options" do
it "no longer converts a spaced token between matched symbols" do
expect(Polytypo.transform("$15 - $20", locale: "en-US")).to eq("$15 - $20")
end

it "still converts one between unmatched symbols, which never became a candidate" do
expect(Polytypo.transform("$15 - €20", locale: "en-US")).to eq("$15—€20")
end
end
end
Loading