Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 123 additions & 3 deletions src/Engine/Rules/DashShared.php
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,108 @@ public static function isDigit(int $cp): bool
* BREAK, including Sentinels::LINE_MARKER: a member of BREAK for every rule everywhere
* (modes.md 3.2).
*/
/**
* ranges.md 3.2a CLOSED-SYMBOL (spec 1.3.0): the symbols conventionally written closed up to
* a number. A literal code-point set, never a Unicode category test -- a category makes the
* verdict depend on which Unicode version a runtime was built against, and the five runtimes
* must agree. The currency part is the U+20A0-U+20CF block by its own bounds, not the subset
* assigned in some Unicode version: the assigned subset drifts between releases, block
* bounds do not.
*/
public static function isClosedUpSymbol(int $cp): bool
{
return $cp === 0x0024
|| ($cp >= 0x00A2 && $cp <= 0x00A5)
|| ($cp >= 0x20A0 && $cp <= 0x20CF)
|| $cp === 0x0025
|| $cp === 0x2030
|| $cp === 0x2031
|| $cp === 0x00B0;
}

/**
* ranges.md 3.2 and 3.2a -- is this token a range candidate, and where are its digit runs?
* null means it is not one, which is the signal that the token belongs to `dashes`.
*
* Both sides are decided from the ORIGINAL left/right, simultaneously; a side consumes a
* closed-up symbol only when the opposite member repeats the same code point. An unmatched
* symbol leaves the flank a non-DIGIT, so `$15-<euro>20` and `15-$20` are not candidates and
* do not change hands.
*
* @param int[] $cp
* @return array{left: int, right: int, a: int, b: int, outerLeft: int, outerRight: int}|null
*/
public static function rangeFlanks(array $cp, int $left, int $right): ?array
{
$n = count($cp);
$innerRight = null;
if (
$right >= 0 && $right < $n && self::isClosedUpSymbol($cp[$right])
&& $right + 1 < $n && self::isDigit($cp[$right + 1])
) {
$innerRight = $cp[$right];
}
$innerLeft = null;
if (
$left > 0 && $left < $n && self::isClosedUpSymbol($cp[$left])
&& self::isDigit($cp[$left - 1])
) {
$innerLeft = $cp[$left];
}

$l = $innerLeft === null ? $left : $left - 1;
$r = $innerRight === null ? $right : $right + 1;
if ($l < 0 || $l >= $n || $r < 0 || $r >= $n) {
return null;
}
if (!self::isDigit($cp[$l]) || !self::isDigit($cp[$r])) {
return null;
}

$a = $l;
while ($a > 0 && self::isDigit($cp[$a - 1])) {
$a--;
}
$b = $r;
while ($b + 1 < $n && self::isDigit($cp[$b + 1])) {
$b++;
}

$outerLeft = -1;
if ($innerRight !== null) {
$outerLeft = self::effectiveIndex($cp, $a - 1, -1);
if ($outerLeft < 0 || $cp[$outerLeft] !== $innerRight) {
return null;
}
}
$outerRight = -1;
if ($innerLeft !== null) {
$outerRight = self::effectiveIndex($cp, $b + 1, 1);
if ($outerRight < 0 || $cp[$outerRight] !== $innerLeft) {
return null;
}
}

return ['left' => $l, 'right' => $r, 'a' => $a, 'b' => $b,
'outerLeft' => $outerLeft, 'outerRight' => $outerRight];
}

/**
* T1's reach is transparent to one CLOSED-SYMBOL on either end of a digit run (spec 1.3.0):
* the run it protects may be a range member carrying an outer symbol.
*
* @param int[] $cp
*/
public static function skipClosedUpSymbol(array $cp, int $from, int $step): int
{
$i = self::effectiveIndex($cp, $from, $step);
if ($i < 0 || !self::isClosedUpSymbol($cp[$i])) {
return $from;
}

return $i + $step;
}

public static function isBreak(int $cp): bool
{
return $cp === self::LF || $cp === self::CR || $cp === self::VT || $cp === self::FF ||
Expand Down Expand Up @@ -284,12 +386,27 @@ public static function isSpacingTransitionBlocked(array $cp, int $left, int $rig
{
$n = count($cp);

// Spec 1.3.0, position p1: step over a CLOSED-SYMBOL between the token and the run.
if (
$left > 0 && $left < $n && self::isClosedUpSymbol($cp[$left])
&& self::isDigit($cp[$left - 1])
) {
$left--;
}
if (
$right >= 0 && $right < $n && self::isClosedUpSymbol($cp[$right])
&& $right + 1 < $n && self::isDigit($cp[$right + 1])
) {
$right++;
}

if ($left >= 0 && $left < $n && self::isDigit($cp[$left])) {
$d = $left;
while ($d > 0 && self::isDigit($cp[$d - 1])) {
$d--;
}
$i1 = self::effectiveIndex($cp, $d - 1, -1);
// Position p2: and over one at the far end of the run.
$i1 = self::effectiveIndex($cp, self::skipClosedUpSymbol($cp, $d - 1, -1), -1);
$one = $i1 >= 0 ? $cp[$i1] : Sentinels::NONE;
$two = $i1 >= 0 ? self::effectiveNeighbour($cp, $i1 - 1, -1) : Sentinels::NONE;
if (self::isDashUnion($one)) {
Expand All @@ -305,7 +422,7 @@ public static function isSpacingTransitionBlocked(array $cp, int $left, int $rig
while ($d + 1 < $n && self::isDigit($cp[$d + 1])) {
$d++;
}
$i1 = self::effectiveIndex($cp, $d + 1, 1);
$i1 = self::effectiveIndex($cp, self::skipClosedUpSymbol($cp, $d + 1, 1), 1);
$one = $i1 >= 0 ? $cp[$i1] : Sentinels::NONE;
$two = $i1 >= 0 ? self::effectiveNeighbour($cp, $i1 + 1, 1) : Sentinels::NONE;
if (self::isDashUnion($one)) {
Expand Down Expand Up @@ -423,7 +540,10 @@ public static function findTokens(array $cp): array
$leftCp = $cp[$left];
$rightCp = $cp[$right];

if ($crossedJoiner && !(self::isDigit($leftCp) && self::isDigit($rightCp))) {
// dashes.md 3.2a: re-entry across a joiner is only ever a bound range `ranges`
// produced on an earlier pass. Spec 1.3.0 reads that condition after the
// closed-up-symbol walk, so a bound range carrying symbols re-enters the same way.
if ($crossedJoiner && self::rangeFlanks($cp, $left, $right) === null) {
continue;
}
if (self::isBreak($leftCp) || self::isBreak($rightCp)) {
Expand Down
5 changes: 4 additions & 1 deletion src/Engine/Rules/DashesRule.php
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,10 @@ public static function scan(array $cp, array $localeData, RuleContext $ctx): arr
// A digit-flanked token is `ranges`' territory, never `dashes`' -- declined
// unconditionally, whether or not `ranges` is enabled (operator decision, spec
// 0.5.0).
if (DashShared::isDigit($token['leftCp']) && DashShared::isDigit($token['rightCp'])) {
// A range candidate is `ranges`' territory, never `dashes`'. Since spec 1.3.0 a
// candidate may carry a matched closed-up symbol on a flank (ranges.md 3.2a), which
// is why this is rangeFlanks rather than a digit test on both flanks.
if (DashShared::rangeFlanks($cp, $token['left'], $token['right']) !== null) {
continue;
}

Expand Down
46 changes: 24 additions & 22 deletions src/Engine/Rules/RangesRule.php
Original file line number Diff line number Diff line change
Expand Up @@ -54,25 +54,24 @@ private static function isNonDecreasing(array $cp, int $leftStart, int $rightSta
}

/**
* ranges.md 3.3, G1-G5. left/right are the token's post-joiner-walk flank indices (both
* already known to be DIGIT by the caller).
* ranges.md 3.2, G1-G5, over the flanks and digit runs 3.2a's walk produced. before/after
* read past a matched outer closed-up symbol, so G1-G3 judge the text in front of the whole
* member rather than the symbol itself -- which is what declines `US$15-$20` on G1.
*
* @param int[] $cp
* @param array{left: int, right: int, a: int, b: int, outerLeft: int, outerRight: int} $flanks
*/
private static function guardsPass(array $cp, int $left, int $right): bool
private static function guardsPass(array $cp, array $flanks): bool
{
$n = count($cp);
$a = $left;
while ($a > 0 && DashShared::isDigit($cp[$a - 1])) {
$a--;
}
$b = $right;
while ($b + 1 < $n && DashShared::isDigit($cp[$b + 1])) {
$b++;
}
$left = $flanks['left'];
$right = $flanks['right'];
$a = $flanks['a'];
$b = $flanks['b'];

$before = DashShared::effectiveNeighbour($cp, $a - 1, -1);
$after = DashShared::effectiveNeighbour($cp, $b + 1, 1);
$beforeFrom = $flanks['outerLeft'] >= 0 ? $flanks['outerLeft'] : $a;
$afterFrom = $flanks['outerRight'] >= 0 ? $flanks['outerRight'] : $b;
$before = DashShared::effectiveNeighbour($cp, $beforeFrom - 1, -1);
$after = DashShared::effectiveNeighbour($cp, $afterFrom + 1, 1);

// G1 -- no letter adjacency.
if (UnicodeUtil::isLetter($before)) {
Expand Down Expand Up @@ -119,16 +118,17 @@ public static function scan(array $cp, array $localeData, RuleContext $ctx): arr
$style = $localeData['dash']['range'];

foreach (DashShared::findTokens($cp) as $token) {
// ranges.md 3.2 -- a range candidate iff both flanks are DIGIT. `ranges` never
// processes any other token shape; that is `dashes`' territory, and `dashes`
// declines a digit-flanked token unconditionally too (operator decision, spec
// ranges.md 3.2, 3.2a -- a candidate iff both flanks are DIGIT once a matched
// closed-up symbol has been walked over. Anything else is `dashes`' territory, and
// `dashes` declines a candidate unconditionally too (operator decision, spec
// 0.5.0) -- neither rule reinterprets the other's shape, whether or not `ranges` is
// enabled.
if (!DashShared::isDigit($token['leftCp']) || !DashShared::isDigit($token['rightCp'])) {
$flanks = DashShared::rangeFlanks($cp, $token['left'], $token['right']);
if ($flanks === null) {
continue;
}

if (!self::guardsPass($cp, $token['left'], $token['right'])) {
if (!self::guardsPass($cp, $flanks)) {
continue;
}

Expand All @@ -141,18 +141,20 @@ public static function scan(array $cp, array $localeData, RuleContext $ctx): arr
if (DashShared::isSpacedStyle($style)) {
// T1: a tight token may not become spaced across a digit run that has a far
// dash.
// T1/T2 read the walked flanks: ranges.md 3.2a makes cp[L']/cp[R'] what every
// shared guard sees once a closed-up symbol has been consumed.
if (
$token['lsp'] === 0 && $token['rsp'] === 0 &&
DashShared::isSpacingTransitionBlocked($cp, $token['left'], $token['right'])
DashShared::isSpacingTransitionBlocked($cp, $flanks['left'], $flanks['right'])
) {
continue;
}
// T2: the emitted U+0020 must not land where `spaces` (order 10) would delete
// it.
if (DashShared::isStripBeforeOrCloseBracket($token['rightCp'])) {
if (DashShared::isStripBeforeOrCloseBracket($cp[$flanks['right']])) {
continue;
}
if (DashShared::isOpenBracket($token['leftCp'])) {
if (DashShared::isOpenBracket($cp[$flanks['left']])) {
continue;
}
}
Expand Down
118 changes: 118 additions & 0 deletions tests/ClosedUpSymbolRangesTest.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
<?php

declare(strict_types=1);

namespace Polytypo\Tests;

use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\TestCase;
use Polytypo\Polytypo;

/**
* spec/rules/ranges.md §3.2a (spec 1.3.0) -- a symbol repeated closed up on both members, and the
* dashes.md §3.2 step 8 amendment it forced.
*/
final class ClosedUpSymbolRangesTest extends TestCase
{
private const WJ = "\u{2060}";
private const EN = "\u{2013}";
private const EM = "\u{2014}";

private static function ranges(string $input, string $locale = 'en-US'): string
{
return Polytypo::transform($input, $locale, 'text', null, ['ranges' => true]);
}

/** @return array<string, array{0: string, 1: string, 2: string}> */
public static function candidacyCases(): array
{
$wj = self::WJ;
$en = self::EN;
$em = self::EM;

return [
'repeated prefix' => ['en-US', '$15-$20', "\$15{$wj}{$en}{$wj}\$20"],
'repeated prefix, euro' => ['de-DE', '€15-€20', "€15{$wj}{$en}{$wj}€20"],
'repeated suffix' => ['en-US', '35%-50%', "35%{$wj}{$en}{$wj}50%"],
'repeated suffix, em locale' => ['ru', '35%-50%', "35%{$wj}{$em}{$wj}50%"],
'degree sign' => ['en-US', '15°-20°', "15°{$wj}{$en}{$wj}20°"],
'mismatched symbols decline' => ['en-US', '$15-€20', '$15-€20'],
'half-written declines' => ['en-US', '15-$20', '15-$20'],
'half-written suffix declines' => ['en-US', '15%-20', '15%-20'],
'elided prefix still converts' => ['en-US', '$15-20', "\$15{$wj}{$en}{$wj}20"],
'elided suffix still converts' => ['en-US', '15-20%', "15{$wj}{$en}{$wj}20%"],
'multi-code-point prefix declines' => ['en-US', 'US$15-US$20', 'US$15-US$20'],
'multi-code-point suffix declines' => ['en-US', '15°C-20°C', '15°C-20°C'],
'both flanks decided together' => ['en-US', '%15%-%20%', "%15%{$wj}{$en}{$wj}%20%"],
'spaced token is now a range' => ['en-US', '$15 - $20', "\$15{$wj}{$en}{$wj}\$20"],
'hyphen between a joiner pair' => ['en-US', "\$15{$wj}-{$wj}\$20", "\$15{$wj}{$en}{$wj}\$20"],
];
}

#[DataProvider('candidacyCases')]
public function testCandidacy(string $locale, string $input, string $want): void
{
$this->assertSame($want, self::ranges($input, $locale));
}

/**
* CLOSED-SYMBOL is the U+20A0-U+20CF block by its bounds plus a fixed list, never an Sc
* category test: a port enumerating only the symbols it saw in tests fails the first two, and
* a port using a category test fails the last two.
*/
public function testSymbolSetBoundary(): void
{
$wj = self::WJ;
$en = self::EN;
$this->assertSame("₹15{$wj}{$en}{$wj}₹20", self::ranges('₹15-₹20'));
$this->assertSame("₴100{$wj}{$en}{$wj}₴200", self::ranges('₴100-₴200'));
$this->assertSame('֏15-֏20', self::ranges('֏15-֏20'));
$this->assertSame('¥15-¥20', self::ranges('¥15-¥20'));
}

/**
* dashes.md §3.2 step 8: widening a range member widens what a `dashes` edit elsewhere can
* disturb. Each witness drifted on the second pass before T1's reach became
* CLOSED-SYMBOL-transparent, and each is inert in its all-digit shape.
*/
public function testT1IsAFixedPointOnEveryWitness(): void
{
$witnesses = ['a—$15-$20', '35%-50%—b', 'a--15% - 20%', '$1 - $1--a'];
foreach (['de-DE', 'ru', 'en-GB', 'fi'] as $locale) {
foreach ($witnesses as $input) {
$once = self::ranges($input, $locale);
$this->assertSame($once, self::ranges($once, $locale), "{$locale} {$input}");
}
}

// Both transparency positions on one side at once -- eleven tokens, out of reach of the
// canonical exhaustive sweep.
foreach (['a--$15% - $20%', '$15% - $20%--a', 'a--$15% - $20%--b'] as $input) {
$once = self::ranges($input, 'de-DE');
$this->assertSame($once, self::ranges($once, 'de-DE'), $input);
}
}

public function testT1Boundary(): void
{
// T1 applies only when the chosen form is spaced, which is why the repair is T1 and not
// the unconditional cluster guard: that one would have taken en-US with it.
$this->assertSame('price—$50—drop', Polytypo::transform('price--$50--drop', 'en-US'));
$this->assertSame('Anstieg--50%--war', Polytypo::transform('Anstieg--50%--war', 'de-DE'));
$this->assertSame('Anstieg--50--war', Polytypo::transform('Anstieg--50--war', 'de-DE'));

// The accepted cost, with default options, and its limit: only a MATCHED symbol moves a
// token between rules.
$this->assertSame('$15 - $20', Polytypo::transform('$15 - $20', 'en-US'));
$this->assertSame('$15—€20', Polytypo::transform('$15 - €20', 'en-US'));
}

public function testRightBranchReadsEffectiveNeighbours(): void
{
// dashes.md §3.2 step 8 disagreed with §3.2b in its own text through spec 1.2.0; no
// implementation ever did. The space ends the cluster, so step 7 does not cover this.
$wj = self::WJ;
$this->assertSame("a--15{$wj} - 20", self::ranges("a--15{$wj} - 20", 'de-DE'));
$this->assertSame("a--\$15{$wj}-{$wj}\$20", self::ranges("a--\$15{$wj}-{$wj}\$20", 'de-DE'));
}
}
Loading