From 5844a0298d727de4351229b19174264c3a66a323 Mon Sep 17 00:00:00 2001 From: Iurii Rogulia Date: Fri, 18 Sep 2026 19:25:10 +0300 Subject: [PATCH] Add Polytypo::analyze(), the reported-edits entry point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements spec 1.3.0's spec/rules/analyze.md (polytypo/polytypo@691c180, canonical issue #15), porting polytypo-js@1a2ef57. Same pipeline as transform(), reporting what it would do instead of doing it: Polytypo::analyze('Wait... "really"?', 'en-US') // [ Change { ruleId: "ellipsis", start: 4, end: 7, before: "...", after: "โ€ฆ" }, โ€ฆ ] Nothing in the engine changed to support it. Rules already produce edits and the pipeline already applies them, so the whole feature is one structure: an origin map travelling alongside the code-point array (Engine\Origin), holding the input offset each code point came from. A same-length replacement keeps its origins position by position, which is what makes a converted space still point at the space it converted. Two places this could quietly go wrong in PHP specifically, both tested: - Offsets are code points, not bytes. `strpos` on `๐Ÿ˜€ and "this"` says 9 and analyze() says 6. Every ASCII test passes either way, which is what makes it worth an explicit case rather than a comment. - In html mode the rules run over the marker-joined concatenation of spans, so reporting those offsets would pass every text-mode test and be useless to a caller (analyze.md ยง6). Spans::originOfSpans maps them back to document offsets, asserted in a three-span document as well as a single-span one. markdown is rejected by analyze() exactly as transform() rejects it, through the same transformMarkdown() โ€” A1 requires the two entry points to reject the same arguments with the same code, and this runtime implements no markdown dialect. Seventeen tests, including A3 against the entire vendored fixture corpus โ€” empty list exactly when in == out โ€” and analyze.md ยง5's French overlap at the spec's own offsets, spaces at 3โ€“4 and nbsp at 4โ€“4 on `Oui : non`. Full suite 1119 tests, 2907 assertions, 21 skipped (the markdown cases this runtime declines); PHPStan level 8 and phpcs clean. transform()'s behaviour is untouched. Co-Authored-By: Claude Opus 5 --- README.md | 18 ++++ src/Change.php | 23 +++++ src/Engine/Origin.php | 115 +++++++++++++++++++++ src/Engine/Pipeline.php | 41 ++++++++ src/Modes/Runner.php | 40 ++++++++ src/Modes/Spans.php | 28 ++++++ src/Polytypo.php | 72 ++++++++++++++ tests/AnalyzeTest.php | 214 ++++++++++++++++++++++++++++++++++++++++ 8 files changed, 551 insertions(+) create mode 100644 src/Change.php create mode 100644 src/Engine/Origin.php create mode 100644 tests/AnalyzeTest.php diff --git a/README.md b/README.md index 61f77ed..4d209e1 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,24 @@ accepts PHP named arguments: Polytypo::transform($input, 'fr', mode: 'html', rules: ['ranges' => true]); ``` +`Polytypo::analyze` runs the same pipeline and reports what it would do instead of doing it โ€” one +`Polytypo\Change` per edit, each with the rule that made it and code-point offsets into the input +you passed (into the **document**, in `html` mode, not into a span): + +```php +Polytypo::analyze('Wait... "really"?', 'en-US'); +// => [ Change { ruleId: "ellipsis", start: 4, end: 7, before: "...", after: "โ€ฆ" }, +// Change { ruleId: "quotes", start: 8, end: 9, before: "\"", after: "โ€œ" }, ... ] +``` + +Offsets are code points, not bytes: in `๐Ÿ˜€ and "this"` the opening quotation mark is reported at +6, where `strpos` would say 9. It is a report, not a patch. The list is empty exactly when +`Polytypo::transform` would return the input unchanged, and every `ruleId` is a rule that was +enabled for that call โ€” but two rules may touch the same original range (French `spaces` deletes +the space before `:` and `nbsp` puts a no-break one back), so replaying the list is not guaranteed +to reproduce the output. Call `Polytypo::transform` for the text. Full contract: +`spec/rules/analyze.md`. + ### Errors Every error `Polytypo::transform` raises is a `Polytypo\PolytypoException` carrying one of seven diff --git a/src/Change.php b/src/Change.php new file mode 100644 index 0000000..f09ad1e --- /dev/null +++ b/src/Change.php @@ -0,0 +1,23 @@ + U+00A0) still point at the character it converted; anything longer is synthetic + * beyond the positions it covers. + * + * @param int[] $origin + * @param Edit[] $edits + * @return int[] + */ + public static function applyEditsToOrigin(array $origin, array $edits): array + { + if ($edits === []) { + return $origin; + } + + $out = []; + $cursor = 0; + foreach ($edits as $edit) { + for ($i = $cursor; $i < $edit->start; $i++) { + $out[] = $origin[$i]; + } + $length = count($edit->replacement); + for ($k = 0; $k < $length; $k++) { + $source = $edit->start + $k; + $out[] = $source < $edit->end ? $origin[$source] : self::NO_ORIGIN; + } + $cursor = $edit->end; + } + $total = count($origin); + for ($i = $cursor; $i < $total; $i++) { + $out[] = $origin[$i]; + } + + return $out; + } + + /** + * One rule's edits, in the coordinates that rule saw, rendered as Changes in input + * coordinates. $before is the text this rule replaced and $after what it replaced it with + * (analyze.md section 2), so on a text two rules have both touched, $before is what the second + * rule saw rather than what the caller typed -- section 5 says so and shows the French case + * where it matters. + * + * @param int[] $cp + * @param Edit[] $edits + * @param int[] $origin + * @return Change[] + */ + public static function recordChanges( + array $cp, + array $edits, + array $origin, + int $inputLength, + string $ruleId, + ): array { + $changes = []; + foreach ($edits as $edit) { + $changes[] = new Change( + ruleId: $ruleId, + start: self::originAt($origin, $edit->start, $inputLength), + end: self::originAt($origin, $edit->end, $inputLength), + before: Codepoints::fromCodepoints(array_slice($cp, $edit->start, $edit->end - $edit->start)), + after: Codepoints::fromCodepoints($edit->replacement), + ); + } + + return $changes; + } +} diff --git a/src/Engine/Pipeline.php b/src/Engine/Pipeline.php index 6d4cee6..df227a9 100644 --- a/src/Engine/Pipeline.php +++ b/src/Engine/Pipeline.php @@ -86,4 +86,45 @@ public static function runRules(array $cp, array $plan, array $localeData, RuleC return $current; } + + /** + * runRules, keeping the edits instead of discarding them (analyze.md section 1: same + * pipeline, same order, reporting rather than applying). The origin map travels alongside the + * array so every change comes back in input coordinates, and $filterEdits is the hook the + * span-runner needs for modes.md 3.4's boundary filters -- text mode passes null and gets the + * identity. + * + * @param int[] $cp + * @param string[] $plan + * @param array $localeData + * @param int[] $origin + * @param null|callable(int[], Edit[]): Edit[] $filterEdits + * @return \Polytypo\Change[] + */ + public static function runRulesRecording( + array $cp, + array $plan, + array $localeData, + RuleContext $ctx, + array $origin, + int $inputLength, + ?callable $filterEdits = null, + ): array { + $current = $cp; + $currentOrigin = $origin; + $changes = []; + foreach ($plan as $ruleId) { + $fn = Registry::rule($ruleId); + $produced = $fn($current, $localeData, $ctx); + $edits = $filterEdits === null ? $produced : $filterEdits($current, $produced); + if ($edits === []) { + continue; + } + array_push($changes, ...Origin::recordChanges($current, $edits, $currentOrigin, $inputLength, $ruleId)); + $currentOrigin = Origin::applyEditsToOrigin($currentOrigin, $edits); + $current = Edits::applyEdits($current, $edits, $ruleId); + } + + return $changes; + } } diff --git a/src/Modes/Runner.php b/src/Modes/Runner.php index a88c90a..2c6e19f 100644 --- a/src/Modes/Runner.php +++ b/src/Modes/Runner.php @@ -5,6 +5,7 @@ namespace Polytypo\Modes; use Polytypo\Engine\Edits; +use Polytypo\Engine\Pipeline; use Polytypo\Engine\Registry; use Polytypo\Engine\RuleContext; @@ -90,4 +91,43 @@ public static function runOverSpans( return \Polytypo\Engine\Codepoints::fromCodepoints($out); } + + /** + * runOverSpans, reporting instead of applying (analyze.md section 1). The span table supplies + * the origin map, so every change comes back in DOCUMENT coordinates -- analyze.md section 6 + * names a runtime that reports span-local offsets here as the mistake that passes every + * text-mode test. + * + * @param int[] $sourceCp + * @param Span[] $spans + * @param string[] $plan + * @param array $localeData + * @return \Polytypo\Change[] + */ + public static function analyzeOverSpans( + array $sourceCp, + array $spans, + array $plan, + array $localeData, + RuleContext $ctx, + ): array { + $normalized = Spans::normalizeSpans($spans); + if ($normalized === []) { + return []; + } + + return Pipeline::runRulesRecording( + Spans::concatenateSpans($sourceCp, $normalized), + $plan, + $localeData, + $ctx, + Spans::originOfSpans($normalized), + count($sourceCp), + static fn (array $current, array $edits): array => Spans::filterBoundaryEdits( + $current, + $edits, + Spans::spanRangesOf($current), + ), + ); + } } diff --git a/src/Modes/Spans.php b/src/Modes/Spans.php index 9c20fa9..bb16b72 100644 --- a/src/Modes/Spans.php +++ b/src/Modes/Spans.php @@ -5,6 +5,7 @@ namespace Polytypo\Modes; use Polytypo\Engine\Edit; +use Polytypo\Engine\Origin; use Polytypo\Engine\Sentinels; use Polytypo\PolytypoException; @@ -94,6 +95,33 @@ public static function concatenateSpans(array $sourceCp, array $spans): array return $cp; } + /** + * The origin map for concatenateSpans (analyze.md section 2): for every code point of the + * joined array, the code-point offset of the character it came from IN THE DOCUMENT, and + * Origin::NO_ORIGIN for the markers, which came from nowhere. A Span's bounds are already + * code-point offsets (the mode adapter converts its parser's byte offsets away before + * constructing one), so no coordinate conversion belongs here. + * + * @param Span[] $spans + * @return int[] + */ + public static function originOfSpans(array $spans): array + { + $origin = []; + $first = true; + foreach ($spans as $span) { + if (!$first) { + $origin[] = Origin::NO_ORIGIN; + } + for ($i = $span->start; $i < $span->end; $i++) { + $origin[] = $i; + } + $first = false; + } + + return $origin; + } + /** * The span extents of the array as it stands. Recomputed after every rule, because applying * edits shifts every index after the first one -- the markers themselves always survive, diff --git a/src/Polytypo.php b/src/Polytypo.php index 16b96ec..ed42cb6 100644 --- a/src/Polytypo.php +++ b/src/Polytypo.php @@ -60,6 +60,40 @@ public static function transform( }; } + /** + * Runs the same pipeline as transform() and reports what it would do instead of doing it + * (spec/rules/analyze.md). Offsets are code-point offsets into $input in every mode -- into + * the document, in "html" mode, not into a span. + * + * What it guarantees: the list is empty exactly when transform() would return the input + * unchanged, every ruleId was enabled for the call, and every offset is inside the input. + * What it does not: the list is a report, not a patch -- two rules may touch the same + * original range, so replaying it is not guaranteed to reproduce transform()'s output. Call + * transform() for the text (analyze.md sections 4 and 5). + * + * Pure on the same terms as transform(). + * + * @param array|null $rules + * @return Change[] + */ + public static function analyze( + string $input, + string $locale, + string $mode = 'text', + ?string $dialect = null, + ?array $rules = null, + ): array { + $resolvedMode = self::resolveMode($mode); + + return match ($resolvedMode) { + 'text' => self::analyzeText($input, $locale, $dialect, $rules), + 'html' => self::analyzeHtml($input, $locale, $dialect, $rules), + // 'markdown': not implemented by this runtime, exactly as for transform() -- A1 + // requires analyze() to reject what transform() rejects, with the same code. + default => self::transformMarkdown($input, $locale, $dialect, $rules), + }; + } + private static function resolveMode(string $mode): string { return match ($mode) { @@ -124,4 +158,42 @@ private static function transformMarkdown(string $input, string $locale, ?string . 'See spec/CONFORMANCE.md in github.com/polytypo/polytypo for what this runtime implements.', ); } + + /** + * @param array|null $rules + * @return Change[] + */ + private static function analyzeText(string $input, string $locale, ?string $dialect, ?array $rules): array + { + if ($dialect !== null) { + throw new PolytypoException( + PolytypoException::CODE_INVALID_DIALECT, + '"dialect" is only valid when mode is "markdown"', + ); + } + [, $localeData, $plan] = Pipeline::prepare($locale, $rules); + $cp = Codepoints::toCodepoints($input); + $ctx = new RuleContext(mode: 'text', dialect: null, locale: $locale); + + return Pipeline::runRulesRecording($cp, $plan, $localeData, $ctx, array_keys($cp), count($cp)); + } + + /** + * @param array|null $rules + * @return Change[] + */ + private static function analyzeHtml(string $input, string $locale, ?string $dialect, ?array $rules): array + { + if ($dialect !== null) { + throw new PolytypoException( + PolytypoException::CODE_INVALID_DIALECT, + '"dialect" is only valid when mode is "markdown"', + ); + } + [$resolvedLocale, $localeData, $plan] = Pipeline::prepare($locale, $rules); + $spans = Html::htmlSpans($input); + $ctx = new RuleContext(mode: 'html', dialect: null, locale: $resolvedLocale); + + return Runner::analyzeOverSpans(Codepoints::toCodepoints($input), $spans, $plan, $localeData, $ctx); + } } diff --git a/tests/AnalyzeTest.php b/tests/AnalyzeTest.php new file mode 100644 index 0000000..123656c --- /dev/null +++ b/tests/AnalyzeTest.php @@ -0,0 +1,214 @@ + $c->ruleId, $changes); + } + + public function testRejectsAnUnknownLocaleLikeTransform(): void + { + try { + Polytypo::analyze('x', 'xx'); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_UNKNOWN_LOCALE, $e->getErrorCode()); + } + } + + public function testAnUnknownRuleWinsOverAnUnknownLocale(): void + { + try { + Polytypo::analyze('x', 'xx', rules: ['nope' => true]); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_UNKNOWN_RULE, $e->getErrorCode()); + } + } + + public function testRejectsAnUnknownMode(): void + { + try { + Polytypo::analyze('x', 'en-US', 'yaml'); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_INVALID_MODE, $e->getErrorCode()); + } + } + + public function testRejectsMarkdownExactlyAsTransformDoes(): void + { + try { + Polytypo::analyze('x', 'en-US', 'markdown', 'commonmark'); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_INVALID_DIALECT, $e->getErrorCode()); + } + } + + public function testRejectsADialectOutsideMarkdown(): void + { + try { + Polytypo::analyze('x', 'en-US', 'text', 'commonmark'); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_INVALID_DIALECT, $e->getErrorCode()); + } + } + + public function testIsPure(): void + { + $input = 'She said "hi" -- really...'; + $before = Polytypo::transform($input, 'en-US'); + + $once = Polytypo::analyze($input, 'en-US'); + $twice = Polytypo::analyze($input, 'en-US'); + $this->assertEquals($once, $twice); + + $this->assertSame($before, Polytypo::transform($input, 'en-US')); + } + + public function testIsEmptyForTextThatNeedsNothing(): void + { + $this->assertSame([], Polytypo::analyze('Nothing to do here.', 'en-US')); + } + + public function testIsNonEmptyForTextThatNeedsSomething(): void + { + $this->assertNotSame([], Polytypo::analyze('Wait...', 'en-US')); + } + + /** analyze.md section 6: the whole vendored corpus, the cheap strong version of A3. */ + public function testAgreesWithTransformOnEveryFixture(): void + { + $offenders = []; + foreach (ConformanceTest::cases() as $key => [$basename, $case]) { + if (isset($case['throws']) || $case['mode'] === 'markdown') { + continue; + } + $locale = $case['locale']; + $rules = $case['rules'] ?? null; + $changed = Polytypo::transform($case['in'], $locale, $case['mode'], null, $rules) !== $case['in']; + $reported = Polytypo::analyze($case['in'], $locale, $case['mode'], null, $rules) !== []; + if ($changed !== $reported) { + $offenders[] = $key; + } + } + $this->assertSame([], $offenders); + } + + public function testNeverReportsARuleTheCallerDisabled(): void + { + $ids = self::ruleIds(Polytypo::analyze('She said "hi"...', 'en-US', rules: ['quotes' => false])); + $this->assertNotContains('quotes', $ids); + $this->assertContains('ellipsis', $ids); + } + + public function testNeverReportsRangesUnlessTurnedOn(): void + { + $input = 'chapters 3-5'; + $this->assertNotContains('ranges', self::ruleIds(Polytypo::analyze($input, 'en-US'))); + $this->assertContains( + 'ranges', + self::ruleIds(Polytypo::analyze($input, 'en-US', rules: ['ranges' => true])), + ); + } + + public function testStaysWithinBoundsOnAstralInput(): void + { + $input = 'A ๐Ÿ˜€ says "hi" and waits...'; + $length = mb_strlen($input, 'UTF-8'); + foreach (Polytypo::analyze($input, 'en-US') as $change) { + $this->assertGreaterThanOrEqual(0, $change->start); + $this->assertLessThanOrEqual($length, $change->end); + $this->assertLessThanOrEqual($change->end, $change->start); + } + } + + public function testReportsCodePointOffsetsNotByteOffsets(): void + { + // The emoji is four UTF-8 bytes and one code point, so the opening quotation mark sits at + // code point 6 and at byte index 9. A runtime reporting native string offsets -- which in + // PHP means bytes -- says 9. + $input = '๐Ÿ˜€ and "this"'; + $changes = Polytypo::analyze($input, 'en-US'); + $this->assertNotSame([], $changes); + $this->assertSame('quotes', $changes[0]->ruleId); + $this->assertSame(6, $changes[0]->start); + $this->assertSame(9, strpos($input, '"'), 'test premise: the byte offset differs from the code-point one'); + } + + /** analyze.md section 6: the mistake that passes every text-mode test. */ + public function testHtmlModeReportsDocumentOffsets(): void + { + $input = '

Wait...

'; + $changes = Polytypo::analyze($input, 'en-US', 'html'); + $this->assertNotSame([], $changes); + $this->assertSame('ellipsis', $changes[0]->ruleId); + $this->assertSame(mb_strpos($input, '...', 0, 'UTF-8'), $changes[0]->start); + $this->assertSame('...', $changes[0]->before); + $this->assertSame("\u{2026}", $changes[0]->after); + } + + public function testHtmlModeReportsDocumentOffsetsInALaterSpan(): void + { + // Three spans, and the change is in the third: the two markers before it are the only + // code points in the joined array with no origin, so a doubled or dropped one shifts this + // offset and nothing in a one- or two-span document would notice. + $input = '

one

two

Wait... three

'; + $changes = Polytypo::analyze($input, 'en-US', 'html'); + $this->assertNotSame([], $changes); + $this->assertSame('ellipsis', $changes[0]->ruleId); + $this->assertSame(mb_strpos($input, '...', 0, 'UTF-8'), $changes[0]->start); + } + + public function testReportsRulesInPipelineOrder(): void + { + $ids = self::ruleIds(Polytypo::analyze('She said "hi" -- wait...', 'en-US')); + $sorted = $ids; + usort($sorted, static fn (string $a, string $b): int + => array_search($a, self::RULE_ORDER, true) <=> array_search($b, self::RULE_ORDER, true)); + $this->assertSame($sorted, $ids); + } + + public function testReportsBothRulesOnTheSameOriginalRange(): void + { + // The French case analyze.md section 5 is written around: two rules, one original index. + // `spaces` removes the space at 3-4 and `nbsp` inserts at 4-4, in front of the colon the + // caller wrote at 4 -- the second change's position is the colon's, not the deleted + // space's. + $changes = Polytypo::analyze('Oui : non', 'fr'); + $this->assertSame(['spaces', 'nbsp'], self::ruleIds($changes)); + $this->assertSame(' ', $changes[0]->before); + $this->assertSame('', $changes[0]->after); + $this->assertSame([3, 4], [$changes[0]->start, $changes[0]->end]); + $this->assertSame("\u{00a0}", $changes[1]->after); + $this->assertSame([4, 4], [$changes[1]->start, $changes[1]->end]); + } +}