diff --git a/README.md b/README.md index 61f77ed..4d209e1 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,24 @@ accepts PHP named arguments: Polytypo::transform($input, 'fr', mode: 'html', rules: ['ranges' => true]); ``` +`Polytypo::analyze` runs the same pipeline and reports what it would do instead of doing it — one +`Polytypo\Change` per edit, each with the rule that made it and code-point offsets into the input +you passed (into the **document**, in `html` mode, not into a span): + +```php +Polytypo::analyze('Wait... "really"?', 'en-US'); +// => [ Change { ruleId: "ellipsis", start: 4, end: 7, before: "...", after: "…" }, +// Change { ruleId: "quotes", start: 8, end: 9, before: "\"", after: "“" }, ... ] +``` + +Offsets are code points, not bytes: in `😀 and "this"` the opening quotation mark is reported at +6, where `strpos` would say 9. It is a report, not a patch. The list is empty exactly when +`Polytypo::transform` would return the input unchanged, and every `ruleId` is a rule that was +enabled for that call — but two rules may touch the same original range (French `spaces` deletes +the space before `:` and `nbsp` puts a no-break one back), so replaying the list is not guaranteed +to reproduce the output. Call `Polytypo::transform` for the text. Full contract: +`spec/rules/analyze.md`. + ### Errors Every error `Polytypo::transform` raises is a `Polytypo\PolytypoException` carrying one of seven diff --git a/src/Change.php b/src/Change.php new file mode 100644 index 0000000..f09ad1e --- /dev/null +++ b/src/Change.php @@ -0,0 +1,23 @@ + U+00A0) still point at the character it converted; anything longer is synthetic + * beyond the positions it covers. + * + * @param int[] $origin + * @param Edit[] $edits + * @return int[] + */ + public static function applyEditsToOrigin(array $origin, array $edits): array + { + if ($edits === []) { + return $origin; + } + + $out = []; + $cursor = 0; + foreach ($edits as $edit) { + for ($i = $cursor; $i < $edit->start; $i++) { + $out[] = $origin[$i]; + } + $length = count($edit->replacement); + for ($k = 0; $k < $length; $k++) { + $source = $edit->start + $k; + $out[] = $source < $edit->end ? $origin[$source] : self::NO_ORIGIN; + } + $cursor = $edit->end; + } + $total = count($origin); + for ($i = $cursor; $i < $total; $i++) { + $out[] = $origin[$i]; + } + + return $out; + } + + /** + * One rule's edits, in the coordinates that rule saw, rendered as Changes in input + * coordinates. $before is the text this rule replaced and $after what it replaced it with + * (analyze.md section 2), so on a text two rules have both touched, $before is what the second + * rule saw rather than what the caller typed -- section 5 says so and shows the French case + * where it matters. + * + * @param int[] $cp + * @param Edit[] $edits + * @param int[] $origin + * @return Change[] + */ + public static function recordChanges( + array $cp, + array $edits, + array $origin, + int $inputLength, + string $ruleId, + ): array { + $changes = []; + foreach ($edits as $edit) { + $changes[] = new Change( + ruleId: $ruleId, + start: self::originAt($origin, $edit->start, $inputLength), + end: self::originAt($origin, $edit->end, $inputLength), + before: Codepoints::fromCodepoints(array_slice($cp, $edit->start, $edit->end - $edit->start)), + after: Codepoints::fromCodepoints($edit->replacement), + ); + } + + return $changes; + } +} diff --git a/src/Engine/Pipeline.php b/src/Engine/Pipeline.php index 6d4cee6..df227a9 100644 --- a/src/Engine/Pipeline.php +++ b/src/Engine/Pipeline.php @@ -86,4 +86,45 @@ public static function runRules(array $cp, array $plan, array $localeData, RuleC return $current; } + + /** + * runRules, keeping the edits instead of discarding them (analyze.md section 1: same + * pipeline, same order, reporting rather than applying). The origin map travels alongside the + * array so every change comes back in input coordinates, and $filterEdits is the hook the + * span-runner needs for modes.md 3.4's boundary filters -- text mode passes null and gets the + * identity. + * + * @param int[] $cp + * @param string[] $plan + * @param array $localeData + * @param int[] $origin + * @param null|callable(int[], Edit[]): Edit[] $filterEdits + * @return \Polytypo\Change[] + */ + public static function runRulesRecording( + array $cp, + array $plan, + array $localeData, + RuleContext $ctx, + array $origin, + int $inputLength, + ?callable $filterEdits = null, + ): array { + $current = $cp; + $currentOrigin = $origin; + $changes = []; + foreach ($plan as $ruleId) { + $fn = Registry::rule($ruleId); + $produced = $fn($current, $localeData, $ctx); + $edits = $filterEdits === null ? $produced : $filterEdits($current, $produced); + if ($edits === []) { + continue; + } + array_push($changes, ...Origin::recordChanges($current, $edits, $currentOrigin, $inputLength, $ruleId)); + $currentOrigin = Origin::applyEditsToOrigin($currentOrigin, $edits); + $current = Edits::applyEdits($current, $edits, $ruleId); + } + + return $changes; + } } diff --git a/src/Modes/Runner.php b/src/Modes/Runner.php index a88c90a..2c6e19f 100644 --- a/src/Modes/Runner.php +++ b/src/Modes/Runner.php @@ -5,6 +5,7 @@ namespace Polytypo\Modes; use Polytypo\Engine\Edits; +use Polytypo\Engine\Pipeline; use Polytypo\Engine\Registry; use Polytypo\Engine\RuleContext; @@ -90,4 +91,43 @@ public static function runOverSpans( return \Polytypo\Engine\Codepoints::fromCodepoints($out); } + + /** + * runOverSpans, reporting instead of applying (analyze.md section 1). The span table supplies + * the origin map, so every change comes back in DOCUMENT coordinates -- analyze.md section 6 + * names a runtime that reports span-local offsets here as the mistake that passes every + * text-mode test. + * + * @param int[] $sourceCp + * @param Span[] $spans + * @param string[] $plan + * @param array $localeData + * @return \Polytypo\Change[] + */ + public static function analyzeOverSpans( + array $sourceCp, + array $spans, + array $plan, + array $localeData, + RuleContext $ctx, + ): array { + $normalized = Spans::normalizeSpans($spans); + if ($normalized === []) { + return []; + } + + return Pipeline::runRulesRecording( + Spans::concatenateSpans($sourceCp, $normalized), + $plan, + $localeData, + $ctx, + Spans::originOfSpans($normalized), + count($sourceCp), + static fn (array $current, array $edits): array => Spans::filterBoundaryEdits( + $current, + $edits, + Spans::spanRangesOf($current), + ), + ); + } } diff --git a/src/Modes/Spans.php b/src/Modes/Spans.php index 9c20fa9..bb16b72 100644 --- a/src/Modes/Spans.php +++ b/src/Modes/Spans.php @@ -5,6 +5,7 @@ namespace Polytypo\Modes; use Polytypo\Engine\Edit; +use Polytypo\Engine\Origin; use Polytypo\Engine\Sentinels; use Polytypo\PolytypoException; @@ -94,6 +95,33 @@ public static function concatenateSpans(array $sourceCp, array $spans): array return $cp; } + /** + * The origin map for concatenateSpans (analyze.md section 2): for every code point of the + * joined array, the code-point offset of the character it came from IN THE DOCUMENT, and + * Origin::NO_ORIGIN for the markers, which came from nowhere. A Span's bounds are already + * code-point offsets (the mode adapter converts its parser's byte offsets away before + * constructing one), so no coordinate conversion belongs here. + * + * @param Span[] $spans + * @return int[] + */ + public static function originOfSpans(array $spans): array + { + $origin = []; + $first = true; + foreach ($spans as $span) { + if (!$first) { + $origin[] = Origin::NO_ORIGIN; + } + for ($i = $span->start; $i < $span->end; $i++) { + $origin[] = $i; + } + $first = false; + } + + return $origin; + } + /** * The span extents of the array as it stands. Recomputed after every rule, because applying * edits shifts every index after the first one -- the markers themselves always survive, diff --git a/src/Polytypo.php b/src/Polytypo.php index 16b96ec..ed42cb6 100644 --- a/src/Polytypo.php +++ b/src/Polytypo.php @@ -60,6 +60,40 @@ public static function transform( }; } + /** + * Runs the same pipeline as transform() and reports what it would do instead of doing it + * (spec/rules/analyze.md). Offsets are code-point offsets into $input in every mode -- into + * the document, in "html" mode, not into a span. + * + * What it guarantees: the list is empty exactly when transform() would return the input + * unchanged, every ruleId was enabled for the call, and every offset is inside the input. + * What it does not: the list is a report, not a patch -- two rules may touch the same + * original range, so replaying it is not guaranteed to reproduce transform()'s output. Call + * transform() for the text (analyze.md sections 4 and 5). + * + * Pure on the same terms as transform(). + * + * @param array|null $rules + * @return Change[] + */ + public static function analyze( + string $input, + string $locale, + string $mode = 'text', + ?string $dialect = null, + ?array $rules = null, + ): array { + $resolvedMode = self::resolveMode($mode); + + return match ($resolvedMode) { + 'text' => self::analyzeText($input, $locale, $dialect, $rules), + 'html' => self::analyzeHtml($input, $locale, $dialect, $rules), + // 'markdown': not implemented by this runtime, exactly as for transform() -- A1 + // requires analyze() to reject what transform() rejects, with the same code. + default => self::transformMarkdown($input, $locale, $dialect, $rules), + }; + } + private static function resolveMode(string $mode): string { return match ($mode) { @@ -124,4 +158,42 @@ private static function transformMarkdown(string $input, string $locale, ?string . 'See spec/CONFORMANCE.md in github.com/polytypo/polytypo for what this runtime implements.', ); } + + /** + * @param array|null $rules + * @return Change[] + */ + private static function analyzeText(string $input, string $locale, ?string $dialect, ?array $rules): array + { + if ($dialect !== null) { + throw new PolytypoException( + PolytypoException::CODE_INVALID_DIALECT, + '"dialect" is only valid when mode is "markdown"', + ); + } + [, $localeData, $plan] = Pipeline::prepare($locale, $rules); + $cp = Codepoints::toCodepoints($input); + $ctx = new RuleContext(mode: 'text', dialect: null, locale: $locale); + + return Pipeline::runRulesRecording($cp, $plan, $localeData, $ctx, array_keys($cp), count($cp)); + } + + /** + * @param array|null $rules + * @return Change[] + */ + private static function analyzeHtml(string $input, string $locale, ?string $dialect, ?array $rules): array + { + if ($dialect !== null) { + throw new PolytypoException( + PolytypoException::CODE_INVALID_DIALECT, + '"dialect" is only valid when mode is "markdown"', + ); + } + [$resolvedLocale, $localeData, $plan] = Pipeline::prepare($locale, $rules); + $spans = Html::htmlSpans($input); + $ctx = new RuleContext(mode: 'html', dialect: null, locale: $resolvedLocale); + + return Runner::analyzeOverSpans(Codepoints::toCodepoints($input), $spans, $plan, $localeData, $ctx); + } } diff --git a/tests/AnalyzeTest.php b/tests/AnalyzeTest.php new file mode 100644 index 0000000..123656c --- /dev/null +++ b/tests/AnalyzeTest.php @@ -0,0 +1,214 @@ + $c->ruleId, $changes); + } + + public function testRejectsAnUnknownLocaleLikeTransform(): void + { + try { + Polytypo::analyze('x', 'xx'); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_UNKNOWN_LOCALE, $e->getErrorCode()); + } + } + + public function testAnUnknownRuleWinsOverAnUnknownLocale(): void + { + try { + Polytypo::analyze('x', 'xx', rules: ['nope' => true]); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_UNKNOWN_RULE, $e->getErrorCode()); + } + } + + public function testRejectsAnUnknownMode(): void + { + try { + Polytypo::analyze('x', 'en-US', 'yaml'); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_INVALID_MODE, $e->getErrorCode()); + } + } + + public function testRejectsMarkdownExactlyAsTransformDoes(): void + { + try { + Polytypo::analyze('x', 'en-US', 'markdown', 'commonmark'); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_INVALID_DIALECT, $e->getErrorCode()); + } + } + + public function testRejectsADialectOutsideMarkdown(): void + { + try { + Polytypo::analyze('x', 'en-US', 'text', 'commonmark'); + $this->fail('expected an exception'); + } catch (PolytypoException $e) { + $this->assertSame(PolytypoException::CODE_INVALID_DIALECT, $e->getErrorCode()); + } + } + + public function testIsPure(): void + { + $input = 'She said "hi" -- really...'; + $before = Polytypo::transform($input, 'en-US'); + + $once = Polytypo::analyze($input, 'en-US'); + $twice = Polytypo::analyze($input, 'en-US'); + $this->assertEquals($once, $twice); + + $this->assertSame($before, Polytypo::transform($input, 'en-US')); + } + + public function testIsEmptyForTextThatNeedsNothing(): void + { + $this->assertSame([], Polytypo::analyze('Nothing to do here.', 'en-US')); + } + + public function testIsNonEmptyForTextThatNeedsSomething(): void + { + $this->assertNotSame([], Polytypo::analyze('Wait...', 'en-US')); + } + + /** analyze.md section 6: the whole vendored corpus, the cheap strong version of A3. */ + public function testAgreesWithTransformOnEveryFixture(): void + { + $offenders = []; + foreach (ConformanceTest::cases() as $key => [$basename, $case]) { + if (isset($case['throws']) || $case['mode'] === 'markdown') { + continue; + } + $locale = $case['locale']; + $rules = $case['rules'] ?? null; + $changed = Polytypo::transform($case['in'], $locale, $case['mode'], null, $rules) !== $case['in']; + $reported = Polytypo::analyze($case['in'], $locale, $case['mode'], null, $rules) !== []; + if ($changed !== $reported) { + $offenders[] = $key; + } + } + $this->assertSame([], $offenders); + } + + public function testNeverReportsARuleTheCallerDisabled(): void + { + $ids = self::ruleIds(Polytypo::analyze('She said "hi"...', 'en-US', rules: ['quotes' => false])); + $this->assertNotContains('quotes', $ids); + $this->assertContains('ellipsis', $ids); + } + + public function testNeverReportsRangesUnlessTurnedOn(): void + { + $input = 'chapters 3-5'; + $this->assertNotContains('ranges', self::ruleIds(Polytypo::analyze($input, 'en-US'))); + $this->assertContains( + 'ranges', + self::ruleIds(Polytypo::analyze($input, 'en-US', rules: ['ranges' => true])), + ); + } + + public function testStaysWithinBoundsOnAstralInput(): void + { + $input = 'A 😀 says "hi" and waits...'; + $length = mb_strlen($input, 'UTF-8'); + foreach (Polytypo::analyze($input, 'en-US') as $change) { + $this->assertGreaterThanOrEqual(0, $change->start); + $this->assertLessThanOrEqual($length, $change->end); + $this->assertLessThanOrEqual($change->end, $change->start); + } + } + + public function testReportsCodePointOffsetsNotByteOffsets(): void + { + // The emoji is four UTF-8 bytes and one code point, so the opening quotation mark sits at + // code point 6 and at byte index 9. A runtime reporting native string offsets -- which in + // PHP means bytes -- says 9. + $input = '😀 and "this"'; + $changes = Polytypo::analyze($input, 'en-US'); + $this->assertNotSame([], $changes); + $this->assertSame('quotes', $changes[0]->ruleId); + $this->assertSame(6, $changes[0]->start); + $this->assertSame(9, strpos($input, '"'), 'test premise: the byte offset differs from the code-point one'); + } + + /** analyze.md section 6: the mistake that passes every text-mode test. */ + public function testHtmlModeReportsDocumentOffsets(): void + { + $input = '

Wait...

'; + $changes = Polytypo::analyze($input, 'en-US', 'html'); + $this->assertNotSame([], $changes); + $this->assertSame('ellipsis', $changes[0]->ruleId); + $this->assertSame(mb_strpos($input, '...', 0, 'UTF-8'), $changes[0]->start); + $this->assertSame('...', $changes[0]->before); + $this->assertSame("\u{2026}", $changes[0]->after); + } + + public function testHtmlModeReportsDocumentOffsetsInALaterSpan(): void + { + // Three spans, and the change is in the third: the two markers before it are the only + // code points in the joined array with no origin, so a doubled or dropped one shifts this + // offset and nothing in a one- or two-span document would notice. + $input = '

one

two

Wait... three

'; + $changes = Polytypo::analyze($input, 'en-US', 'html'); + $this->assertNotSame([], $changes); + $this->assertSame('ellipsis', $changes[0]->ruleId); + $this->assertSame(mb_strpos($input, '...', 0, 'UTF-8'), $changes[0]->start); + } + + public function testReportsRulesInPipelineOrder(): void + { + $ids = self::ruleIds(Polytypo::analyze('She said "hi" -- wait...', 'en-US')); + $sorted = $ids; + usort($sorted, static fn (string $a, string $b): int + => array_search($a, self::RULE_ORDER, true) <=> array_search($b, self::RULE_ORDER, true)); + $this->assertSame($sorted, $ids); + } + + public function testReportsBothRulesOnTheSameOriginalRange(): void + { + // The French case analyze.md section 5 is written around: two rules, one original index. + // `spaces` removes the space at 3-4 and `nbsp` inserts at 4-4, in front of the colon the + // caller wrote at 4 -- the second change's position is the colon's, not the deleted + // space's. + $changes = Polytypo::analyze('Oui : non', 'fr'); + $this->assertSame(['spaces', 'nbsp'], self::ruleIds($changes)); + $this->assertSame(' ', $changes[0]->before); + $this->assertSame('', $changes[0]->after); + $this->assertSame([3, 4], [$changes[0]->start, $changes[0]->end]); + $this->assertSame("\u{00a0}", $changes[1]->after); + $this->assertSame([4, 4], [$changes[1]->start, $changes[1]->end]); + } +}