From 5906059e64a614c6cf22f20ee6a69437eea7da4c Mon Sep 17 00:00:00 2001 From: Iurii Rogulia Date: Fri, 18 Sep 2026 19:05:13 +0300 Subject: [PATCH] Add analyze(), the reported-edits entry point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements spec 1.3.0's spec/rules/analyze.md (polytypo/polytypo@691c180, canonical issue #15), porting polytypo-js@1a2ef57. Same pipeline as transform, reporting what it would do instead of doing it: polytypo.analyze('Wait... "really"?', locale="en-US") # [Change(rule_id='ellipsis', start=4, end=7, before='...', after='…'), …] Nothing in the engine changed to support it. Rules already produce edits and the pipeline already applies them, so the whole feature is one structure: an origin map travelling alongside the code-point array (_engine/origin.py), holding the input offset each code point came from. A same-length replacement keeps its origins position by position, which is what makes a converted space still point at the space it converted. The mode adapters are where this could quietly go wrong, and analyze.md §6 says so: in html and markdown the rules run over the marker-joined concatenation of spans, so reporting those offsets would pass every text-mode test and be useless to a caller. origin_of_spans() maps them back to document offsets. A Python Span carries code-point offsets already, so that conversion is a range() here where polytypo-js needs a UTF-16 index table — the cheap half of this port, and the reason the offset tests assert document positions in a three-span document too, not only in a single-span one. Eighteen tests, one per contract clause plus the three that matter most: A3 against the entire vendored fixture corpus (empty list exactly when in == out), A5 across the mode adapters, and analyze.md §5's French overlap asserted at the spec's own offsets — spaces at 3–4 and nbsp at 4–4 on `Oui : non`. Full suite 1176 pass, packaging 4 pass, ruff and mypy --strict clean. transform's behaviour is untouched. Co-Authored-By: Claude Opus 5 --- README.md | 18 +++ src/polytypo/__init__.py | 30 ++++- src/polytypo/_engine/edits.py | 5 +- src/polytypo/_engine/mode_pipelines.py | 44 ++++++- src/polytypo/_engine/origin.py | 77 ++++++++++++ src/polytypo/_engine/pipeline.py | 32 ++++- src/polytypo/_engine/span_runner.py | 28 +++++ src/polytypo/_modes/spans.py | 17 +++ src/polytypo/html.py | 12 +- src/polytypo/markdown.py | 14 ++- src/polytypo/text.py | 11 +- tests/engine/test_analyze.py | 166 +++++++++++++++++++++++++ 12 files changed, 441 insertions(+), 13 deletions(-) create mode 100644 src/polytypo/_engine/origin.py create mode 100644 tests/engine/test_analyze.py diff --git a/README.md b/README.md index 080e3cf..c3ef0ca 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,24 @@ import polytypo polytypo.transform("...", locale="fr", mode="markdown", dialect="commonmark") ``` +`analyze()` runs the same pipeline and reports what it would do instead of doing it — one record +per edit, each with the rule that made it and code-point offsets into the input you passed (into +the **document**, in `html` and `markdown` mode, not into a span): + +```python +import polytypo + +polytypo.analyze('Wait... "really"?', locale="en-US") +# [Change(rule_id='ellipsis', start=4, end=7, before='...', after='…'), +# Change(rule_id='quotes', start=8, end=9, before='"', after='“'), …] +``` + +It is a report, not a patch. The list is empty exactly when `transform` would return the input +unchanged, and every `rule_id` is a rule that was enabled for that call — but two rules may touch +the same original range (French `spaces` deletes the space before `:` and `nbsp` puts a no-break +one back), so replaying the list is not guaranteed to reproduce the output. Call `transform` for +the text. Full contract: `spec/rules/analyze.md`. + ## Supported dialects `markdown` mode requires a `dialect` keyword, exactly as the spec requires (no default, diff --git a/src/polytypo/__init__.py b/src/polytypo/__init__.py index 78a01c7..7e39468 100644 --- a/src/polytypo/__init__.py +++ b/src/polytypo/__init__.py @@ -9,13 +9,17 @@ from __future__ import annotations from polytypo._engine.mode_pipelines import ( + analyze_html_pipeline, + analyze_markdown_pipeline, + analyze_text_pipeline, run_html_pipeline, run_markdown_pipeline, run_text_pipeline, ) +from polytypo._engine.origin import Change from polytypo.errors import POLYTYPO_INVALID_MODE, PolytypoError -__all__ = ["PolytypoError", "transform"] +__all__ = ["Change", "PolytypoError", "analyze", "transform"] __version__ = "0.0.0" @@ -44,3 +48,27 @@ def transform( if resolved_mode == "html": return run_html_pipeline(input, locale=locale, rules=rules) return run_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules) + + +def analyze( + input: str, + *, + locale: str, + mode: str | None = None, + dialect: str | None = None, + rules: dict[str, bool] | None = None, +) -> list[Change]: + """The same pipeline as `transform`, reporting what it would do instead of doing it + (spec/rules/analyze.md). Offsets are code-point offsets into `input` in every mode. + + What it guarantees: the list is empty exactly when `transform` would return the input + unchanged, every `rule_id` was enabled for the call, and every offset is inside the input. + What it does not: the list is a report, not a patch -- two rules may touch the same original + range, so replaying it is not guaranteed to reproduce `transform`'s output. Call `transform` + for the text (analyze.md sections 4 and 5).""" + resolved_mode = _resolve_mode(mode) + if resolved_mode == "text": + return analyze_text_pipeline(input, locale=locale, rules=rules) + if resolved_mode == "html": + return analyze_html_pipeline(input, locale=locale, rules=rules) + return analyze_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules) diff --git a/src/polytypo/_engine/edits.py b/src/polytypo/_engine/edits.py index 3d63ad1..96eadaf 100644 --- a/src/polytypo/_engine/edits.py +++ b/src/polytypo/_engine/edits.py @@ -1,7 +1,6 @@ """The Edit contract (mirrors polytypo-js's src/engine/edits.ts). Rules produce edits; the -pipeline applies them -- this separation is the one piece of v1 internal structure explicitly -authorized ahead of need (docs/ARCHITECTURE.md section 7.1's reserved, not-yet-implemented -``analyze()`` API).""" +pipeline applies them -- the separation docs/ARCHITECTURE.md section 7.1 reserved the engine's +shape for, and what ``analyze()`` (spec/rules/analyze.md, origin.py) reports without applying.""" from __future__ import annotations diff --git a/src/polytypo/_engine/mode_pipelines.py b/src/polytypo/_engine/mode_pipelines.py index 19ad736..777662d 100644 --- a/src/polytypo/_engine/mode_pipelines.py +++ b/src/polytypo/_engine/mode_pipelines.py @@ -11,9 +11,10 @@ from __future__ import annotations from polytypo._engine.codepoints import from_codepoints, to_codepoints -from polytypo._engine.pipeline import prepare, run_rules +from polytypo._engine.origin import Change +from polytypo._engine.pipeline import prepare, run_rules, run_rules_recording from polytypo._engine.registry import RuleContext -from polytypo._engine.span_runner import run_over_spans +from polytypo._engine.span_runner import analyze_over_spans, run_over_spans from polytypo._modes.spans import normalize_spans @@ -43,3 +44,42 @@ def run_markdown_pipeline( ctx: RuleContext = {"mode": "markdown", "dialect": dialect, "locale": resolved_locale} spans = normalize_spans(markdown_spans(input_text, dialect)) return run_over_spans(input_text, spans, plan, locale_data, ctx) + + +def analyze_text_pipeline( + input_text: str, *, locale: object, rules: dict[str, bool] | None +) -> list[Change]: + """analyze.md section 1: the same pipeline as `run_text_pipeline`, reporting instead of + applying.""" + resolved_locale, locale_data, plan = prepare(locale, rules) + cp = to_codepoints(input_text) + ctx: RuleContext = {"mode": "text", "dialect": None, "locale": resolved_locale} + return run_rules_recording(cp, plan, locale_data, ctx, list(range(len(cp))), len(cp)) + + +def analyze_html_pipeline( + input_text: str, *, locale: object, rules: dict[str, bool] | None +) -> list[Change]: + """analyze.md section 1, `html` mode: offsets are into the document, not into a span + (analyze.md section 6).""" + from polytypo._modes.html import html_spans + + resolved_locale, locale_data, plan = prepare(locale, rules) + ctx: RuleContext = {"mode": "html", "dialect": None, "locale": resolved_locale} + spans = normalize_spans(html_spans(input_text)) + return analyze_over_spans(input_text, spans, plan, locale_data, ctx) + + +def analyze_markdown_pipeline( + input_text: str, *, locale: object, dialect: str | None, rules: dict[str, bool] | None +) -> list[Change]: + """analyze.md section 1, `markdown` mode. Dialect validation happens here exactly as it does + for `run_markdown_pipeline`, so an absent or unsupported dialect raises + POLYTYPO_INVALID_DIALECT from `analyze` too (analyze.md section 4, A1).""" + from polytypo._modes.markdown import markdown_spans, resolve_dialect + + resolved_locale, locale_data, plan = prepare(locale, rules) + resolve_dialect(dialect) + ctx: RuleContext = {"mode": "markdown", "dialect": dialect, "locale": resolved_locale} + spans = normalize_spans(markdown_spans(input_text, dialect)) + return analyze_over_spans(input_text, spans, plan, locale_data, ctx) diff --git a/src/polytypo/_engine/origin.py b/src/polytypo/_engine/origin.py new file mode 100644 index 0000000..a68d641 --- /dev/null +++ b/src/polytypo/_engine/origin.py @@ -0,0 +1,77 @@ +"""analyze.md section 2: every reported offset is a code-point offset into the input the caller +passed, in every mode. The rules, however, run over an array that is not the input -- in `text` +mode edits from earlier rules have already shifted it, and in `html`/`markdown` mode it is the +marker-joined concatenation of the processable spans (modes.md 3.5). This module carries the one +structure that bridges the two: an origin map, parallel to the current code-point array, holding +the input offset each code point came from, or NO_ORIGIN for one the pipeline itself produced. +Mirrors polytypo-js's src/engine/origin.ts.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from polytypo._engine.codepoints import from_codepoints +from polytypo._engine.edits import Edit + +NO_ORIGIN = -1 + + +@dataclass(frozen=True, slots=True) +class Change: + """One entry of ``analyze``'s result, in input coordinates (analyze.md section 2).""" + + rule_id: str + start: int + end: int + before: str + after: str + + +def origin_at(origin: list[int], index: int, input_length: int) -> int: + """The input offset an edit boundary at ``index`` addresses. Synthetic code points have no + origin of their own, so the scan runs forward to the first that has one -- an insertion + between two earlier insertions still lands where the next real character is. Falling off the + end means the boundary is at the end of the input.""" + for i in range(index, len(origin)): + if origin[i] != NO_ORIGIN: + return origin[i] + return input_length + + +def apply_edits_to_origin(origin: list[int], edits: list[Edit]) -> list[int]: + """The origin map for the array ``apply_edits`` is about to produce. A replacement of equal + length keeps its origins position by position, which is what makes a conversion + (U+0020 -> U+00A0) still point at the character it converted; anything longer is synthetic + beyond the positions it covers.""" + if not edits: + return list(origin) + out: list[int] = [] + cursor = 0 + for edit in edits: + out.extend(origin[cursor : edit.start]) + for k in range(len(edit.replacement)): + source = edit.start + k + out.append(origin[source] if source < edit.end else NO_ORIGIN) + cursor = edit.end + out.extend(origin[cursor:]) + return out + + +def record_changes( + cp: list[int], edits: list[Edit], origin: list[int], input_length: int, rule_id: str +) -> list[Change]: + """One rule's edits, in the coordinates that rule saw, rendered as Changes in input + coordinates. ``before`` is the text this rule replaced and ``after`` what it replaced it with + (analyze.md section 2), so on a text two rules have both touched, ``before`` is what the + second rule saw rather than what the caller typed -- section 5 says so and shows the French + case where it matters.""" + return [ + Change( + rule_id=rule_id, + start=origin_at(origin, edit.start, input_length), + end=origin_at(origin, edit.end, input_length), + before=from_codepoints(cp[edit.start : edit.end]), + after=from_codepoints(edit.replacement), + ) + for edit in edits + ] diff --git a/src/polytypo/_engine/pipeline.py b/src/polytypo/_engine/pipeline.py index 73a230e..076fbdf 100644 --- a/src/polytypo/_engine/pipeline.py +++ b/src/polytypo/_engine/pipeline.py @@ -6,14 +6,16 @@ from __future__ import annotations +from collections.abc import Callable from typing import Any # Import side effect: registers every rule id with the registry (mirrors polytypo-js's # registry.ts, which statically imports every rule module for the same reason: importing the # engine must be enough, without a separate caller-side step). import polytypo._engine.rules # noqa: F401,E402 -from polytypo._engine.edits import apply_edits +from polytypo._engine.edits import Edit, apply_edits from polytypo._engine.locale import get_locale_data, resolve +from polytypo._engine.origin import Change, apply_edits_to_origin, record_changes from polytypo._engine.registry import ( RuleContext, get_rule, @@ -64,3 +66,31 @@ def run_rules( if edits: current = apply_edits(current, edits, rule_id) return current + + +def run_rules_recording( + cp: list[int], + plan: list[str], + locale_data: dict[str, Any], + ctx: RuleContext, + origin: list[int], + input_length: int, + filter_edits: Callable[[list[int], list[Edit]], list[Edit]] | None = None, +) -> list[Change]: + """`run_rules`, keeping the edits instead of discarding them (analyze.md section 1: same + pipeline, same order, reporting rather than applying). The origin map travels alongside the + array so every change comes back in input coordinates, and `filter_edits` is the hook the + span-runner needs for modes.md 3.4's boundary filters -- text mode passes nothing and gets + the identity.""" + current = cp + current_origin = origin + changes: list[Change] = [] + for rule_id in plan: + rule_fn = get_rule(rule_id) + produced = rule_fn(current, locale_data, ctx) + edits = produced if filter_edits is None else filter_edits(current, produced) + if edits: + changes.extend(record_changes(current, edits, current_origin, input_length, rule_id)) + current_origin = apply_edits_to_origin(current_origin, edits) + current = apply_edits(current, edits, rule_id) + return changes diff --git a/src/polytypo/_engine/span_runner.py b/src/polytypo/_engine/span_runner.py index c1bf609..8a51f60 100644 --- a/src/polytypo/_engine/span_runner.py +++ b/src/polytypo/_engine/span_runner.py @@ -9,12 +9,15 @@ from polytypo._engine.codepoints import from_codepoints from polytypo._engine.edits import apply_edits +from polytypo._engine.origin import Change +from polytypo._engine.pipeline import run_rules_recording from polytypo._engine.registry import RuleContext, get_rule from polytypo._modes.spans import ( Span, concatenate_spans, filter_boundary_edits, normalize_spans, + origin_of_spans, span_ranges_of, split_on_marker, ) @@ -65,3 +68,28 @@ def run_over_spans( cursor = span.end out.append(source[cursor:]) return "".join(out) + + +def analyze_over_spans( + source: str, + spans: list[Span], + plan: list[str], + locale_data: dict[str, Any], + ctx: RuleContext, +) -> list[Change]: + """`run_over_spans`, reporting instead of applying (analyze.md section 1). The span table + supplies the origin map, so every change comes back in DOCUMENT coordinates -- analyze.md + section 6 names a runtime that reports span-local offsets here as the mistake that passes + every text-mode test.""" + normalized = normalize_spans(spans) + if not normalized: + return [] + return run_rules_recording( + concatenate_spans(source, normalized), + plan, + locale_data, + ctx, + origin_of_spans(source, normalized), + len(source), + lambda current, edits: filter_boundary_edits(current, edits, span_ranges_of(current)), + ) diff --git a/src/polytypo/_modes/spans.py b/src/polytypo/_modes/spans.py index de1cd44..6f172bc 100644 --- a/src/polytypo/_modes/spans.py +++ b/src/polytypo/_modes/spans.py @@ -8,6 +8,7 @@ from polytypo._engine.codepoints import to_codepoints from polytypo._engine.edits import Edit +from polytypo._engine.origin import NO_ORIGIN from polytypo._engine.sentinels import LINE_MARKER, MARKER, is_marker from polytypo.errors import POLYTYPO_RULE_CONTRACT, PolytypoError @@ -136,3 +137,19 @@ def split_on_marker(cp: list[int], expected: int) -> list[list[int]]: f"found {len(pieces)}", ) return pieces + + +def origin_of_spans(source: str, spans: list[Span]) -> list[int]: + """The origin map for `concatenate_spans` (analyze.md section 2): for every code point of the + joined array, the code-point offset of the character it came from IN THE DOCUMENT, and + NO_ORIGIN for the markers, which came from nowhere. A Python `Span`'s bounds are already + code-point offsets, so unlike polytypo-js's `originOfSpans` there is no UTF-16 index to + convert here.""" + origin: list[int] = [] + previous: Span | None = None + for span in spans: + if previous is not None: + origin.append(NO_ORIGIN) + origin.extend(range(span.start, span.end)) + previous = span + return origin diff --git a/src/polytypo/html.py b/src/polytypo/html.py index 057e3a0..d958917 100644 --- a/src/polytypo/html.py +++ b/src/polytypo/html.py @@ -8,11 +8,19 @@ from __future__ import annotations -from polytypo._engine.mode_pipelines import run_html_pipeline +from polytypo._engine.mode_pipelines import analyze_html_pipeline, run_html_pipeline +from polytypo._engine.origin import Change from polytypo.errors import PolytypoError -__all__ = ["PolytypoError", "transform"] +__all__ = ["Change", "PolytypoError", "analyze", "transform"] def transform(input: str, *, locale: str, rules: dict[str, bool] | None = None) -> str: return run_html_pipeline(input, locale=locale, rules=rules) + + +def analyze(input: str, *, locale: str, rules: dict[str, bool] | None = None) -> list[Change]: + """The same pipeline as this entry's `transform`, reporting instead of applying + (spec/rules/analyze.md). Offsets are code-point offsets into the **document**, not into a + span (analyze.md section 2).""" + return analyze_html_pipeline(input, locale=locale, rules=rules) diff --git a/src/polytypo/markdown.py b/src/polytypo/markdown.py index 5d34ffa..7829cb4 100644 --- a/src/polytypo/markdown.py +++ b/src/polytypo/markdown.py @@ -11,13 +11,23 @@ from __future__ import annotations -from polytypo._engine.mode_pipelines import run_markdown_pipeline +from polytypo._engine.mode_pipelines import analyze_markdown_pipeline, run_markdown_pipeline +from polytypo._engine.origin import Change from polytypo.errors import PolytypoError -__all__ = ["PolytypoError", "transform"] +__all__ = ["Change", "PolytypoError", "analyze", "transform"] def transform( input: str, *, locale: str, dialect: str, rules: dict[str, bool] | None = None ) -> str: return run_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules) + + +def analyze( + input: str, *, locale: str, dialect: str, rules: dict[str, bool] | None = None +) -> list[Change]: + """The same pipeline as this entry's `transform`, reporting instead of applying + (spec/rules/analyze.md). Offsets are code-point offsets into the **document**, not into a + span (analyze.md section 2).""" + return analyze_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules) diff --git a/src/polytypo/text.py b/src/polytypo/text.py index ecd381b..1df94e2 100644 --- a/src/polytypo/text.py +++ b/src/polytypo/text.py @@ -8,11 +8,18 @@ from __future__ import annotations -from polytypo._engine.mode_pipelines import run_text_pipeline +from polytypo._engine.mode_pipelines import analyze_text_pipeline, run_text_pipeline +from polytypo._engine.origin import Change from polytypo.errors import PolytypoError -__all__ = ["PolytypoError", "transform"] +__all__ = ["Change", "PolytypoError", "analyze", "transform"] def transform(input: str, *, locale: str, rules: dict[str, bool] | None = None) -> str: return run_text_pipeline(input, locale=locale, rules=rules) + + +def analyze(input: str, *, locale: str, rules: dict[str, bool] | None = None) -> list[Change]: + """The same pipeline as this entry's `transform`, reporting instead of applying + (spec/rules/analyze.md). Offsets are code-point offsets into `input`.""" + return analyze_text_pipeline(input, locale=locale, rules=rules) diff --git a/tests/engine/test_analyze.py b/tests/engine/test_analyze.py new file mode 100644 index 0000000..25eed48 --- /dev/null +++ b/tests/engine/test_analyze.py @@ -0,0 +1,166 @@ +"""spec/rules/analyze.md -- the contract is A1...A5; the decomposition is observation. + +Mirrors polytypo-js's tests/engine/analyze.test.ts case for case, including the two that are +cheap to get wrong (analyze.md section 6): A3 over the whole vendored corpus, and document +offsets under the mode adapters.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +import polytypo +from polytypo.errors import PolytypoError + +FIXTURES_DIR = Path(__file__).resolve().parents[2] / "vendor" / "polytypo-spec" / "fixtures" + +ORDER = [ + "spaces", + "ellipsis", + "ranges", + "dashes", + "hyphen", + "quotes", + "apostrophe", + "symbols", + "nbsp", +] + + +class TestA1AcceptsAndRejectsWhatTransformDoes: + def test_unknown_locale(self) -> None: + with pytest.raises(PolytypoError) as excinfo: + polytypo.analyze("x", locale="xx") + assert excinfo.value.code == "POLYTYPO_UNKNOWN_LOCALE" + + def test_unknown_rule_wins_over_unknown_locale(self) -> None: + with pytest.raises(PolytypoError) as excinfo: + polytypo.analyze("x", locale="xx", rules={"nope": True}) + assert excinfo.value.code == "POLYTYPO_UNKNOWN_RULE" + + def test_unknown_mode(self) -> None: + with pytest.raises(PolytypoError) as excinfo: + polytypo.analyze("x", locale="en-US", mode="yaml") + assert excinfo.value.code == "POLYTYPO_INVALID_MODE" + + def test_markdown_requires_a_dialect(self) -> None: + with pytest.raises(PolytypoError) as excinfo: + polytypo.analyze("x", locale="en-US", mode="markdown") + assert excinfo.value.code == "POLYTYPO_INVALID_DIALECT" + + +class TestA2Pure: + def test_same_list_for_the_same_arguments(self) -> None: + once = polytypo.analyze('She said "hi" -- really...', locale="en-US") + twice = polytypo.analyze('She said "hi" -- really...', locale="en-US") + assert twice == once + + def test_does_not_change_what_transform_returns(self) -> None: + text = 'She said "hi" -- really...' + before = polytypo.transform(text, locale="en-US") + polytypo.analyze(text, locale="en-US") + assert polytypo.transform(text, locale="en-US") == before + + +class TestA3EmptyExactlyWhenTransformChangesNothing: + def test_empty_for_text_that_needs_nothing(self) -> None: + assert polytypo.analyze("Nothing to do here.", locale="en-US") == [] + + def test_non_empty_for_text_that_needs_something(self) -> None: + assert len(polytypo.analyze("Wait...", locale="en-US")) > 0 + + def test_agrees_with_transform_on_every_canonical_fixture(self) -> None: + """analyze.md section 6: the whole canonical corpus, the cheap strong version of A3.""" + offenders: list[str] = [] + for path in sorted(FIXTURES_DIR.glob("*.json")): + if path.name == "locale-resolution.json": + continue + data: dict[str, Any] = json.loads(path.read_text("utf-8")) + for case in data["cases"]: + if "throws" in case or case.get("dialect") == "mdx": + continue + kwargs: dict[str, Any] = {"locale": data["locale"], "mode": case["mode"]} + if case.get("dialect") is not None: + kwargs["dialect"] = case["dialect"] + if case.get("rules") is not None: + kwargs["rules"] = case["rules"] + changed = polytypo.transform(case["in"], **kwargs) != case["in"] + reported = len(polytypo.analyze(case["in"], **kwargs)) > 0 + if changed != reported: + offenders.append(f"{data['locale']}/{case['id']}") + assert offenders == [] + + +class TestA4OnlyRulesEnabledForTheCall: + def test_never_reports_a_rule_the_caller_disabled(self) -> None: + ids = [ + c.rule_id + for c in polytypo.analyze('She said "hi"...', locale="en-US", rules={"quotes": False}) + ] + assert "quotes" not in ids + assert "ellipsis" in ids + + def test_never_reports_ranges_unless_turned_on(self) -> None: + text = "chapters 3-5" + assert "ranges" not in [c.rule_id for c in polytypo.analyze(text, locale="en-US")] + turned_on = polytypo.analyze(text, locale="en-US", rules={"ranges": True}) + assert "ranges" in [c.rule_id for c in turned_on] + + +class TestA5OffsetsAreCodePointsInsideTheInput: + def test_stays_within_bounds_with_astral_characters(self) -> None: + text = 'A \U0001f600 says "hi" and waits...' + for change in polytypo.analyze(text, locale="en-US"): + assert 0 <= change.start <= change.end <= len(text) + + def test_reports_code_point_offsets(self) -> None: + """The emoji is one code point here and two UTF-16 units in the JS runtime; both report + 6, which is what makes the offsets portable rather than string-representation-specific.""" + text = '\U0001f600 and "this"' + first = polytypo.analyze(text, locale="en-US")[0] + assert first.rule_id == "quotes" + assert first.start == 6 + + def test_html_mode_reports_document_offsets(self) -> None: + """analyze.md section 6: the mistake that passes every text-mode test.""" + text = '

Wait...

' + first = polytypo.analyze(text, locale="en-US", mode="html")[0] + assert first.rule_id == "ellipsis" + assert first.start == text.index("...") + assert first.before == "..." + assert first.after == "…" + + def test_html_mode_reports_document_offsets_in_a_later_span(self) -> None: + """Three spans, and the change is in the third: the two markers before it are the only + code points in the joined array with no origin, so a doubled or dropped one shifts this + offset and nothing in a one- or two-span document would notice.""" + text = "

one

two

Wait... three

" + first = polytypo.analyze(text, locale="en-US", mode="html")[0] + assert first.rule_id == "ellipsis" + assert first.start == text.index("...") + + def test_markdown_mode_reports_document_offsets(self) -> None: + text = "# Title\n\nWait... here\n" + first = polytypo.analyze(text, locale="en-US", mode="markdown", dialect="commonmark")[0] + assert first.start == text.index("...") + + +class TestPipelineOrderAndOverlap: + def test_reports_rules_in_spec_order(self) -> None: + ids = [c.rule_id for c in polytypo.analyze('She said "hi" -- wait...', locale="en-US")] + assert ids == sorted(ids, key=ORDER.index) + + def test_reports_both_rules_when_they_touch_the_same_original_range(self) -> None: + """The French case analyze.md section 5 is written around: two rules, one original index. + `spaces` removes the space at 3-4 and `nbsp` inserts at 4-4, in front of the colon the + caller wrote at 4 -- the second change's position is the colon's, not the deleted + space's.""" + changes = polytypo.analyze("Oui : non", locale="fr") + assert [c.rule_id for c in changes] == ["spaces", "nbsp"] + assert (changes[0].before, changes[0].after) == (" ", "") + assert (changes[0].start, changes[0].end) == (3, 4) + assert changes[1].after == " " + assert (changes[1].start, changes[1].end) == (4, 4)