Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,24 @@ import polytypo
polytypo.transform("...", locale="fr", mode="markdown", dialect="commonmark")
```

`analyze()` runs the same pipeline and reports what it would do instead of doing it — one record
per edit, each with the rule that made it and code-point offsets into the input you passed (into
the **document**, in `html` and `markdown` mode, not into a span):

```python
import polytypo

polytypo.analyze('Wait... "really"?', locale="en-US")
# [Change(rule_id='ellipsis', start=4, end=7, before='...', after='…'),
# Change(rule_id='quotes', start=8, end=9, before='"', after='“'), …]
```

It is a report, not a patch. The list is empty exactly when `transform` would return the input
unchanged, and every `rule_id` is a rule that was enabled for that call — but two rules may touch
the same original range (French `spaces` deletes the space before `:` and `nbsp` puts a no-break
one back), so replaying the list is not guaranteed to reproduce the output. Call `transform` for
the text. Full contract: `spec/rules/analyze.md`.

## Supported dialects

`markdown` mode requires a `dialect` keyword, exactly as the spec requires (no default,
Expand Down
30 changes: 29 additions & 1 deletion src/polytypo/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,17 @@
from __future__ import annotations

from polytypo._engine.mode_pipelines import (
analyze_html_pipeline,
analyze_markdown_pipeline,
analyze_text_pipeline,
run_html_pipeline,
run_markdown_pipeline,
run_text_pipeline,
)
from polytypo._engine.origin import Change
from polytypo.errors import POLYTYPO_INVALID_MODE, PolytypoError

__all__ = ["PolytypoError", "transform"]
__all__ = ["Change", "PolytypoError", "analyze", "transform"]

__version__ = "0.0.0"

Expand Down Expand Up @@ -44,3 +48,27 @@ def transform(
if resolved_mode == "html":
return run_html_pipeline(input, locale=locale, rules=rules)
return run_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules)


def analyze(
input: str,
*,
locale: str,
mode: str | None = None,
dialect: str | None = None,
rules: dict[str, bool] | None = None,
) -> list[Change]:
"""The same pipeline as `transform`, reporting what it would do instead of doing it
(spec/rules/analyze.md). Offsets are code-point offsets into `input` in every mode.

What it guarantees: the list is empty exactly when `transform` would return the input
unchanged, every `rule_id` was enabled for the call, and every offset is inside the input.
What it does not: the list is a report, not a patch -- two rules may touch the same original
range, so replaying it is not guaranteed to reproduce `transform`'s output. Call `transform`
for the text (analyze.md sections 4 and 5)."""
resolved_mode = _resolve_mode(mode)
if resolved_mode == "text":
return analyze_text_pipeline(input, locale=locale, rules=rules)
if resolved_mode == "html":
return analyze_html_pipeline(input, locale=locale, rules=rules)
return analyze_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules)
5 changes: 2 additions & 3 deletions src/polytypo/_engine/edits.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
"""The Edit contract (mirrors polytypo-js's src/engine/edits.ts). Rules produce edits; the
pipeline applies them -- this separation is the one piece of v1 internal structure explicitly
authorized ahead of need (docs/ARCHITECTURE.md section 7.1's reserved, not-yet-implemented
``analyze()`` API)."""
pipeline applies them -- the separation docs/ARCHITECTURE.md section 7.1 reserved the engine's
shape for, and what ``analyze()`` (spec/rules/analyze.md, origin.py) reports without applying."""

from __future__ import annotations

Expand Down
44 changes: 42 additions & 2 deletions src/polytypo/_engine/mode_pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@
from __future__ import annotations

from polytypo._engine.codepoints import from_codepoints, to_codepoints
from polytypo._engine.pipeline import prepare, run_rules
from polytypo._engine.origin import Change
from polytypo._engine.pipeline import prepare, run_rules, run_rules_recording
from polytypo._engine.registry import RuleContext
from polytypo._engine.span_runner import run_over_spans
from polytypo._engine.span_runner import analyze_over_spans, run_over_spans
from polytypo._modes.spans import normalize_spans


Expand Down Expand Up @@ -43,3 +44,42 @@ def run_markdown_pipeline(
ctx: RuleContext = {"mode": "markdown", "dialect": dialect, "locale": resolved_locale}
spans = normalize_spans(markdown_spans(input_text, dialect))
return run_over_spans(input_text, spans, plan, locale_data, ctx)


def analyze_text_pipeline(
input_text: str, *, locale: object, rules: dict[str, bool] | None
) -> list[Change]:
"""analyze.md section 1: the same pipeline as `run_text_pipeline`, reporting instead of
applying."""
resolved_locale, locale_data, plan = prepare(locale, rules)
cp = to_codepoints(input_text)
ctx: RuleContext = {"mode": "text", "dialect": None, "locale": resolved_locale}
return run_rules_recording(cp, plan, locale_data, ctx, list(range(len(cp))), len(cp))


def analyze_html_pipeline(
input_text: str, *, locale: object, rules: dict[str, bool] | None
) -> list[Change]:
"""analyze.md section 1, `html` mode: offsets are into the document, not into a span
(analyze.md section 6)."""
from polytypo._modes.html import html_spans

resolved_locale, locale_data, plan = prepare(locale, rules)
ctx: RuleContext = {"mode": "html", "dialect": None, "locale": resolved_locale}
spans = normalize_spans(html_spans(input_text))
return analyze_over_spans(input_text, spans, plan, locale_data, ctx)


def analyze_markdown_pipeline(
input_text: str, *, locale: object, dialect: str | None, rules: dict[str, bool] | None
) -> list[Change]:
"""analyze.md section 1, `markdown` mode. Dialect validation happens here exactly as it does
for `run_markdown_pipeline`, so an absent or unsupported dialect raises
POLYTYPO_INVALID_DIALECT from `analyze` too (analyze.md section 4, A1)."""
from polytypo._modes.markdown import markdown_spans, resolve_dialect

resolved_locale, locale_data, plan = prepare(locale, rules)
resolve_dialect(dialect)
ctx: RuleContext = {"mode": "markdown", "dialect": dialect, "locale": resolved_locale}
spans = normalize_spans(markdown_spans(input_text, dialect))
return analyze_over_spans(input_text, spans, plan, locale_data, ctx)
77 changes: 77 additions & 0 deletions src/polytypo/_engine/origin.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
"""analyze.md section 2: every reported offset is a code-point offset into the input the caller
passed, in every mode. The rules, however, run over an array that is not the input -- in `text`
mode edits from earlier rules have already shifted it, and in `html`/`markdown` mode it is the
marker-joined concatenation of the processable spans (modes.md 3.5). This module carries the one
structure that bridges the two: an origin map, parallel to the current code-point array, holding
the input offset each code point came from, or NO_ORIGIN for one the pipeline itself produced.
Mirrors polytypo-js's src/engine/origin.ts."""

from __future__ import annotations

from dataclasses import dataclass

from polytypo._engine.codepoints import from_codepoints
from polytypo._engine.edits import Edit

NO_ORIGIN = -1


@dataclass(frozen=True, slots=True)
class Change:
"""One entry of ``analyze``'s result, in input coordinates (analyze.md section 2)."""

rule_id: str
start: int
end: int
before: str
after: str


def origin_at(origin: list[int], index: int, input_length: int) -> int:
"""The input offset an edit boundary at ``index`` addresses. Synthetic code points have no
origin of their own, so the scan runs forward to the first that has one -- an insertion
between two earlier insertions still lands where the next real character is. Falling off the
end means the boundary is at the end of the input."""
for i in range(index, len(origin)):
if origin[i] != NO_ORIGIN:
return origin[i]
return input_length


def apply_edits_to_origin(origin: list[int], edits: list[Edit]) -> list[int]:
"""The origin map for the array ``apply_edits`` is about to produce. A replacement of equal
length keeps its origins position by position, which is what makes a conversion
(U+0020 -> U+00A0) still point at the character it converted; anything longer is synthetic
beyond the positions it covers."""
if not edits:
return list(origin)
out: list[int] = []
cursor = 0
for edit in edits:
out.extend(origin[cursor : edit.start])
for k in range(len(edit.replacement)):
source = edit.start + k
out.append(origin[source] if source < edit.end else NO_ORIGIN)
cursor = edit.end
out.extend(origin[cursor:])
return out


def record_changes(
cp: list[int], edits: list[Edit], origin: list[int], input_length: int, rule_id: str
) -> list[Change]:
"""One rule's edits, in the coordinates that rule saw, rendered as Changes in input
coordinates. ``before`` is the text this rule replaced and ``after`` what it replaced it with
(analyze.md section 2), so on a text two rules have both touched, ``before`` is what the
second rule saw rather than what the caller typed -- section 5 says so and shows the French
case where it matters."""
return [
Change(
rule_id=rule_id,
start=origin_at(origin, edit.start, input_length),
end=origin_at(origin, edit.end, input_length),
before=from_codepoints(cp[edit.start : edit.end]),
after=from_codepoints(edit.replacement),
)
for edit in edits
]
32 changes: 31 additions & 1 deletion src/polytypo/_engine/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,16 @@

from __future__ import annotations

from collections.abc import Callable
from typing import Any

# Import side effect: registers every rule id with the registry (mirrors polytypo-js's
# registry.ts, which statically imports every rule module for the same reason: importing the
# engine must be enough, without a separate caller-side step).
import polytypo._engine.rules # noqa: F401,E402
from polytypo._engine.edits import apply_edits
from polytypo._engine.edits import Edit, apply_edits
from polytypo._engine.locale import get_locale_data, resolve
from polytypo._engine.origin import Change, apply_edits_to_origin, record_changes
from polytypo._engine.registry import (
RuleContext,
get_rule,
Expand Down Expand Up @@ -64,3 +66,31 @@ def run_rules(
if edits:
current = apply_edits(current, edits, rule_id)
return current


def run_rules_recording(
cp: list[int],
plan: list[str],
locale_data: dict[str, Any],
ctx: RuleContext,
origin: list[int],
input_length: int,
filter_edits: Callable[[list[int], list[Edit]], list[Edit]] | None = None,
) -> list[Change]:
"""`run_rules`, keeping the edits instead of discarding them (analyze.md section 1: same
pipeline, same order, reporting rather than applying). The origin map travels alongside the
array so every change comes back in input coordinates, and `filter_edits` is the hook the
span-runner needs for modes.md 3.4's boundary filters -- text mode passes nothing and gets
the identity."""
current = cp
current_origin = origin
changes: list[Change] = []
for rule_id in plan:
rule_fn = get_rule(rule_id)
produced = rule_fn(current, locale_data, ctx)
edits = produced if filter_edits is None else filter_edits(current, produced)
if edits:
changes.extend(record_changes(current, edits, current_origin, input_length, rule_id))
current_origin = apply_edits_to_origin(current_origin, edits)
current = apply_edits(current, edits, rule_id)
return changes
28 changes: 28 additions & 0 deletions src/polytypo/_engine/span_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,15 @@

from polytypo._engine.codepoints import from_codepoints
from polytypo._engine.edits import apply_edits
from polytypo._engine.origin import Change
from polytypo._engine.pipeline import run_rules_recording
from polytypo._engine.registry import RuleContext, get_rule
from polytypo._modes.spans import (
Span,
concatenate_spans,
filter_boundary_edits,
normalize_spans,
origin_of_spans,
span_ranges_of,
split_on_marker,
)
Expand Down Expand Up @@ -65,3 +68,28 @@ def run_over_spans(
cursor = span.end
out.append(source[cursor:])
return "".join(out)


def analyze_over_spans(
source: str,
spans: list[Span],
plan: list[str],
locale_data: dict[str, Any],
ctx: RuleContext,
) -> list[Change]:
"""`run_over_spans`, reporting instead of applying (analyze.md section 1). The span table
supplies the origin map, so every change comes back in DOCUMENT coordinates -- analyze.md
section 6 names a runtime that reports span-local offsets here as the mistake that passes
every text-mode test."""
normalized = normalize_spans(spans)
if not normalized:
return []
return run_rules_recording(
concatenate_spans(source, normalized),
plan,
locale_data,
ctx,
origin_of_spans(source, normalized),
len(source),
lambda current, edits: filter_boundary_edits(current, edits, span_ranges_of(current)),
)
17 changes: 17 additions & 0 deletions src/polytypo/_modes/spans.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from polytypo._engine.codepoints import to_codepoints
from polytypo._engine.edits import Edit
from polytypo._engine.origin import NO_ORIGIN
from polytypo._engine.sentinels import LINE_MARKER, MARKER, is_marker
from polytypo.errors import POLYTYPO_RULE_CONTRACT, PolytypoError

Expand Down Expand Up @@ -136,3 +137,19 @@ def split_on_marker(cp: list[int], expected: int) -> list[list[int]]:
f"found {len(pieces)}",
)
return pieces


def origin_of_spans(source: str, spans: list[Span]) -> list[int]:
"""The origin map for `concatenate_spans` (analyze.md section 2): for every code point of the
joined array, the code-point offset of the character it came from IN THE DOCUMENT, and
NO_ORIGIN for the markers, which came from nowhere. A Python `Span`'s bounds are already
code-point offsets, so unlike polytypo-js's `originOfSpans` there is no UTF-16 index to
convert here."""
origin: list[int] = []
previous: Span | None = None
for span in spans:
if previous is not None:
origin.append(NO_ORIGIN)
origin.extend(range(span.start, span.end))
previous = span
return origin
12 changes: 10 additions & 2 deletions src/polytypo/html.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,19 @@

from __future__ import annotations

from polytypo._engine.mode_pipelines import run_html_pipeline
from polytypo._engine.mode_pipelines import analyze_html_pipeline, run_html_pipeline
from polytypo._engine.origin import Change
from polytypo.errors import PolytypoError

__all__ = ["PolytypoError", "transform"]
__all__ = ["Change", "PolytypoError", "analyze", "transform"]


def transform(input: str, *, locale: str, rules: dict[str, bool] | None = None) -> str:
return run_html_pipeline(input, locale=locale, rules=rules)


def analyze(input: str, *, locale: str, rules: dict[str, bool] | None = None) -> list[Change]:
"""The same pipeline as this entry's `transform`, reporting instead of applying
(spec/rules/analyze.md). Offsets are code-point offsets into the **document**, not into a
span (analyze.md section 2)."""
return analyze_html_pipeline(input, locale=locale, rules=rules)
14 changes: 12 additions & 2 deletions src/polytypo/markdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,23 @@

from __future__ import annotations

from polytypo._engine.mode_pipelines import run_markdown_pipeline
from polytypo._engine.mode_pipelines import analyze_markdown_pipeline, run_markdown_pipeline
from polytypo._engine.origin import Change
from polytypo.errors import PolytypoError

__all__ = ["PolytypoError", "transform"]
__all__ = ["Change", "PolytypoError", "analyze", "transform"]


def transform(
input: str, *, locale: str, dialect: str, rules: dict[str, bool] | None = None
) -> str:
return run_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules)


def analyze(
input: str, *, locale: str, dialect: str, rules: dict[str, bool] | None = None
) -> list[Change]:
"""The same pipeline as this entry's `transform`, reporting instead of applying
(spec/rules/analyze.md). Offsets are code-point offsets into the **document**, not into a
span (analyze.md section 2)."""
return analyze_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules)
Loading
Loading