Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 16 additions & 6 deletions src/polytypo/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,20 @@ def transform(
mode: str | None = None,
dialect: str | None = None,
rules: dict[str, bool] | None = None,
narrow_nbsp: str | None = None,
) -> str:
"""`narrow_nbsp="nbsp"` makes the engine emit U+00A0 everywhere it would emit U+202F
(nbsp.md 3.1a). It moves the rule's target rather than post-processing the output, so the
result stays a fixed point. Validation order is mode -> narrow_nbsp -> rules -> locale ->
dialect, and the check runs whether or not `nbsp` is enabled."""
resolved_mode = _resolve_mode(mode)
if resolved_mode == "text":
return run_text_pipeline(input, locale=locale, rules=rules)
return run_text_pipeline(input, locale=locale, rules=rules, narrow_nbsp=narrow_nbsp)
if resolved_mode == "html":
return run_html_pipeline(input, locale=locale, rules=rules)
return run_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules)
return run_html_pipeline(input, locale=locale, rules=rules, narrow_nbsp=narrow_nbsp)
return run_markdown_pipeline(
input, locale=locale, dialect=dialect, rules=rules, narrow_nbsp=narrow_nbsp
)


def analyze(
Expand All @@ -57,6 +64,7 @@ def analyze(
mode: str | None = None,
dialect: str | None = None,
rules: dict[str, bool] | None = None,
narrow_nbsp: str | None = None,
) -> list[Change]:
"""The same pipeline as `transform`, reporting what it would do instead of doing it
(spec/rules/analyze.md). Offsets are code-point offsets into `input` in every mode.
Expand All @@ -68,7 +76,9 @@ def analyze(
for the text (analyze.md sections 4 and 5)."""
resolved_mode = _resolve_mode(mode)
if resolved_mode == "text":
return analyze_text_pipeline(input, locale=locale, rules=rules)
return analyze_text_pipeline(input, locale=locale, rules=rules, narrow_nbsp=narrow_nbsp)
if resolved_mode == "html":
return analyze_html_pipeline(input, locale=locale, rules=rules)
return analyze_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules)
return analyze_html_pipeline(input, locale=locale, rules=rules, narrow_nbsp=narrow_nbsp)
return analyze_markdown_pipeline(
input, locale=locale, dialect=dialect, rules=rules, narrow_nbsp=narrow_nbsp
)
91 changes: 79 additions & 12 deletions src/polytypo/_engine/mode_pipelines.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,75 +11,142 @@
from __future__ import annotations

from polytypo._engine.codepoints import from_codepoints, to_codepoints
from polytypo._engine.narrow_target import resolve_narrow_target
from polytypo._engine.origin import Change
from polytypo._engine.pipeline import prepare, run_rules, run_rules_recording
from polytypo._engine.registry import RuleContext
from polytypo._engine.span_runner import analyze_over_spans, run_over_spans
from polytypo._modes.spans import normalize_spans


def run_text_pipeline(input_text: str, *, locale: object, rules: dict[str, bool] | None) -> str:
def run_text_pipeline(
input_text: str,
*,
locale: object,
rules: dict[str, bool] | None,
narrow_nbsp: str | None = None,
) -> str:
narrow_target = resolve_narrow_target(narrow_nbsp)
resolved_locale, locale_data, plan = prepare(locale, rules)
cp = to_codepoints(input_text)
ctx: RuleContext = {"mode": "text", "dialect": None, "locale": resolved_locale}
ctx: RuleContext = {
"mode": "text",
"dialect": None,
"locale": resolved_locale,
"narrow_target": narrow_target,
}
return from_codepoints(run_rules(cp, plan, locale_data, ctx))


def run_html_pipeline(input_text: str, *, locale: object, rules: dict[str, bool] | None) -> str:
def run_html_pipeline(
input_text: str,
*,
locale: object,
rules: dict[str, bool] | None,
narrow_nbsp: str | None = None,
) -> str:
from polytypo._modes.html import html_spans

narrow_target = resolve_narrow_target(narrow_nbsp)
resolved_locale, locale_data, plan = prepare(locale, rules)
ctx: RuleContext = {"mode": "html", "dialect": None, "locale": resolved_locale}
ctx: RuleContext = {
"mode": "html",
"dialect": None,
"locale": resolved_locale,
"narrow_target": narrow_target,
}
spans = normalize_spans(html_spans(input_text))
return run_over_spans(input_text, spans, plan, locale_data, ctx)


def run_markdown_pipeline(
input_text: str, *, locale: object, dialect: str | None, rules: dict[str, bool] | None
input_text: str,
*,
locale: object,
dialect: str | None,
rules: dict[str, bool] | None,
narrow_nbsp: str | None = None,
) -> str:
from polytypo._modes.markdown import markdown_spans, resolve_dialect

narrow_target = resolve_narrow_target(narrow_nbsp)
resolved_locale, locale_data, plan = prepare(locale, rules)
resolve_dialect(dialect)
ctx: RuleContext = {"mode": "markdown", "dialect": dialect, "locale": resolved_locale}
ctx: RuleContext = {
"mode": "markdown",
"dialect": dialect,
"locale": resolved_locale,
"narrow_target": narrow_target,
}
spans = normalize_spans(markdown_spans(input_text, dialect))
return run_over_spans(input_text, spans, plan, locale_data, ctx)


def analyze_text_pipeline(
input_text: str, *, locale: object, rules: dict[str, bool] | None
input_text: str,
*,
locale: object,
rules: dict[str, bool] | None,
narrow_nbsp: str | None = None,
) -> list[Change]:
"""analyze.md section 1: the same pipeline as `run_text_pipeline`, reporting instead of
applying."""
narrow_target = resolve_narrow_target(narrow_nbsp)
resolved_locale, locale_data, plan = prepare(locale, rules)
cp = to_codepoints(input_text)
ctx: RuleContext = {"mode": "text", "dialect": None, "locale": resolved_locale}
ctx: RuleContext = {
"mode": "text",
"dialect": None,
"locale": resolved_locale,
"narrow_target": narrow_target,
}
return run_rules_recording(cp, plan, locale_data, ctx, list(range(len(cp))), len(cp))


def analyze_html_pipeline(
input_text: str, *, locale: object, rules: dict[str, bool] | None
input_text: str,
*,
locale: object,
rules: dict[str, bool] | None,
narrow_nbsp: str | None = None,
) -> list[Change]:
"""analyze.md section 1, `html` mode: offsets are into the document, not into a span
(analyze.md section 6)."""
from polytypo._modes.html import html_spans

narrow_target = resolve_narrow_target(narrow_nbsp)
resolved_locale, locale_data, plan = prepare(locale, rules)
ctx: RuleContext = {"mode": "html", "dialect": None, "locale": resolved_locale}
ctx: RuleContext = {
"mode": "html",
"dialect": None,
"locale": resolved_locale,
"narrow_target": narrow_target,
}
spans = normalize_spans(html_spans(input_text))
return analyze_over_spans(input_text, spans, plan, locale_data, ctx)


def analyze_markdown_pipeline(
input_text: str, *, locale: object, dialect: str | None, rules: dict[str, bool] | None
input_text: str,
*,
locale: object,
dialect: str | None,
rules: dict[str, bool] | None,
narrow_nbsp: str | None = None,
) -> list[Change]:
"""analyze.md section 1, `markdown` mode. Dialect validation happens here exactly as it does
for `run_markdown_pipeline`, so an absent or unsupported dialect raises
POLYTYPO_INVALID_DIALECT from `analyze` too (analyze.md section 4, A1)."""
from polytypo._modes.markdown import markdown_spans, resolve_dialect

narrow_target = resolve_narrow_target(narrow_nbsp)
resolved_locale, locale_data, plan = prepare(locale, rules)
resolve_dialect(dialect)
ctx: RuleContext = {"mode": "markdown", "dialect": dialect, "locale": resolved_locale}
ctx: RuleContext = {
"mode": "markdown",
"dialect": dialect,
"locale": resolved_locale,
"narrow_target": narrow_target,
}
spans = normalize_spans(markdown_spans(input_text, dialect))
return analyze_over_spans(input_text, spans, plan, locale_data, ctx)
26 changes: 26 additions & 0 deletions src/polytypo/_engine/narrow_target.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
"""nbsp.md 3.1a: resolve the ``narrow_nbsp`` option to the code point the rule writes.

Done once, at the call boundary, so no rule ever sees the option's string. Checked immediately
after ``mode`` and before ``rules`` -- the two checks that read nothing but the call itself come
first (ARCHITECTURE.md section 7). It runs whether or not ``nbsp`` is enabled, so a misspelled
value still raises rather than being silently ignored."""

from __future__ import annotations

from polytypo.errors import POLYTYPO_INVALID_OPTION, PolytypoError

#: U+202F, the narrow no-break space -- nbsp.md 3.1a's default NARROW-TARGET.
NARROW_NO_BREAK_SPACE = 0x202F
#: U+00A0, what ``narrow_nbsp="nbsp"`` substitutes for it.
NO_BREAK_SPACE = 0x00A0


def resolve_narrow_target(value: str | None) -> int:
if value is None or value == "narrow":
return NARROW_NO_BREAK_SPACE
if value == "nbsp":
return NO_BREAK_SPACE
raise PolytypoError(
POLYTYPO_INVALID_OPTION,
f'Unknown narrow_nbsp {value!r}. Expected "narrow" or "nbsp".',
)
3 changes: 3 additions & 0 deletions src/polytypo/_engine/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ class RuleContext(TypedDict):
mode: str
dialect: str | None
locale: str
# nbsp.md 3.1a NARROW-TARGET, already resolved to a code point: U+202F by default, U+00A0
# when the caller passed narrow_nbsp="nbsp". A rule reads a code point and never the option.
narrow_target: int


RuleFn = Callable[[list[int], dict[str, Any], RuleContext], list[Edit]]
Expand Down
24 changes: 17 additions & 7 deletions src/polytypo/_engine/rules/nbsp.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,15 @@ def scan(cp: list[int], locale_data: dict[str, Any], ctx: RuleContext) -> list[E

sub_rule_edits = [
_n1_before_punctuation(cp, before_punct, narrow_before_punct, openish, closeish),
_n2_narrow_before_punctuation(cp, before_punct, narrow_before_punct, openish, closeish),
_n2_narrow_before_punctuation(
cp, before_punct, narrow_before_punct, openish, closeish, ctx["narrow_target"]
),
_n3_after_short_words(cp, nbsp_data["afterShortWords"], openish),
_n4_abbreviations(cp, nbsp_data["abbreviations"]),
_n5_before_units(cp, nbsp_data["beforeUnits"]),
_n6_after_symbols(cp, nbsp_data["afterSymbols"]),
_n7_initial_binding(cp, nbsp_data["initialBinding"], openish),
_n8_quotes_inner_space(cp, locale_data["quotes"]),
_n8_quotes_inner_space(cp, locale_data["quotes"], ctx["narrow_target"]),
_n9_before_number(cp, nbsp_data["beforeNumber"], openish),
_n10_before_word(cp, nbsp_data["beforeWord"], openish, nbsp_data["initialBinding"]),
]
Expand Down Expand Up @@ -200,16 +202,22 @@ def _n2_narrow_before_punctuation(
narrow_before_punct: set[int],
openish: frozenset[int],
closeish: frozenset[int],
narrow_target: int,
) -> list[Edit]:
"""nbsp.md 3.4 with NARROW-TARGET (3.1a) in place of U+202F: the already-correct state, the
conversion target and the insertion all move together. `other` is the NOBREAK member that is
not the target, which is what the sub-rule converts -- with the substitution on, N2 and N1
want the same character, never different ones."""
other = NNBSP if narrow_target == NBSP else NBSP
return _before_punctuation_common(
cp,
narrow_before_punct,
before_punct,
openish,
closeish,
NNBSP,
frozenset({SP, NBSP}),
NNBSP,
narrow_target,
frozenset({SP, other}),
narrow_target,
)


Expand Down Expand Up @@ -461,15 +469,17 @@ def _chain_confirmed(cp: list[int], p: int, openish: frozenset[int]) -> bool:
# ---------------------------------------------------------------- N8


def _n8_quotes_inner_space(cp: list[int], quotes_data: dict[str, Any]) -> list[Edit]:
def _n8_quotes_inner_space(
cp: list[int], quotes_data: dict[str, Any], narrow_target: int
) -> list[Edit]:
edits: list[Edit] = []
n = len(cp)
for pair_key in ("primary", "secondary"):
pair = quotes_data[pair_key]
inner = pair["innerSpace"]
if inner == "none":
continue
target = NBSP if inner == "nbsp" else NNBSP
target = NBSP if inner == "nbsp" else narrow_target
open_glyph = ord(pair["open"])
close_glyph = ord(pair["close"])
if open_glyph == close_glyph:
Expand Down
5 changes: 4 additions & 1 deletion src/polytypo/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,13 @@ class is enough: the stable machine-readable ``code`` is the contract, not the e
POLYTYPO_MALFORMED_LOCALE_DATA = "POLYTYPO_MALFORMED_LOCALE_DATA"
POLYTYPO_RULE_CONTRACT = "POLYTYPO_RULE_CONTRACT"
POLYTYPO_MALFORMED_INPUT = "POLYTYPO_MALFORMED_INPUT"
# Spec 1.3.0. Deliberately general: every option added from 1.3.0 on shares this code, while
# `mode` and `dialect` keep their own because callers branch on them.
POLYTYPO_INVALID_OPTION = "POLYTYPO_INVALID_OPTION"


class PolytypoError(Exception):
"""Carries a stable ``code`` from the seven above. No other exception type from this package
"""Carries a stable ``code`` from the eight above. No other exception type from this package
or its dependencies is ever allowed to escape :func:`polytypo.transform`."""

def __init__(self, code: str, message: str) -> None:
Expand Down
24 changes: 18 additions & 6 deletions src/polytypo/html.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,24 @@
__all__ = ["Change", "PolytypoError", "analyze", "transform"]


def transform(input: str, *, locale: str, rules: dict[str, bool] | None = None) -> str:
return run_html_pipeline(input, locale=locale, rules=rules)


def analyze(input: str, *, locale: str, rules: dict[str, bool] | None = None) -> list[Change]:
def transform(
input: str,
*,
locale: str,
rules: dict[str, bool] | None = None,
narrow_nbsp: str | None = None,
) -> str:
return run_html_pipeline(input, locale=locale, rules=rules, narrow_nbsp=narrow_nbsp)


def analyze(
input: str,
*,
locale: str,
rules: dict[str, bool] | None = None,
narrow_nbsp: str | None = None,
) -> list[Change]:
"""The same pipeline as this entry's `transform`, reporting instead of applying
(spec/rules/analyze.md). Offsets are code-point offsets into the **document**, not into a
span (analyze.md section 2)."""
return analyze_html_pipeline(input, locale=locale, rules=rules)
return analyze_html_pipeline(input, locale=locale, rules=rules, narrow_nbsp=narrow_nbsp)
22 changes: 18 additions & 4 deletions src/polytypo/markdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,29 @@


def transform(
input: str, *, locale: str, dialect: str, rules: dict[str, bool] | None = None
input: str,
*,
locale: str,
dialect: str,
rules: dict[str, bool] | None = None,
narrow_nbsp: str | None = None,
) -> str:
return run_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules)
return run_markdown_pipeline(
input, locale=locale, dialect=dialect, rules=rules, narrow_nbsp=narrow_nbsp
)


def analyze(
input: str, *, locale: str, dialect: str, rules: dict[str, bool] | None = None
input: str,
*,
locale: str,
dialect: str,
rules: dict[str, bool] | None = None,
narrow_nbsp: str | None = None,
) -> list[Change]:
"""The same pipeline as this entry's `transform`, reporting instead of applying
(spec/rules/analyze.md). Offsets are code-point offsets into the **document**, not into a
span (analyze.md section 2)."""
return analyze_markdown_pipeline(input, locale=locale, dialect=dialect, rules=rules)
return analyze_markdown_pipeline(
input, locale=locale, dialect=dialect, rules=rules, narrow_nbsp=narrow_nbsp
)
Loading
Loading