Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 28 additions & 20 deletions lib/polytypo.rb
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,23 @@ module Polytypo
#
# Only `mode: "text"`/`"html"` ever touch this file's own requires; `mode: "markdown"` lazily
# requires "commonmarker" from within Modes::Markdown, never at load time of this file.
def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil)
def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil, narrow_nbsp: nil)
resolved_mode = resolve_mode(mode)
narrow_target = Engine.resolve_narrow_target(narrow_nbsp)

case resolved_mode
when "text"
if dialect
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
end
transform_text(input, locale, rules)
transform_text(input, locale, rules, narrow_target)
when "html"
if dialect
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
end
transform_html(input, locale, rules)
transform_html(input, locale, rules, narrow_target)
else # "markdown"
transform_markdown(input, locale, dialect, rules)
transform_markdown(input, locale, dialect, rules, narrow_target)
end
end

Expand All @@ -59,22 +60,23 @@ def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil)
# for the text (analyze.md sections 4 and 5).
#
# Pure and thread-safe on the same terms as .transform.
def self.analyze(input, locale:, mode: "text", dialect: nil, rules: nil)
def self.analyze(input, locale:, mode: "text", dialect: nil, rules: nil, narrow_nbsp: nil)
resolved_mode = resolve_mode(mode)
narrow_target = Engine.resolve_narrow_target(narrow_nbsp)

case resolved_mode
when "text"
if dialect
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
end
analyze_text(input, locale, rules)
analyze_text(input, locale, rules, narrow_target)
when "html"
if dialect
raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"')
end
analyze_html(input, locale, rules)
analyze_html(input, locale, rules, narrow_target)
else # "markdown"
analyze_markdown(input, locale, dialect, rules)
analyze_markdown(input, locale, dialect, rules, narrow_target)
end
end

Expand All @@ -90,61 +92,67 @@ def self.resolve_mode(mode)
end
private_class_method :resolve_mode

def self.transform_text(input, locale, rules)
def self.transform_text(input, locale, rules, narrow_target)
_resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
cp = Engine::Codepoints.to_codepoints(input)
ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale)
ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale,
narrow_target: narrow_target)
result = Engine::Pipeline.run_rules(cp, plan, locale_data, ctx)
Engine::Codepoints.from_codepoints(result)
end
private_class_method :transform_text

def self.transform_html(input, locale, rules)
def self.transform_html(input, locale, rules, narrow_target)
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
spans = Modes::Html.html_spans(input)
ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale)
ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale,
narrow_target: narrow_target)
cp = Engine::Codepoints.to_codepoints(input)
Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx)
end
private_class_method :transform_html

def self.transform_markdown(input, locale, dialect, rules)
def self.transform_markdown(input, locale, dialect, rules, narrow_target)
# Validation order is public, tested behaviour, identical across every runtime: rules (an
# unknown rule id), then locale (an unknown locale), then dialect/parsing.
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
require_relative "polytypo/modes/markdown"
Modes::Markdown.resolve_dialect(dialect)
spans = Modes::Markdown.markdown_spans(input)
ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale)
ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale,
narrow_target: narrow_target)
cp = Engine::Codepoints.to_codepoints(input)
Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx)
end
private_class_method :transform_markdown

def self.analyze_text(input, locale, rules)
def self.analyze_text(input, locale, rules, narrow_target)
_resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
cp = Engine::Codepoints.to_codepoints(input)
ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale)
ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale,
narrow_target: narrow_target)
Engine::Pipeline.run_rules_recording(cp, plan, locale_data, ctx, (0...cp.length).to_a, cp.length)
end
private_class_method :analyze_text

def self.analyze_html(input, locale, rules)
def self.analyze_html(input, locale, rules, narrow_target)
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
spans = Modes::Html.html_spans(input)
ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale)
ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale,
narrow_target: narrow_target)
Modes::Runner.analyze_over_spans(Engine::Codepoints.to_codepoints(input), spans, plan, locale_data, ctx)
end
private_class_method :analyze_html

def self.analyze_markdown(input, locale, dialect, rules)
def self.analyze_markdown(input, locale, dialect, rules, narrow_target)
# Validation order is public, tested behaviour and is shared with .transform: rules, then
# locale, then dialect/parsing (analyze.md section 4, A1).
resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules)
require_relative "polytypo/modes/markdown"
Modes::Markdown.resolve_dialect(dialect)
spans = Modes::Markdown.markdown_spans(input)
ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale)
ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale,
narrow_target: narrow_target)
Modes::Runner.analyze_over_spans(Engine::Codepoints.to_codepoints(input), spans, plan, locale_data, ctx)
end
private_class_method :analyze_markdown
Expand Down
22 changes: 21 additions & 1 deletion lib/polytypo/engine/pipeline.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,27 @@
module Polytypo
module Engine
# Per-call context every rule reads, alongside the code-point array and the locale data.
RuleContext = Struct.new(:mode, :dialect, :locale, keyword_init: true)
# :narrow_target is nbsp.md 3.1a's NARROW-TARGET, already resolved to a code point: U+202F
# by default, U+00A0 when the caller passed narrow_nbsp: "nbsp". A rule reads a code point
# and never the option, so the string never reaches the pipeline.
RuleContext = Struct.new(:mode, :dialect, :locale, :narrow_target, keyword_init: true)

# nbsp.md 3.1a: resolve the narrow_nbsp option to the code point the rule writes. Checked
# immediately after mode and before rules (ARCHITECTURE.md 7) -- the two checks that read
# nothing but the call itself come first -- and it runs whether or not `nbsp` is enabled, so
# a misspelled value still raises rather than being silently ignored.
NARROW_NO_BREAK_SPACE = 0x202F
NO_BREAK_SPACE = 0x00A0

def self.resolve_narrow_target(value)
return NARROW_NO_BREAK_SPACE if value.nil? || value == "narrow"
return NO_BREAK_SPACE if value == "nbsp"

raise Polytypo::Error.new(
Polytypo::CODE_INVALID_OPTION,
"Unknown narrow_nbsp #{value.inspect}. Expected \"narrow\" or \"nbsp\".",
)
end

# Resolves the locale, builds the rule plan, and runs each enabled rule in
# spec/rules/order.json order over a code-point array, applying its edits before the next
Expand Down
19 changes: 13 additions & 6 deletions lib/polytypo/engine/rules/nbsp.rb
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,8 @@ def self.sentence_dash?(cp)
:before_punctuation, :narrow_before_punctuation, :short_words, :abbreviations,
:units, :before_number, :before_word, :symbols, :initial_binding, :opens, :closes,
:quote_pairs,
# nbsp.md 3.1a NARROW-TARGET: what N2 writes, and what N8 writes for a narrow-nbsp pair.
:narrow_target,
keyword_init: true,
)

Expand All @@ -145,7 +147,7 @@ def self.prepare_list(entries)
# Resolves the locale's nbsp and quotes fields to code points once. nbsp.md 2 lists the
# fields; 2.1 explains why the mechanism (U+00A0 vs U+202F, convert-only vs insert)
# lives here and not in the locale file.
def self.prepare(locale_data)
def self.prepare(locale_data, narrow_target = NNBSP)
data = locale_data["nbsp"]
before_punctuation = data["beforePunctuation"].map { |e| single_code_point(e, "beforePunctuation") }
narrow_before_punctuation =
Expand Down Expand Up @@ -191,7 +193,7 @@ def self.prepare(locale_data)
# (7.5).
next if open_cp == close_cp

target = pair["innerSpace"] == "nbsp" ? NBSP : NNBSP
target = pair["innerSpace"] == "nbsp" ? NBSP : narrow_target
quote_pairs << QuoteTarget.new(open_cp, close_cp, target)
end

Expand All @@ -208,6 +210,7 @@ def self.prepare(locale_data)
opens: opens,
closes: closes,
quote_pairs: quote_pairs,
narrow_target: narrow_target,
)
end

Expand Down Expand Up @@ -610,12 +613,16 @@ def self.forward_binding_sub_rule(cp, prep, claims, patterns, wants_digit)
# yielding the index to a lower-priority sub-rule. Sub-rules wanting *different* code
# points at a shared index are therefore made disjoint by construction elsewhere
# (N1/N2's quote-glyph guard, nbsp.md 3.10.1) rather than relying on ordering alone.
def self.scan(cp, locale_data, _ctx)
prep = prepare(locale_data)
def self.scan(cp, locale_data, ctx)
prep = prepare(locale_data, ctx.narrow_target)
claims = Array.new(cp.length + 1)

punctuation_sub_rule(cp, prep, claims, prep.before_punctuation, NBSP, NNBSP) # N1
punctuation_sub_rule(cp, prep, claims, prep.narrow_before_punctuation, NNBSP, NBSP) # N2
punctuation_sub_rule(cp, prep, claims, prep.before_punctuation, NBSP, NNBSP) # N1
# N2's target is NARROW-TARGET (nbsp.md 3.1a); `other` is the NOBREAK member that is
# not the target, which is what the sub-rule converts. With the substitution on, N2 and
# N1 want the same character -- never different ones.
other = prep.narrow_target == NBSP ? NNBSP : NBSP
punctuation_sub_rule(cp, prep, claims, prep.narrow_before_punctuation, prep.narrow_target, other) # N2
short_words_sub_rule(cp, prep, claims) # N3
abbreviations_sub_rule(cp, prep, claims) # N4
units_sub_rule(cp, prep, claims) # N5
Expand Down
3 changes: 3 additions & 0 deletions lib/polytypo/errors.rb
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ module Polytypo
CODE_MALFORMED_LOCALE_DATA = "POLYTYPO_MALFORMED_LOCALE_DATA"
CODE_RULE_CONTRACT = "POLYTYPO_RULE_CONTRACT"
CODE_MALFORMED_INPUT = "POLYTYPO_MALFORMED_INPUT"
# Spec 1.3.0. Deliberately general: every option added from 1.3.0 on shares this code, while
# `mode` and `dialect` keep their own because callers branch on them.
CODE_INVALID_OPTION = "POLYTYPO_INVALID_OPTION"

# The only error type Transform ever raises. One class is enough: the stable machine-readable
# +code+ is the contract, not the exception type or the message text.
Expand Down
5 changes: 4 additions & 1 deletion spec/conformance_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
opts = { locale: locale, mode: c["mode"] }
opts[:dialect] = c["dialect"] if c["dialect"]
opts[:rules] = c["rules"] if c["rules"]
opts[:narrow_nbsp] = c["narrowNbsp"] if c["narrowNbsp"]

if c["throws"]
expect { Polytypo.transform(c["in"], **opts) }
Expand All @@ -39,7 +40,9 @@
expect(escape_non_ascii(got)).to eq(escape_non_ascii(expected)),
"out = #{escape_non_ascii(got)}, want #{escape_non_ascii(expected)}"

# Free coverage, and the most common port bug (ARCHITECTURE.md 6.1).
# Free coverage, and the most common port bug (ARCHITECTURE.md 6.1). The re-run carries
# the case's OWN options: a case with narrowNbsp is a fixed point under that option and
# not under the defaults, so spreading **opts here is contract, not convenience.
twice = Polytypo.transform(expected, **opts)
expect(escape_non_ascii(twice)).to eq(escape_non_ascii(expected)),
"not idempotent: transform(out) = #{escape_non_ascii(twice)}"
Expand Down
97 changes: 97 additions & 0 deletions spec/narrow_nbsp_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
# frozen_string_literal: true

require "spec_helper"

# spec/rules/nbsp.md 3.1a -- `narrow_nbsp` moves NARROW-TARGET, it does not post-process.
RSpec.describe "Polytypo narrow_nbsp" do
nbsp = " "
nnbsp = " "

def fr(text, **kwargs)
Polytypo.transform(text, locale: "fr", **kwargs)
end

describe "what it changes" do
it "writes U+00A0 where N2 would write U+202F" do
text = "Un délai ? Vraiment ! Et puis ; voilà."
expect(fr(text)).to eq("Un délai#{nnbsp}? Vraiment#{nnbsp}! Et puis#{nnbsp}; voilà.")
expect(fr(text, narrow_nbsp: "nbsp"))
.to eq("Un délai#{nbsp}? Vraiment#{nbsp}! Et puis#{nbsp}; voilà.")
end

it "normalises an authored narrow space at a claimed index" do
expect(fr("Oui#{nnbsp}?", narrow_nbsp: "nbsp")).to eq("Oui#{nbsp}?")
expect(fr("Oui#{nnbsp}?")).to eq("Oui#{nnbsp}?")
end

it "leaves an authored narrow space alone elsewhere" do
expect(fr("mot#{nnbsp}mot", narrow_nbsp: "nbsp")).to eq("mot#{nnbsp}mot")
end
end

describe "what it does not change" do
it "claims the same indices under the same guards" do
expect(fr("12:30 et http://x ; oui", narrow_nbsp: "nbsp")).to eq("12:30 et http://x#{nbsp}; oui")
end

it "leaves a sub-rule whose target was already U+00A0" do
# N1 (the colon) and N8 (fr's primary pair, innerSpace "nbsp") do not move.
expect(fr("Il a dit : « oui » ; puis ?", narrow_nbsp: "nbsp"))
.to eq("Il a dit#{nbsp}: «#{nbsp}oui#{nbsp}»#{nbsp}; puis#{nbsp}?")
end

%w[en-US de-DE ru].each do |locale|
it "is a no-op in #{locale}, which never emits U+202F" do
text = "She said “hi” — really..."
expect(Polytypo.transform(text, locale: locale, narrow_nbsp: "nbsp"))
.to eq(Polytypo.transform(text, locale: locale))
end
end
end

describe "idempotency" do
it "is a fixed point under the option" do
["Un délai ? Vraiment !", "Oui#{nnbsp}?", "Il a dit : « oui » ;"].each do |text|
once = fr(text, narrow_nbsp: "nbsp")
expect(fr(once, narrow_nbsp: "nbsp")).to eq(once)
end
end

it "shows why: post-processing the default output is not one" do
# A caller's gsub is stable only as long as it always runs. Feed it back through the
# default pipeline and N2 converts it straight back.
post_processed = fr("Un délai ?").gsub(nnbsp, nbsp)
expect(post_processed).to eq("Un délai#{nbsp}?")
expect(fr(post_processed)).to eq("Un délai#{nnbsp}?")
end
end

describe "validation" do
def code_of(**kwargs)
Polytypo.transform("x", **kwargs)
"NO THROW"
rescue Polytypo::Error => e
e.code
end

it "raises POLYTYPO_INVALID_OPTION for an unknown value" do
expect(code_of(locale: "fr", narrow_nbsp: "wide")).to eq(Polytypo::CODE_INVALID_OPTION)
end

it "is checked after mode and before rules and locale" do
expect(code_of(locale: "fr", mode: "yaml", narrow_nbsp: "wide")).to eq(Polytypo::CODE_INVALID_MODE)
expect(code_of(locale: "fr", narrow_nbsp: "wide", rules: { "nope" => true }))
.to eq(Polytypo::CODE_INVALID_OPTION)
expect(code_of(locale: "xx", narrow_nbsp: "wide")).to eq(Polytypo::CODE_INVALID_OPTION)
end

it "raises even when nbsp is disabled, because the check belongs to the call" do
expect(code_of(locale: "fr", narrow_nbsp: "wide", rules: { "nbsp" => false }))
.to eq(Polytypo::CODE_INVALID_OPTION)
end

it "accepts the explicit default" do
expect(fr("Un délai ?", narrow_nbsp: "narrow")).to eq("Un délai#{nnbsp}?")
end
end
end
Loading