diff --git a/lib/polytypo.rb b/lib/polytypo.rb index 4d4379a..2517e8d 100644 --- a/lib/polytypo.rb +++ b/lib/polytypo.rb @@ -28,22 +28,23 @@ module Polytypo # # Only `mode: "text"`/`"html"` ever touch this file's own requires; `mode: "markdown"` lazily # requires "commonmarker" from within Modes::Markdown, never at load time of this file. - def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil) + def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil, narrow_nbsp: nil) resolved_mode = resolve_mode(mode) + narrow_target = Engine.resolve_narrow_target(narrow_nbsp) case resolved_mode when "text" if dialect raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') end - transform_text(input, locale, rules) + transform_text(input, locale, rules, narrow_target) when "html" if dialect raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') end - transform_html(input, locale, rules) + transform_html(input, locale, rules, narrow_target) else # "markdown" - transform_markdown(input, locale, dialect, rules) + transform_markdown(input, locale, dialect, rules, narrow_target) end end @@ -59,22 +60,23 @@ def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil) # for the text (analyze.md sections 4 and 5). # # Pure and thread-safe on the same terms as .transform. - def self.analyze(input, locale:, mode: "text", dialect: nil, rules: nil) + def self.analyze(input, locale:, mode: "text", dialect: nil, rules: nil, narrow_nbsp: nil) resolved_mode = resolve_mode(mode) + narrow_target = Engine.resolve_narrow_target(narrow_nbsp) case resolved_mode when "text" if dialect raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') end - analyze_text(input, locale, rules) + analyze_text(input, locale, rules, narrow_target) when "html" if dialect raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') end - analyze_html(input, locale, rules) + analyze_html(input, locale, rules, narrow_target) else # "markdown" - analyze_markdown(input, locale, dialect, rules) + analyze_markdown(input, locale, dialect, rules, narrow_target) end end @@ -90,61 +92,67 @@ def self.resolve_mode(mode) end private_class_method :resolve_mode - def self.transform_text(input, locale, rules) + def self.transform_text(input, locale, rules, narrow_target) _resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules) cp = Engine::Codepoints.to_codepoints(input) - ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale) + ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale, + narrow_target: narrow_target) result = Engine::Pipeline.run_rules(cp, plan, locale_data, ctx) Engine::Codepoints.from_codepoints(result) end private_class_method :transform_text - def self.transform_html(input, locale, rules) + def self.transform_html(input, locale, rules, narrow_target) resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) spans = Modes::Html.html_spans(input) - ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale) + ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale, + narrow_target: narrow_target) cp = Engine::Codepoints.to_codepoints(input) Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx) end private_class_method :transform_html - def self.transform_markdown(input, locale, dialect, rules) + def self.transform_markdown(input, locale, dialect, rules, narrow_target) # Validation order is public, tested behaviour, identical across every runtime: rules (an # unknown rule id), then locale (an unknown locale), then dialect/parsing. resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) require_relative "polytypo/modes/markdown" Modes::Markdown.resolve_dialect(dialect) spans = Modes::Markdown.markdown_spans(input) - ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale) + ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale, + narrow_target: narrow_target) cp = Engine::Codepoints.to_codepoints(input) Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx) end private_class_method :transform_markdown - def self.analyze_text(input, locale, rules) + def self.analyze_text(input, locale, rules, narrow_target) _resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules) cp = Engine::Codepoints.to_codepoints(input) - ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale) + ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale, + narrow_target: narrow_target) Engine::Pipeline.run_rules_recording(cp, plan, locale_data, ctx, (0...cp.length).to_a, cp.length) end private_class_method :analyze_text - def self.analyze_html(input, locale, rules) + def self.analyze_html(input, locale, rules, narrow_target) resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) spans = Modes::Html.html_spans(input) - ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale) + ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale, + narrow_target: narrow_target) Modes::Runner.analyze_over_spans(Engine::Codepoints.to_codepoints(input), spans, plan, locale_data, ctx) end private_class_method :analyze_html - def self.analyze_markdown(input, locale, dialect, rules) + def self.analyze_markdown(input, locale, dialect, rules, narrow_target) # Validation order is public, tested behaviour and is shared with .transform: rules, then # locale, then dialect/parsing (analyze.md section 4, A1). resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) require_relative "polytypo/modes/markdown" Modes::Markdown.resolve_dialect(dialect) spans = Modes::Markdown.markdown_spans(input) - ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale) + ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale, + narrow_target: narrow_target) Modes::Runner.analyze_over_spans(Engine::Codepoints.to_codepoints(input), spans, plan, locale_data, ctx) end private_class_method :analyze_markdown diff --git a/lib/polytypo/engine/pipeline.rb b/lib/polytypo/engine/pipeline.rb index c85192d..d62023e 100644 --- a/lib/polytypo/engine/pipeline.rb +++ b/lib/polytypo/engine/pipeline.rb @@ -9,7 +9,27 @@ module Polytypo module Engine # Per-call context every rule reads, alongside the code-point array and the locale data. - RuleContext = Struct.new(:mode, :dialect, :locale, keyword_init: true) + # :narrow_target is nbsp.md 3.1a's NARROW-TARGET, already resolved to a code point: U+202F + # by default, U+00A0 when the caller passed narrow_nbsp: "nbsp". A rule reads a code point + # and never the option, so the string never reaches the pipeline. + RuleContext = Struct.new(:mode, :dialect, :locale, :narrow_target, keyword_init: true) + + # nbsp.md 3.1a: resolve the narrow_nbsp option to the code point the rule writes. Checked + # immediately after mode and before rules (ARCHITECTURE.md 7) -- the two checks that read + # nothing but the call itself come first -- and it runs whether or not `nbsp` is enabled, so + # a misspelled value still raises rather than being silently ignored. + NARROW_NO_BREAK_SPACE = 0x202F + NO_BREAK_SPACE = 0x00A0 + + def self.resolve_narrow_target(value) + return NARROW_NO_BREAK_SPACE if value.nil? || value == "narrow" + return NO_BREAK_SPACE if value == "nbsp" + + raise Polytypo::Error.new( + Polytypo::CODE_INVALID_OPTION, + "Unknown narrow_nbsp #{value.inspect}. Expected \"narrow\" or \"nbsp\".", + ) + end # Resolves the locale, builds the rule plan, and runs each enabled rule in # spec/rules/order.json order over a code-point array, applying its edits before the next diff --git a/lib/polytypo/engine/rules/nbsp.rb b/lib/polytypo/engine/rules/nbsp.rb index 43d163b..932df1d 100644 --- a/lib/polytypo/engine/rules/nbsp.rb +++ b/lib/polytypo/engine/rules/nbsp.rb @@ -122,6 +122,8 @@ def self.sentence_dash?(cp) :before_punctuation, :narrow_before_punctuation, :short_words, :abbreviations, :units, :before_number, :before_word, :symbols, :initial_binding, :opens, :closes, :quote_pairs, + # nbsp.md 3.1a NARROW-TARGET: what N2 writes, and what N8 writes for a narrow-nbsp pair. + :narrow_target, keyword_init: true, ) @@ -145,7 +147,7 @@ def self.prepare_list(entries) # Resolves the locale's nbsp and quotes fields to code points once. nbsp.md 2 lists the # fields; 2.1 explains why the mechanism (U+00A0 vs U+202F, convert-only vs insert) # lives here and not in the locale file. - def self.prepare(locale_data) + def self.prepare(locale_data, narrow_target = NNBSP) data = locale_data["nbsp"] before_punctuation = data["beforePunctuation"].map { |e| single_code_point(e, "beforePunctuation") } narrow_before_punctuation = @@ -191,7 +193,7 @@ def self.prepare(locale_data) # (7.5). next if open_cp == close_cp - target = pair["innerSpace"] == "nbsp" ? NBSP : NNBSP + target = pair["innerSpace"] == "nbsp" ? NBSP : narrow_target quote_pairs << QuoteTarget.new(open_cp, close_cp, target) end @@ -208,6 +210,7 @@ def self.prepare(locale_data) opens: opens, closes: closes, quote_pairs: quote_pairs, + narrow_target: narrow_target, ) end @@ -610,12 +613,16 @@ def self.forward_binding_sub_rule(cp, prep, claims, patterns, wants_digit) # yielding the index to a lower-priority sub-rule. Sub-rules wanting *different* code # points at a shared index are therefore made disjoint by construction elsewhere # (N1/N2's quote-glyph guard, nbsp.md 3.10.1) rather than relying on ordering alone. - def self.scan(cp, locale_data, _ctx) - prep = prepare(locale_data) + def self.scan(cp, locale_data, ctx) + prep = prepare(locale_data, ctx.narrow_target) claims = Array.new(cp.length + 1) - punctuation_sub_rule(cp, prep, claims, prep.before_punctuation, NBSP, NNBSP) # N1 - punctuation_sub_rule(cp, prep, claims, prep.narrow_before_punctuation, NNBSP, NBSP) # N2 + punctuation_sub_rule(cp, prep, claims, prep.before_punctuation, NBSP, NNBSP) # N1 + # N2's target is NARROW-TARGET (nbsp.md 3.1a); `other` is the NOBREAK member that is + # not the target, which is what the sub-rule converts. With the substitution on, N2 and + # N1 want the same character -- never different ones. + other = prep.narrow_target == NBSP ? NNBSP : NBSP + punctuation_sub_rule(cp, prep, claims, prep.narrow_before_punctuation, prep.narrow_target, other) # N2 short_words_sub_rule(cp, prep, claims) # N3 abbreviations_sub_rule(cp, prep, claims) # N4 units_sub_rule(cp, prep, claims) # N5 diff --git a/lib/polytypo/errors.rb b/lib/polytypo/errors.rb index 67e9fbe..dceafd5 100644 --- a/lib/polytypo/errors.rb +++ b/lib/polytypo/errors.rb @@ -10,6 +10,9 @@ module Polytypo CODE_MALFORMED_LOCALE_DATA = "POLYTYPO_MALFORMED_LOCALE_DATA" CODE_RULE_CONTRACT = "POLYTYPO_RULE_CONTRACT" CODE_MALFORMED_INPUT = "POLYTYPO_MALFORMED_INPUT" + # Spec 1.3.0. Deliberately general: every option added from 1.3.0 on shares this code, while + # `mode` and `dialect` keep their own because callers branch on them. + CODE_INVALID_OPTION = "POLYTYPO_INVALID_OPTION" # The only error type Transform ever raises. One class is enough: the stable machine-readable # +code+ is the contract, not the exception type or the message text. diff --git a/spec/conformance_spec.rb b/spec/conformance_spec.rb index 9e4841e..13d72c7 100644 --- a/spec/conformance_spec.rb +++ b/spec/conformance_spec.rb @@ -27,6 +27,7 @@ opts = { locale: locale, mode: c["mode"] } opts[:dialect] = c["dialect"] if c["dialect"] opts[:rules] = c["rules"] if c["rules"] + opts[:narrow_nbsp] = c["narrowNbsp"] if c["narrowNbsp"] if c["throws"] expect { Polytypo.transform(c["in"], **opts) } @@ -39,7 +40,9 @@ expect(escape_non_ascii(got)).to eq(escape_non_ascii(expected)), "out = #{escape_non_ascii(got)}, want #{escape_non_ascii(expected)}" - # Free coverage, and the most common port bug (ARCHITECTURE.md 6.1). + # Free coverage, and the most common port bug (ARCHITECTURE.md 6.1). The re-run carries + # the case's OWN options: a case with narrowNbsp is a fixed point under that option and + # not under the defaults, so spreading **opts here is contract, not convenience. twice = Polytypo.transform(expected, **opts) expect(escape_non_ascii(twice)).to eq(escape_non_ascii(expected)), "not idempotent: transform(out) = #{escape_non_ascii(twice)}" diff --git a/spec/narrow_nbsp_spec.rb b/spec/narrow_nbsp_spec.rb new file mode 100644 index 0000000..17e844e --- /dev/null +++ b/spec/narrow_nbsp_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require "spec_helper" + +# spec/rules/nbsp.md 3.1a -- `narrow_nbsp` moves NARROW-TARGET, it does not post-process. +RSpec.describe "Polytypo narrow_nbsp" do + nbsp = " " + nnbsp = " " + + def fr(text, **kwargs) + Polytypo.transform(text, locale: "fr", **kwargs) + end + + describe "what it changes" do + it "writes U+00A0 where N2 would write U+202F" do + text = "Un délai ? Vraiment ! Et puis ; voilà." + expect(fr(text)).to eq("Un délai#{nnbsp}? Vraiment#{nnbsp}! Et puis#{nnbsp}; voilà.") + expect(fr(text, narrow_nbsp: "nbsp")) + .to eq("Un délai#{nbsp}? Vraiment#{nbsp}! Et puis#{nbsp}; voilà.") + end + + it "normalises an authored narrow space at a claimed index" do + expect(fr("Oui#{nnbsp}?", narrow_nbsp: "nbsp")).to eq("Oui#{nbsp}?") + expect(fr("Oui#{nnbsp}?")).to eq("Oui#{nnbsp}?") + end + + it "leaves an authored narrow space alone elsewhere" do + expect(fr("mot#{nnbsp}mot", narrow_nbsp: "nbsp")).to eq("mot#{nnbsp}mot") + end + end + + describe "what it does not change" do + it "claims the same indices under the same guards" do + expect(fr("12:30 et http://x ; oui", narrow_nbsp: "nbsp")).to eq("12:30 et http://x#{nbsp}; oui") + end + + it "leaves a sub-rule whose target was already U+00A0" do + # N1 (the colon) and N8 (fr's primary pair, innerSpace "nbsp") do not move. + expect(fr("Il a dit : « oui » ; puis ?", narrow_nbsp: "nbsp")) + .to eq("Il a dit#{nbsp}: «#{nbsp}oui#{nbsp}»#{nbsp}; puis#{nbsp}?") + end + + %w[en-US de-DE ru].each do |locale| + it "is a no-op in #{locale}, which never emits U+202F" do + text = "She said “hi” — really..." + expect(Polytypo.transform(text, locale: locale, narrow_nbsp: "nbsp")) + .to eq(Polytypo.transform(text, locale: locale)) + end + end + end + + describe "idempotency" do + it "is a fixed point under the option" do + ["Un délai ? Vraiment !", "Oui#{nnbsp}?", "Il a dit : « oui » ;"].each do |text| + once = fr(text, narrow_nbsp: "nbsp") + expect(fr(once, narrow_nbsp: "nbsp")).to eq(once) + end + end + + it "shows why: post-processing the default output is not one" do + # A caller's gsub is stable only as long as it always runs. Feed it back through the + # default pipeline and N2 converts it straight back. + post_processed = fr("Un délai ?").gsub(nnbsp, nbsp) + expect(post_processed).to eq("Un délai#{nbsp}?") + expect(fr(post_processed)).to eq("Un délai#{nnbsp}?") + end + end + + describe "validation" do + def code_of(**kwargs) + Polytypo.transform("x", **kwargs) + "NO THROW" + rescue Polytypo::Error => e + e.code + end + + it "raises POLYTYPO_INVALID_OPTION for an unknown value" do + expect(code_of(locale: "fr", narrow_nbsp: "wide")).to eq(Polytypo::CODE_INVALID_OPTION) + end + + it "is checked after mode and before rules and locale" do + expect(code_of(locale: "fr", mode: "yaml", narrow_nbsp: "wide")).to eq(Polytypo::CODE_INVALID_MODE) + expect(code_of(locale: "fr", narrow_nbsp: "wide", rules: { "nope" => true })) + .to eq(Polytypo::CODE_INVALID_OPTION) + expect(code_of(locale: "xx", narrow_nbsp: "wide")).to eq(Polytypo::CODE_INVALID_OPTION) + end + + it "raises even when nbsp is disabled, because the check belongs to the call" do + expect(code_of(locale: "fr", narrow_nbsp: "wide", rules: { "nbsp" => false })) + .to eq(Polytypo::CODE_INVALID_OPTION) + end + + it "accepts the explicit default" do + expect(fr("Un délai ?", narrow_nbsp: "narrow")).to eq("Un délai#{nnbsp}?") + end + end +end