From 1b4d15d58eca8c48c2af2659422dc37aaab041b8 Mon Sep 17 00:00:00 2001 From: Iurii Rogulia Date: Sat, 19 Sep 2026 19:37:36 +0300 Subject: [PATCH] Add the narrow_nbsp option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements spec 1.3.0's nbsp.md 3.1a (polytypo/polytypo@3a2355d, canonical issue #31), porting polytypo-js@964db2d. `narrow_nbsp: "nbsp"` makes the engine emit U+00A0 everywhere it would emit U+202F — the narrow no-break space many common faces do not carry, where a browser falls back mid-line and a PDF renderer with no fallback drops the glyph. It rewrites the rule's target rather than post-processing the output, and an example shows why that distinction is the whole feature: `gsub` over the default output produces the same text and is not a fixed point — feed it back through .transform and N2 converts it straight back. The option is resolved once at the call boundary into a code point (Engine.resolve_narrow_target) and reaches the rules as RuleContext#narrow_target, so no rule sees the option's string. CODE_INVALID_OPTION is the eighth error code, general to every option added from 1.3.0 on. Validation order is mode -> narrow_nbsp -> rules -> locale -> dialect, and the check runs whether or not `nbsp` is enabled — it belongs to the call, so `rules: { "nbsp" => false }` does not suppress it. All four precedence pairs are examples. The conformance spec now passes the case's own options on the idempotency re-run, which spec 1.3.0 made contract: a case with narrowNbsp is a fixed point under that option and not under the defaults. This spec already spread **opts there, so the change is the new key plus the reason. Verified beyond the suite: all 1245 runnable canonical fixture cases at spec 1.3.0, across all 18 locales and all three modes, pass and are idempotent against this build. 1155 examples pass on Ruby 3.3.8. Co-Authored-By: Claude Opus 5 --- lib/polytypo.rb | 48 ++++++++------- lib/polytypo/engine/pipeline.rb | 22 ++++++- lib/polytypo/engine/rules/nbsp.rb | 19 ++++-- lib/polytypo/errors.rb | 3 + spec/conformance_spec.rb | 5 +- spec/narrow_nbsp_spec.rb | 97 +++++++++++++++++++++++++++++++ 6 files changed, 166 insertions(+), 28 deletions(-) create mode 100644 spec/narrow_nbsp_spec.rb diff --git a/lib/polytypo.rb b/lib/polytypo.rb index 4d4379a..2517e8d 100644 --- a/lib/polytypo.rb +++ b/lib/polytypo.rb @@ -28,22 +28,23 @@ module Polytypo # # Only `mode: "text"`/`"html"` ever touch this file's own requires; `mode: "markdown"` lazily # requires "commonmarker" from within Modes::Markdown, never at load time of this file. - def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil) + def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil, narrow_nbsp: nil) resolved_mode = resolve_mode(mode) + narrow_target = Engine.resolve_narrow_target(narrow_nbsp) case resolved_mode when "text" if dialect raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') end - transform_text(input, locale, rules) + transform_text(input, locale, rules, narrow_target) when "html" if dialect raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') end - transform_html(input, locale, rules) + transform_html(input, locale, rules, narrow_target) else # "markdown" - transform_markdown(input, locale, dialect, rules) + transform_markdown(input, locale, dialect, rules, narrow_target) end end @@ -59,22 +60,23 @@ def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil) # for the text (analyze.md sections 4 and 5). # # Pure and thread-safe on the same terms as .transform. - def self.analyze(input, locale:, mode: "text", dialect: nil, rules: nil) + def self.analyze(input, locale:, mode: "text", dialect: nil, rules: nil, narrow_nbsp: nil) resolved_mode = resolve_mode(mode) + narrow_target = Engine.resolve_narrow_target(narrow_nbsp) case resolved_mode when "text" if dialect raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') end - analyze_text(input, locale, rules) + analyze_text(input, locale, rules, narrow_target) when "html" if dialect raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') end - analyze_html(input, locale, rules) + analyze_html(input, locale, rules, narrow_target) else # "markdown" - analyze_markdown(input, locale, dialect, rules) + analyze_markdown(input, locale, dialect, rules, narrow_target) end end @@ -90,61 +92,67 @@ def self.resolve_mode(mode) end private_class_method :resolve_mode - def self.transform_text(input, locale, rules) + def self.transform_text(input, locale, rules, narrow_target) _resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules) cp = Engine::Codepoints.to_codepoints(input) - ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale) + ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale, + narrow_target: narrow_target) result = Engine::Pipeline.run_rules(cp, plan, locale_data, ctx) Engine::Codepoints.from_codepoints(result) end private_class_method :transform_text - def self.transform_html(input, locale, rules) + def self.transform_html(input, locale, rules, narrow_target) resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) spans = Modes::Html.html_spans(input) - ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale) + ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale, + narrow_target: narrow_target) cp = Engine::Codepoints.to_codepoints(input) Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx) end private_class_method :transform_html - def self.transform_markdown(input, locale, dialect, rules) + def self.transform_markdown(input, locale, dialect, rules, narrow_target) # Validation order is public, tested behaviour, identical across every runtime: rules (an # unknown rule id), then locale (an unknown locale), then dialect/parsing. resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) require_relative "polytypo/modes/markdown" Modes::Markdown.resolve_dialect(dialect) spans = Modes::Markdown.markdown_spans(input) - ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale) + ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale, + narrow_target: narrow_target) cp = Engine::Codepoints.to_codepoints(input) Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx) end private_class_method :transform_markdown - def self.analyze_text(input, locale, rules) + def self.analyze_text(input, locale, rules, narrow_target) _resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules) cp = Engine::Codepoints.to_codepoints(input) - ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale) + ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale, + narrow_target: narrow_target) Engine::Pipeline.run_rules_recording(cp, plan, locale_data, ctx, (0...cp.length).to_a, cp.length) end private_class_method :analyze_text - def self.analyze_html(input, locale, rules) + def self.analyze_html(input, locale, rules, narrow_target) resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) spans = Modes::Html.html_spans(input) - ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale) + ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale, + narrow_target: narrow_target) Modes::Runner.analyze_over_spans(Engine::Codepoints.to_codepoints(input), spans, plan, locale_data, ctx) end private_class_method :analyze_html - def self.analyze_markdown(input, locale, dialect, rules) + def self.analyze_markdown(input, locale, dialect, rules, narrow_target) # Validation order is public, tested behaviour and is shared with .transform: rules, then # locale, then dialect/parsing (analyze.md section 4, A1). resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) require_relative "polytypo/modes/markdown" Modes::Markdown.resolve_dialect(dialect) spans = Modes::Markdown.markdown_spans(input) - ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale) + ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale, + narrow_target: narrow_target) Modes::Runner.analyze_over_spans(Engine::Codepoints.to_codepoints(input), spans, plan, locale_data, ctx) end private_class_method :analyze_markdown diff --git a/lib/polytypo/engine/pipeline.rb b/lib/polytypo/engine/pipeline.rb index c85192d..d62023e 100644 --- a/lib/polytypo/engine/pipeline.rb +++ b/lib/polytypo/engine/pipeline.rb @@ -9,7 +9,27 @@ module Polytypo module Engine # Per-call context every rule reads, alongside the code-point array and the locale data. - RuleContext = Struct.new(:mode, :dialect, :locale, keyword_init: true) + # :narrow_target is nbsp.md 3.1a's NARROW-TARGET, already resolved to a code point: U+202F + # by default, U+00A0 when the caller passed narrow_nbsp: "nbsp". A rule reads a code point + # and never the option, so the string never reaches the pipeline. + RuleContext = Struct.new(:mode, :dialect, :locale, :narrow_target, keyword_init: true) + + # nbsp.md 3.1a: resolve the narrow_nbsp option to the code point the rule writes. Checked + # immediately after mode and before rules (ARCHITECTURE.md 7) -- the two checks that read + # nothing but the call itself come first -- and it runs whether or not `nbsp` is enabled, so + # a misspelled value still raises rather than being silently ignored. + NARROW_NO_BREAK_SPACE = 0x202F + NO_BREAK_SPACE = 0x00A0 + + def self.resolve_narrow_target(value) + return NARROW_NO_BREAK_SPACE if value.nil? || value == "narrow" + return NO_BREAK_SPACE if value == "nbsp" + + raise Polytypo::Error.new( + Polytypo::CODE_INVALID_OPTION, + "Unknown narrow_nbsp #{value.inspect}. Expected \"narrow\" or \"nbsp\".", + ) + end # Resolves the locale, builds the rule plan, and runs each enabled rule in # spec/rules/order.json order over a code-point array, applying its edits before the next diff --git a/lib/polytypo/engine/rules/nbsp.rb b/lib/polytypo/engine/rules/nbsp.rb index 43d163b..932df1d 100644 --- a/lib/polytypo/engine/rules/nbsp.rb +++ b/lib/polytypo/engine/rules/nbsp.rb @@ -122,6 +122,8 @@ def self.sentence_dash?(cp) :before_punctuation, :narrow_before_punctuation, :short_words, :abbreviations, :units, :before_number, :before_word, :symbols, :initial_binding, :opens, :closes, :quote_pairs, + # nbsp.md 3.1a NARROW-TARGET: what N2 writes, and what N8 writes for a narrow-nbsp pair. + :narrow_target, keyword_init: true, ) @@ -145,7 +147,7 @@ def self.prepare_list(entries) # Resolves the locale's nbsp and quotes fields to code points once. nbsp.md 2 lists the # fields; 2.1 explains why the mechanism (U+00A0 vs U+202F, convert-only vs insert) # lives here and not in the locale file. - def self.prepare(locale_data) + def self.prepare(locale_data, narrow_target = NNBSP) data = locale_data["nbsp"] before_punctuation = data["beforePunctuation"].map { |e| single_code_point(e, "beforePunctuation") } narrow_before_punctuation = @@ -191,7 +193,7 @@ def self.prepare(locale_data) # (7.5). next if open_cp == close_cp - target = pair["innerSpace"] == "nbsp" ? NBSP : NNBSP + target = pair["innerSpace"] == "nbsp" ? NBSP : narrow_target quote_pairs << QuoteTarget.new(open_cp, close_cp, target) end @@ -208,6 +210,7 @@ def self.prepare(locale_data) opens: opens, closes: closes, quote_pairs: quote_pairs, + narrow_target: narrow_target, ) end @@ -610,12 +613,16 @@ def self.forward_binding_sub_rule(cp, prep, claims, patterns, wants_digit) # yielding the index to a lower-priority sub-rule. Sub-rules wanting *different* code # points at a shared index are therefore made disjoint by construction elsewhere # (N1/N2's quote-glyph guard, nbsp.md 3.10.1) rather than relying on ordering alone. - def self.scan(cp, locale_data, _ctx) - prep = prepare(locale_data) + def self.scan(cp, locale_data, ctx) + prep = prepare(locale_data, ctx.narrow_target) claims = Array.new(cp.length + 1) - punctuation_sub_rule(cp, prep, claims, prep.before_punctuation, NBSP, NNBSP) # N1 - punctuation_sub_rule(cp, prep, claims, prep.narrow_before_punctuation, NNBSP, NBSP) # N2 + punctuation_sub_rule(cp, prep, claims, prep.before_punctuation, NBSP, NNBSP) # N1 + # N2's target is NARROW-TARGET (nbsp.md 3.1a); `other` is the NOBREAK member that is + # not the target, which is what the sub-rule converts. With the substitution on, N2 and + # N1 want the same character -- never different ones. + other = prep.narrow_target == NBSP ? NNBSP : NBSP + punctuation_sub_rule(cp, prep, claims, prep.narrow_before_punctuation, prep.narrow_target, other) # N2 short_words_sub_rule(cp, prep, claims) # N3 abbreviations_sub_rule(cp, prep, claims) # N4 units_sub_rule(cp, prep, claims) # N5 diff --git a/lib/polytypo/errors.rb b/lib/polytypo/errors.rb index 67e9fbe..dceafd5 100644 --- a/lib/polytypo/errors.rb +++ b/lib/polytypo/errors.rb @@ -10,6 +10,9 @@ module Polytypo CODE_MALFORMED_LOCALE_DATA = "POLYTYPO_MALFORMED_LOCALE_DATA" CODE_RULE_CONTRACT = "POLYTYPO_RULE_CONTRACT" CODE_MALFORMED_INPUT = "POLYTYPO_MALFORMED_INPUT" + # Spec 1.3.0. Deliberately general: every option added from 1.3.0 on shares this code, while + # `mode` and `dialect` keep their own because callers branch on them. + CODE_INVALID_OPTION = "POLYTYPO_INVALID_OPTION" # The only error type Transform ever raises. One class is enough: the stable machine-readable # +code+ is the contract, not the exception type or the message text. diff --git a/spec/conformance_spec.rb b/spec/conformance_spec.rb index 9e4841e..13d72c7 100644 --- a/spec/conformance_spec.rb +++ b/spec/conformance_spec.rb @@ -27,6 +27,7 @@ opts = { locale: locale, mode: c["mode"] } opts[:dialect] = c["dialect"] if c["dialect"] opts[:rules] = c["rules"] if c["rules"] + opts[:narrow_nbsp] = c["narrowNbsp"] if c["narrowNbsp"] if c["throws"] expect { Polytypo.transform(c["in"], **opts) } @@ -39,7 +40,9 @@ expect(escape_non_ascii(got)).to eq(escape_non_ascii(expected)), "out = #{escape_non_ascii(got)}, want #{escape_non_ascii(expected)}" - # Free coverage, and the most common port bug (ARCHITECTURE.md 6.1). + # Free coverage, and the most common port bug (ARCHITECTURE.md 6.1). The re-run carries + # the case's OWN options: a case with narrowNbsp is a fixed point under that option and + # not under the defaults, so spreading **opts here is contract, not convenience. twice = Polytypo.transform(expected, **opts) expect(escape_non_ascii(twice)).to eq(escape_non_ascii(expected)), "not idempotent: transform(out) = #{escape_non_ascii(twice)}" diff --git a/spec/narrow_nbsp_spec.rb b/spec/narrow_nbsp_spec.rb new file mode 100644 index 0000000..17e844e --- /dev/null +++ b/spec/narrow_nbsp_spec.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require "spec_helper" + +# spec/rules/nbsp.md 3.1a -- `narrow_nbsp` moves NARROW-TARGET, it does not post-process. +RSpec.describe "Polytypo narrow_nbsp" do + nbsp = " " + nnbsp = " " + + def fr(text, **kwargs) + Polytypo.transform(text, locale: "fr", **kwargs) + end + + describe "what it changes" do + it "writes U+00A0 where N2 would write U+202F" do + text = "Un délai ? Vraiment ! Et puis ; voilà." + expect(fr(text)).to eq("Un délai#{nnbsp}? Vraiment#{nnbsp}! Et puis#{nnbsp}; voilà.") + expect(fr(text, narrow_nbsp: "nbsp")) + .to eq("Un délai#{nbsp}? Vraiment#{nbsp}! Et puis#{nbsp}; voilà.") + end + + it "normalises an authored narrow space at a claimed index" do + expect(fr("Oui#{nnbsp}?", narrow_nbsp: "nbsp")).to eq("Oui#{nbsp}?") + expect(fr("Oui#{nnbsp}?")).to eq("Oui#{nnbsp}?") + end + + it "leaves an authored narrow space alone elsewhere" do + expect(fr("mot#{nnbsp}mot", narrow_nbsp: "nbsp")).to eq("mot#{nnbsp}mot") + end + end + + describe "what it does not change" do + it "claims the same indices under the same guards" do + expect(fr("12:30 et http://x ; oui", narrow_nbsp: "nbsp")).to eq("12:30 et http://x#{nbsp}; oui") + end + + it "leaves a sub-rule whose target was already U+00A0" do + # N1 (the colon) and N8 (fr's primary pair, innerSpace "nbsp") do not move. + expect(fr("Il a dit : « oui » ; puis ?", narrow_nbsp: "nbsp")) + .to eq("Il a dit#{nbsp}: «#{nbsp}oui#{nbsp}»#{nbsp}; puis#{nbsp}?") + end + + %w[en-US de-DE ru].each do |locale| + it "is a no-op in #{locale}, which never emits U+202F" do + text = "She said “hi” — really..." + expect(Polytypo.transform(text, locale: locale, narrow_nbsp: "nbsp")) + .to eq(Polytypo.transform(text, locale: locale)) + end + end + end + + describe "idempotency" do + it "is a fixed point under the option" do + ["Un délai ? Vraiment !", "Oui#{nnbsp}?", "Il a dit : « oui » ;"].each do |text| + once = fr(text, narrow_nbsp: "nbsp") + expect(fr(once, narrow_nbsp: "nbsp")).to eq(once) + end + end + + it "shows why: post-processing the default output is not one" do + # A caller's gsub is stable only as long as it always runs. Feed it back through the + # default pipeline and N2 converts it straight back. + post_processed = fr("Un délai ?").gsub(nnbsp, nbsp) + expect(post_processed).to eq("Un délai#{nbsp}?") + expect(fr(post_processed)).to eq("Un délai#{nnbsp}?") + end + end + + describe "validation" do + def code_of(**kwargs) + Polytypo.transform("x", **kwargs) + "NO THROW" + rescue Polytypo::Error => e + e.code + end + + it "raises POLYTYPO_INVALID_OPTION for an unknown value" do + expect(code_of(locale: "fr", narrow_nbsp: "wide")).to eq(Polytypo::CODE_INVALID_OPTION) + end + + it "is checked after mode and before rules and locale" do + expect(code_of(locale: "fr", mode: "yaml", narrow_nbsp: "wide")).to eq(Polytypo::CODE_INVALID_MODE) + expect(code_of(locale: "fr", narrow_nbsp: "wide", rules: { "nope" => true })) + .to eq(Polytypo::CODE_INVALID_OPTION) + expect(code_of(locale: "xx", narrow_nbsp: "wide")).to eq(Polytypo::CODE_INVALID_OPTION) + end + + it "raises even when nbsp is disabled, because the check belongs to the call" do + expect(code_of(locale: "fr", narrow_nbsp: "wide", rules: { "nbsp" => false })) + .to eq(Polytypo::CODE_INVALID_OPTION) + end + + it "accepts the explicit default" do + expect(fr("Un délai ?", narrow_nbsp: "narrow")).to eq("Un délai#{nnbsp}?") + end + end +end