diff --git a/README.md b/README.md index 3815722..81d3f34 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,22 @@ pipeline only. A single `Polytypo.transform` module method (not separate `require` already gets you the same dependency isolation JS/Python's subpath split exists for, without a separate namespace per mode. +`Polytypo.analyze` runs the same pipeline and reports what it would do instead of doing it — one +record per edit, each with the rule that made it and code-point offsets into the input you passed +(into the **document**, in `html` and `markdown` mode, not into a span): + +```ruby +Polytypo.analyze(%q{Wait... "really"?}, locale: "en-US") +# => [#, +# #, ...] +``` + +It is a report, not a patch. The list is empty exactly when `Polytypo.transform` would return the +input unchanged, and every `rule_id` is a rule that was enabled for that call — but two rules may +touch the same original range (French `spaces` deletes the space before `:` and `nbsp` puts a +no-break one back), so replaying the list is not guaranteed to reproduce the output. Call +`Polytypo.transform` for the text. Full contract: `spec/rules/analyze.md`. + ### Errors Every error `Polytypo.transform` raises is a `Polytypo::Error` carrying one of seven stable diff --git a/lib/polytypo.rb b/lib/polytypo.rb index 6271f21..4d4379a 100644 --- a/lib/polytypo.rb +++ b/lib/polytypo.rb @@ -3,6 +3,7 @@ require_relative "polytypo/version" require_relative "polytypo/errors" require_relative "polytypo/engine/codepoints" +require_relative "polytypo/engine/origin" require_relative "polytypo/engine/pipeline" require_relative "polytypo/engine/rules" # side effect: registers all 9 rules require_relative "polytypo/modes/spans" @@ -46,6 +47,37 @@ def self.transform(input, locale:, mode: "text", dialect: nil, rules: nil) end end + # Runs the same pipeline as .transform and reports what it would do instead of doing it + # (spec/rules/analyze.md), returning an Array of Polytypo::Engine::Change. Offsets are + # code-point offsets into +input+ in every mode -- into the document, in "html" and "markdown" + # mode, not into a span. + # + # What it guarantees: the list is empty exactly when .transform would return the input + # unchanged, every +rule_id+ was enabled for the call, and every offset is inside the input. + # What it does not: the list is a report, not a patch -- two rules may touch the same original + # range, so replaying it is not guaranteed to reproduce .transform's output. Call .transform + # for the text (analyze.md sections 4 and 5). + # + # Pure and thread-safe on the same terms as .transform. + def self.analyze(input, locale:, mode: "text", dialect: nil, rules: nil) + resolved_mode = resolve_mode(mode) + + case resolved_mode + when "text" + if dialect + raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') + end + analyze_text(input, locale, rules) + when "html" + if dialect + raise Error.new(CODE_INVALID_DIALECT, '"dialect" is only valid when mode is "markdown"') + end + analyze_html(input, locale, rules) + else # "markdown" + analyze_markdown(input, locale, dialect, rules) + end + end + def self.resolve_mode(mode) case mode when nil, "text" @@ -88,4 +120,32 @@ def self.transform_markdown(input, locale, dialect, rules) Modes::Runner.run_over_spans(cp, spans, plan, locale_data, ctx) end private_class_method :transform_markdown + + def self.analyze_text(input, locale, rules) + _resolved, locale_data, plan = Engine::Pipeline.prepare(locale, rules) + cp = Engine::Codepoints.to_codepoints(input) + ctx = Engine::RuleContext.new(mode: "text", dialect: nil, locale: locale) + Engine::Pipeline.run_rules_recording(cp, plan, locale_data, ctx, (0...cp.length).to_a, cp.length) + end + private_class_method :analyze_text + + def self.analyze_html(input, locale, rules) + resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) + spans = Modes::Html.html_spans(input) + ctx = Engine::RuleContext.new(mode: "html", dialect: nil, locale: resolved_locale) + Modes::Runner.analyze_over_spans(Engine::Codepoints.to_codepoints(input), spans, plan, locale_data, ctx) + end + private_class_method :analyze_html + + def self.analyze_markdown(input, locale, dialect, rules) + # Validation order is public, tested behaviour and is shared with .transform: rules, then + # locale, then dialect/parsing (analyze.md section 4, A1). + resolved_locale, locale_data, plan = Engine::Pipeline.prepare(locale, rules) + require_relative "polytypo/modes/markdown" + Modes::Markdown.resolve_dialect(dialect) + spans = Modes::Markdown.markdown_spans(input) + ctx = Engine::RuleContext.new(mode: "markdown", dialect: dialect, locale: resolved_locale) + Modes::Runner.analyze_over_spans(Engine::Codepoints.to_codepoints(input), spans, plan, locale_data, ctx) + end + private_class_method :analyze_markdown end diff --git a/lib/polytypo/engine/origin.rb b/lib/polytypo/engine/origin.rb new file mode 100644 index 0000000..444d823 --- /dev/null +++ b/lib/polytypo/engine/origin.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +require_relative "codepoints" + +module Polytypo + # One entry of Polytypo.analyze's result, in input coordinates (analyze.md section 2): + # +rule_id+ is a rule id from spec/rules/order.json, +start+/+end+ are code-point offsets into + # the input (inclusive/exclusive), and +before+/+after+ are the text on either side of that one + # edit. +start+ == +end+ is a pure insertion, and +before+ is then empty. + # + # Public API, so it is named in this namespace rather than in Engine, where it is built. + Change = Data.define(:rule_id, :start, :end, :before, :after) + + module Engine + # analyze.md section 2: every reported offset is a code-point offset into the input the + # caller passed, in every mode. The rules, however, run over an array that is not the input + # -- in "text" mode edits from earlier rules have already shifted it, and in "html"/"markdown" + # mode it is the marker-joined concatenation of the processable spans (modes.md 3.5). This + # module carries the one structure that bridges the two: an origin map, parallel to the + # current code-point array, holding the input offset each code point came from, or NO_ORIGIN + # for one the pipeline itself produced. Mirrors polytypo-js's src/engine/origin.ts. + module Origin + NO_ORIGIN = -1 + + # The input offset an edit boundary at +index+ addresses. Synthetic code points have no + # origin of their own, so the scan runs forward to the first that has one -- an insertion + # between two earlier insertions still lands where the next real character is. Falling off + # the end means the boundary is at the end of the input. + def self.origin_at(origin, index, input_length) + (index...origin.length).each do |i| + return origin[i] unless origin[i] == NO_ORIGIN + end + input_length + end + + # The origin map for the array Edits.apply_edits is about to produce. A replacement of + # equal length keeps its origins position by position, which is what makes a conversion + # (U+0020 -> U+00A0) still point at the character it converted; anything longer is + # synthetic beyond the positions it covers. + def self.apply_edits_to_origin(origin, edits) + return origin.dup if edits.empty? + + out = [] + cursor = 0 + edits.each do |edit| + out.concat(origin[cursor...edit.start]) + edit.replacement.each_index do |k| + source = edit.start + k + out << (source < edit.end ? origin[source] : NO_ORIGIN) + end + cursor = edit.end + end + out.concat(origin[cursor..]) + out + end + + # One rule's edits, in the coordinates that rule saw, rendered as Changes in input + # coordinates. +before+ is the text this rule replaced and +after+ what it replaced it with + # (analyze.md section 2), so on a text two rules have both touched, +before+ is what the + # second rule saw rather than what the caller typed -- section 5 says so and shows the + # French case where it matters. + def self.record_changes(cp, edits, origin, input_length, rule_id) + edits.map do |edit| + Polytypo::Change.new( + rule_id: rule_id, + start: origin_at(origin, edit.start, input_length), + end: origin_at(origin, edit.end, input_length), + before: Codepoints.from_codepoints(cp[edit.start...edit.end]), + after: Codepoints.from_codepoints(edit.replacement) + ) + end + end + end + end +end diff --git a/lib/polytypo/engine/pipeline.rb b/lib/polytypo/engine/pipeline.rb index 8bc528d..c85192d 100644 --- a/lib/polytypo/engine/pipeline.rb +++ b/lib/polytypo/engine/pipeline.rb @@ -2,6 +2,7 @@ require_relative "edits" require_relative "locale" +require_relative "origin" require_relative "registry" require_relative "../errors" @@ -56,6 +57,28 @@ def self.run_rules(cp, plan, locale_data, ctx) end current end + + # run_rules, keeping the edits instead of discarding them (analyze.md section 1: same + # pipeline, same order, reporting rather than applying). The origin map travels alongside + # the array so every change comes back in input coordinates, and +filter_edits+ is the hook + # the span-runner needs for modes.md 3.4's boundary filters, passed as a block -- text mode + # passes none and gets the identity. + def self.run_rules_recording(cp, plan, locale_data, ctx, origin, input_length) + current = cp + current_origin = origin + changes = [] + plan.each do |rule_id| + fn = Registry.rule(rule_id) + produced = fn.call(current, locale_data, ctx) + edits = block_given? ? yield(current, produced) : produced + next if edits.empty? + + changes.concat(Origin.record_changes(current, edits, current_origin, input_length, rule_id)) + current_origin = Origin.apply_edits_to_origin(current_origin, edits) + current = Edits.apply_edits(current, edits, rule_id) + end + changes + end end end end diff --git a/lib/polytypo/modes/runner.rb b/lib/polytypo/modes/runner.rb index e3fdac9..2f6f3a0 100644 --- a/lib/polytypo/modes/runner.rb +++ b/lib/polytypo/modes/runner.rb @@ -2,6 +2,7 @@ require_relative "spans" require_relative "../engine/edits" +require_relative "../engine/pipeline" require_relative "../engine/registry" module Polytypo @@ -52,6 +53,22 @@ def self.run_over_spans(source_cp, spans, plan, locale_data, ctx) out.concat(source_cp[cursor..]) out.pack("U*") end + + # run_over_spans, reporting instead of applying (analyze.md section 1). The span table + # supplies the origin map, so every change comes back in DOCUMENT coordinates -- analyze.md + # section 6 names a runtime that reports span-local offsets here as the mistake that passes + # every text-mode test. + def self.analyze_over_spans(source_cp, spans, plan, locale_data, ctx) + normalized = Spans.normalize_spans(spans) + return [] if normalized.empty? + + Engine::Pipeline.run_rules_recording( + Spans.concatenate_spans(source_cp, normalized), + plan, locale_data, ctx, + Spans.origin_of_spans(normalized), + source_cp.length + ) { |current, edits| Spans.filter_boundary_edits(current, edits, Spans.span_ranges_of(current)) } + end end end end diff --git a/lib/polytypo/modes/spans.rb b/lib/polytypo/modes/spans.rb index cae44de..595965f 100644 --- a/lib/polytypo/modes/spans.rb +++ b/lib/polytypo/modes/spans.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require_relative "../engine/origin" require_relative "../engine/sentinels" require_relative "../errors" @@ -68,6 +69,19 @@ def self.concatenate_spans(source_cp, spans) cp end + # The origin map for concatenate_spans (analyze.md section 2): for every code point of the + # joined array, the code-point offset of the character it came from IN THE DOCUMENT, and + # NO_ORIGIN for the markers, which came from nowhere. A Span's bounds are already + # code-point offsets, so no coordinate conversion belongs here. + def self.origin_of_spans(spans) + origin = [] + spans.each_with_index do |span, i| + origin << Engine::Origin::NO_ORIGIN unless i.zero? + origin.concat((span.start...span.end).to_a) + end + origin + end + # The span extents of the array as it stands. Recomputed after every rule, because applying # edits shifts every index after the first one -- the markers themselves always survive, # since no edit may contain one. diff --git a/spec/analyze_spec.rb b/spec/analyze_spec.rb new file mode 100644 index 0000000..eea896e --- /dev/null +++ b/spec/analyze_spec.rb @@ -0,0 +1,170 @@ +# frozen_string_literal: true + +require "spec_helper" +require "json" + +# spec/rules/analyze.md -- the contract is A1...A5; the decomposition is observation. +# +# Mirrors the JS, Python and Go ports' own analyze suites case for case, including the two that +# are cheap to get wrong (analyze.md section 6): A3 over the whole vendored corpus, and document +# offsets under the mode adapters. +RSpec.describe "Polytypo.analyze" do + rule_order = %w[spaces ellipsis ranges dashes hyphen quotes apostrophe symbols nbsp].freeze + + def rule_ids(changes) + changes.map(&:rule_id) + end + + describe "A1 -- accepts and rejects exactly what .transform does" do + it "raises POLYTYPO_UNKNOWN_LOCALE for an unknown locale" do + expect { Polytypo.analyze("x", locale: "xx") } + .to raise_error(Polytypo::Error) { |e| expect(e.code).to eq(Polytypo::CODE_UNKNOWN_LOCALE) } + end + + it "raises POLYTYPO_UNKNOWN_RULE before it raises about the locale" do + expect { Polytypo.analyze("x", locale: "xx", rules: { "nope" => true }) } + .to raise_error(Polytypo::Error) { |e| expect(e.code).to eq(Polytypo::CODE_UNKNOWN_RULE) } + end + + it "raises POLYTYPO_INVALID_MODE for an unknown mode" do + expect { Polytypo.analyze("x", locale: "en-US", mode: "yaml") } + .to raise_error(Polytypo::Error) { |e| expect(e.code).to eq(Polytypo::CODE_INVALID_MODE) } + end + + it "requires a dialect in markdown mode, as .transform does" do + expect { Polytypo.analyze("x", locale: "en-US", mode: "markdown") } + .to raise_error(Polytypo::Error) { |e| expect(e.code).to eq(Polytypo::CODE_INVALID_DIALECT) } + end + + it "rejects a dialect outside markdown mode" do + expect { Polytypo.analyze("x", locale: "en-US", dialect: "commonmark") } + .to raise_error(Polytypo::Error) { |e| expect(e.code).to eq(Polytypo::CODE_INVALID_DIALECT) } + end + end + + describe "A2 -- pure" do + it "returns the same list for the same arguments" do + input = 'She said "hi" -- really...' + expect(Polytypo.analyze(input, locale: "en-US")).to eq(Polytypo.analyze(input, locale: "en-US")) + end + + it "does not change what .transform returns" do + input = 'She said "hi" -- really...' + before = Polytypo.transform(input, locale: "en-US") + Polytypo.analyze(input, locale: "en-US") + expect(Polytypo.transform(input, locale: "en-US")).to eq(before) + end + end + + describe "A3 -- empty exactly when .transform changes nothing" do + it "is empty for text that needs nothing" do + expect(Polytypo.analyze("Nothing to do here.", locale: "en-US")).to eq([]) + end + + it "is non-empty for text that needs something" do + expect(Polytypo.analyze("Wait...", locale: "en-US")).not_to be_empty + end + + it "agrees with .transform on every canonical fixture" do + fixtures_dir = File.expand_path("../lib/polytypo/data/fixtures", __dir__) + files = Dir.glob(File.join(fixtures_dir, "*.json")).reject { |f| f.end_with?("locale-resolution.json") } + raise "no fixture files found" if files.empty? + + offenders = [] + files.sort.each do |path| + data = JSON.parse(File.read(path)) + data["cases"].each do |c| + next if c["throws"] || c["dialect"] == "mdx" + + opts = { locale: data["locale"], mode: c["mode"] } + opts[:dialect] = c["dialect"] if c["dialect"] + opts[:rules] = c["rules"] if c["rules"] + + changed = Polytypo.transform(c["in"], **opts) != c["in"] + reported = !Polytypo.analyze(c["in"], **opts).empty? + offenders << "#{data["locale"]}/#{c["id"]}" if changed != reported + end + end + expect(offenders).to eq([]) + end + end + + describe "A4 -- only rules that were enabled for the call" do + it "never reports a rule the caller disabled" do + ids = rule_ids(Polytypo.analyze('She said "hi"...', locale: "en-US", rules: { "quotes" => false })) + expect(ids).not_to include("quotes") + expect(ids).to include("ellipsis") + end + + it "never reports ranges unless it was turned on" do + input = "chapters 3-5" + expect(rule_ids(Polytypo.analyze(input, locale: "en-US"))).not_to include("ranges") + expect(rule_ids(Polytypo.analyze(input, locale: "en-US", rules: { "ranges" => true }))).to include("ranges") + end + end + + describe "A5 -- offsets are code points inside the input" do + it "stays within bounds on a string with astral characters" do + input = 'A 😀 says "hi" and waits...' + length = input.codepoints.length + Polytypo.analyze(input, locale: "en-US").each do |change| + expect(change.start).to be >= 0 + expect(change.end).to be <= length + expect(change.start).to be <= change.end + end + end + + it "reports code-point offsets" do + # The emoji is one code point here and two UTF-16 units in the JS runtime; both report 6, + # which is what makes the offsets portable rather than string-representation-specific. + first = Polytypo.analyze('😀 and "this"', locale: "en-US").first + expect(first.rule_id).to eq("quotes") + expect(first.start).to eq(6) + end + + # analyze.md section 6: the mistake that passes every text-mode test. + it "reports document offsets in html mode, not span-local ones" do + input = '

Wait...

' + first = Polytypo.analyze(input, locale: "en-US", mode: "html").first + expect(first.rule_id).to eq("ellipsis") + expect(first.start).to eq(input.index("...")) + expect(first.before).to eq("...") + expect(first.after).to eq("…") + end + + it "reports document offsets in a later span too" do + # Three spans, and the change is in the third: the two markers before it are the only code + # points in the joined array with no origin, so a doubled or dropped one shifts this offset + # and nothing in a one- or two-span document would notice. + input = "

one

two

Wait... three

" + first = Polytypo.analyze(input, locale: "en-US", mode: "html").first + expect(first.rule_id).to eq("ellipsis") + expect(first.start).to eq(input.index("...")) + end + + it "reports document offsets in markdown mode too" do + input = "# Title\n\nWait... here\n" + first = Polytypo.analyze(input, locale: "en-US", mode: "markdown", dialect: "commonmark").first + expect(first.start).to eq(input.index("...")) + end + end + + describe "section 3 -- pipeline order, and section 5 -- overlap" do + it "reports rules in spec order" do + ids = rule_ids(Polytypo.analyze('She said "hi" -- wait...', locale: "en-US")) + expect(ids).to eq(ids.sort_by { |id| rule_order.index(id) }) + end + + it "reports both rules when they touch the same original range" do + # The French case analyze.md section 5 is written around: two rules, one original index. + # `spaces` removes the space at 3-4 and `nbsp` inserts at 4-4, in front of the colon the + # caller wrote at 4 -- the second change's position is the colon's, not the deleted space's. + changes = Polytypo.analyze("Oui : non", locale: "fr") + expect(rule_ids(changes)).to eq(%w[spaces nbsp]) + expect([changes[0].before, changes[0].after]).to eq([" ", ""]) + expect([changes[0].start, changes[0].end]).to eq([3, 4]) + expect(changes[1].after).to eq(" ") + expect([changes[1].start, changes[1].end]).to eq([4, 4]) + end + end +end