From fff7f947c94d400105951e3734c759b1bb397a34 Mon Sep 17 00:00:00 2001 From: Iurii Rogulia Date: Fri, 18 Sep 2026 19:38:07 +0300 Subject: [PATCH] analyze: prove document offsets past a non-ASCII character MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every html/markdown offset assertion added with analyze() was ASCII-only, so none of them discriminated a code-point document offset from a byte one through the span adapters — the place a byte-oriented runtime actually gets it wrong. This runtime is not byte-oriented, but the five ports share one proof set. `café` in the first span puts the UTF-8 byte offset one ahead of the code-point offset, and the test now asserts both numbers rather than only the one it wants. Test-only. 18 analyze tests pass. Co-Authored-By: Claude Opus 5 --- tests/engine/test_analyze.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/engine/test_analyze.py b/tests/engine/test_analyze.py index 25eed48..554f50d 100644 --- a/tests/engine/test_analyze.py +++ b/tests/engine/test_analyze.py @@ -134,13 +134,16 @@ def test_html_mode_reports_document_offsets(self) -> None: assert first.after == "…" def test_html_mode_reports_document_offsets_in_a_later_span(self) -> None: - """Three spans, and the change is in the third: the two markers before it are the only - code points in the joined array with no origin, so a doubled or dropped one shifts this - offset and nothing in a one- or two-span document would notice.""" - text = "

one

two

Wait... three

" + """Three spans, a non-ASCII character before the change, and the change in the third + span: the two markers are the only code points in the joined array with no origin, so a + doubled or dropped one shifts this offset and nothing in a one- or two-span document + would notice, and `café` puts the UTF-8 byte offset one ahead of the code-point one, so + a byte offset leaking out of a span adapter cannot pass either.""" + text = "

café

two

Wait... three

" first = polytypo.analyze(text, locale="en-US", mode="html")[0] assert first.rule_id == "ellipsis" assert first.start == text.index("...") + assert len(text[: text.index("...")].encode("utf-8")) == first.start + 1 def test_markdown_mode_reports_document_offsets(self) -> None: text = "# Title\n\nWait... here\n"