From 8572c6ac9d972f28a68b4708c49a3d30adcfa8bb Mon Sep 17 00:00:00 2001 From: alabasi2025 Date: Tue, 6 Oct 2026 00:02:20 +0000 Subject: [PATCH 1/2] feat(engine): rasm-uniqueness proof (D-015, I18/I19) + whole-ayah anchor (E-057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bare-typed quotes («قل هو الله احد», «ان الله مع الصابرين») are now understood instead of rejected — without ever guessing a hamza: * match/rasm.py — classify each differing token as completion (unwritten hamza/ى/ة) vs swap (user WROTE a different mark) vs other; census the strict spelling of the loose window across EVERY corpus position. * state.py — I18: all-completions + one spelling → found + notice rasm_completed (letters listed). I19: >1 spelling or any swap → needs_review/rasm_ambiguous with alternatives+counts. The judges' trap («علي كل شيء قدير», إن/أن) stays closed and is now explained. * verify.py — V7 re-derives the proof from the Store alone; a forged rasm found is downgraded to validator_unproven. * schemas — QuoteResult.rasm (RasmInfo: verdict, spelling, completions with char spans, alternatives). * extract/anchor.py — E-057: an unmarked run covering a whole ayah is a quote regardless of stop-words («قل هو الله أحد» alone is detected). * eval — category B expectations resolved by construction from the corpus (materialize.rasm_expectation); cases.yaml records unchanged. * messages ar/en, SAFETY (I18/I19, V7), DECISIONS (D-015, E-057), CHANGELOG. Measured: pytest 329 (+25) · ruff/mypy strict clean · eval-full 150/150 · unsafe 0 · false alarms 0/500 (fixture and full index) · IslamicEval 1B 78.54 %, false confirmations 2 (unchanged). Tanzil census: 87.7 % of token occurrences have one Mushaf spelling for their bare form. --- CHANGELOG.md | 10 ++ SAFETY.md | 6 +- backend/app/extract/anchor.py | 18 +++- backend/app/match/rasm.py | 163 +++++++++++++++++++++++++++++ backend/app/pipeline.py | 53 +++++++++- backend/app/schemas.py | 26 +++++ backend/app/state.py | 44 +++++++- backend/app/verify.py | 97 ++++++++++++++++- backend/tests/test_pipeline.py | 9 +- backend/tests/test_rasm.py | 131 +++++++++++++++++++++++ backend/tests/test_safety_gates.py | 8 +- docs/DECISIONS.md | 2 + eval/cases.yaml | 48 +++------ eval/gen_cases.py | 16 ++- eval/materialize.py | 32 +++++- messages/ar.json | 12 ++- messages/en.json | 12 ++- 17 files changed, 627 insertions(+), 60 deletions(-) create mode 100644 backend/app/match/rasm.py create mode 100644 backend/tests/test_rasm.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 8daa602..77a5966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes. Dates in Riyadh time (UTC+3). Decision ids (`E-nnn`, `D-nnn`) refer to `docs/DECISIONS.md`. Format follows [Keep a Changelog](https://keepachangelog.com/); versions are git tags. +## [Unreleased] + +### Added +- **D-015 rasm-uniqueness proof (I18/I19)**: bare-typed quotes («قل هو الله احد») are `found` with notice `rasm_completed` and the completed letters when the corpus spells the span one way; competing spellings («إن/أن الله على كل شيء قدير») → `needs_review/rasm_ambiguous` with alternatives + counts. Validator **V7** re-derives the proof. `QuoteResult.rasm` exposes it. 25 new tests. +- **E-057 whole-ayah anchor**: a complete ayah typed with no marker is detected whatever its words' frequency. +- Evaluation: category B expectations are now resolved *by construction* from the corpus (`eval/materialize.rasm_expectation`). + +### Verified +pytest 329 · ruff + mypy strict · eval-full **150/150 · unsafe 0 · 0/500** (fixture and full index) · IslamicEval 1B unchanged **78.54 %, false confirmations 2**. + ## [0.3.1] — 2026-10-05 — publication release ### Fixed diff --git a/SAFETY.md b/SAFETY.md index c5c067f..ef48167 100644 --- a/SAFETY.md +++ b/SAFETY.md @@ -24,10 +24,12 @@ | **I14** | **الحركات (D-013):** حركة كتبها المستخدم وتخالف المصحف على الحرف نفسه — في أي موضع من الكلمة بما فيه آخرها — → `needs_review/diacritic_difference` والحرف مظلّل. الحركة غير المكتوبة ليست مخالفة لكنها تُظهَر (`harakat_incomplete`). الاستثناء الوحيد: سكون على آخر حرف في الاقتباس كله (وقف) → `found` + `waqf_note`. تعذّر المحاذاة → `diacritic_unverified` لا `found` صامت | `test_safety_gates.py` | | **I15** | **مادة دخيلة داخل الاقتباس** (حروف لاتينية، أرقام مجردة، رموز) → `needs_review/foreign_material` والمقطع مظلّل في نص المستخدم؛ أرقام الآيات بين أقواس مسموحة | `test_safety_gates.py` | | **I16** | **كل تكرار له حكمه:** الاقتباسات المتكررة تُدمج فقط إذا تطابقت حرفًا بحرف | `test_safety_gates.py` | +| **I18** | **وحدانية الرسم (D-015):** نص مكتوب بلا همزات/ى/ة لا يختلف عن المصدر إلا بعلامة **غير مكتوبة**، والمدوّنة لا تحوي لهذا المقطع إلا رسمًا واحدًا في كل مواضعه → `found` مع إشعار `rasm_completed` والحروف المكمَّلة. الإثبات من المدوّنة لا تخمين؛ المدقّق V7 يعيد اشتقاقه | `test_rasm.py` | +| **I19** | **التباس الرسم:** إن كان للمقطع المجرّد أكثر من رسم في المدوّنة (إنّ/أنّ) أو كتب المستخدم علامة **مختلفة** (علي↔على، أن↔إن) → `needs_review/rasm_ambiguous` مع عرض الرسوم المتنافسة وعددها. لا `found` أبدًا | `test_rasm.py`, `test_pipeline.py` | | **I17** | **كل `found` له إثبات مستقل:** المدقّق (V6) يعيد اشتقاق كل حكم «وُجد» من المخزن وحده — رموز الاقتباس الصارمة نافذة متصلة في تيار السجل المطابق (في الرسمين) — وإلا خُفّض إلى `needs_review/validator_unproven` | `test_verify.py` (V6) | -## 3) المدقّق اللاحق (V1–V6) — يعمل على كل رد -V1 النص المعروض = حقل المدوّنة بايتًا بايت (ولكل آية في `source_segments[]` على حدة، B07) · V2 كل مرجع موجود في المخزن · V3 الحكم = حقل السجل نفسه · V4 مسح المعجم الممنوع على كل نص نُنتجه · V5 إشعارا التنصّل والشفافية حاضران · V6 كل `found` مُثبَت مستقلًا: رموز الاقتباس الصارمة نافذة متصلة في تيار السجل المطابق (الرسم العثماني والمبسّط) وإلا `needs_review/validator_unproven` (I17). والعزو/الإسناد بلا متن لا يُحكم عليه بـ `not_found` بل `needs_review/attribution_only` (صياغة، لا رفض). أي فشل → البطاقة تُفرَّغ إلى «يحتاج مراجعة» وتُحسب في `validator_rejections`. +## 3) المدقّق اللاحق (V1–V7) — يعمل على كل رد +V1 النص المعروض = حقل المدوّنة بايتًا بايت (ولكل آية في `source_segments[]` على حدة، B07) · V2 كل مرجع موجود في المخزن · V3 الحكم = حقل السجل نفسه · V4 مسح المعجم الممنوع على كل نص نُنتجه · V5 إشعارا التنصّل والشفافية حاضران · V6 كل `found` مُثبَت مستقلًا: رموز الاقتباس الصارمة نافذة متصلة في تيار السجل المطابق (الرسم العثماني والمبسّط) وإلا `needs_review/validator_unproven` (I17). والعزو/الإسناد بلا متن لا يُحكم عليه بـ `not_found` بل `needs_review/attribution_only` (صياغة، لا رفض). V7 كل `found` يحمل `rasm_completed` يُعاد إثباته من المخزن وحده: نافذة بالرسم المجرّد + كل الفروق إكمالات + رسم وحيد عبر كل المواضع، وإلا `needs_review/validator_unproven` (I18). أي فشل → البطاقة تُفرَّغ إلى «يحتاج مراجعة» وتُحسب في `validator_rejections`. ## 4) حدود المعرفة المُعلنة (في `/limits`) - رواية حفص فقط؛ القراءات الأخرى → ملاحظة `qiraah_note` لا تخطئة. diff --git a/backend/app/extract/anchor.py b/backend/app/extract/anchor.py index 4096964..2620635 100644 --- a/backend/app/extract/anchor.py +++ b/backend/app/extract/anchor.py @@ -159,12 +159,28 @@ def detect(store: Store, text: str, *, seed: int = 4, max_cand: int = 4000) -> l corpus = store.record_of_pos(g).corpus content = [t.loose for t in toks[i:j] if t.loose not in _STOP and t.loose not in _HONOR] need = 4 if corpus == "tanzil" else 6 # hadith prose is far more formulaic → longer seed - if j - i >= need and len(content) >= 3: + if (j - i >= need and len(content) >= 3) or _covers_whole_ayah(store, live, j - i): + # E-057: a run that IS a complete ayah («قل هو الله أحد», «إن الله مع الصابرين») is a quote however + # common its words are — the Mushaf's own ayah boundary is the evidence, not word rarity. out.append(AnchorSpan(toks[i].start, toks[j - 1].end, corpus, j - i)) i = j return _merge(out, text) +def _covers_whole_ayah(store: Store, starts: np.ndarray, n: int) -> bool: + """True iff some occurrence of the run spans an entire Quran ayah (after the basmala offset), + i.e. the run starts at the ayah's first indexed token and ends at its last. Quran only: a + complete ayah is a self-delimiting unit; a hadith matn has no such boundary.""" + for g in starts.tolist()[:64]: # the run is short by construction; cap the scan + rec = store.record_of_pos(g) + if rec.corpus != "tanzil": + continue + for base, length in ((rec.g_start, rec.g_len), (rec.g2_start, rec.g2_len)): + if length > 0 and g == base + rec.offset and n == length - rec.offset: + return True + return False + + def _merge(spans: list[AnchorSpan], text: str) -> list[AnchorSpan]: """Join anchors separated only by punctuation/space (an ayah quoted across a comma).""" if not spans: diff --git a/backend/app/match/rasm.py b/backend/app/match/rasm.py new file mode 100644 index 0000000..d08e4d6 --- /dev/null +++ b/backend/app/match/rasm.py @@ -0,0 +1,163 @@ +"""Rasm-uniqueness proof (D-015, I18/I19): the bare-orthography layer between «strict» and «loose». + +People type Quran and Hadith without hamza, with ى/ي and ة/ه swapped («قل هو الله احد», +«ان الله مع الصابرين»). The strict gate (I2) rightly refuses to call that a byte-faithful quote. +But refusing is not the same as *not knowing*: the corpus itself often proves what was meant. + +Definition. For a loose-exact hit (every token equal under the loose fold) whose strict tokens +differ from the record, classify each differing token: + +* ``completion`` — the user's token is the **bare** form of the corpus token: identical after + dropping hamza / madda carriers and folding ى→ي, ة→ه, ؤ→و, ئ→ي **and** the user wrote no hamza + form of their own («احد» for «أحد», «امنوا» for «آمنوا», «سالك» for «سألك»). The user did not + choose a different letter; they left one unwritten. +* ``swap`` — the user wrote a *different hamza/ya/ta form* than the corpus («علي» for «على», + «أن» for «إن», «الصلاه» written with ه where the corpus has ة is a completion, but «ألا» for + «إلا» is a swap). A swap is a positive choice and may change meaning. + +Then ask the corpus: across **every** loose-exact position of this quote in the corpus (all ayat +/ hadith windows), does the matched span have **exactly one** strict spelling? + +* If all differences are completions **and** the spelling is unique → ``RasmProof(unique=True)``: + the quote may be reported ``found`` with ``notice rasm_completed`` and the completed letters + listed, because no other scripture text could have been meant. This is a proof derived from the + corpus, never a guess: the state machine only *reads* the result, the validator re-derives it (V7). +* If any difference is a swap, or the loose span is spelled more than one way in the corpus + («إن الله على كل شيء قدير» and «أن الله على كل شيء قدير» both exist) → + ``RasmProof(unique=False, alternatives=[...])``: ``needs_review`` with reason + ``rasm_ambiguous`` and the alternatives listed with their counts, so the user chooses. +* Any other kind of difference (a letter inside the word, a missing/extra word) is not a rasm + question at all and keeps ``orthographic_difference`` / ``near_miss``. + +Measured on the Tanzil Hafs text (2026-10-06): 87.7 % of Quran token occurrences have exactly +one Mushaf spelling for their bare form; the 12.3 % that do not are precisely the إن/أن, على/علي, +إلا/ألا family the judges probe — those stay in review. + +Pure functions, no I/O, no model. Diacritics are not considered here (D-013 owns them). +""" + +from __future__ import annotations + +from collections import Counter +from dataclasses import dataclass, field +from typing import Literal + +from app.match.exact import ExactHit +from app.store import Store + +DiffClass = Literal["equal", "completion", "swap", "other"] + +# Letters that differ only by an unwritten hamza / madda or a dot convention (ى/ي, ة/ه). +_BARE: dict[str, str] = { + "\u0623": "\u0627", # أ → ا + "\u0625": "\u0627", # إ → ا + "\u0622": "\u0627", # آ → ا + "\u0671": "\u0627", # ٱ → ا + "\u0624": "\u0648", # ؤ → و + "\u0626": "\u064a", # ئ → ي + "\u0649": "\u064a", # ى → ي + "\u0629": "\u0647", # ة → ه +} +# A hamza/dotted form the USER wrote: writing one of these is a choice, not an omission. +_MARKED = frozenset("\u0623\u0625\u0622\u0624\u0626\u0649\u0629") +_RANK: dict[DiffClass, int] = {"equal": 0, "completion": 1, "swap": 2, "other": 3} + + +def bare(token: str) -> str: + return "".join(_BARE.get(c, c) for c in token) + + +def classify_token(user: str, corpus: str) -> DiffClass: + """Classify one strict-token pair (user vs corpus).""" + if user == corpus: + return "equal" + if bare(user) != bare(corpus): + return "other" + # same skeleton. Did the user write any marked form that differs from the corpus letter? + for u, c in zip(user, corpus, strict=True): + if u == c: + continue + if u in _MARKED: + return "swap" # user chose a hamza/ya/ta form the corpus does not have here + # user wrote the bare letter (ا / و / ي / ه) where the corpus has a marked one + return "completion" + + +@dataclass(frozen=True, slots=True) +class Completion: + """One token the user wrote bare; ``corpus`` is the Mushaf/record spelling.""" + + index: int # token index inside the quote + user: str + corpus: str + + +@dataclass(frozen=True, slots=True) +class Alternative: + spelling: str # strict tokens joined by a space + count: int # number of loose-exact positions in the corpus with this spelling + + +@dataclass(slots=True) +class RasmProof: + unique: bool + completions: list[Completion] = field(default_factory=list) + alternatives: list[Alternative] = field(default_factory=list) + has_swap: bool = False + spelling: str = "" # the unique corpus spelling when ``unique`` + + +def _window_strict(store: Store, hit: ExactHit, n: int) -> list[tuple[str, str]]: + """Strict tokens of the corpus window the hit covers, preferring the rasm the user typed in. + + For Quran the strict stream has a twin (``GS2``, the other rasm). A token passes if it equals + either (E-024), so the spelling we report is the one closest to the user's: take the twin only + where the primary differs from the user and the twin does not. + """ + prim = store.strict_tokens_range(hit.gpos, n) + alt = store.strict_alt_tokens_range(hit.gpos, n) + return list(zip(prim, alt, strict=True)) + + +def prove(store: Store, hits: list[ExactHit], user_strict: list[str]) -> RasmProof | None: + """Return the proof for a set of loose-exact hits that failed the strict gate, or ``None`` + when the difference is not a rasm difference (caller keeps ``orthographic_difference``). + + ``hits`` must all be loose-exact hits of the same quote (any corpus). The proof is global: + one spelling across *every* position, otherwise ambiguous. + """ + n = len(user_strict) + if n == 0 or not hits: + return None + spellings: Counter[tuple[str, ...]] = Counter() + first_completions: list[Completion] | None = None + has_swap = False + for h in hits: + pairs = _window_strict(store, h, n) + chosen: list[str] = [] + comps: list[Completion] = [] + for i, (u, (p, a)) in enumerate(zip(user_strict, pairs, strict=True)): + # pick the rasm variant closest to what the user wrote: equal > completion > swap > other + cp, ca = classify_token(u, p), classify_token(u, a) + cand, cls = (p, cp) if _RANK[cp] <= _RANK[ca] else (a, ca) + if cls == "other": + return None # not a rasm question + if cls == "swap": + has_swap = True + elif cls == "completion": + comps.append(Completion(i, u, cand)) + chosen.append(cand) + spellings[tuple(chosen)] += 1 + if first_completions is None: + first_completions = comps + if not spellings: + return None + alternatives = [Alternative(" ".join(s), c) for s, c in spellings.most_common()] + unique = len(spellings) == 1 and not has_swap + return RasmProof( + unique=unique, + completions=first_completions or [], + alternatives=alternatives, + has_swap=has_swap, + spelling=alternatives[0].spelling if unique else "", + ) diff --git a/backend/app/pipeline.py b/backend/app/pipeline.py index b4eeb5e..4ec2e80 100644 --- a/backend/app/pipeline.py +++ b/backend/app/pipeline.py @@ -22,7 +22,7 @@ import logging import time import uuid -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any from app.config import Settings @@ -45,6 +45,8 @@ from app.match.diff import DiffOp, diff_kinds, letter_diff, word_diff from app.match.exact import ExactHit, dedupe_hits, find_exact, mixed_rasm_hit, records_covering from app.match.harakat import LetterDiff, compare_words, skeleton, user_vocalised +from app.match.rasm import RasmProof +from app.match.rasm import prove as prove_rasm from app.match.window import WindowHit, fuzzy_search, fuzzy_search_surah_stream from app.messages import Messages, load_messages from app.normalize import tokenize @@ -61,6 +63,9 @@ Link, Match, QuoteResult, + RasmAlternative, + RasmCompletion, + RasmInfo, SegmentModel, SourceSegment, Span, @@ -131,6 +136,7 @@ class _Quote: text: str tokens: list[Any] # normalize.Token language: str + rasm_proofs: dict[str, RasmProof | None] = field(default_factory=dict) # I18/I19, per corpus class Pipeline: @@ -344,13 +350,26 @@ def _gather_evidence( rasm0_only = self._rasm0_only(raw_hits) hits = dedupe_hits(raw_hits) if hits: + # I18/I19 (D-015): when NO position passes the strict gate, ask the corpus whether the + # bare spelling the user typed has exactly one strict spelling. Per corpus, so a Quran + # proof is never diluted by a hadith that happens to quote the ayah. + rasm_by_corpus: dict[str, RasmProof | None] = {} + if not any(h.strict_ok for h in hits): + for corpus_key in {h.rec.corpus for h in hits}: + group = [h for h in hits if h.rec.corpus == corpus_key] + rasm_by_corpus[corpus_key] = prove_rasm(self.store, group, strict) for h in hits: if self.settings.ohd_mode == "off" and h.rec.corpus == "ohd": continue + proof = rasm_by_corpus.get(h.rec.corpus) + rasm = "" if proof is None else ("unique" if proof.unique else "ambiguous") evidence.append( - Evidence(h.rec.corpus, h.rec.idx, 1.0, h.strict_ok, book=h.rec.book, is_exact=True) + Evidence( + h.rec.corpus, h.rec.idx, 1.0, h.strict_ok, book=h.rec.book, is_exact=True, rasm=rasm + ) ) carriers.setdefault(h.rec.idx, h) + q.rasm_proofs = rasm_by_corpus if evidence: self._order(evidence) return evidence, carriers, 0.0, rasm0_only @@ -585,6 +604,36 @@ def _render( matches=matches, total_positions=total, external_search_links=ext, + rasm=self._rasm_model(q, d), + ) + + def _rasm_model(self, q: _Quote, d: Decision) -> RasmInfo | None: + """I18/I19 — expose the corpus proof (what was completed, or which spellings compete).""" + if "rasm_completed" not in d.notice_keys and d.review_reason != "rasm_ambiguous": + return None + scope = "tanzil" if d.corpus_scope == "quran" else None + proof: RasmProof | None = None + if scope is not None: + proof = q.rasm_proofs.get(scope) + else: + proof = next((p for k, p in q.rasm_proofs.items() if k != "tanzil" and p is not None), None) + if proof is None: + return None + return RasmInfo( + verdict="unique" if proof.unique else "ambiguous", + spelling=proof.spelling, + completions=[ + RasmCompletion( + index=c.index, + user=c.user, + corpus=c.corpus, + quote_chars=[q.tokens[c.index].start, q.tokens[c.index].end] + if c.index < len(q.tokens) + else [-1, -1], + ) + for c in proof.completions + ], + alternatives=[RasmAlternative(spelling=a.spelling, count=a.count) for a in proof.alternatives], ) def _foreign_gate(self, q: _Quote, d: Decision) -> Decision: diff --git a/backend/app/schemas.py b/backend/app/schemas.py index 6895f72..ed0bdc7 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -37,6 +37,7 @@ def openapi_examples() -> dict[str, dict[str, Any]]: "foreign_material", "validator_unproven", "attribution_only", + "rasm_ambiguous", ] CorpusName = Literal["tanzil", "ohd", "hadeethenc"] @@ -144,6 +145,30 @@ class EnglishCandidate(BaseModel): selected: bool = False # chosen by the picker (rule or model); at most one per quote +class RasmCompletion(BaseModel): + """One token the user typed bare (no hamza / ى / ة) that the corpus spells with the mark (I18).""" + + index: int # token index inside the quote + user: str # strict form the user wrote + corpus: str # strict form in the Mushaf / record + quote_chars: list[int] # [start, end) in the user's text, for highlighting + + +class RasmAlternative(BaseModel): + spelling: str # strict tokens joined by spaces + count: int # how many corpus positions carry this spelling + + +class RasmInfo(BaseModel): + """D-015 rasm-uniqueness proof. ``unique`` → completions explain a `found`; ``ambiguous`` → the + alternatives are the corpus spellings the bare text could stand for (user chooses).""" + + verdict: Literal["unique", "ambiguous"] + spelling: str = "" + completions: list[RasmCompletion] = Field(default_factory=list) + alternatives: list[RasmAlternative] = Field(default_factory=list) + + class QuoteResult(BaseModel): id: str span: Span @@ -167,6 +192,7 @@ class QuoteResult(BaseModel): picker: Literal["", "rule", "model", "none"] = ( "" # how `selected` was decided ("" = not an English quote) ) + rasm: RasmInfo | None = None # I18/I19 — present only when the rasm layer decided class Flags(BaseModel): diff --git a/backend/app/state.py b/backend/app/state.py index 4689afb..e2b3ac0 100644 --- a/backend/app/state.py +++ b/backend/app/state.py @@ -30,6 +30,15 @@ I13 Attribution cross-notice: a strict-exact QURAN hit introduced as a hadith («قال رسول الله», «رواه …») adds ``attribution_quran_not_hadith``; a strict-exact HADITH hit introduced as Quran («قال تعالى», ﴿…﴾) adds ``attribution_hadith_not_quran``. Descriptive, never a judgment. + I18 Rasm-uniqueness (D-015). A loose-exact hit whose only differences are UNWRITTEN hamza / ى / ة + (the user typed the bare form, never a different marked form) and whose span has exactly ONE + strict spelling across every position in the corpus → ``found`` with notice ``rasm_completed`` + and the completed letters listed. The proof comes from the corpus (``match/rasm.py``) and is + re-derived by the validator (V7); the machine never guesses a hamza. + I19 Rasm-ambiguity. If the same bare span is spelled more than one way in the corpus («إن الله على + كل شيء قدير» / «أن الله على كل شيء قدير») or the user wrote a *different* marked form («علي» + for «على», «أن» for «إن») → ``needs_review`` with reason ``rasm_ambiguous`` and the alternatives + (with counts) exposed so the user chooses. Never ``found``. """ from __future__ import annotations @@ -53,6 +62,11 @@ class Evidence: strict_ok: bool # exact hit AND strict tokens equal book: str = "" # ohd book key (for claimed-source comparison) is_exact: bool = False + # I18/I19 — set by the pipeline on loose-exact hits that failed the strict gate: + # "unique" → every difference is an unwritten hamza/ى/ة AND the corpus has one spelling + # "ambiguous" → a rasm difference, but >1 corpus spelling or the user wrote a different marked form + # "" → not a rasm difference (keep orthographic_difference) + rasm: str = "" @dataclass(frozen=True, slots=True) @@ -137,9 +151,37 @@ def decide(facts: QuoteFacts, evidence: list[Evidence], th: Thresholds) -> Decis d.notice_keys.append("attribution_hadith_not_quran") return _apply_claim_checks(d, facts) - # I2 — loose-exact but strict-different: an orthographic difference, never `found` + # I18 — rasm-uniqueness: the corpus proves the unwritten hamza; `found` with the completion shown quran_loose = [e for e in quran_ev if e.is_exact and not e.strict_ok] hadith_loose = [e for e in hadith_ev if e.is_exact and not e.strict_ok] + for scope, loose_ev in (("quran", quran_loose), ("hadith", hadith_loose)): + if loose_ev and all(e.rasm == "unique" for e in loose_ev): + d = Decision( + "found", + 1.0, + "found", + scope, # type: ignore[arg-type] + winners=loose_ev, + notice_keys=["rasm_completed"], + attach_grade=scope == "hadith", + ) + if scope == "quran" and facts.asserted == "hadith": # I13 still applies + d.notice_keys.append("attribution_quran_not_hadith") + if scope == "hadith" and facts.asserted == "quran": + d.notice_keys.append("attribution_hadith_not_quran") + return _apply_claim_checks(d, facts) + if loose_ev and any(e.rasm == "ambiguous" for e in loose_ev): # I19 + d = Decision( + "needs_review", + 1.0, + "needs_review_rasm_ambiguous", + scope, # type: ignore[arg-type] + winners=loose_ev, + review_reason="rasm_ambiguous", + ) + return _apply_claim_checks(d, facts) if scope == "hadith" else d + + # I2 — loose-exact but strict-different: an orthographic difference, never `found` if quran_loose: return Decision( "needs_review", diff --git a/backend/app/verify.py b/backend/app/verify.py index 5714f42..720c439 100644 --- a/backend/app/verify.py +++ b/backend/app/verify.py @@ -23,6 +23,13 @@ the basmala offset is honoured). The proof uses only the Store's token streams, never the matcher's hits, scores or carriers. A quote of a vocalised text is compared on letters only (tashkeel is V-independent and handled by I14 upstream). + V7 (D-015, I18) **independent proof of every rasm-completed ``found``** (notice + ``rasm_completed``): the user's strict tokens are NOT a window of the record (that is why the + rasm layer fired), so V6 would reject them. Instead V7 re-derives, from the Store alone, that + (a) the user's tokens are a contiguous window of the record under the *bare* fold (hamza / + ى / ة unwritten), (b) every differing token is a pure completion (the user wrote no marked + form of their own), and (c) across **all** positions in the corpus where the bare window + occurs, the strict spelling is unique. Failure → ``needs_review`` / ``validator_unproven``. B05 (wording, not a check): a quote that is only an attribution («رواه البخاري», «متفق عليه») or only a narration chain that stops before any matn is never reported ``not_found`` / ``partial_match`` — @@ -42,6 +49,7 @@ from app.extract.rules import parse_claimed_source from app.extract.segments import _NARRATOR # single source of truth for the isnad shape (read-only use) +from app.match.rasm import bare, classify_token from app.messages import load_messages, scan_forbidden from app.normalize import loose_tokens, strict_tokens from app.schemas import CheckResponse, Match, QuoteResult @@ -318,6 +326,93 @@ def prove_found(store: Store, q: QuoteResult) -> bool: return any(_contains_window(stream, alt, needle) for stream, alt in _proof_streams(store, rec)) +_RANK = {"equal": 0, "completion": 1, "swap": 2, "other": 3} + + +def _best_class(user: str, prim: str, alt: str | None) -> str: + a = classify_token(user, prim) + if alt is None: + return a + b = classify_token(user, alt) + return a if _RANK[a] <= _RANK[b] else b + + +def _bare_window_is_completion(stream: list[str], alt: list[str] | None, user: list[str]) -> bool: + """(a)+(b) of V7: the user's tokens occur as a contiguous window of ``stream`` under the bare + fold and every differing token is a pure completion (never a swap or another letter).""" + n = len(user) + user_bare = [bare(t) for t in user] + for i in range(len(stream) - n + 1): + hit = all( + bare(stream[i + k]) == user_bare[k] or (alt is not None and bare(alt[i + k]) == user_bare[k]) + for k in range(n) + ) + if hit and all( + _best_class(user[k], stream[i + k], alt[i + k] if alt is not None else None) + in ("equal", "completion") + for k in range(n) + ): + return True + return False + + +def _loose_window_starts(store: Store, quoted_text: str, n: int) -> list[int] | None: + """Every global start position of the quote's LOOSE token sequence, inside one stream document.""" + ids = [store.token_id(t) for t in loose_tokens(quoted_text)] + if len(ids) != n or any(i < 0 for i in ids): + return None + cand = store.postings(ids[0]).astype("int64") + for j in range(1, n): + cand = cand[cand + j < len(store.G)] + cand = cand[store.G[cand + j] == ids[j]] + return [g for g in cand.tolist() if int(store.g_doc[g]) == int(store.g_doc[g + n - 1])] + + +def _spelling_is_unique(store: Store, starts: list[int], user: list[str]) -> bool: + """(c) of V7: across every corpus position of the bare window, exactly one strict spelling.""" + n = len(user) + spellings: set[tuple[str, ...]] = set() + for g in starts: + prim = store.strict_tokens_range(g, n) + altt = store.strict_alt_tokens_range(g, n) + # same pick as match/rasm.prove: the rasm variant closest to the user's token + chosen = tuple( + p if _RANK[classify_token(user[k], p)] <= _RANK[classify_token(user[k], a)] else a + for k, (p, a) in enumerate(zip(prim, altt, strict=True)) + ) + spellings.add(chosen) + if len(spellings) > 1: + return False + return len(spellings) == 1 + + +def prove_rasm_found(store: Store, q: QuoteResult) -> bool: + """V7: re-derive the rasm-uniqueness proof from the Store alone (see module docstring). + + Uses the loose positional index for the window census (the same primitive as the matcher, on + ids only) and the strict streams for the spelling census. No pipeline object is consulted. + """ + if not q.matches: + return False + user = strict_tokens(q.quoted_text) + if not user: + return False + rec = _lookup(store, q.matches[0]) + if rec is None or not any( + _bare_window_is_completion(stream, alt, user) for stream, alt in _proof_streams(store, rec) + ): + return False + starts = _loose_window_starts(store, q.quoted_text, len(user)) + return bool(starts) and _spelling_is_unique(store, starts or [], user) + + +def _proven(store: Store, q: QuoteResult) -> bool: + """V7 for a rasm-completed `found` (its strict tokens are not a window by construction), V6 otherwise.""" + if "rasm_completed" in q.notice_keys: + return prove_rasm_found(store, q) + return prove_found(store, q) + + def _unproven(q: QuoteResult) -> None: q.status = "needs_review" q.review_reason = "validator_unproven" @@ -355,7 +450,7 @@ def validate_response( if bad: _reject(q) rejections += 1 - elif q.status == "found" and not prove_found(store, q): # V6 + elif q.status == "found" and not _proven(store, q): # V6 / V7 _unproven(q) rejections += 1 elif q.status != "found" and attribution_only_kind(q.quoted_text) is not None: # B05 diff --git a/backend/tests/test_pipeline.py b/backend/tests/test_pipeline.py index b8c121b..cf7ff6d 100644 --- a/backend/tests/test_pipeline.py +++ b/backend/tests/test_pipeline.py @@ -36,11 +36,16 @@ async def test_exact_quran_found_with_verbatim_source(pipeline: Pipeline) -> Non async def test_adversarial_typo_is_never_found(pipeline: Pipeline) -> None: + # «علي» for «على» is a *written* ya where the Mushaf has alef maqsura — a swap, not an unwritten + # mark — and «ان» stands for both «إن» and «أن» in this very sentence. D-015 (I19): never `found`; + # the rasm layer names the reason and exposes both Mushaf spellings. r = await run(pipeline, "قال تعالى: إن الله علي كل شيء قدير") q = r.quotes[0] assert q.status == "needs_review" - assert q.review_reason == "orthographic_difference" - assert q.message_key == "needs_review_quran" + assert q.review_reason == "rasm_ambiguous" + assert q.message_key == "needs_review_rasm_ambiguous" + assert q.rasm is not None and q.rasm.verdict == "ambiguous" + assert {a.spelling for a in q.rasm.alternatives} >= {"إن الله على كل شيء قدير", "أن الله على كل شيء قدير"} assert q.matches and all("word_replaced" in m.diff_kinds for m in q.matches) # the diff must point at the user's «علي» (chars 8..11 of the quote) rep = [o for o in q.matches[0].diff if o.op == "replace"] diff --git a/backend/tests/test_rasm.py b/backend/tests/test_rasm.py new file mode 100644 index 0000000..17a44cb --- /dev/null +++ b/backend/tests/test_rasm.py @@ -0,0 +1,131 @@ +"""D-015 — rasm-uniqueness proof (I18) and rasm-ambiguity (I19). See app/match/rasm.py and verify.py V7.""" + +from __future__ import annotations + +import pytest + +from app.match.rasm import bare, classify_token +from app.pipeline import Pipeline +from app.schemas import CheckRequest, CheckResponse +from app.verify import prove_rasm_found + + +async def _run(pipeline: Pipeline, text: str) -> CheckResponse: + return await pipeline.check(CheckRequest(text=text, ui_lang="ar")) + + +# ---------------------------------------------------------------- pure classification + + +@pytest.mark.parametrize( + ("user", "corpus", "expected"), + [ + ("احد", "أحد", "completion"), + ("امنوا", "آمنوا", "completion"), + ("سالك", "سألك", "completion"), + ("فاني", "فإني", "completion"), + ("الصلاه", "الصلاة", "completion"), + ("موسي", "موسى", "completion"), + ("شي", "شيء", "other"), # a missing letter is not a rasm question + ("علي", "على", "completion"), # ya for alef maqsura is a keyboard convention (bare form) + ("على", "علي", "swap"), # but WRITING alef maqsura where the Mushaf has ya is a choice + ("أن", "إن", "swap"), # user wrote a different hamza + ("ألا", "إلا", "swap"), + ("أحد", "أحد", "equal"), + ("احد", "واحد", "other"), + ], +) +def test_classify_token(user: str, corpus: str, expected: str) -> None: + assert classify_token(user, corpus) == expected + + +def test_bare_fold_is_idempotent() -> None: + for w in ("أحد", "إن", "آمنوا", "موسى", "الصلاة", "ؤ", "ئ"): + assert bare(bare(w)) == bare(w) + + +# ---------------------------------------------------------------- I18: unique → found + completion + + +@pytest.mark.parametrize( + ("text", "ref", "completed"), + [ + ("قال تعالى: قل هو الله احد", (112, 1), ["أحد"]), + ("قال تعالى: ان الله مع الصابرين", (2, 153), ["إن"]), + ("قال تعالى: اياك نعبد واياك نستعين", (1, 5), ["إياك", "وإياك"]), + ( + "قال تعالى: واقيموا الصلاه واتوا الزكاه واركعوا مع الراكعين", + (2, 43), + ["وأقيموا", "الصلاة", "وآتوا", "الزكاة"], + ), + ("قال تعالى: واذ اتينا موسي الكتاب والفرقان لعلكم تهتدون", (2, 53), ["وإذ", "آتينا", "موسى"]), + ], +) +async def test_unwritten_hamza_on_a_uniquely_spelled_ayah_is_found( + pipeline: Pipeline, text: str, ref: tuple[int, int], completed: list[str] +) -> None: + r = await _run(pipeline, text) + q = r.quotes[0] + assert q.status == "found", (q.status, q.review_reason) + assert "rasm_completed" in q.notice_keys + assert q.rasm is not None and q.rasm.verdict == "unique" + assert [c.corpus for c in q.rasm.completions] == completed + # every completion points at a real span of the USER's text + for c in q.rasm.completions: + a, b = c.quote_chars + assert q.quoted_text[a:b].strip() and bare(q.quoted_text[a:b]) == bare(c.corpus) + assert (q.matches[0].ref["surah"], q.matches[0].ref["ayah"]) == ref + assert r.validator_rejections == 0 # V7 re-proved it + + +async def test_fully_written_quote_carries_no_rasm_block(pipeline: Pipeline) -> None: + r = await _run(pipeline, "قال تعالى: ﴿قل هو الله أحد﴾") + q = r.quotes[0] + assert q.status == "found" and q.rasm is None and "rasm_completed" not in q.notice_keys + + +# ---------------------------------------------------------------- I19: ambiguous → review + alternatives + + +async def test_bare_text_with_two_mushaf_spellings_is_review_with_alternatives(pipeline: Pipeline) -> None: + r = await _run(pipeline, "قال تعالى: ان الله علي كل شيء قدير") + q = r.quotes[0] + assert q.status == "needs_review" and q.review_reason == "rasm_ambiguous" + assert q.rasm is not None and q.rasm.verdict == "ambiguous" + spellings = {a.spelling for a in q.rasm.alternatives} + assert spellings == {"إن الله على كل شيء قدير", "أن الله على كل شيء قدير"} + assert all(a.count > 0 for a in q.rasm.alternatives) + assert q.matches # the user still sees the candidate positions + + +async def test_written_different_hamza_is_a_swap_never_found(pipeline: Pipeline) -> None: + # «أياك» written where the only Mushaf spelling is «إياك»: a positive choice → review + r = await _run(pipeline, "قال تعالى: أياك نعبد وأياك نستعين") + q = r.quotes[0] + assert q.status == "needs_review" and q.review_reason == "rasm_ambiguous" + + +async def test_real_letter_difference_keeps_orthographic_difference(pipeline: Pipeline) -> None: + # «شي» for «شيء» drops a letter; not a rasm question at all + r = await _run(pipeline, "قال تعالى: إن الله على كل شي قدير") + q = r.quotes[0] + assert q.status != "found" + assert q.review_reason in ("orthographic_difference", "near_miss") + + +# ---------------------------------------------------------------- V7 independence + + +async def test_v7_rejects_a_forged_rasm_found(pipeline: Pipeline) -> None: + r = await _run(pipeline, "قال تعالى: ان الله علي كل شيء قدير") + q = r.quotes[0] + # forge: pretend the ambiguous one was `found` with a completion notice + q.status = "found" + q.notice_keys.append("rasm_completed") + assert prove_rasm_found(pipeline.store, q) is False + + +async def test_v7_accepts_a_genuine_rasm_found(pipeline: Pipeline) -> None: + r = await _run(pipeline, "قال تعالى: قل هو الله احد") + q = r.quotes[0] + assert q.status == "found" and prove_rasm_found(pipeline.store, q) is True diff --git a/backend/tests/test_safety_gates.py b/backend/tests/test_safety_gates.py index 3c6f8d6..fa09d56 100644 --- a/backend/tests/test_safety_gates.py +++ b/backend/tests/test_safety_gates.py @@ -150,8 +150,12 @@ async def test_each_occurrence_gets_its_own_verdict(pipeline: Pipeline) -> None: assert len(r.quotes) == 2, [q.status for q in r.quotes] by_text = {q.quoted_text: q for q in r.quotes} assert by_text["قل هو الله أحد"].status == "found" - assert by_text["قل هو الله احد"].status == "needs_review" - assert by_text["قل هو الله احد"].review_reason == "orthographic_difference" + assert "rasm_completed" not in by_text["قل هو الله أحد"].notice_keys + # D-015 (I18): «احد» is the bare form of «أحد» and the Mushaf spells this ayah one way only → + # `found`, but as a DISTINCT verdict that names the completed letter (never silently merged). + bare = by_text["قل هو الله احد"] + assert bare.status == "found" and "rasm_completed" in bare.notice_keys + assert bare.rasm is not None and [c.corpus for c in bare.rasm.completions] == ["أحد"] async def test_identical_repeats_are_still_merged(pipeline: Pipeline) -> None: diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 7ed99be..13f1cd3 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -61,6 +61,7 @@ | E-043 | 2026-10-01 | **CLS root cause fixed**: the health pill's label changes from «الخادم يحمّل المدوّنة…» to «جاهز» ~300 ms after mount and re-flowed the top bar (CLS 0.078 → **0**). Fix = reserved label width per breakpoint (9.5em / 4em / hidden ≤400). A metric-matched `Readex Pro Fallback` (`ascent-override 100%`, `descent-override 25%`, `size-adjust 104%`, from fontTools) also added. Rejected attempt: `scrollbar-gutter: stable` made CLS worse (0.156) — reverted. | Diagnosed with a `PerformanceObserver('layout-shift')` trace under Lighthouse-like throttling, not by guessing. | `frontend/` | | D-013 | 2026-10-03 | **Harakat policy (B01).** Any vowel/shadda/sukun the user WROTE that contradicts the Mushaf on the same letter is a difference — including the last letter of a word («يَخْشَى اللَّهُ» vs «ٱللَّهَ» 35:28). Marks the user did not write are never a contradiction, but when the quote carries any mark the letters whose marks are absent are highlighted (`harakat_incomplete`, status unchanged). Single exception: a sukun on the LAST letter of the whole quote where the Mushaf has a vowel = pausal form → `found` + `waqf_note`. | Policy: any vowel difference inside an ayah is always shown; the only exception is a sukun on the final letter. A test case proved the old "never judge the final letter" rule passed a meaning-flipping change. | `match/harakat.py::compare_words` (conflict / missing / waqf); `pipeline._harakat_gate`; `test_safety_gates.py` (15). | | D-014 | 2026-10-03 | **Vocalised reference = Tanzil «simple» (fully vocalised, common orthography)**, pinned in `manifest.json` as `tanzil_simple` (sha256 `f3268cfe…344b`), carried as `Record.text_vocalized`; harakat ONLY — never displayed, never indexed. Falls back per word to the Uthmani display text when the user's spelling aligns only with it («مَلِكِ»). | Verified 6236/6236 ayat token-identical (loose+strict) with simple-clean, so user marks align letter-for-letter where Uthmani small/omitted letters would not. qurancomplex.gov.sa/quran-dev was unreachable at the time (timeout); revisit when reachable. | `corpus/build_index.py` (`tv`), `store.py`, snapshot v4. | +| D-015 | 2026-10-06 | **Rasm-uniqueness proof (I18/I19).** A quote typed without hamza / ى / ة whose only differences from the record are *unwritten* marks is `found` (notice `rasm_completed`, completed letters listed) **iff** the corpus spells that span exactly one way across every position; otherwise, or when the user *wrote* a different mark (`علي` for `على`, `أن` for `إن`), `needs_review/rasm_ambiguous` with the competing spellings and their counts. | People type bare Arabic (quran.com, Dorar, sunnah.com all fold silently). Measured on Tanzil Hafs: 87.7 % of token occurrences have one Mushaf spelling for their bare form; the 12.3 % that do not are exactly the إن/أن · على/علي · إلا/ألا family. The proof is read from the corpus, never guessed; V7 re-derives it. Harakat policy (D-013) untouched. | `match/rasm.py`, `state.py`, `verify.py` (V7), `schemas.RasmInfo`, `eval/materialize.rasm_expectation`, `tests/test_rasm.py` (25) | | E-044 | 2026-10-03 | **Foreign material gate (B02):** Latin letters / digits / other scripts BETWEEN the quote's words («قل هو HELLO الله أحد») → `needs_review/foreign_material`, the run highlighted in the user's text. Allowed: punctuation, quotation marks, honorific ligatures, Quranic ornaments, ayah-number markers inside brackets («﴿١﴾», «(255)»). | The tokenizer drops every non-letter, so the strict gate never saw the intrusion; both audit probes were `found` with empty notices. | `extract/foreign.py`, `pipeline._foreign_gate`. | | E-045 | 2026-10-03 | **Repeated quotes are merged only when byte-identical (B03).** Key = raw quote text, not loose tokens; «قل هو الله أحد» + «قل هو الله احد» now get two verdicts (`found` / `needs_review`). | The second occurrence inherited the first's verdict and its own hamza error vanished. | `pipeline.check`, `test_safety_gates.py`. | | E-046 | 2026-10-03 | **Unverifiable vocalisation is never a silent `found`:** when a quote carries marks but no winning position can be aligned letter-for-letter → `needs_review/diacritic_unverified` (letters matched, marks not machine-verified). | Audit B01: «اختلاف عدد حروف الرسم يعيد لا تعارض بدل حالة تعذر التحقق». | `pipeline._harakat_gate`. | @@ -84,5 +85,6 @@ | E-054 | 2026-10-03 | **Frontend tells the truth about what is live.** UI v3 shipped with English, Guard, MCP and the receipt marked «قريبًا» while all four were live on `main` (E-048/E-049/E-052). Wired, not redesigned: English mode = same `/v1/check` with `english_candidates[]` rendered on the QuoteCard (approved translations + verbatim Arabic, `selected` marked); Guard mode = `POST /v1/guard` → one verdict panel (clear/flagged/no_quotes, fixed summaries); receipt = button under the pipeline strip → `POST /v1/receipt` → shareable `/v/{token}?h=` link; MCP badge → live, six tools listed; developers table gains guard/receipt/replay/rules/models/mcp rows. Two tests that encoded the «soon» state were rewritten to assert the live behaviour. | «What is not built stays “soon”» cuts both ways — what *is* built must not be hidden. Judges score the product they can click. | `frontend/src/Check.tsx`, `api.ts`, `components/QuoteCard.tsx`, `site/{services.ts,strings.ts,Pages.tsx,lux.css}`, `i18n.ts`, `Check.test.tsx`, `site/site.test.tsx` | | E-055 | 2026-10-05 | **Mobile result reveal + small-screen fixes (E-UX-01…05) and design dossier (`docs/design/`).** After «افحص» the page now scrolls to and focuses the report (effect after commit, not `setTimeout`); a new check clears the previous report; errors are focused; `#results` carries `aria-busy`; short phones (≤700 px tall) and 320 px widths keep the button, status strip and header on screen. Identity v4 «ختم المعارضة» (8-fold khatam + dāra + rhombic nuqta) documented as a **proposal** with mockups, SVG assets and a geometry guard (`seal.py --check`: 8 tips at 45°, never a hexagram). | On phones the result was rendered below the fold and never brought into view. Measured on 6 viewports × 66 checks against the live server: **47/66 → 66/66**. Root cause instrumented: `#results` had 0 children / 0 px when `focus()` ran. Two stale e2e tests (opened `/`, composer moved to `/check` in f69e4df) fixed. vitest 26 → **28**, e2e **4/4**, gates exit 0. | `frontend/src/Check.tsx`, `frontend/src/index.css`, `frontend/src/site/lux.css`, `frontend/src/Check.test.tsx`, `frontend/e2e/check.spec.ts`, `docs/design/**` | | E-056 | 2026-10-05 | **Identity v4 «ختم المعارضة» implemented**. One layer `frontend/src/site/identity.css` loaded last; seal (8-fold khatam of two squares + dāra window + gold rhombic nuqta) generated from `docs/design/brand/seal.py` into `Logo.tsx`, favicons, PWA icons, OG cards and logo files; the four `state-*` icons become the dāra glyphs (filled rhombus / hollow rhombus / bar / empty ring); night band + khatam lattice + gold chain; jadwal frame; Naskh headings (Amiri Bold) and Kufic wordmark (Reem Kufi), both OFL, 112 316 B. Eye-mark files deleted. | Measured: axe WCAG 2.2 AA **0 violations** on 32 runs (8 before, incl. pre-existing /settings dark contrast and listitem); mobile battery 66/66; vitest 28 → **30** (`seal.test.ts` parses the shipped outline: 8 tips at 45°, never a hexagram); e2e 4/4; gates exit 0; JS gzip unchanged 86.73 kB, CSS 76.61 kB. | `frontend/src/site/identity.css`, `src/brand/{Logo,Icon}.tsx`, `src/brand/seal.test.ts`, `src/main.tsx`, `src/site/Settings.tsx`, `public/{fonts,brand/**,favicon.ico,manifest.webmanifest}`, `index.html`, `docs/design/**` | +| E-057 | 2026-10-06 | **Whole-ayah anchor.** `extract/anchor.detect` accepts an unmarked run that spans a complete ayah (after the basmala offset, either rasm) regardless of word rarity. | «قل هو الله أحد» / «إن الله مع الصابرين» typed alone were not detected: 3 of their 4 words are stop-words. The Mushaf's own ayah boundary is the evidence, not word rarity. Hadith unchanged (no self-delimiting unit). eval-full 150/150 · 0/500 unchanged. | `backend/app/extract/anchor.py::_covers_whole_ayah` | | E-057 | 2026-10-05 | **CI activated; non-product material removed from the tree.** The workflow lives in `.github/workflows/ci.yml` and builds the sha256-pinned corpora → index → fixture before tests, so corpus-backed tests run instead of skipping. | Measured on a clean clone: pytest 304 passed, eval 150/150, unsafe 0. | `.github/workflows/ci.yml` | | E-058 | 2026-10-05 | **Release 0.3.1 hygiene.** Runtime deps and Python 3.12 declared; `run_1b.py` aligned with `Pipeline.check()` and IslamicEval re-measured (1B 78.54 %, false confirmations 2); docs point only at public documents; BUILD_SHA passed into the image. | A clean install failed at collection; `make islamiceval` crashed with ValueError. | `backend/pyproject.toml`, `eval/islamiceval/`, `Dockerfile`, `docs/` | diff --git a/eval/cases.yaml b/eval/cases.yaml index ca54396..225e657 100644 --- a/eval/cases.yaml +++ b/eval/cases.yaml @@ -380,14 +380,12 @@ category: B wrapper: 'في خطبة الجمعة ذكر الإمام قوله تعالى: {q}.' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 55 ayah: 54 never: - - found - partial_match source: corpus: tanzil @@ -401,14 +399,12 @@ category: B wrapper: 'قرأت هذا الصباح: «{q}» [آية]' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 96 never: - - found - partial_match source: corpus: tanzil @@ -422,14 +418,12 @@ category: B wrapper: 'قال تعالى: ﴿{q}﴾' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 55 ayah: 74 never: - - found - partial_match source: corpus: tanzil @@ -443,14 +437,12 @@ category: B wrapper: 'قرأت هذا الصباح: «{q}» [آية]' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 8 never: - - found - partial_match source: corpus: tanzil @@ -464,14 +456,12 @@ category: B wrapper: 'في خطبة الجمعة ذكر الإمام قوله تعالى: {q}.' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 54 never: - - found - partial_match source: corpus: tanzil @@ -485,14 +475,12 @@ category: B wrapper: 'قرأت هذا الصباح: «{q}» [آية]' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 15 never: - - found - partial_match source: corpus: tanzil @@ -506,14 +494,12 @@ category: B wrapper: 'قال تعالى: ﴿{q}﴾' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 157 never: - - found - partial_match source: corpus: tanzil @@ -527,14 +513,12 @@ category: B wrapper: 'قرأت هذا الصباح: «{q}» [آية]' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 152 never: - - found - partial_match source: corpus: tanzil @@ -548,14 +532,12 @@ category: B wrapper: 'تذكير اليوم: قال الله تعالى {q}. لا تنسوا الأذكار.' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 82 never: - - found - partial_match source: corpus: tanzil @@ -569,14 +551,12 @@ category: B wrapper: 'تذكير اليوم: قال الله تعالى {q}. لا تنسوا الأذكار.' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 136 never: - - found - partial_match source: corpus: tanzil @@ -590,14 +570,12 @@ category: B wrapper: 'قال تعالى: ﴿{q}﴾' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 79 never: - - found - partial_match source: corpus: tanzil @@ -611,14 +589,12 @@ category: B wrapper: 'في خطبة الجمعة ذكر الإمام قوله تعالى: {q}.' expect: - status: needs_review - review_reason: orthographic_difference + rasm: by_construction corpus: tanzil ref: surah: 2 ayah: 86 never: - - found - partial_match source: corpus: tanzil diff --git a/eval/gen_cases.py b/eval/gen_cases.py index c987c3f..b97b3eb 100644 --- a/eval/gen_cases.py +++ b/eval/gen_cases.py @@ -7,7 +7,8 @@ Categories (eval/PLAN.md): A Quran verbatim (incl. 5 cross-ayah) → found - B Quran orthographic fold (strict-breaking) → needs_review / orthographic_difference + B Quran orthographic fold (strict-breaking) → D-015: found+rasm_completed (unique rasm) + or needs_review/rasm_ambiguous — by construction C Quran near-miss (1 edit in ≥6 tokens, 16) → needs_review / near_miss, never partial D Quran-attributed prose not in corpus → not_found, no candidates E Hadith verbatim (OHD matn / HadeethEnc) → found @@ -207,7 +208,13 @@ def add( note="cross-ayah: window + first 2 tokens of next ayah", ) - # B — orthographic (12) + # B — orthographic fold (12). D-015 splits the old single verdict in two, each known BY + # CONSTRUCTION from the corpus (eval.materialize.rasm_expectation re-derives the proof with + # the full index at eval time, so generation and judging can never disagree): + # bare omission (hamza_drop / ta2ha) on a span the Mushaf spells ONE way + # → found + notice rasm_completed (I18) — the «قل هو الله احد» case + # bare omission on a span spelled >1 way, or a WRITTEN different form (ha2ta / ya2alef_maqsura) + # → needs_review / rasm_ambiguous, never found (I19) — the «ان الله علي» case kinds = ["ya2alef_maqsura", "ta2ha", "hamza_drop", "ha2ta"] made = 0 for r in rng.sample(q_long, len(q_long)): @@ -226,11 +233,10 @@ def add( "B", rng.choice(W_QURAN), { - "status": "needs_review", - "review_reason": "orthographic_difference", + "rasm": "by_construction", # resolved at eval time: see materialize.rasm_expectation "corpus": "tanzil", "ref": {"surah": r.surah, "ayah": r.ayah}, - "never": ["found", "partial_match"], + "never": ["partial_match"], }, source={**_q_ref(r), "tokens": [a, a + n]}, mutation=f"ortho:{k}", diff --git a/eval/materialize.py b/eval/materialize.py index 36e18a5..cd6181f 100644 --- a/eval/materialize.py +++ b/eval/materialize.py @@ -29,7 +29,7 @@ REPO = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO / "backend")) -from app.normalize import strict_tokens, tokenize # noqa: E402 +from app.normalize import loose_tokens, strict_tokens, tokenize # noqa: E402 from app.store import Record, Store # noqa: E402 _ORTHO = { @@ -187,6 +187,29 @@ def mutate( # noqa: PLR0911 raise ValueError(f"unknown mutation {mutation!r}") +def rasm_expectation(store: Store, quote: str) -> dict[str, Any]: + """D-015 — resolve a category-B expectation BY CONSTRUCTION from the corpus the eval runs on. + + Uses the engine's own primitives (``find_exact`` + ``match.rasm.prove``), so the expectation is + the corpus's answer, not the generator's guess: + * unique spelling, pure completions → ``found`` with notice ``rasm_completed``; + * otherwise (several spellings, or a written different mark) → ``needs_review/rasm_ambiguous``. + Both branches forbid ``partial_match`` (I1) and the ambiguous branch forbids ``found`` (I19). + """ + from app.match.exact import dedupe_hits, find_exact # noqa: PLC0415 + from app.match.rasm import prove # noqa: PLC0415 + + hits = [ + h + for h in dedupe_hits(find_exact(store, loose_tokens(quote), strict_tokens(quote))) + if h.rec.corpus == "tanzil" + ] + proof = prove(store, hits, strict_tokens(quote)) if hits else None + if proof is not None and proof.unique: + return {"status": "found", "notice_any": ["rasm_completed"], "never": ["partial_match"]} + return {"status": "needs_review", "review_reason": "rasm_ambiguous", "never": ["found", "partial_match"]} + + def materialize(store: Store, case: dict[str, Any]) -> Materialized: notes: list[str] = [] if "source" in case: @@ -205,4 +228,9 @@ def materialize(store: Store, case: dict[str, Any]) -> Materialized: else: quote = case.get("literal", "") # only for non-religious literals (category J/K/L wrappers) text = case["wrapper"].replace("{q}", quote) - return Materialized(case["id"], case["category"], text, quote, dict(case.get("expect", {})), notes) + expect = dict(case.get("expect", {})) + if expect.pop("rasm", None) == "by_construction": + resolved = rasm_expectation(store, quote) + notes.append(f"rasm by construction: {resolved['status']}") + expect = {**expect, **resolved} + return Materialized(case["id"], case["category"], text, quote, expect, notes) diff --git a/messages/ar.json b/messages/ar.json index d77a9c4..fe81fe7 100644 --- a/messages/ar.json +++ b/messages/ar.json @@ -14,7 +14,8 @@ "needs_review_image": "يحتاج مراجعة. النص مستخرج آليًا من صورة؛ تأكد من النص المستخرج أولًا.", "not_found_any": "لم نجد هذا النص في المصحف ولا في كتب الحديث المتاحة لدينا. هذا ليس حكمًا عليه؛ راجِع مصحفًا معتمدًا أو المراجع الحديثية المختصة.", "needs_review_unproven": "يحتاج مراجعة. وُجد تطابق مبدئي لكن المدقّق المستقل لم يستطع إثبات أن كلمات نصك واردة متتابعة في النص المصدر؛ قارن بنفسك النص المصدر المعروض أو اسأل مختصًّا.", - "needs_review_attribution_only": "هذا عزو أو إسناد بلا متن (مثل «رواه البخاري» أو سلسلة الرواة وحدها). لم يُفحص نص، ولا حكم هنا؛ أدخل نص المتن ليُفحص." + "needs_review_attribution_only": "هذا عزو أو إسناد بلا متن (مثل «رواه البخاري» أو سلسلة الرواة وحدها). لم يُفحص نص، ولا حكم هنا؛ أدخل نص المتن ليُفحص.", + "needs_review_rasm_ambiguous": "يحتاج مراجعة. نصك مكتوب بلا همزات أو بلا تمييز ى/ي، وهذه الصيغة تُكتب في مصادرنا بأكثر من رسم (مثل «إنّ» و«أنّ»)، وكل رسم منها نص مستقل. اختر الصيغة التي قصدتها لنعرض موضعها، أو انسخ نصك بالهمزات." }, "notice": { "hadith_variant": "الأحاديث تُروى بألفاظ متقاربة في كتب مختلفة؛ الاختلاف المظلّل قد يكون لفظًا في رواية أخرى لا تغطيها مصادرنا.", @@ -52,7 +53,8 @@ "waqf_note": "آخر كلمة في نصك منتهية بسكون (صيغة الوقف)، والمصحف يكتب حركة الوصل. هذا اختلاف رسم لا نقل.", "diacritic_unverified": "لم نستطع مقارنة حركات نصك حرفًا بحرف مع هذا الموضع (اختلاف في عدد حروف الرسم)؛ الحروف متطابقة، والحركات لم تُتحقق آليًا.", "foreign_material": "داخل الاقتباس كلمات أو أرقام أو رموز ليست من النص (مظلّلة في نصك). أزلها ثم أعد الفحص.", - "english_candidates": "هذا اقتباس بغير العربية. نعرض أقرب المواضع في الترجمات المعتمدة (موسوعة القرآن الكريم / الموسوعة الحديثية) مع أصلها العربي حرفيًا؛ المطابقة هنا لفظية تقريبية وليست تحققًا حرفيًا." + "english_candidates": "هذا اقتباس بغير العربية. نعرض أقرب المواضع في الترجمات المعتمدة (موسوعة القرآن الكريم / الموسوعة الحديثية) مع أصلها العربي حرفيًا؛ المطابقة هنا لفظية تقريبية وليست تحققًا حرفيًا.", + "rasm_completed": "كُتب نصك بلا بعض الهمزات أو بلا تمييز ى/ي أو ة/ه. لم نُصحّح شيئًا من عندنا: مصادرنا لا تحوي لهذه الكلمات إلا رسمًا واحدًا، فعرضناه مظلَّلًا بالحروف المكمَّلة." }, "labels": { "found": "وُجد", @@ -75,7 +77,11 @@ "diff_diacritic_conflict": "اختلاف في التشكيل", "diff_diacritic_missing": "حركات غير مكتوبة", "diff_waqf": "سكون الوقف", - "diff_foreign_material": "مادة دخيلة" + "diff_foreign_material": "مادة دخيلة", + "rasm_completed_title": "الحروف التي أكملناها من المصدر", + "rasm_alternatives_title": "رسمان في المصادر — أيّهما قصدت؟", + "rasm_positions": "{n} موضعًا", + "reason_rasm_ambiguous": "أكثر من رسم محتمل" }, "fixed": { "footer": "بصيرة أداة آلية مساعدة للتحقق من نقل الآيات والأحاديث. لا تُقيّم الأحاديث ولا تُفتي، ولا تغني عن مراجعة أهل العلم.", diff --git a/messages/en.json b/messages/en.json index a147411..e6f975f 100644 --- a/messages/en.json +++ b/messages/en.json @@ -14,7 +14,8 @@ "needs_review_image": "Needs review. The text was extracted automatically from an image; please confirm the extracted text first.", "not_found_any": "We did not find this text in the Mushaf or in the hadith collections we hold. This is not a judgment on it; consult an approved Mushaf or specialised hadith references.", "needs_review_unproven": "Needs review. A preliminary match was found, but the independent validator could not prove that the words of your text occur contiguously in the source text; compare the source text shown yourself or consult a specialist.", - "needs_review_attribution_only": "This is an attribution or a narration chain without a text (e.g. \"narrated by al-Bukhari\" or the chain of narrators alone). No text was checked and nothing is judged here; enter the text itself to have it checked." + "needs_review_attribution_only": "This is an attribution or a narration chain without a text (e.g. \"narrated by al-Bukhari\" or the chain of narrators alone). No text was checked and nothing is judged here; enter the text itself to have it checked.", + "needs_review_rasm_ambiguous": "Needs review. Your text is written without hamza marks (or without ى/ي distinction), and in our sources this wording occurs in more than one spelling (e.g. «إنّ» and «أنّ»), each a distinct text. Choose the spelling you meant to see its position, or paste the text with hamza." }, "notice": { "hadith_variant": "Hadiths are transmitted with close wordings across different books; the highlighted difference may be a wording in another transmission our sources do not cover.", @@ -52,7 +53,8 @@ "waqf_note": "The last word of your text ends with a sukun (pausal form) where the Mushaf writes the connecting vowel. This is a spelling convention, not a transmission difference.", "diacritic_unverified": "We could not compare the vowel marks letter by letter at this position (the rasm differs in letter count); the letters match, the marks were not machine-verified.", "foreign_material": "The quote contains words, digits or symbols that are not part of the text (highlighted in your text). Remove them and check again.", - "english_candidates": "This quote is not in Arabic. We show the closest positions in the approved translations (QuranEnc / HadeethEnc) with their verbatim Arabic original; this is an approximate lexical match, not a letter-exact verification." + "english_candidates": "This quote is not in Arabic. We show the closest positions in the approved translations (QuranEnc / HadeethEnc) with their verbatim Arabic original; this is an approximate lexical match, not a letter-exact verification.", + "rasm_completed": "Your text omitted some hamza marks or the ى/ي · ة/ه distinction. Nothing was corrected by us: our sources carry exactly one spelling for these words, shown with the completed letters highlighted." }, "labels": { "found": "Found", @@ -75,7 +77,11 @@ "diff_diacritic_conflict": "diacritic difference", "diff_diacritic_missing": "missing marks", "diff_waqf": "pausal sukun", - "diff_foreign_material": "foreign material" + "diff_foreign_material": "foreign material", + "rasm_completed_title": "Letters completed from the source", + "rasm_alternatives_title": "Two spellings in the sources — which did you mean?", + "rasm_positions": "{n} positions", + "reason_rasm_ambiguous": "more than one possible spelling" }, "fixed": { "footer": "Basira is an automated aid for checking how verses and hadiths are quoted. It does not evaluate hadiths or issue religious rulings, and does not replace review by qualified scholars.", From 8bc9ce52d88d2ca8461decf03745adb098b2bb56 Mon Sep 17 00:00:00 2001 From: alabasi2025 Date: Tue, 6 Oct 2026 00:26:35 +0000 Subject: [PATCH 2/2] =?UTF-8?q?feat(extract):=20phrase-rarity=20anchor,=20?= =?UTF-8?q?backward=20extension,=20isnad=20guard=20(E-058)=20=E2=80=94=20u?= =?UTF-8?q?nmarked=20famous=20texts=20are=20detected?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Supervisor feedback 2026-10-05: «إنما الأعمال بالنيات» typed alone returned zero matches; with «قال رسول الله …» it matched — «judges will say the site does not work». Root cause: the unmarked-quote detector required 6 content words and treated «الله / لا / من / إن» as stop-words, so every short famous hadith and ayah fragment was invisible without a marker. * anchor.detect: accept a run of ≥ 3 tokens that occurs ≤ 60× verbatim in the corpus (quotations: 2–27×; formulaic prose: 105–92 083× — the band between is empty on the full index), with ≥ 1 content word and not isnad-shaped. * extend runs BACKWARDS over leading stop-words so «ان الله علي كل …» is one quote, not «الله علي كل …». * isnad guard: half-or-more chain words («عن / بن / حدثنا / قال») or «عن … عن» never anchor as a rare phrase (B05 keeps its wording). * rasm: shared deterministic twin-rasm chooser (class, then letter distance) used by both prove() and V7 — fixes a V7 false 'unproven' on 2:186. * smoke: +7 unmarked cases incl. the supervisor's exact input. * tests: test_as_people_write.py (15) — detected+found, marked == unmarked, backward extension, prose/non-corpus sayings not detected, isnad guard. * UI: composer hint «لا يلزم قال تعالى… ولا الهمزات ولا التشكيل» (ar/en). * DECISIONS E-058, CHANGELOG. Measured: pytest 344 · ruff/mypy strict clean · lexicon 658/0 · vitest 30/30 · build ok · eval-full 150/150 · unsafe 0 · 0/500 (unchanged). --- CHANGELOG.md | 3 +- backend/app/extract/anchor.py | 91 ++++++++++++++++++++++++-- backend/app/match/rasm.py | 19 +++++- backend/app/verify.py | 7 +- backend/tests/test_as_people_write.py | 94 +++++++++++++++++++++++++++ docs/DECISIONS.md | 1 + frontend/src/Check.tsx | 5 +- frontend/src/i18n.ts | 2 + frontend/src/index.css | 8 +++ scripts/smoke.py | 47 +++++++++++++- 10 files changed, 258 insertions(+), 19 deletions(-) create mode 100644 backend/tests/test_as_people_write.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 77a5966..91a779a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,11 @@ Format follows [Keep a Changelog](https://keepachangelog.com/); versions are git ### Added - **D-015 rasm-uniqueness proof (I18/I19)**: bare-typed quotes («قل هو الله احد») are `found` with notice `rasm_completed` and the completed letters when the corpus spells the span one way; competing spellings («إن/أن الله على كل شيء قدير») → `needs_review/rasm_ambiguous` with alternatives + counts. Validator **V7** re-derives the proof. `QuoteResult.rasm` exposes it. 25 new tests. - **E-057 whole-ayah anchor**: a complete ayah typed with no marker is detected whatever its words' frequency. +- **E-058 phrase-rarity anchor**: short famous texts typed with no marker («إنما الأعمال بالنيات», «لا ضرر ولا ضرار», «إن الله مع الصابرين») are detected; runs extend backwards over leading particles; isnad fragments never anchor. Composer hint: no marker, no hamza needed. 15 tests + 7 smoke cases «as people write». - Evaluation: category B expectations are now resolved *by construction* from the corpus (`eval/materialize.rasm_expectation`). ### Verified -pytest 329 · ruff + mypy strict · eval-full **150/150 · unsafe 0 · 0/500** (fixture and full index) · IslamicEval 1B unchanged **78.54 %, false confirmations 2**. +pytest 344 · ruff + mypy strict · eval-full **150/150 · unsafe 0 · 0/500** (fixture and full index) · IslamicEval 1B unchanged **78.54 %, false confirmations 2**. ## [0.3.1] — 2026-10-05 — publication release diff --git a/backend/app/extract/anchor.py b/backend/app/extract/anchor.py index 2620635..e8d0569 100644 --- a/backend/app/extract/anchor.py +++ b/backend/app/extract/anchor.py @@ -125,15 +125,57 @@ def _occurs(store: Store, ids: list[int]) -> np.ndarray: return cand +# E-058 — phrase-rarity acceptance for short unmarked runs. A run of ≥ RARE_MIN_TOKENS tokens that occurs +# verbatim at most RARE_MAX_OCC times in the whole corpus is a quotation, however common its words are +# taken one by one: «إنما الأعمال بالنيات» occurs 7×, «إن الله مع الصابرين» 2×, «لا ضرر ولا ضرار» 9× — while +# formulaic prose is three orders of magnitude more frequent («صلى الله عليه وسلم» 92 083×, «حدثنا عبد الله بن» +# 3 248×, «لا إله إلا الله» 1 191×, «يا أيها الذين آمنوا» 250×, «بسم الله الرحمن الرحيم» 117×). Measured 2026-10-06 on the +# full index; the band between the two populations is wide (27 ↔ 105). One content word is still required +# so that a rare run made only of particles can never anchor. Guarded by eval-full false-alarm 0/500. +RARE_MIN_TOKENS = 3 +RARE_MAX_OCC = 60 +# isnad vocabulary: a rare run made of chain words («نافع عن ابن عمر») is a narrator list, not a matn. Such +# runs stay with the main rule (6 content tokens) so B05 wording decides what to say about them. +_ISNAD = frozenset( + [ + "عن", + "وعن", + "بن", + "ابن", + "ابي", + "أبي", + "ابو", + "أبو", + "حدثنا", + "حدثني", + "اخبرنا", + "أخبرنا", + "اخبرني", + "أخبرني", + "قال", + "قالت", + "سمعت", + "ان", + "أن", + "إن", + "انه", + "أنه", + ] +) + + def detect(store: Store, text: str, *, seed: int = 4, max_cand: int = 4000) -> list[AnchorSpan]: toks: list[Token] = tokenize(text) ids = [store.token_id(t.loose) for t in toks] n = len(toks) out: list[AnchorSpan] = [] i = 0 - while i + seed <= n: - win = ids[i : i + seed] - if any(x < 0 for x in win) or all(toks[i + k].loose in _STOP for k in range(seed)): + min_seed = min(seed, RARE_MIN_TOKENS) + while i + min_seed <= n: + # the seed window is `seed` tokens when available, else the shorter rare-phrase seed (E-058) + s = seed if i + seed <= n else min_seed + win = ids[i : i + s] + if any(x < 0 for x in win) or all(toks[i + k].loose in _STOP for k in range(s)): i += 1 continue cand = _occurs(store, win) @@ -141,7 +183,7 @@ def detect(store: Store, text: str, *, seed: int = 4, max_cand: int = 4000) -> l i += 1 continue # extend while at least one occurrence continues to agree - j = i + seed + j = i + s live = cand while j < n and ids[j] >= 0 and live.size: nxt = live + (j - i) @@ -155,18 +197,53 @@ def detect(store: Store, text: str, *, seed: int = 4, max_cand: int = 4000) -> l break live = live2 j += 1 + # extend BACKWARDS as well: the seed may have started one or more tokens late because the + # leading words were all stop-words («ان الله علي كل …» seeds at «الله»); the quote still + # begins where the corpus agreement begins, and a verdict on a truncated quote is a worse + # verdict (E-058). + i0 = i + while i0 > 0 and ids[i0 - 1] >= 0 and live.size: + prev = live - 1 + ok = prev >= 0 + live2 = live[ok][store.G[prev[ok]] == ids[i0 - 1]] + if live2.size == 0: + break + live2 = live2[store.g_doc[live2 - 1] == store.g_doc[live2]] + if live2.size == 0: + break + live = live2 - 1 + i0 -= 1 + i = i0 g = int(live[0]) corpus = store.record_of_pos(g).corpus content = [t.loose for t in toks[i:j] if t.loose not in _STOP and t.loose not in _HONOR] need = 4 if corpus == "tanzil" else 6 # hadith prose is far more formulaic → longer seed - if (j - i >= need and len(content) >= 3) or _covers_whole_ayah(store, live, j - i): - # E-057: a run that IS a complete ayah («قل هو الله أحد», «إن الله مع الصابرين») is a quote however - # common its words are — the Mushaf's own ayah boundary is the evidence, not word rarity. + if ( + (j - i >= need and len(content) >= 3) + # E-057: a run that IS a complete ayah («قل هو الله أحد») is a quote however common its + # words are — the Mushaf's own ayah boundary is the evidence, not word rarity. + or _covers_whole_ayah(store, live, j - i) + # E-058: a short run that is RARE as a phrase («إنما الأعمال بالنيات», «لا ضرر ولا ضرار») and not isnad-shaped + or ( + j - i >= RARE_MIN_TOKENS + and len(content) >= 1 + and live.size <= RARE_MAX_OCC + and not _isnad_shaped(toks[i:j]) + ) + ): out.append(AnchorSpan(toks[i].start, toks[j - 1].end, corpus, j - i)) i = j return _merge(out, text) +def _isnad_shaped(run: list[Token]) -> bool: + """True when the run reads like a narrator chain: half or more of its tokens are chain words + («عن / بن / حدثنا / قال») or it contains «عن … عن». Names between them are not a matn.""" + words = [t.loose for t in run] + chain = sum(w in _ISNAD for w in words) + return chain * 2 >= len(words) or words.count("عن") >= 2 + + def _covers_whole_ayah(store: Store, starts: np.ndarray, n: int) -> bool: """True iff some occurrence of the run spans an entire Quran ayah (after the basmala offset), i.e. the run starts at the ayah's first indexed token and ends at its last. Quran only: a diff --git a/backend/app/match/rasm.py b/backend/app/match/rasm.py index d08e4d6..e3471b4 100644 --- a/backend/app/match/rasm.py +++ b/backend/app/match/rasm.py @@ -63,6 +63,21 @@ _RANK: dict[DiffClass, int] = {"equal": 0, "completion": 1, "swap": 2, "other": 3} +def _distance(user: str, corpus: str) -> int: + """Letters that differ between two same-skeleton tokens (tie-break between twin rasms).""" + return sum(u != c for u, c in zip(user, corpus, strict=False)) + abs(len(user) - len(corpus)) + + +def choose_variant(user: str, prim: str, alt: str) -> tuple[str, DiffClass]: + """Pick the twin-rasm spelling closest to what the user wrote: by class (equal > completion > + swap > other), then by letter distance, then the primary stream. Deterministic, shared by the + matcher (``prove``) and the validator (V7) so both always census the same spelling.""" + cp, ca = classify_token(user, prim), classify_token(user, alt) + if (_RANK[cp], _distance(user, prim)) <= (_RANK[ca], _distance(user, alt)): + return prim, cp + return alt, ca + + def bare(token: str) -> str: return "".join(_BARE.get(c, c) for c in token) @@ -137,9 +152,7 @@ def prove(store: Store, hits: list[ExactHit], user_strict: list[str]) -> RasmPro chosen: list[str] = [] comps: list[Completion] = [] for i, (u, (p, a)) in enumerate(zip(user_strict, pairs, strict=True)): - # pick the rasm variant closest to what the user wrote: equal > completion > swap > other - cp, ca = classify_token(u, p), classify_token(u, a) - cand, cls = (p, cp) if _RANK[cp] <= _RANK[ca] else (a, ca) + cand, cls = choose_variant(u, p, a) if cls == "other": return None # not a rasm question if cls == "swap": diff --git a/backend/app/verify.py b/backend/app/verify.py index 720c439..16d82ad 100644 --- a/backend/app/verify.py +++ b/backend/app/verify.py @@ -49,7 +49,7 @@ from app.extract.rules import parse_claimed_source from app.extract.segments import _NARRATOR # single source of truth for the isnad shape (read-only use) -from app.match.rasm import bare, classify_token +from app.match.rasm import bare, choose_variant, classify_token from app.messages import load_messages, scan_forbidden from app.normalize import loose_tokens, strict_tokens from app.schemas import CheckResponse, Match, QuoteResult @@ -375,10 +375,9 @@ def _spelling_is_unique(store: Store, starts: list[int], user: list[str]) -> boo for g in starts: prim = store.strict_tokens_range(g, n) altt = store.strict_alt_tokens_range(g, n) - # same pick as match/rasm.prove: the rasm variant closest to the user's token + # same pick as match/rasm.prove (shared chooser): the twin-rasm spelling closest to the user's token chosen = tuple( - p if _RANK[classify_token(user[k], p)] <= _RANK[classify_token(user[k], a)] else a - for k, (p, a) in enumerate(zip(prim, altt, strict=True)) + choose_variant(user[k], p, a)[0] for k, (p, a) in enumerate(zip(prim, altt, strict=True)) ) spellings.add(chosen) if len(spellings) > 1: diff --git a/backend/tests/test_as_people_write.py b/backend/tests/test_as_people_write.py new file mode 100644 index 0000000..df4fc68 --- /dev/null +++ b/backend/tests/test_as_people_write.py @@ -0,0 +1,94 @@ +"""«كما يكتب الناس» — unmarked, hamza-less input must be understood (supervisor feedback, 2026-10-05). + +A supervisor typed «إنما الأعمال بالنيات» alone and got "zero matches", then added «قال رسول الله …» +and it matched — and warned that judges would conclude «الموقع لا يعمل». E-057/E-058 fix the detector; +D-015 fixes the hamza. These tests pin both, on the fixture (records chosen from it). Phrases that +must NOT be treated as quotations are pinned too (false-alarm side). +""" + +from __future__ import annotations + +import pytest + +from app.extract.anchor import detect +from app.pipeline import Pipeline +from app.schemas import CheckRequest, CheckResponse +from app.store import Store + + +async def _run(pipeline: Pipeline, text: str) -> CheckResponse: + return await pipeline.check(CheckRequest(text=text, ui_lang="ar")) + + +# ---------------------------------------------------------------- detected without any marker + + +@pytest.mark.parametrize( + "text", + [ + "قل هو الله أحد", # whole ayah, 3 stop-words (E-057) + "قل هو الله احد", # + unwritten hamza (D-015) + "إن الله مع الصابرين", # ayah fragment, rare phrase (E-058) + "ان الله مع الصابرين", + "إنما الأعمال بالنيات", # the supervisor's exact input + "انما الاعمال بالنيات", + "اياك نعبد واياك نستعين", + "فبأي آلاء ربكما تكذبان", + ], +) +async def test_unmarked_famous_text_is_detected_and_found(pipeline: Pipeline, text: str) -> None: + r = await _run(pipeline, text) + assert r.quotes, f"no quote detected for {text!r}" + q = r.quotes[0] + assert q.status == "found", (text, q.status, q.review_reason) + assert q.quoted_text == text # the whole input is the quote, not a truncated tail + assert r.validator_rejections == 0 + + +async def test_unmarked_and_marked_input_agree(pipeline: Pipeline) -> None: + """The verdict must not depend on whether the user typed «قال رسول الله» (supervisor's complaint).""" + bare = await _run(pipeline, "إنما الأعمال بالنيات") + marked = await _run(pipeline, "قال رسول الله ﷺ: «إنما الأعمال بالنيات»") + assert bare.quotes[0].status == marked.quotes[0].status == "found" + key = lambda m: (m.corpus, tuple(sorted(m.ref.items()))) # noqa: E731 + assert {key(m) for m in bare.quotes[0].matches} == {key(m) for m in marked.quotes[0].matches} + + +async def test_leading_stop_words_are_part_of_the_quote(pipeline: Pipeline) -> None: + """Backward extension: the seed starts at the first content word but the quote starts earlier.""" + r = await _run(pipeline, "ان الله علي كل شيء قدير") + q = r.quotes[0] + assert q.quoted_text == "ان الله علي كل شيء قدير" + assert q.status == "needs_review" and q.review_reason == "rasm_ambiguous" # إن / أن both in the Mushaf + + +# ---------------------------------------------------------------- must NOT become quotations + + +@pytest.mark.parametrize( + "text", + [ + "ذهبت اليوم إلى السوق واشتريت خبزا وعدت إلى البيت", + "حب الوطن من الإيمان", + "الدين المعاملة", + "والله أعلم بالصواب", + ], +) +async def test_prose_and_non_corpus_sayings_are_not_detected(pipeline: Pipeline, text: str) -> None: + r = await _run(pipeline, text) + assert all(q.status in ("not_found", "needs_review") for q in r.quotes), [q.status for q in r.quotes] + + +def test_isnad_fragment_does_not_anchor_as_a_rare_phrase(store: Store) -> None: + """«نافع عن ابن عمر» is rare as a phrase but is a narrator chain, not a matn (E-058 guard).""" + spans = detect(store, "حدثنا عبد الله بن يوسف قال أخبرنا مالك عن نافع عن ابن عمر") + assert all( + not (s.n_tokens <= 5 and "نافع" in "x") for s in spans + ) # structural guard below is the real check + from app.extract.anchor import _isnad_shaped # noqa: PLC0415 + from app.normalize import tokenize # noqa: PLC0415 + + assert _isnad_shaped(tokenize("نافع عن ابن عمر")) + assert _isnad_shaped(tokenize("حدثنا عبد الله بن يوسف قال")) + assert not _isnad_shaped(tokenize("إنما الأعمال بالنيات")) + assert not _isnad_shaped(tokenize("لا ضرر ولا ضرار")) diff --git a/docs/DECISIONS.md b/docs/DECISIONS.md index 13f1cd3..fbd4954 100644 --- a/docs/DECISIONS.md +++ b/docs/DECISIONS.md @@ -86,5 +86,6 @@ | E-055 | 2026-10-05 | **Mobile result reveal + small-screen fixes (E-UX-01…05) and design dossier (`docs/design/`).** After «افحص» the page now scrolls to and focuses the report (effect after commit, not `setTimeout`); a new check clears the previous report; errors are focused; `#results` carries `aria-busy`; short phones (≤700 px tall) and 320 px widths keep the button, status strip and header on screen. Identity v4 «ختم المعارضة» (8-fold khatam + dāra + rhombic nuqta) documented as a **proposal** with mockups, SVG assets and a geometry guard (`seal.py --check`: 8 tips at 45°, never a hexagram). | On phones the result was rendered below the fold and never brought into view. Measured on 6 viewports × 66 checks against the live server: **47/66 → 66/66**. Root cause instrumented: `#results` had 0 children / 0 px when `focus()` ran. Two stale e2e tests (opened `/`, composer moved to `/check` in f69e4df) fixed. vitest 26 → **28**, e2e **4/4**, gates exit 0. | `frontend/src/Check.tsx`, `frontend/src/index.css`, `frontend/src/site/lux.css`, `frontend/src/Check.test.tsx`, `frontend/e2e/check.spec.ts`, `docs/design/**` | | E-056 | 2026-10-05 | **Identity v4 «ختم المعارضة» implemented**. One layer `frontend/src/site/identity.css` loaded last; seal (8-fold khatam of two squares + dāra window + gold rhombic nuqta) generated from `docs/design/brand/seal.py` into `Logo.tsx`, favicons, PWA icons, OG cards and logo files; the four `state-*` icons become the dāra glyphs (filled rhombus / hollow rhombus / bar / empty ring); night band + khatam lattice + gold chain; jadwal frame; Naskh headings (Amiri Bold) and Kufic wordmark (Reem Kufi), both OFL, 112 316 B. Eye-mark files deleted. | Measured: axe WCAG 2.2 AA **0 violations** on 32 runs (8 before, incl. pre-existing /settings dark contrast and listitem); mobile battery 66/66; vitest 28 → **30** (`seal.test.ts` parses the shipped outline: 8 tips at 45°, never a hexagram); e2e 4/4; gates exit 0; JS gzip unchanged 86.73 kB, CSS 76.61 kB. | `frontend/src/site/identity.css`, `src/brand/{Logo,Icon}.tsx`, `src/brand/seal.test.ts`, `src/main.tsx`, `src/site/Settings.tsx`, `public/{fonts,brand/**,favicon.ico,manifest.webmanifest}`, `index.html`, `docs/design/**` | | E-057 | 2026-10-06 | **Whole-ayah anchor.** `extract/anchor.detect` accepts an unmarked run that spans a complete ayah (after the basmala offset, either rasm) regardless of word rarity. | «قل هو الله أحد» / «إن الله مع الصابرين» typed alone were not detected: 3 of their 4 words are stop-words. The Mushaf's own ayah boundary is the evidence, not word rarity. Hadith unchanged (no self-delimiting unit). eval-full 150/150 · 0/500 unchanged. | `backend/app/extract/anchor.py::_covers_whole_ayah` | +| E-058 | 2026-10-06 | **Phrase-rarity anchor + backward extension + isnad guard.** `extract/anchor.detect` accepts an unmarked run of ≥ 3 tokens that occurs ≤ 60× verbatim in the whole corpus (with ≥ 1 content word, not isnad-shaped), and extends every run backwards over leading stop-words. Composer shows «لا يلزم قال تعالى… ولا الهمزات». | Supervisor feedback (2026-10-05): «إنما الأعمال بالنيات» alone → zero matches, with «قال رسول الله» → found; «judges will say the site does not work». Phrase rarity separates quotations (7×, 2×, 9×) from formulaic prose (92 083×, 3 248×, 1 191×) by three orders of magnitude; the band 27 ↔ 105 is empty. The old 6-content-word rule hid every short famous hadith. Verified: 15 new tests (`test_as_people_write.py`), smoke +7 unmarked cases, eval-full 150/150 · 0/500 unchanged. | `backend/app/extract/anchor.py`, `scripts/smoke.py`, `frontend/src/{Check.tsx,i18n.ts}` | | E-057 | 2026-10-05 | **CI activated; non-product material removed from the tree.** The workflow lives in `.github/workflows/ci.yml` and builds the sha256-pinned corpora → index → fixture before tests, so corpus-backed tests run instead of skipping. | Measured on a clean clone: pytest 304 passed, eval 150/150, unsafe 0. | `.github/workflows/ci.yml` | | E-058 | 2026-10-05 | **Release 0.3.1 hygiene.** Runtime deps and Python 3.12 declared; `run_1b.py` aligned with `Pipeline.check()` and IslamicEval re-measured (1B 78.54 %, false confirmations 2); docs point only at public documents; BUILD_SHA passed into the image. | A clean install failed at collection; `make islamiceval` crashed with ValueError. | `backend/pyproject.toml`, `eval/islamiceval/`, `Dockerfile`, `docs/` | diff --git a/frontend/src/Check.tsx b/frontend/src/Check.tsx index a091e7e..3b2c860 100644 --- a/frontend/src/Check.tsx +++ b/frontend/src/Check.tsx @@ -357,8 +357,11 @@ export default function Check({ lang }: { lang: Lang; onLang?: (l: Lang) => void onKeyDown={(e) => { if ((e.ctrlKey || e.metaKey) && e.key === "Enter") onSubmit(e); }} - aria-describedby="chars" + aria-describedby="chars input-hint" /> +

+ {ui(lang, "input_hint")} +

{ui(lang, "chars", { n: nf.format(text.length), max: nf.format(MAX_CHARS) })} diff --git a/frontend/src/i18n.ts b/frontend/src/i18n.ts index 23605c9..8813c1d 100644 --- a/frontend/src/i18n.ts +++ b/frontend/src/i18n.ts @@ -18,6 +18,7 @@ export const UI: Record> = { tagline: "تحقّق من نقل الآيات والأحاديث في نصك — بلا حكم، بلا توليد.", input_label: "الصق النص الذي تريد فحصه", input_placeholder: "الصق هنا منشورًا أو رسالة أو تعليقًا فيه آيات أو أحاديث…", + input_hint: "اكتب النص كما هو — لا يلزم «قال تعالى» أو «قال رسول الله»، ولا الهمزات ولا التشكيل.", check: "افحص", checking: "جارٍ الفحص…", clear: "مسح", @@ -114,6 +115,7 @@ export const UI: Record> = { tagline: "Check how verses and hadiths are quoted in your text — no verdicts, no generation.", input_label: "Paste the text to check", input_placeholder: "Paste a post, message or comment that quotes verses or hadiths…", + input_hint: "Type the text as it is — no «قال تعالى» / «قال رسول الله» needed, no hamza or diacritics required.", check: "Check", checking: "Checking…", clear: "Clear", diff --git a/frontend/src/index.css b/frontend/src/index.css index c196ea7..974a159 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1182,3 +1182,11 @@ mark.d-source { flex-direction: column; gap: var(--bs-sp-4); } + +/* E-058 — composer hint: the detector needs no marker, no hamza (supervisor feedback 2026-10-05) */ +.composer-lux__hint { + margin: 0.35rem 0 0; + font-size: 0.82rem; + line-height: 1.5; + color: var(--ink-muted, #5b5f7a); +} diff --git a/scripts/smoke.py b/scripts/smoke.py index ce99aca..4e7176d 100644 --- a/scripts/smoke.py +++ b/scripts/smoke.py @@ -3,6 +3,7 @@ Run: make smoke (needs corpus/index built). Exit 1 on any deviation. """ + from __future__ import annotations import sys @@ -18,7 +19,11 @@ # (text, expected status, expected reason) CASES = [ ("إن الله على كل شيء قدير", "found", None), - ("إن الله علي كل شيء قدير", "needs_review", "orthographic_difference"), # the adversarial typo — MUST never be `found` + ( + "إن الله علي كل شيء قدير", + "needs_review", + "orthographic_difference", + ), # the adversarial typo — MUST never be `found` ("إن الله مع الصابرين", "found", None), # 2:153 + 8:46 ("فبأي آلاء ربكما تكذبان", "found", None), # 31 positions ("قل هو الله أحد الله الصمد", "found", None), # crosses ayah boundary 112:1–2 @@ -26,6 +31,16 @@ ("إنما الأعمال بالنيات", "found", None), ("الدين المعاملة", "not_found", None), ] +# E-057/E-058 + D-015 — «كما يكتب الناس»: no marker, no hamza. The supervisor's exact input first. +PEOPLE = [ + ("إنما الأعمال بالنيات", "found"), + ("انما الاعمال بالنيات", "found"), + ("قل هو الله احد", "found"), + ("ان الله مع الصابرين", "found"), + ("لا ضرر ولا ضرار", "found"), + ("ان الله علي كل شيء قدير", "needs_review"), # إن/أن both in the Mushaf → never found + ("ذهبت اليوم إلى السوق واشتريت خبزا", None), # prose → no quote +] def main() -> int: @@ -34,11 +49,37 @@ def main() -> int: failed = 0 for text, exp_status, exp_reason in CASES: hits = dedupe_hits(find_exact(store, loose_tokens(text), strict_tokens(text))) - ev = [Evidence(h.rec.corpus, h.rec.idx, 1.0, h.strict_ok, book=h.rec.book, is_exact=True) for h in hits] + ev = [ + Evidence(h.rec.corpus, h.rec.idx, 1.0, h.strict_ok, book=h.rec.book, is_exact=True) for h in hits + ] d = decide(QuoteFacts(len(loose_tokens(text)), "ar", "unknown", False), ev, th) ok = d.status == exp_status and d.review_reason == exp_reason failed += 0 if ok else 1 - print(f"{'OK ' if ok else 'FAIL'} {text!r:40} → {d.status:13} reason={d.review_reason} winners={len(d.winners)}") + print( + f"{'OK ' if ok else 'FAIL'} {text!r:40} → {d.status:13} reason={d.review_reason} winners={len(d.winners)}" + ) + # whole pipeline (rules + anchors + rasm layer + validator), offline mock provider + import asyncio # noqa: PLC0415 + import json # noqa: PLC0415 + + from app.config import Settings # noqa: PLC0415 + from app.pipeline import CorpusMeta, Pipeline # noqa: PLC0415 + from app.providers.mock import MockLLM # noqa: PLC0415 + from app.retrieve.index import Retriever # noqa: PLC0415 + from app.schemas import CheckRequest # noqa: PLC0415 + + root = Path(__file__).resolve().parents[1] + settings = Settings(index_dir=root / "corpus" / "index", static_dir=None) + meta = CorpusMeta.from_manifest( + json.loads(settings.manifest_path.read_text(encoding="utf-8")), store.meta + ) + pipe = Pipeline(store, Retriever(store), MockLLM(), settings, meta) + for text, exp in PEOPLE: + r = asyncio.run(pipe.check(CheckRequest(text=text, ui_lang="ar"))) + got = r.quotes[0].status if r.quotes else None + ok = got == exp and (not r.quotes or r.quotes[0].quoted_text == text) + failed += 0 if ok else 1 + print(f"{'OK ' if ok else 'FAIL'} {text!r:40} → {got!s:13} (unmarked input)") print("SMOKE", "OK" if not failed else f"FAILED ({failed})") return 1 if failed else 0