From e977850ec1707f0489b54b434cc7151222b9f976 Mon Sep 17 00:00:00 2001 From: PiratesIRC Date: Sun, 28 Jun 2026 12:48:50 -0500 Subject: [PATCH 1/2] Migrate matcher onto the shared vendored core (FuzzyMatcherCore) Lineuparr is the canonical generator the core was extracted from, so its FuzzyMatcher now subclasses FuzzyMatcherCore and deletes the 10 inherited primitives (normalize_name, calculate_similarity, the callsign ladder, process_string_for_matching, _has_token_overlap, _length_scaled_threshold, _trailing_number) plus the dead module-level pattern constants. The country- detection layer, has_upgrade_quality, _is_group_header, _channel_number_boost, the cache glue, and the entry points (alias_match/fuzzy_match/match_all_streams) stay Lineuparr-specific. Decorative helpers are re-exported from the core. Adopts the core's two intended behavior changes (golden baseline regenerated): - East/West stripping is now gated on ignore_regional (was unconditional), so a regional_off normalize keeps "HBO West"/"HBO East". - process_string keeps "+", so Discovery+/Disney+/Paramount+ stay distinct (the agreed 4-of-4 superset). To preserve Lineuparr's region-filter design (strip bare Pacific/Central/ Mountain/Atlantic for scoring, enforce correctness via the post-match filter on the original names), the core gained an opt-in strip_bare_region resolved from a _STRIP_BARE_REGION class attr; Lineuparr sets it True. Stream-Mapparr leaves it off (keeps bug-066: "Comedy Central" survives). Vendored core hash-pinned in .github/scripts/core_manifest.json (check_core_parity.py + .gitattributes LF); CI byte-compiles it and runs the parity gate. Full local suite 430 green. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01N2rpwFn3AHoNvsbTrEwgwd --- .gitattributes | 5 + .github/scripts/check_core_parity.py | 48 ++ .github/scripts/check_matcher_golden.py | 141 ++++ .github/scripts/core_manifest.json | 3 + .github/scripts/matcher_golden_baseline.json | 300 +++++++ .github/scripts/sync_core.py | 107 +++ .github/workflows/validate.yml | 26 +- Lineuparr/fuzzy_matcher.py | 675 +--------------- Lineuparr/matching_core.py | 793 +++++++++++++++++++ Lineuparr/plugin.json | 2 +- Lineuparr/plugin.py | 2 +- 11 files changed, 1449 insertions(+), 653 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/scripts/check_core_parity.py create mode 100644 .github/scripts/check_matcher_golden.py create mode 100644 .github/scripts/core_manifest.json create mode 100644 .github/scripts/matcher_golden_baseline.json create mode 100644 .github/scripts/sync_core.py create mode 100644 Lineuparr/matching_core.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d98ca44 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# The vendored shared core is sha256-pinned by .github/scripts/check_core_parity.py +# against .github/scripts/core_manifest.json, so it MUST stay LF on every OS - a CRLF +# checkout (Windows autocrlf) would change the hash and break the drift gate. Same for +# any future vendored shared file (e.g. aliases_base.py). +Lineuparr/matching_core.py text eol=lf diff --git a/.github/scripts/check_core_parity.py b/.github/scripts/check_core_parity.py new file mode 100644 index 0000000..8fc67d9 --- /dev/null +++ b/.github/scripts/check_core_parity.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Vendored-core drift gate (Layer A) - runs in CI, needs no workspace _shared/. + +The committed vendored shared file (Lineuparr/matching_core.py) must hash-match the +sha256 pinned in core_manifest.json. This catches a hand-edit to the vendored copy that +silently diverges from the _shared source of truth. To land an INTENDED core change: +edit /_shared/matching_core.py, re-run .github/scripts/sync_core.py (re-vendors ++ rewrites the manifest), regenerate the golden baseline if behavior changed, and commit +all of it together. See MATCHER-STANDARDIZATION-PLAN.md. + +Exit 0 = match; exit 1 = drift/missing. +""" +import hashlib +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parent.parent # /.github/scripts -> +INNER = REPO_ROOT / REPO_ROOT.name # /Lineuparr (flat deploy folder) +MANIFEST = HERE / "core_manifest.json" + + +def main() -> int: + pins = json.loads(MANIFEST.read_text(encoding="utf-8")) + failed = False + for fname, expected in sorted(pins.items()): + path = INNER / fname + if not path.exists(): + print(f"MISSING vendored {fname} (run .github/scripts/sync_core.py)") + failed = True + continue + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + print(f"DRIFT {fname}: {actual} != pinned {expected}") + print(" If intended: edit _shared/, re-run sync_core.py, commit the new manifest.") + failed = True + else: + print(f"OK {fname}: {actual[:16]}...") + if failed: + print("Core parity gate FAILED.") + return 1 + print("Core parity gate passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/check_matcher_golden.py b/.github/scripts/check_matcher_golden.py new file mode 100644 index 0000000..fcd656d --- /dev/null +++ b/.github/scripts/check_matcher_golden.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Golden drift gate for Lineuparr's pure matcher primitives (Stage 0). + +Lineuparr keeps its unit suite OUT of git (tests/ is gitignored) and CI runs only the +static validate_plugin.py, so the matcher golden gate lives here in .github/scripts +instead of a pytest file. It is the Lineuparr equivalent of +Stream/Channel/EPG's tests/test_matcher_golden.py. + +It loads Lineuparr/fuzzy_matcher.py directly, runs a shared corpus through the PURE +primitives, and compares against the committed matcher_golden_baseline.json beside this +script. Any unreviewed change to match behavior fails CI. An INTENDED de-drift change is +landed by re-running this with --write (or tools/matcher_parity_check.py --write at the +workspace root) and committing the updated baseline in the same change. + +Keep the corpus below in lockstep with tools/matcher_parity_check.py and the per-plugin +tests/test_matcher_golden.py. Needs rapidfuzz installed to match the production path. +See MATCHER-STANDARDIZATION-PLAN.md §7. +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent # /.github/scripts -> +INNER = ROOT / "Lineuparr" +BASELINE = HERE / "matcher_golden_baseline.json" + +# --- shared corpus (keep identical to tools/matcher_parity_check.py) --------- +NAMES = [ + "US: USA Network HD", "US| ESPN", "[US] CNN", "UK: BBC One", "UK| ITV 1", + "Discovery Channel 4K", "HBO HD", "ESPN [FHD]", "Cinemax HD", "TNT UHD RAW", + "BBC Three", "BBC Four", "Three Angels Broadcasting Network", "ESPN 2", "HBO 2", + "JusticeCentral.TV", "DangerTV", "NewsNation", + "HBO East", "HBO West", "HBO (W)", "Fox Sports West", "ESPN Pacific", + "(PRIME) FOX News", "(D1) CBS", + "Disney+", "Discovery+", "Paramount+", "Disney Channel", "Discovery Channel", + "Justice Central", "Justice Central.TV", "Justice Central TV", "True Crime Network", + "WABC-TV", "KCBS", "KING 5", "WAVE 3", "WOOD TV8", "WHO 13", "KOMO News", + "\U0001f174\U0001f182\U0001f17f\U0001f175", "┃US┃ ESPN", "★ CNN ★", + "Россия 1", "France 2", "beИN SPORTS", + "HLN", "MTV", "getTV", "TUDN", "SEC Network", "NHL Network", "BBC News", +] +PAIRS = [ + ("usanetwork", "usanetwork"), ("espn", "espn2"), ("hbo", "hbo2"), + ("bbcone", "bbctwo"), ("disney", "disneyplus"), ("foxnews", "foxnews"), + ("cnn", "cnninternational"), ("discoverychannel", "discovery"), ("e", "ae"), + ("paramount", "paramountnetwork"), ("nflnetwork", "nhlnetwork"), + ("justicecentral", "truecrimenetwork"), ("a", "a"), ("", ""), +] +FLAG_COMBOS = [ + ("all_on", dict(ignore_quality=True, ignore_regional=True, ignore_geographic=True, ignore_misc=True)), + ("regional_off", dict(ignore_quality=True, ignore_regional=False, ignore_geographic=True, ignore_misc=True)), +] + + +def _safe(fn, *args, **kwargs): + try: + val = fn(*args, **kwargs) + except Exception as exc: + return f"__ERROR__: {type(exc).__name__}: {exc}" + if isinstance(val, float): + return round(val, 9) + return val + + +def run_corpus(matcher): + out = { + "process_string": {n: _safe(matcher.process_string_for_matching, n) for n in NAMES}, + "normalize_name": {}, + "calculate_similarity": {f"{a}|{b}": _safe(matcher.calculate_similarity, a, b) for a, b in PAIRS}, + "extract_callsign": {n: _safe(matcher.extract_callsign, n) for n in NAMES}, + "normalize_callsign": {n: _safe(matcher.normalize_callsign, n) for n in NAMES}, + } + for label, flags in FLAG_COMBOS: + out["normalize_name"][label] = {n: _safe(matcher.normalize_name, n, **flags) for n in NAMES} + return out + + +def _flatten(d, prefix=""): + for k in sorted(d): + v = d[k] + if isinstance(v, dict): + yield from _flatten(v, f"{prefix}{k}.") + else: + yield (f"{prefix}{k}", v) + + +def load_matcher(): + path = INNER / "fuzzy_matcher.py" + saved_path = list(sys.path) + saved_aliases = sys.modules.pop("aliases", None) + sys.path.insert(0, str(INNER)) + try: + spec = importlib.util.spec_from_file_location("fm_golden_under_test", str(path)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + finally: + sys.path[:] = saved_path + sys.modules.pop("aliases", None) + if saved_aliases is not None: + sys.modules["aliases"] = saved_aliases + return mod.FuzzyMatcher() + + +def main() -> int: + ap = argparse.ArgumentParser(description="Lineuparr matcher golden drift gate") + ap.add_argument("--write", action="store_true", help="(re)generate the baseline from current code") + args = ap.parse_args() + + current = run_corpus(load_matcher()) + if args.write: + BASELINE.write_text( + json.dumps(current, indent=2, ensure_ascii=False, sort_keys=True), encoding="utf-8" + ) + print(f"wrote {BASELINE}") + return 0 + + if not BASELINE.exists(): + print(f"MATCHER GOLDEN: no baseline at {BASELINE} (run with --write first)") + return 1 + base_flat = dict(_flatten(json.loads(BASELINE.read_text(encoding="utf-8")))) + cur_flat = dict(_flatten(current)) + diffs = [(k, base_flat.get(k, ""), cur_flat.get(k, "")) + for k in sorted(set(base_flat) | set(cur_flat)) + if base_flat.get(k, "") != cur_flat.get(k, "")] + if diffs: + print(f"MATCHER GOLDEN DRIFT ({len(diffs)} primitive output(s) changed):") + for key, old, new in diffs[:30]: + print(f" {key}: {old!r} -> {new!r}") + print("If intended, re-run with --write and commit the updated baseline.") + return 1 + print(f"Matcher golden gate passed ({len(base_flat)} primitive outputs match baseline).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/core_manifest.json b/.github/scripts/core_manifest.json new file mode 100644 index 0000000..7d8ff5b --- /dev/null +++ b/.github/scripts/core_manifest.json @@ -0,0 +1,3 @@ +{ + "matching_core.py": "2ac7ac4cb283306152701bfd977ff2f13700d63d8023fdfa280b1480f5a969dc" +} diff --git a/.github/scripts/matcher_golden_baseline.json b/.github/scripts/matcher_golden_baseline.json new file mode 100644 index 0000000..b02725b --- /dev/null +++ b/.github/scripts/matcher_golden_baseline.json @@ -0,0 +1,300 @@ +{ + "calculate_similarity": { + "a|a": 1.0, + "bbcone|bbctwo": 0.5, + "cnn|cnninternational": 0.1875, + "discoverychannel|discovery": 0.5625, + "disney|disneyplus": 0.6, + "espn|espn2": 0.8, + "e|ae": 0.5, + "foxnews|foxnews": 1.0, + "hbo|hbo2": 0.75, + "justicecentral|truecrimenetwork": 0.25, + "nflnetwork|nhlnetwork": 0.9, + "paramount|paramountnetwork": 0.5625, + "usanetwork|usanetwork": 1.0, + "|": 0.0 + }, + "extract_callsign": { + "(D1) CBS": null, + "(PRIME) FOX News": null, + "BBC Four": null, + "BBC News": null, + "BBC Three": null, + "Cinemax HD": null, + "DangerTV": null, + "Discovery Channel": null, + "Discovery Channel 4K": null, + "Discovery+": null, + "Disney Channel": null, + "Disney+": null, + "ESPN 2": null, + "ESPN Pacific": null, + "ESPN [FHD]": null, + "Fox Sports West": null, + "France 2": null, + "HBO (W)": null, + "HBO 2": null, + "HBO East": null, + "HBO HD": null, + "HBO West": null, + "HLN": null, + "Justice Central": null, + "Justice Central TV": null, + "Justice Central.TV": null, + "JusticeCentral.TV": null, + "KCBS": "KCBS", + "KING 5": null, + "KOMO News": "KOMO", + "MTV": null, + "NHL Network": null, + "NewsNation": null, + "Paramount+": null, + "SEC Network": null, + "TNT UHD RAW": null, + "TUDN": null, + "Three Angels Broadcasting Network": null, + "True Crime Network": null, + "UK: BBC One": null, + "UK| ITV 1": null, + "US: USA Network HD": null, + "US| ESPN": null, + "WABC-TV": "WABC-TV", + "WAVE 3": null, + "WHO 13": null, + "WOOD TV8": null, + "[US] CNN": null, + "beИN SPORTS": null, + "getTV": null, + "Россия 1": null, + "┃US┃ ESPN": null, + "★ CNN ★": null, + "🅴🆂🅿🅵": null + }, + "normalize_callsign": { + "(D1) CBS": "(D1) CBS", + "(PRIME) FOX News": "(PRIME) FOX News", + "BBC Four": "BBC Four", + "BBC News": "BBC News", + "BBC Three": "BBC Three", + "Cinemax HD": "Cinemax HD", + "DangerTV": "DangerTV", + "Discovery Channel": "Discovery Channel", + "Discovery Channel 4K": "Discovery Channel 4K", + "Discovery+": "Discovery+", + "Disney Channel": "Disney Channel", + "Disney+": "Disney+", + "ESPN 2": "ESPN 2", + "ESPN Pacific": "ESPN Pacific", + "ESPN [FHD]": "ESPN [FHD]", + "Fox Sports West": "Fox Sports West", + "France 2": "France 2", + "HBO (W)": "HBO (W)", + "HBO 2": "HBO 2", + "HBO East": "HBO East", + "HBO HD": "HBO HD", + "HBO West": "HBO West", + "HLN": "HLN", + "Justice Central": "Justice Central", + "Justice Central TV": "Justice Central TV", + "Justice Central.TV": "Justice Central.TV", + "JusticeCentral.TV": "JusticeCentral.TV", + "KCBS": "KCBS", + "KING 5": "KING 5", + "KOMO News": "KOMO News", + "MTV": "MTV", + "NHL Network": "NHL Network", + "NewsNation": "NewsNation", + "Paramount+": "Paramount+", + "SEC Network": "SEC Network", + "TNT UHD RAW": "TNT UHD RAW", + "TUDN": "TUDN", + "Three Angels Broadcasting Network": "Three Angels Broadcasting Network", + "True Crime Network": "True Crime Network", + "UK: BBC One": "UK: BBC One", + "UK| ITV 1": "UK| ITV 1", + "US: USA Network HD": "US: USA Network HD", + "US| ESPN": "US| ESPN", + "WABC-TV": "WABC", + "WAVE 3": "WAVE 3", + "WHO 13": "WHO 13", + "WOOD TV8": "WOOD TV8", + "[US] CNN": "[US] CNN", + "beИN SPORTS": "beИN SPORTS", + "getTV": "getTV", + "Россия 1": "Россия 1", + "┃US┃ ESPN": "┃US┃ ESPN", + "★ CNN ★": "★ CNN ★", + "🅴🆂🅿🅵": "🅴🆂🅿🅵" + }, + "normalize_name": { + "all_on": { + "(D1) CBS": "CBS", + "(PRIME) FOX News": "FOX News", + "BBC Four": "BBC 4", + "BBC News": "BBC News", + "BBC Three": "BBC 3", + "Cinemax HD": "Cinemax", + "DangerTV": "Danger", + "Discovery Channel": "Discovery", + "Discovery Channel 4K": "Discovery", + "Discovery+": "Discovery+", + "Disney Channel": "Disney", + "Disney+": "Disney+", + "ESPN 2": "ESPN 2", + "ESPN Pacific": "ESPN", + "ESPN [FHD]": "ESPN", + "Fox Sports West": "Fox Sports", + "France 2": "France 2", + "HBO (W)": "HBO", + "HBO 2": "HBO 2", + "HBO East": "HBO", + "HBO HD": "HBO", + "HBO West": "HBO", + "HLN": "HLN", + "Justice Central": "Justice", + "Justice Central TV": "Justice", + "Justice Central.TV": "Justice", + "JusticeCentral.TV": "Justice", + "KCBS": "KCBS", + "KING 5": "KING 5", + "KOMO News": "KOMO News", + "MTV": "MTV", + "NHL Network": "NHL", + "NewsNation": "News Nation", + "Paramount+": "Paramount+", + "SEC Network": "SEC", + "TNT UHD RAW": "TNTRAW", + "TUDN": "TUDN", + "Three Angels Broadcasting Network": "3 Angels Broadcasting", + "True Crime Network": "True Crime", + "UK: BBC One": "BBC 1", + "UK| ITV 1": "ITV 1", + "US: USA Network HD": "USA", + "US| ESPN": "ESPN", + "WABC-TV": "WABC", + "WAVE 3": "WAVE 3", + "WHO 13": "WHO 13", + "WOOD TV8": "WOOD TV 8", + "[US] CNN": "CNN", + "beИN SPORTS": "beИN SPORTS", + "getTV": "getTV", + "Россия 1": "Россия 1", + "┃US┃ ESPN": "ESPN", + "★ CNN ★": "CNN", + "🅴🆂🅿🅵": "" + }, + "regional_off": { + "(D1) CBS": "CBS", + "(PRIME) FOX News": "FOX News", + "BBC Four": "BBC 4", + "BBC News": "BBC News", + "BBC Three": "BBC 3", + "Cinemax HD": "Cinemax", + "DangerTV": "Danger", + "Discovery Channel": "Discovery", + "Discovery Channel 4K": "Discovery", + "Discovery+": "Discovery+", + "Disney Channel": "Disney", + "Disney+": "Disney+", + "ESPN 2": "ESPN 2", + "ESPN Pacific": "ESPN Pacific", + "ESPN [FHD]": "ESPN", + "Fox Sports West": "Fox Sports West", + "France 2": "France 2", + "HBO (W)": "HBO (W)", + "HBO 2": "HBO 2", + "HBO East": "HBO East", + "HBO HD": "HBO", + "HBO West": "HBO West", + "HLN": "HLN", + "Justice Central": "Justice Central", + "Justice Central TV": "Justice Central", + "Justice Central.TV": "Justice Central", + "JusticeCentral.TV": "Justice Central", + "KCBS": "KCBS", + "KING 5": "KING 5", + "KOMO News": "KOMO News", + "MTV": "MTV", + "NHL Network": "NHL", + "NewsNation": "News Nation", + "Paramount+": "Paramount+", + "SEC Network": "SEC", + "TNT UHD RAW": "TNTRAW", + "TUDN": "TUDN", + "Three Angels Broadcasting Network": "3 Angels Broadcasting", + "True Crime Network": "True Crime", + "UK: BBC One": "BBC 1", + "UK| ITV 1": "ITV 1", + "US: USA Network HD": "USA", + "US| ESPN": "ESPN", + "WABC-TV": "WABC", + "WAVE 3": "WAVE 3", + "WHO 13": "WHO 13", + "WOOD TV8": "WOOD TV 8", + "[US] CNN": "CNN", + "beИN SPORTS": "beИN SPORTS", + "getTV": "getTV", + "Россия 1": "Россия 1", + "┃US┃ ESPN": "ESPN", + "★ CNN ★": "CNN", + "🅴🆂🅿🅵": "" + } + }, + "process_string": { + "(D1) CBS": "1 cbs d", + "(PRIME) FOX News": "fox news prime", + "BBC Four": "bbc four", + "BBC News": "bbc news", + "BBC Three": "bbc three", + "Cinemax HD": "cinemax hd", + "DangerTV": "dangertv", + "Discovery Channel": "channel discovery", + "Discovery Channel 4K": "4k channel discovery", + "Discovery+": "discovery+", + "Disney Channel": "channel disney", + "Disney+": "disney+", + "ESPN 2": "2 espn", + "ESPN Pacific": "espn pacific", + "ESPN [FHD]": "espn fhd", + "Fox Sports West": "fox sports west", + "France 2": "2 france", + "HBO (W)": "hbo w", + "HBO 2": "2 hbo", + "HBO East": "east hbo", + "HBO HD": "hbo hd", + "HBO West": "hbo west", + "HLN": "hln", + "Justice Central": "central justice", + "Justice Central TV": "central justice tv", + "Justice Central.TV": "central justice tv", + "JusticeCentral.TV": "justicecentral tv", + "KCBS": "kcbs", + "KING 5": "5 king", + "KOMO News": "komo news", + "MTV": "mtv", + "NHL Network": "network nhl", + "NewsNation": "newsnation", + "Paramount+": "paramount+", + "SEC Network": "network sec", + "TNT UHD RAW": "raw tnt uhd", + "TUDN": "tudn", + "Three Angels Broadcasting Network": "angels broadcasting network three", + "True Crime Network": "crime network true", + "UK: BBC One": "bbc one uk", + "UK| ITV 1": "1 itv uk", + "US: USA Network HD": "hd network us usa", + "US| ESPN": "espn us", + "WABC-TV": "tv wabc", + "WAVE 3": "3 wave", + "WHO 13": "13 who", + "WOOD TV8": "8 tv wood", + "[US] CNN": "cnn us", + "beИN SPORTS": "beиn sports", + "getTV": "gettv", + "Россия 1": "1 россия", + "┃US┃ ESPN": "espn us", + "★ CNN ★": "cnn", + "🅴🆂🅿🅵": "" + } +} \ No newline at end of file diff --git a/.github/scripts/sync_core.py b/.github/scripts/sync_core.py new file mode 100644 index 0000000..7f11e10 --- /dev/null +++ b/.github/scripts/sync_core.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Vendor the shared matching core into Lineuparr's inner folder (local dev tool). + +The source of truth is /_shared/matching_core.py. This re-vendors it into the +deployable inner folder and (re)writes .github/scripts/core_manifest.json with its sha256. +It self-locates assuming the layout: + /_shared/matching_core.py + /Lineuparr/.github/scripts/sync_core.py (this file) + /Lineuparr/Lineuparr/ (flat inner = deploy artifact) + +Modes: + (default) write: copy _shared/matching_core.py into the inner folder and rewrite the + manifest with its sha256 hash. + --check verify the vendored copy is byte-identical to _shared (the "forgot to sync" + gate). Needs _shared present, so it runs LOCALLY only, never in GitHub CI. + --dry-run print what write would do without touching anything. + +The per-plugin CI gate is SEPARATE and needs no _shared: .github/scripts/check_core_parity.py +asserts sha256(inner/matching_core.py) == core_manifest.json. Keep the vendored file OUT of +any bump_version lockstep stamping, or the hash gate fails forever. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import sys +from pathlib import Path + +SHARED_FILES = ["matching_core.py"] + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def locate(): + """Return (inner_dir, shared_dir, manifest_path).""" + here = Path(__file__).resolve() + repo_root = here.parents[2] # .github/scripts/sync_core.py -> / + workspace = repo_root.parent # / + shared_dir = workspace / "_shared" + inner_dir = repo_root / repo_root.name # /Lineuparr/ (flat inner folder) + manifest_path = here.parent / "core_manifest.json" + return inner_dir, shared_dir, manifest_path + + +def do_write(inner_dir: Path, shared_dir: Path, manifest_path: Path, dry_run: bool) -> int: + manifest = {} + for fname in SHARED_FILES: + src = shared_dir / fname + if not src.exists(): + print(f"ERROR: shared source missing: {src}") + return 1 + digest = file_sha256(src) + manifest[fname] = digest + dst = inner_dir / fname + if dry_run: + print(f" would copy {src} -> {dst} ({digest[:12]}...)") + else: + shutil.copyfile(src, dst) + print(f" vendored {fname} ({digest[:12]}...)") + if dry_run: + print(f" would write manifest {manifest_path}") + return 0 + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f" wrote manifest {manifest_path}") + return 0 + + +def do_check(inner_dir: Path, shared_dir: Path) -> int: + drift = 0 + for fname in SHARED_FILES: + src, dst = shared_dir / fname, inner_dir / fname + if not src.exists(): + print(f"ERROR: shared source missing: {src}") + return 1 + if not dst.exists(): + print(f"DRIFT: vendored copy missing: {dst} (run sync_core.py)") + drift += 1 + continue + if file_sha256(src) != file_sha256(dst): + print(f"DRIFT: {fname} vendored copy differs from _shared (run sync_core.py)") + drift += 1 + if drift: + print(f"sync check FAILED: {drift} file(s) out of sync") + return 1 + print("sync check OK: vendored core matches _shared") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description="Vendor the shared matching core into Lineuparr.") + ap.add_argument("--check", action="store_true", help="verify vendored == _shared, don't write") + ap.add_argument("--dry-run", action="store_true", help="show what write would do") + args = ap.parse_args() + + inner_dir, shared_dir, manifest_path = locate() + if args.check: + return do_check(inner_dir, shared_dir) + return do_write(inner_dir, shared_dir, manifest_path, dry_run=args.dry_run) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 0901642..1b5ed5a 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1,9 +1,9 @@ name: Validate plugin # Static validation of the committed plugin runtime (compile, lineup JSON, -# manifest, em-dash ban). Catches a shipped-broken file before a user installs -# from GitHub. The full matcher regression suite is intentionally kept local -# and is not run here. +# manifest, em-dash ban) plus the matcher golden drift gate. Catches a +# shipped-broken file before a user installs from GitHub. The full matcher +# regression suite is intentionally kept local and is not run here. on: push: @@ -12,12 +12,20 @@ on: - "Lineuparr/**" - "README.md" - ".github/scripts/validate_plugin.py" + - ".github/scripts/check_matcher_golden.py" + - ".github/scripts/matcher_golden_baseline.json" + - ".github/scripts/check_core_parity.py" + - ".github/scripts/core_manifest.json" - ".github/workflows/validate.yml" pull_request: paths: - "Lineuparr/**" - "README.md" - ".github/scripts/validate_plugin.py" + - ".github/scripts/check_matcher_golden.py" + - ".github/scripts/matcher_golden_baseline.json" + - ".github/scripts/check_core_parity.py" + - ".github/scripts/core_manifest.json" - ".github/workflows/validate.yml" workflow_dispatch: @@ -31,3 +39,15 @@ jobs: python-version: "3.12" - name: Validate plugin runtime run: python .github/scripts/validate_plugin.py + # The vendored shared matching core must byte-compile and stay byte-identical to its + # pinned hash (catches a CRLF checkout or a stray hand-edit to the vendored copy). + - name: Vendored-core compile + parity gate + run: | + python -m py_compile Lineuparr/matching_core.py + python .github/scripts/check_core_parity.py + # The matcher golden gate needs rapidfuzz to reproduce the production + # similarity path that the committed baseline was generated against. + - name: Install matcher-check deps + run: pip install rapidfuzz + - name: Matcher golden drift gate + run: python .github/scripts/check_matcher_golden.py diff --git a/Lineuparr/fuzzy_matcher.py b/Lineuparr/fuzzy_matcher.py index 97fc8b6..54e1599 100644 --- a/Lineuparr/fuzzy_matcher.py +++ b/Lineuparr/fuzzy_matcher.py @@ -10,14 +10,24 @@ import logging import unicodedata -# Optional C-accelerated Levenshtein. When present, the matcher uses rapidfuzz's -# normalized_similarity (1 - distance/max(len)); the pure-Python fallback below -# computes the identical value (bug-026). rapidfuzz is an OPTIONAL runtime dep. +# The pure matching primitives (normalize_name, calculate_similarity, the callsign +# ladder, ...) live in the vendored shared core. The rapidfuzz fast path and its +# pure-Python fallback are inside FuzzyMatcherCore.calculate_similarity. The decorative +# helpers are re-exported so tests/callers that reference them keep working. try: - from rapidfuzz.distance import Levenshtein as _rf_lev - _USE_RAPIDFUZZ = True -except ImportError: - _USE_RAPIDFUZZ = False + from .matching_core import ( + FuzzyMatcherCore, + _is_decorative_char, + _normalize_emoji, + _strip_stylized_tokens, + ) +except ImportError: # script/test context without the package parent on sys.path + from matching_core import ( + FuzzyMatcherCore, + _is_decorative_char, + _normalize_emoji, + _strip_stylized_tokens, + ) __version__ = "1.3.4" @@ -28,48 +38,12 @@ LOGGER.addHandler(_handler) LOGGER.setLevel(logging.DEBUG) -# --- Pattern categories for normalization --- - -# Unicode categories considered decorative/badge characters by IPTV providers. -# So = Other Symbol (◉), No = Other Number (², ³), Lm = Modifier Letter -# (ᴿᴬᵂ, ᴴᴰ, ⱽᴵᴾ superscripts), Sk = Modifier Symbol. -# Accented letters (é, î, ü…) are Ll/Lu and are NOT in this set. -# Sm (Math Symbol) is intentionally EXCLUDED: it contains "+", which is a -# meaningful, channel-distinguishing character (Canal+, Three Stooges+, -# Comedy Central+). Stripping it regresses those matches. -_DECORATOR_CATS = frozenset({'So', 'No', 'Lm', 'Sk'}) - -# Tokens that are non-distinctive stream-label variants (e.g. "ABC News Live" -# should still match "ABC News"). Used by the subset/divergent guards. -_NON_DISTINCTIVE_TOKENS = frozenset({"live", "now", "new"}) - -def _is_distinctive(t): - """Return True if token t is distinctive enough to matter in subset/divergent guards.""" - return t not in _NON_DISTINCTIVE_TOKENS and (len(t) >= 4 or (t.isdigit() and len(t) >= 2)) # Matches "+1" / "+2" time-shift suffixes in the ORIGINAL (pre-normalization) # channel name. Must be checked before normalization because "+" is in the Sm # category and gets stripped, making "+1" indistinguishable from "1" afterward. _PLUS_SHIFT_RE = re.compile(r'\+\s{0,2}\d{1,2}\b') -QUALITY_PATTERNS = [ - r'\s*\[(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\]\s*', - r'\s*\((4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\)\s*', - r'^\s*(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s*', - r'\s*\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)$', - r'\s+\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s+', -] - -# Numeric resolution markers the keyword QUALITY_PATTERNS miss: 720p, 1080p/i, 2160p, -# 3840P, 480p, etc. — a 3-4 digit run glued directly to p/i. The 3-digit lower bound -# excludes 2-digit noise; the 4-digit upper bound excludes 5-digit numbers (10800p won't -# match). The p/i must be GLUED to the digits (no space): real markers are always written -# "720P"/"3840P", and requiring the glue avoids stripping a spaced standalone P/I such as a -# roman numeral ("Volume 100 I"). The p/i \b anchor keeps bare numbers (1080, "Channel 4") -# intact. Applied with re.IGNORECASE in the ignore_quality block, like QUALITY_PATTERNS. -RESOLUTION_PATTERNS = [ - r'\b\d{3,4}[pi]\b', -] # Quality markers that distinguish an upgrade-tier channel from its standard twin. # HD/FHD/HEVC are intentionally excluded - they don't create a separate twin channel. @@ -81,19 +55,6 @@ def has_upgrade_quality(name: str) -> bool: return bool(_UPGRADE_QUALITY_RE.search(name)) -REGIONAL_PATTERNS = [ - # East/West are intentionally NOT stripped - they distinguish separate channel feeds - # (e.g., "HBO East" and "HBO West" are different channels) - r'\s[Pp][Aa][Cc][Ii][Ff][Ii][Cc]', - r'\s[Cc][Ee][Nn][Tt][Rr][Aa][Ll]', - r'\s[Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]', - r'\s[Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]', - r'\s*\([Pp][Aa][Cc][Ii][Ff][Ii][Cc]\)\s*', - r'\s*\([Cc][Ee][Nn][Tt][Rr][Aa][Ll]\)\s*', - r'\s*\([Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]\)\s*', - r'\s*\([Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]\)\s*', -] - # Country tokens for the delimited provider-prefix patterns below; curated so a # bare delimited word ("(SPORTS)") isn't misread as a country. Keep in sync with # detect_stream_country(). @@ -114,55 +75,6 @@ def _balanced_delim(token): # Leading country tag in a matched delimiter pair; one capture group fires. _BRACKETED_CC_RE = re.compile(r'^\s*' + _balanced_delim(r'([A-Za-z]{2,3})')) -# Strip a leading box-bar bouquet/source tag with arbitrary inner text -# ("┃CANAL+┃ NPO 1" -> "NPO 1"); box bars never occur in real names, so this is -# always safe and also covers leading "┃XX┃" country tags. detect_stream_country -# reads the raw name, so detection is unaffected. -_LEADING_BAR_TAG_RE = re.compile(r'^\s*[┃│]\s*[^┃│]*[┃│]\s*') - -GEOGRAPHIC_PATTERNS = [ - r'\b[A-Z]{2,3}[:┃│]\s*', - r'\b[A-Z]{2,3}\s*-\s*', - # Matched bar pair only ("|US|", "┃US┃") - a mismatched "|US┃" is noise. - r'(?:\|[A-Z]{2,3}\||┃[A-Z]{2,3}┃|│[A-Z]{2,3}│)\s*', - r'\[[A-Z]{2,3}\]\s*', -] - -# Enhanced provider prefix patterns for IPTV-specific naming -PROVIDER_PREFIX_PATTERNS = [ - r'^(?:' + _PREFIX_COUNTRY + r')\s*[:\-\|┃│]\s*', - # Bare country tag + whitespace, no separator (e.g. "US Racer Network", - # "FR beIN SPORTS MAX", "MEX Bein Sports"). Restricted to a curated set so - # it cannot eat a real channel name: "USA Network" (USA != US + space), - # "In Country Television" ("IN") and "IT Crowd" ("IT") are all safe too. - # Keep this set in sync with detect_stream_country()'s bare-space branch. - r'^(?:US|UK|CA|AU|FR|DE|MX|MEX|FRA|GER)\s+', - # "USA " space prefix as a US country tag ("USA ABC", "USA BET"). A - # negative lookahead for NETWORK protects the real channel "USA Network" - # (these feeds tag that one as "US ..."/"US: ...", never bare "USA "). - # Keep in sync with detect_stream_country()'s USA branch. - r'^USA\s+(?!NETWORK\b)', - # Country code glued directly to a quality tag with no separator - # ("UKSD: Sky Sports", "UKHD ESPN", "USFHD ..."). Detection mirrors this - # in detect_stream_country() so these can't leak as wildcards. - r'^(?:US|UK)(?:SD|HD|FHD|UHD|FD|HEVC|4K|8K)\b\s*[:\-\|┃│]?\s*', - # Bracketed/piped country tag with a MATCHED delimiter pair ("(US)", "│US│"). - r'^\s*' + _balanced_delim(_PREFIX_COUNTRY) + r'\s*', - r'\s*[\|┃│]\s*(?:' + _PREFIX_COUNTRY + r')\s*$', - # Content-category group prefixes used by some IPTV providers. - r'^(?:ADULT|EROTIC|PRIME|GOLD)\s*[:\-\|┃│]\s*', - # FAST streaming-platform source tags (Roku, Tubi, Pluto, etc.). These mark - # the distribution platform, not the channel or its country, so strip them - # for matching ("RK: beIN Sports Xtra" -> "beIN Sports Xtra"). A separator - # is required so this can't eat real names like "GOLF" or "PLEX TV Movies". - # NOT a country signal: detect_stream_country() ignores these (correctly, - # since a platform like Roku spans US/CA/UK). - r'^(?:RK|GO|TUBI|PLUTO|XUMO|PLEX|STIRR|FREEVEE|GLANCE)\s*[:\-\|┃│]\s*', -] - -MISC_PATTERNS = [ - r'\s*\([^)]*\)\s*', -] # ISO-2 country codes Lineuparr lineup filenames use (keep in sync with # PROVIDER_PREFIX_PATTERNS above and PluginConfig.COUNTRY_DIR_MAP). @@ -378,116 +290,23 @@ def country_codes_in_text(text): return codes -# --------------------------------------------------------------------------- # -# Stylized-Unicode decoration stripping -# --------------------------------------------------------------------------- # -# Streams tag names with stylized-Unicode tier/format markers (superscript -# "WEATHERNATION RAW", small-cap "FHD", bullet-prefixed "CNN") that the ASCII tag -# regexes below cannot see. We drop whole tokens that are pure decoration BEFORE -# the ASCII pipeline runs. Detection is by Unicode character *name* (not code-point -# ranges), so it covers superscripts, "modifier letter" superscript capitals, and -# Latin small-caps wherever they live (e.g. small-cap H is U+029C in IPA Extensions -# and modifier V is U+2C7D in Latin-Ext-C, both outside the obvious blocks). - -# Ornament glyphs whose Unicode name carries no decoration keyword. -_DECORATIVE_SYMBOLS = frozenset("◉") # FISHEYE; add individual chars (not strings) here - - -def _is_decorative_char(ch): - """True for a stylized letterform/ornament that carries no semantic content in a - channel name (superscripts, subscripts, modifier-letter superscript capitals, - Latin small-capitals, curated bullets). ASCII and ordinary letters return False.""" - if ch.isascii(): - return False - if ch in _DECORATIVE_SYMBOLS: - return True - try: - nm = unicodedata.name(ch) - except ValueError: - # unnamed code point (control char / lone surrogate) -> not decoration - return False - return ('SUPERSCRIPT' in nm or 'SUBSCRIPT' in nm - or 'SMALL CAPITAL' in nm or 'MODIFIER LETTER' in nm) - - -def _strip_stylized_tokens(name): - """Drop whitespace tokens that are pure stylized decoration, then NFKD-canonicalize - the remainder. A token is decoration when it has >=1 decorative char, no ASCII - alphanumeric, and every char is decorative or ASCII punctuation (so a bullet glued - to a colon, or "HD/RAW" written in superscripts, are dropped too). Real ASCII words - (Gold/VIP) and non-Latin letters (Arabic/Cyrillic/CJK) are always kept. ASCII-only - input is returned unchanged via the fast path (no per-char work; NFKD is a no-op - on ASCII, so skipping it changes nothing).""" - if name.isascii(): - return name - kept = [] - for tok in name.split(): - has_decorative = any(_is_decorative_char(c) for c in tok) - has_ascii_alnum = any(c.isascii() and c.isalnum() for c in tok) - only_decorative_or_punct = all( - _is_decorative_char(c) or (c.isascii() and not c.isalnum()) for c in tok - ) - if has_decorative and only_decorative_or_punct and not has_ascii_alnum: - continue # pure decoration -> drop the whole token - kept.append(tok) - return unicodedata.normalize('NFKD', ' '.join(kept)) - - -# --------------------------------------------------------------------------- # -# Emoji-as-letter + emoji decoration normalization -# --------------------------------------------------------------------------- # -# Some streams use an emoji AS A LETTER inside a word: "SP⚽RTS" / "Sp⚽rts" where the -# soccer ball stands in for 'o' (= SPORTS, the beIN family). _strip_stylized_tokens keeps -# the token (it has ASCII alnum) and process_string_for_matching would turn the ball into a -# space ("sp rts"), so it never matches "sports". We substitute the glyph for the letter it -# replaces (only when flanked by ASCII letters) and strip emoji used purely as decoration. - -# Emoji that visually replace an ASCII letter when embedded in a word. Extensible. -_EMOJI_LETTER_MAP = {'⚽': 'o'} # SOCCER BALL = 'o' (SP⚽RTS -> SPORTS) -# Pictographic ornaments to delete. NOTE: ⚽ is intentionally in BOTH maps — the letter -# map handles it mid-word (-> 'o'); here it catches any ⚽ NOT flanked by ASCII letters -# (standalone/edge), which the substitution above leaves untouched. -_EMOJI_ORNAMENTS = frozenset('♬☾⚽') # beamed notes, last-quarter moon, soccer ball -# Zero-width / invisible code points that only add noise to a name. -_ZERO_WIDTH = ('️', '‍') # VARIATION SELECTOR-16, ZERO WIDTH JOINER - - -def _normalize_emoji(name): - """Map emoji-as-letters to their letter and strip emoji decoration. - - The letter substitution fires ONLY when the glyph is flanked by ASCII letters - (so "SP⚽RTS" -> "SPoRTS" but a standalone/edge "⚽" is treated as decoration and - dropped). Zero-width selectors and ornament pictographs are deleted outright. - ASCII-only input is returned unchanged (no emoji possible).""" - if name.isascii(): - return name - for zw in _ZERO_WIDTH: - if zw in name: - name = name.replace(zw, '') - for glyph, letter in _EMOJI_LETTER_MAP.items(): - if glyph in name: - name = re.sub(r'(?<=[A-Za-z])' + re.escape(glyph) + r'(?=[A-Za-z])', letter, name) - if any(c in _EMOJI_ORNAMENTS for c in name): - name = ''.join(c for c in name if c not in _EMOJI_ORNAMENTS) - return name - - -class FuzzyMatcher: +class FuzzyMatcher(FuzzyMatcherCore): """Handles fuzzy matching for Lineuparr with alias support and channel number boosting.""" + # Lineuparr strips bare time-zone region words (Pacific/Central/Mountain/Atlantic) for + # SCORING and enforces region correctness via its post-match region filter + # (match_all_streams reads the ORIGINAL names). See REGIONAL_BARE_PATTERNS in the core. + _STRIP_BARE_REGION = True + def __init__(self, match_threshold=80, logger=None): - self.match_threshold = match_threshold - self.logger = logger or LOGGER - # Cache for pre-normalized stream names (performance optimization) - self._norm_cache = {} # raw_name -> normalized_lower - self._norm_nospace_cache = {} # raw_name -> normalized_nospace - self._processed_cache = {} # raw_name -> processed_for_matching + # The core seeds match_threshold, logger, the four normalization/callsign + # caches, and the known-callsign rescue slot (unused here - Lineuparr has no + # channel DB). Only the country-filter counter is Lineuparr-specific. + super().__init__(match_threshold=match_threshold, logger=logger or LOGGER) # Cumulative count of candidates dropped by the country filter across # match_all_streams calls. Callers reset this before a matching loop # and read it after, so they can log a summary. self.country_filter_drops = 0 - # Cache for callsign extraction: raw_name -> (callsign|None, is_high_conf) - self._callsign_cache = {} def precompute_normalizations(self, names, user_ignored_tags=None): """ @@ -531,340 +350,6 @@ def _get_cached_processed(self, name, user_ignored_tags=None): return None return self.process_string_for_matching(norm) - def normalize_name(self, name, user_ignored_tags=None, ignore_quality=True, ignore_regional=True, - ignore_geographic=True, ignore_misc=True): - """ - Normalize channel or stream name for matching by removing tags, prefixes, and noise. - """ - if user_ignored_tags is None: - user_ignored_tags = [] - - original_name = name - - name = _LEADING_BAR_TAG_RE.sub('', name) # leading "┃CANAL+┃" bouquet tag - - # Map emoji-as-letters (⚽ = 'o' in "SP⚽RTS") and strip emoji decoration, before - # the stylized-Unicode strip and ASCII regexes below — so "beIN SP⚽RTS" -> "beIN sports". - name = _normalize_emoji(name) - - # Strip stylized-Unicode decoration (superscript/small-cap tier markers, - # bullets) up front so the ASCII tag regexes below see plain text. Runs - # unconditionally: a token written in superscript/small-caps is decoration - # regardless of tag_handling, and it would otherwise block matches - # (e.g. a superscript-RAW suffix never matches channel "WeatherNation"). - name = _strip_stylized_tokens(name) - - # Strip IPTV provider prefixes BEFORE hyphen normalization so that - # "FR - Canal+ FHD" loses its "FR - " while the hyphen is still a - # hyphen. After hyphen normalization the separator would become a space - # and the pattern would fail to match, leaving "FR" as a stray token. - for pattern in PROVIDER_PREFIX_PATTERNS: - name = re.sub(pattern, '', name, flags=re.IGNORECASE) - - # Quality patterns (before space normalization). Loop until stable so - # chained suffixes like "4K HDR" or "UHD HDR" are fully stripped in - # successive passes (each pass may expose a token for the next). - if ignore_quality: - prev = None - while prev != name: - prev = name - # Strip numeric resolution markers (3840P/2160P/1080P/720P/...) before the - # digit/letter spacer below would split "3840P" into "3840 P". - # Must run before QUALITY_PATTERNS so that removing " 4K " does not glue - # "SPoRTS" to "3840P" and break the word-boundary anchor. - for pattern in RESOLUTION_PATTERNS: - name = re.sub(pattern, '', name, flags=re.IGNORECASE) - for pattern in QUALITY_PATTERNS: - name = re.sub(pattern, '', name, flags=re.IGNORECASE) - - # Normalize spacing around numbers - name = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', name) - name = re.sub(r'(\d)([a-zA-Z])', r'\1 \2', name) - - # Normalize hyphens to spaces - name = re.sub(r'-', ' ', name) - - # Replace dots between word chars with spaces (e.g. "JusticeCentral.TV" - # → "JusticeCentral TV"). Keeps the dot-suffix variant equivalent to - # the spaced form for matching purposes. - name = re.sub(r'(?<=\w)\.(?=\w)', ' ', name) - - # Normalize number-words to digits so "BBC Three" and "BBC 3" share - # tokens. Critical for cases like "Three Angels Broadcasting Network" - # vs "3 Angels Broadcasting Network", and for BBC One/Two/Three/Four - # vs BBC 1/2/3/4. Bounded by word boundaries so brand names with - # embedded letters (e.g. "Onesimus") aren't corrupted. - _NUM_WORDS = { - "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", - "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10", - "eleven": "11", "twelve": "12", - } - def _num_repl(m): - return _NUM_WORDS[m.group(0).lower()] - name = re.sub( - r'\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\b', - _num_repl, name, flags=re.IGNORECASE, - ) - - # Split CamelCase boundaries so "JusticeCentral" becomes "Justice - # Central" and "DangerTV" becomes "Danger TV". Two separate patterns: - # 1. lower → Upper followed by lower (Justice|Central, Danger|Iq → no, etc.) - # 2. lower(4+) → UPPER acronym at word boundary (Danger|TV, Beauty|IQ) - # The 4-char floor on rule 2 protects short brand names like "MeTV" and - # "truTV" whose existing EPG matches rely on the un-split form. - name = re.sub(r'([a-z])([A-Z][a-z])', r'\1 \2', name) - name = re.sub(r'([a-z]{4,})([A-Z]{2,})\b', r'\1 \2', name) - - # Strip region tokens (East / Eastern / West and the (E)/(W) - # abbreviations) for SCORING. Region CORRECTNESS - East vs West vs - # Pacific - is enforced separately by the post-match region filter in - # match_all_streams, which reads the ORIGINAL un-normalized names, not - # this output. Stripping here lets a regionless lineup channel - # ("Food Network") still score-match a region-tagged stream - # ("Food Network Eastern" / "... East") instead of being dragged below - # threshold by the extra token; the filter then accepts it (regionless - # defaults to East). "Western"/"Westerns" is a movie genre, NOT a - # region, so bare "west" is stripped only on a word boundary (\bwest\b - # does not match inside "western") and "western" is never touched. - name = re.sub(r'\(\s*(?:eastern|east|e|west|w)\s*\)', ' ', name, flags=re.IGNORECASE) - name = re.sub(r'\b(?:eastern|east|west)\b', ' ', name, flags=re.IGNORECASE) - - # Remove leading parenthetical prefixes - while name.lstrip().startswith('('): - new_name = re.sub(r'^\s*\([^\)]+\)\s*', '', name) - if new_name == name: - break - name = new_name - - # Build pattern list based on flags - patterns_to_apply = [] - if ignore_regional: - patterns_to_apply.extend(REGIONAL_PATTERNS) - if ignore_geographic: - patterns_to_apply.extend(GEOGRAPHIC_PATTERNS) - if ignore_misc and ignore_regional: - patterns_to_apply.extend(MISC_PATTERNS) - - for pattern in patterns_to_apply: - name = re.sub(pattern, '', name, flags=re.IGNORECASE) - - # Apply user-configured ignored tags - for tag in user_ignored_tags: - escaped_tag = re.escape(tag) - if '[' in tag or ']' in tag or '(' in tag or ')' in tag: - name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) - else: - if re.match(r'^\w+$', tag): - name = re.sub(r'\b' + escaped_tag + r'\b', '', name, flags=re.IGNORECASE) - else: - name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) - - # Remove callsigns in parentheses - if ignore_regional: - name = re.sub(r'\([KW][A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) - else: - name = re.sub(r'\([KW](?!EST\)|ACIFIC\)|ENTRAL\)|OUNTAIN\)|TLANTIC\))[A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) - - if ignore_regional: - name = re.sub(r'\([A-Z0-9]+\)', '', name) - - # Remove common suffixes/prefixes - name = re.sub(r'^The\s+', '', name, flags=re.IGNORECASE) - name = re.sub(r'\s+Network\s*$', '', name, flags=re.IGNORECASE) - name = re.sub(r'\s+Channel\s*$', '', name, flags=re.IGNORECASE) - name = re.sub(r'\s+TV\s*$', '', name, flags=re.IGNORECASE) - - # Strip decorative Unicode markers (◉, ², superscript letters ᴿᴬᵂ…) - # that some IPTV providers append as quality/status badges. Only - # characters in decorator categories are removed; accented letters - # (é, î, ü…) are in Ll/Lu and are preserved. - name = ''.join( - ' ' if unicodedata.category(c) in _DECORATOR_CATS else c - for c in name - ) - - # Clean up whitespace - name = re.sub(r'\s+', ' ', name).strip() - - if not name: - self.logger.debug(f"normalize_name returned empty for: '{original_name}'") - - return name - - def calculate_similarity(self, str1, str2, min_ratio=0.0): - """Levenshtein similarity ratio (0.0-1.0), defined as 1 - distance/max(len). - If min_ratio > 0, returns 0.0 early when the result can't reach it. - - bug-026: the ratio MUST be distance/max(len), matching rapidfuzz - Levenshtein.normalized_similarity. The old (len1+len2-distance)/(len1+len2) - formula scored higher for the same edit distance and let numbered siblings - ("Fox Sports 1" vs "2") pass threshold 95. The rapidfuzz fast path and the - pure-Python fallback below compute the identical value. - """ - if len(str1) == 0 or len(str2) == 0: - return 0.0 - - # Fast path: C-accelerated rapidfuzz when available (same definition). - if _USE_RAPIDFUZZ: - # No score_cutoff: rapidfuzz's cutoff rounding zeroes a score landing - # exactly on min_ratio, but the pure-Python path returns it. Dropping - # the cutoff makes the two paths agree at the threshold boundary; - # callers apply their own >= comparison. - return _rf_lev.normalized_similarity(str1, str2) - - if len(str1) < len(str2): - str1, str2 = str2, str1 - len1, len2 = len(str1), len(str2) - max_len = len1 # the longer string after the swap - - # Length-difference pre-check: minimum possible distance is (len1 - len2), - # so the max possible ratio is (max_len - (len1 - len2)) / max_len. - if min_ratio > 0: - max_possible = (max_len - (len1 - len2)) / max_len - if max_possible < min_ratio: - return 0.0 - - previous_row = list(range(len2 + 1)) - for i, c1 in enumerate(str1): - current_row = [i + 1] - for j, c2 in enumerate(str2): - insertions = previous_row[j + 1] + 1 - deletions = current_row[j] + 1 - substitutions = previous_row[j] + (c1 != c2) - current_row.append(min(insertions, deletions, substitutions)) - # Early termination: a lower bound on the final distance is the current - # row minimum minus the str1 chars still unprocessed. - if min_ratio > 0: - min_distance_so_far = min(current_row) - remaining = len1 - i - 1 - best_possible_distance = max(0, min_distance_so_far - remaining) - best_possible_ratio = (max_len - best_possible_distance) / max_len - if best_possible_ratio < min_ratio: - return 0.0 - previous_row = current_row - - distance = previous_row[-1] - return (max_len - distance) / max_len - - @staticmethod - def _length_scaled_threshold(base_threshold, shorter_len): - """Require higher similarity for shorter strings to avoid false positives.""" - if shorter_len <= 4: - return max(base_threshold, 95) - elif shorter_len <= 8: - return max(base_threshold, 90) - return base_threshold - - @staticmethod - def _has_token_overlap(str_a, str_b, min_token_len=4, require_majority=False): - """Check that distinctive tokens are shared between two strings. - - Basic mode: at least one token (>= min_token_len) must be shared. - Majority mode: uses all tokens (>= 2 chars) and requires that more than - half of the smaller set overlaps. Catches false positives like - "america racing" vs "america bbc" while allowing single-token matches. - """ - # "network"/"channel"/"television" are generic brand-suffix words, not - # distinctive - treat as common so the subset guard does not reject - # cases like "FanDuel Sports Cincinnati" vs "FanDuel Sports Network - # Cincinnati". (They're already stripped from end-of-string by - # normalize_name; this catches mid-string occurrences.) - common_words = { - "the", "and", "of", "in", "on", "at", "to", "for", "a", "an", - "network", "channel", "television", - } - - if require_majority: - # Use all meaningful tokens (>= 2 chars) for stricter checking. - # Single-digit tokens (1, 2, 3, ...) are kept because they're - # channel-distinguishing (BBC 1 vs BBC 2, ESPN 1 vs ESPN 2 etc.) - # even though they're only 1 char. - def _meaningful(t): - if t in common_words: - return False - return len(t) >= 2 or t.isdigit() - tokens_a = {t for t in str_a.split() if _meaningful(t)} - tokens_b = {t for t in str_b.split() if _meaningful(t)} - if not tokens_a or not tokens_b: - return True - shared = tokens_a & tokens_b - if not shared: - return False - smaller = min(len(tokens_a), len(tokens_b)) - if not len(shared) > smaller / 2: - return False - - unique_a = tokens_a - tokens_b - unique_b = tokens_b - tokens_a - - # Subset guard: when one side is a strict subset of the other and - # the larger side has a distinctive token the smaller lacks, the - # candidate is a more specific channel than the query and the high - # fuzzy score is a false positive. Catches e.g. "Nickelodeon" vs - # "Nickelodeon Teen". Threshold is >=4 chars; "live"/"now" are - # explicitly non-distinctive (stream label variants) so "ABC News" - # still matches "ABC News Live". Pure-digit tokens >=2 chars - # (e.g. "360") are also distinctive: "Canal+Sport" must not match - # "Canal+Sport 360". - if not unique_a: - if any(_is_distinctive(t) for t in unique_b): - return False - elif not unique_b: - if any(_is_distinctive(t) for t in unique_a): - return False - - # Divergent guard: when BOTH sides have unique tokens AND at least - # one of those unique tokens is a distinctive (>=4 char) word, the - # strings describe different brands and the fuzzy score is - # misleading. Catches "Sky Cinema Disney" vs "Sky Cinema Decades" - # (decades = 7 chars), "Sky Cinema Fast" vs "Sky Cinema Family", - # and "BBC One vs BBC Two" once number-words become digits earlier. - # Number-word→digit normalization (in normalize_name) reduces this - # guard's risk: "Three Angels" / "3 Angels" both become "3 angels" - # before reaching here, so they never trigger this case. - if unique_a and unique_b: - if any(len(t) >= 4 for t in unique_a | unique_b): - return False - - # Numeric/ordinal divergent guard: when BOTH sides have a numeric - # or ordinal token unique to them, they are sibling channels - # distinguished by number (BBC One vs BBC Two; ESPN 1 vs ESPN 2). - # Short tokens like "one"/"two" wouldn't trip the divergent guard - # above so they need their own check. - _NUMERIC = { - "one", "two", "three", "four", "five", "six", "seven", "eight", - "nine", "ten", "eleven", "twelve", - "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", - "first", "second", "third", "fourth", "fifth", - } - if (unique_a & _NUMERIC) and (unique_b & _NUMERIC): - return False - - return True - - # Basic mode: at least one long token shared - tokens_a = {t for t in str_a.split() if t not in common_words and len(t) >= min_token_len} - tokens_b = {t for t in str_b.split() if t not in common_words and len(t) >= min_token_len} - if not tokens_a or not tokens_b: - return True - return bool(tokens_a & tokens_b) - - def process_string_for_matching(self, s): - """Normalize for token-sort matching: NFKD-fold (so "ＨＢＯ"->"hbo", "²"->"2", - "fi"->"fi", accents drop), lowercase, keep alphanumerics of any script - (isalnum keeps Cyrillic/CJK/Arabic rather than erasing them), sort tokens.""" - s = unicodedata.normalize('NFKD', s) - s = ''.join(char for char in s if unicodedata.category(char) != 'Mn') - s = s.lower() - s = re.sub(r'([^\W\d_])(\d)', r'\1 \2', s) # split letter-glued digit, any script - cleaned_s = "" - for char in s: - if char.isalnum(): - cleaned_s += char - else: - cleaned_s += ' ' - tokens = sorted([token for token in cleaned_s.split() if token]) - return " ".join(tokens) @staticmethod def _is_group_header(name): @@ -875,15 +360,6 @@ def _is_group_header(name): is a common in-name separator ("001 | Team A vs Team B").""" return bool(re.search(r'[#=*]{2,}|\|{3,}', name or "")) - @staticmethod - def _trailing_number(name): - """Return the integer value of a space-separated, purely-numeric - trailing token, or None. 'HBO 2' -> 2, 'DIRECTV 4K Live 1' -> 1, - 'ESPN' -> None, 'ESPN2' -> None (digit not space-separated). Used to - reject 'Foo 1' vs 'Foo 2' false positives -- different channels that - otherwise fuzzy-match almost perfectly.""" - m = re.search(r'(?:^|\s)(\d{1,4})\s*$', name or "") - return int(m.group(1)) if m else None def _channel_number_boost(self, stream_name, expected_number): """ @@ -902,103 +378,6 @@ def _channel_number_boost(self, stream_name, expected_number): return 5 return 0 - # Words that match the callsign regex shape but are never US broadcast - # callsigns. A US callsign (K/W + 2-4 letters) is shape-identical to many - # common English words, so the loose Priority-4 pattern mis-extracts them - - # e.g. "with" in "Bizarre Foods with Andrew Zimmern" becomes callsign - # "WITH". Regex alone cannot tell "WITH" from "WABC"; frequent K/W-initial - # words are denied explicitly so they never extract as a callsign. WWE/WWF/ - # WCW stop wrestling show names extracting as false-positive callsigns. - _CALLSIGN_DENYLIST = frozenset({ - 'WWE', 'WWF', 'WCW', 'EAST', - 'WAR', 'WARS', 'WARM', 'WASH', 'WATCH', 'WAVE', 'WAVES', 'WAY', 'WAYS', - 'WEB', 'WEEK', 'WELL', 'WENT', 'WERE', 'WEST', 'WHAT', 'WHEN', 'WHERE', - 'WHICH', 'WHILE', 'WHITE', 'WHO', 'WHY', 'WIDE', 'WIFE', 'WILD', 'WILL', - 'WIND', 'WINE', 'WING', 'WINGS', 'WINS', 'WIRE', 'WISE', 'WISH', 'WITH', - 'WOLF', 'WOMAN', 'WOMEN', 'WOOD', 'WORD', 'WORDS', 'WORK', 'WORKS', - 'WORLD', 'WORM', 'WORN', 'WRAP', - 'KEEN', 'KEEP', 'KEPT', 'KEY', 'KEYS', 'KICK', 'KID', 'KIDS', 'KILL', - 'KIND', 'KING', 'KINGS', 'KISS', 'KITE', 'KNEE', 'KNEW', 'KNOW', 'KNOWN', - }) - - def _compute_callsign_with_confidence(self, channel_name): - """ - Extract US TV callsign with a confidence flag. - - Returns (callsign, is_high_confidence). High confidence = - Priorities 1-3 (parenthesized / suffixed-paren / end-of-name). - Priority 4 (any loose word) is low confidence. (None, False) - when nothing extractable. - """ - # Remove common provider prefixes - channel_name = re.sub(r'^D\d+-', '', channel_name) - channel_name = re.sub(r'^USA?\s*[^a-zA-Z0-9]*\s*', '', channel_name, flags=re.IGNORECASE) - - # Priority 1: Callsigns in parentheses (most reliable) - paren_match = re.search(r'\(([KW][A-Z]{3})(?:-[A-Z\s]+)?\)', channel_name, re.IGNORECASE) - if paren_match: - callsign = paren_match.group(1).upper() - if callsign not in self._CALLSIGN_DENYLIST: - return callsign, True - - # Priority 1b: grandfathered 3-letter callsigns in parentheses without a suffix - # (WWL/WJZ/KYW/WRC). Suffixed forms fall through to Priority 2. bug-062. - paren3_match = re.search(r'\(([KW][A-Z]{2})\)', channel_name, re.IGNORECASE) - if paren3_match: - callsign = paren3_match.group(1).upper() - if callsign not in self._CALLSIGN_DENYLIST: - return callsign, True - - # Priority 2: Callsigns with suffix in parentheses - paren_suffix_match = re.search(r'\(([KW][A-Z]{2,4}-(?:TV|CD|LP|DT|LD))\)', channel_name, re.IGNORECASE) - if paren_suffix_match: - return paren_suffix_match.group(1).upper(), True - - # Priority 3: Callsigns at the end - end_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\s*(?:\.[a-z]+)?\s*$', channel_name, re.IGNORECASE) - if end_match: - callsign = end_match.group(1).upper() - if callsign not in self._CALLSIGN_DENYLIST: - return callsign, True - - # Priority 4: Any word matching callsign pattern (low confidence) - word_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\b', channel_name, re.IGNORECASE) - if word_match: - callsign = word_match.group(1).upper() - if callsign not in self._CALLSIGN_DENYLIST: - return callsign, False - - return None, False - - def _extract_callsign_with_confidence(self, channel_name): - """ - Cached wrapper around _compute_callsign_with_confidence. - - Extraction is pure in channel_name, so results are memoized - the - anchor calls this once per stream over a fixed candidate list, which - is otherwise massively redundant across the per-channel matching - loop. Cache is cleared by precompute_normalizations. - """ - cached = self._callsign_cache.get(channel_name) - if cached is not None: - return cached - result = self._compute_callsign_with_confidence(channel_name) - self._callsign_cache[channel_name] = result - return result - - def extract_callsign(self, channel_name): - """ - Extract US TV callsign from channel name with priority order. - Returns None if common false positives appear alone. - """ - callsign, _ = self._extract_callsign_with_confidence(channel_name) - return callsign - - def normalize_callsign(self, callsign): - """Remove the broadcast suffix (-TV/-CD/-LP/-DT/-LD) from a callsign.""" - if callsign: - callsign = re.sub(r'-(?:TV|CD|LP|DT|LD)$', '', callsign) - return callsign def alias_match(self, lineup_name, candidate_names, alias_map, user_ignored_tags=None): """ diff --git a/Lineuparr/matching_core.py b/Lineuparr/matching_core.py new file mode 100644 index 0000000..d7e9147 --- /dev/null +++ b/Lineuparr/matching_core.py @@ -0,0 +1,793 @@ +""" +matching_core.py - shared pure matching primitives for the Dispatcharr plugins. + +The canonical, vendored core extracted from Lineuparr's fuzzy_matcher.py (v1.3.4, the +newest generation). It is PURE and stateless string-in/string-out: stdlib + optional +rapidfuzz only, with no Django / ORM / filesystem coupling. Each plugin SUBCLASSES +FuzzyMatcherCore and layers its own orchestration on top (DB load, matching entry points, +and one feature block: zones / token-index / country / OTA). See +MATCHER-STANDARDIZATION-PLAN.md. + +Authored faithfully from Lineuparr with exactly these deliberate changes: + - class renamed FuzzyMatcher -> FuzzyMatcherCore; Lineuparr's country layer, the + matching entry points (alias_match / fuzzy_match / match_all_streams), the cache glue + (precompute_normalizations / _get_cached_*), _channel_number_boost, _is_group_header, + and has_upgrade_quality are NOT included - they stay plugin-side. + - process_string_for_matching KEEPS '+' so Disney+ / Discovery+ / Paramount+ stay + distinct from their base channels (the 4-of-4 superset decision). + - the callsign confidence ladder consults an optional self._known_callsigns set so a + plugin can rescue a denylisted-but-real station (KING / WAVE / WOOD / WHO) from its own + channel DB. With the default (None) the ladder is identical to Lineuparr's pure path. + +This file is the SOURCE OF TRUTH. It is vendored (copied byte-identically) into each +plugin's flat inner folder at release time and hash-pinned; never edit a vendored copy - +edit this file and re-sync. +""" + +import logging +import re +import unicodedata + +# Optional C-accelerated Levenshtein. When present, the matcher uses rapidfuzz's +# normalized_similarity (1 - distance/max(len)); the pure-Python fallback below +# computes the identical value (bug-026). rapidfuzz is an OPTIONAL runtime dep. +try: + from rapidfuzz.distance import Levenshtein as _rf_lev + _USE_RAPIDFUZZ = True +except ImportError: + _USE_RAPIDFUZZ = False + +__version__ = "0.1.0" + +LOGGER = logging.getLogger("matching_core") +if not LOGGER.handlers: + _handler = logging.StreamHandler() + _handler.setFormatter(logging.Formatter("%(levelname)s %(name)s %(message)s")) + LOGGER.addHandler(_handler) +# WARNING by default so a vendored core does not spam each plugin's logs with the +# per-name "normalize_name returned empty" debug line; a plugin can pass its own +# logger at a lower level when diagnosing. +LOGGER.setLevel(logging.WARNING) + +# --- Pattern categories for normalization --- + +# Unicode categories considered decorative/badge characters by IPTV providers. +# So = Other Symbol (◉), No = Other Number (², ³), Lm = Modifier Letter +# (ᴿᴬᵂ, ᴴᴰ, ⱽᴵᴾ superscripts), Sk = Modifier Symbol. +# Accented letters (é, î, ü…) are Ll/Lu and are NOT in this set. +# Sm (Math Symbol) is intentionally EXCLUDED: it contains "+", which is a +# meaningful, channel-distinguishing character (Canal+, Three Stooges+, +# Comedy Central+). Stripping it regresses those matches. +_DECORATOR_CATS = frozenset({'So', 'No', 'Lm', 'Sk'}) + +# Tokens that are non-distinctive stream-label variants (e.g. "ABC News Live" +# should still match "ABC News"). Used by the subset/divergent guards. +_NON_DISTINCTIVE_TOKENS = frozenset({"live", "now", "new"}) + +def _is_distinctive(t): + """Return True if token t is distinctive enough to matter in subset/divergent guards.""" + return t not in _NON_DISTINCTIVE_TOKENS and (len(t) >= 4 or (t.isdigit() and len(t) >= 2)) + +QUALITY_PATTERNS = [ + r'\s*\[(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\]\s*', + r'\s*\((4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\)\s*', + r'^\s*(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s*', + r'\s*\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)$', + r'\s+\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s+', +] + +# Numeric resolution markers the keyword QUALITY_PATTERNS miss: 720p, 1080p/i, 2160p, +# 3840P, 480p, etc. - a 3-4 digit run glued directly to p/i. The 3-digit lower bound +# excludes 2-digit noise; the 4-digit upper bound excludes 5-digit numbers (10800p won't +# match). The p/i must be GLUED to the digits (no space): real markers are always written +# "720P"/"3840P", and requiring the glue avoids stripping a spaced standalone P/I such as a +# roman numeral ("Volume 100 I"). The p/i \b anchor keeps bare numbers (1080, "Channel 4") +# intact. Applied with re.IGNORECASE in the ignore_quality block, like QUALITY_PATTERNS. +RESOLUTION_PATTERNS = [ + r'\b\d{3,4}[pi]\b', +] + +REGIONAL_PATTERNS = [ + # East/West are NOT stripped here - they distinguish separate channel feeds + # ("HBO East" vs "HBO West"); normalize_name strips bare East/West separately, + # gated on ignore_regional. + # bug-066: bare " Pacific"/" Central"/" Mountain"/" Atlantic" are brand tokens far + # more often than feed markers ("Comedy Central", "The Atlantic"), so they are + # stripped ONLY in their parenthesized form, never as bare words. + r'\s*\([Pp][Aa][Cc][Ii][Ff][Ii][Cc]\)\s*', + r'\s*\([Cc][Ee][Nn][Tt][Rr][Aa][Ll]\)\s*', + r'\s*\([Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]\)\s*', + r'\s*\([Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]\)\s*', +] + +# bug-066 (opt-in half): the BARE-word forms of the time-zone region markers. OFF by +# default - a plugin enables them by setting the class attr _STRIP_BARE_REGION = True +# (or passing strip_bare_region=True). Lineuparr opts in: it strips these for SCORING +# and relies on its post-match region filter - which reads the ORIGINAL un-normalized +# name - to enforce region correctness, so a "Sportsnet Pacific" stream can score-match +# a "Sportsnet (P)" lineup and an "HBO West" lineup keeps the "HBO Pacific" feed. Plugins +# WITHOUT that filter (Stream-Mapparr) leave it off, or brand tokens like "Comedy Central" +# / "The Atlantic" would be wrongly truncated. +REGIONAL_BARE_PATTERNS = [ + r'\s[Pp][Aa][Cc][Ii][Ff][Ii][Cc]', + r'\s[Cc][Ee][Nn][Tt][Rr][Aa][Ll]', + r'\s[Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]', + r'\s[Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]', +] + +# Country tokens for the delimited provider-prefix patterns below; curated so a +# bare delimited word ("(SPORTS)") isn't misread as a country. +_PREFIX_COUNTRY = r'US|USA|UK|CA|AU|FR|DE|ES|IT|NL|BR|MX|IN' + +# (open, close) pairs; open and close must MATCH, so "(US]" / "│US)" are rejected. +_DELIM_PAIRS = ((r'\(', r'\)'), (r'\[', r'\]'), (r'\|', r'\|'), ('┃', '┃'), ('│', '│')) + + +def _balanced_delim(token): + """Regex fragment matching `token` wrapped in one MATCHED delimiter pair: + (token) [token] |token| ┃token┃ │token│. `token` may contain a capture group.""" + return '(?:' + '|'.join( + o + r'\s*(?:' + token + r')\s*' + c for o, c in _DELIM_PAIRS + ) + ')' + + +# Strip a leading box-bar bouquet/source tag with arbitrary inner text +# ("┃CANAL+┃ NPO 1" -> "NPO 1"); box bars never occur in real names, so this is +# always safe and also covers leading "┃XX┃" country tags. +_LEADING_BAR_TAG_RE = re.compile(r'^\s*[┃│]\s*[^┃│]*[┃│]\s*') + +GEOGRAPHIC_PATTERNS = [ + r'\b[A-Z]{2,3}[:┃│]\s*', + r'\b[A-Z]{2,3}\s*-\s*', + # Matched bar pair only ("|US|", "┃US┃") - a mismatched "|US┃" is noise. + r'(?:\|[A-Z]{2,3}\||┃[A-Z]{2,3}┃|│[A-Z]{2,3}│)\s*', + r'\[[A-Z]{2,3}\]\s*', +] + +# Enhanced provider prefix patterns for IPTV-specific naming +PROVIDER_PREFIX_PATTERNS = [ + r'^(?:' + _PREFIX_COUNTRY + r')\s*[:\-\|┃│]\s*', + # Bare country tag + whitespace, no separator (e.g. "US Racer Network", + # "FR beIN SPORTS MAX", "MEX Bein Sports"). Restricted to a curated set so + # it cannot eat a real channel name: "USA Network" (USA != US + space), + # "In Country Television" ("IN") and "IT Crowd" ("IT") are all safe too. + r'^(?:US|UK|CA|AU|FR|DE|MX|MEX|FRA|GER)\s+', + # "USA " space prefix as a US country tag ("USA ABC", "USA BET"). A + # negative lookahead for NETWORK protects the real channel "USA Network" + # (these feeds tag that one as "US ..."/"US: ...", never bare "USA "). + r'^USA\s+(?!NETWORK\b)', + # Country code glued directly to a quality tag with no separator + # ("UKSD: Sky Sports", "UKHD ESPN", "USFHD ..."). + r'^(?:US|UK)(?:SD|HD|FHD|UHD|FD|HEVC|4K|8K)\b\s*[:\-\|┃│]?\s*', + # Bracketed/piped country tag with a MATCHED delimiter pair ("(US)", "│US│"). + r'^\s*' + _balanced_delim(_PREFIX_COUNTRY) + r'\s*', + r'\s*[\|┃│]\s*(?:' + _PREFIX_COUNTRY + r')\s*$', + # Content-category group prefixes used by some IPTV providers. + r'^(?:ADULT|EROTIC|PRIME|GOLD)\s*[:\-\|┃│]\s*', + # FAST streaming-platform source tags (Roku, Tubi, Pluto, etc.). These mark + # the distribution platform, not the channel or its country, so strip them + # for matching ("RK: beIN Sports Xtra" -> "beIN Sports Xtra"). A separator + # is required so this can't eat real names like "GOLF" or "PLEX TV Movies". + r'^(?:RK|GO|TUBI|PLUTO|XUMO|PLEX|STIRR|FREEVEE|GLANCE)\s*[:\-\|┃│]\s*', +] + +MISC_PATTERNS = [ + r'\s*\([^)]*\)\s*', +] + +# --------------------------------------------------------------------------- # +# Stylized-Unicode decoration stripping +# --------------------------------------------------------------------------- # +# Streams tag names with stylized-Unicode tier/format markers (superscript +# "WEATHERNATION RAW", small-cap "FHD", bullet-prefixed "CNN") that the ASCII tag +# regexes below cannot see. We drop whole tokens that are pure decoration BEFORE +# the ASCII pipeline runs. Detection is by Unicode character *name* (not code-point +# ranges), so it covers superscripts, "modifier letter" superscript capitals, and +# Latin small-caps wherever they live (e.g. small-cap H is U+029C in IPA Extensions +# and modifier V is U+2C7D in Latin-Ext-C, both outside the obvious blocks). + +# Ornament glyphs whose Unicode name carries no decoration keyword. +_DECORATIVE_SYMBOLS = frozenset("◉") # FISHEYE; add individual chars (not strings) here + + +def _is_decorative_char(ch): + """True for a stylized letterform/ornament that carries no semantic content in a + channel name (superscripts, subscripts, modifier-letter superscript capitals, + Latin small-capitals, curated bullets). ASCII and ordinary letters return False.""" + if ch.isascii(): + return False + if ch in _DECORATIVE_SYMBOLS: + return True + try: + nm = unicodedata.name(ch) + except ValueError: + # unnamed code point (control char / lone surrogate) -> not decoration + return False + return ('SUPERSCRIPT' in nm or 'SUBSCRIPT' in nm + or 'SMALL CAPITAL' in nm or 'MODIFIER LETTER' in nm) + + +def _strip_stylized_tokens(name): + """Drop whitespace tokens that are pure stylized decoration, then NFKD-canonicalize + the remainder. A token is decoration when it has >=1 decorative char, no ASCII + alphanumeric, and every char is decorative or ASCII punctuation (so a bullet glued + to a colon, or "HD/RAW" written in superscripts, are dropped too). Real ASCII words + (Gold/VIP) and non-Latin letters (Arabic/Cyrillic/CJK) are always kept. ASCII-only + input is returned unchanged via the fast path (no per-char work; NFKD is a no-op + on ASCII, so skipping it changes nothing).""" + if name.isascii(): + return name + kept = [] + for tok in name.split(): + has_decorative = any(_is_decorative_char(c) for c in tok) + has_ascii_alnum = any(c.isascii() and c.isalnum() for c in tok) + only_decorative_or_punct = all( + _is_decorative_char(c) or (c.isascii() and not c.isalnum()) for c in tok + ) + if has_decorative and only_decorative_or_punct and not has_ascii_alnum: + continue # pure decoration -> drop the whole token + kept.append(tok) + return unicodedata.normalize('NFKD', ' '.join(kept)) + + +# --------------------------------------------------------------------------- # +# Emoji-as-letter + emoji decoration normalization +# --------------------------------------------------------------------------- # +# Some streams use an emoji AS A LETTER inside a word: "SP⚽RTS" / "Sp⚽rts" where the +# soccer ball stands in for 'o' (= SPORTS, the beIN family). _strip_stylized_tokens keeps +# the token (it has ASCII alnum) and process_string_for_matching would turn the ball into a +# space ("sp rts"), so it never matches "sports". We substitute the glyph for the letter it +# replaces (only when flanked by ASCII letters) and strip emoji used purely as decoration. + +# Emoji that visually replace an ASCII letter when embedded in a word. Extensible. +_EMOJI_LETTER_MAP = {'⚽': 'o'} # SOCCER BALL = 'o' (SP⚽RTS -> SPORTS) +# Pictographic ornaments to delete. NOTE: ⚽ is intentionally in BOTH maps - the letter +# map handles it mid-word (-> 'o'); here it catches any ⚽ NOT flanked by ASCII letters +# (standalone/edge), which the substitution above leaves untouched. +_EMOJI_ORNAMENTS = frozenset('♬☾⚽') # beamed notes, last-quarter moon, soccer ball +# Zero-width / invisible code points that only add noise to a name. +_ZERO_WIDTH = ('️', '‍') # VARIATION SELECTOR-16, ZERO WIDTH JOINER + + +def _normalize_emoji(name): + """Map emoji-as-letters to their letter and strip emoji decoration. + + The letter substitution fires ONLY when the glyph is flanked by ASCII letters + (so "SP⚽RTS" -> "SPoRTS" but a standalone/edge "⚽" is treated as decoration and + dropped). Zero-width selectors and ornament pictographs are deleted outright. + ASCII-only input is returned unchanged (no emoji possible).""" + if name.isascii(): + return name + for zw in _ZERO_WIDTH: + if zw in name: + name = name.replace(zw, '') + for glyph, letter in _EMOJI_LETTER_MAP.items(): + if glyph in name: + name = re.sub(r'(?<=[A-Za-z])' + re.escape(glyph) + r'(?=[A-Za-z])', letter, name) + if any(c in _EMOJI_ORNAMENTS for c in name): + name = ''.join(c for c in name if c not in _EMOJI_ORNAMENTS) + return name + + +class FuzzyMatcherCore: + """Pure, stateless matching primitives shared across the Dispatcharr plugins. + + Subclass this and add the plugin-specific orchestration layer (DB load, matching + entry points, and one feature block). The core references self only for self.logger + and the per-instance caches; it never reads plugin_dir / a channel DB / alias maps. + """ + + def __init__(self, match_threshold=80, logger=None): + self.match_threshold = match_threshold + self.logger = logger or LOGGER + # Cache for pre-normalized stream names (performance optimization). The core + # initializes the storage; the cache-filling glue (precompute_normalizations / + # _get_cached_*) is plugin-side because its flag handling is plugin-specific. + self._norm_cache = {} # raw_name -> normalized_lower + self._norm_nospace_cache = {} # raw_name -> normalized_nospace + self._processed_cache = {} # raw_name -> processed_for_matching + # Cache for callsign extraction: raw_name -> (callsign|None, is_high_conf) + self._callsign_cache = {} + # Optional set of known-real callsigns a plugin supplies from its channel DB so + # the ladder can rescue a denylisted-but-real station (KING/WAVE/WOOD/WHO). None + # = no rescue, identical to the pure Lineuparr ladder. Assign it via + # set_known_callsigns() so the callsign cache is cleared when it changes. + self._known_callsigns = None + + def set_known_callsigns(self, known): + """Supply the plugin's known-real callsign set (or None to disable rescue). + + Clears the callsign cache so a changed set takes effect on the next extraction. + """ + self._known_callsigns = frozenset(known) if known else None + self._callsign_cache.clear() + + def normalize_name(self, name, user_ignored_tags=None, ignore_quality=True, ignore_regional=True, + ignore_geographic=True, ignore_misc=True, remove_cinemax=False, + remove_country_prefix=False, strip_bare_region=None): + """ + Normalize channel or stream name for matching by removing tags, prefixes, and noise. + + remove_cinemax / remove_country_prefix are optional opt-in superset behaviors (a + plugin sets them True); both default False, so the default path is unchanged. + + strip_bare_region opts into stripping BARE time-zone region words (Pacific/Central/ + Mountain/Atlantic) for scoring - see REGIONAL_BARE_PATTERNS. None (the default) + resolves it from the subclass class attr _STRIP_BARE_REGION (default False), so a + plugin enables it once via that attr instead of threading it through every call. + """ + if user_ignored_tags is None: + user_ignored_tags = [] + if strip_bare_region is None: + strip_bare_region = getattr(self, "_STRIP_BARE_REGION", False) + + original_name = name + + name = _LEADING_BAR_TAG_RE.sub('', name) # leading "┃CANAL+┃" bouquet tag + + # Map emoji-as-letters (⚽ = 'o' in "SP⚽RTS") and strip emoji decoration, before + # the stylized-Unicode strip and ASCII regexes below - so "beIN SP⚽RTS" -> "beIN sports". + name = _normalize_emoji(name) + + # Strip stylized-Unicode decoration (superscript/small-cap tier markers, + # bullets) up front so the ASCII tag regexes below see plain text. Runs + # unconditionally: a token written in superscript/small-caps is decoration + # regardless of tag_handling, and it would otherwise block matches + # (e.g. a superscript-RAW suffix never matches channel "WeatherNation"). + name = _strip_stylized_tokens(name) + + # Strip IPTV provider prefixes BEFORE hyphen normalization so that + # "FR - Canal+ FHD" loses its "FR - " while the hyphen is still a + # hyphen. After hyphen normalization the separator would become a space + # and the pattern would fail to match, leaving "FR" as a stray token. + for pattern in PROVIDER_PREFIX_PATTERNS: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + + # Quality patterns (before space normalization). Loop until stable so + # chained suffixes like "4K HDR" or "UHD HDR" are fully stripped in + # successive passes (each pass may expose a token for the next). + if ignore_quality: + prev = None + while prev != name: + prev = name + # Strip numeric resolution markers (3840P/2160P/1080P/720P/...) before the + # digit/letter spacer below would split "3840P" into "3840 P". + # Must run before QUALITY_PATTERNS so that removing " 4K " does not glue + # "SPoRTS" to "3840P" and break the word-boundary anchor. + for pattern in RESOLUTION_PATTERNS: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + for pattern in QUALITY_PATTERNS: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + + # Normalize spacing around numbers + name = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', name) + name = re.sub(r'(\d)([a-zA-Z])', r'\1 \2', name) + + # Normalize hyphens to spaces + name = re.sub(r'-', ' ', name) + + # Replace dots between word chars with spaces (e.g. "JusticeCentral.TV" + # → "JusticeCentral TV"). Keeps the dot-suffix variant equivalent to + # the spaced form for matching purposes. + name = re.sub(r'(?<=\w)\.(?=\w)', ' ', name) + + # Normalize number-words to digits so "BBC Three" and "BBC 3" share + # tokens. Critical for cases like "Three Angels Broadcasting Network" + # vs "3 Angels Broadcasting Network", and for BBC One/Two/Three/Four + # vs BBC 1/2/3/4. Bounded by word boundaries so brand names with + # embedded letters (e.g. "Onesimus") aren't corrupted. + _NUM_WORDS = { + "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", + "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10", + "eleven": "11", "twelve": "12", + } + def _num_repl(m): + return _NUM_WORDS[m.group(0).lower()] + name = re.sub( + r'\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\b', + _num_repl, name, flags=re.IGNORECASE, + ) + + # Split CamelCase boundaries so "JusticeCentral" becomes "Justice + # Central" and "DangerTV" becomes "Danger TV". Two separate patterns: + # 1. lower → Upper followed by lower (Justice|Central, Danger|Iq → no, etc.) + # 2. lower(4+) → UPPER acronym at word boundary (Danger|TV, Beauty|IQ) + # The 4-char floor on rule 2 protects short brand names like "MeTV" and + # "truTV" whose existing EPG matches rely on the un-split form. + name = re.sub(r'([a-z])([A-Z][a-z])', r'\1 \2', name) + name = re.sub(r'([a-z]{4,})([A-Z]{2,})\b', r'\1 \2', name) + + # Strip region tokens (East / Eastern / West and the (E)/(W) + # abbreviations) for SCORING. Region CORRECTNESS - East vs West vs + # Pacific - is enforced separately by the post-match region filter in + # match_all_streams, which reads the ORIGINAL un-normalized names, not + # this output. Stripping here lets a regionless lineup channel + # ("Food Network") still score-match a region-tagged stream + # ("Food Network Eastern" / "... East") instead of being dragged below + # threshold by the extra token; the filter then accepts it (regionless + # defaults to East). "Western"/"Westerns" is a movie genre, NOT a + # region, so bare "west" is stripped only on a word boundary (\bwest\b + # does not match inside "western") and "western" is never touched. + if ignore_regional: + name = re.sub(r'\(\s*(?:eastern|east|e|west|w)\s*\)', ' ', name, flags=re.IGNORECASE) + name = re.sub(r'\b(?:eastern|east|west)\b', ' ', name, flags=re.IGNORECASE) + + # Remove leading parenthetical prefixes + while name.lstrip().startswith('('): + new_name = re.sub(r'^\s*\([^\)]+\)\s*', '', name) + if new_name == name: + break + name = new_name + + # Opt-in: remove a country-code prefix (multi-country DBs). Strips a 2-3 + # letter colon/space prefix unless it is a quality tag. Off by default; + # PROVIDER_PREFIX_PATTERNS above already removes the curated country + # prefixes, so this only catches the remainder. (Stream-Mapparr opts in.) + if remove_country_prefix: + quality_tags = {'HD', 'SD', 'FD', 'UHD', 'FHD'} + prefix_match = re.match(r'^([A-Z]{2,3})[:\s]\s*', name) + if prefix_match: + prefix = prefix_match.group(1).upper() + if prefix not in quality_tags: + name = name[len(prefix_match.group(0)):] + + # Opt-in: remove a leading "Cinemax" (for channels containing "max"). Off + # by default. (Stream-Mapparr opts in.) + if remove_cinemax: + name = re.sub(r'\bCinemax\b\s*', '', name, flags=re.IGNORECASE) + + # Build pattern list based on flags + patterns_to_apply = [] + if ignore_regional: + patterns_to_apply.extend(REGIONAL_PATTERNS) + if strip_bare_region: + patterns_to_apply.extend(REGIONAL_BARE_PATTERNS) + if ignore_geographic: + patterns_to_apply.extend(GEOGRAPHIC_PATTERNS) + if ignore_misc and ignore_regional: + patterns_to_apply.extend(MISC_PATTERNS) + + for pattern in patterns_to_apply: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + + # Apply user-configured ignored tags + for tag in user_ignored_tags: + escaped_tag = re.escape(tag) + if '[' in tag or ']' in tag or '(' in tag or ')' in tag: + name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) + else: + if re.match(r'^\w+$', tag): + name = re.sub(r'\b' + escaped_tag + r'\b', '', name, flags=re.IGNORECASE) + else: + name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) + + # Remove callsigns in parentheses + if ignore_regional: + name = re.sub(r'\([KW][A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) + else: + name = re.sub(r'\([KW](?!EST\)|ACIFIC\)|ENTRAL\)|OUNTAIN\)|TLANTIC\))[A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) + + if ignore_regional: + name = re.sub(r'\([A-Z0-9]+\)', '', name) + + # Remove common suffixes/prefixes + name = re.sub(r'^The\s+', '', name, flags=re.IGNORECASE) + name = re.sub(r'\s+Network\s*$', '', name, flags=re.IGNORECASE) + name = re.sub(r'\s+Channel\s*$', '', name, flags=re.IGNORECASE) + name = re.sub(r'\s+TV\s*$', '', name, flags=re.IGNORECASE) + + # Strip decorative Unicode markers (◉, ², superscript letters ᴿᴬᵂ…) + # that some IPTV providers append as quality/status badges. Only + # characters in decorator categories are removed; accented letters + # (é, î, ü…) are in Ll/Lu and are preserved. + name = ''.join( + ' ' if unicodedata.category(c) in _DECORATOR_CATS else c + for c in name + ) + + # Clean up whitespace + name = re.sub(r'\s+', ' ', name).strip() + + if not name: + self.logger.debug(f"normalize_name returned empty for: '{original_name}'") + + return name + + def calculate_similarity(self, str1, str2, min_ratio=0.0): + """Levenshtein similarity ratio (0.0-1.0), defined as 1 - distance/max(len). + If min_ratio > 0, returns 0.0 early when the result can't reach it. + + bug-026: the ratio MUST be distance/max(len), matching rapidfuzz + Levenshtein.normalized_similarity. The old (len1+len2-distance)/(len1+len2) + formula scored higher for the same edit distance and let numbered siblings + ("Fox Sports 1" vs "2") pass threshold 95. The rapidfuzz fast path and the + pure-Python fallback below compute the identical value. + """ + if len(str1) == 0 or len(str2) == 0: + return 0.0 + + # Fast path: C-accelerated rapidfuzz when available (same definition). + if _USE_RAPIDFUZZ: + # Compute the true similarity, then apply the SAME gate as the pure-Python + # early-exit below: a score that cannot reach min_ratio is 0.0 on both paths + # (raw agreement below threshold), but a score landing EXACTLY on min_ratio is + # KEPT (>=, inclusive). We deliberately do NOT use rapidfuzz's own score_cutoff: + # it treats the cutoff as strict-greater and quantizes it onto the achievable + # distance grid, so a true 0.8 against a 0.8 cutoff is spuriously zeroed while the + # pure-Python path returns it. With min_ratio=0.0 the gate is a no-op. + sim = _rf_lev.normalized_similarity(str1, str2) + return sim if (min_ratio <= 0.0 or sim >= min_ratio) else 0.0 + + if len(str1) < len(str2): + str1, str2 = str2, str1 + len1, len2 = len(str1), len(str2) + max_len = len1 # the longer string after the swap + + # Length-difference pre-check: minimum possible distance is (len1 - len2), + # so the max possible ratio is (max_len - (len1 - len2)) / max_len. + if min_ratio > 0: + max_possible = (max_len - (len1 - len2)) / max_len + if max_possible < min_ratio: + return 0.0 + + previous_row = list(range(len2 + 1)) + for i, c1 in enumerate(str1): + current_row = [i + 1] + for j, c2 in enumerate(str2): + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + # Early termination: a lower bound on the final distance is the current + # row minimum minus the str1 chars still unprocessed. + if min_ratio > 0: + min_distance_so_far = min(current_row) + remaining = len1 - i - 1 + best_possible_distance = max(0, min_distance_so_far - remaining) + best_possible_ratio = (max_len - best_possible_distance) / max_len + if best_possible_ratio < min_ratio: + return 0.0 + previous_row = current_row + + distance = previous_row[-1] + return (max_len - distance) / max_len + + @staticmethod + def _length_scaled_threshold(base_threshold, shorter_len): + """Require higher similarity for shorter strings to avoid false positives.""" + if shorter_len <= 4: + return max(base_threshold, 95) + elif shorter_len <= 8: + return max(base_threshold, 90) + return base_threshold + + @staticmethod + def _has_token_overlap(str_a, str_b, min_token_len=4, require_majority=False): + """Check that distinctive tokens are shared between two strings. + + Basic mode: at least one token (>= min_token_len) must be shared. + Majority mode: uses all tokens (>= 2 chars) and requires that more than + half of the smaller set overlaps. Catches false positives like + "america racing" vs "america bbc" while allowing single-token matches. + """ + # "network"/"channel"/"television" are generic brand-suffix words, not + # distinctive - treat as common so the subset guard does not reject + # cases like "FanDuel Sports Cincinnati" vs "FanDuel Sports Network + # Cincinnati". (They're already stripped from end-of-string by + # normalize_name; this catches mid-string occurrences.) + common_words = { + "the", "and", "of", "in", "on", "at", "to", "for", "a", "an", + "network", "channel", "television", + } + + if require_majority: + # Use all meaningful tokens (>= 2 chars) for stricter checking. + # Single-digit tokens (1, 2, 3, ...) are kept because they're + # channel-distinguishing (BBC 1 vs BBC 2, ESPN 1 vs ESPN 2 etc.) + # even though they're only 1 char. + def _meaningful(t): + if t in common_words: + return False + return len(t) >= 2 or t.isdigit() + tokens_a = {t for t in str_a.split() if _meaningful(t)} + tokens_b = {t for t in str_b.split() if _meaningful(t)} + if not tokens_a or not tokens_b: + return True + shared = tokens_a & tokens_b + if not shared: + return False + smaller = min(len(tokens_a), len(tokens_b)) + if not len(shared) > smaller / 2: + return False + + unique_a = tokens_a - tokens_b + unique_b = tokens_b - tokens_a + + # Subset guard: when one side is a strict subset of the other and + # the larger side has a distinctive token the smaller lacks, the + # candidate is a more specific channel than the query and the high + # fuzzy score is a false positive. Catches e.g. "Nickelodeon" vs + # "Nickelodeon Teen". Threshold is >=4 chars; "live"/"now" are + # explicitly non-distinctive (stream label variants) so "ABC News" + # still matches "ABC News Live". Pure-digit tokens >=2 chars + # (e.g. "360") are also distinctive: "Canal+Sport" must not match + # "Canal+Sport 360". + if not unique_a: + if any(_is_distinctive(t) for t in unique_b): + return False + elif not unique_b: + if any(_is_distinctive(t) for t in unique_a): + return False + + # Divergent guard: when BOTH sides have unique tokens AND at least + # one of those unique tokens is a distinctive (>=4 char) word, the + # strings describe different brands and the fuzzy score is + # misleading. Catches "Sky Cinema Disney" vs "Sky Cinema Decades" + # (decades = 7 chars), "Sky Cinema Fast" vs "Sky Cinema Family", + # and "BBC One vs BBC Two" once number-words become digits earlier. + # Number-word→digit normalization (in normalize_name) reduces this + # guard's risk: "Three Angels" / "3 Angels" both become "3 angels" + # before reaching here, so they never trigger this case. + if unique_a and unique_b: + if any(len(t) >= 4 for t in unique_a | unique_b): + return False + + # Numeric/ordinal divergent guard: when BOTH sides have a numeric + # or ordinal token unique to them, they are sibling channels + # distinguished by number (BBC One vs BBC Two; ESPN 1 vs ESPN 2). + # Short tokens like "one"/"two" wouldn't trip the divergent guard + # above so they need their own check. + _NUMERIC = { + "one", "two", "three", "four", "five", "six", "seven", "eight", + "nine", "ten", "eleven", "twelve", + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", + "first", "second", "third", "fourth", "fifth", + } + if (unique_a & _NUMERIC) and (unique_b & _NUMERIC): + return False + + return True + + # Basic mode: at least one long token shared + tokens_a = {t for t in str_a.split() if t not in common_words and len(t) >= min_token_len} + tokens_b = {t for t in str_b.split() if t not in common_words and len(t) >= min_token_len} + if not tokens_a or not tokens_b: + return True + return bool(tokens_a & tokens_b) + + def process_string_for_matching(self, s): + """Normalize for token-sort matching: NFKD-fold (so "ＨＢＯ"->"hbo", "²"->"2", + "fi"->"fi", accents drop), lowercase, keep alphanumerics of any script + (isalnum keeps Cyrillic/CJK/Arabic rather than erasing them) plus '+' + (Disney+/Discovery+/Paramount+ stay distinct), sort tokens.""" + s = unicodedata.normalize('NFKD', s) + s = ''.join(char for char in s if unicodedata.category(char) != 'Mn') + s = s.lower() + s = re.sub(r'([^\W\d_])(\d)', r'\1 \2', s) # split letter-glued digit, any script + cleaned_s = "" + for char in s: + if char.isalnum() or char == '+': + cleaned_s += char + else: + cleaned_s += ' ' + tokens = sorted([token for token in cleaned_s.split() if token]) + return " ".join(tokens) + + @staticmethod + def _trailing_number(name): + """Return the integer value of a space-separated, purely-numeric + trailing token, or None. 'HBO 2' -> 2, 'DIRECTV 4K Live 1' -> 1, + 'ESPN' -> None, 'ESPN2' -> None (digit not space-separated). Used to + reject 'Foo 1' vs 'Foo 2' false positives -- different channels that + otherwise fuzzy-match almost perfectly.""" + m = re.search(r'(?:^|\s)(\d{1,4})\s*$', name or "") + return int(m.group(1)) if m else None + + # Words that match the callsign regex shape but are never US broadcast + # callsigns. A US callsign (K/W + 2-4 letters) is shape-identical to many + # common English words, so the loose Priority-4 pattern mis-extracts them - + # e.g. "with" in "Bizarre Foods with Andrew Zimmern" becomes callsign + # "WITH". Regex alone cannot tell "WITH" from "WABC"; frequent K/W-initial + # words are denied explicitly so they never extract as a callsign. WWE/WWF/ + # WCW stop wrestling show names extracting as false-positive callsigns. + _CALLSIGN_DENYLIST = frozenset({ + 'WWE', 'WWF', 'WCW', 'EAST', + 'WAR', 'WARS', 'WARM', 'WASH', 'WATCH', 'WAVE', 'WAVES', 'WAY', 'WAYS', + 'WEB', 'WEEK', 'WELL', 'WENT', 'WERE', 'WEST', 'WHAT', 'WHEN', 'WHERE', + 'WHICH', 'WHILE', 'WHITE', 'WHO', 'WHY', 'WIDE', 'WIFE', 'WILD', 'WILL', + 'WIND', 'WINE', 'WING', 'WINGS', 'WINS', 'WIRE', 'WISE', 'WISH', 'WITH', + 'WOLF', 'WOMAN', 'WOMEN', 'WOOD', 'WORD', 'WORDS', 'WORK', 'WORKS', + 'WORLD', 'WORM', 'WORN', 'WRAP', + 'KEEN', 'KEEP', 'KEPT', 'KEY', 'KEYS', 'KICK', 'KID', 'KIDS', 'KILL', + 'KIND', 'KING', 'KINGS', 'KISS', 'KITE', 'KNEE', 'KNEW', 'KNOW', 'KNOWN', + }) + + def _is_callsign_allowed(self, callsign): + """A candidate callsign is allowed if it is not denylisted, OR the plugin + supplied a known-real callsign set that contains it (DB rescue).""" + return (callsign not in self._CALLSIGN_DENYLIST + or (self._known_callsigns is not None and callsign in self._known_callsigns)) + + def _compute_callsign_with_confidence(self, channel_name): + """ + Extract US TV callsign with a confidence flag. + + Returns (callsign, is_high_confidence). High confidence = + Priorities 1-3 (parenthesized / suffixed-paren / end-of-name). + Priority 4 (any loose word) is low confidence. (None, False) + when nothing extractable. + + A denylisted word is rejected UNLESS the plugin supplied a known-real + callsign set (set_known_callsigns) that contains it - that is the DB + rescue for real stations whose callsign collides with a common word + (KING/WAVE/WOOD/WHO). With no set supplied this is the pure ladder. + """ + # Remove common provider prefixes + channel_name = re.sub(r'^D\d+-', '', channel_name) + channel_name = re.sub(r'^USA?\s*[^a-zA-Z0-9]*\s*', '', channel_name, flags=re.IGNORECASE) + + # Priority 1: Callsigns in parentheses (most reliable) + paren_match = re.search(r'\(([KW][A-Z]{3})(?:-[A-Z\s]+)?\)', channel_name, re.IGNORECASE) + if paren_match: + callsign = paren_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, True + + # Priority 1b: grandfathered 3-letter callsigns in parentheses without a suffix + # (WWL/WJZ/KYW/WRC). Suffixed forms fall through to Priority 2. bug-062. + paren3_match = re.search(r'\(([KW][A-Z]{2})\)', channel_name, re.IGNORECASE) + if paren3_match: + callsign = paren3_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, True + + # Priority 2: Callsigns with suffix in parentheses + paren_suffix_match = re.search(r'\(([KW][A-Z]{2,4}-(?:TV|CD|LP|DT|LD))\)', channel_name, re.IGNORECASE) + if paren_suffix_match: + return paren_suffix_match.group(1).upper(), True + + # Priority 3: Callsigns at the end + end_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\s*(?:\.[a-z]+)?\s*$', channel_name, re.IGNORECASE) + if end_match: + callsign = end_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, True + + # Priority 4: Any word matching callsign pattern (low confidence) + word_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\b', channel_name, re.IGNORECASE) + if word_match: + callsign = word_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, False + + return None, False + + def _extract_callsign_with_confidence(self, channel_name): + """ + Cached wrapper around _compute_callsign_with_confidence. + + Extraction is pure in channel_name (and in self._known_callsigns, which is + held fixed between set_known_callsigns calls), so results are memoized - the + anchor calls this once per stream over a fixed candidate list, which is + otherwise massively redundant across the per-channel matching loop. + """ + cached = self._callsign_cache.get(channel_name) + if cached is not None: + return cached + result = self._compute_callsign_with_confidence(channel_name) + self._callsign_cache[channel_name] = result + return result + + def extract_callsign(self, channel_name): + """ + Extract US TV callsign from channel name with priority order. + Returns None if common false positives appear alone. + """ + callsign, _ = self._extract_callsign_with_confidence(channel_name) + return callsign + + def normalize_callsign(self, callsign): + """Remove the broadcast suffix (-TV/-CD/-LP/-DT/-LD) from a callsign.""" + if callsign: + callsign = re.sub(r'-(?:TV|CD|LP|DT|LD)$', '', callsign) + return callsign diff --git a/Lineuparr/plugin.json b/Lineuparr/plugin.json index 0d00625..0d718e6 100644 --- a/Lineuparr/plugin.json +++ b/Lineuparr/plugin.json @@ -1,6 +1,6 @@ { "name": "Lineuparr", - "version": "1.26.1641222", + "version": "1.26.1791747", "description": "Mirror real-world provider channel lineups by creating channel groups, channels, and fuzzy-matching IPTV streams to them.", "author": "PiratesIRC", "license": "MIT", diff --git a/Lineuparr/plugin.py b/Lineuparr/plugin.py index 0bb2c86..480afad 100644 --- a/Lineuparr/plugin.py +++ b/Lineuparr/plugin.py @@ -64,7 +64,7 @@ def _clean_json_text(s): class PluginConfig: - PLUGIN_VERSION = "1.26.1641222" + PLUGIN_VERSION = "1.26.1791747" DEFAULT_FUZZY_MATCH_THRESHOLD = 80 DEFAULT_PRIORITIZE_QUALITY = True From 5d606707d55ced17d1e99794f1bc774d2fb3a24d Mon Sep 17 00:00:00 2001 From: PiratesIRC Date: Sun, 28 Jun 2026 15:47:14 -0500 Subject: [PATCH 2/2] Fix matcher-gate inner-folder resolution for GitHub CI The golden/parity gates resolved the inner deploy folder as repo_root/basename(repo_root), assuming the repo directory name equals the inner folder name. That holds locally and for Stream-Mapparr (repo == "Stream-Mapparr"), but on GitHub the repos check out as "Dispatcharr--Plugin" while the inner folder is "", so the gates looked for .../Dispatcharr-X-Plugin/Dispatcharr-X-Plugin/ fuzzy_matcher.py and failed test collection. Resolve the inner folder by finding the subdir that actually contains fuzzy_matcher.py (with the old assumption as fallback). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01N2rpwFn3AHoNvsbTrEwgwd --- .github/scripts/check_core_parity.py | 2 +- .github/scripts/sync_core.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/check_core_parity.py b/.github/scripts/check_core_parity.py index 8fc67d9..b4017b8 100644 --- a/.github/scripts/check_core_parity.py +++ b/.github/scripts/check_core_parity.py @@ -17,7 +17,7 @@ HERE = Path(__file__).resolve().parent REPO_ROOT = HERE.parent.parent # /.github/scripts -> -INNER = REPO_ROOT / REPO_ROOT.name # /Lineuparr (flat deploy folder) +INNER = next((p.parent for p in REPO_ROOT.glob("*/fuzzy_matcher.py")), REPO_ROOT / REPO_ROOT.name) MANIFEST = HERE / "core_manifest.json" diff --git a/.github/scripts/sync_core.py b/.github/scripts/sync_core.py index 7f11e10..8f1241c 100644 --- a/.github/scripts/sync_core.py +++ b/.github/scripts/sync_core.py @@ -41,7 +41,7 @@ def locate(): repo_root = here.parents[2] # .github/scripts/sync_core.py -> / workspace = repo_root.parent # / shared_dir = workspace / "_shared" - inner_dir = repo_root / repo_root.name # /Lineuparr/ (flat inner folder) + inner_dir = next((p.parent for p in repo_root.glob("*/fuzzy_matcher.py")), repo_root / repo_root.name) manifest_path = here.parent / "core_manifest.json" return inner_dir, shared_dir, manifest_path