diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d98ca44 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# The vendored shared core is sha256-pinned by .github/scripts/check_core_parity.py +# against .github/scripts/core_manifest.json, so it MUST stay LF on every OS - a CRLF +# checkout (Windows autocrlf) would change the hash and break the drift gate. Same for +# any future vendored shared file (e.g. aliases_base.py). +Lineuparr/matching_core.py text eol=lf diff --git a/.github/scripts/check_core_parity.py b/.github/scripts/check_core_parity.py new file mode 100644 index 0000000..b4017b8 --- /dev/null +++ b/.github/scripts/check_core_parity.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Vendored-core drift gate (Layer A) - runs in CI, needs no workspace _shared/. + +The committed vendored shared file (Lineuparr/matching_core.py) must hash-match the +sha256 pinned in core_manifest.json. This catches a hand-edit to the vendored copy that +silently diverges from the _shared source of truth. To land an INTENDED core change: +edit /_shared/matching_core.py, re-run .github/scripts/sync_core.py (re-vendors ++ rewrites the manifest), regenerate the golden baseline if behavior changed, and commit +all of it together. See MATCHER-STANDARDIZATION-PLAN.md. + +Exit 0 = match; exit 1 = drift/missing. +""" +import hashlib +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parent.parent # /.github/scripts -> +INNER = next((p.parent for p in REPO_ROOT.glob("*/fuzzy_matcher.py")), REPO_ROOT / REPO_ROOT.name) +MANIFEST = HERE / "core_manifest.json" + + +def main() -> int: + pins = json.loads(MANIFEST.read_text(encoding="utf-8")) + failed = False + for fname, expected in sorted(pins.items()): + path = INNER / fname + if not path.exists(): + print(f"MISSING vendored {fname} (run .github/scripts/sync_core.py)") + failed = True + continue + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if actual != expected: + print(f"DRIFT {fname}: {actual} != pinned {expected}") + print(" If intended: edit _shared/, re-run sync_core.py, commit the new manifest.") + failed = True + else: + print(f"OK {fname}: {actual[:16]}...") + if failed: + print("Core parity gate FAILED.") + return 1 + print("Core parity gate passed.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/check_matcher_golden.py b/.github/scripts/check_matcher_golden.py new file mode 100644 index 0000000..fcd656d --- /dev/null +++ b/.github/scripts/check_matcher_golden.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Golden drift gate for Lineuparr's pure matcher primitives (Stage 0). + +Lineuparr keeps its unit suite OUT of git (tests/ is gitignored) and CI runs only the +static validate_plugin.py, so the matcher golden gate lives here in .github/scripts +instead of a pytest file. It is the Lineuparr equivalent of +Stream/Channel/EPG's tests/test_matcher_golden.py. + +It loads Lineuparr/fuzzy_matcher.py directly, runs a shared corpus through the PURE +primitives, and compares against the committed matcher_golden_baseline.json beside this +script. Any unreviewed change to match behavior fails CI. An INTENDED de-drift change is +landed by re-running this with --write (or tools/matcher_parity_check.py --write at the +workspace root) and committing the updated baseline in the same change. + +Keep the corpus below in lockstep with tools/matcher_parity_check.py and the per-plugin +tests/test_matcher_golden.py. Needs rapidfuzz installed to match the production path. +See MATCHER-STANDARDIZATION-PLAN.md §7. +""" +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent # /.github/scripts -> +INNER = ROOT / "Lineuparr" +BASELINE = HERE / "matcher_golden_baseline.json" + +# --- shared corpus (keep identical to tools/matcher_parity_check.py) --------- +NAMES = [ + "US: USA Network HD", "US| ESPN", "[US] CNN", "UK: BBC One", "UK| ITV 1", + "Discovery Channel 4K", "HBO HD", "ESPN [FHD]", "Cinemax HD", "TNT UHD RAW", + "BBC Three", "BBC Four", "Three Angels Broadcasting Network", "ESPN 2", "HBO 2", + "JusticeCentral.TV", "DangerTV", "NewsNation", + "HBO East", "HBO West", "HBO (W)", "Fox Sports West", "ESPN Pacific", + "(PRIME) FOX News", "(D1) CBS", + "Disney+", "Discovery+", "Paramount+", "Disney Channel", "Discovery Channel", + "Justice Central", "Justice Central.TV", "Justice Central TV", "True Crime Network", + "WABC-TV", "KCBS", "KING 5", "WAVE 3", "WOOD TV8", "WHO 13", "KOMO News", + "\U0001f174\U0001f182\U0001f17f\U0001f175", "┃US┃ ESPN", "★ CNN ★", + "Россия 1", "France 2", "beИN SPORTS", + "HLN", "MTV", "getTV", "TUDN", "SEC Network", "NHL Network", "BBC News", +] +PAIRS = [ + ("usanetwork", "usanetwork"), ("espn", "espn2"), ("hbo", "hbo2"), + ("bbcone", "bbctwo"), ("disney", "disneyplus"), ("foxnews", "foxnews"), + ("cnn", "cnninternational"), ("discoverychannel", "discovery"), ("e", "ae"), + ("paramount", "paramountnetwork"), ("nflnetwork", "nhlnetwork"), + ("justicecentral", "truecrimenetwork"), ("a", "a"), ("", ""), +] +FLAG_COMBOS = [ + ("all_on", dict(ignore_quality=True, ignore_regional=True, ignore_geographic=True, ignore_misc=True)), + ("regional_off", dict(ignore_quality=True, ignore_regional=False, ignore_geographic=True, ignore_misc=True)), +] + + +def _safe(fn, *args, **kwargs): + try: + val = fn(*args, **kwargs) + except Exception as exc: + return f"__ERROR__: {type(exc).__name__}: {exc}" + if isinstance(val, float): + return round(val, 9) + return val + + +def run_corpus(matcher): + out = { + "process_string": {n: _safe(matcher.process_string_for_matching, n) for n in NAMES}, + "normalize_name": {}, + "calculate_similarity": {f"{a}|{b}": _safe(matcher.calculate_similarity, a, b) for a, b in PAIRS}, + "extract_callsign": {n: _safe(matcher.extract_callsign, n) for n in NAMES}, + "normalize_callsign": {n: _safe(matcher.normalize_callsign, n) for n in NAMES}, + } + for label, flags in FLAG_COMBOS: + out["normalize_name"][label] = {n: _safe(matcher.normalize_name, n, **flags) for n in NAMES} + return out + + +def _flatten(d, prefix=""): + for k in sorted(d): + v = d[k] + if isinstance(v, dict): + yield from _flatten(v, f"{prefix}{k}.") + else: + yield (f"{prefix}{k}", v) + + +def load_matcher(): + path = INNER / "fuzzy_matcher.py" + saved_path = list(sys.path) + saved_aliases = sys.modules.pop("aliases", None) + sys.path.insert(0, str(INNER)) + try: + spec = importlib.util.spec_from_file_location("fm_golden_under_test", str(path)) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + finally: + sys.path[:] = saved_path + sys.modules.pop("aliases", None) + if saved_aliases is not None: + sys.modules["aliases"] = saved_aliases + return mod.FuzzyMatcher() + + +def main() -> int: + ap = argparse.ArgumentParser(description="Lineuparr matcher golden drift gate") + ap.add_argument("--write", action="store_true", help="(re)generate the baseline from current code") + args = ap.parse_args() + + current = run_corpus(load_matcher()) + if args.write: + BASELINE.write_text( + json.dumps(current, indent=2, ensure_ascii=False, sort_keys=True), encoding="utf-8" + ) + print(f"wrote {BASELINE}") + return 0 + + if not BASELINE.exists(): + print(f"MATCHER GOLDEN: no baseline at {BASELINE} (run with --write first)") + return 1 + base_flat = dict(_flatten(json.loads(BASELINE.read_text(encoding="utf-8")))) + cur_flat = dict(_flatten(current)) + diffs = [(k, base_flat.get(k, ""), cur_flat.get(k, "")) + for k in sorted(set(base_flat) | set(cur_flat)) + if base_flat.get(k, "") != cur_flat.get(k, "")] + if diffs: + print(f"MATCHER GOLDEN DRIFT ({len(diffs)} primitive output(s) changed):") + for key, old, new in diffs[:30]: + print(f" {key}: {old!r} -> {new!r}") + print("If intended, re-run with --write and commit the updated baseline.") + return 1 + print(f"Matcher golden gate passed ({len(base_flat)} primitive outputs match baseline).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/core_manifest.json b/.github/scripts/core_manifest.json new file mode 100644 index 0000000..7d8ff5b --- /dev/null +++ b/.github/scripts/core_manifest.json @@ -0,0 +1,3 @@ +{ + "matching_core.py": "2ac7ac4cb283306152701bfd977ff2f13700d63d8023fdfa280b1480f5a969dc" +} diff --git a/.github/scripts/matcher_golden_baseline.json b/.github/scripts/matcher_golden_baseline.json new file mode 100644 index 0000000..b02725b --- /dev/null +++ b/.github/scripts/matcher_golden_baseline.json @@ -0,0 +1,300 @@ +{ + "calculate_similarity": { + "a|a": 1.0, + "bbcone|bbctwo": 0.5, + "cnn|cnninternational": 0.1875, + "discoverychannel|discovery": 0.5625, + "disney|disneyplus": 0.6, + "espn|espn2": 0.8, + "e|ae": 0.5, + "foxnews|foxnews": 1.0, + "hbo|hbo2": 0.75, + "justicecentral|truecrimenetwork": 0.25, + "nflnetwork|nhlnetwork": 0.9, + "paramount|paramountnetwork": 0.5625, + "usanetwork|usanetwork": 1.0, + "|": 0.0 + }, + "extract_callsign": { + "(D1) CBS": null, + "(PRIME) FOX News": null, + "BBC Four": null, + "BBC News": null, + "BBC Three": null, + "Cinemax HD": null, + "DangerTV": null, + "Discovery Channel": null, + "Discovery Channel 4K": null, + "Discovery+": null, + "Disney Channel": null, + "Disney+": null, + "ESPN 2": null, + "ESPN Pacific": null, + "ESPN [FHD]": null, + "Fox Sports West": null, + "France 2": null, + "HBO (W)": null, + "HBO 2": null, + "HBO East": null, + "HBO HD": null, + "HBO West": null, + "HLN": null, + "Justice Central": null, + "Justice Central TV": null, + "Justice Central.TV": null, + "JusticeCentral.TV": null, + "KCBS": "KCBS", + "KING 5": null, + "KOMO News": "KOMO", + "MTV": null, + "NHL Network": null, + "NewsNation": null, + "Paramount+": null, + "SEC Network": null, + "TNT UHD RAW": null, + "TUDN": null, + "Three Angels Broadcasting Network": null, + "True Crime Network": null, + "UK: BBC One": null, + "UK| ITV 1": null, + "US: USA Network HD": null, + "US| ESPN": null, + "WABC-TV": "WABC-TV", + "WAVE 3": null, + "WHO 13": null, + "WOOD TV8": null, + "[US] CNN": null, + "beИN SPORTS": null, + "getTV": null, + "Россия 1": null, + "┃US┃ ESPN": null, + "★ CNN ★": null, + "🅴🆂🅿🅵": null + }, + "normalize_callsign": { + "(D1) CBS": "(D1) CBS", + "(PRIME) FOX News": "(PRIME) FOX News", + "BBC Four": "BBC Four", + "BBC News": "BBC News", + "BBC Three": "BBC Three", + "Cinemax HD": "Cinemax HD", + "DangerTV": "DangerTV", + "Discovery Channel": "Discovery Channel", + "Discovery Channel 4K": "Discovery Channel 4K", + "Discovery+": "Discovery+", + "Disney Channel": "Disney Channel", + "Disney+": "Disney+", + "ESPN 2": "ESPN 2", + "ESPN Pacific": "ESPN Pacific", + "ESPN [FHD]": "ESPN [FHD]", + "Fox Sports West": "Fox Sports West", + "France 2": "France 2", + "HBO (W)": "HBO (W)", + "HBO 2": "HBO 2", + "HBO East": "HBO East", + "HBO HD": "HBO HD", + "HBO West": "HBO West", + "HLN": "HLN", + "Justice Central": "Justice Central", + "Justice Central TV": "Justice Central TV", + "Justice Central.TV": "Justice Central.TV", + "JusticeCentral.TV": "JusticeCentral.TV", + "KCBS": "KCBS", + "KING 5": "KING 5", + "KOMO News": "KOMO News", + "MTV": "MTV", + "NHL Network": "NHL Network", + "NewsNation": "NewsNation", + "Paramount+": "Paramount+", + "SEC Network": "SEC Network", + "TNT UHD RAW": "TNT UHD RAW", + "TUDN": "TUDN", + "Three Angels Broadcasting Network": "Three Angels Broadcasting Network", + "True Crime Network": "True Crime Network", + "UK: BBC One": "UK: BBC One", + "UK| ITV 1": "UK| ITV 1", + "US: USA Network HD": "US: USA Network HD", + "US| ESPN": "US| ESPN", + "WABC-TV": "WABC", + "WAVE 3": "WAVE 3", + "WHO 13": "WHO 13", + "WOOD TV8": "WOOD TV8", + "[US] CNN": "[US] CNN", + "beИN SPORTS": "beИN SPORTS", + "getTV": "getTV", + "Россия 1": "Россия 1", + "┃US┃ ESPN": "┃US┃ ESPN", + "★ CNN ★": "★ CNN ★", + "🅴🆂🅿🅵": "🅴🆂🅿🅵" + }, + "normalize_name": { + "all_on": { + "(D1) CBS": "CBS", + "(PRIME) FOX News": "FOX News", + "BBC Four": "BBC 4", + "BBC News": "BBC News", + "BBC Three": "BBC 3", + "Cinemax HD": "Cinemax", + "DangerTV": "Danger", + "Discovery Channel": "Discovery", + "Discovery Channel 4K": "Discovery", + "Discovery+": "Discovery+", + "Disney Channel": "Disney", + "Disney+": "Disney+", + "ESPN 2": "ESPN 2", + "ESPN Pacific": "ESPN", + "ESPN [FHD]": "ESPN", + "Fox Sports West": "Fox Sports", + "France 2": "France 2", + "HBO (W)": "HBO", + "HBO 2": "HBO 2", + "HBO East": "HBO", + "HBO HD": "HBO", + "HBO West": "HBO", + "HLN": "HLN", + "Justice Central": "Justice", + "Justice Central TV": "Justice", + "Justice Central.TV": "Justice", + "JusticeCentral.TV": "Justice", + "KCBS": "KCBS", + "KING 5": "KING 5", + "KOMO News": "KOMO News", + "MTV": "MTV", + "NHL Network": "NHL", + "NewsNation": "News Nation", + "Paramount+": "Paramount+", + "SEC Network": "SEC", + "TNT UHD RAW": "TNTRAW", + "TUDN": "TUDN", + "Three Angels Broadcasting Network": "3 Angels Broadcasting", + "True Crime Network": "True Crime", + "UK: BBC One": "BBC 1", + "UK| ITV 1": "ITV 1", + "US: USA Network HD": "USA", + "US| ESPN": "ESPN", + "WABC-TV": "WABC", + "WAVE 3": "WAVE 3", + "WHO 13": "WHO 13", + "WOOD TV8": "WOOD TV 8", + "[US] CNN": "CNN", + "beИN SPORTS": "beИN SPORTS", + "getTV": "getTV", + "Россия 1": "Россия 1", + "┃US┃ ESPN": "ESPN", + "★ CNN ★": "CNN", + "🅴🆂🅿🅵": "" + }, + "regional_off": { + "(D1) CBS": "CBS", + "(PRIME) FOX News": "FOX News", + "BBC Four": "BBC 4", + "BBC News": "BBC News", + "BBC Three": "BBC 3", + "Cinemax HD": "Cinemax", + "DangerTV": "Danger", + "Discovery Channel": "Discovery", + "Discovery Channel 4K": "Discovery", + "Discovery+": "Discovery+", + "Disney Channel": "Disney", + "Disney+": "Disney+", + "ESPN 2": "ESPN 2", + "ESPN Pacific": "ESPN Pacific", + "ESPN [FHD]": "ESPN", + "Fox Sports West": "Fox Sports West", + "France 2": "France 2", + "HBO (W)": "HBO (W)", + "HBO 2": "HBO 2", + "HBO East": "HBO East", + "HBO HD": "HBO", + "HBO West": "HBO West", + "HLN": "HLN", + "Justice Central": "Justice Central", + "Justice Central TV": "Justice Central", + "Justice Central.TV": "Justice Central", + "JusticeCentral.TV": "Justice Central", + "KCBS": "KCBS", + "KING 5": "KING 5", + "KOMO News": "KOMO News", + "MTV": "MTV", + "NHL Network": "NHL", + "NewsNation": "News Nation", + "Paramount+": "Paramount+", + "SEC Network": "SEC", + "TNT UHD RAW": "TNTRAW", + "TUDN": "TUDN", + "Three Angels Broadcasting Network": "3 Angels Broadcasting", + "True Crime Network": "True Crime", + "UK: BBC One": "BBC 1", + "UK| ITV 1": "ITV 1", + "US: USA Network HD": "USA", + "US| ESPN": "ESPN", + "WABC-TV": "WABC", + "WAVE 3": "WAVE 3", + "WHO 13": "WHO 13", + "WOOD TV8": "WOOD TV 8", + "[US] CNN": "CNN", + "beИN SPORTS": "beИN SPORTS", + "getTV": "getTV", + "Россия 1": "Россия 1", + "┃US┃ ESPN": "ESPN", + "★ CNN ★": "CNN", + "🅴🆂🅿🅵": "" + } + }, + "process_string": { + "(D1) CBS": "1 cbs d", + "(PRIME) FOX News": "fox news prime", + "BBC Four": "bbc four", + "BBC News": "bbc news", + "BBC Three": "bbc three", + "Cinemax HD": "cinemax hd", + "DangerTV": "dangertv", + "Discovery Channel": "channel discovery", + "Discovery Channel 4K": "4k channel discovery", + "Discovery+": "discovery+", + "Disney Channel": "channel disney", + "Disney+": "disney+", + "ESPN 2": "2 espn", + "ESPN Pacific": "espn pacific", + "ESPN [FHD]": "espn fhd", + "Fox Sports West": "fox sports west", + "France 2": "2 france", + "HBO (W)": "hbo w", + "HBO 2": "2 hbo", + "HBO East": "east hbo", + "HBO HD": "hbo hd", + "HBO West": "hbo west", + "HLN": "hln", + "Justice Central": "central justice", + "Justice Central TV": "central justice tv", + "Justice Central.TV": "central justice tv", + "JusticeCentral.TV": "justicecentral tv", + "KCBS": "kcbs", + "KING 5": "5 king", + "KOMO News": "komo news", + "MTV": "mtv", + "NHL Network": "network nhl", + "NewsNation": "newsnation", + "Paramount+": "paramount+", + "SEC Network": "network sec", + "TNT UHD RAW": "raw tnt uhd", + "TUDN": "tudn", + "Three Angels Broadcasting Network": "angels broadcasting network three", + "True Crime Network": "crime network true", + "UK: BBC One": "bbc one uk", + "UK| ITV 1": "1 itv uk", + "US: USA Network HD": "hd network us usa", + "US| ESPN": "espn us", + "WABC-TV": "tv wabc", + "WAVE 3": "3 wave", + "WHO 13": "13 who", + "WOOD TV8": "8 tv wood", + "[US] CNN": "cnn us", + "beИN SPORTS": "beиn sports", + "getTV": "gettv", + "Россия 1": "1 россия", + "┃US┃ ESPN": "espn us", + "★ CNN ★": "cnn", + "🅴🆂🅿🅵": "" + } +} \ No newline at end of file diff --git a/.github/scripts/sync_core.py b/.github/scripts/sync_core.py new file mode 100644 index 0000000..8f1241c --- /dev/null +++ b/.github/scripts/sync_core.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Vendor the shared matching core into Lineuparr's inner folder (local dev tool). + +The source of truth is /_shared/matching_core.py. This re-vendors it into the +deployable inner folder and (re)writes .github/scripts/core_manifest.json with its sha256. +It self-locates assuming the layout: + /_shared/matching_core.py + /Lineuparr/.github/scripts/sync_core.py (this file) + /Lineuparr/Lineuparr/ (flat inner = deploy artifact) + +Modes: + (default) write: copy _shared/matching_core.py into the inner folder and rewrite the + manifest with its sha256 hash. + --check verify the vendored copy is byte-identical to _shared (the "forgot to sync" + gate). Needs _shared present, so it runs LOCALLY only, never in GitHub CI. + --dry-run print what write would do without touching anything. + +The per-plugin CI gate is SEPARATE and needs no _shared: .github/scripts/check_core_parity.py +asserts sha256(inner/matching_core.py) == core_manifest.json. Keep the vendored file OUT of +any bump_version lockstep stamping, or the hash gate fails forever. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import sys +from pathlib import Path + +SHARED_FILES = ["matching_core.py"] + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def locate(): + """Return (inner_dir, shared_dir, manifest_path).""" + here = Path(__file__).resolve() + repo_root = here.parents[2] # .github/scripts/sync_core.py -> / + workspace = repo_root.parent # / + shared_dir = workspace / "_shared" + inner_dir = next((p.parent for p in repo_root.glob("*/fuzzy_matcher.py")), repo_root / repo_root.name) + manifest_path = here.parent / "core_manifest.json" + return inner_dir, shared_dir, manifest_path + + +def do_write(inner_dir: Path, shared_dir: Path, manifest_path: Path, dry_run: bool) -> int: + manifest = {} + for fname in SHARED_FILES: + src = shared_dir / fname + if not src.exists(): + print(f"ERROR: shared source missing: {src}") + return 1 + digest = file_sha256(src) + manifest[fname] = digest + dst = inner_dir / fname + if dry_run: + print(f" would copy {src} -> {dst} ({digest[:12]}...)") + else: + shutil.copyfile(src, dst) + print(f" vendored {fname} ({digest[:12]}...)") + if dry_run: + print(f" would write manifest {manifest_path}") + return 0 + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f" wrote manifest {manifest_path}") + return 0 + + +def do_check(inner_dir: Path, shared_dir: Path) -> int: + drift = 0 + for fname in SHARED_FILES: + src, dst = shared_dir / fname, inner_dir / fname + if not src.exists(): + print(f"ERROR: shared source missing: {src}") + return 1 + if not dst.exists(): + print(f"DRIFT: vendored copy missing: {dst} (run sync_core.py)") + drift += 1 + continue + if file_sha256(src) != file_sha256(dst): + print(f"DRIFT: {fname} vendored copy differs from _shared (run sync_core.py)") + drift += 1 + if drift: + print(f"sync check FAILED: {drift} file(s) out of sync") + return 1 + print("sync check OK: vendored core matches _shared") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description="Vendor the shared matching core into Lineuparr.") + ap.add_argument("--check", action="store_true", help="verify vendored == _shared, don't write") + ap.add_argument("--dry-run", action="store_true", help="show what write would do") + args = ap.parse_args() + + inner_dir, shared_dir, manifest_path = locate() + if args.check: + return do_check(inner_dir, shared_dir) + return do_write(inner_dir, shared_dir, manifest_path, dry_run=args.dry_run) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 0901642..1b5ed5a 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1,9 +1,9 @@ name: Validate plugin # Static validation of the committed plugin runtime (compile, lineup JSON, -# manifest, em-dash ban). Catches a shipped-broken file before a user installs -# from GitHub. The full matcher regression suite is intentionally kept local -# and is not run here. +# manifest, em-dash ban) plus the matcher golden drift gate. Catches a +# shipped-broken file before a user installs from GitHub. The full matcher +# regression suite is intentionally kept local and is not run here. on: push: @@ -12,12 +12,20 @@ on: - "Lineuparr/**" - "README.md" - ".github/scripts/validate_plugin.py" + - ".github/scripts/check_matcher_golden.py" + - ".github/scripts/matcher_golden_baseline.json" + - ".github/scripts/check_core_parity.py" + - ".github/scripts/core_manifest.json" - ".github/workflows/validate.yml" pull_request: paths: - "Lineuparr/**" - "README.md" - ".github/scripts/validate_plugin.py" + - ".github/scripts/check_matcher_golden.py" + - ".github/scripts/matcher_golden_baseline.json" + - ".github/scripts/check_core_parity.py" + - ".github/scripts/core_manifest.json" - ".github/workflows/validate.yml" workflow_dispatch: @@ -31,3 +39,15 @@ jobs: python-version: "3.12" - name: Validate plugin runtime run: python .github/scripts/validate_plugin.py + # The vendored shared matching core must byte-compile and stay byte-identical to its + # pinned hash (catches a CRLF checkout or a stray hand-edit to the vendored copy). + - name: Vendored-core compile + parity gate + run: | + python -m py_compile Lineuparr/matching_core.py + python .github/scripts/check_core_parity.py + # The matcher golden gate needs rapidfuzz to reproduce the production + # similarity path that the committed baseline was generated against. + - name: Install matcher-check deps + run: pip install rapidfuzz + - name: Matcher golden drift gate + run: python .github/scripts/check_matcher_golden.py diff --git a/Lineuparr/fuzzy_matcher.py b/Lineuparr/fuzzy_matcher.py index 97fc8b6..54e1599 100644 --- a/Lineuparr/fuzzy_matcher.py +++ b/Lineuparr/fuzzy_matcher.py @@ -10,14 +10,24 @@ import logging import unicodedata -# Optional C-accelerated Levenshtein. When present, the matcher uses rapidfuzz's -# normalized_similarity (1 - distance/max(len)); the pure-Python fallback below -# computes the identical value (bug-026). rapidfuzz is an OPTIONAL runtime dep. +# The pure matching primitives (normalize_name, calculate_similarity, the callsign +# ladder, ...) live in the vendored shared core. The rapidfuzz fast path and its +# pure-Python fallback are inside FuzzyMatcherCore.calculate_similarity. The decorative +# helpers are re-exported so tests/callers that reference them keep working. try: - from rapidfuzz.distance import Levenshtein as _rf_lev - _USE_RAPIDFUZZ = True -except ImportError: - _USE_RAPIDFUZZ = False + from .matching_core import ( + FuzzyMatcherCore, + _is_decorative_char, + _normalize_emoji, + _strip_stylized_tokens, + ) +except ImportError: # script/test context without the package parent on sys.path + from matching_core import ( + FuzzyMatcherCore, + _is_decorative_char, + _normalize_emoji, + _strip_stylized_tokens, + ) __version__ = "1.3.4" @@ -28,48 +38,12 @@ LOGGER.addHandler(_handler) LOGGER.setLevel(logging.DEBUG) -# --- Pattern categories for normalization --- - -# Unicode categories considered decorative/badge characters by IPTV providers. -# So = Other Symbol (◉), No = Other Number (², ³), Lm = Modifier Letter -# (ᴿᴬᵂ, ᴴᴰ, ⱽᴵᴾ superscripts), Sk = Modifier Symbol. -# Accented letters (é, î, ü…) are Ll/Lu and are NOT in this set. -# Sm (Math Symbol) is intentionally EXCLUDED: it contains "+", which is a -# meaningful, channel-distinguishing character (Canal+, Three Stooges+, -# Comedy Central+). Stripping it regresses those matches. -_DECORATOR_CATS = frozenset({'So', 'No', 'Lm', 'Sk'}) - -# Tokens that are non-distinctive stream-label variants (e.g. "ABC News Live" -# should still match "ABC News"). Used by the subset/divergent guards. -_NON_DISTINCTIVE_TOKENS = frozenset({"live", "now", "new"}) - -def _is_distinctive(t): - """Return True if token t is distinctive enough to matter in subset/divergent guards.""" - return t not in _NON_DISTINCTIVE_TOKENS and (len(t) >= 4 or (t.isdigit() and len(t) >= 2)) # Matches "+1" / "+2" time-shift suffixes in the ORIGINAL (pre-normalization) # channel name. Must be checked before normalization because "+" is in the Sm # category and gets stripped, making "+1" indistinguishable from "1" afterward. _PLUS_SHIFT_RE = re.compile(r'\+\s{0,2}\d{1,2}\b') -QUALITY_PATTERNS = [ - r'\s*\[(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\]\s*', - r'\s*\((4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\)\s*', - r'^\s*(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s*', - r'\s*\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)$', - r'\s+\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s+', -] - -# Numeric resolution markers the keyword QUALITY_PATTERNS miss: 720p, 1080p/i, 2160p, -# 3840P, 480p, etc. — a 3-4 digit run glued directly to p/i. The 3-digit lower bound -# excludes 2-digit noise; the 4-digit upper bound excludes 5-digit numbers (10800p won't -# match). The p/i must be GLUED to the digits (no space): real markers are always written -# "720P"/"3840P", and requiring the glue avoids stripping a spaced standalone P/I such as a -# roman numeral ("Volume 100 I"). The p/i \b anchor keeps bare numbers (1080, "Channel 4") -# intact. Applied with re.IGNORECASE in the ignore_quality block, like QUALITY_PATTERNS. -RESOLUTION_PATTERNS = [ - r'\b\d{3,4}[pi]\b', -] # Quality markers that distinguish an upgrade-tier channel from its standard twin. # HD/FHD/HEVC are intentionally excluded - they don't create a separate twin channel. @@ -81,19 +55,6 @@ def has_upgrade_quality(name: str) -> bool: return bool(_UPGRADE_QUALITY_RE.search(name)) -REGIONAL_PATTERNS = [ - # East/West are intentionally NOT stripped - they distinguish separate channel feeds - # (e.g., "HBO East" and "HBO West" are different channels) - r'\s[Pp][Aa][Cc][Ii][Ff][Ii][Cc]', - r'\s[Cc][Ee][Nn][Tt][Rr][Aa][Ll]', - r'\s[Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]', - r'\s[Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]', - r'\s*\([Pp][Aa][Cc][Ii][Ff][Ii][Cc]\)\s*', - r'\s*\([Cc][Ee][Nn][Tt][Rr][Aa][Ll]\)\s*', - r'\s*\([Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]\)\s*', - r'\s*\([Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]\)\s*', -] - # Country tokens for the delimited provider-prefix patterns below; curated so a # bare delimited word ("(SPORTS)") isn't misread as a country. Keep in sync with # detect_stream_country(). @@ -114,55 +75,6 @@ def _balanced_delim(token): # Leading country tag in a matched delimiter pair; one capture group fires. _BRACKETED_CC_RE = re.compile(r'^\s*' + _balanced_delim(r'([A-Za-z]{2,3})')) -# Strip a leading box-bar bouquet/source tag with arbitrary inner text -# ("┃CANAL+┃ NPO 1" -> "NPO 1"); box bars never occur in real names, so this is -# always safe and also covers leading "┃XX┃" country tags. detect_stream_country -# reads the raw name, so detection is unaffected. -_LEADING_BAR_TAG_RE = re.compile(r'^\s*[┃│]\s*[^┃│]*[┃│]\s*') - -GEOGRAPHIC_PATTERNS = [ - r'\b[A-Z]{2,3}[:┃│]\s*', - r'\b[A-Z]{2,3}\s*-\s*', - # Matched bar pair only ("|US|", "┃US┃") - a mismatched "|US┃" is noise. - r'(?:\|[A-Z]{2,3}\||┃[A-Z]{2,3}┃|│[A-Z]{2,3}│)\s*', - r'\[[A-Z]{2,3}\]\s*', -] - -# Enhanced provider prefix patterns for IPTV-specific naming -PROVIDER_PREFIX_PATTERNS = [ - r'^(?:' + _PREFIX_COUNTRY + r')\s*[:\-\|┃│]\s*', - # Bare country tag + whitespace, no separator (e.g. "US Racer Network", - # "FR beIN SPORTS MAX", "MEX Bein Sports"). Restricted to a curated set so - # it cannot eat a real channel name: "USA Network" (USA != US + space), - # "In Country Television" ("IN") and "IT Crowd" ("IT") are all safe too. - # Keep this set in sync with detect_stream_country()'s bare-space branch. - r'^(?:US|UK|CA|AU|FR|DE|MX|MEX|FRA|GER)\s+', - # "USA " space prefix as a US country tag ("USA ABC", "USA BET"). A - # negative lookahead for NETWORK protects the real channel "USA Network" - # (these feeds tag that one as "US ..."/"US: ...", never bare "USA "). - # Keep in sync with detect_stream_country()'s USA branch. - r'^USA\s+(?!NETWORK\b)', - # Country code glued directly to a quality tag with no separator - # ("UKSD: Sky Sports", "UKHD ESPN", "USFHD ..."). Detection mirrors this - # in detect_stream_country() so these can't leak as wildcards. - r'^(?:US|UK)(?:SD|HD|FHD|UHD|FD|HEVC|4K|8K)\b\s*[:\-\|┃│]?\s*', - # Bracketed/piped country tag with a MATCHED delimiter pair ("(US)", "│US│"). - r'^\s*' + _balanced_delim(_PREFIX_COUNTRY) + r'\s*', - r'\s*[\|┃│]\s*(?:' + _PREFIX_COUNTRY + r')\s*$', - # Content-category group prefixes used by some IPTV providers. - r'^(?:ADULT|EROTIC|PRIME|GOLD)\s*[:\-\|┃│]\s*', - # FAST streaming-platform source tags (Roku, Tubi, Pluto, etc.). These mark - # the distribution platform, not the channel or its country, so strip them - # for matching ("RK: beIN Sports Xtra" -> "beIN Sports Xtra"). A separator - # is required so this can't eat real names like "GOLF" or "PLEX TV Movies". - # NOT a country signal: detect_stream_country() ignores these (correctly, - # since a platform like Roku spans US/CA/UK). - r'^(?:RK|GO|TUBI|PLUTO|XUMO|PLEX|STIRR|FREEVEE|GLANCE)\s*[:\-\|┃│]\s*', -] - -MISC_PATTERNS = [ - r'\s*\([^)]*\)\s*', -] # ISO-2 country codes Lineuparr lineup filenames use (keep in sync with # PROVIDER_PREFIX_PATTERNS above and PluginConfig.COUNTRY_DIR_MAP). @@ -378,116 +290,23 @@ def country_codes_in_text(text): return codes -# --------------------------------------------------------------------------- # -# Stylized-Unicode decoration stripping -# --------------------------------------------------------------------------- # -# Streams tag names with stylized-Unicode tier/format markers (superscript -# "WEATHERNATION RAW", small-cap "FHD", bullet-prefixed "CNN") that the ASCII tag -# regexes below cannot see. We drop whole tokens that are pure decoration BEFORE -# the ASCII pipeline runs. Detection is by Unicode character *name* (not code-point -# ranges), so it covers superscripts, "modifier letter" superscript capitals, and -# Latin small-caps wherever they live (e.g. small-cap H is U+029C in IPA Extensions -# and modifier V is U+2C7D in Latin-Ext-C, both outside the obvious blocks). - -# Ornament glyphs whose Unicode name carries no decoration keyword. -_DECORATIVE_SYMBOLS = frozenset("◉") # FISHEYE; add individual chars (not strings) here - - -def _is_decorative_char(ch): - """True for a stylized letterform/ornament that carries no semantic content in a - channel name (superscripts, subscripts, modifier-letter superscript capitals, - Latin small-capitals, curated bullets). ASCII and ordinary letters return False.""" - if ch.isascii(): - return False - if ch in _DECORATIVE_SYMBOLS: - return True - try: - nm = unicodedata.name(ch) - except ValueError: - # unnamed code point (control char / lone surrogate) -> not decoration - return False - return ('SUPERSCRIPT' in nm or 'SUBSCRIPT' in nm - or 'SMALL CAPITAL' in nm or 'MODIFIER LETTER' in nm) - - -def _strip_stylized_tokens(name): - """Drop whitespace tokens that are pure stylized decoration, then NFKD-canonicalize - the remainder. A token is decoration when it has >=1 decorative char, no ASCII - alphanumeric, and every char is decorative or ASCII punctuation (so a bullet glued - to a colon, or "HD/RAW" written in superscripts, are dropped too). Real ASCII words - (Gold/VIP) and non-Latin letters (Arabic/Cyrillic/CJK) are always kept. ASCII-only - input is returned unchanged via the fast path (no per-char work; NFKD is a no-op - on ASCII, so skipping it changes nothing).""" - if name.isascii(): - return name - kept = [] - for tok in name.split(): - has_decorative = any(_is_decorative_char(c) for c in tok) - has_ascii_alnum = any(c.isascii() and c.isalnum() for c in tok) - only_decorative_or_punct = all( - _is_decorative_char(c) or (c.isascii() and not c.isalnum()) for c in tok - ) - if has_decorative and only_decorative_or_punct and not has_ascii_alnum: - continue # pure decoration -> drop the whole token - kept.append(tok) - return unicodedata.normalize('NFKD', ' '.join(kept)) - - -# --------------------------------------------------------------------------- # -# Emoji-as-letter + emoji decoration normalization -# --------------------------------------------------------------------------- # -# Some streams use an emoji AS A LETTER inside a word: "SP⚽RTS" / "Sp⚽rts" where the -# soccer ball stands in for 'o' (= SPORTS, the beIN family). _strip_stylized_tokens keeps -# the token (it has ASCII alnum) and process_string_for_matching would turn the ball into a -# space ("sp rts"), so it never matches "sports". We substitute the glyph for the letter it -# replaces (only when flanked by ASCII letters) and strip emoji used purely as decoration. - -# Emoji that visually replace an ASCII letter when embedded in a word. Extensible. -_EMOJI_LETTER_MAP = {'⚽': 'o'} # SOCCER BALL = 'o' (SP⚽RTS -> SPORTS) -# Pictographic ornaments to delete. NOTE: ⚽ is intentionally in BOTH maps — the letter -# map handles it mid-word (-> 'o'); here it catches any ⚽ NOT flanked by ASCII letters -# (standalone/edge), which the substitution above leaves untouched. -_EMOJI_ORNAMENTS = frozenset('♬☾⚽') # beamed notes, last-quarter moon, soccer ball -# Zero-width / invisible code points that only add noise to a name. -_ZERO_WIDTH = ('️', '‍') # VARIATION SELECTOR-16, ZERO WIDTH JOINER - - -def _normalize_emoji(name): - """Map emoji-as-letters to their letter and strip emoji decoration. - - The letter substitution fires ONLY when the glyph is flanked by ASCII letters - (so "SP⚽RTS" -> "SPoRTS" but a standalone/edge "⚽" is treated as decoration and - dropped). Zero-width selectors and ornament pictographs are deleted outright. - ASCII-only input is returned unchanged (no emoji possible).""" - if name.isascii(): - return name - for zw in _ZERO_WIDTH: - if zw in name: - name = name.replace(zw, '') - for glyph, letter in _EMOJI_LETTER_MAP.items(): - if glyph in name: - name = re.sub(r'(?<=[A-Za-z])' + re.escape(glyph) + r'(?=[A-Za-z])', letter, name) - if any(c in _EMOJI_ORNAMENTS for c in name): - name = ''.join(c for c in name if c not in _EMOJI_ORNAMENTS) - return name - - -class FuzzyMatcher: +class FuzzyMatcher(FuzzyMatcherCore): """Handles fuzzy matching for Lineuparr with alias support and channel number boosting.""" + # Lineuparr strips bare time-zone region words (Pacific/Central/Mountain/Atlantic) for + # SCORING and enforces region correctness via its post-match region filter + # (match_all_streams reads the ORIGINAL names). See REGIONAL_BARE_PATTERNS in the core. + _STRIP_BARE_REGION = True + def __init__(self, match_threshold=80, logger=None): - self.match_threshold = match_threshold - self.logger = logger or LOGGER - # Cache for pre-normalized stream names (performance optimization) - self._norm_cache = {} # raw_name -> normalized_lower - self._norm_nospace_cache = {} # raw_name -> normalized_nospace - self._processed_cache = {} # raw_name -> processed_for_matching + # The core seeds match_threshold, logger, the four normalization/callsign + # caches, and the known-callsign rescue slot (unused here - Lineuparr has no + # channel DB). Only the country-filter counter is Lineuparr-specific. + super().__init__(match_threshold=match_threshold, logger=logger or LOGGER) # Cumulative count of candidates dropped by the country filter across # match_all_streams calls. Callers reset this before a matching loop # and read it after, so they can log a summary. self.country_filter_drops = 0 - # Cache for callsign extraction: raw_name -> (callsign|None, is_high_conf) - self._callsign_cache = {} def precompute_normalizations(self, names, user_ignored_tags=None): """ @@ -531,340 +350,6 @@ def _get_cached_processed(self, name, user_ignored_tags=None): return None return self.process_string_for_matching(norm) - def normalize_name(self, name, user_ignored_tags=None, ignore_quality=True, ignore_regional=True, - ignore_geographic=True, ignore_misc=True): - """ - Normalize channel or stream name for matching by removing tags, prefixes, and noise. - """ - if user_ignored_tags is None: - user_ignored_tags = [] - - original_name = name - - name = _LEADING_BAR_TAG_RE.sub('', name) # leading "┃CANAL+┃" bouquet tag - - # Map emoji-as-letters (⚽ = 'o' in "SP⚽RTS") and strip emoji decoration, before - # the stylized-Unicode strip and ASCII regexes below — so "beIN SP⚽RTS" -> "beIN sports". - name = _normalize_emoji(name) - - # Strip stylized-Unicode decoration (superscript/small-cap tier markers, - # bullets) up front so the ASCII tag regexes below see plain text. Runs - # unconditionally: a token written in superscript/small-caps is decoration - # regardless of tag_handling, and it would otherwise block matches - # (e.g. a superscript-RAW suffix never matches channel "WeatherNation"). - name = _strip_stylized_tokens(name) - - # Strip IPTV provider prefixes BEFORE hyphen normalization so that - # "FR - Canal+ FHD" loses its "FR - " while the hyphen is still a - # hyphen. After hyphen normalization the separator would become a space - # and the pattern would fail to match, leaving "FR" as a stray token. - for pattern in PROVIDER_PREFIX_PATTERNS: - name = re.sub(pattern, '', name, flags=re.IGNORECASE) - - # Quality patterns (before space normalization). Loop until stable so - # chained suffixes like "4K HDR" or "UHD HDR" are fully stripped in - # successive passes (each pass may expose a token for the next). - if ignore_quality: - prev = None - while prev != name: - prev = name - # Strip numeric resolution markers (3840P/2160P/1080P/720P/...) before the - # digit/letter spacer below would split "3840P" into "3840 P". - # Must run before QUALITY_PATTERNS so that removing " 4K " does not glue - # "SPoRTS" to "3840P" and break the word-boundary anchor. - for pattern in RESOLUTION_PATTERNS: - name = re.sub(pattern, '', name, flags=re.IGNORECASE) - for pattern in QUALITY_PATTERNS: - name = re.sub(pattern, '', name, flags=re.IGNORECASE) - - # Normalize spacing around numbers - name = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', name) - name = re.sub(r'(\d)([a-zA-Z])', r'\1 \2', name) - - # Normalize hyphens to spaces - name = re.sub(r'-', ' ', name) - - # Replace dots between word chars with spaces (e.g. "JusticeCentral.TV" - # → "JusticeCentral TV"). Keeps the dot-suffix variant equivalent to - # the spaced form for matching purposes. - name = re.sub(r'(?<=\w)\.(?=\w)', ' ', name) - - # Normalize number-words to digits so "BBC Three" and "BBC 3" share - # tokens. Critical for cases like "Three Angels Broadcasting Network" - # vs "3 Angels Broadcasting Network", and for BBC One/Two/Three/Four - # vs BBC 1/2/3/4. Bounded by word boundaries so brand names with - # embedded letters (e.g. "Onesimus") aren't corrupted. - _NUM_WORDS = { - "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", - "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10", - "eleven": "11", "twelve": "12", - } - def _num_repl(m): - return _NUM_WORDS[m.group(0).lower()] - name = re.sub( - r'\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\b', - _num_repl, name, flags=re.IGNORECASE, - ) - - # Split CamelCase boundaries so "JusticeCentral" becomes "Justice - # Central" and "DangerTV" becomes "Danger TV". Two separate patterns: - # 1. lower → Upper followed by lower (Justice|Central, Danger|Iq → no, etc.) - # 2. lower(4+) → UPPER acronym at word boundary (Danger|TV, Beauty|IQ) - # The 4-char floor on rule 2 protects short brand names like "MeTV" and - # "truTV" whose existing EPG matches rely on the un-split form. - name = re.sub(r'([a-z])([A-Z][a-z])', r'\1 \2', name) - name = re.sub(r'([a-z]{4,})([A-Z]{2,})\b', r'\1 \2', name) - - # Strip region tokens (East / Eastern / West and the (E)/(W) - # abbreviations) for SCORING. Region CORRECTNESS - East vs West vs - # Pacific - is enforced separately by the post-match region filter in - # match_all_streams, which reads the ORIGINAL un-normalized names, not - # this output. Stripping here lets a regionless lineup channel - # ("Food Network") still score-match a region-tagged stream - # ("Food Network Eastern" / "... East") instead of being dragged below - # threshold by the extra token; the filter then accepts it (regionless - # defaults to East). "Western"/"Westerns" is a movie genre, NOT a - # region, so bare "west" is stripped only on a word boundary (\bwest\b - # does not match inside "western") and "western" is never touched. - name = re.sub(r'\(\s*(?:eastern|east|e|west|w)\s*\)', ' ', name, flags=re.IGNORECASE) - name = re.sub(r'\b(?:eastern|east|west)\b', ' ', name, flags=re.IGNORECASE) - - # Remove leading parenthetical prefixes - while name.lstrip().startswith('('): - new_name = re.sub(r'^\s*\([^\)]+\)\s*', '', name) - if new_name == name: - break - name = new_name - - # Build pattern list based on flags - patterns_to_apply = [] - if ignore_regional: - patterns_to_apply.extend(REGIONAL_PATTERNS) - if ignore_geographic: - patterns_to_apply.extend(GEOGRAPHIC_PATTERNS) - if ignore_misc and ignore_regional: - patterns_to_apply.extend(MISC_PATTERNS) - - for pattern in patterns_to_apply: - name = re.sub(pattern, '', name, flags=re.IGNORECASE) - - # Apply user-configured ignored tags - for tag in user_ignored_tags: - escaped_tag = re.escape(tag) - if '[' in tag or ']' in tag or '(' in tag or ')' in tag: - name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) - else: - if re.match(r'^\w+$', tag): - name = re.sub(r'\b' + escaped_tag + r'\b', '', name, flags=re.IGNORECASE) - else: - name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) - - # Remove callsigns in parentheses - if ignore_regional: - name = re.sub(r'\([KW][A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) - else: - name = re.sub(r'\([KW](?!EST\)|ACIFIC\)|ENTRAL\)|OUNTAIN\)|TLANTIC\))[A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) - - if ignore_regional: - name = re.sub(r'\([A-Z0-9]+\)', '', name) - - # Remove common suffixes/prefixes - name = re.sub(r'^The\s+', '', name, flags=re.IGNORECASE) - name = re.sub(r'\s+Network\s*$', '', name, flags=re.IGNORECASE) - name = re.sub(r'\s+Channel\s*$', '', name, flags=re.IGNORECASE) - name = re.sub(r'\s+TV\s*$', '', name, flags=re.IGNORECASE) - - # Strip decorative Unicode markers (◉, ², superscript letters ᴿᴬᵂ…) - # that some IPTV providers append as quality/status badges. Only - # characters in decorator categories are removed; accented letters - # (é, î, ü…) are in Ll/Lu and are preserved. - name = ''.join( - ' ' if unicodedata.category(c) in _DECORATOR_CATS else c - for c in name - ) - - # Clean up whitespace - name = re.sub(r'\s+', ' ', name).strip() - - if not name: - self.logger.debug(f"normalize_name returned empty for: '{original_name}'") - - return name - - def calculate_similarity(self, str1, str2, min_ratio=0.0): - """Levenshtein similarity ratio (0.0-1.0), defined as 1 - distance/max(len). - If min_ratio > 0, returns 0.0 early when the result can't reach it. - - bug-026: the ratio MUST be distance/max(len), matching rapidfuzz - Levenshtein.normalized_similarity. The old (len1+len2-distance)/(len1+len2) - formula scored higher for the same edit distance and let numbered siblings - ("Fox Sports 1" vs "2") pass threshold 95. The rapidfuzz fast path and the - pure-Python fallback below compute the identical value. - """ - if len(str1) == 0 or len(str2) == 0: - return 0.0 - - # Fast path: C-accelerated rapidfuzz when available (same definition). - if _USE_RAPIDFUZZ: - # No score_cutoff: rapidfuzz's cutoff rounding zeroes a score landing - # exactly on min_ratio, but the pure-Python path returns it. Dropping - # the cutoff makes the two paths agree at the threshold boundary; - # callers apply their own >= comparison. - return _rf_lev.normalized_similarity(str1, str2) - - if len(str1) < len(str2): - str1, str2 = str2, str1 - len1, len2 = len(str1), len(str2) - max_len = len1 # the longer string after the swap - - # Length-difference pre-check: minimum possible distance is (len1 - len2), - # so the max possible ratio is (max_len - (len1 - len2)) / max_len. - if min_ratio > 0: - max_possible = (max_len - (len1 - len2)) / max_len - if max_possible < min_ratio: - return 0.0 - - previous_row = list(range(len2 + 1)) - for i, c1 in enumerate(str1): - current_row = [i + 1] - for j, c2 in enumerate(str2): - insertions = previous_row[j + 1] + 1 - deletions = current_row[j] + 1 - substitutions = previous_row[j] + (c1 != c2) - current_row.append(min(insertions, deletions, substitutions)) - # Early termination: a lower bound on the final distance is the current - # row minimum minus the str1 chars still unprocessed. - if min_ratio > 0: - min_distance_so_far = min(current_row) - remaining = len1 - i - 1 - best_possible_distance = max(0, min_distance_so_far - remaining) - best_possible_ratio = (max_len - best_possible_distance) / max_len - if best_possible_ratio < min_ratio: - return 0.0 - previous_row = current_row - - distance = previous_row[-1] - return (max_len - distance) / max_len - - @staticmethod - def _length_scaled_threshold(base_threshold, shorter_len): - """Require higher similarity for shorter strings to avoid false positives.""" - if shorter_len <= 4: - return max(base_threshold, 95) - elif shorter_len <= 8: - return max(base_threshold, 90) - return base_threshold - - @staticmethod - def _has_token_overlap(str_a, str_b, min_token_len=4, require_majority=False): - """Check that distinctive tokens are shared between two strings. - - Basic mode: at least one token (>= min_token_len) must be shared. - Majority mode: uses all tokens (>= 2 chars) and requires that more than - half of the smaller set overlaps. Catches false positives like - "america racing" vs "america bbc" while allowing single-token matches. - """ - # "network"/"channel"/"television" are generic brand-suffix words, not - # distinctive - treat as common so the subset guard does not reject - # cases like "FanDuel Sports Cincinnati" vs "FanDuel Sports Network - # Cincinnati". (They're already stripped from end-of-string by - # normalize_name; this catches mid-string occurrences.) - common_words = { - "the", "and", "of", "in", "on", "at", "to", "for", "a", "an", - "network", "channel", "television", - } - - if require_majority: - # Use all meaningful tokens (>= 2 chars) for stricter checking. - # Single-digit tokens (1, 2, 3, ...) are kept because they're - # channel-distinguishing (BBC 1 vs BBC 2, ESPN 1 vs ESPN 2 etc.) - # even though they're only 1 char. - def _meaningful(t): - if t in common_words: - return False - return len(t) >= 2 or t.isdigit() - tokens_a = {t for t in str_a.split() if _meaningful(t)} - tokens_b = {t for t in str_b.split() if _meaningful(t)} - if not tokens_a or not tokens_b: - return True - shared = tokens_a & tokens_b - if not shared: - return False - smaller = min(len(tokens_a), len(tokens_b)) - if not len(shared) > smaller / 2: - return False - - unique_a = tokens_a - tokens_b - unique_b = tokens_b - tokens_a - - # Subset guard: when one side is a strict subset of the other and - # the larger side has a distinctive token the smaller lacks, the - # candidate is a more specific channel than the query and the high - # fuzzy score is a false positive. Catches e.g. "Nickelodeon" vs - # "Nickelodeon Teen". Threshold is >=4 chars; "live"/"now" are - # explicitly non-distinctive (stream label variants) so "ABC News" - # still matches "ABC News Live". Pure-digit tokens >=2 chars - # (e.g. "360") are also distinctive: "Canal+Sport" must not match - # "Canal+Sport 360". - if not unique_a: - if any(_is_distinctive(t) for t in unique_b): - return False - elif not unique_b: - if any(_is_distinctive(t) for t in unique_a): - return False - - # Divergent guard: when BOTH sides have unique tokens AND at least - # one of those unique tokens is a distinctive (>=4 char) word, the - # strings describe different brands and the fuzzy score is - # misleading. Catches "Sky Cinema Disney" vs "Sky Cinema Decades" - # (decades = 7 chars), "Sky Cinema Fast" vs "Sky Cinema Family", - # and "BBC One vs BBC Two" once number-words become digits earlier. - # Number-word→digit normalization (in normalize_name) reduces this - # guard's risk: "Three Angels" / "3 Angels" both become "3 angels" - # before reaching here, so they never trigger this case. - if unique_a and unique_b: - if any(len(t) >= 4 for t in unique_a | unique_b): - return False - - # Numeric/ordinal divergent guard: when BOTH sides have a numeric - # or ordinal token unique to them, they are sibling channels - # distinguished by number (BBC One vs BBC Two; ESPN 1 vs ESPN 2). - # Short tokens like "one"/"two" wouldn't trip the divergent guard - # above so they need their own check. - _NUMERIC = { - "one", "two", "three", "four", "five", "six", "seven", "eight", - "nine", "ten", "eleven", "twelve", - "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", - "first", "second", "third", "fourth", "fifth", - } - if (unique_a & _NUMERIC) and (unique_b & _NUMERIC): - return False - - return True - - # Basic mode: at least one long token shared - tokens_a = {t for t in str_a.split() if t not in common_words and len(t) >= min_token_len} - tokens_b = {t for t in str_b.split() if t not in common_words and len(t) >= min_token_len} - if not tokens_a or not tokens_b: - return True - return bool(tokens_a & tokens_b) - - def process_string_for_matching(self, s): - """Normalize for token-sort matching: NFKD-fold (so "ＨＢＯ"->"hbo", "²"->"2", - "fi"->"fi", accents drop), lowercase, keep alphanumerics of any script - (isalnum keeps Cyrillic/CJK/Arabic rather than erasing them), sort tokens.""" - s = unicodedata.normalize('NFKD', s) - s = ''.join(char for char in s if unicodedata.category(char) != 'Mn') - s = s.lower() - s = re.sub(r'([^\W\d_])(\d)', r'\1 \2', s) # split letter-glued digit, any script - cleaned_s = "" - for char in s: - if char.isalnum(): - cleaned_s += char - else: - cleaned_s += ' ' - tokens = sorted([token for token in cleaned_s.split() if token]) - return " ".join(tokens) @staticmethod def _is_group_header(name): @@ -875,15 +360,6 @@ def _is_group_header(name): is a common in-name separator ("001 | Team A vs Team B").""" return bool(re.search(r'[#=*]{2,}|\|{3,}', name or "")) - @staticmethod - def _trailing_number(name): - """Return the integer value of a space-separated, purely-numeric - trailing token, or None. 'HBO 2' -> 2, 'DIRECTV 4K Live 1' -> 1, - 'ESPN' -> None, 'ESPN2' -> None (digit not space-separated). Used to - reject 'Foo 1' vs 'Foo 2' false positives -- different channels that - otherwise fuzzy-match almost perfectly.""" - m = re.search(r'(?:^|\s)(\d{1,4})\s*$', name or "") - return int(m.group(1)) if m else None def _channel_number_boost(self, stream_name, expected_number): """ @@ -902,103 +378,6 @@ def _channel_number_boost(self, stream_name, expected_number): return 5 return 0 - # Words that match the callsign regex shape but are never US broadcast - # callsigns. A US callsign (K/W + 2-4 letters) is shape-identical to many - # common English words, so the loose Priority-4 pattern mis-extracts them - - # e.g. "with" in "Bizarre Foods with Andrew Zimmern" becomes callsign - # "WITH". Regex alone cannot tell "WITH" from "WABC"; frequent K/W-initial - # words are denied explicitly so they never extract as a callsign. WWE/WWF/ - # WCW stop wrestling show names extracting as false-positive callsigns. - _CALLSIGN_DENYLIST = frozenset({ - 'WWE', 'WWF', 'WCW', 'EAST', - 'WAR', 'WARS', 'WARM', 'WASH', 'WATCH', 'WAVE', 'WAVES', 'WAY', 'WAYS', - 'WEB', 'WEEK', 'WELL', 'WENT', 'WERE', 'WEST', 'WHAT', 'WHEN', 'WHERE', - 'WHICH', 'WHILE', 'WHITE', 'WHO', 'WHY', 'WIDE', 'WIFE', 'WILD', 'WILL', - 'WIND', 'WINE', 'WING', 'WINGS', 'WINS', 'WIRE', 'WISE', 'WISH', 'WITH', - 'WOLF', 'WOMAN', 'WOMEN', 'WOOD', 'WORD', 'WORDS', 'WORK', 'WORKS', - 'WORLD', 'WORM', 'WORN', 'WRAP', - 'KEEN', 'KEEP', 'KEPT', 'KEY', 'KEYS', 'KICK', 'KID', 'KIDS', 'KILL', - 'KIND', 'KING', 'KINGS', 'KISS', 'KITE', 'KNEE', 'KNEW', 'KNOW', 'KNOWN', - }) - - def _compute_callsign_with_confidence(self, channel_name): - """ - Extract US TV callsign with a confidence flag. - - Returns (callsign, is_high_confidence). High confidence = - Priorities 1-3 (parenthesized / suffixed-paren / end-of-name). - Priority 4 (any loose word) is low confidence. (None, False) - when nothing extractable. - """ - # Remove common provider prefixes - channel_name = re.sub(r'^D\d+-', '', channel_name) - channel_name = re.sub(r'^USA?\s*[^a-zA-Z0-9]*\s*', '', channel_name, flags=re.IGNORECASE) - - # Priority 1: Callsigns in parentheses (most reliable) - paren_match = re.search(r'\(([KW][A-Z]{3})(?:-[A-Z\s]+)?\)', channel_name, re.IGNORECASE) - if paren_match: - callsign = paren_match.group(1).upper() - if callsign not in self._CALLSIGN_DENYLIST: - return callsign, True - - # Priority 1b: grandfathered 3-letter callsigns in parentheses without a suffix - # (WWL/WJZ/KYW/WRC). Suffixed forms fall through to Priority 2. bug-062. - paren3_match = re.search(r'\(([KW][A-Z]{2})\)', channel_name, re.IGNORECASE) - if paren3_match: - callsign = paren3_match.group(1).upper() - if callsign not in self._CALLSIGN_DENYLIST: - return callsign, True - - # Priority 2: Callsigns with suffix in parentheses - paren_suffix_match = re.search(r'\(([KW][A-Z]{2,4}-(?:TV|CD|LP|DT|LD))\)', channel_name, re.IGNORECASE) - if paren_suffix_match: - return paren_suffix_match.group(1).upper(), True - - # Priority 3: Callsigns at the end - end_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\s*(?:\.[a-z]+)?\s*$', channel_name, re.IGNORECASE) - if end_match: - callsign = end_match.group(1).upper() - if callsign not in self._CALLSIGN_DENYLIST: - return callsign, True - - # Priority 4: Any word matching callsign pattern (low confidence) - word_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\b', channel_name, re.IGNORECASE) - if word_match: - callsign = word_match.group(1).upper() - if callsign not in self._CALLSIGN_DENYLIST: - return callsign, False - - return None, False - - def _extract_callsign_with_confidence(self, channel_name): - """ - Cached wrapper around _compute_callsign_with_confidence. - - Extraction is pure in channel_name, so results are memoized - the - anchor calls this once per stream over a fixed candidate list, which - is otherwise massively redundant across the per-channel matching - loop. Cache is cleared by precompute_normalizations. - """ - cached = self._callsign_cache.get(channel_name) - if cached is not None: - return cached - result = self._compute_callsign_with_confidence(channel_name) - self._callsign_cache[channel_name] = result - return result - - def extract_callsign(self, channel_name): - """ - Extract US TV callsign from channel name with priority order. - Returns None if common false positives appear alone. - """ - callsign, _ = self._extract_callsign_with_confidence(channel_name) - return callsign - - def normalize_callsign(self, callsign): - """Remove the broadcast suffix (-TV/-CD/-LP/-DT/-LD) from a callsign.""" - if callsign: - callsign = re.sub(r'-(?:TV|CD|LP|DT|LD)$', '', callsign) - return callsign def alias_match(self, lineup_name, candidate_names, alias_map, user_ignored_tags=None): """ diff --git a/Lineuparr/matching_core.py b/Lineuparr/matching_core.py new file mode 100644 index 0000000..d7e9147 --- /dev/null +++ b/Lineuparr/matching_core.py @@ -0,0 +1,793 @@ +""" +matching_core.py - shared pure matching primitives for the Dispatcharr plugins. + +The canonical, vendored core extracted from Lineuparr's fuzzy_matcher.py (v1.3.4, the +newest generation). It is PURE and stateless string-in/string-out: stdlib + optional +rapidfuzz only, with no Django / ORM / filesystem coupling. Each plugin SUBCLASSES +FuzzyMatcherCore and layers its own orchestration on top (DB load, matching entry points, +and one feature block: zones / token-index / country / OTA). See +MATCHER-STANDARDIZATION-PLAN.md. + +Authored faithfully from Lineuparr with exactly these deliberate changes: + - class renamed FuzzyMatcher -> FuzzyMatcherCore; Lineuparr's country layer, the + matching entry points (alias_match / fuzzy_match / match_all_streams), the cache glue + (precompute_normalizations / _get_cached_*), _channel_number_boost, _is_group_header, + and has_upgrade_quality are NOT included - they stay plugin-side. + - process_string_for_matching KEEPS '+' so Disney+ / Discovery+ / Paramount+ stay + distinct from their base channels (the 4-of-4 superset decision). + - the callsign confidence ladder consults an optional self._known_callsigns set so a + plugin can rescue a denylisted-but-real station (KING / WAVE / WOOD / WHO) from its own + channel DB. With the default (None) the ladder is identical to Lineuparr's pure path. + +This file is the SOURCE OF TRUTH. It is vendored (copied byte-identically) into each +plugin's flat inner folder at release time and hash-pinned; never edit a vendored copy - +edit this file and re-sync. +""" + +import logging +import re +import unicodedata + +# Optional C-accelerated Levenshtein. When present, the matcher uses rapidfuzz's +# normalized_similarity (1 - distance/max(len)); the pure-Python fallback below +# computes the identical value (bug-026). rapidfuzz is an OPTIONAL runtime dep. +try: + from rapidfuzz.distance import Levenshtein as _rf_lev + _USE_RAPIDFUZZ = True +except ImportError: + _USE_RAPIDFUZZ = False + +__version__ = "0.1.0" + +LOGGER = logging.getLogger("matching_core") +if not LOGGER.handlers: + _handler = logging.StreamHandler() + _handler.setFormatter(logging.Formatter("%(levelname)s %(name)s %(message)s")) + LOGGER.addHandler(_handler) +# WARNING by default so a vendored core does not spam each plugin's logs with the +# per-name "normalize_name returned empty" debug line; a plugin can pass its own +# logger at a lower level when diagnosing. +LOGGER.setLevel(logging.WARNING) + +# --- Pattern categories for normalization --- + +# Unicode categories considered decorative/badge characters by IPTV providers. +# So = Other Symbol (◉), No = Other Number (², ³), Lm = Modifier Letter +# (ᴿᴬᵂ, ᴴᴰ, ⱽᴵᴾ superscripts), Sk = Modifier Symbol. +# Accented letters (é, î, ü…) are Ll/Lu and are NOT in this set. +# Sm (Math Symbol) is intentionally EXCLUDED: it contains "+", which is a +# meaningful, channel-distinguishing character (Canal+, Three Stooges+, +# Comedy Central+). Stripping it regresses those matches. +_DECORATOR_CATS = frozenset({'So', 'No', 'Lm', 'Sk'}) + +# Tokens that are non-distinctive stream-label variants (e.g. "ABC News Live" +# should still match "ABC News"). Used by the subset/divergent guards. +_NON_DISTINCTIVE_TOKENS = frozenset({"live", "now", "new"}) + +def _is_distinctive(t): + """Return True if token t is distinctive enough to matter in subset/divergent guards.""" + return t not in _NON_DISTINCTIVE_TOKENS and (len(t) >= 4 or (t.isdigit() and len(t) >= 2)) + +QUALITY_PATTERNS = [ + r'\s*\[(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\]\s*', + r'\s*\((4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\)\s*', + r'^\s*(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s*', + r'\s*\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)$', + r'\s+\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s+', +] + +# Numeric resolution markers the keyword QUALITY_PATTERNS miss: 720p, 1080p/i, 2160p, +# 3840P, 480p, etc. - a 3-4 digit run glued directly to p/i. The 3-digit lower bound +# excludes 2-digit noise; the 4-digit upper bound excludes 5-digit numbers (10800p won't +# match). The p/i must be GLUED to the digits (no space): real markers are always written +# "720P"/"3840P", and requiring the glue avoids stripping a spaced standalone P/I such as a +# roman numeral ("Volume 100 I"). The p/i \b anchor keeps bare numbers (1080, "Channel 4") +# intact. Applied with re.IGNORECASE in the ignore_quality block, like QUALITY_PATTERNS. +RESOLUTION_PATTERNS = [ + r'\b\d{3,4}[pi]\b', +] + +REGIONAL_PATTERNS = [ + # East/West are NOT stripped here - they distinguish separate channel feeds + # ("HBO East" vs "HBO West"); normalize_name strips bare East/West separately, + # gated on ignore_regional. + # bug-066: bare " Pacific"/" Central"/" Mountain"/" Atlantic" are brand tokens far + # more often than feed markers ("Comedy Central", "The Atlantic"), so they are + # stripped ONLY in their parenthesized form, never as bare words. + r'\s*\([Pp][Aa][Cc][Ii][Ff][Ii][Cc]\)\s*', + r'\s*\([Cc][Ee][Nn][Tt][Rr][Aa][Ll]\)\s*', + r'\s*\([Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]\)\s*', + r'\s*\([Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]\)\s*', +] + +# bug-066 (opt-in half): the BARE-word forms of the time-zone region markers. OFF by +# default - a plugin enables them by setting the class attr _STRIP_BARE_REGION = True +# (or passing strip_bare_region=True). Lineuparr opts in: it strips these for SCORING +# and relies on its post-match region filter - which reads the ORIGINAL un-normalized +# name - to enforce region correctness, so a "Sportsnet Pacific" stream can score-match +# a "Sportsnet (P)" lineup and an "HBO West" lineup keeps the "HBO Pacific" feed. Plugins +# WITHOUT that filter (Stream-Mapparr) leave it off, or brand tokens like "Comedy Central" +# / "The Atlantic" would be wrongly truncated. +REGIONAL_BARE_PATTERNS = [ + r'\s[Pp][Aa][Cc][Ii][Ff][Ii][Cc]', + r'\s[Cc][Ee][Nn][Tt][Rr][Aa][Ll]', + r'\s[Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]', + r'\s[Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]', +] + +# Country tokens for the delimited provider-prefix patterns below; curated so a +# bare delimited word ("(SPORTS)") isn't misread as a country. +_PREFIX_COUNTRY = r'US|USA|UK|CA|AU|FR|DE|ES|IT|NL|BR|MX|IN' + +# (open, close) pairs; open and close must MATCH, so "(US]" / "│US)" are rejected. +_DELIM_PAIRS = ((r'\(', r'\)'), (r'\[', r'\]'), (r'\|', r'\|'), ('┃', '┃'), ('│', '│')) + + +def _balanced_delim(token): + """Regex fragment matching `token` wrapped in one MATCHED delimiter pair: + (token) [token] |token| ┃token┃ │token│. `token` may contain a capture group.""" + return '(?:' + '|'.join( + o + r'\s*(?:' + token + r')\s*' + c for o, c in _DELIM_PAIRS + ) + ')' + + +# Strip a leading box-bar bouquet/source tag with arbitrary inner text +# ("┃CANAL+┃ NPO 1" -> "NPO 1"); box bars never occur in real names, so this is +# always safe and also covers leading "┃XX┃" country tags. +_LEADING_BAR_TAG_RE = re.compile(r'^\s*[┃│]\s*[^┃│]*[┃│]\s*') + +GEOGRAPHIC_PATTERNS = [ + r'\b[A-Z]{2,3}[:┃│]\s*', + r'\b[A-Z]{2,3}\s*-\s*', + # Matched bar pair only ("|US|", "┃US┃") - a mismatched "|US┃" is noise. + r'(?:\|[A-Z]{2,3}\||┃[A-Z]{2,3}┃|│[A-Z]{2,3}│)\s*', + r'\[[A-Z]{2,3}\]\s*', +] + +# Enhanced provider prefix patterns for IPTV-specific naming +PROVIDER_PREFIX_PATTERNS = [ + r'^(?:' + _PREFIX_COUNTRY + r')\s*[:\-\|┃│]\s*', + # Bare country tag + whitespace, no separator (e.g. "US Racer Network", + # "FR beIN SPORTS MAX", "MEX Bein Sports"). Restricted to a curated set so + # it cannot eat a real channel name: "USA Network" (USA != US + space), + # "In Country Television" ("IN") and "IT Crowd" ("IT") are all safe too. + r'^(?:US|UK|CA|AU|FR|DE|MX|MEX|FRA|GER)\s+', + # "USA " space prefix as a US country tag ("USA ABC", "USA BET"). A + # negative lookahead for NETWORK protects the real channel "USA Network" + # (these feeds tag that one as "US ..."/"US: ...", never bare "USA "). + r'^USA\s+(?!NETWORK\b)', + # Country code glued directly to a quality tag with no separator + # ("UKSD: Sky Sports", "UKHD ESPN", "USFHD ..."). + r'^(?:US|UK)(?:SD|HD|FHD|UHD|FD|HEVC|4K|8K)\b\s*[:\-\|┃│]?\s*', + # Bracketed/piped country tag with a MATCHED delimiter pair ("(US)", "│US│"). + r'^\s*' + _balanced_delim(_PREFIX_COUNTRY) + r'\s*', + r'\s*[\|┃│]\s*(?:' + _PREFIX_COUNTRY + r')\s*$', + # Content-category group prefixes used by some IPTV providers. + r'^(?:ADULT|EROTIC|PRIME|GOLD)\s*[:\-\|┃│]\s*', + # FAST streaming-platform source tags (Roku, Tubi, Pluto, etc.). These mark + # the distribution platform, not the channel or its country, so strip them + # for matching ("RK: beIN Sports Xtra" -> "beIN Sports Xtra"). A separator + # is required so this can't eat real names like "GOLF" or "PLEX TV Movies". + r'^(?:RK|GO|TUBI|PLUTO|XUMO|PLEX|STIRR|FREEVEE|GLANCE)\s*[:\-\|┃│]\s*', +] + +MISC_PATTERNS = [ + r'\s*\([^)]*\)\s*', +] + +# --------------------------------------------------------------------------- # +# Stylized-Unicode decoration stripping +# --------------------------------------------------------------------------- # +# Streams tag names with stylized-Unicode tier/format markers (superscript +# "WEATHERNATION RAW", small-cap "FHD", bullet-prefixed "CNN") that the ASCII tag +# regexes below cannot see. We drop whole tokens that are pure decoration BEFORE +# the ASCII pipeline runs. Detection is by Unicode character *name* (not code-point +# ranges), so it covers superscripts, "modifier letter" superscript capitals, and +# Latin small-caps wherever they live (e.g. small-cap H is U+029C in IPA Extensions +# and modifier V is U+2C7D in Latin-Ext-C, both outside the obvious blocks). + +# Ornament glyphs whose Unicode name carries no decoration keyword. +_DECORATIVE_SYMBOLS = frozenset("◉") # FISHEYE; add individual chars (not strings) here + + +def _is_decorative_char(ch): + """True for a stylized letterform/ornament that carries no semantic content in a + channel name (superscripts, subscripts, modifier-letter superscript capitals, + Latin small-capitals, curated bullets). ASCII and ordinary letters return False.""" + if ch.isascii(): + return False + if ch in _DECORATIVE_SYMBOLS: + return True + try: + nm = unicodedata.name(ch) + except ValueError: + # unnamed code point (control char / lone surrogate) -> not decoration + return False + return ('SUPERSCRIPT' in nm or 'SUBSCRIPT' in nm + or 'SMALL CAPITAL' in nm or 'MODIFIER LETTER' in nm) + + +def _strip_stylized_tokens(name): + """Drop whitespace tokens that are pure stylized decoration, then NFKD-canonicalize + the remainder. A token is decoration when it has >=1 decorative char, no ASCII + alphanumeric, and every char is decorative or ASCII punctuation (so a bullet glued + to a colon, or "HD/RAW" written in superscripts, are dropped too). Real ASCII words + (Gold/VIP) and non-Latin letters (Arabic/Cyrillic/CJK) are always kept. ASCII-only + input is returned unchanged via the fast path (no per-char work; NFKD is a no-op + on ASCII, so skipping it changes nothing).""" + if name.isascii(): + return name + kept = [] + for tok in name.split(): + has_decorative = any(_is_decorative_char(c) for c in tok) + has_ascii_alnum = any(c.isascii() and c.isalnum() for c in tok) + only_decorative_or_punct = all( + _is_decorative_char(c) or (c.isascii() and not c.isalnum()) for c in tok + ) + if has_decorative and only_decorative_or_punct and not has_ascii_alnum: + continue # pure decoration -> drop the whole token + kept.append(tok) + return unicodedata.normalize('NFKD', ' '.join(kept)) + + +# --------------------------------------------------------------------------- # +# Emoji-as-letter + emoji decoration normalization +# --------------------------------------------------------------------------- # +# Some streams use an emoji AS A LETTER inside a word: "SP⚽RTS" / "Sp⚽rts" where the +# soccer ball stands in for 'o' (= SPORTS, the beIN family). _strip_stylized_tokens keeps +# the token (it has ASCII alnum) and process_string_for_matching would turn the ball into a +# space ("sp rts"), so it never matches "sports". We substitute the glyph for the letter it +# replaces (only when flanked by ASCII letters) and strip emoji used purely as decoration. + +# Emoji that visually replace an ASCII letter when embedded in a word. Extensible. +_EMOJI_LETTER_MAP = {'⚽': 'o'} # SOCCER BALL = 'o' (SP⚽RTS -> SPORTS) +# Pictographic ornaments to delete. NOTE: ⚽ is intentionally in BOTH maps - the letter +# map handles it mid-word (-> 'o'); here it catches any ⚽ NOT flanked by ASCII letters +# (standalone/edge), which the substitution above leaves untouched. +_EMOJI_ORNAMENTS = frozenset('♬☾⚽') # beamed notes, last-quarter moon, soccer ball +# Zero-width / invisible code points that only add noise to a name. +_ZERO_WIDTH = ('️', '‍') # VARIATION SELECTOR-16, ZERO WIDTH JOINER + + +def _normalize_emoji(name): + """Map emoji-as-letters to their letter and strip emoji decoration. + + The letter substitution fires ONLY when the glyph is flanked by ASCII letters + (so "SP⚽RTS" -> "SPoRTS" but a standalone/edge "⚽" is treated as decoration and + dropped). Zero-width selectors and ornament pictographs are deleted outright. + ASCII-only input is returned unchanged (no emoji possible).""" + if name.isascii(): + return name + for zw in _ZERO_WIDTH: + if zw in name: + name = name.replace(zw, '') + for glyph, letter in _EMOJI_LETTER_MAP.items(): + if glyph in name: + name = re.sub(r'(?<=[A-Za-z])' + re.escape(glyph) + r'(?=[A-Za-z])', letter, name) + if any(c in _EMOJI_ORNAMENTS for c in name): + name = ''.join(c for c in name if c not in _EMOJI_ORNAMENTS) + return name + + +class FuzzyMatcherCore: + """Pure, stateless matching primitives shared across the Dispatcharr plugins. + + Subclass this and add the plugin-specific orchestration layer (DB load, matching + entry points, and one feature block). The core references self only for self.logger + and the per-instance caches; it never reads plugin_dir / a channel DB / alias maps. + """ + + def __init__(self, match_threshold=80, logger=None): + self.match_threshold = match_threshold + self.logger = logger or LOGGER + # Cache for pre-normalized stream names (performance optimization). The core + # initializes the storage; the cache-filling glue (precompute_normalizations / + # _get_cached_*) is plugin-side because its flag handling is plugin-specific. + self._norm_cache = {} # raw_name -> normalized_lower + self._norm_nospace_cache = {} # raw_name -> normalized_nospace + self._processed_cache = {} # raw_name -> processed_for_matching + # Cache for callsign extraction: raw_name -> (callsign|None, is_high_conf) + self._callsign_cache = {} + # Optional set of known-real callsigns a plugin supplies from its channel DB so + # the ladder can rescue a denylisted-but-real station (KING/WAVE/WOOD/WHO). None + # = no rescue, identical to the pure Lineuparr ladder. Assign it via + # set_known_callsigns() so the callsign cache is cleared when it changes. + self._known_callsigns = None + + def set_known_callsigns(self, known): + """Supply the plugin's known-real callsign set (or None to disable rescue). + + Clears the callsign cache so a changed set takes effect on the next extraction. + """ + self._known_callsigns = frozenset(known) if known else None + self._callsign_cache.clear() + + def normalize_name(self, name, user_ignored_tags=None, ignore_quality=True, ignore_regional=True, + ignore_geographic=True, ignore_misc=True, remove_cinemax=False, + remove_country_prefix=False, strip_bare_region=None): + """ + Normalize channel or stream name for matching by removing tags, prefixes, and noise. + + remove_cinemax / remove_country_prefix are optional opt-in superset behaviors (a + plugin sets them True); both default False, so the default path is unchanged. + + strip_bare_region opts into stripping BARE time-zone region words (Pacific/Central/ + Mountain/Atlantic) for scoring - see REGIONAL_BARE_PATTERNS. None (the default) + resolves it from the subclass class attr _STRIP_BARE_REGION (default False), so a + plugin enables it once via that attr instead of threading it through every call. + """ + if user_ignored_tags is None: + user_ignored_tags = [] + if strip_bare_region is None: + strip_bare_region = getattr(self, "_STRIP_BARE_REGION", False) + + original_name = name + + name = _LEADING_BAR_TAG_RE.sub('', name) # leading "┃CANAL+┃" bouquet tag + + # Map emoji-as-letters (⚽ = 'o' in "SP⚽RTS") and strip emoji decoration, before + # the stylized-Unicode strip and ASCII regexes below - so "beIN SP⚽RTS" -> "beIN sports". + name = _normalize_emoji(name) + + # Strip stylized-Unicode decoration (superscript/small-cap tier markers, + # bullets) up front so the ASCII tag regexes below see plain text. Runs + # unconditionally: a token written in superscript/small-caps is decoration + # regardless of tag_handling, and it would otherwise block matches + # (e.g. a superscript-RAW suffix never matches channel "WeatherNation"). + name = _strip_stylized_tokens(name) + + # Strip IPTV provider prefixes BEFORE hyphen normalization so that + # "FR - Canal+ FHD" loses its "FR - " while the hyphen is still a + # hyphen. After hyphen normalization the separator would become a space + # and the pattern would fail to match, leaving "FR" as a stray token. + for pattern in PROVIDER_PREFIX_PATTERNS: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + + # Quality patterns (before space normalization). Loop until stable so + # chained suffixes like "4K HDR" or "UHD HDR" are fully stripped in + # successive passes (each pass may expose a token for the next). + if ignore_quality: + prev = None + while prev != name: + prev = name + # Strip numeric resolution markers (3840P/2160P/1080P/720P/...) before the + # digit/letter spacer below would split "3840P" into "3840 P". + # Must run before QUALITY_PATTERNS so that removing " 4K " does not glue + # "SPoRTS" to "3840P" and break the word-boundary anchor. + for pattern in RESOLUTION_PATTERNS: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + for pattern in QUALITY_PATTERNS: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + + # Normalize spacing around numbers + name = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', name) + name = re.sub(r'(\d)([a-zA-Z])', r'\1 \2', name) + + # Normalize hyphens to spaces + name = re.sub(r'-', ' ', name) + + # Replace dots between word chars with spaces (e.g. "JusticeCentral.TV" + # → "JusticeCentral TV"). Keeps the dot-suffix variant equivalent to + # the spaced form for matching purposes. + name = re.sub(r'(?<=\w)\.(?=\w)', ' ', name) + + # Normalize number-words to digits so "BBC Three" and "BBC 3" share + # tokens. Critical for cases like "Three Angels Broadcasting Network" + # vs "3 Angels Broadcasting Network", and for BBC One/Two/Three/Four + # vs BBC 1/2/3/4. Bounded by word boundaries so brand names with + # embedded letters (e.g. "Onesimus") aren't corrupted. + _NUM_WORDS = { + "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", + "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10", + "eleven": "11", "twelve": "12", + } + def _num_repl(m): + return _NUM_WORDS[m.group(0).lower()] + name = re.sub( + r'\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\b', + _num_repl, name, flags=re.IGNORECASE, + ) + + # Split CamelCase boundaries so "JusticeCentral" becomes "Justice + # Central" and "DangerTV" becomes "Danger TV". Two separate patterns: + # 1. lower → Upper followed by lower (Justice|Central, Danger|Iq → no, etc.) + # 2. lower(4+) → UPPER acronym at word boundary (Danger|TV, Beauty|IQ) + # The 4-char floor on rule 2 protects short brand names like "MeTV" and + # "truTV" whose existing EPG matches rely on the un-split form. + name = re.sub(r'([a-z])([A-Z][a-z])', r'\1 \2', name) + name = re.sub(r'([a-z]{4,})([A-Z]{2,})\b', r'\1 \2', name) + + # Strip region tokens (East / Eastern / West and the (E)/(W) + # abbreviations) for SCORING. Region CORRECTNESS - East vs West vs + # Pacific - is enforced separately by the post-match region filter in + # match_all_streams, which reads the ORIGINAL un-normalized names, not + # this output. Stripping here lets a regionless lineup channel + # ("Food Network") still score-match a region-tagged stream + # ("Food Network Eastern" / "... East") instead of being dragged below + # threshold by the extra token; the filter then accepts it (regionless + # defaults to East). "Western"/"Westerns" is a movie genre, NOT a + # region, so bare "west" is stripped only on a word boundary (\bwest\b + # does not match inside "western") and "western" is never touched. + if ignore_regional: + name = re.sub(r'\(\s*(?:eastern|east|e|west|w)\s*\)', ' ', name, flags=re.IGNORECASE) + name = re.sub(r'\b(?:eastern|east|west)\b', ' ', name, flags=re.IGNORECASE) + + # Remove leading parenthetical prefixes + while name.lstrip().startswith('('): + new_name = re.sub(r'^\s*\([^\)]+\)\s*', '', name) + if new_name == name: + break + name = new_name + + # Opt-in: remove a country-code prefix (multi-country DBs). Strips a 2-3 + # letter colon/space prefix unless it is a quality tag. Off by default; + # PROVIDER_PREFIX_PATTERNS above already removes the curated country + # prefixes, so this only catches the remainder. (Stream-Mapparr opts in.) + if remove_country_prefix: + quality_tags = {'HD', 'SD', 'FD', 'UHD', 'FHD'} + prefix_match = re.match(r'^([A-Z]{2,3})[:\s]\s*', name) + if prefix_match: + prefix = prefix_match.group(1).upper() + if prefix not in quality_tags: + name = name[len(prefix_match.group(0)):] + + # Opt-in: remove a leading "Cinemax" (for channels containing "max"). Off + # by default. (Stream-Mapparr opts in.) + if remove_cinemax: + name = re.sub(r'\bCinemax\b\s*', '', name, flags=re.IGNORECASE) + + # Build pattern list based on flags + patterns_to_apply = [] + if ignore_regional: + patterns_to_apply.extend(REGIONAL_PATTERNS) + if strip_bare_region: + patterns_to_apply.extend(REGIONAL_BARE_PATTERNS) + if ignore_geographic: + patterns_to_apply.extend(GEOGRAPHIC_PATTERNS) + if ignore_misc and ignore_regional: + patterns_to_apply.extend(MISC_PATTERNS) + + for pattern in patterns_to_apply: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + + # Apply user-configured ignored tags + for tag in user_ignored_tags: + escaped_tag = re.escape(tag) + if '[' in tag or ']' in tag or '(' in tag or ')' in tag: + name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) + else: + if re.match(r'^\w+$', tag): + name = re.sub(r'\b' + escaped_tag + r'\b', '', name, flags=re.IGNORECASE) + else: + name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) + + # Remove callsigns in parentheses + if ignore_regional: + name = re.sub(r'\([KW][A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) + else: + name = re.sub(r'\([KW](?!EST\)|ACIFIC\)|ENTRAL\)|OUNTAIN\)|TLANTIC\))[A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) + + if ignore_regional: + name = re.sub(r'\([A-Z0-9]+\)', '', name) + + # Remove common suffixes/prefixes + name = re.sub(r'^The\s+', '', name, flags=re.IGNORECASE) + name = re.sub(r'\s+Network\s*$', '', name, flags=re.IGNORECASE) + name = re.sub(r'\s+Channel\s*$', '', name, flags=re.IGNORECASE) + name = re.sub(r'\s+TV\s*$', '', name, flags=re.IGNORECASE) + + # Strip decorative Unicode markers (◉, ², superscript letters ᴿᴬᵂ…) + # that some IPTV providers append as quality/status badges. Only + # characters in decorator categories are removed; accented letters + # (é, î, ü…) are in Ll/Lu and are preserved. + name = ''.join( + ' ' if unicodedata.category(c) in _DECORATOR_CATS else c + for c in name + ) + + # Clean up whitespace + name = re.sub(r'\s+', ' ', name).strip() + + if not name: + self.logger.debug(f"normalize_name returned empty for: '{original_name}'") + + return name + + def calculate_similarity(self, str1, str2, min_ratio=0.0): + """Levenshtein similarity ratio (0.0-1.0), defined as 1 - distance/max(len). + If min_ratio > 0, returns 0.0 early when the result can't reach it. + + bug-026: the ratio MUST be distance/max(len), matching rapidfuzz + Levenshtein.normalized_similarity. The old (len1+len2-distance)/(len1+len2) + formula scored higher for the same edit distance and let numbered siblings + ("Fox Sports 1" vs "2") pass threshold 95. The rapidfuzz fast path and the + pure-Python fallback below compute the identical value. + """ + if len(str1) == 0 or len(str2) == 0: + return 0.0 + + # Fast path: C-accelerated rapidfuzz when available (same definition). + if _USE_RAPIDFUZZ: + # Compute the true similarity, then apply the SAME gate as the pure-Python + # early-exit below: a score that cannot reach min_ratio is 0.0 on both paths + # (raw agreement below threshold), but a score landing EXACTLY on min_ratio is + # KEPT (>=, inclusive). We deliberately do NOT use rapidfuzz's own score_cutoff: + # it treats the cutoff as strict-greater and quantizes it onto the achievable + # distance grid, so a true 0.8 against a 0.8 cutoff is spuriously zeroed while the + # pure-Python path returns it. With min_ratio=0.0 the gate is a no-op. + sim = _rf_lev.normalized_similarity(str1, str2) + return sim if (min_ratio <= 0.0 or sim >= min_ratio) else 0.0 + + if len(str1) < len(str2): + str1, str2 = str2, str1 + len1, len2 = len(str1), len(str2) + max_len = len1 # the longer string after the swap + + # Length-difference pre-check: minimum possible distance is (len1 - len2), + # so the max possible ratio is (max_len - (len1 - len2)) / max_len. + if min_ratio > 0: + max_possible = (max_len - (len1 - len2)) / max_len + if max_possible < min_ratio: + return 0.0 + + previous_row = list(range(len2 + 1)) + for i, c1 in enumerate(str1): + current_row = [i + 1] + for j, c2 in enumerate(str2): + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + # Early termination: a lower bound on the final distance is the current + # row minimum minus the str1 chars still unprocessed. + if min_ratio > 0: + min_distance_so_far = min(current_row) + remaining = len1 - i - 1 + best_possible_distance = max(0, min_distance_so_far - remaining) + best_possible_ratio = (max_len - best_possible_distance) / max_len + if best_possible_ratio < min_ratio: + return 0.0 + previous_row = current_row + + distance = previous_row[-1] + return (max_len - distance) / max_len + + @staticmethod + def _length_scaled_threshold(base_threshold, shorter_len): + """Require higher similarity for shorter strings to avoid false positives.""" + if shorter_len <= 4: + return max(base_threshold, 95) + elif shorter_len <= 8: + return max(base_threshold, 90) + return base_threshold + + @staticmethod + def _has_token_overlap(str_a, str_b, min_token_len=4, require_majority=False): + """Check that distinctive tokens are shared between two strings. + + Basic mode: at least one token (>= min_token_len) must be shared. + Majority mode: uses all tokens (>= 2 chars) and requires that more than + half of the smaller set overlaps. Catches false positives like + "america racing" vs "america bbc" while allowing single-token matches. + """ + # "network"/"channel"/"television" are generic brand-suffix words, not + # distinctive - treat as common so the subset guard does not reject + # cases like "FanDuel Sports Cincinnati" vs "FanDuel Sports Network + # Cincinnati". (They're already stripped from end-of-string by + # normalize_name; this catches mid-string occurrences.) + common_words = { + "the", "and", "of", "in", "on", "at", "to", "for", "a", "an", + "network", "channel", "television", + } + + if require_majority: + # Use all meaningful tokens (>= 2 chars) for stricter checking. + # Single-digit tokens (1, 2, 3, ...) are kept because they're + # channel-distinguishing (BBC 1 vs BBC 2, ESPN 1 vs ESPN 2 etc.) + # even though they're only 1 char. + def _meaningful(t): + if t in common_words: + return False + return len(t) >= 2 or t.isdigit() + tokens_a = {t for t in str_a.split() if _meaningful(t)} + tokens_b = {t for t in str_b.split() if _meaningful(t)} + if not tokens_a or not tokens_b: + return True + shared = tokens_a & tokens_b + if not shared: + return False + smaller = min(len(tokens_a), len(tokens_b)) + if not len(shared) > smaller / 2: + return False + + unique_a = tokens_a - tokens_b + unique_b = tokens_b - tokens_a + + # Subset guard: when one side is a strict subset of the other and + # the larger side has a distinctive token the smaller lacks, the + # candidate is a more specific channel than the query and the high + # fuzzy score is a false positive. Catches e.g. "Nickelodeon" vs + # "Nickelodeon Teen". Threshold is >=4 chars; "live"/"now" are + # explicitly non-distinctive (stream label variants) so "ABC News" + # still matches "ABC News Live". Pure-digit tokens >=2 chars + # (e.g. "360") are also distinctive: "Canal+Sport" must not match + # "Canal+Sport 360". + if not unique_a: + if any(_is_distinctive(t) for t in unique_b): + return False + elif not unique_b: + if any(_is_distinctive(t) for t in unique_a): + return False + + # Divergent guard: when BOTH sides have unique tokens AND at least + # one of those unique tokens is a distinctive (>=4 char) word, the + # strings describe different brands and the fuzzy score is + # misleading. Catches "Sky Cinema Disney" vs "Sky Cinema Decades" + # (decades = 7 chars), "Sky Cinema Fast" vs "Sky Cinema Family", + # and "BBC One vs BBC Two" once number-words become digits earlier. + # Number-word→digit normalization (in normalize_name) reduces this + # guard's risk: "Three Angels" / "3 Angels" both become "3 angels" + # before reaching here, so they never trigger this case. + if unique_a and unique_b: + if any(len(t) >= 4 for t in unique_a | unique_b): + return False + + # Numeric/ordinal divergent guard: when BOTH sides have a numeric + # or ordinal token unique to them, they are sibling channels + # distinguished by number (BBC One vs BBC Two; ESPN 1 vs ESPN 2). + # Short tokens like "one"/"two" wouldn't trip the divergent guard + # above so they need their own check. + _NUMERIC = { + "one", "two", "three", "four", "five", "six", "seven", "eight", + "nine", "ten", "eleven", "twelve", + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", + "first", "second", "third", "fourth", "fifth", + } + if (unique_a & _NUMERIC) and (unique_b & _NUMERIC): + return False + + return True + + # Basic mode: at least one long token shared + tokens_a = {t for t in str_a.split() if t not in common_words and len(t) >= min_token_len} + tokens_b = {t for t in str_b.split() if t not in common_words and len(t) >= min_token_len} + if not tokens_a or not tokens_b: + return True + return bool(tokens_a & tokens_b) + + def process_string_for_matching(self, s): + """Normalize for token-sort matching: NFKD-fold (so "ＨＢＯ"->"hbo", "²"->"2", + "fi"->"fi", accents drop), lowercase, keep alphanumerics of any script + (isalnum keeps Cyrillic/CJK/Arabic rather than erasing them) plus '+' + (Disney+/Discovery+/Paramount+ stay distinct), sort tokens.""" + s = unicodedata.normalize('NFKD', s) + s = ''.join(char for char in s if unicodedata.category(char) != 'Mn') + s = s.lower() + s = re.sub(r'([^\W\d_])(\d)', r'\1 \2', s) # split letter-glued digit, any script + cleaned_s = "" + for char in s: + if char.isalnum() or char == '+': + cleaned_s += char + else: + cleaned_s += ' ' + tokens = sorted([token for token in cleaned_s.split() if token]) + return " ".join(tokens) + + @staticmethod + def _trailing_number(name): + """Return the integer value of a space-separated, purely-numeric + trailing token, or None. 'HBO 2' -> 2, 'DIRECTV 4K Live 1' -> 1, + 'ESPN' -> None, 'ESPN2' -> None (digit not space-separated). Used to + reject 'Foo 1' vs 'Foo 2' false positives -- different channels that + otherwise fuzzy-match almost perfectly.""" + m = re.search(r'(?:^|\s)(\d{1,4})\s*$', name or "") + return int(m.group(1)) if m else None + + # Words that match the callsign regex shape but are never US broadcast + # callsigns. A US callsign (K/W + 2-4 letters) is shape-identical to many + # common English words, so the loose Priority-4 pattern mis-extracts them - + # e.g. "with" in "Bizarre Foods with Andrew Zimmern" becomes callsign + # "WITH". Regex alone cannot tell "WITH" from "WABC"; frequent K/W-initial + # words are denied explicitly so they never extract as a callsign. WWE/WWF/ + # WCW stop wrestling show names extracting as false-positive callsigns. + _CALLSIGN_DENYLIST = frozenset({ + 'WWE', 'WWF', 'WCW', 'EAST', + 'WAR', 'WARS', 'WARM', 'WASH', 'WATCH', 'WAVE', 'WAVES', 'WAY', 'WAYS', + 'WEB', 'WEEK', 'WELL', 'WENT', 'WERE', 'WEST', 'WHAT', 'WHEN', 'WHERE', + 'WHICH', 'WHILE', 'WHITE', 'WHO', 'WHY', 'WIDE', 'WIFE', 'WILD', 'WILL', + 'WIND', 'WINE', 'WING', 'WINGS', 'WINS', 'WIRE', 'WISE', 'WISH', 'WITH', + 'WOLF', 'WOMAN', 'WOMEN', 'WOOD', 'WORD', 'WORDS', 'WORK', 'WORKS', + 'WORLD', 'WORM', 'WORN', 'WRAP', + 'KEEN', 'KEEP', 'KEPT', 'KEY', 'KEYS', 'KICK', 'KID', 'KIDS', 'KILL', + 'KIND', 'KING', 'KINGS', 'KISS', 'KITE', 'KNEE', 'KNEW', 'KNOW', 'KNOWN', + }) + + def _is_callsign_allowed(self, callsign): + """A candidate callsign is allowed if it is not denylisted, OR the plugin + supplied a known-real callsign set that contains it (DB rescue).""" + return (callsign not in self._CALLSIGN_DENYLIST + or (self._known_callsigns is not None and callsign in self._known_callsigns)) + + def _compute_callsign_with_confidence(self, channel_name): + """ + Extract US TV callsign with a confidence flag. + + Returns (callsign, is_high_confidence). High confidence = + Priorities 1-3 (parenthesized / suffixed-paren / end-of-name). + Priority 4 (any loose word) is low confidence. (None, False) + when nothing extractable. + + A denylisted word is rejected UNLESS the plugin supplied a known-real + callsign set (set_known_callsigns) that contains it - that is the DB + rescue for real stations whose callsign collides with a common word + (KING/WAVE/WOOD/WHO). With no set supplied this is the pure ladder. + """ + # Remove common provider prefixes + channel_name = re.sub(r'^D\d+-', '', channel_name) + channel_name = re.sub(r'^USA?\s*[^a-zA-Z0-9]*\s*', '', channel_name, flags=re.IGNORECASE) + + # Priority 1: Callsigns in parentheses (most reliable) + paren_match = re.search(r'\(([KW][A-Z]{3})(?:-[A-Z\s]+)?\)', channel_name, re.IGNORECASE) + if paren_match: + callsign = paren_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, True + + # Priority 1b: grandfathered 3-letter callsigns in parentheses without a suffix + # (WWL/WJZ/KYW/WRC). Suffixed forms fall through to Priority 2. bug-062. + paren3_match = re.search(r'\(([KW][A-Z]{2})\)', channel_name, re.IGNORECASE) + if paren3_match: + callsign = paren3_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, True + + # Priority 2: Callsigns with suffix in parentheses + paren_suffix_match = re.search(r'\(([KW][A-Z]{2,4}-(?:TV|CD|LP|DT|LD))\)', channel_name, re.IGNORECASE) + if paren_suffix_match: + return paren_suffix_match.group(1).upper(), True + + # Priority 3: Callsigns at the end + end_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\s*(?:\.[a-z]+)?\s*$', channel_name, re.IGNORECASE) + if end_match: + callsign = end_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, True + + # Priority 4: Any word matching callsign pattern (low confidence) + word_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\b', channel_name, re.IGNORECASE) + if word_match: + callsign = word_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, False + + return None, False + + def _extract_callsign_with_confidence(self, channel_name): + """ + Cached wrapper around _compute_callsign_with_confidence. + + Extraction is pure in channel_name (and in self._known_callsigns, which is + held fixed between set_known_callsigns calls), so results are memoized - the + anchor calls this once per stream over a fixed candidate list, which is + otherwise massively redundant across the per-channel matching loop. + """ + cached = self._callsign_cache.get(channel_name) + if cached is not None: + return cached + result = self._compute_callsign_with_confidence(channel_name) + self._callsign_cache[channel_name] = result + return result + + def extract_callsign(self, channel_name): + """ + Extract US TV callsign from channel name with priority order. + Returns None if common false positives appear alone. + """ + callsign, _ = self._extract_callsign_with_confidence(channel_name) + return callsign + + def normalize_callsign(self, callsign): + """Remove the broadcast suffix (-TV/-CD/-LP/-DT/-LD) from a callsign.""" + if callsign: + callsign = re.sub(r'-(?:TV|CD|LP|DT|LD)$', '', callsign) + return callsign diff --git a/Lineuparr/plugin.json b/Lineuparr/plugin.json index 0d00625..0d718e6 100644 --- a/Lineuparr/plugin.json +++ b/Lineuparr/plugin.json @@ -1,6 +1,6 @@ { "name": "Lineuparr", - "version": "1.26.1641222", + "version": "1.26.1791747", "description": "Mirror real-world provider channel lineups by creating channel groups, channels, and fuzzy-matching IPTV streams to them.", "author": "PiratesIRC", "license": "MIT", diff --git a/Lineuparr/plugin.py b/Lineuparr/plugin.py index 0bb2c86..480afad 100644 --- a/Lineuparr/plugin.py +++ b/Lineuparr/plugin.py @@ -64,7 +64,7 @@ def _clean_json_text(s): class PluginConfig: - PLUGIN_VERSION = "1.26.1641222" + PLUGIN_VERSION = "1.26.1791747" DEFAULT_FUZZY_MATCH_THRESHOLD = 80 DEFAULT_PRIORITIZE_QUALITY = True