From 2dd462236a6a6bab825d659e3a26e5d2ff2b8a40 Mon Sep 17 00:00:00 2001 From: PiratesIRC Date: Sat, 27 Jun 2026 13:00:21 -0500 Subject: [PATCH 1/3] Fix wedge: lazy-load channel databases in FuzzyMatcher FuzzyMatcher.__init__ eager-loaded all ~42k channel records (12 country JSONs + networks.json) in its constructor. Dispatcharr re-instantiates every Plugin on each plugin discovery, which cascades across all uWSGI/Celery workers via /data/plugins/.reload_token, so this burned CPU loading 42k channels on each worker's single gevent thread. Under live channel-zapping plus a reload cascade it pinned the workers and wedged all UI + streaming (ops incident 2026-06-27: two autoheal restarts in 7 min). The data is only needed when a mapping run calls reload_databases(), which clears and reloads the user-selected countries anyway, so the constructor's eager load was always discarded. Drop the eager call; construction is now cheap and databases load on demand. Adds tests/test_lazy_db_load.py: construction must not load the databases; reload_databases() must still populate them on demand. Full suite 197 passing (was 195). Co-Authored-By: Claude Opus 4.8 (1M context) --- Channel-Maparr/fuzzy_matcher.py | 13 +++++++++---- Channel-Maparr/plugin.json | 2 +- Channel-Maparr/plugin.py | 2 +- tests/test_lazy_db_load.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 tests/test_lazy_db_load.py diff --git a/Channel-Maparr/fuzzy_matcher.py b/Channel-Maparr/fuzzy_matcher.py index da2e6bc..246e3c2 100644 --- a/Channel-Maparr/fuzzy_matcher.py +++ b/Channel-Maparr/fuzzy_matcher.py @@ -292,10 +292,15 @@ def __init__(self, plugin_dir=None, match_threshold=80, logger=None): self._token_index = {} # token -> set of original candidate names self._indexed_names = set() # all names in the token index - # Load all channel databases if plugin_dir is provided - if self.plugin_dir: - self._load_channel_databases() - + # NOTE: channel databases are intentionally NOT loaded here. Loading all + # ~42k records is expensive, and Dispatcharr re-instantiates every Plugin + # on each plugin discovery (cascading across all uWSGI/Celery workers via + # .reload_token). Eager-loading in the constructor pinned the gevent + # workers and wedged streaming (ops incident 2026-06-27). The run path + # calls reload_databases() with the user-selected countries before + # matching (plugin.py), which loads on demand — so the eager load was + # always discarded anyway. Construct cheap; load when actually matching. + def precompute_normalizations(self, names, user_ignored_tags=None): """ Pre-normalize a list of names and cache the results. diff --git a/Channel-Maparr/plugin.json b/Channel-Maparr/plugin.json index 392a90f..8619576 100644 --- a/Channel-Maparr/plugin.json +++ b/Channel-Maparr/plugin.json @@ -1,6 +1,6 @@ { "name": "Channel Mapparr", - "version": "1.26.1701952", + "version": "1.26.1781253", "description": "Standardizes broadcast (OTA) and premium/cable channel names using network data and channel lists. Supports M3U stream import, category organization, and fuzzy matching across 42K+ channels in 11 countries.", "author": "PiratesIRC", "license": "MIT", diff --git a/Channel-Maparr/plugin.py b/Channel-Maparr/plugin.py index 47a5fa2..6bd4a5a 100644 --- a/Channel-Maparr/plugin.py +++ b/Channel-Maparr/plugin.py @@ -44,7 +44,7 @@ class PluginConfig: """Configuration constants for Channel Maparr.""" - PLUGIN_VERSION = "1.26.1701952" + PLUGIN_VERSION = "1.26.1781253" # Channel Database Settings DEFAULT_CHANNEL_DATABASES = "US" diff --git a/tests/test_lazy_db_load.py b/tests/test_lazy_db_load.py new file mode 100644 index 0000000..c220445 --- /dev/null +++ b/tests/test_lazy_db_load.py @@ -0,0 +1,28 @@ +"""Regression: constructing FuzzyMatcher must NOT eager-load the channel DBs. + +Eager-loading all ~42k channel records in ``FuzzyMatcher.__init__`` meant every +Dispatcharr plugin *discovery* — which re-instantiates every Plugin and cascades +across all uWSGI/Celery workers via ``/data/plugins/.reload_token`` — burned CPU +loading 42k channels on each worker's single gevent thread. Under channel-zapping +plus a reload cascade this pinned the workers and wedged all UI + streaming (ops +incident 2026-06-27: two autoheal restarts in 7 min). The channel data is only +needed when a mapping run calls ``reload_databases()`` (plugin.py:756), which +clears and reloads the user-selected countries anyway — so the constructor's +eager load was always discarded. Keep construction cheap; load on demand. +""" + + +def test_construction_does_not_eager_load_databases(fuzzy_module, plugin_dir): + """Building the matcher must not touch the channel JSON databases.""" + fm = fuzzy_module.FuzzyMatcher(plugin_dir=str(plugin_dir), match_threshold=80) + assert fm.premium_channels == [] + assert fm.premium_channels_full == [] + assert fm.broadcast_channels == [] + + +def test_reload_databases_still_loads_on_demand(fuzzy_module, plugin_dir): + """The explicit run-path load (reload_databases) must still populate data.""" + fm = fuzzy_module.FuzzyMatcher(plugin_dir=str(plugin_dir), match_threshold=80) + assert fm.reload_databases(["US"]) is True + assert len(fm.premium_channels) > 0 + assert len(fm.broadcast_channels) > 0 From 5bb2fa0fae6d177523b71ef13228a358c0c8506f Mon Sep 17 00:00:00 2001 From: PiratesIRC Date: Sun, 28 Jun 2026 13:25:00 -0500 Subject: [PATCH 2/3] Migrate matcher onto the shared vendored core (partial: shared primitives) Channel-Maparr is the most divergent matcher, so this is a PARTIAL subclass like EPG-Janitor: FuzzyMatcher subclasses FuzzyMatcherCore and inherits the body-compatible primitives (calculate_similarity, process_string_for_matching, _length_scaled_threshold, _trailing_number, extract_callsign, normalize_callsign, _CALLSIGN_DENYLIST, and the decorative helpers re-exported for the conftest tests). It KEEPS its divergent domain logic: normalize_name (+ patterns), the single-digit token-overlap guard, and the callsign ladder (_compute/_extract_callsign_with_confidence) which rescues denylisted-but-real callsigns via channel_lookup ONLY at parenthesized Priority 1/1b -- the core's _is_callsign_allowed rescues at ALL priorities, and broadening it here would mis-extract prose like "King of the Hill". Also kept: the stateful machinery (inverted token index build_token_index/get_candidates, stateful aliases set_user_aliases/_rebuild_reverse_alias_index/ alias_match, channel-DB layer). The subclass __init__ PRESERVES the lazy-load fix (v1.26.1781253): it calls super().__init__() and sets cheap state only -- it does NOT load the ~42k channel DB, which Dispatcharr's per-discovery re-instantiation would otherwise replay across all gevent workers and wedge streaming (ops incident 2026-06-27). Verified: a fresh FuzzyMatcher() has empty channel_lookup/broadcast/premium. process_string now keeps "+" (Discovery+/Disney+/Paramount+ stay distinct -- the agreed 4-of-4 superset; golden baseline regenerated for those 3 entries), and inheriting calculate_similarity adopts the core's boundary-gate fix. Vendored core hash-pinned (scripts/core_manifest.json + tests/test_core_parity.py; *.py is already eol=lf); CI byte-compiles it and runs the parity gate. Full suite 285 green, lazy-load preserved. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01N2rpwFn3AHoNvsbTrEwgwd --- .github/workflows/tests.yml | 13 +- Channel-Maparr/fuzzy_matcher.py | 271 +--------- Channel-Maparr/matching_core.py | 793 +++++++++++++++++++++++++++++ Channel-Maparr/plugin.json | 2 +- Channel-Maparr/plugin.py | 2 +- scripts/core_manifest.json | 3 + scripts/sync_core.py | 115 +++++ tests/matcher_golden_baseline.json | 300 +++++++++++ tests/test_core_parity.py | 31 ++ tests/test_matcher_golden.py | 125 +++++ 10 files changed, 1407 insertions(+), 248 deletions(-) create mode 100644 Channel-Maparr/matching_core.py create mode 100644 scripts/core_manifest.json create mode 100644 scripts/sync_core.py create mode 100644 tests/matcher_golden_baseline.json create mode 100644 tests/test_core_parity.py create mode 100644 tests/test_matcher_golden.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da5e6de..76a81d3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,7 +29,18 @@ jobs: pip install -r requirements-dev.txt - name: Compile-check shipped plugin sources - run: python -m py_compile Channel-Maparr/plugin.py Channel-Maparr/fuzzy_matcher.py Channel-Maparr/logo_matcher.py Channel-Maparr/progress_status.py Channel-Maparr/aliases.py + run: python -m py_compile Channel-Maparr/plugin.py Channel-Maparr/fuzzy_matcher.py Channel-Maparr/matching_core.py Channel-Maparr/logo_matcher.py Channel-Maparr/progress_status.py Channel-Maparr/aliases.py + + # Explicit, named drift gate: freezes the pure matcher primitives against + # tests/matcher_golden_baseline.json (Stage 0 of the matcher standardization + # plan). -o addopts="" neutralizes any inherited ini options for a clean run. + - name: Matcher golden drift gate + run: python -m pytest tests/test_matcher_golden.py -q -o addopts="" + + # The vendored shared matching core must stay byte-identical to its pinned hash + # (catches a stray hand-edit to the vendored copy or a CRLF checkout). + - name: Vendored-core parity gate + run: python -m pytest tests/test_core_parity.py -q -o addopts="" - name: Run test suite run: python -m pytest diff --git a/Channel-Maparr/fuzzy_matcher.py b/Channel-Maparr/fuzzy_matcher.py index 246e3c2..eaa8bf3 100644 --- a/Channel-Maparr/fuzzy_matcher.py +++ b/Channel-Maparr/fuzzy_matcher.py @@ -8,7 +8,6 @@ import re import json import logging -import unicodedata from glob import glob try: @@ -30,14 +29,26 @@ # Setup logging LOGGER = logging.getLogger("plugins.fuzzy_matcher") -# Optional C-accelerated Levenshtein. When present, the matcher uses rapidfuzz's -# normalized_similarity (1 - distance/max(len)); the pure-Python fallback below -# computes the identical value (bug-026). rapidfuzz is an OPTIONAL runtime dep. +# The shared matching primitives (calculate_similarity with its rapidfuzz fast path + +# pure-Python fallback, process_string_for_matching, the length/trailing-number helpers, +# the callsign denylist + extract/normalize) live in the vendored core. The decorative +# helpers are re-exported so the conftest/unit tests that reference them keep working. +# Channel-Maparr keeps its own normalize_name, single-digit token-overlap guard, and the +# callsign ladder (channel_lookup rescue, parenthesized-only), which diverge from the core. try: - from rapidfuzz.distance import Levenshtein as _rf_lev - _USE_RAPIDFUZZ = True -except ImportError: - _USE_RAPIDFUZZ = False + from .matching_core import ( + FuzzyMatcherCore, + _is_decorative_char, # noqa: F401 re-exported for the decoration unit tests + _normalize_emoji, # noqa: F401 + _strip_stylized_tokens, # noqa: F401 + ) +except ImportError: # script/test context without the package parent on sys.path + from matching_core import ( + FuzzyMatcherCore, + _is_decorative_char, # noqa: F401 + _normalize_emoji, # noqa: F401 + _strip_stylized_tokens, # noqa: F401 + ) # Categorized regex patterns for granular control during fuzzy matching # Note: All patterns are applied with re.IGNORECASE flag in normalize_name() @@ -152,101 +163,7 @@ ) -# --------------------------------------------------------------------------- # -# Stylized-Unicode decoration stripping -# --------------------------------------------------------------------------- # -# Streams tag names with stylized-Unicode tier/format markers (superscript -# "WEATHERNATION RAW", small-cap "FHD", bullet-prefixed "CNN") that the ASCII tag -# regexes below cannot see. We drop whole tokens that are pure decoration BEFORE -# the ASCII pipeline runs. Detection is by Unicode character *name* (not code-point -# ranges), so it covers superscripts, "modifier letter" superscript capitals, and -# Latin small-caps wherever they live (e.g. small-cap H is U+029C in IPA Extensions -# and modifier V is U+2C7D in Latin-Ext-C, both outside the obvious blocks). - -# Ornament glyphs whose Unicode name carries no decoration keyword. -_DECORATIVE_SYMBOLS = frozenset("◉") # FISHEYE; add individual chars (not strings) here - - -def _is_decorative_char(ch): - """True for a stylized letterform/ornament that carries no semantic content in a - channel name (superscripts, subscripts, modifier-letter superscript capitals, - Latin small-capitals, curated bullets). ASCII and ordinary letters return False.""" - if ch.isascii(): - return False - if ch in _DECORATIVE_SYMBOLS: - return True - try: - nm = unicodedata.name(ch) - except ValueError: - # unnamed code point (control char / lone surrogate) -> not decoration - return False - return ('SUPERSCRIPT' in nm or 'SUBSCRIPT' in nm - or 'SMALL CAPITAL' in nm or 'MODIFIER LETTER' in nm) - - -def _strip_stylized_tokens(name): - """Drop whitespace tokens that are pure stylized decoration, then NFKD-canonicalize - the remainder. A token is decoration when it has >=1 decorative char, no ASCII - alphanumeric, and every char is decorative or ASCII punctuation (so a bullet glued - to a colon, or "HD/RAW" written in superscripts, are dropped too). Real ASCII words - (Gold/VIP) and non-Latin letters (Arabic/Cyrillic/CJK) are always kept. ASCII-only - input is returned unchanged via the fast path (no per-char work; NFKD is a no-op - on ASCII, so skipping it changes nothing).""" - if name.isascii(): - return name - kept = [] - for tok in name.split(): - has_decorative = any(_is_decorative_char(c) for c in tok) - has_ascii_alnum = any(c.isascii() and c.isalnum() for c in tok) - only_decorative_or_punct = all( - _is_decorative_char(c) or (c.isascii() and not c.isalnum()) for c in tok - ) - if has_decorative and only_decorative_or_punct and not has_ascii_alnum: - continue # pure decoration -> drop the whole token - kept.append(tok) - return unicodedata.normalize('NFKD', ' '.join(kept)) - - -# --------------------------------------------------------------------------- # -# Emoji-as-letter + emoji decoration normalization -# --------------------------------------------------------------------------- # -# Some streams use an emoji AS A LETTER inside a word: "SP⚽RTS" / "Sp⚽rts" where the -# soccer ball stands in for 'o' (= SPORTS, the beIN family). _strip_stylized_tokens keeps -# the token (it has ASCII alnum) and process_string_for_matching would turn the ball into a -# space ("sp rts"), so it never matches "sports". We substitute the glyph for the letter it -# replaces (only when flanked by ASCII letters) and strip emoji used purely as decoration. - -# Emoji that visually replace an ASCII letter when embedded in a word. Extensible. -_EMOJI_LETTER_MAP = {'⚽': 'o'} # SOCCER BALL = 'o' (SP⚽RTS -> SPORTS) -# Pictographic ornaments to delete. NOTE: ⚽ is intentionally in BOTH maps — the letter -# map handles it mid-word (-> 'o'); here it catches any ⚽ NOT flanked by ASCII letters -# (standalone/edge), which the substitution above leaves untouched. -_EMOJI_ORNAMENTS = frozenset('♬☾⚽') # beamed notes, last-quarter moon, soccer ball -# Zero-width / invisible code points that only add noise to a name. -_ZERO_WIDTH = ('️', '‍') # VARIATION SELECTOR-16, ZERO WIDTH JOINER - - -def _normalize_emoji(name): - """Map emoji-as-letters to their letter and strip emoji decoration. - - The letter substitution fires ONLY when the glyph is flanked by ASCII letters - (so "SP⚽RTS" -> "SPoRTS" but a standalone/edge "⚽" is treated as decoration and - dropped). Zero-width selectors and ornament pictographs are deleted outright. - ASCII-only input is returned unchanged (no emoji possible).""" - if name.isascii(): - return name - for zw in _ZERO_WIDTH: - if zw in name: - name = name.replace(zw, '') - for glyph, letter in _EMOJI_LETTER_MAP.items(): - if glyph in name: - name = re.sub(r'(?<=[A-Za-z])' + re.escape(glyph) + r'(?=[A-Za-z])', letter, name) - if any(c in _EMOJI_ORNAMENTS for c in name): - name = ''.join(c for c in name if c not in _EMOJI_ORNAMENTS) - return name - - -class FuzzyMatcher: +class FuzzyMatcher(FuzzyMatcherCore): """Handles fuzzy matching for channel and stream names with normalization and database loading.""" # Common words excluded from token indexing (too generic to be discriminating) @@ -261,9 +178,12 @@ def __init__(self, plugin_dir=None, match_threshold=80, logger=None): match_threshold: Minimum similarity score (0-100) for a match to be accepted logger: Logger instance (optional) """ + # The core seeds match_threshold, logger, the four normalization/callsign caches, + # and the _known_callsigns rescue slot. Channel-Maparr's callsign ladder rescues via + # channel_lookup instead, so _known_callsigns stays unused. Channel databases are + # still NOT loaded here (see the NOTE below) - the constructor must stay cheap. + super().__init__(match_threshold=match_threshold, logger=logger or LOGGER) self.plugin_dir = plugin_dir or os.path.dirname(__file__) - self.match_threshold = match_threshold - self.logger = logger or LOGGER # Channel data storage self.broadcast_channels = [] # Channels with callsigns @@ -272,12 +192,6 @@ def __init__(self, plugin_dir=None, match_threshold=80, logger=None): self.channel_lookup = {} # Callsign -> channel data mapping self.country_codes = None # Track which country databases are currently loaded - # Cache for pre-normalized stream names (performance optimization) - self._norm_cache = {} # raw_name -> normalized_lower - self._norm_nospace_cache = {} # raw_name -> normalized_nospace - self._processed_cache = {} # raw_name -> processed_for_matching - self._callsign_cache = {} # raw_name -> (callsign, is_high_confidence) - # Alias map: channel_name -> [stream-name variants]. Builtins ship in # aliases.py; users can extend at runtime via set_user_aliases(). # _reverse_alias_index maps normalized-variant -> canonical channel, @@ -629,21 +543,6 @@ def reload_databases(self, country_codes=None): self.logger.info(f"Total channels loaded: {total_broadcast} broadcast, {total_premium} premium") return True - # Words shape-identical to US callsigns (K/W + 2-4 letters) that are never - # real broadcast callsigns. The Priority 4 loose pattern would otherwise - # mis-extract them — e.g. "with" in "Bizarre Foods with Andrew Zimmern". - # Sourced from Lineuparr's denylist; WWE/WWF/WCW are wrestling brands. - _CALLSIGN_DENYLIST = frozenset({ - 'WWE', 'WWF', 'WCW', 'EAST', - 'WAR', 'WARS', 'WARM', 'WASH', 'WATCH', 'WAVE', 'WAVES', 'WAY', 'WAYS', - 'WEB', 'WEEK', 'WELL', 'WENT', 'WERE', 'WEST', 'WHAT', 'WHEN', 'WHERE', - 'WHICH', 'WHILE', 'WHITE', 'WHO', 'WHY', 'WIDE', 'WIFE', 'WILD', 'WILL', - 'WIND', 'WINE', 'WING', 'WINGS', 'WINS', 'WIRE', 'WISE', 'WISH', 'WITH', - 'WOLF', 'WOMAN', 'WOMEN', 'WOOD', 'WORD', 'WORDS', 'WORK', 'WORKS', - 'WORLD', 'WORM', 'WORN', 'WRAP', - 'KEEN', 'KEEP', 'KEPT', 'KEY', 'KEYS', 'KICK', 'KID', 'KIDS', 'KILL', - 'KIND', 'KING', 'KINGS', 'KISS', 'KITE', 'KNEE', 'KNEW', 'KNOW', 'KNOWN', - }) def _compute_callsign_with_confidence(self, channel_name): """ @@ -713,13 +612,6 @@ def _extract_callsign_with_confidence(self, channel_name): self._callsign_cache[channel_name] = result return result - def extract_callsign(self, channel_name): - """ - Extract US TV callsign from channel name with priority order. - Returns None if no callsign found or only a denylisted word was matched. - """ - callsign, _ = self._extract_callsign_with_confidence(channel_name) - return callsign def set_user_aliases(self, user_aliases): """Merge user-supplied aliases on top of the builtin set. Pass a dict @@ -785,11 +677,6 @@ def alias_match(self, query_name, candidate_names, user_ignored_tags=None): return cand, 100, "alias" return None, 0, None - def normalize_callsign(self, callsign): - """Remove suffix from callsign for display.""" - if callsign: - callsign = re.sub(r'-(?:TV|CD|LP|DT|LD)$', '', callsign) - return callsign def normalize_name(self, name, user_ignored_tags=None, ignore_quality=True, ignore_regional=True, ignore_geographic=True, ignore_misc=True, remove_cinemax=False, remove_country_prefix=False): @@ -1057,70 +944,7 @@ def extract_tags(self, name, user_ignored_tags=None): return regional, extra_tags, quality_tags - def calculate_similarity(self, str1, str2, min_ratio=0.0): - """Levenshtein similarity ratio (0.0-1.0), defined as 1 - distance/max(len). - If min_ratio > 0, returns 0.0 early when the result can't reach it. - - bug-026: the ratio MUST be distance/max(len), matching rapidfuzz - Levenshtein.normalized_similarity. The old (len1+len2-distance)/(len1+len2) - formula scored higher for the same edit distance and let numbered siblings - ("Fox Sports 1" vs "2") pass threshold 95. The rapidfuzz fast path and the - pure-Python fallback below compute the identical value. - """ - if len(str1) == 0 or len(str2) == 0: - return 0.0 - - # Fast path: C-accelerated rapidfuzz when available (same definition). - if _USE_RAPIDFUZZ: - # No score_cutoff: rapidfuzz's cutoff rounding zeroes a score landing - # exactly on min_ratio, but the pure-Python path returns it. Dropping - # the cutoff makes the two paths agree at the threshold boundary; - # callers apply their own >= comparison. - return _rf_lev.normalized_similarity(str1, str2) - - if len(str1) < len(str2): - str1, str2 = str2, str1 - len1, len2 = len(str1), len(str2) - max_len = len1 # the longer string after the swap - - # Length-difference pre-check: minimum possible distance is (len1 - len2), - # so the max possible ratio is (max_len - (len1 - len2)) / max_len. - if min_ratio > 0: - max_possible = (max_len - (len1 - len2)) / max_len - if max_possible < min_ratio: - return 0.0 - - previous_row = list(range(len2 + 1)) - for i, c1 in enumerate(str1): - current_row = [i + 1] - for j, c2 in enumerate(str2): - insertions = previous_row[j + 1] + 1 - deletions = current_row[j] + 1 - substitutions = previous_row[j] + (c1 != c2) - current_row.append(min(insertions, deletions, substitutions)) - # Early termination: a lower bound on the final distance is the current - # row minimum minus the str1 chars still unprocessed. - if min_ratio > 0: - min_distance_so_far = min(current_row) - remaining = len1 - i - 1 - best_possible_distance = max(0, min_distance_so_far - remaining) - best_possible_ratio = (max_len - best_possible_distance) / max_len - if best_possible_ratio < min_ratio: - return 0.0 - previous_row = current_row - - distance = previous_row[-1] - return (max_len - distance) / max_len - @staticmethod - def _length_scaled_threshold(base_threshold, shorter_len): - """Require higher similarity for shorter strings to avoid false positives. - Short names (<=4 chars) need 95%, medium (<=8) need 90%.""" - if shorter_len <= 4: - return max(base_threshold, 95) - elif shorter_len <= 8: - return max(base_threshold, 90) - return base_threshold @staticmethod def _has_token_overlap(str_a, str_b, min_token_len=4, require_majority=False): @@ -1208,50 +1032,7 @@ def _meaningful(t): return True return bool(tokens_a & tokens_b) - @staticmethod - def _trailing_number(name): - """Integer value of a space-separated, purely-numeric trailing token, - or None. 'HBO 2' → 2, 'ESPN' → None, 'ESPN2' → None (digit not - space-separated). Used to reject 'Foo 1' vs 'Foo 2' false positives - — sibling channels that otherwise fuzzy-match almost perfectly.""" - m = re.search(r'(?:^|\s)(\d{1,4})\s*$', name or "") - return int(m.group(1)) if m else None - - def process_string_for_matching(self, s): - """ - Normalize a string for token-sort fuzzy matching. - Lowercases, removes accents, removes punctuation, sorts tokens. - Properly handles Unicode characters (e.g., French accents). - Normalizes spacing around numbers to handle "ITV1" vs "ITV 1" cases. - """ - # First, NFKD-fold: compatibility decomposition so "ＨＢＯ"->"HBO", "²"->"2", - # "fi"->"fi", and accents split into base + combining mark (dropped below). - s = unicodedata.normalize('NFKD', s) - - # Remove combining characters (accent marks) - # Keep only base characters - s = ''.join(char for char in s if unicodedata.category(char) != 'Mn') - - # Convert to lowercase - s = s.lower() - - # Normalize spacing around numbers: add space before numbers if not already present - # This makes "itv1" and "itv 1" equivalent after tokenization - # Pattern: letter (any script) followed immediately by digit -> insert space - s = re.sub(r'([^\W\d_])(\d)', r'\1 \2', s) - - # Replace non-alphanumeric with space. isalnum() keeps alphanumerics of any - # script (Cyrillic/CJK/Arabic) instead of erasing them to '' (false matches). - cleaned_s = "" - for char in s: - if char.isalnum(): - cleaned_s += char - else: - cleaned_s += ' ' - - # Split, sort, and rejoin - tokens = sorted([token for token in cleaned_s.split() if token]) - return " ".join(tokens) + def find_best_match(self, query_name, candidate_names, user_ignored_tags=None, remove_cinemax=False, ignore_quality=True, ignore_regional=True, ignore_geographic=True, ignore_misc=True): diff --git a/Channel-Maparr/matching_core.py b/Channel-Maparr/matching_core.py new file mode 100644 index 0000000..d7e9147 --- /dev/null +++ b/Channel-Maparr/matching_core.py @@ -0,0 +1,793 @@ +""" +matching_core.py - shared pure matching primitives for the Dispatcharr plugins. + +The canonical, vendored core extracted from Lineuparr's fuzzy_matcher.py (v1.3.4, the +newest generation). It is PURE and stateless string-in/string-out: stdlib + optional +rapidfuzz only, with no Django / ORM / filesystem coupling. Each plugin SUBCLASSES +FuzzyMatcherCore and layers its own orchestration on top (DB load, matching entry points, +and one feature block: zones / token-index / country / OTA). See +MATCHER-STANDARDIZATION-PLAN.md. + +Authored faithfully from Lineuparr with exactly these deliberate changes: + - class renamed FuzzyMatcher -> FuzzyMatcherCore; Lineuparr's country layer, the + matching entry points (alias_match / fuzzy_match / match_all_streams), the cache glue + (precompute_normalizations / _get_cached_*), _channel_number_boost, _is_group_header, + and has_upgrade_quality are NOT included - they stay plugin-side. + - process_string_for_matching KEEPS '+' so Disney+ / Discovery+ / Paramount+ stay + distinct from their base channels (the 4-of-4 superset decision). + - the callsign confidence ladder consults an optional self._known_callsigns set so a + plugin can rescue a denylisted-but-real station (KING / WAVE / WOOD / WHO) from its own + channel DB. With the default (None) the ladder is identical to Lineuparr's pure path. + +This file is the SOURCE OF TRUTH. It is vendored (copied byte-identically) into each +plugin's flat inner folder at release time and hash-pinned; never edit a vendored copy - +edit this file and re-sync. +""" + +import logging +import re +import unicodedata + +# Optional C-accelerated Levenshtein. When present, the matcher uses rapidfuzz's +# normalized_similarity (1 - distance/max(len)); the pure-Python fallback below +# computes the identical value (bug-026). rapidfuzz is an OPTIONAL runtime dep. +try: + from rapidfuzz.distance import Levenshtein as _rf_lev + _USE_RAPIDFUZZ = True +except ImportError: + _USE_RAPIDFUZZ = False + +__version__ = "0.1.0" + +LOGGER = logging.getLogger("matching_core") +if not LOGGER.handlers: + _handler = logging.StreamHandler() + _handler.setFormatter(logging.Formatter("%(levelname)s %(name)s %(message)s")) + LOGGER.addHandler(_handler) +# WARNING by default so a vendored core does not spam each plugin's logs with the +# per-name "normalize_name returned empty" debug line; a plugin can pass its own +# logger at a lower level when diagnosing. +LOGGER.setLevel(logging.WARNING) + +# --- Pattern categories for normalization --- + +# Unicode categories considered decorative/badge characters by IPTV providers. +# So = Other Symbol (◉), No = Other Number (², ³), Lm = Modifier Letter +# (ᴿᴬᵂ, ᴴᴰ, ⱽᴵᴾ superscripts), Sk = Modifier Symbol. +# Accented letters (é, î, ü…) are Ll/Lu and are NOT in this set. +# Sm (Math Symbol) is intentionally EXCLUDED: it contains "+", which is a +# meaningful, channel-distinguishing character (Canal+, Three Stooges+, +# Comedy Central+). Stripping it regresses those matches. +_DECORATOR_CATS = frozenset({'So', 'No', 'Lm', 'Sk'}) + +# Tokens that are non-distinctive stream-label variants (e.g. "ABC News Live" +# should still match "ABC News"). Used by the subset/divergent guards. +_NON_DISTINCTIVE_TOKENS = frozenset({"live", "now", "new"}) + +def _is_distinctive(t): + """Return True if token t is distinctive enough to matter in subset/divergent guards.""" + return t not in _NON_DISTINCTIVE_TOKENS and (len(t) >= 4 or (t.isdigit() and len(t) >= 2)) + +QUALITY_PATTERNS = [ + r'\s*\[(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\]\s*', + r'\s*\((4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead|Backup)\)\s*', + r'^\s*(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s*', + r'\s*\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)$', + r'\s+\b(4K|8K|UHD|FHD|HD|HDR|HEVC|SD|FD|Unknown|Unk|Slow|Dead)\b\s+', +] + +# Numeric resolution markers the keyword QUALITY_PATTERNS miss: 720p, 1080p/i, 2160p, +# 3840P, 480p, etc. - a 3-4 digit run glued directly to p/i. The 3-digit lower bound +# excludes 2-digit noise; the 4-digit upper bound excludes 5-digit numbers (10800p won't +# match). The p/i must be GLUED to the digits (no space): real markers are always written +# "720P"/"3840P", and requiring the glue avoids stripping a spaced standalone P/I such as a +# roman numeral ("Volume 100 I"). The p/i \b anchor keeps bare numbers (1080, "Channel 4") +# intact. Applied with re.IGNORECASE in the ignore_quality block, like QUALITY_PATTERNS. +RESOLUTION_PATTERNS = [ + r'\b\d{3,4}[pi]\b', +] + +REGIONAL_PATTERNS = [ + # East/West are NOT stripped here - they distinguish separate channel feeds + # ("HBO East" vs "HBO West"); normalize_name strips bare East/West separately, + # gated on ignore_regional. + # bug-066: bare " Pacific"/" Central"/" Mountain"/" Atlantic" are brand tokens far + # more often than feed markers ("Comedy Central", "The Atlantic"), so they are + # stripped ONLY in their parenthesized form, never as bare words. + r'\s*\([Pp][Aa][Cc][Ii][Ff][Ii][Cc]\)\s*', + r'\s*\([Cc][Ee][Nn][Tt][Rr][Aa][Ll]\)\s*', + r'\s*\([Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]\)\s*', + r'\s*\([Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]\)\s*', +] + +# bug-066 (opt-in half): the BARE-word forms of the time-zone region markers. OFF by +# default - a plugin enables them by setting the class attr _STRIP_BARE_REGION = True +# (or passing strip_bare_region=True). Lineuparr opts in: it strips these for SCORING +# and relies on its post-match region filter - which reads the ORIGINAL un-normalized +# name - to enforce region correctness, so a "Sportsnet Pacific" stream can score-match +# a "Sportsnet (P)" lineup and an "HBO West" lineup keeps the "HBO Pacific" feed. Plugins +# WITHOUT that filter (Stream-Mapparr) leave it off, or brand tokens like "Comedy Central" +# / "The Atlantic" would be wrongly truncated. +REGIONAL_BARE_PATTERNS = [ + r'\s[Pp][Aa][Cc][Ii][Ff][Ii][Cc]', + r'\s[Cc][Ee][Nn][Tt][Rr][Aa][Ll]', + r'\s[Mm][Oo][Uu][Nn][Tt][Aa][Ii][Nn]', + r'\s[Aa][Tt][Ll][Aa][Nn][Tt][Ii][Cc]', +] + +# Country tokens for the delimited provider-prefix patterns below; curated so a +# bare delimited word ("(SPORTS)") isn't misread as a country. +_PREFIX_COUNTRY = r'US|USA|UK|CA|AU|FR|DE|ES|IT|NL|BR|MX|IN' + +# (open, close) pairs; open and close must MATCH, so "(US]" / "│US)" are rejected. +_DELIM_PAIRS = ((r'\(', r'\)'), (r'\[', r'\]'), (r'\|', r'\|'), ('┃', '┃'), ('│', '│')) + + +def _balanced_delim(token): + """Regex fragment matching `token` wrapped in one MATCHED delimiter pair: + (token) [token] |token| ┃token┃ │token│. `token` may contain a capture group.""" + return '(?:' + '|'.join( + o + r'\s*(?:' + token + r')\s*' + c for o, c in _DELIM_PAIRS + ) + ')' + + +# Strip a leading box-bar bouquet/source tag with arbitrary inner text +# ("┃CANAL+┃ NPO 1" -> "NPO 1"); box bars never occur in real names, so this is +# always safe and also covers leading "┃XX┃" country tags. +_LEADING_BAR_TAG_RE = re.compile(r'^\s*[┃│]\s*[^┃│]*[┃│]\s*') + +GEOGRAPHIC_PATTERNS = [ + r'\b[A-Z]{2,3}[:┃│]\s*', + r'\b[A-Z]{2,3}\s*-\s*', + # Matched bar pair only ("|US|", "┃US┃") - a mismatched "|US┃" is noise. + r'(?:\|[A-Z]{2,3}\||┃[A-Z]{2,3}┃|│[A-Z]{2,3}│)\s*', + r'\[[A-Z]{2,3}\]\s*', +] + +# Enhanced provider prefix patterns for IPTV-specific naming +PROVIDER_PREFIX_PATTERNS = [ + r'^(?:' + _PREFIX_COUNTRY + r')\s*[:\-\|┃│]\s*', + # Bare country tag + whitespace, no separator (e.g. "US Racer Network", + # "FR beIN SPORTS MAX", "MEX Bein Sports"). Restricted to a curated set so + # it cannot eat a real channel name: "USA Network" (USA != US + space), + # "In Country Television" ("IN") and "IT Crowd" ("IT") are all safe too. + r'^(?:US|UK|CA|AU|FR|DE|MX|MEX|FRA|GER)\s+', + # "USA " space prefix as a US country tag ("USA ABC", "USA BET"). A + # negative lookahead for NETWORK protects the real channel "USA Network" + # (these feeds tag that one as "US ..."/"US: ...", never bare "USA "). + r'^USA\s+(?!NETWORK\b)', + # Country code glued directly to a quality tag with no separator + # ("UKSD: Sky Sports", "UKHD ESPN", "USFHD ..."). + r'^(?:US|UK)(?:SD|HD|FHD|UHD|FD|HEVC|4K|8K)\b\s*[:\-\|┃│]?\s*', + # Bracketed/piped country tag with a MATCHED delimiter pair ("(US)", "│US│"). + r'^\s*' + _balanced_delim(_PREFIX_COUNTRY) + r'\s*', + r'\s*[\|┃│]\s*(?:' + _PREFIX_COUNTRY + r')\s*$', + # Content-category group prefixes used by some IPTV providers. + r'^(?:ADULT|EROTIC|PRIME|GOLD)\s*[:\-\|┃│]\s*', + # FAST streaming-platform source tags (Roku, Tubi, Pluto, etc.). These mark + # the distribution platform, not the channel or its country, so strip them + # for matching ("RK: beIN Sports Xtra" -> "beIN Sports Xtra"). A separator + # is required so this can't eat real names like "GOLF" or "PLEX TV Movies". + r'^(?:RK|GO|TUBI|PLUTO|XUMO|PLEX|STIRR|FREEVEE|GLANCE)\s*[:\-\|┃│]\s*', +] + +MISC_PATTERNS = [ + r'\s*\([^)]*\)\s*', +] + +# --------------------------------------------------------------------------- # +# Stylized-Unicode decoration stripping +# --------------------------------------------------------------------------- # +# Streams tag names with stylized-Unicode tier/format markers (superscript +# "WEATHERNATION RAW", small-cap "FHD", bullet-prefixed "CNN") that the ASCII tag +# regexes below cannot see. We drop whole tokens that are pure decoration BEFORE +# the ASCII pipeline runs. Detection is by Unicode character *name* (not code-point +# ranges), so it covers superscripts, "modifier letter" superscript capitals, and +# Latin small-caps wherever they live (e.g. small-cap H is U+029C in IPA Extensions +# and modifier V is U+2C7D in Latin-Ext-C, both outside the obvious blocks). + +# Ornament glyphs whose Unicode name carries no decoration keyword. +_DECORATIVE_SYMBOLS = frozenset("◉") # FISHEYE; add individual chars (not strings) here + + +def _is_decorative_char(ch): + """True for a stylized letterform/ornament that carries no semantic content in a + channel name (superscripts, subscripts, modifier-letter superscript capitals, + Latin small-capitals, curated bullets). ASCII and ordinary letters return False.""" + if ch.isascii(): + return False + if ch in _DECORATIVE_SYMBOLS: + return True + try: + nm = unicodedata.name(ch) + except ValueError: + # unnamed code point (control char / lone surrogate) -> not decoration + return False + return ('SUPERSCRIPT' in nm or 'SUBSCRIPT' in nm + or 'SMALL CAPITAL' in nm or 'MODIFIER LETTER' in nm) + + +def _strip_stylized_tokens(name): + """Drop whitespace tokens that are pure stylized decoration, then NFKD-canonicalize + the remainder. A token is decoration when it has >=1 decorative char, no ASCII + alphanumeric, and every char is decorative or ASCII punctuation (so a bullet glued + to a colon, or "HD/RAW" written in superscripts, are dropped too). Real ASCII words + (Gold/VIP) and non-Latin letters (Arabic/Cyrillic/CJK) are always kept. ASCII-only + input is returned unchanged via the fast path (no per-char work; NFKD is a no-op + on ASCII, so skipping it changes nothing).""" + if name.isascii(): + return name + kept = [] + for tok in name.split(): + has_decorative = any(_is_decorative_char(c) for c in tok) + has_ascii_alnum = any(c.isascii() and c.isalnum() for c in tok) + only_decorative_or_punct = all( + _is_decorative_char(c) or (c.isascii() and not c.isalnum()) for c in tok + ) + if has_decorative and only_decorative_or_punct and not has_ascii_alnum: + continue # pure decoration -> drop the whole token + kept.append(tok) + return unicodedata.normalize('NFKD', ' '.join(kept)) + + +# --------------------------------------------------------------------------- # +# Emoji-as-letter + emoji decoration normalization +# --------------------------------------------------------------------------- # +# Some streams use an emoji AS A LETTER inside a word: "SP⚽RTS" / "Sp⚽rts" where the +# soccer ball stands in for 'o' (= SPORTS, the beIN family). _strip_stylized_tokens keeps +# the token (it has ASCII alnum) and process_string_for_matching would turn the ball into a +# space ("sp rts"), so it never matches "sports". We substitute the glyph for the letter it +# replaces (only when flanked by ASCII letters) and strip emoji used purely as decoration. + +# Emoji that visually replace an ASCII letter when embedded in a word. Extensible. +_EMOJI_LETTER_MAP = {'⚽': 'o'} # SOCCER BALL = 'o' (SP⚽RTS -> SPORTS) +# Pictographic ornaments to delete. NOTE: ⚽ is intentionally in BOTH maps - the letter +# map handles it mid-word (-> 'o'); here it catches any ⚽ NOT flanked by ASCII letters +# (standalone/edge), which the substitution above leaves untouched. +_EMOJI_ORNAMENTS = frozenset('♬☾⚽') # beamed notes, last-quarter moon, soccer ball +# Zero-width / invisible code points that only add noise to a name. +_ZERO_WIDTH = ('️', '‍') # VARIATION SELECTOR-16, ZERO WIDTH JOINER + + +def _normalize_emoji(name): + """Map emoji-as-letters to their letter and strip emoji decoration. + + The letter substitution fires ONLY when the glyph is flanked by ASCII letters + (so "SP⚽RTS" -> "SPoRTS" but a standalone/edge "⚽" is treated as decoration and + dropped). Zero-width selectors and ornament pictographs are deleted outright. + ASCII-only input is returned unchanged (no emoji possible).""" + if name.isascii(): + return name + for zw in _ZERO_WIDTH: + if zw in name: + name = name.replace(zw, '') + for glyph, letter in _EMOJI_LETTER_MAP.items(): + if glyph in name: + name = re.sub(r'(?<=[A-Za-z])' + re.escape(glyph) + r'(?=[A-Za-z])', letter, name) + if any(c in _EMOJI_ORNAMENTS for c in name): + name = ''.join(c for c in name if c not in _EMOJI_ORNAMENTS) + return name + + +class FuzzyMatcherCore: + """Pure, stateless matching primitives shared across the Dispatcharr plugins. + + Subclass this and add the plugin-specific orchestration layer (DB load, matching + entry points, and one feature block). The core references self only for self.logger + and the per-instance caches; it never reads plugin_dir / a channel DB / alias maps. + """ + + def __init__(self, match_threshold=80, logger=None): + self.match_threshold = match_threshold + self.logger = logger or LOGGER + # Cache for pre-normalized stream names (performance optimization). The core + # initializes the storage; the cache-filling glue (precompute_normalizations / + # _get_cached_*) is plugin-side because its flag handling is plugin-specific. + self._norm_cache = {} # raw_name -> normalized_lower + self._norm_nospace_cache = {} # raw_name -> normalized_nospace + self._processed_cache = {} # raw_name -> processed_for_matching + # Cache for callsign extraction: raw_name -> (callsign|None, is_high_conf) + self._callsign_cache = {} + # Optional set of known-real callsigns a plugin supplies from its channel DB so + # the ladder can rescue a denylisted-but-real station (KING/WAVE/WOOD/WHO). None + # = no rescue, identical to the pure Lineuparr ladder. Assign it via + # set_known_callsigns() so the callsign cache is cleared when it changes. + self._known_callsigns = None + + def set_known_callsigns(self, known): + """Supply the plugin's known-real callsign set (or None to disable rescue). + + Clears the callsign cache so a changed set takes effect on the next extraction. + """ + self._known_callsigns = frozenset(known) if known else None + self._callsign_cache.clear() + + def normalize_name(self, name, user_ignored_tags=None, ignore_quality=True, ignore_regional=True, + ignore_geographic=True, ignore_misc=True, remove_cinemax=False, + remove_country_prefix=False, strip_bare_region=None): + """ + Normalize channel or stream name for matching by removing tags, prefixes, and noise. + + remove_cinemax / remove_country_prefix are optional opt-in superset behaviors (a + plugin sets them True); both default False, so the default path is unchanged. + + strip_bare_region opts into stripping BARE time-zone region words (Pacific/Central/ + Mountain/Atlantic) for scoring - see REGIONAL_BARE_PATTERNS. None (the default) + resolves it from the subclass class attr _STRIP_BARE_REGION (default False), so a + plugin enables it once via that attr instead of threading it through every call. + """ + if user_ignored_tags is None: + user_ignored_tags = [] + if strip_bare_region is None: + strip_bare_region = getattr(self, "_STRIP_BARE_REGION", False) + + original_name = name + + name = _LEADING_BAR_TAG_RE.sub('', name) # leading "┃CANAL+┃" bouquet tag + + # Map emoji-as-letters (⚽ = 'o' in "SP⚽RTS") and strip emoji decoration, before + # the stylized-Unicode strip and ASCII regexes below - so "beIN SP⚽RTS" -> "beIN sports". + name = _normalize_emoji(name) + + # Strip stylized-Unicode decoration (superscript/small-cap tier markers, + # bullets) up front so the ASCII tag regexes below see plain text. Runs + # unconditionally: a token written in superscript/small-caps is decoration + # regardless of tag_handling, and it would otherwise block matches + # (e.g. a superscript-RAW suffix never matches channel "WeatherNation"). + name = _strip_stylized_tokens(name) + + # Strip IPTV provider prefixes BEFORE hyphen normalization so that + # "FR - Canal+ FHD" loses its "FR - " while the hyphen is still a + # hyphen. After hyphen normalization the separator would become a space + # and the pattern would fail to match, leaving "FR" as a stray token. + for pattern in PROVIDER_PREFIX_PATTERNS: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + + # Quality patterns (before space normalization). Loop until stable so + # chained suffixes like "4K HDR" or "UHD HDR" are fully stripped in + # successive passes (each pass may expose a token for the next). + if ignore_quality: + prev = None + while prev != name: + prev = name + # Strip numeric resolution markers (3840P/2160P/1080P/720P/...) before the + # digit/letter spacer below would split "3840P" into "3840 P". + # Must run before QUALITY_PATTERNS so that removing " 4K " does not glue + # "SPoRTS" to "3840P" and break the word-boundary anchor. + for pattern in RESOLUTION_PATTERNS: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + for pattern in QUALITY_PATTERNS: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + + # Normalize spacing around numbers + name = re.sub(r'([a-zA-Z])(\d)', r'\1 \2', name) + name = re.sub(r'(\d)([a-zA-Z])', r'\1 \2', name) + + # Normalize hyphens to spaces + name = re.sub(r'-', ' ', name) + + # Replace dots between word chars with spaces (e.g. "JusticeCentral.TV" + # → "JusticeCentral TV"). Keeps the dot-suffix variant equivalent to + # the spaced form for matching purposes. + name = re.sub(r'(?<=\w)\.(?=\w)', ' ', name) + + # Normalize number-words to digits so "BBC Three" and "BBC 3" share + # tokens. Critical for cases like "Three Angels Broadcasting Network" + # vs "3 Angels Broadcasting Network", and for BBC One/Two/Three/Four + # vs BBC 1/2/3/4. Bounded by word boundaries so brand names with + # embedded letters (e.g. "Onesimus") aren't corrupted. + _NUM_WORDS = { + "one": "1", "two": "2", "three": "3", "four": "4", "five": "5", + "six": "6", "seven": "7", "eight": "8", "nine": "9", "ten": "10", + "eleven": "11", "twelve": "12", + } + def _num_repl(m): + return _NUM_WORDS[m.group(0).lower()] + name = re.sub( + r'\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\b', + _num_repl, name, flags=re.IGNORECASE, + ) + + # Split CamelCase boundaries so "JusticeCentral" becomes "Justice + # Central" and "DangerTV" becomes "Danger TV". Two separate patterns: + # 1. lower → Upper followed by lower (Justice|Central, Danger|Iq → no, etc.) + # 2. lower(4+) → UPPER acronym at word boundary (Danger|TV, Beauty|IQ) + # The 4-char floor on rule 2 protects short brand names like "MeTV" and + # "truTV" whose existing EPG matches rely on the un-split form. + name = re.sub(r'([a-z])([A-Z][a-z])', r'\1 \2', name) + name = re.sub(r'([a-z]{4,})([A-Z]{2,})\b', r'\1 \2', name) + + # Strip region tokens (East / Eastern / West and the (E)/(W) + # abbreviations) for SCORING. Region CORRECTNESS - East vs West vs + # Pacific - is enforced separately by the post-match region filter in + # match_all_streams, which reads the ORIGINAL un-normalized names, not + # this output. Stripping here lets a regionless lineup channel + # ("Food Network") still score-match a region-tagged stream + # ("Food Network Eastern" / "... East") instead of being dragged below + # threshold by the extra token; the filter then accepts it (regionless + # defaults to East). "Western"/"Westerns" is a movie genre, NOT a + # region, so bare "west" is stripped only on a word boundary (\bwest\b + # does not match inside "western") and "western" is never touched. + if ignore_regional: + name = re.sub(r'\(\s*(?:eastern|east|e|west|w)\s*\)', ' ', name, flags=re.IGNORECASE) + name = re.sub(r'\b(?:eastern|east|west)\b', ' ', name, flags=re.IGNORECASE) + + # Remove leading parenthetical prefixes + while name.lstrip().startswith('('): + new_name = re.sub(r'^\s*\([^\)]+\)\s*', '', name) + if new_name == name: + break + name = new_name + + # Opt-in: remove a country-code prefix (multi-country DBs). Strips a 2-3 + # letter colon/space prefix unless it is a quality tag. Off by default; + # PROVIDER_PREFIX_PATTERNS above already removes the curated country + # prefixes, so this only catches the remainder. (Stream-Mapparr opts in.) + if remove_country_prefix: + quality_tags = {'HD', 'SD', 'FD', 'UHD', 'FHD'} + prefix_match = re.match(r'^([A-Z]{2,3})[:\s]\s*', name) + if prefix_match: + prefix = prefix_match.group(1).upper() + if prefix not in quality_tags: + name = name[len(prefix_match.group(0)):] + + # Opt-in: remove a leading "Cinemax" (for channels containing "max"). Off + # by default. (Stream-Mapparr opts in.) + if remove_cinemax: + name = re.sub(r'\bCinemax\b\s*', '', name, flags=re.IGNORECASE) + + # Build pattern list based on flags + patterns_to_apply = [] + if ignore_regional: + patterns_to_apply.extend(REGIONAL_PATTERNS) + if strip_bare_region: + patterns_to_apply.extend(REGIONAL_BARE_PATTERNS) + if ignore_geographic: + patterns_to_apply.extend(GEOGRAPHIC_PATTERNS) + if ignore_misc and ignore_regional: + patterns_to_apply.extend(MISC_PATTERNS) + + for pattern in patterns_to_apply: + name = re.sub(pattern, '', name, flags=re.IGNORECASE) + + # Apply user-configured ignored tags + for tag in user_ignored_tags: + escaped_tag = re.escape(tag) + if '[' in tag or ']' in tag or '(' in tag or ')' in tag: + name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) + else: + if re.match(r'^\w+$', tag): + name = re.sub(r'\b' + escaped_tag + r'\b', '', name, flags=re.IGNORECASE) + else: + name = re.sub(escaped_tag + r'\s*', '', name, flags=re.IGNORECASE) + + # Remove callsigns in parentheses + if ignore_regional: + name = re.sub(r'\([KW][A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) + else: + name = re.sub(r'\([KW](?!EST\)|ACIFIC\)|ENTRAL\)|OUNTAIN\)|TLANTIC\))[A-Z]{3}(?:-(?:TV|CD|LP|DT|LD))?\)', '', name, flags=re.IGNORECASE) + + if ignore_regional: + name = re.sub(r'\([A-Z0-9]+\)', '', name) + + # Remove common suffixes/prefixes + name = re.sub(r'^The\s+', '', name, flags=re.IGNORECASE) + name = re.sub(r'\s+Network\s*$', '', name, flags=re.IGNORECASE) + name = re.sub(r'\s+Channel\s*$', '', name, flags=re.IGNORECASE) + name = re.sub(r'\s+TV\s*$', '', name, flags=re.IGNORECASE) + + # Strip decorative Unicode markers (◉, ², superscript letters ᴿᴬᵂ…) + # that some IPTV providers append as quality/status badges. Only + # characters in decorator categories are removed; accented letters + # (é, î, ü…) are in Ll/Lu and are preserved. + name = ''.join( + ' ' if unicodedata.category(c) in _DECORATOR_CATS else c + for c in name + ) + + # Clean up whitespace + name = re.sub(r'\s+', ' ', name).strip() + + if not name: + self.logger.debug(f"normalize_name returned empty for: '{original_name}'") + + return name + + def calculate_similarity(self, str1, str2, min_ratio=0.0): + """Levenshtein similarity ratio (0.0-1.0), defined as 1 - distance/max(len). + If min_ratio > 0, returns 0.0 early when the result can't reach it. + + bug-026: the ratio MUST be distance/max(len), matching rapidfuzz + Levenshtein.normalized_similarity. The old (len1+len2-distance)/(len1+len2) + formula scored higher for the same edit distance and let numbered siblings + ("Fox Sports 1" vs "2") pass threshold 95. The rapidfuzz fast path and the + pure-Python fallback below compute the identical value. + """ + if len(str1) == 0 or len(str2) == 0: + return 0.0 + + # Fast path: C-accelerated rapidfuzz when available (same definition). + if _USE_RAPIDFUZZ: + # Compute the true similarity, then apply the SAME gate as the pure-Python + # early-exit below: a score that cannot reach min_ratio is 0.0 on both paths + # (raw agreement below threshold), but a score landing EXACTLY on min_ratio is + # KEPT (>=, inclusive). We deliberately do NOT use rapidfuzz's own score_cutoff: + # it treats the cutoff as strict-greater and quantizes it onto the achievable + # distance grid, so a true 0.8 against a 0.8 cutoff is spuriously zeroed while the + # pure-Python path returns it. With min_ratio=0.0 the gate is a no-op. + sim = _rf_lev.normalized_similarity(str1, str2) + return sim if (min_ratio <= 0.0 or sim >= min_ratio) else 0.0 + + if len(str1) < len(str2): + str1, str2 = str2, str1 + len1, len2 = len(str1), len(str2) + max_len = len1 # the longer string after the swap + + # Length-difference pre-check: minimum possible distance is (len1 - len2), + # so the max possible ratio is (max_len - (len1 - len2)) / max_len. + if min_ratio > 0: + max_possible = (max_len - (len1 - len2)) / max_len + if max_possible < min_ratio: + return 0.0 + + previous_row = list(range(len2 + 1)) + for i, c1 in enumerate(str1): + current_row = [i + 1] + for j, c2 in enumerate(str2): + insertions = previous_row[j + 1] + 1 + deletions = current_row[j] + 1 + substitutions = previous_row[j] + (c1 != c2) + current_row.append(min(insertions, deletions, substitutions)) + # Early termination: a lower bound on the final distance is the current + # row minimum minus the str1 chars still unprocessed. + if min_ratio > 0: + min_distance_so_far = min(current_row) + remaining = len1 - i - 1 + best_possible_distance = max(0, min_distance_so_far - remaining) + best_possible_ratio = (max_len - best_possible_distance) / max_len + if best_possible_ratio < min_ratio: + return 0.0 + previous_row = current_row + + distance = previous_row[-1] + return (max_len - distance) / max_len + + @staticmethod + def _length_scaled_threshold(base_threshold, shorter_len): + """Require higher similarity for shorter strings to avoid false positives.""" + if shorter_len <= 4: + return max(base_threshold, 95) + elif shorter_len <= 8: + return max(base_threshold, 90) + return base_threshold + + @staticmethod + def _has_token_overlap(str_a, str_b, min_token_len=4, require_majority=False): + """Check that distinctive tokens are shared between two strings. + + Basic mode: at least one token (>= min_token_len) must be shared. + Majority mode: uses all tokens (>= 2 chars) and requires that more than + half of the smaller set overlaps. Catches false positives like + "america racing" vs "america bbc" while allowing single-token matches. + """ + # "network"/"channel"/"television" are generic brand-suffix words, not + # distinctive - treat as common so the subset guard does not reject + # cases like "FanDuel Sports Cincinnati" vs "FanDuel Sports Network + # Cincinnati". (They're already stripped from end-of-string by + # normalize_name; this catches mid-string occurrences.) + common_words = { + "the", "and", "of", "in", "on", "at", "to", "for", "a", "an", + "network", "channel", "television", + } + + if require_majority: + # Use all meaningful tokens (>= 2 chars) for stricter checking. + # Single-digit tokens (1, 2, 3, ...) are kept because they're + # channel-distinguishing (BBC 1 vs BBC 2, ESPN 1 vs ESPN 2 etc.) + # even though they're only 1 char. + def _meaningful(t): + if t in common_words: + return False + return len(t) >= 2 or t.isdigit() + tokens_a = {t for t in str_a.split() if _meaningful(t)} + tokens_b = {t for t in str_b.split() if _meaningful(t)} + if not tokens_a or not tokens_b: + return True + shared = tokens_a & tokens_b + if not shared: + return False + smaller = min(len(tokens_a), len(tokens_b)) + if not len(shared) > smaller / 2: + return False + + unique_a = tokens_a - tokens_b + unique_b = tokens_b - tokens_a + + # Subset guard: when one side is a strict subset of the other and + # the larger side has a distinctive token the smaller lacks, the + # candidate is a more specific channel than the query and the high + # fuzzy score is a false positive. Catches e.g. "Nickelodeon" vs + # "Nickelodeon Teen". Threshold is >=4 chars; "live"/"now" are + # explicitly non-distinctive (stream label variants) so "ABC News" + # still matches "ABC News Live". Pure-digit tokens >=2 chars + # (e.g. "360") are also distinctive: "Canal+Sport" must not match + # "Canal+Sport 360". + if not unique_a: + if any(_is_distinctive(t) for t in unique_b): + return False + elif not unique_b: + if any(_is_distinctive(t) for t in unique_a): + return False + + # Divergent guard: when BOTH sides have unique tokens AND at least + # one of those unique tokens is a distinctive (>=4 char) word, the + # strings describe different brands and the fuzzy score is + # misleading. Catches "Sky Cinema Disney" vs "Sky Cinema Decades" + # (decades = 7 chars), "Sky Cinema Fast" vs "Sky Cinema Family", + # and "BBC One vs BBC Two" once number-words become digits earlier. + # Number-word→digit normalization (in normalize_name) reduces this + # guard's risk: "Three Angels" / "3 Angels" both become "3 angels" + # before reaching here, so they never trigger this case. + if unique_a and unique_b: + if any(len(t) >= 4 for t in unique_a | unique_b): + return False + + # Numeric/ordinal divergent guard: when BOTH sides have a numeric + # or ordinal token unique to them, they are sibling channels + # distinguished by number (BBC One vs BBC Two; ESPN 1 vs ESPN 2). + # Short tokens like "one"/"two" wouldn't trip the divergent guard + # above so they need their own check. + _NUMERIC = { + "one", "two", "three", "four", "five", "six", "seven", "eight", + "nine", "ten", "eleven", "twelve", + "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", + "first", "second", "third", "fourth", "fifth", + } + if (unique_a & _NUMERIC) and (unique_b & _NUMERIC): + return False + + return True + + # Basic mode: at least one long token shared + tokens_a = {t for t in str_a.split() if t not in common_words and len(t) >= min_token_len} + tokens_b = {t for t in str_b.split() if t not in common_words and len(t) >= min_token_len} + if not tokens_a or not tokens_b: + return True + return bool(tokens_a & tokens_b) + + def process_string_for_matching(self, s): + """Normalize for token-sort matching: NFKD-fold (so "ＨＢＯ"->"hbo", "²"->"2", + "fi"->"fi", accents drop), lowercase, keep alphanumerics of any script + (isalnum keeps Cyrillic/CJK/Arabic rather than erasing them) plus '+' + (Disney+/Discovery+/Paramount+ stay distinct), sort tokens.""" + s = unicodedata.normalize('NFKD', s) + s = ''.join(char for char in s if unicodedata.category(char) != 'Mn') + s = s.lower() + s = re.sub(r'([^\W\d_])(\d)', r'\1 \2', s) # split letter-glued digit, any script + cleaned_s = "" + for char in s: + if char.isalnum() or char == '+': + cleaned_s += char + else: + cleaned_s += ' ' + tokens = sorted([token for token in cleaned_s.split() if token]) + return " ".join(tokens) + + @staticmethod + def _trailing_number(name): + """Return the integer value of a space-separated, purely-numeric + trailing token, or None. 'HBO 2' -> 2, 'DIRECTV 4K Live 1' -> 1, + 'ESPN' -> None, 'ESPN2' -> None (digit not space-separated). Used to + reject 'Foo 1' vs 'Foo 2' false positives -- different channels that + otherwise fuzzy-match almost perfectly.""" + m = re.search(r'(?:^|\s)(\d{1,4})\s*$', name or "") + return int(m.group(1)) if m else None + + # Words that match the callsign regex shape but are never US broadcast + # callsigns. A US callsign (K/W + 2-4 letters) is shape-identical to many + # common English words, so the loose Priority-4 pattern mis-extracts them - + # e.g. "with" in "Bizarre Foods with Andrew Zimmern" becomes callsign + # "WITH". Regex alone cannot tell "WITH" from "WABC"; frequent K/W-initial + # words are denied explicitly so they never extract as a callsign. WWE/WWF/ + # WCW stop wrestling show names extracting as false-positive callsigns. + _CALLSIGN_DENYLIST = frozenset({ + 'WWE', 'WWF', 'WCW', 'EAST', + 'WAR', 'WARS', 'WARM', 'WASH', 'WATCH', 'WAVE', 'WAVES', 'WAY', 'WAYS', + 'WEB', 'WEEK', 'WELL', 'WENT', 'WERE', 'WEST', 'WHAT', 'WHEN', 'WHERE', + 'WHICH', 'WHILE', 'WHITE', 'WHO', 'WHY', 'WIDE', 'WIFE', 'WILD', 'WILL', + 'WIND', 'WINE', 'WING', 'WINGS', 'WINS', 'WIRE', 'WISE', 'WISH', 'WITH', + 'WOLF', 'WOMAN', 'WOMEN', 'WOOD', 'WORD', 'WORDS', 'WORK', 'WORKS', + 'WORLD', 'WORM', 'WORN', 'WRAP', + 'KEEN', 'KEEP', 'KEPT', 'KEY', 'KEYS', 'KICK', 'KID', 'KIDS', 'KILL', + 'KIND', 'KING', 'KINGS', 'KISS', 'KITE', 'KNEE', 'KNEW', 'KNOW', 'KNOWN', + }) + + def _is_callsign_allowed(self, callsign): + """A candidate callsign is allowed if it is not denylisted, OR the plugin + supplied a known-real callsign set that contains it (DB rescue).""" + return (callsign not in self._CALLSIGN_DENYLIST + or (self._known_callsigns is not None and callsign in self._known_callsigns)) + + def _compute_callsign_with_confidence(self, channel_name): + """ + Extract US TV callsign with a confidence flag. + + Returns (callsign, is_high_confidence). High confidence = + Priorities 1-3 (parenthesized / suffixed-paren / end-of-name). + Priority 4 (any loose word) is low confidence. (None, False) + when nothing extractable. + + A denylisted word is rejected UNLESS the plugin supplied a known-real + callsign set (set_known_callsigns) that contains it - that is the DB + rescue for real stations whose callsign collides with a common word + (KING/WAVE/WOOD/WHO). With no set supplied this is the pure ladder. + """ + # Remove common provider prefixes + channel_name = re.sub(r'^D\d+-', '', channel_name) + channel_name = re.sub(r'^USA?\s*[^a-zA-Z0-9]*\s*', '', channel_name, flags=re.IGNORECASE) + + # Priority 1: Callsigns in parentheses (most reliable) + paren_match = re.search(r'\(([KW][A-Z]{3})(?:-[A-Z\s]+)?\)', channel_name, re.IGNORECASE) + if paren_match: + callsign = paren_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, True + + # Priority 1b: grandfathered 3-letter callsigns in parentheses without a suffix + # (WWL/WJZ/KYW/WRC). Suffixed forms fall through to Priority 2. bug-062. + paren3_match = re.search(r'\(([KW][A-Z]{2})\)', channel_name, re.IGNORECASE) + if paren3_match: + callsign = paren3_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, True + + # Priority 2: Callsigns with suffix in parentheses + paren_suffix_match = re.search(r'\(([KW][A-Z]{2,4}-(?:TV|CD|LP|DT|LD))\)', channel_name, re.IGNORECASE) + if paren_suffix_match: + return paren_suffix_match.group(1).upper(), True + + # Priority 3: Callsigns at the end + end_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\s*(?:\.[a-z]+)?\s*$', channel_name, re.IGNORECASE) + if end_match: + callsign = end_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, True + + # Priority 4: Any word matching callsign pattern (low confidence) + word_match = re.search(r'\b([KW][A-Z]{2,4}(?:-(?:TV|CD|LP|DT|LD))?)\b', channel_name, re.IGNORECASE) + if word_match: + callsign = word_match.group(1).upper() + if self._is_callsign_allowed(callsign): + return callsign, False + + return None, False + + def _extract_callsign_with_confidence(self, channel_name): + """ + Cached wrapper around _compute_callsign_with_confidence. + + Extraction is pure in channel_name (and in self._known_callsigns, which is + held fixed between set_known_callsigns calls), so results are memoized - the + anchor calls this once per stream over a fixed candidate list, which is + otherwise massively redundant across the per-channel matching loop. + """ + cached = self._callsign_cache.get(channel_name) + if cached is not None: + return cached + result = self._compute_callsign_with_confidence(channel_name) + self._callsign_cache[channel_name] = result + return result + + def extract_callsign(self, channel_name): + """ + Extract US TV callsign from channel name with priority order. + Returns None if common false positives appear alone. + """ + callsign, _ = self._extract_callsign_with_confidence(channel_name) + return callsign + + def normalize_callsign(self, callsign): + """Remove the broadcast suffix (-TV/-CD/-LP/-DT/-LD) from a callsign.""" + if callsign: + callsign = re.sub(r'-(?:TV|CD|LP|DT|LD)$', '', callsign) + return callsign diff --git a/Channel-Maparr/plugin.json b/Channel-Maparr/plugin.json index 8619576..7b2aa49 100644 --- a/Channel-Maparr/plugin.json +++ b/Channel-Maparr/plugin.json @@ -1,6 +1,6 @@ { "name": "Channel Mapparr", - "version": "1.26.1781253", + "version": "1.26.1791324", "description": "Standardizes broadcast (OTA) and premium/cable channel names using network data and channel lists. Supports M3U stream import, category organization, and fuzzy matching across 42K+ channels in 11 countries.", "author": "PiratesIRC", "license": "MIT", diff --git a/Channel-Maparr/plugin.py b/Channel-Maparr/plugin.py index 6bd4a5a..cfd4749 100644 --- a/Channel-Maparr/plugin.py +++ b/Channel-Maparr/plugin.py @@ -44,7 +44,7 @@ class PluginConfig: """Configuration constants for Channel Maparr.""" - PLUGIN_VERSION = "1.26.1781253" + PLUGIN_VERSION = "1.26.1791324" # Channel Database Settings DEFAULT_CHANNEL_DATABASES = "US" diff --git a/scripts/core_manifest.json b/scripts/core_manifest.json new file mode 100644 index 0000000..7d8ff5b --- /dev/null +++ b/scripts/core_manifest.json @@ -0,0 +1,3 @@ +{ + "matching_core.py": "2ac7ac4cb283306152701bfd977ff2f13700d63d8023fdfa280b1480f5a969dc" +} diff --git a/scripts/sync_core.py b/scripts/sync_core.py new file mode 100644 index 0000000..8a3312b --- /dev/null +++ b/scripts/sync_core.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Vendor the shared matching core into a plugin's flat inner folder (Stage 0.5 tooling). + +This is the CANONICAL template. At each plugin's migration it is copied verbatim into +that plugin's tracked ``scripts/sync_core.py`` and run by the /release flow. It self- +locates assuming the layout ``//scripts/sync_core.py`` with the shared +source of truth at ``/_shared/`` and the deployable inner folder at +``///``. + +Modes: + (default) write: copy _shared/{matching_core,aliases_base}.py into the inner folder + and (re)write scripts/core_manifest.json with their sha256 hashes. + --check verify the vendored copies are byte-identical to _shared (the "forgot to + sync" gate). Needs _shared present, so it runs LOCALLY (release skill / + pre-commit), never in per-plugin GitHub CI. Exit != 0 on any drift. + --dry-run print what write would do (hashes + targets) without touching anything. + +The per-plugin CI gate is SEPARATE and needs no _shared: it asserts +sha256(inner/) == core_manifest.json (see tests/test_core_parity.py). Keep the two +vendored files OUT of any bump_version lockstep stamping, or the hash gate fails forever. +See MATCHER-STANDARDIZATION-PLAN.md §6. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +import sys +from pathlib import Path + +# The matcher is vendored now; aliases_base.py joins this list at the alias-migration +# stage (when each plugin's aliases.py becomes a de-drifted base + per-market overlay). +SHARED_FILES = ["matching_core.py"] + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def locate(): + """Return (inner_dir, shared_dir, manifest_path) from this script's location. + + Layout: //scripts/sync_core.py + /_shared/{matching_core,aliases_base}.py + /// (flat inner folder = deploy artifact) + """ + here = Path(__file__).resolve() + repo_root = here.parent.parent # / + workspace = repo_root.parent # / + shared_dir = workspace / "_shared" + inner_dir = repo_root / repo_root.name # // + manifest_path = here.parent / "core_manifest.json" + return inner_dir, shared_dir, manifest_path + + +def do_write(inner_dir: Path, shared_dir: Path, manifest_path: Path, dry_run: bool) -> int: + manifest = {} + for fname in SHARED_FILES: + src = shared_dir / fname + if not src.exists(): + print(f"ERROR: shared source missing: {src}") + return 1 + digest = file_sha256(src) + manifest[fname] = digest + dst = inner_dir / fname + if dry_run: + print(f" would copy {src} -> {dst} ({digest[:12]}…)") + else: + shutil.copyfile(src, dst) + print(f" vendored {fname} ({digest[:12]}…)") + if dry_run: + print(f" would write manifest {manifest_path}") + return 0 + manifest_path.parent.mkdir(parents=True, exist_ok=True) + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(f" wrote manifest {manifest_path}") + return 0 + + +def do_check(inner_dir: Path, shared_dir: Path) -> int: + drift = 0 + for fname in SHARED_FILES: + src, dst = shared_dir / fname, inner_dir / fname + if not src.exists(): + print(f"ERROR: shared source missing: {src}") + return 1 + if not dst.exists(): + print(f"DRIFT: vendored copy missing: {dst} (run sync_core.py)") + drift += 1 + continue + if file_sha256(src) != file_sha256(dst): + print(f"DRIFT: {fname} vendored copy differs from _shared (run sync_core.py)") + drift += 1 + if drift: + print(f"sync check FAILED: {drift} file(s) out of sync") + return 1 + print("sync check OK: vendored core matches _shared") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description="Vendor the shared matching core into this plugin.") + ap.add_argument("--check", action="store_true", help="verify vendored == _shared, don't write") + ap.add_argument("--dry-run", action="store_true", help="show what write would do") + args = ap.parse_args() + + inner_dir, shared_dir, manifest_path = locate() + if args.check: + return do_check(inner_dir, shared_dir) + return do_write(inner_dir, shared_dir, manifest_path, dry_run=args.dry_run) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/matcher_golden_baseline.json b/tests/matcher_golden_baseline.json new file mode 100644 index 0000000..bd37f1d --- /dev/null +++ b/tests/matcher_golden_baseline.json @@ -0,0 +1,300 @@ +{ + "calculate_similarity": { + "a|a": 1.0, + "bbcone|bbctwo": 0.5, + "cnn|cnninternational": 0.1875, + "discoverychannel|discovery": 0.5625, + "disney|disneyplus": 0.6, + "espn|espn2": 0.8, + "e|ae": 0.5, + "foxnews|foxnews": 1.0, + "hbo|hbo2": 0.75, + "justicecentral|truecrimenetwork": 0.25, + "nflnetwork|nhlnetwork": 0.9, + "paramount|paramountnetwork": 0.5625, + "usanetwork|usanetwork": 1.0, + "|": 0.0 + }, + "extract_callsign": { + "(D1) CBS": null, + "(PRIME) FOX News": null, + "BBC Four": null, + "BBC News": null, + "BBC Three": null, + "Cinemax HD": null, + "DangerTV": null, + "Discovery Channel": null, + "Discovery Channel 4K": null, + "Discovery+": null, + "Disney Channel": null, + "Disney+": null, + "ESPN 2": null, + "ESPN Pacific": null, + "ESPN [FHD]": null, + "Fox Sports West": null, + "France 2": null, + "HBO (W)": null, + "HBO 2": null, + "HBO East": null, + "HBO HD": null, + "HBO West": null, + "HLN": null, + "Justice Central": null, + "Justice Central TV": null, + "Justice Central.TV": null, + "JusticeCentral.TV": null, + "KCBS": "KCBS", + "KING 5": null, + "KOMO News": "KOMO", + "MTV": null, + "NHL Network": null, + "NewsNation": null, + "Paramount+": null, + "SEC Network": null, + "TNT UHD RAW": null, + "TUDN": null, + "Three Angels Broadcasting Network": null, + "True Crime Network": null, + "UK: BBC One": null, + "UK| ITV 1": null, + "US: USA Network HD": null, + "US| ESPN": null, + "WABC-TV": "WABC-TV", + "WAVE 3": null, + "WHO 13": null, + "WOOD TV8": null, + "[US] CNN": null, + "beИN SPORTS": null, + "getTV": null, + "Россия 1": null, + "┃US┃ ESPN": null, + "★ CNN ★": null, + "🅴🆂🅿🅵": null + }, + "normalize_callsign": { + "(D1) CBS": "(D1) CBS", + "(PRIME) FOX News": "(PRIME) FOX News", + "BBC Four": "BBC Four", + "BBC News": "BBC News", + "BBC Three": "BBC Three", + "Cinemax HD": "Cinemax HD", + "DangerTV": "DangerTV", + "Discovery Channel": "Discovery Channel", + "Discovery Channel 4K": "Discovery Channel 4K", + "Discovery+": "Discovery+", + "Disney Channel": "Disney Channel", + "Disney+": "Disney+", + "ESPN 2": "ESPN 2", + "ESPN Pacific": "ESPN Pacific", + "ESPN [FHD]": "ESPN [FHD]", + "Fox Sports West": "Fox Sports West", + "France 2": "France 2", + "HBO (W)": "HBO (W)", + "HBO 2": "HBO 2", + "HBO East": "HBO East", + "HBO HD": "HBO HD", + "HBO West": "HBO West", + "HLN": "HLN", + "Justice Central": "Justice Central", + "Justice Central TV": "Justice Central TV", + "Justice Central.TV": "Justice Central.TV", + "JusticeCentral.TV": "JusticeCentral.TV", + "KCBS": "KCBS", + "KING 5": "KING 5", + "KOMO News": "KOMO News", + "MTV": "MTV", + "NHL Network": "NHL Network", + "NewsNation": "NewsNation", + "Paramount+": "Paramount+", + "SEC Network": "SEC Network", + "TNT UHD RAW": "TNT UHD RAW", + "TUDN": "TUDN", + "Three Angels Broadcasting Network": "Three Angels Broadcasting Network", + "True Crime Network": "True Crime Network", + "UK: BBC One": "UK: BBC One", + "UK| ITV 1": "UK| ITV 1", + "US: USA Network HD": "US: USA Network HD", + "US| ESPN": "US| ESPN", + "WABC-TV": "WABC", + "WAVE 3": "WAVE 3", + "WHO 13": "WHO 13", + "WOOD TV8": "WOOD TV8", + "[US] CNN": "[US] CNN", + "beИN SPORTS": "beИN SPORTS", + "getTV": "getTV", + "Россия 1": "Россия 1", + "┃US┃ ESPN": "┃US┃ ESPN", + "★ CNN ★": "★ CNN ★", + "🅴🆂🅿🅵": "🅴🆂🅿🅵" + }, + "normalize_name": { + "all_on": { + "(D1) CBS": "CBS", + "(PRIME) FOX News": "FOX News", + "BBC Four": "BBC 4", + "BBC News": "BBC News", + "BBC Three": "BBC 3", + "Cinemax HD": "Cinemax", + "DangerTV": "Danger", + "Discovery Channel": "Discovery", + "Discovery Channel 4K": "Discovery", + "Discovery+": "Discovery+", + "Disney Channel": "Disney", + "Disney+": "Disney+", + "ESPN 2": "ESPN 2", + "ESPN Pacific": "ESPN Pacific", + "ESPN [FHD]": "ESPN", + "Fox Sports West": "Fox Sports West", + "France 2": "France 2", + "HBO (W)": "HBO West", + "HBO 2": "HBO 2", + "HBO East": "HBO East", + "HBO HD": "HBO", + "HBO West": "HBO West", + "HLN": "HLN", + "Justice Central": "Justice Central", + "Justice Central TV": "Justice Central", + "Justice Central.TV": "Justice Central", + "JusticeCentral.TV": "Justice Central", + "KCBS": "KCBS", + "KING 5": "KING 5", + "KOMO News": "KOMO News", + "MTV": "MTV", + "NHL Network": "NHL", + "NewsNation": "News Nation", + "Paramount+": "Paramount+", + "SEC Network": "SEC", + "TNT UHD RAW": "TNTRAW", + "TUDN": "TUDN", + "Three Angels Broadcasting Network": "3 Angels Broadcasting", + "True Crime Network": "True Crime", + "UK: BBC One": "BBC 1", + "UK| ITV 1": "ITV 1", + "US: USA Network HD": "USA", + "US| ESPN": "ESPN", + "WABC-TV": "WABC", + "WAVE 3": "WAVE 3", + "WHO 13": "WHO 13", + "WOOD TV8": "WOOD TV 8", + "[US] CNN": "CNN", + "beИN SPORTS": "beИN SPORTS", + "getTV": "getTV", + "Россия 1": "Россия 1", + "┃US┃ ESPN": "ESPN", + "★ CNN ★": "★ CNN ★", + "🅴🆂🅿🅵": "🅴🆂🅿🅵" + }, + "regional_off": { + "(D1) CBS": "CBS", + "(PRIME) FOX News": "FOX News", + "BBC Four": "BBC 4", + "BBC News": "BBC News", + "BBC Three": "BBC 3", + "Cinemax HD": "Cinemax", + "DangerTV": "Danger", + "Discovery Channel": "Discovery", + "Discovery Channel 4K": "Discovery", + "Discovery+": "Discovery+", + "Disney Channel": "Disney", + "Disney+": "Disney+", + "ESPN 2": "ESPN 2", + "ESPN Pacific": "ESPN Pacific", + "ESPN [FHD]": "ESPN", + "Fox Sports West": "Fox Sports West", + "France 2": "France 2", + "HBO (W)": "HBO West", + "HBO 2": "HBO 2", + "HBO East": "HBO East", + "HBO HD": "HBO", + "HBO West": "HBO West", + "HLN": "HLN", + "Justice Central": "Justice Central", + "Justice Central TV": "Justice Central", + "Justice Central.TV": "Justice Central", + "JusticeCentral.TV": "Justice Central", + "KCBS": "KCBS", + "KING 5": "KING 5", + "KOMO News": "KOMO News", + "MTV": "MTV", + "NHL Network": "NHL", + "NewsNation": "News Nation", + "Paramount+": "Paramount+", + "SEC Network": "SEC", + "TNT UHD RAW": "TNTRAW", + "TUDN": "TUDN", + "Three Angels Broadcasting Network": "3 Angels Broadcasting", + "True Crime Network": "True Crime", + "UK: BBC One": "BBC 1", + "UK| ITV 1": "ITV 1", + "US: USA Network HD": "USA", + "US| ESPN": "ESPN", + "WABC-TV": "WABC", + "WAVE 3": "WAVE 3", + "WHO 13": "WHO 13", + "WOOD TV8": "WOOD TV 8", + "[US] CNN": "CNN", + "beИN SPORTS": "beИN SPORTS", + "getTV": "getTV", + "Россия 1": "Россия 1", + "┃US┃ ESPN": "ESPN", + "★ CNN ★": "★ CNN ★", + "🅴🆂🅿🅵": "🅴🆂🅿🅵" + } + }, + "process_string": { + "(D1) CBS": "1 cbs d", + "(PRIME) FOX News": "fox news prime", + "BBC Four": "bbc four", + "BBC News": "bbc news", + "BBC Three": "bbc three", + "Cinemax HD": "cinemax hd", + "DangerTV": "dangertv", + "Discovery Channel": "channel discovery", + "Discovery Channel 4K": "4k channel discovery", + "Discovery+": "discovery+", + "Disney Channel": "channel disney", + "Disney+": "disney+", + "ESPN 2": "2 espn", + "ESPN Pacific": "espn pacific", + "ESPN [FHD]": "espn fhd", + "Fox Sports West": "fox sports west", + "France 2": "2 france", + "HBO (W)": "hbo w", + "HBO 2": "2 hbo", + "HBO East": "east hbo", + "HBO HD": "hbo hd", + "HBO West": "hbo west", + "HLN": "hln", + "Justice Central": "central justice", + "Justice Central TV": "central justice tv", + "Justice Central.TV": "central justice tv", + "JusticeCentral.TV": "justicecentral tv", + "KCBS": "kcbs", + "KING 5": "5 king", + "KOMO News": "komo news", + "MTV": "mtv", + "NHL Network": "network nhl", + "NewsNation": "newsnation", + "Paramount+": "paramount+", + "SEC Network": "network sec", + "TNT UHD RAW": "raw tnt uhd", + "TUDN": "tudn", + "Three Angels Broadcasting Network": "angels broadcasting network three", + "True Crime Network": "crime network true", + "UK: BBC One": "bbc one uk", + "UK| ITV 1": "1 itv uk", + "US: USA Network HD": "hd network us usa", + "US| ESPN": "espn us", + "WABC-TV": "tv wabc", + "WAVE 3": "3 wave", + "WHO 13": "13 who", + "WOOD TV8": "8 tv wood", + "[US] CNN": "cnn us", + "beИN SPORTS": "beиn sports", + "getTV": "gettv", + "Россия 1": "1 россия", + "┃US┃ ESPN": "espn us", + "★ CNN ★": "cnn", + "🅴🆂🅿🅵": "" + } +} \ No newline at end of file diff --git a/tests/test_core_parity.py b/tests/test_core_parity.py new file mode 100644 index 0000000..82e0d4e --- /dev/null +++ b/tests/test_core_parity.py @@ -0,0 +1,31 @@ +"""Vendored-core drift gate (Layer A) — runs in CI, needs no workspace _shared/. + +The committed vendored shared files (matching_core.py, and later aliases_base.py) must +hash-match scripts/core_manifest.json. This catches a hand-edit to a vendored copy that +silently diverges from the _shared source of truth. To land an intended core change: +edit _shared/, re-run scripts/sync_core.py (re-vendors + rewrites the manifest), and +commit both. See MATCHER-STANDARDIZATION-PLAN.md §6. +""" +import hashlib +import json +import os + +import pytest + +_REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +_INNER = os.path.join(_REPO, os.path.basename(_REPO)) +_MANIFEST = os.path.join(_REPO, "scripts", "core_manifest.json") + +with open(_MANIFEST, encoding="utf-8") as _fh: + _PINS = json.load(_fh) + + +@pytest.mark.parametrize("fname", sorted(_PINS)) +def test_vendored_core_matches_manifest(fname): + path = os.path.join(_INNER, fname) + assert os.path.exists(path), f"vendored {fname} is missing (run scripts/sync_core.py)" + digest = hashlib.sha256(open(path, "rb").read()).hexdigest() + assert digest == _PINS[fname], ( + f"{fname} vendored copy drifted from its pinned hash. If intended, edit _shared/, " + f"re-run scripts/sync_core.py, and commit the updated core_manifest.json." + ) diff --git a/tests/test_matcher_golden.py b/tests/test_matcher_golden.py new file mode 100644 index 0000000..a2565a8 --- /dev/null +++ b/tests/test_matcher_golden.py @@ -0,0 +1,125 @@ +"""Golden drift gate for the pure matcher primitives (Stage 0). + +Freezes the output of fuzzy_matcher.py's PURE primitives (normalize_name, +process_string_for_matching, calculate_similarity, extract_callsign, +normalize_callsign) over a shared corpus, so any unreviewed change to match behavior +fails CI. During the matcher-standardization migration, an INTENDED de-drift change is +landed by re-running ``python tools/matcher_parity_check.py --write`` at the workspace +root and committing the updated ``matcher_golden_baseline.json`` in the same change — +never by editing expectations ad hoc. + +This file is intentionally self-contained and IDENTICAL across Stream-Mapparr, +Channel-Maparr, and EPG-Janitor (CI runs per-repo and cannot see the workspace tool); it +loads fuzzy_matcher.py directly from the inner folder, so it needs no conftest fixture. +Only the committed baseline JSON differs per plugin. Keep the corpus below in lockstep +with tools/matcher_parity_check.py. See MATCHER-STANDARDIZATION-PLAN.md §7. +""" +import importlib.util +import json +import os +import sys + +import pytest + +# --- shared corpus (keep identical to tools/matcher_parity_check.py) --------- +NAMES = [ + "US: USA Network HD", "US| ESPN", "[US] CNN", "UK: BBC One", "UK| ITV 1", + "Discovery Channel 4K", "HBO HD", "ESPN [FHD]", "Cinemax HD", "TNT UHD RAW", + "BBC Three", "BBC Four", "Three Angels Broadcasting Network", "ESPN 2", "HBO 2", + "JusticeCentral.TV", "DangerTV", "NewsNation", + "HBO East", "HBO West", "HBO (W)", "Fox Sports West", "ESPN Pacific", + "(PRIME) FOX News", "(D1) CBS", + "Disney+", "Discovery+", "Paramount+", "Disney Channel", "Discovery Channel", + "Justice Central", "Justice Central.TV", "Justice Central TV", "True Crime Network", + "WABC-TV", "KCBS", "KING 5", "WAVE 3", "WOOD TV8", "WHO 13", "KOMO News", + "\U0001f174\U0001f182\U0001f17f\U0001f175", "┃US┃ ESPN", "★ CNN ★", + "Россия 1", "France 2", "beИN SPORTS", + "HLN", "MTV", "getTV", "TUDN", "SEC Network", "NHL Network", "BBC News", +] +PAIRS = [ + ("usanetwork", "usanetwork"), ("espn", "espn2"), ("hbo", "hbo2"), + ("bbcone", "bbctwo"), ("disney", "disneyplus"), ("foxnews", "foxnews"), + ("cnn", "cnninternational"), ("discoverychannel", "discovery"), ("e", "ae"), + ("paramount", "paramountnetwork"), ("nflnetwork", "nhlnetwork"), + ("justicecentral", "truecrimenetwork"), ("a", "a"), ("", ""), +] +FLAG_COMBOS = [ + ("all_on", dict(ignore_quality=True, ignore_regional=True, ignore_geographic=True, ignore_misc=True)), + ("regional_off", dict(ignore_quality=True, ignore_regional=False, ignore_geographic=True, ignore_misc=True)), +] + + +def _safe(fn, *args, **kwargs): + try: + val = fn(*args, **kwargs) + except Exception as exc: + return f"__ERROR__: {type(exc).__name__}: {exc}" + if isinstance(val, float): + return round(val, 9) + return val + + +def run_corpus(matcher): + out = { + "process_string": {n: _safe(matcher.process_string_for_matching, n) for n in NAMES}, + "normalize_name": {}, + "calculate_similarity": {f"{a}|{b}": _safe(matcher.calculate_similarity, a, b) for a, b in PAIRS}, + "extract_callsign": {n: _safe(matcher.extract_callsign, n) for n in NAMES}, + "normalize_callsign": {n: _safe(matcher.normalize_callsign, n) for n in NAMES}, + } + for label, flags in FLAG_COMBOS: + out["normalize_name"][label] = {n: _safe(matcher.normalize_name, n, **flags) for n in NAMES} + return out + + +def _flatten(d, prefix=""): + for k in sorted(d): + v = d[k] + if isinstance(v, dict): + yield from _flatten(v, f"{prefix}{k}.") + else: + yield (f"{prefix}{k}", v) + + +def _load_fuzzy_matcher(): + """Load fuzzy_matcher.py from the inner folder, conftest-free. + + Layout: /tests/test_matcher_golden.py and //fuzzy_matcher.py. + """ + repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + inner = os.path.join(repo_root, os.path.basename(repo_root)) + path = os.path.join(inner, "fuzzy_matcher.py") + saved_path = list(sys.path) + saved_aliases = sys.modules.pop("aliases", None) + sys.path.insert(0, inner) + try: + spec = importlib.util.spec_from_file_location("fm_golden_under_test", path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + finally: + sys.path[:] = saved_path + sys.modules.pop("aliases", None) + if saved_aliases is not None: + sys.modules["aliases"] = saved_aliases + return mod + + +_FM_MOD = _load_fuzzy_matcher() + +_BASELINE_FILE = os.path.join(os.path.dirname(__file__), "matcher_golden_baseline.json") +with open(_BASELINE_FILE, encoding="utf-8") as _fh: + _BASELINE_FLAT = dict(_flatten(json.load(_fh))) + + +@pytest.fixture(scope="module") +def current(): + return dict(_flatten(run_corpus(_FM_MOD.FuzzyMatcher()))) + + +@pytest.mark.parametrize("key", sorted(_BASELINE_FLAT)) +def test_primitive_matches_golden(current, key): + assert current.get(key, "") == _BASELINE_FLAT[key], ( + f"matcher primitive drifted at {key!r}: " + f"baseline={_BASELINE_FLAT[key]!r} now={current.get(key, '')!r}. " + f"If intended, re-run tools/matcher_parity_check.py --write and commit the baseline." + ) From 287df19908f54d797d03492269499b2d35c55129 Mon Sep 17 00:00:00 2001 From: PiratesIRC Date: Sun, 28 Jun 2026 15:47:13 -0500 Subject: [PATCH 3/3] Fix matcher-gate inner-folder resolution for GitHub CI The golden/parity gates resolved the inner deploy folder as repo_root/basename(repo_root), assuming the repo directory name equals the inner folder name. That holds locally and for Stream-Mapparr (repo == "Stream-Mapparr"), but on GitHub the repos check out as "Dispatcharr--Plugin" while the inner folder is "", so the gates looked for .../Dispatcharr-X-Plugin/Dispatcharr-X-Plugin/ fuzzy_matcher.py and failed test collection. Resolve the inner folder by finding the subdir that actually contains fuzzy_matcher.py (with the old assumption as fallback). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01N2rpwFn3AHoNvsbTrEwgwd --- scripts/sync_core.py | 2 +- tests/test_core_parity.py | 6 +++++- tests/test_matcher_golden.py | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/sync_core.py b/scripts/sync_core.py index 8a3312b..13b38a1 100644 --- a/scripts/sync_core.py +++ b/scripts/sync_core.py @@ -49,7 +49,7 @@ def locate(): repo_root = here.parent.parent # / workspace = repo_root.parent # / shared_dir = workspace / "_shared" - inner_dir = repo_root / repo_root.name # // + inner_dir = next((p.parent for p in repo_root.glob("*/fuzzy_matcher.py")), repo_root / repo_root.name) manifest_path = here.parent / "core_manifest.json" return inner_dir, shared_dir, manifest_path diff --git a/tests/test_core_parity.py b/tests/test_core_parity.py index 82e0d4e..187a20a 100644 --- a/tests/test_core_parity.py +++ b/tests/test_core_parity.py @@ -13,7 +13,11 @@ import pytest _REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -_INNER = os.path.join(_REPO, os.path.basename(_REPO)) +_INNER = next( + (os.path.join(_REPO, _e) for _e in sorted(os.listdir(_REPO)) + if os.path.isfile(os.path.join(_REPO, _e, "fuzzy_matcher.py"))), + os.path.join(_REPO, os.path.basename(_REPO)), +) _MANIFEST = os.path.join(_REPO, "scripts", "core_manifest.json") with open(_MANIFEST, encoding="utf-8") as _fh: diff --git a/tests/test_matcher_golden.py b/tests/test_matcher_golden.py index a2565a8..4eb585b 100644 --- a/tests/test_matcher_golden.py +++ b/tests/test_matcher_golden.py @@ -87,7 +87,11 @@ def _load_fuzzy_matcher(): Layout: /tests/test_matcher_golden.py and //fuzzy_matcher.py. """ repo_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - inner = os.path.join(repo_root, os.path.basename(repo_root)) + inner = next( + (os.path.join(repo_root, _e) for _e in sorted(os.listdir(repo_root)) + if os.path.isfile(os.path.join(repo_root, _e, "fuzzy_matcher.py"))), + os.path.join(repo_root, os.path.basename(repo_root)), + ) path = os.path.join(inner, "fuzzy_matcher.py") saved_path = list(sys.path) saved_aliases = sys.modules.pop("aliases", None)