fix(yara): reduce packaged malware-signature false positives (#236)

src/skillspector/nodes/analyzers/static_yara.py

@@ -22,8 +22,11 @@
 
 from __future__ import annotations
 
+import base64
+import binascii
 import hashlib
 from pathlib import Path
+from tempfile import TemporaryDirectory
 
 import yara
 
@@ -40,7 +43,8 @@
 
 _BUILTIN_RULES_DIR = Path(__file__).resolve().parent.parent.parent / "yara_rules"
 
-_RULE_EXTENSIONS = ("*.yar", "*.yara")
+_RULE_EXTENSIONS = ("*.yar", "*.yara", "*.yar.b64", "*.yara.b64")
+_ENCODED_RULE_SUFFIXES = (".yar.b64", ".yara.b64")
 
 _CATEGORY_MAP: dict[str, tuple[str, Severity]] = {
     "malware": ("YR1", Severity.CRITICAL),
@@ -82,15 +86,47 @@ def _content_hash(rule_files: list[Path]) -> str:
     return h.hexdigest()
 
 
-def _build_namespace_map(rule_files: list[Path]) -> dict[str, str]:
-    """Build a {namespace: filepath} dict from rule files, deduplicating namespace names."""
+def _rule_namespace(rule_file: Path) -> str:
+    """Derive a stable namespace from a rule file name."""
+    for suffix in _ENCODED_RULE_SUFFIXES:
+        if rule_file.name.endswith(suffix):
+            return rule_file.name[: -len(suffix)]
+    return rule_file.stem
+
+
+def _materialize_rule_file(
+    rule_file: Path, temp_dir: Path | None = None, namespace: str | None = None
+) -> Path:
+    """Return a compile-ready rule path, decoding embedded sources when needed."""
+    if not rule_file.name.endswith(_ENCODED_RULE_SUFFIXES):
+        return rule_file
+    if temp_dir is None:
+        raise ValueError("temp_dir is required for encoded rule files")
+
+    encoded_source = rule_file.read_text(encoding="utf-8")
+    decoded_source = base64.b64decode("".join(encoded_source.split())).decode("utf-8")
+    temp_name = (namespace or _rule_namespace(rule_file)).replace("/", "__")
+    temp_file = temp_dir / f"{temp_name}.yar"
+    temp_file.write_text(decoded_source, encoding="utf-8")
+    return temp_file
+
+
+def _build_namespace_map(
+    rule_files: list[Path], temp_dir: Path | None = None
+) -> tuple[dict[str, str], int]:
+    """Build a {namespace: filepath} dict and count malformed encoded files."""
     filepaths: dict[str, str] = {}
+    skipped = 0
     for rf in rule_files:
-        ns = rf.stem
+        ns = _rule_namespace(rf)
         if ns in filepaths:
-            ns = f"{rf.parent.name}/{rf.stem}"
-        filepaths[ns] = str(rf)
-    return filepaths
+            ns = f"{rf.parent.name}/{ns}"
+        try:
+            filepaths[ns] = str(_materialize_rule_file(rf, temp_dir, ns))
+        except (binascii.Error, UnicodeDecodeError) as exc:
+            skipped += 1
+            logger.debug("%s: skipping malformed encoded rule %s: %s", ANALYZER_ID, rf, exc)
+    return filepaths, skipped
 
 
 def _compile_rules(filepaths: dict[str, str]) -> tuple[yara.Rules | None, int]:
@@ -140,18 +176,22 @@ def _load_rules(extra_dir: Path | None = None) -> yara.Rules | None:
     if _compiled_rules is not None and _rules_hash == current_hash:
         return _compiled_rules
 
-    filepaths = _build_namespace_map(rule_files)
-    compiled, skipped = _compile_rules(filepaths)
+    with TemporaryDirectory() as temp_dir_name:
+        temp_dir = Path(temp_dir_name)
+        filepaths, materialize_skipped = _build_namespace_map(rule_files, temp_dir)
 
-    if compiled is None:
-        logger.warning("%s: failed to compile any YARA rules", ANALYZER_ID)
-        return None
+        compiled, compile_skipped = _compile_rules(filepaths)
+        skipped = materialize_skipped + compile_skipped
+
+        if compiled is None:
+            logger.warning("%s: failed to compile any YARA rules", ANALYZER_ID)
+            return None
 
-    _compiled_rules = compiled
-    _rules_hash = current_hash
-    loaded = len(filepaths) - skipped
-    logger.info("%s: compiled %d YARA rule file(s) (%d skipped)", ANALYZER_ID, loaded, skipped)
-    return compiled
+        _compiled_rules = compiled
+        _rules_hash = current_hash
+        loaded = len(filepaths) - compile_skipped
+        logger.info("%s: compiled %d YARA rule file(s) (%d skipped)", ANALYZER_ID, loaded, skipped)
+        return compiled
 
 
 def _extract_match_strings(match: yara.Match) -> tuple[int, str | None]:

src/skillspector/yara_rules/malware.yar.b64

@@ -0,0 +1,89 @@
+LyoNCiAgICBNYWx3YXJlIGluZGljYXRvciBydWxlcyBmb3Igc291cmNlIGNvZGUgc2Nhbm5pbmcu
+DQogICAgQmFzZWQgb24gcGF0dGVybnMgZnJvbSBOZW8yM3gwL3NpZ25hdHVyZS1iYXNlIGFuZCBj
+b21tdW5pdHkgcmVzZWFyY2guDQogICAgQ292ZXJzIHJldmVyc2Ugc2hlbGxzLCBiYWNrZG9vcnMs
+IGtleWxvZ2dlcnMsIHJhbnNvbXdhcmUtbGlrZSBiZWhhdmlvciwNCiAgICBhbmQgQzIgZnJhbWV3
+b3JrIGluZGljYXRvcnMgZm91bmQgaW4gc291cmNlL3NjcmlwdCBmaWxlcy4NCiovDQoNCnJ1bGUg
+cmV2ZXJzZV9zaGVsbA0Kew0KICAgIG1ldGE6DQogICAgICAgIGRlc2NyaXB0aW9uID0gIlJldmVy
+c2Ugc2hlbGwgcGF0dGVybnMgaW4gc2NyaXB0cyBvciBzb3VyY2UgY29kZSINCiAgICAgICAgY2F0
+ZWdvcnkgPSAibWFsd2FyZSINCiAgICAgICAgc2V2ZXJpdHkgPSAiQ1JJVElDQUwiDQogICAgICAg
+IGNvbmZpZGVuY2UgPSAiMC44NSINCiAgICAgICAgcmVmZXJlbmNlID0gImh0dHBzOi8vZ2l0aHVi
+LmNvbS9OZW8yM3gwL3NpZ25hdHVyZS1iYXNlIg0KICAgIHN0cmluZ3M6DQogICAgICAgICRiYXNo
+X3JldnNoZWxsICAgID0gL2Jhc2hccystaVxzKz4mXHMqXC9kZXZcL3RjcFwvLyBub2Nhc2UNCiAg
+ICAgICAgJG5jX3NoZWxsICAgICAgICAgPSAvbmNccy4qLWVccypcL2JpblwvKGJhKT9zaC8gbm9j
+YXNlDQogICAgICAgICRuY2F0X3NoZWxsICAgICAgID0gL25jYXRccy4qLWVccypcL2JpblwvKGJh
+KT9zaC8gbm9jYXNlDQogICAgICAgICRweXRob25fc29ja2V0ICAgID0gL3NvY2tldFwuc29ja2V0
+XCguKlNPQ0tfU1RSRUFNLipcLmNvbm5lY3RcKC8NCiAgICAgICAgJHBlcmxfc29ja2V0ICAgICAg
+PSAvdXNlXHMrU29ja2V0Oy4qc29ja2V0XHMqXChccypTT0NLLw0KICAgICAgICAkcGhwX2Zzb2Nr
+ICAgICAgICA9IC9mc29ja29wZW5ccypcKC4qZXhlY1xzKlwoLyBub2Nhc2UNCiAgICAgICAgJHJ1
+YnlfdGNwc29ja2V0ICAgPSAvVENQU29ja2V0XC5ccypuZXdccypcKC4qZXhlY1xzKlwoLw0KICAg
+ICAgICAkcG93ZXJzaGVsbF90Y3AgICA9IC9OZXctT2JqZWN0XHMrU3lzdGVtXC5OZXRcLlNvY2tl
+dHNcLlRDUENsaWVudC8gbm9jYXNlDQogICAgICAgICRzb2NhdF9zaGVsbCAgICAgID0gL3NvY2F0
+XHMrLipFWEVDLipcL2JpblwvKGJhKT9zaC8gbm9jYXNlDQogICAgICAgICRta2ZpZm9fc2hlbGwg
+ICAgID0gL21rZmlmb1xzKy4qXHxccypcL2JpblwvKGJhKT9zaC8NCiAgICBjb25kaXRpb246DQog
+ICAgICAgIGFueSBvZiB0aGVtDQp9DQoNCnJ1bGUgYmFja2Rvb3JfcGVyc2lzdGVuY2UNCnsNCiAg
+ICBtZXRhOg0KICAgICAgICBkZXNjcmlwdGlvbiA9ICJCYWNrZG9vciBwZXJzaXN0ZW5jZSB3aXRo
+IG1hbGljaW91cyBwYXlsb2FkcyAoc2hlbGwgY29tbWFuZHMsIFNTSCBrZXkgaW5qZWN0aW9uLCBo
+aWRkZW4gcm9vdCB1c2VycykiDQogICAgICAgIGNhdGVnb3J5ID0gIm1hbHdhcmUiDQogICAgICAg
+IHNldmVyaXR5ID0gIkhJR0giDQogICAgICAgIGNvbmZpZGVuY2UgPSAiMC43NSINCiAgICAgICAg
+cmVmZXJlbmNlID0gImh0dHBzOi8vZ2l0aHViLmNvbS9OZW8yM3gwL3NpZ25hdHVyZS1iYXNlIg0K
+ICAgIHN0cmluZ3M6DQogICAgICAgICRoaWRkZW5fdXNlciAgICAgICA9IC91c2VyYWRkXHMrLiot
+b1xzKy11XHMrMC8gbm9jYXNlDQogICAgICAgICRjcm9uX3BlcnNpc3QgICAgICA9IC9jcm9udGFi
+XHMuKihjdXJsfHdnZXR8bmN8YmFzaHxweXRob24pLyBub2Nhc2UNCiAgICAgICAgJHNzaF9pbmpl
+Y3QgICAgICAgID0gL2VjaG9ccysuKj4+P1xzKi4qXC5zc2hcL2F1dGhvcml6ZWRfa2V5cy8gbm9j
+YXNlDQogICAgICAgICRzeXN0ZW1kX3BlcnNpc3QgICA9IC9cW1NlcnZpY2VcXS4qRXhlY1N0YXJ0
+LioobmN8YmFzaHxweXRob258Y3VybCkvIG5vY2FzZQ0KICAgICAgICAkYmFzaHJjX3BlcnNpc3Qg
+ICAgPSAvZWNob1xzKy4qPj4/XHMqLipcLmJhc2hyYy8gbm9jYXNlDQogICAgICAgICRwcm9maWxl
+X3BlcnNpc3QgICA9IC9lY2hvXHMrLio+Pj9ccyouKlwucHJvZmlsZS8gbm9jYXNlDQogICAgICAg
+ICRpbml0X3BlcnNpc3QgICAgICA9IC9cL2V0Y1wvaW5pdFwuZFwvLioobmN8YmFzaHxyZXZlcnNl
+KS8gbm9jYXNlDQogICAgICAgICRsZF9wcmVsb2FkICAgICAgICA9IC9MRF9QUkVMT0FELipcLnNv
+LyBub2Nhc2UNCiAgICBjb25kaXRpb246DQogICAgICAgIGFueSBvZiB0aGVtDQp9DQoNCnJ1bGUg
+a2V5bG9nZ2VyX2luZGljYXRvcnMNCnsNCiAgICBtZXRhOg0KICAgICAgICBkZXNjcmlwdGlvbiA9
+ICJLZXlsb2dnZXIgZnVuY3Rpb25hbGl0eSBpbiBzY3JpcHRzIG9yIHNvdXJjZSBjb2RlIg0KICAg
+ICAgICBjYXRlZ29yeSA9ICJtYWx3YXJlIg0KICAgICAgICBzZXZlcml0eSA9ICJISUdIIg0KICAg
+ICAgICBjb25maWRlbmNlID0gIjAuNyINCiAgICBzdHJpbmdzOg0KICAgICAgICAkcHlucHV0ICAg
+ICAgICAgPSAvZnJvbVxzK3B5bnB1dFwua2V5Ym9hcmRccytpbXBvcnQvIG5vY2FzZQ0KICAgICAg
+ICAka2V5Ym9hcmRfaG9vayAgPSAva2V5Ym9hcmRcLihvbl9wcmVzc3xob29rfG9uX3JlbGVhc2Up
+LyBub2Nhc2UNCiAgICAgICAgJHhpbnB1dF90ZXN0ICAgID0gL3hpbnB1dFxzK3Rlc3QvIG5vY2Fz
+ZQ0KICAgICAgICAkbG9na2V5cyAgICAgICAgPSAvbG9na2V5c1xzKy0tc3RhcnQvIG5vY2FzZQ0K
+ICAgICAgICAka2V5YmRfZXZlbnQgICAgPSAvKEdldEFzeW5jS2V5U3RhdGV8U2V0V2luZG93c0hv
+b2tFeC4qV0hfS0VZQk9BUkQpLyBub2Nhc2UNCiAgICBjb25kaXRpb246DQogICAgICAgIGFueSBv
+ZiB0aGVtDQp9DQoNCnJ1bGUgcmFuc29td2FyZV9iZWhhdmlvcg0Kew0KICAgIG1ldGE6DQogICAg
+ICAgIGRlc2NyaXB0aW9uID0gIlJhbnNvbXdhcmUtbGlrZSBwYXR0ZXJucyAobWFzcyBlbmNyeXB0
+aW9uLCByYW5zb20gbm90ZXMpIg0KICAgICAgICBjYXRlZ29yeSA9ICJtYWx3YXJlIg0KICAgICAg
+ICBzZXZlcml0eSA9ICJDUklUSUNBTCINCiAgICAgICAgY29uZmlkZW5jZSA9ICIwLjgiDQogICAg
+c3RyaW5nczoNCiAgICAgICAgJHdhbGtfZW5jcnlwdCAgID0gL29zXC53YWxrXHMqXCguKlwuKGVu
+Y3J5cHR8Y2lwaGVyKS8NCiAgICAgICAgJHJhbnNvbV9ub3RlICAgID0gLyh5b3VyXHMrZmlsZXNc
+cysoaGF2ZVxzK2JlZW58YXJlKVxzK2VuY3J5cHRlZHxwYXlccysuKmJpdGNvaW58c2VuZFxzKy4q
+YnRjKS8gbm9jYXNlDQogICAgICAgICRleHRfcmVuYW1lICAgICA9IC9vc1wucmVuYW1lXHMqXCgu
+KlwrXHMqWyciXVwuKGxvY2tlZHxlbmNyeXB0ZWR8Y3J5cHR8ZW5jKVsnIl1ccypcKS8NCiAgICAg
+ICAgJG1hc3Nfb3ZlcndyaXRlID0gL29zXC53YWxrXHMqXCguKm9wZW5ccypcKC4qWydcIl13Ylsn
+XCJdXCkvDQogICAgY29uZGl0aW9uOg0KICAgICAgICBhbnkgb2YgdGhlbQ0KfQ0KDQpydWxlIGMy
+X2ZyYW1ld29ya19pbmRpY2F0b3JzDQp7DQogICAgbWV0YToNCiAgICAgICAgZGVzY3JpcHRpb24g
+PSAiQ29tbWFuZC1hbmQtY29udHJvbCBmcmFtZXdvcmsgaW5kaWNhdG9ycyAoQ29iYWx0IFN0cmlr
+ZSwgTWV0YXNwbG9pdCwgU2xpdmVyLCBldGMuKSINCiAgICAgICAgY2F0ZWdvcnkgPSAibWFsd2Fy
+ZSINCiAgICAgICAgc2V2ZXJpdHkgPSAiQ1JJVElDQUwiDQogICAgICAgIGNvbmZpZGVuY2UgPSAi
+MC44NSINCiAgICAgICAgcmVmZXJlbmNlID0gImh0dHBzOi8vZ2l0aHViLmNvbS9OZW8yM3gwL3Np
+Z25hdHVyZS1iYXNlIg0KICAgIHN0cmluZ3M6DQogICAgICAgICRjb2JhbHRfc3RyaWtlICA9ICJj
+b2JhbHRzdHJpa2UiIG5vY2FzZQ0KICAgICAgICAkbWV0ZXJwcmV0ZXIgICAgPSAibWV0ZXJwcmV0
+ZXIiIG5vY2FzZQ0KICAgICAgICAkbWV0YXNwbG9pdCAgICAgPSAvbWV0YXNwbG9pdC4qKHBheWxv
+YWR8ZXhwbG9pdHxzdGFnZXIpLyBub2Nhc2UNCiAgICAgICAgJGVtcGlyZSAgICAgICAgID0gL3Bv
+d2Vyc2hlbGwuKmVtcGlyZS8gbm9jYXNlDQogICAgICAgICRzbGl2ZXJfYzIgICAgICA9IC9zbGl2
+ZXIuKihpbXBsYW50fGJlYWNvbnxzZXNzaW9uKS8gbm9jYXNlDQogICAgICAgICRjb3ZlbmFudCAg
+ICAgICA9IC9Db3ZlbmFudC4qKEdydW50fExpc3RlbmVyKS8gbm9jYXNlDQogICAgICAgICRoYXZv
+Y19jMiAgICAgICA9IC9oYXZvYy4qKGRlbW9ufHRlYW1zZXJ2ZXIpLyBub2Nhc2UNCiAgICAgICAg
+JGJlYWNvbl9jb25maWcgID0gLyhCZWFjb25UeXBlfEMyU2VydmVyfFB1YmxpY0tleS4qd2F0ZXJt
+YXJrKS8gbm9jYXNlDQogICAgY29uZGl0aW9uOg0KICAgICAgICBhbnkgb2YgdGhlbQ0KfQ0KDQpy
+dWxlIGluZm9fc3RlYWxlcg0Kew0KICAgIG1ldGE6DQogICAgICAgIGRlc2NyaXB0aW9uID0gIklu
+Zm9ybWF0aW9uIHN0ZWFsZXIgcGF0dGVybnMgKGNyZWRlbnRpYWwgaGFydmVzdGluZywgYnJvd3Nl
+ciBkYXRhIHRoZWZ0KSINCiAgICAgICAgY2F0ZWdvcnkgPSAibWFsd2FyZSINCiAgICAgICAgc2V2
+ZXJpdHkgPSAiSElHSCINCiAgICAgICAgY29uZmlkZW5jZSA9ICIwLjc1Ig0KICAgICAgICByZWZl
+cmVuY2UgPSAiaHR0cHM6Ly9naXRodWIuY29tL05lbzIzeDAvc2lnbmF0dXJlLWJhc2UiDQogICAg
+c3RyaW5nczoNCiAgICAgICAgJGNocm9tZV9sb2dpbiAgICAgPSAvQ2hyb21lLipMb2dpblxzKkRh
+dGEvIG5vY2FzZQ0KICAgICAgICAkZmlyZWZveF9sb2dpbnMgICA9IC9sb2dpbnNcLmpzb24uKmZp
+cmVmb3gvIG5vY2FzZQ0KICAgICAgICAkYnJvd3Nlcl9jb29raWVzICA9IC9Db29raWVzLiooY2hy
+b21lfGZpcmVmb3h8ZWRnZXxvcGVyYSkvIG5vY2FzZQ0KICAgICAgICAkd2FsbGV0X3N0ZWFsICAg
+ICA9IC93YWxsZXRcLmRhdC8gbm9jYXNlDQogICAgICAgICRtaW1pa2F0eiAgICAgICAgID0gIm1p
+bWlrYXR6IiBub2Nhc2UNCiAgICAgICAgJGxhemFnbmUgICAgICAgICAgPSAibGF6YWduZSIgbm9j
+YXNlDQogICAgICAgICRjcmVkX2R1bXBfc2FtICAgID0gL3JlZ1xzK3NhdmVccysuKlxcc2FtLyBu
+b2Nhc2UNCiAgICAgICAgJG50ZHNfZHVtcCAgICAgICAgPSAvbnRkc1wuZGl0LyBub2Nhc2UNCiAg
+ICBjb25kaXRpb246DQogICAgICAgIGFueSBvZiB0aGVtDQp9DQo=