feat(research): scan the files a skill ships, not just the file it documents

.github/workflows/measure-unread-surface.yml

@@ -18,6 +18,11 @@ on:
         description: "Sampling seed, fixed so the number is reproducible"
         type: string
         default: "20260816"
+      mode:
+        description: "measure = count the surface; scan = fetch and scan those files"
+        type: choice
+        options: ["measure", "scan"]
+        default: "measure"
 
 permissions:
   contents: read
@@ -35,8 +40,8 @@ jobs:
         with:
           python-version: "3.13"
 
-      - name: Install httpx
-        run: pip install httpx
+      - name: Install malwar
+        run: pip install -e "." && pip install httpx
 
       - name: Fetch the current snapshot
         run: |
@@ -46,7 +51,8 @@ jobs:
             > data/registry-snapshots/latest.json
           echo "Snapshot: $(jq -r .created_at data/registry-snapshots/latest.json)"
 
-      - name: Measure
+      - name: Measure the surface
+        if: ${{ github.event.inputs.mode != 'scan' }}
         env:
           SAMPLE: ${{ github.event.inputs.sample }}
           SEED: ${{ github.event.inputs.seed }}
@@ -56,6 +62,17 @@ jobs:
             --sample "$SAMPLE" --seed "$SEED" \
             --json referenced-files.json
 
+      - name: Scan the referenced files
+        if: ${{ github.event.inputs.mode == 'scan' }}
+        env:
+          SAMPLE: ${{ github.event.inputs.sample }}
+          SEED: ${{ github.event.inputs.seed }}
+        run: |
+          python scripts/scan-referenced-files.py \
+            data/registry-snapshots/latest.json \
+            --sample "$SAMPLE" --seed "$SEED" \
+            --json referenced-files.json
+
       - name: Upload the path map
         if: always()
         uses: actions/upload-artifact@v4

scripts/extract-referenced-files.py

@@ -31,79 +31,14 @@
 import asyncio
 import json
 import random
-import re
 import sys
 from collections import Counter
 from pathlib import Path
 from typing import Any
 
-BASE_URL = "https://clawhub.ai/api/v1"
-
-# Interpreters and runners whose argument is a file the skill expects to run.
-_RUNNERS = r"(?:node|python3?|bun|deno|ts-node|tsx|bash|sh|zsh|ruby|perl|php|Rscript)"
-
-# Ways a SKILL.md names a file it ships. Each must capture the path in group 1.
-_PATTERNS: list[tuple[re.Pattern[str], str]] = [
-    # Executed by an interpreter: `node scripts/foo.js`, `python3 ./bin/x.py`.
-    # ${SKILL_DIR}/ and ./ prefixes are stripped by _clean.
-    (
-        re.compile(
-            rf"\b{_RUNNERS}\s+(?:\$\{{SKILL_DIR\}}/|\./)?"
-            r"([\w./-]+\.(?:js|mjs|cjs|ts|tsx|py|sh|bash|rb|pl|php|R))\b"
-        ),
-        "executed",
-    ),
-    # Explicit "Read `path`" instructions, which the agent will follow.
-    (
-        re.compile(r"\b(?:Read|read|Load|load|See|see)\s+`([\w./-]+\.[\w]+)`"),
-        "read-instruction",
-    ),
-    # Markdown link or bare mention of a repo-relative source/doc file.
-    (
-        re.compile(
-            r"[\(\[`\s](?:\./)?"
-            r"((?:scripts|bin|lib|src|tools|references|assets|specs)/[\w./-]+"
-            r"\.(?:js|mjs|cjs|ts|tsx|py|sh|bash|rb|pl|php|R|md|json|ya?ml|toml))"
-        ),
-        "referenced",
-    ),
-]
-
-# Paths that are not part of the skill: the file we already scan, and things
-# that belong to the user's project rather than the package.
-_SKIP = re.compile(
-    r"^(?:SKILL\.md|README\.md|package(?:-lock)?\.json|tsconfig\.json"
-    r"|\.env(?:\.example)?|node_modules/.*|\.\./.*)$",
-    re.IGNORECASE,
-)
-
-
-def _clean(path: str) -> str | None:
-    """Normalise a captured path, or return None if it is not skill-local."""
-    path = path.strip().strip("`'\"")
-    path = re.sub(r"^\$\{SKILL_DIR\}/", "", path)
-    path = re.sub(r"^\./", "", path)
-    if not path or path.startswith(("/", "~", "http")) or ".." in path:
-        return None
-    if _SKIP.match(path):
-        return None
-    # A bare filename with no directory and no extension we recognise is more
-    # likely prose than a shipped file.
-    if "." not in path:
-        return None
-    return path
-
-
-def extract(text: str) -> dict[str, str]:
-    """Return {path: how it was referenced} for one SKILL.md."""
-    found: dict[str, str] = {}
-    for pattern, kind in _PATTERNS:
-        for match in pattern.finditer(text):
-            cleaned = _clean(match.group(1))
-            if cleaned and cleaned not in found:
-                found[cleaned] = kind
-    return found
+from malwar.research.references import extract, is_executable
 
+BASE_URL = "https://clawhub.ai/api/v1"
 
 async def sample_registry(slugs: list[str]) -> dict[str, str]:
     """Fetch SKILL.md for each slug. Requires network reach to the registry."""
@@ -196,11 +131,7 @@ def main() -> int:
     for ext, n in exts.most_common(12):
         print(f"  {ext or '(none)':<10} {n:,}")
 
-    executable = sum(
-        1 for v in refs.values()
-        for p in v if Path(p).suffix.lower() in
-        {".js", ".mjs", ".cjs", ".ts", ".tsx", ".py", ".sh", ".bash", ".rb", ".pl", ".php"}
-    )
+    executable = sum(1 for v in refs.values() for p in v if is_executable(p))
     print()
     print(f"of those, executable code:              {executable:,}")
 

scripts/scan-referenced-files.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""Scan the files a skill ships, and report where they disagree with its docs.
+
+Step one measured the surface: 47% of a 290-skill sample name files no scan has
+ever read, 182 of them executable. This reads them.
+
+The output that matters is the **divergence set**: skills whose SKILL.md is
+clean but whose referenced code is not. That set is hidden behaviour by
+construction -- the documentation a reviewer or user would read says one thing,
+the code that actually runs says another -- and it is the only shape of finding
+worth publishing as malicious.
+
+Two things this deliberately does not do:
+
+* It does not convict. The rule engine is tuned for SKILL.md prose; run against
+  JavaScript and Python it has an unmeasured false-positive profile, and this
+  project has already published one inflated number by assuming otherwise.
+  Findings here are leads to read by hand, and the report says so.
+* It does not treat a 404 as absence of code. A referenced path that fails to
+  fetch is counted separately, because "the skill names a script we could not
+  retrieve" and "the skill ships nothing" are different facts.
+
+Usage:
+    scan-referenced-files.py <snapshot.json> --sample N [--json out.json]
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import random
+import sys
+from collections import Counter
+from pathlib import Path
+from typing import Any
+
+from malwar.research.references import extract, is_executable
+
+BASE_URL = "https://clawhub.ai/api/v1"
+
+# Paced to stay inside the registry's ~120 req/min limit.
+_DELAY = 0.55
+
+
+async def _get(client: Any, slug: str, path: str) -> tuple[int, str]:
+    resp = await client.get(
+        f"{BASE_URL}/skills/{slug}/file", params={"path": path}
+    )
+    return resp.status_code, resp.text
+
+
+async def scan_skill(client: Any, slug: str) -> dict[str, Any] | None:
+    """Fetch a skill's SKILL.md and everything it names, scanning each."""
+    from malwar.sdk import scan
+
+    try:
+        status, skill_md = await _get(client, slug, "SKILL.md")
+    except Exception as exc:
+        return {"slug": slug, "error": f"{type(exc).__name__}: {exc}"}
+    if status != 200:
+        return None
+    await asyncio.sleep(_DELAY)
+
+    doc = await scan(skill_md, file_name=f"{slug}/SKILL.md", use_llm=False, use_urls=False)
+    refs = extract(skill_md)
+
+    files: list[dict[str, Any]] = []
+    for path in refs:
+        try:
+            status, body = await _get(client, slug, path)
+        except Exception as exc:
+            files.append({"path": path, "status": "error", "detail": str(exc)[:120]})
+            await asyncio.sleep(_DELAY)
+            continue
+        await asyncio.sleep(_DELAY)
+        if status != 200:
+            # Named but not retrievable. Recorded, never counted as clean.
+            files.append({"path": path, "status": status})
+            continue
+        res = await scan(body, file_name=f"{slug}/{path}", use_llm=False, use_urls=False)
+        files.append({
+            "path": path,
+            "status": 200,
+            "kind": refs[path],
+            "executable": is_executable(path),
+            "verdict": res.verdict,
+            "risk": res.risk_score,
+            "rules": sorted({f.rule_id for f in res.findings if not f.suppressed}),
+            "bytes": len(body),
+        })
+
+    return {
+        "slug": slug,
+        "doc_verdict": doc.verdict,
+        "doc_risk": doc.risk_score,
+        "doc_rules": sorted({f.rule_id for f in doc.findings if not f.suppressed}),
+        "referenced": files,
+    }
+
+
+async def run(slugs: list[str]) -> list[dict[str, Any]]:
+    import httpx
+
+    out: list[dict[str, Any]] = []
+    async with httpx.AsyncClient(timeout=20.0, follow_redirects=True) as client:
+        for i, slug in enumerate(slugs, 1):
+            result = await scan_skill(client, slug)
+            if result:
+                out.append(result)
+            if i % 10 == 0:
+                print(f"  ...{i}/{len(slugs)} skills", flush=True)
+    return out
+
+
+def report(results: list[dict[str, Any]]) -> None:
+    fetched = [r for r in results if "referenced" in r]
+    with_refs = [r for r in fetched if r["referenced"]]
+    all_files = [f for r in with_refs for f in r["referenced"]]
+    ok = [f for f in all_files if f.get("status") == 200]
+    unreachable = [f for f in all_files if f.get("status") != 200]
+
+    print(f"\nskills scanned:              {len(fetched):,}")
+    print(f"  naming referenced files:   {len(with_refs):,}")
+    print(f"referenced paths tried:      {len(all_files):,}")
+    print(f"  fetched:                   {len(ok):,}")
+    print(f"  unreachable (named, 404):  {len(unreachable):,}")
+
+    if ok:
+        print("\nverdicts on referenced files:")
+        for v, n in Counter(f["verdict"] for f in ok).most_common():
+            print(f"  {v:<12} {n:,}")
+
+    # The divergence set: clean docs, flagged code.
+    divergent = [
+        (r, f)
+        for r in with_refs
+        if r["doc_verdict"] in ("CLEAN", "UNKNOWN")
+        for f in r["referenced"]
+        if f.get("status") == 200 and f.get("verdict") not in ("CLEAN", "UNKNOWN")
+    ]
+    print(f"\n{'=' * 62}")
+    print(f"DIVERGENCE  (SKILL.md clean, referenced file flagged): {len(divergent)}")
+    print(f"{'=' * 62}")
+    if not divergent:
+        print("  none in this sample.")
+        print("  A clean result here is a real result: it says the unread")
+        print("  surface is large and, in this sample, not hiding anything.")
+    for r, f in divergent[:40]:
+        rules = ", ".join(x.replace("MALWAR-", "") for x in f["rules"])
+        print(f"  {r['slug'][:34]:<34} {f['path'][:30]:<30} "
+              f"{f['verdict']:<11} risk={f['risk']:<4} [{rules}]")
+
+    print("\nThese are leads, not verdicts: the rule engine is calibrated for")
+    print("SKILL.md prose and its false-positive profile on code is unmeasured.")
+    print("Every one needs reading by hand before it is called anything.")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("snapshot", type=Path)
+    ap.add_argument("--sample", type=int, default=60)
+    ap.add_argument("--seed", type=int, default=20260816)
+    ap.add_argument("--json", type=Path)
+    args = ap.parse_args()
+
+    skills = json.loads(args.snapshot.read_text(encoding="utf-8")).get("skills", {})
+    # Reproducible sampling, not cryptography: the seed is published so anyone
+    # can redraw the same sample and check the result.
+    rng = random.Random(args.seed)  # noqa: S311
+    chosen = rng.sample(sorted(skills), min(args.sample, len(skills)))
+    print(f"sampling {len(chosen)} skills (seed {args.seed})\n")
+
+    results = asyncio.run(run(chosen))
+    report(results)
+
+    if args.json:
+        args.json.write_text(json.dumps(results, indent=2), encoding="utf-8")
+        print(f"\nwrote {args.json}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/malwar/research/__init__.py

@@ -0,0 +1,6 @@
+"""Research tooling that reaches beyond a single scanned file.
+
+The sweep scans one file per skill (SKILL.md). Everything here exists to look
+at what a skill actually ships alongside that file, which is where behaviour
+diverging from the documentation would live.
+"""