From 59aee61c06b00a791218ed3cc96505ed6fb2e6cc Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 11 Sep 2026 22:42:01 +0000 Subject: [PATCH] research: name every flagged referenced file, not just divergent ones The first step-2 scan (60 skills, seed 20260816) returned a divergence set of zero with three referenced files flagged SUSPICIOUS. Both facts matter, but the report only printed the first, so the three were invisible: which skills, which paths, which rules, all unrecoverable without re-running. A file whose SKILL.md already flagged is not concealment -- documentation and code agreeing is the honest case, and it is precisely why those three are outside the divergence set. That is a reason to label them, not to drop them. A flag you cannot see is a flag you cannot check. Also lists the named-but-unreachable paths by skill. Thirteen of 120 paths 404'd; they are already excluded from the clean count, but a reader has no way to tell which skills carry that unknown. Raises the job timeout to 120 minutes so a 300-skill scan (~900 registry fetches at 0.55s, plus rule-engine time per file) fits. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01DNoTXU8k3pfSBzR7aJubqL --- .github/workflows/measure-unread-surface.yml | 4 ++- scripts/scan-referenced-files.py | 31 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/.github/workflows/measure-unread-surface.yml b/.github/workflows/measure-unread-surface.yml index b9f5f67..5be29f0 100644 --- a/.github/workflows/measure-unread-surface.yml +++ b/.github/workflows/measure-unread-surface.yml @@ -31,7 +31,9 @@ jobs: measure: name: Sample SKILL.md and extract referenced paths runs-on: ubuntu-latest - timeout-minutes: 45 + # A scan of 300 skills is ~900 registry fetches at 0.55s plus rule-engine + # time on every file, so it needs materially longer than a measure run. + timeout-minutes: 120 steps: - uses: actions/checkout@v4 diff --git a/scripts/scan-referenced-files.py b/scripts/scan-referenced-files.py index 1e066ba..85feb83 100644 --- a/scripts/scan-referenced-files.py +++ b/scripts/scan-referenced-files.py @@ -151,6 +151,37 @@ def report(results: list[dict[str, Any]]) -> None: print(f" {r['slug'][:34]:<34} {f['path'][:30]:<30} " f"{f['verdict']:<11} risk={f['risk']:<4} [{rules}]") + # Every flagged file, divergent or not. A file whose docs already flagged + # is not concealment -- documentation and code agreeing is the honest case + # -- but a report that prints only the divergence set makes those invisible, + # and a flag you cannot see is a flag you cannot check. + flagged = [ + (r, f) + for r in with_refs + for f in r["referenced"] + if f.get("status") == 200 and f.get("verdict") not in ("CLEAN", "UNKNOWN") + ] + if flagged: + print(f"\nall flagged referenced files ({len(flagged)}), with their doc verdict:") + for r, f in flagged[:60]: + rules = ", ".join(x.replace("MALWAR-", "") for x in f["rules"]) + print(f" {r['slug'][:30]:<30} {f['path'][:28]:<28} " + f"doc={r['doc_verdict']:<11} file={f['verdict']:<11} " + f"risk={f['risk']:<4} [{rules}]") + + # Named but not retrievable. Not evidence of anything on its own; recorded + # because "we could not read it" and "there was nothing to read" are + # different facts and only one of them supports a clean claim. + if unreachable: + print(f"\nnamed but unreachable ({len(unreachable)}) -- unknown, not clean:") + by_slug: Counter[str] = Counter() + for r in with_refs: + for f in r["referenced"]: + if f.get("status") != 200: + by_slug[r["slug"]] += 1 + for slug, n in by_slug.most_common(20): + print(f" {slug[:44]:<44} {n}") + print("\nThese are leads, not verdicts: the rule engine is calibrated for") print("SKILL.md prose and its false-positive profile on code is unmeasured.") print("Every one needs reading by hand before it is called anything.")