From bd1fa3eb6df7c45297b43f193e598f9fa7ec5a48 Mon Sep 17 00:00:00 2001 From: Kazim Ali Date: Fri, 13 Nov 2026 03:23:48 +1100 Subject: [PATCH] Add repo security and permission checks to github-seo Read-only security section in check_repo_seo.py (branch and Pages rules, secret scanning, push protection, Dependabot, private vulnerability reporting, Actions permissions), checklist reference, SKILL.md step, evals and changelog. Co-Authored-By: Claude Sonnet 5.5 --- CHANGELOG.md | 6 + skills/github-seo/SKILL.md | 18 ++- skills/github-seo/evals/evals.json | 15 ++ .../references/repo-security-checklist.md | 40 +++++ skills/github-seo/scripts/check_repo_seo.py | 138 +++++++++++++++++- 5 files changed, 207 insertions(+), 10 deletions(-) create mode 100644 skills/github-seo/references/repo-security-checklist.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2abf192..a21af6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes are listed here. The format follows [Keep a Changelog](https ## [Unreleased] +### Added +- Repo security and permission checks in `check_repo_seo.py` (read-only, GET only): default branch and Pages publishing branch rules (PR required, deletion and force-push blocked), secret scanning, push protection, Dependabot, private vulnerability reporting, Actions token permissions and fork PR approval. Checks that need admin print MANUAL. +- `--security-only` and `--skip-security` flags. +- `references/repo-security-checklist.md` with evidence labels and the exact `gh api` commands (checked 2026-11-13). +- SKILL.md Step 1b, and evals for security audit prompts. The existing `set up branch protection rules on main` should_not_trigger case stays valid: the skill audits and recommends, it never configures. + ## [0.2.0] - 2026-11-02 ### Added diff --git a/skills/github-seo/SKILL.md b/skills/github-seo/SKILL.md index 542046d..3acc2b0 100644 --- a/skills/github-seo/SKILL.md +++ b/skills/github-seo/SKILL.md @@ -42,6 +42,16 @@ Never skip this, and never let distribution excitement jump ahead of it. Details and the publish-or-not framework: `references/publish-safety.md`. +## Step 1b: repo security and permissions (report only) + +Public does not mean safe to push to. Check who can change the default branch and the Pages +publishing branch, and which security features are on. Run +`python3 scripts/check_repo_seo.py owner/repo --security-only`. It covers branch rules (PR +required, deletion and force-push blocked), the Pages branch, secret scanning, push protection, +Dependabot, private vulnerability reporting and Actions token permissions. Items that need admin +print MANUAL: say "could not check, needs admin" and ask the owner. Never change settings +yourself, only recommend. Commands and evidence labels: `references/repo-security-checklist.md`. + ## Step 2: audit what exists Run the read-only audit. It needs the GitHub CLI (`gh`) logged in. @@ -51,7 +61,8 @@ python3 scripts/check_repo_seo.py owner/repo ``` It reports pass, warn or fail for the description, topics, homepage, license, README shape, -and community files, and prints the one lever it cannot check (the social preview image). +and community files, then runs the security section from Step 1b (skip it with +`--skip-security`), and prints the one lever it cannot check (the social preview image). Fix failures first, then warnings. ## Step A: plain repo checklist @@ -104,5 +115,6 @@ build and label, leaked-history scrub) are in the install reference. ## Reporting back When you finish, give the owner three lists: what you changed, what only they can do (social -preview upload, pinning, account-bound registry logins), and which advice is verified, observed -or general. +preview upload, pinning, account-bound registry logins, repo security settings you recommended), +and which advice is verified, observed or general. List every security check that returned +MANUAL as unchecked, not as passed. diff --git a/skills/github-seo/evals/evals.json b/skills/github-seo/evals/evals.json index 574f8bd..7a0f732 100644 --- a/skills/github-seo/evals/evals.json +++ b/skills/github-seo/evals/evals.json @@ -11,6 +11,8 @@ { "type": "should_trigger", "prompt": "is it safe to make this private repo public" }, { "type": "should_trigger", "prompt": "submit my mcp server to smithery and glama" }, { "type": "should_trigger", "prompt": "audit the discoverability of github.com/acme/widget-cli" }, + { "type": "should_trigger", "prompt": "before we go public, check who can push to main and gh-pages on our repo" }, + { "type": "should_trigger", "prompt": "is secret scanning and push protection turned on for this public repo, and is the pages branch protected" }, { "type": "should_not_trigger", "prompt": "improve the SEO of my marketing website" }, { "type": "should_not_trigger", "prompt": "write a blog post about our product" }, { "type": "should_not_trigger", "prompt": "fix the failing unit test in this repo" }, @@ -19,6 +21,8 @@ { "type": "should_not_trigger", "prompt": "build an MCP server that wraps our REST API" }, { "type": "should_not_trigger", "prompt": "generate a changelog from the git log" }, { "type": "should_not_trigger", "prompt": "run keyword research for an ecommerce store" }, + { "type": "should_not_trigger", "prompt": "enable branch protection on main with required reviews using the API" }, + { "type": "should_not_trigger", "prompt": "write a GitHub Actions workflow that runs CodeQL" }, { "type": "quality", "prompt": "We want to open-source our internal CLI. What do we do before and after we flip it public?", @@ -60,6 +64,17 @@ "tells the owner that mcp-publisher login is interactive and theirs to run", "labels registry facts with a verification date rather than stating them from memory" ] + }, + { + "type": "quality", + "prompt": "Check the security settings of github.com/acme/widget-cli before we open-source it", + "expected_behavior": [ + "runs scripts/check_repo_seo.py --security-only rather than guessing at the repo state", + "checks the default branch and the Pages publishing branch for deletion, force-push and pull request rules", + "reports anything needing admin as could not check, needs admin, not as passed", + "recommends settings changes and the Settings path but does not change any repo setting itself", + "labels evidence as verified, observed or general" + ] } ] } diff --git a/skills/github-seo/references/repo-security-checklist.md b/skills/github-seo/references/repo-security-checklist.md new file mode 100644 index 0000000..945567e --- /dev/null +++ b/skills/github-seo/references/repo-security-checklist.md @@ -0,0 +1,40 @@ +# Repo security and permission checklist + +Run by `python3 scripts/check_repo_seo.py owner/repo --security-only`. The skill reports and +recommends. It never changes repo settings: the owner applies fixes in Settings. + +Labels: **verified** = primary doc checked 2026-11-13; **observed** = live `gh api` GET on +2026-11-13; **general** = common practice. Sources are under docs.github.com. + +## Who can see what + +- Admin token: every endpoint below is readable [observed]. +- Read-only token: `rules/branches/{branch}`, `rulesets` and `private-vulnerability-reporting` + return 200. The Actions permission endpoints return 403. `security_and_analysis` is absent from + the repo object, and `vulnerability-alerts`, `automated-security-fixes`, `secret-scanning/alerts` + and `pages` return a 404 that cannot be told apart from "off" [observed, 2026-11-13]. +- So a 403 or an ambiguous 404 is reported as MANUAL ("could not check, needs admin"), never + as a pass or a fail. + +## Checks + +| Check | Command (GET only) | Notes | +|---|---|---| +| Effective rules on a branch | `gh api repos/OWNER/REPO/rules/branches/BRANCH` | Look for `deletion`, `non_fast_forward`, `pull_request`. Includes org rulesets; excludes evaluate and disabled ones [verified: /en/rest/repos/rules]. | +| Rulesets list | `gh api repos/OWNER/REPO/rulesets` | `enforcement` must be `active` [observed]. | +| Legacy branch protection | `gh api repos/OWNER/REPO/branches/BRANCH/protection` | Admin only in practice. 404 "Branch not protected" does not mean unprotected when a ruleset exists [observed]. The doc does not define 404 [verified: /en/rest/branches/branch-protection]. | +| Pages enabled and source | `gh api repos/OWNER/REPO --jq .has_pages` then `gh api repos/OWNER/REPO/pages` | `source.branch` is the publishing branch (any branch, root or /docs) [verified]. The docs do not tell you to protect it, but anyone who can push there can change the live site [general]. Apply deletion and force-push blocking at least. A PR rule can break deploy bots, so it is not required for this branch. | +| Pages via Actions | `gh api repos/OWNER/REPO/pages --jq .build_type` | `workflow`: restrict the `github-pages` environment to the default branch [verified: Pages publishing source doc]. Not readable here, so MANUAL. | +| Secret scanning, push protection, Dependabot security updates | `gh api repos/OWNER/REPO --jq .security_and_analysis` | Free on public repos [verified: /en/code-security/getting-started/github-security-features]. Field only appears for admin [observed]. | +| Dependabot alerts | `gh api -i repos/OWNER/REPO/vulnerability-alerts` | 204 on, 404 "Vulnerability alerts are disabled." off, for admin [observed]. | +| Private vulnerability reporting | `gh api repos/OWNER/REPO/private-vulnerability-reporting` | `{"enabled": bool}`, readable without admin [observed]. Pair with a SECURITY.md. | +| Actions default token | `gh api repos/OWNER/REPO/actions/permissions/workflow` | Want `default_workflow_permissions: read` and `can_approve_pull_request_reviews: false`. Defaults are not documented [verified: /en/rest/actions/permissions]. | +| Fork PR approval | `gh api repos/OWNER/REPO/actions/permissions/fork-pr-contributor-approval` | `approval_policy`: `first_time_contributors_new_to_github` (weakest), `first_time_contributors`, `all_external_contributors` [verified]. | +| Code scanning | `gh api repos/OWNER/REPO/code-scanning/alerts` | 404 "no analysis found" means not set up [observed]. Not scored by the script. | +| CODEOWNERS | `gh api repos/OWNER/REPO/contents/.github/CODEOWNERS` | Optional for solo repos. 404 means absent [observed]. Not scored. | + +## Reporting + +- Failures first, then warnings, then MANUAL items the owner must check in Settings. +- Recommend, do not apply. Tell the owner the path (Settings, Rules, Rulesets) and the setting. +- Say plainly which checks ran with a read-only token. diff --git a/skills/github-seo/scripts/check_repo_seo.py b/skills/github-seo/scripts/check_repo_seo.py index e65178d..0235927 100644 --- a/skills/github-seo/scripts/check_repo_seo.py +++ b/skills/github-seo/scripts/check_repo_seo.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 -"""Read-only GitHub repo SEO audit. Usage: check_repo_seo.py [owner/repo] +"""Read-only GitHub repo SEO and security audit. +Usage: check_repo_seo.py [owner/repo] [--security-only | --skip-security] -Needs the GitHub CLI (`gh`) logged in. Makes no changes. Exits 1 if any check fails. +Needs the GitHub CLI (`gh`) logged in. Makes no changes (GET requests only). Exits 1 if any check fails. +Security checks that need admin rights print MANUAL instead of guessing. """ import base64 import json import re import subprocess import sys +from urllib.parse import quote results = [] @@ -23,15 +26,135 @@ def record(level, name, detail): results.append((level, name, detail)) -target = sys.argv[1:2] +def api(path): + """GET via gh. Returns (status, parsed JSON or None). Never raises, never prints credentials.""" + out = subprocess.run(["gh", "api", path], capture_output=True, text=True) + if out.returncode == 0: + try: + return 200, json.loads(out.stdout) if out.stdout.strip() else {} + except ValueError: + return 200, None + m = re.search(r"HTTP (\d{3})", out.stderr) + return (int(m.group(1)) if m else 0), None + + +def security_checks(slug, default_branch): + """Repo security and permission checks. Reports only; never changes settings.""" + need = "could not check, needs admin" + st, repo = api(f"repos/{slug}") + if st != 200 or not isinstance(repo, dict): + record("MANUAL", "security", f"could not read repo settings (HTTP {st or 'error'})") + return + default_branch = repo.get("default_branch") or default_branch + admin = bool((repo.get("permissions") or {}).get("admin")) + sa = repo.get("security_and_analysis") + record("PASS" if admin else "WARN", "token permission", "admin" if admin else "read or write only. Settings checks below are limited.") + + def branch_rules(branch): + st_r, rules = api(f"repos/{slug}/rules/branches/{quote(branch, safe='')}") + types = {r.get("type") for r in rules} if st_r == 200 and isinstance(rules, list) else set() + legacy = None + if admin: + st_p, prot = api(f"repos/{slug}/branches/{quote(branch, safe='')}/protection") + if st_p == 200 and isinstance(prot, dict): + legacy = prot + if prot.get("required_pull_request_reviews"): + types.add("pull_request") + if not (prot.get("allow_deletions") or {}).get("enabled"): + types.add("deletion") + if not (prot.get("allow_force_pushes") or {}).get("enabled"): + types.add("non_fast_forward") + return st_r, types, legacy + + def report_branch(label, branch, need_pr): + st_r, types, _ = branch_rules(branch) + if st_r != 200: + record("MANUAL", label, f"rules endpoint returned HTTP {st_r or 'error'} for {branch}") + return + wanted = ["deletion", "non_fast_forward"] + (["pull_request"] if need_pr else []) + missing = [t for t in wanted if t not in types] + names = {"deletion": "deletion blocked", "non_fast_forward": "force-push blocked", "pull_request": "PR required"} + if not missing: + record("PASS", label, f"{branch}: " + ", ".join(names[t] for t in wanted)) + elif not types and not admin: + record("MANUAL", label, f"{branch}: no active ruleset seen; legacy branch protection {need}") + else: + note = "" if admin else f" (legacy branch protection {need})" + record("FAIL" if not types else "WARN", label, f"{branch}: missing " + ", ".join(names[t] for t in missing) + note) + + report_branch("default branch rules", default_branch, True) + + if repo.get("has_pages"): + st_g, pg = api(f"repos/{slug}/pages") + src = (pg or {}).get("source") if st_g == 200 and isinstance(pg, dict) else None + if src and src.get("branch"): + if src["branch"] == default_branch: + record("PASS", "pages branch rules", f"publishes from {default_branch}, covered above") + else: + report_branch("pages branch rules", src["branch"], False) + elif st_g == 200 and isinstance(pg, dict): + record("MANUAL", "pages branch rules", f"build_type {pg.get('build_type')}: check the github-pages environment deployment rule in Settings > Environments") + else: + record("MANUAL", "pages branch rules", f"Pages is on but the source could not be read (HTTP {st_g or 'error'})") + + if isinstance(sa, dict): + for key, label in (("secret_scanning", "secret scanning"), ("secret_scanning_push_protection", "push protection"), ("dependabot_security_updates", "dependabot security updates")): + status = (sa.get(key) or {}).get("status") + record("PASS" if status == "enabled" else "WARN", label, status or "not reported") + else: + for label in ("secret scanning", "push protection", "dependabot security updates"): + record("MANUAL", label, need) + if admin: + st_v, _ = api(f"repos/{slug}/vulnerability-alerts") + record("PASS" if st_v == 200 else "WARN", "dependabot alerts", "enabled" if st_v == 200 else "disabled (HTTP 404)" if st_v == 404 else f"HTTP {st_v or 'error'}") + else: + record("MANUAL", "dependabot alerts", need) + + st_p, pvr = api(f"repos/{slug}/private-vulnerability-reporting") + if st_p == 200 and isinstance(pvr, dict): + record("PASS" if pvr.get("enabled") else "WARN", "private vulnerability reporting", "enabled" if pvr.get("enabled") else "disabled. SECURITY.md should say where to report.") + else: + record("MANUAL", "private vulnerability reporting", f"HTTP {st_p or 'error'}, {need}") + + st_w, wf = api(f"repos/{slug}/actions/permissions/workflow") + if st_w == 200 and isinstance(wf, dict): + record("PASS" if wf.get("default_workflow_permissions") == "read" else "WARN", "actions default token", f"default_workflow_permissions={wf.get('default_workflow_permissions')}") + record("PASS" if not wf.get("can_approve_pull_request_reviews") else "WARN", "actions approve PRs", f"can_approve_pull_request_reviews={wf.get('can_approve_pull_request_reviews')}") + else: + record("MANUAL", "actions workflow permissions", f"HTTP {st_w or 'error'}, {need}") + st_f, fp = api(f"repos/{slug}/actions/permissions/fork-pr-contributor-approval") + if st_f == 200 and isinstance(fp, dict): + pol = fp.get("approval_policy") + record("PASS" if pol in ("first_time_contributors", "all_external_contributors") else "WARN", "fork PR approval", str(pol)) + else: + record("MANUAL", "fork PR approval", f"HTTP {st_f or 'error'}, {need}") + + +def finish(slug, meta): + width = max(len(n) for _, n, _ in results) + for level, n, d in results: + print(f"{level:<6} {n:<{width}} {d}") + return slug, meta + + +flags = [a for a in sys.argv[1:] if a.startswith("--")] +target = [a for a in sys.argv[1:] if not a.startswith("--")][:1] +if "--security-only" in flags and "--skip-security" in flags: + sys.exit("use only one of --security-only and --skip-security") meta = json.loads( gh( "repo", "view", *target, "--json", - "nameWithOwner,name,description,homepageUrl,licenseInfo,repositoryTopics,isPrivate", + "nameWithOwner,name,description,homepageUrl,licenseInfo,repositoryTopics,isPrivate,defaultBranchRef", ) ) slug = meta["nameWithOwner"] name = meta["name"] +default_branch = (meta.get("defaultBranchRef") or {}).get("name") or "main" + +if "--security-only" in flags: + security_checks(slug, default_branch) + finish(slug, meta) + sys.exit(1 if any(l == "FAIL" for l, _, _ in results) else 0) # Description desc = (meta["description"] or "").strip() @@ -136,9 +259,10 @@ def record(level, name, detail): elif any(p.get("registryType") == "npm" for p in pkgs): record("PASS", "npm mcpName", "check package.json mcpName equals server.json name (not read by this script)") -width = max(len(n) for _, n, _ in results) -for level, n, d in results: - print(f"{level:<5} {n:<{width}} {d}") +if "--skip-security" not in flags: + security_checks(slug, default_branch) + +finish(slug, meta) print(f"\nMANUAL social preview: cannot be read or set by CLI. Settings > General > Social preview (1280x640) for {slug}.") if meta["isPrivate"]: print("NOTE repo is private, so it is invisible to search until it is made public.")