From 376dcac8503523a99f60a5eddb34549e3b1052ee Mon Sep 17 00:00:00 2001 From: Mike Odnis Date: Tue, 4 Aug 2026 07:23:47 -0400 Subject: [PATCH 1/3] feat(standards): add a scheduled org-wide conformance sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit repo-standards catches drift on the PR that introduces it. It cannot catch a repo that already drifted and simply is not being touched — which is exactly how four repos ended up with a security scan that had never run once. Nobody had opened a PR against them since the day it broke. Runs the same checks against every non-archived repo weekly and reports a job summary. Read-only. Two things learned by actually running it: * A repo the token cannot read must be reported as UNREADABLE, never as clean. Silence is not conformance — that is the same failure mode the checks exist to catch, and it would be self-inflicted. * `head -1` and `grep -m1` in a pipe close it early, the upstream writer takes SIGPIPE, and under `set -o pipefail` the whole sweep aborts with exit 141 having assessed nothing. First run did precisely that. Sliced with parameter expansion instead, and no-match greps now tolerate their exit 1. Also fixes a false positive in the merged repo-standards check: it matched `zima` anywhere in a runs-on line, including the dynamic USE_SELF_HOSTED toggle. That form embeds the label in an expression actionlint cannot evaluate, so it never errors there — landing was being flagged for a problem it does not have. Both now match only the literal array form. Verified against the live org: 21 repos scanned, and it found vcpkg — a fifth repo whose security workflow has been startup_failure since July, which the manual pass missed entirely because it only sampled known repos. --- .github/workflows/org-conformance-sweep.yml | 189 ++++++++++++++++++++ .github/workflows/repo-standards.yml | 6 +- 2 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/org-conformance-sweep.yml diff --git a/.github/workflows/org-conformance-sweep.yml b/.github/workflows/org-conformance-sweep.yml new file mode 100644 index 0000000..e73efab --- /dev/null +++ b/.github/workflows/org-conformance-sweep.yml @@ -0,0 +1,189 @@ +# Copyright 2026 ResQ Software +# SPDX-License-Identifier: Apache-2.0 +# +# Org-wide CI-configuration sweep. +# +# repo-standards.yml catches drift on the PR that introduces it. It cannot +# catch a repo that already drifted and simply is not being touched — which is +# how the 2026-08 triage found four repos whose security scan had never run +# once. Nobody had opened a PR against them since the day it broke. +# +# This closes that hole: the same checks, applied to every non-archived repo in +# the org on a schedule, reported as a job summary. +# +# Read-only — it reads repo contents through the API and writes nothing back. +# +# TOKEN: needs read access to sibling repos, which GITHUB_TOKEN does not have. +# Set an `ORG_READ_TOKEN` secret (fine-grained, org-wide, Contents: Read). +# Without it the sweep reports every repo as UNREADABLE rather than clean — +# silence must never be mistaken for conformance. + +name: org-conformance-sweep + +on: + schedule: + - cron: "0 7 * * 1" # Mondays 07:00 UTC + workflow_dispatch: + inputs: + fail-on-findings: + description: "Exit non-zero if any repo has findings. Default: report only." + type: boolean + required: false + default: false + +permissions: + contents: read + +jobs: + sweep: + name: org-conformance-sweep + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Harden Runner + uses: step-security/harden-runner@f808768d1510423e83855289c910610ca9b43176 # v2 + with: + egress-policy: audit + + - name: Sweep org repositories + env: + GH_TOKEN: ${{ secrets.ORG_READ_TOKEN || github.token }} + FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} + ORG: ${{ github.repository_owner }} + run: | + set -euo pipefail + + # Print a repo file's contents; empty if absent OR unreadable. The + # caller must have already established readability via ls_workflows, + # so an empty result here means "absent". + fetch() { + gh api "/repos/$ORG/$1/contents/$2" --jq '.content' 2>/dev/null \ + | base64 -d 2>/dev/null || true + } + + # List a repo's workflow filenames. Exits non-zero when the directory + # cannot be read at all, which is how access failures are told apart + # from a repo that genuinely has no workflows. + ls_workflows() { + gh api "/repos/$ORG/$1/contents/.github/workflows" \ + --jq '[.[].name] | join(" ")' 2>/dev/null + } + + repos=$(gh repo list "$ORG" --limit 200 --no-archived \ + --json name --jq '.[].name' | sort) + + total=0 affected=0 unreadable=0 + { + echo "## Org CI-configuration sweep" + echo + echo "| repo | finding |" + echo "| --- | --- |" + } >> "$GITHUB_STEP_SUMMARY" + + for r in $repos; do + total=$((total + 1)) + findings="" + + # Establish readability first. A repo we cannot read is reported as + # such — never counted as clean. + if ! wf=$(ls_workflows "$r"); then + unreadable=$((unreadable + 1)) + echo "| \`$r\` | **UNREADABLE** — token lacks access; not assessed |" >> "$GITHUB_STEP_SUMMARY" + echo "::warning title=org-conformance-sweep::$r unreadable — not assessed" + continue + fi + [ -n "$wf" ] || continue + + # 1. security.yml calling the org scan must grant `actions: read`, + # else the run is rejected at creation and produces no logs. + case " $wf " in + *" security.yml "*) + sec=$(fetch "$r" ".github/workflows/security.yml") + if printf '%s' "$sec" | grep -q 'security-scan\.yml' && + ! printf '%s' "$sec" | grep -qE '^[[:space:]]+actions:[[:space:]]*read'; then + findings="${findings}security.yml missing \`actions: read\` (startup_failure); " + fi + ;; + esac + + locks="" + for f in $wf; do + case "$f" in *.lock.yml) locks="$locks $f" ;; esac + done + + if [ -n "$locks" ]; then + # 2. The Dependabot ignore must carry the trailing wildcard. + dep=$(fetch "$r" ".github/dependabot.yml") + if [ -n "$dep" ] && ! printf '%s' "$dep" | grep -q 'github/gh-aw-actions\*'; then + findings="${findings}dependabot.yml missing \`github/gh-aw-actions*\` ignore; " + fi + + # 3. Lock files whose compiler disagrees with the pinned setup. + for f in $locks; do + body=$(fetch "$r" ".github/workflows/$f") + [ -n "$body" ] || continue + # No `head -1` / `grep -m1` in a pipe here: they close the pipe + # early, the upstream writer takes SIGPIPE, and under + # `set -o pipefail` that aborts the whole sweep. Slice the first + # line with parameter expansion instead, and tolerate no-match + # greps, which exit 1 and would otherwise trip `set -e`. + first=${body%%$'\n'*} + cv=$(printf '%s' "$first" \ + | grep -oE '"compiler_version":"[^"]*"' | cut -d'"' -f4 || true) + bv=$(printf '%s' "$body" \ + | grep -oE 'gh-aw-actions/setup@[a-f0-9]+ # v[0-9.]+' \ + | sed -n '1s/.*# //p' || true) + if [ -n "$cv" ] && [ -n "$bv" ] && [ "$cv" != "$bv" ]; then + findings="${findings}$f drift ($cv vs $bv); " + fi + done + fi + + # 4. `zima` used as a LITERAL runs-on label without actionlint + # declaring it. Match only the LITERAL array form: the + # dynamic USE_SELF_HOSTED toggle in landing/resQ ci.yml embeds + # the label inside an expression actionlint cannot evaluate, so + # it never errors on those. Matching them would report repos + # that are actually fine. + uses_zima=no + for f in $wf; do + case "$f" in *.lock.yml) continue ;; esac + wfbody=$(fetch "$r" ".github/workflows/$f") + if printf '%s' "$wfbody" \ + | grep -qE 'runs-on:[[:space:]]*\[[^]]*\bzima\b'; then + uses_zima=yes + break + fi + done + if [ "$uses_zima" = yes ]; then + al=$(fetch "$r" ".github/actionlint.yaml") + [ -n "$al" ] || al=$(fetch "$r" ".github/actionlint.yml") + if ! printf '%s' "$al" | grep -qE '^[[:space:]]*-[[:space:]]*zima[[:space:]]*$'; then + findings="${findings}uses \`zima\` without actionlint declaring it; " + fi + fi + + if [ -n "$findings" ]; then + affected=$((affected + 1)) + echo "| \`$r\` | ${findings%; } |" >> "$GITHUB_STEP_SUMMARY" + echo "::warning title=org-conformance-sweep::$r — ${findings%; }" + fi + done + + if [ "$affected" -eq 0 ] && [ "$unreadable" -eq 0 ]; then + echo "| _none_ | all $total repos clean |" >> "$GITHUB_STEP_SUMMARY" + fi + { + echo + echo "Scanned **$total** repos — **$affected** with findings, **$unreadable** unreadable." + } >> "$GITHUB_STEP_SUMMARY" + + echo "sweep: $affected/$total with findings, $unreadable unreadable" + + # Unreadable repos are a failure of the sweep itself, not a clean + # result, so they count toward the gate. + if [ "${FAIL_ON_FINDINGS:-false}" = "true" ] && + { [ "$affected" -gt 0 ] || [ "$unreadable" -gt 0 ]; }; then + echo "::error title=org-conformance-sweep::$affected with findings, $unreadable unreadable." + exit 1 + fi diff --git a/.github/workflows/repo-standards.yml b/.github/workflows/repo-standards.yml index f083d59..ec0e31f 100644 --- a/.github/workflows/repo-standards.yml +++ b/.github/workflows/repo-standards.yml @@ -199,7 +199,11 @@ jobs: # `label "zima" is unknown` the moment it is switched on. # Existence is not enough — an actionlint.yaml that omits the label # still fails. Require the label to actually be declared. - if grep -rqE 'runs-on:.*\bzima\b' .github/workflows/ 2>/dev/null; then + # Match only the LITERAL array form of runs-on. The dynamic + # USE_SELF_HOSTED toggle embeds the label inside an expression + # actionlint cannot evaluate, so it never errors on those — and + # matching them would warn repos that are actually fine. + if grep -rqE 'runs-on:[[:space:]]*\[[^]]*\bzima\b' .github/workflows/ 2>/dev/null; then al="" for f in .github/actionlint.yaml .github/actionlint.yml; do if [ -f "$f" ]; then al="$f"; break; fi From fb1ac65de6519104796c968031f6ea1df7408c8d Mon Sep 17 00:00:00 2001 From: Mike Odnis Date: Tue, 4 Aug 2026 08:00:35 -0400 Subject: [PATCH 2/3] feat(standards): let the sweep skip known-inaccessible repos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `scripts` is intentionally out of reach, so the sweep reported it UNREADABLE on every run. A weekly report that cries wolf is one people stop reading — which is the same failure mode these checks exist to prevent, just self inflicted. Adds a `skip-repos` input defaulting to `scripts`. Scheduled runs pass no inputs, so the default is applied in `env:` too. Forwarded through env rather than inlined into the shell, matching the pattern the other workflows use. Verified live: 20 repos scanned, scripts skipped, 0 unreadable, 2 findings — resQ (pending resQ#722) and vcpkg (pending vcpkg#34). --- .github/workflows/org-conformance-sweep.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/org-conformance-sweep.yml b/.github/workflows/org-conformance-sweep.yml index e73efab..deb34af 100644 --- a/.github/workflows/org-conformance-sweep.yml +++ b/.github/workflows/org-conformance-sweep.yml @@ -30,6 +30,11 @@ on: type: boolean required: false default: false + skip-repos: + description: "Space-separated repos to skip entirely (known-inaccessible or out of scope)." + type: string + required: false + default: "scripts" permissions: contents: read @@ -49,6 +54,10 @@ jobs: env: GH_TOKEN: ${{ secrets.ORG_READ_TOKEN || github.token }} FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} + # Scheduled runs pass no inputs, so default here too — otherwise the + # weekly report would flag known-inaccessible repos every Monday, and + # a report that cries wolf is one people stop reading. + SKIP_REPOS: ${{ inputs.skip-repos || 'scripts' }} ORG: ${{ github.repository_owner }} run: | set -euo pipefail @@ -81,6 +90,15 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" for r in $repos; do + skip=no + for sk in ${SKIP_REPOS:-}; do + [ "$r" = "$sk" ] && { skip=yes; break; } + done + if [ "$skip" = yes ]; then + echo "sweep: skipping $r (skip-repos)" + continue + fi + total=$((total + 1)) findings="" From a38d034da21ca41f7b642cba0d068baaac63e909 Mon Sep 17 00:00:00 2001 From: Mike Odnis Date: Tue, 4 Aug 2026 08:05:52 -0400 Subject: [PATCH 3/3] fix(standards): four correctness holes in the sweep, all false-assurance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every one of these could report a repo as fine when it is not — the failure mode this sweep exists to prevent. Readability was tested by listing .github/workflows, but a repo without that directory returns 404, which is indistinguishable from an access failure by exit status alone. `scripts` was being reported UNREADABLE while being perfectly readable — it simply has no workflows. Readability is now established against the repo object, and a missing directory means "no workflows". That also removes the need for the skip-repos default added a commit ago: scripts now scans clean on its own. The input stays as an escape hatch but defaults to empty — a repo suppressed there is a repo nobody is checking. The actions: read check was the loose file-wide grep, i.e. the exact bug caught in repo-standards during #42 review and fixed there. I had copied the pre-fix version into the sweep. It now uses the same caller-scoped awk, so a workflow whose caller job overrides permissions without actions: read is caught. A failed or partial `gh repo list` silently produced a short sweep that read as all-clean. It now errors out, including when enumeration returns fewer than two repos, which is the signature of GITHUB_TOKEN being used without ORG_READ_TOKEN. Limit raised 200 -> 1000. Verified live with and without skip-repos: 21 repos scanned, 0 unreadable, 2 findings (resQ pending resQ#722, vcpkg pending vcpkg#34). --- .github/workflows/org-conformance-sweep.yml | 74 ++++++++++++++++----- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/.github/workflows/org-conformance-sweep.yml b/.github/workflows/org-conformance-sweep.yml index deb34af..5cfc20c 100644 --- a/.github/workflows/org-conformance-sweep.yml +++ b/.github/workflows/org-conformance-sweep.yml @@ -31,10 +31,10 @@ on: required: false default: false skip-repos: - description: "Space-separated repos to skip entirely (known-inaccessible or out of scope)." + description: "Space-separated repos to skip entirely. Normally empty — an escape hatch, not a way to hide findings." type: string required: false - default: "scripts" + default: "" permissions: contents: read @@ -54,10 +54,10 @@ jobs: env: GH_TOKEN: ${{ secrets.ORG_READ_TOKEN || github.token }} FAIL_ON_FINDINGS: ${{ inputs.fail-on-findings }} - # Scheduled runs pass no inputs, so default here too — otherwise the - # weekly report would flag known-inaccessible repos every Monday, and - # a report that cries wolf is one people stop reading. - SKIP_REPOS: ${{ inputs.skip-repos || 'scripts' }} + # Escape hatch only. Deliberately empty: a repo suppressed here is a + # repo nobody is checking, so a finding should be fixed rather than + # skipped. + SKIP_REPOS: ${{ inputs.skip-repos }} ORG: ${{ github.repository_owner }} run: | set -euo pipefail @@ -70,16 +70,59 @@ jobs: | base64 -d 2>/dev/null || true } - # List a repo's workflow filenames. Exits non-zero when the directory - # cannot be read at all, which is how access failures are told apart - # from a repo that genuinely has no workflows. + # Readability is established against the repo object itself, not the + # workflows directory: a repo with no .github/workflows returns 404, + # which is indistinguishable from an access failure if you only look + # at the exit status. Conflating them would report a perfectly + # readable repo as UNREADABLE. + repo_readable() { + gh api "/repos/$ORG/$1" --jq '.name' >/dev/null 2>&1 + } + + # Workflow filenames, or empty when the directory does not exist. ls_workflows() { gh api "/repos/$ORG/$1/contents/.github/workflows" \ - --jq '[.[].name] | join(" ")' 2>/dev/null + --jq '[.[].name] | join(" ")' 2>/dev/null || true + } + + # Effective `actions: read` for the job that calls the reusable scan. + # Job-level permissions REPLACE the top-level block, so a file-wide + # grep would pass a workflow whose caller job has its own narrower + # block. Mirrors the check in repo-standards.yml. + caller_grants_actions_read() { + awk ' + /^permissions:[[:space:]]*$/ { intop=1; next } + /^[^[:space:]]/ { intop=0 } + intop && /^[[:space:]]+actions:[[:space:]]*read/ { topok=1 } + /^[[:space:]][[:space:]][A-Za-z0-9_-]+:[[:space:]]*$/ { + job=$1; sub(":","",job); injobperm=0 + } + job && /^[[:space:]]{4}permissions:[[:space:]]*$/ { + injobperm=1; hasown[job]=1; next + } + injobperm && /^[[:space:]]{6}actions:[[:space:]]*read/ { ok[job]=1 } + injobperm && /^[[:space:]]{4}[^[:space:]]/ { injobperm=0 } + /security-scan\.yml@/ { caller=job } + END { + if (caller == "") exit 0 + if (hasown[caller]) exit (ok[caller] ? 0 : 1) + exit (topok ? 0 : 1) + } + ' } - repos=$(gh repo list "$ORG" --limit 200 --no-archived \ - --json name --jq '.[].name' | sort) + # A partial or failed enumeration must not read as "all clean" — that + # is the exact false-assurance mode this sweep exists to prevent. + if ! repos=$(gh repo list "$ORG" --limit 1000 --no-archived \ + --json name --jq '.[].name' | sort); then + echo "::error title=org-conformance-sweep::cannot enumerate $ORG — set ORG_READ_TOKEN (org-wide, Contents: Read)." + exit 1 + fi + repo_count=$(printf '%s\n' "$repos" | grep -c . || true) + if [ "${repo_count:-0}" -lt 2 ]; then + echo "::error title=org-conformance-sweep::enumerated only ${repo_count:-0} repo(s) — GITHUB_TOKEN cannot list the org. Set ORG_READ_TOKEN." + exit 1 + fi total=0 affected=0 unreadable=0 { @@ -104,12 +147,13 @@ jobs: # Establish readability first. A repo we cannot read is reported as # such — never counted as clean. - if ! wf=$(ls_workflows "$r"); then + if ! repo_readable "$r"; then unreadable=$((unreadable + 1)) echo "| \`$r\` | **UNREADABLE** — token lacks access; not assessed |" >> "$GITHUB_STEP_SUMMARY" echo "::warning title=org-conformance-sweep::$r unreadable — not assessed" continue fi + wf=$(ls_workflows "$r") [ -n "$wf" ] || continue # 1. security.yml calling the org scan must grant `actions: read`, @@ -118,8 +162,8 @@ jobs: *" security.yml "*) sec=$(fetch "$r" ".github/workflows/security.yml") if printf '%s' "$sec" | grep -q 'security-scan\.yml' && - ! printf '%s' "$sec" | grep -qE '^[[:space:]]+actions:[[:space:]]*read'; then - findings="${findings}security.yml missing \`actions: read\` (startup_failure); " + ! printf '%s' "$sec" | caller_grants_actions_read; then + findings="${findings}security.yml caller job lacks effective \`actions: read\` (startup_failure); " fi ;; esac