diff --git a/.github/scripts/refresh-only-head.sh b/.github/scripts/refresh-only-head.sh new file mode 100755 index 0000000000..b50335e651 --- /dev/null +++ b/.github/scripts/refresh-only-head.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +# refresh-only-head.sh - decide whether a pull_request run is on a +# refresh-only head, so the heavy jobs (Docker multi-arch build, FOSSA, +# Mutation) can skip it. See src/docs/ci-refresh-only-heads.md. +# +# Workflow form of the proof in src/pkg/github/pr_decision_base_refresh.go +# (HeadIsBaseRefreshOf). A head is refresh-only when ALL of these hold: +# 1. the run is a same-repository pull_request `synchronize`; +# 2. every hop walked back from the new head (at most MAX_HOPS, the first +# one being the event's previous head) is a refresh: a two-parent merge +# commit whose signature GitHub verified, authored by the hive App bot +# and committed by web-flow (or the bot itself), whose other parent is a +# commit on the base branch and the merge-base of the hop with the base, +# and whose three-dot diff against the base is identical, blob for blob, +# to the parent head's; +# 3. the walk reaches a head on which this same workflow concluded +# `success` (a run cancelled by the refresh is not a pass, so the walk +# continues to that head's own parent only while it, too, is a refresh); +# 4. the cumulative base delta brought in since that passing head passes +# DELTA_POLICY: +# docs-only (docker.yml): EVERY changed path matches DOCS_ONLY below +# (CHANGELOG.md, changelog.d/, any *.md, docs/, src/docs/); any +# other path, a Makefile or a .go file included, means run; +# deny (default; FOSSA, Mutation): NO changed path matches SENSITIVE +# below (dependency manifests and lock files anywhere, .fossa.yml, +# Dockerfiles, src/deploy/, CI definitions and this script). +# +# Fail safe: every lookup failure, unexpected shape or doubt answers false, +# and false means "run everything". The script never fails its step. +# +# Inputs (env): EVENT_NAME, EVENT_ACTION, BEFORE, HEAD_SHA, BASE_REF, +# HEAD_REPO, REPO, WORKFLOW_PATH, GH_TOKEN, optional DELTA_POLICY, +# APP_BOT_LOGIN and GITHUB_SERVER_URL. Outputs: refresh_only=true|false and reason= on +# $GITHUB_OUTPUT, one summary line on $GITHUB_STEP_SUMMARY. +set -uo pipefail + +# The App whose update-branch merges count as base refreshes. +DEFAULT_APP_BOT_LOGIN='hivecommons-hive[bot]' +# GitHub's committer login on commits it creates and signs itself. +WEB_FLOW_LOGIN='web-flow' +# Parent count of a merge commit. +MERGE_PARENTS=2 +# Most consecutive refreshes walked back to find a passing head (same bound +# as baseRefreshMaxHops in pr_decision_base_refresh.go). +MAX_HOPS=5 +# Workflow runs listed per head SHA; one head never has more. +RUNS_PER_PAGE=100 +# A full lowercase commit SHA. +SHA_RE='^[0-9a-f]{40}$' +ZERO_SHA='0000000000000000000000000000000000000000' +# Base-delta paths that always force a full run (extended regex, matched +# against each changed path). +SENSITIVE='(^|/)(go\.mod|go\.sum|go\.work|go\.work\.sum|package[^/]*\.json|npm-shrinkwrap\.json|yarn\.lock|pnpm-lock\.yaml|requirements[^/]*|pyproject\.toml|poetry\.lock|Cargo\.[^/]*|\.fossa\.ya?ml|\.dockerignore)$|(^|/)[^/]*[Dd]ockerfile[^/]*$|^src/deploy/|^\.github/workflows/|^\.github/actions/|^\.github/scripts/refresh-only-head\.sh$' +# The only base-delta paths a docs-only policy accepts (extended regex). +DOCS_ONLY='^CHANGELOG\.md$|^changelog\.d/|\.md$|^docs/|^src/docs/' + +out=${GITHUB_OUTPUT:-/dev/null} +summary=${GITHUB_STEP_SUMMARY:-/dev/null} +workdir='' +cleanup() { [ -n "$workdir" ] && rm -rf "$workdir"; } +trap cleanup EXIT + +decide() { + # $1 true|false, $2 reason + echo "refresh_only=$1" >> "$out" + echo "reason=$2" >> "$out" + if [ "$1" = true ]; then + echo "skipped: refresh-only head. $2 The heavy jobs in this workflow are skipped; build-and-test, test shards and coverage still run." >> "$summary" + else + echo "refresh-only check: running everything ($2)" >> "$summary" + fi + echo "refresh_only=$1: $2" + exit 0 +} +# ctx prefixes the reason once the walk has gone past the first hop. +ctx='' +no() { decide false "${ctx}$1"; } + +APP_BOT_LOGIN=${APP_BOT_LOGIN:-$DEFAULT_APP_BOT_LOGIN} +SERVER=${GITHUB_SERVER_URL:-https://github.com} + +[ "${EVENT_NAME:-}" = pull_request ] || no "event is ${EVENT_NAME:-unset}, not pull_request" +[ "${EVENT_ACTION:-}" = synchronize ] || no "action is ${EVENT_ACTION:-unset}, not synchronize" +[ -n "${REPO:-}" ] && [ "${HEAD_REPO:-}" = "$REPO" ] || no "head repository is not this repository" +[[ "${BEFORE:-}" =~ $SHA_RE ]] && [ "$BEFORE" != "$ZERO_SHA" ] || no "no previous head" +[[ "${HEAD_SHA:-}" =~ $SHA_RE ]] && [ "$HEAD_SHA" != "$BEFORE" ] || no "no new head" +[ -n "${BASE_REF:-}" ] || no "base ref unknown" +[ -n "${WORKFLOW_PATH:-}" ] || no "workflow path unknown" +[ -n "${GH_TOKEN:-}" ] || no "no token for the commit lookup" +command -v gh >/dev/null && command -v jq >/dev/null && command -v git >/dev/null || no "gh, jq or git missing" + +# Blobless clone: --raw and --name-only diffs need trees only. Fetching the +# new head brings every older hop with it. +workdir=$(mktemp -d) || no "no temp dir" +g() { git -C "$workdir" -c core.hooksPath=/dev/null "$@"; } +g init -q 2>/dev/null || no "git init failed" +auth=$(printf 'x-access-token:%s' "$GH_TOKEN" | base64 | tr -d '\n') +g -c "http.$SERVER/.extraheader=AUTHORIZATION: basic $auth" fetch -q --no-tags --filter=blob:none \ + "$SERVER/$REPO.git" "$HEAD_SHA" "$BEFORE" "+refs/heads/$BASE_REF:refs/remotes/origin/base" 2>/dev/null \ + || no "fetch of the heads and base failed" +raw() { g diff --raw --no-renames --no-abbrev --no-ext-diff --no-textconv "$1" "$2"; } + +# passed SHA: this workflow concluded success on SHA. +passed() { + local n + n=$(gh api "repos/$REPO/actions/runs?head_sha=$1&event=pull_request&per_page=$RUNS_PER_PAGE" \ + --jq "[.workflow_runs[] | select((.path | split(\"@\")[0]) == \"$WORKFLOW_PATH\" and .conclusion == \"success\")] | length" 2>/dev/null) || return 1 + [[ "$n" =~ ^[0-9]+$ ]] && [ "$n" -ge 1 ] +} + +# hop CUR PREV: prove CUR is a refresh of PREV (PREV empty = use the first +# parent, as update-branch puts the head side first). Sets hop_prev, +# hop_base_parent and hop_merge_base, or calls no. +hop() { + local cur=$1 want=$2 commit_json facts verified by_app by_web parents p1 p2 mb_new old_diff new_diff + commit_json=$(gh api "repos/$REPO/commits/$cur" 2>/dev/null) || no "commit lookup for ${cur:0:12} failed" + facts=$(jq -r --arg app "$APP_BOT_LOGIN" --arg web "$WEB_FLOW_LOGIN" ' + def norm: ascii_downcase | sub("\\[bot\\]$"; ""); + [ (.commit.verification.verified == true), + ((.author.login // "" | norm) == ($app | norm)), + ((.committer.login // "" | norm) as $c | $c == $web or $c == ($app | norm)), + (.parents | length) ] | map(tostring) | join(" ")' <<<"$commit_json" 2>/dev/null) || no "commit lookup for ${cur:0:12} unreadable" + read -r verified by_app by_web parents <<<"$facts" + [ "$verified" = true ] || no "${cur:0:12} signature is not GitHub-verified" + [ "$by_app" = true ] || no "${cur:0:12} is not authored by $APP_BOT_LOGIN" + [ "$by_web" = true ] || no "${cur:0:12} was not committed by GitHub for the App" + [ "$parents" = "$MERGE_PARENTS" ] || no "${cur:0:12} is not a two-parent merge" + p1=$(jq -r '.parents[0].sha // ""' <<<"$commit_json") + p2=$(jq -r '.parents[1].sha // ""' <<<"$commit_json") + case "${want:-$p1}" in + "$p1") hop_prev=$p1; hop_base_parent=$p2 ;; + "$p2") hop_prev=$p2; hop_base_parent=$p1 ;; + *) no "previous head is not a parent of ${cur:0:12}" ;; + esac + [[ "$hop_prev" =~ $SHA_RE ]] && [[ "$hop_base_parent" =~ $SHA_RE ]] && [ "$hop_prev" != "$hop_base_parent" ] \ + || no "parents of ${cur:0:12} unknown" + g merge-base --is-ancestor "$hop_base_parent" refs/remotes/origin/base 2>/dev/null \ + || no "base parent of ${cur:0:12} is not on $BASE_REF" + mb_new=$(g merge-base --all "$cur" refs/remotes/origin/base 2>/dev/null) || no "no merge-base for ${cur:0:12}" + [ "$mb_new" = "$hop_base_parent" ] || no "merge-base of ${cur:0:12} is not its merged base commit" + hop_merge_base=$(g merge-base --all "$hop_prev" "$hop_base_parent" 2>/dev/null) || no "no merge-base for ${hop_prev:0:12}" + [[ "$hop_merge_base" =~ $SHA_RE ]] || no "${hop_prev:0:12} has no single merge-base" + old_diff=$(raw "$hop_merge_base" "$hop_prev" 2>/dev/null) || no "diff of ${hop_prev:0:12} failed" + new_diff=$(raw "$hop_base_parent" "$cur" 2>/dev/null) || no "diff of ${cur:0:12} failed" + [ -n "$new_diff" ] || no "${cur:0:12} has an empty diff" + [ "$old_diff" = "$new_diff" ] || no "three-dot diff changed at ${cur:0:12} (the merge altered PR content)" +} + +cur=$HEAD_SHA +want=$BEFORE +newest_base_parent='' +found='' +for ((i = 0; i < MAX_HOPS; i++)); do + hop "$cur" "$want" + [ -n "$newest_base_parent" ] || newest_base_parent=$hop_base_parent + if passed "$hop_prev"; then + found=$hop_prev + break + fi + ctx="$WORKFLOW_PATH did not pass on ${hop_prev:0:12} and the walk further back stopped: " + cur=$hop_prev + want='' +done +ctx='' +[ -n "$found" ] || no "$WORKFLOW_PATH has no successful run on a head within $MAX_HOPS refreshes" + +delta=$(g diff --name-only --no-renames "$hop_merge_base" "$newest_base_parent" 2>/dev/null) || no "base delta unreadable" +case "${DELTA_POLICY:-deny}" in + docs-only) + hit=$(grep -v -E -m1 "$DOCS_ONLY" <<<"$delta" || true) + [ -z "$hit" ] || no "base delta is not docs/changelog-only: it touches $hit" + delta_words='touches only docs and changelog files' + ;; + deny) + hit=$(grep -E -m1 "$SENSITIVE" <<<"$delta" || true) + [ -z "$hit" ] || no "base delta touches $hit" + delta_words='touches no build, dependency, deploy or CI path' + ;; + *) no "unknown DELTA_POLICY ${DELTA_POLICY}" ;; +esac + +decide true "Head ${HEAD_SHA:0:12} is a verified $APP_BOT_LOGIN merge of $BASE_REF into the PR with an unchanged three-dot diff, $WORKFLOW_PATH passed on head ${found:0:12}, and the base delta since then $delta_words." diff --git a/.github/scripts/test-refresh-only-head.sh b/.github/scripts/test-refresh-only-head.sh new file mode 100755 index 0000000000..01a546957e --- /dev/null +++ b/.github/scripts/test-refresh-only-head.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# test-refresh-only-head.sh - exercise .github/scripts/refresh-only-head.sh +# against a local git fixture (served over file://) and a stubbed `gh`. +# Every negative case must answer refresh_only=false: false means "run +# everything", so a regression that answers true wrongly skips CI. +set -euo pipefail + +HERE=$(cd "$(dirname "$0")" && pwd) +SCRIPT="$HERE/refresh-only-head.sh" +BOT='hivecommons-hive[bot]' +WF='.github/workflows/docker.yml' + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +fail=0 + +# --- git fixture: owner/repo.git served at file://$tmp/srv --------------- +srv="$tmp/srv" +repo="$srv/owner/repo.git" +mkdir -p "$repo" +git -C "$repo" init -q -b v5 +gitc() { git -C "$repo" -c user.name=t -c user.email=t@example.com -c commit.gpgsign=false "$@"; } +gitc config uploadpack.allowFilter true +gitc config uploadpack.allowAnySHA1InWant true +put() { mkdir -p "$repo/$(dirname "$1")"; printf '%s\n' "$2" > "$repo/$1"; gitc add "$1"; } +put a.txt base; gitc commit -qm A +A=$(gitc rev-parse HEAD) +gitc checkout -qb pr +put pr.txt change; gitc commit -qm P +P=$(gitc rev-parse HEAD) +gitc checkout -q v5 +put base.txt one; gitc commit -qm B +B=$(gitc rev-parse HEAD) +# M: clean refresh of P onto B. +gitc checkout -q pr +gitc merge -q --no-ff --no-edit v5 +M=$(gitc rev-parse HEAD) +# B2 / M2: second clean refresh of M. +gitc checkout -q v5 +put base2.txt two; gitc commit -qm B2 +B2=$(gitc rev-parse HEAD) +gitc checkout -q pr +gitc merge -q --no-ff --no-edit v5 +M2=$(gitc rev-parse HEAD) +# X: a refresh whose merge also changed PR content. +gitc checkout -q -b evil "$P" +gitc merge -q --no-ff --no-commit "$B" || true +put pr.txt altered +gitc commit -qm X +X=$(gitc rev-parse HEAD) +# D: a refresh whose base delta touches a Dockerfile. +gitc checkout -q -b v5d "$B" +put src/Dockerfile 'FROM scratch'; gitc commit -qm BD +BD=$(gitc rev-parse HEAD) +gitc checkout -q -b prd "$P" +gitc merge -q --no-ff --no-edit "$BD" +D=$(gitc rev-parse HEAD) +# One refresh of P per kind of base delta: base commit B on a branch +# off A (so the delta holds only its own files), refresh R merging it +# into P. Arguments after the kind are path/content pairs. +refresh_with() { + local kind=$1; shift + gitc checkout -q -b "v5$kind" "$A" + while [ "$#" -ge 2 ]; do put "$1" "$2"; shift 2; done + gitc commit -qm "B$kind" + printf -v "B$kind" '%s' "$(gitc rev-parse HEAD)" + gitc checkout -q -b "pr$kind" "$P" + gitc merge -q --no-ff --no-edit "v5$kind" + printf -v "R$kind" '%s' "$(gitc rev-parse HEAD)" +} +refresh_with doc docs/guide.md docs changelog.d/fixed-x.md '- x' src/docs/y.md y CHANGELOG.md c README.md r +refresh_with mk Makefile 'all:' +refresh_with go src/pkg/x/x.go 'package x' +refresh_with pkg web/ui/package-lock.json '{}' +# The base branch the script fetches: v5 must contain every base parent. +gitc checkout -q v5 +for b in "$BD" "$Bdoc" "$Bmk" "$Bgo" "$Bpkg"; do gitc merge -q --no-ff --no-edit "$b"; done + +# --- gh stub: answers from $tmp/fix ------------------------------------ +fix="$tmp/fix" +mkdir -p "$fix" "$tmp/bin" +cat > "$tmp/bin/gh" <<'STUB' +#!/usr/bin/env bash +[ "$1" = api ] || exit 1 +path=$2; shift 2 +jqexpr='' +if [ "${1:-}" = --jq ]; then jqexpr=$2; fi +case "$path" in + repos/*/commits/*) f="$FIX/commit-${path##*/}.json" ;; + repos/*/actions/runs\?head_sha=*) sha=${path#*head_sha=}; sha=${sha%%&*}; f="$FIX/runs-$sha.json" + [ -f "$f" ] || { echo '{"workflow_runs":[]}' > "$f"; } ;; + *) exit 1 ;; +esac +[ -f "$f" ] || exit 1 +if [ -n "$jqexpr" ]; then jq -r "$jqexpr" "$f"; else cat "$f"; fi +STUB +chmod +x "$tmp/bin/gh" + +commit() { # sha verified author committer parent1 parent2 + printf '{"commit":{"verification":{"verified":%s}},"author":{"login":"%s"},"committer":{"login":"%s"},"parents":[{"sha":"%s"},{"sha":"%s"}]}\n' \ + "$2" "$3" "$4" "$5" "$6" > "$fix/commit-$1.json" +} +runs() { # sha conclusion + printf '{"workflow_runs":[{"path":"%s","conclusion":"%s"}]}\n' "$WF" "$2" > "$fix/runs-$1.json" +} +reset() { rm -f "$fix"/*.json; } + +check() { # name want before head [extra env...] + local name=$1 want=$2 before=$3 head=$4 got + shift 4 + got=$(env -i PATH="$tmp/bin:$PATH" HOME="$tmp" FIX="$fix" GITHUB_SERVER_URL="file://$srv" \ + EVENT_NAME=pull_request EVENT_ACTION=synchronize BEFORE="$before" HEAD_SHA="$head" \ + BASE_REF=v5 HEAD_REPO=owner/repo REPO=owner/repo WORKFLOW_PATH="$WF" GH_TOKEN=t \ + GITHUB_OUTPUT="$tmp/out" GITHUB_STEP_SUMMARY="$tmp/summary" "$@" bash "$SCRIPT" | sed -n 's/^refresh_only=\([a-z]*\):.*/\1/p') + if [ "$got" = "$want" ]; then + echo "ok $name" + else + echo "FAIL $name: want $want, got '${got}'"; sed 's/^/ /' "$tmp/summary" 2>/dev/null || true + fail=1 + fi + rm -f "$tmp/out" "$tmp/summary" +} + +reset; commit "$M" true "$BOT" web-flow "$P" "$B"; runs "$P" success +check "clean refresh of a passing head" true "$P" "$M" +check "non-synchronize event" false "$P" "$M" EVENT_ACTION=opened +check "fork head" false "$P" "$M" HEAD_REPO=fork/repo +check "zero previous head" false 0000000000000000000000000000000000000000 "$M" +check "no token" false "$P" "$M" GH_TOKEN= +check "previous head not a parent" false "$A" "$M" + +reset; commit "$M" false "$BOT" web-flow "$P" "$B"; runs "$P" success +check "unverified signature" false "$P" "$M" + +reset; commit "$M" true someone-else web-flow "$P" "$B"; runs "$P" success +check "not authored by the App" false "$P" "$M" + +reset; commit "$M" true "$BOT" someone-else "$P" "$B"; runs "$P" success +check "not committed by web-flow" false "$P" "$M" + +reset; commit "$M" true "$BOT" web-flow "$P" "$B"; runs "$P" cancelled +check "previous head cancelled, not a refresh" false "$P" "$M" + +reset; commit "$X" true "$BOT" web-flow "$P" "$B"; runs "$P" success +check "merge altered PR content" false "$P" "$X" + +reset; commit "$D" true "$BOT" web-flow "$P" "$BD"; runs "$P" success +check "base delta touches a Dockerfile" false "$P" "$D" + +reset; commit "$M2" true "$BOT" web-flow "$M" "$B2"; commit "$M" true "$BOT" web-flow "$P" "$B" +runs "$M" cancelled; runs "$P" success +check "two refresh hops back to a passing head" true "$M" "$M2" + +reset; commit "$M2" true "$BOT" web-flow "$M" "$B2"; commit "$M" false "$BOT" web-flow "$P" "$B" +runs "$M" cancelled; runs "$P" success +check "unverified hop in the walk" false "$M" "$M2" + +# docs-only (docker.yml): every base-delta path must be docs or changelog. +reset; commit "$M" true "$BOT" web-flow "$P" "$B"; runs "$P" success +check "docs-only: base delta with a plain file runs" false "$P" "$M" DELTA_POLICY=docs-only +reset; commit "$Rdoc" true "$BOT" web-flow "$P" "$Bdoc"; runs "$P" success +check "docs-only: docs and changelog delta may skip" true "$P" "$Rdoc" DELTA_POLICY=docs-only +reset; commit "$Rmk" true "$BOT" web-flow "$P" "$Bmk"; runs "$P" success +check "docs-only: Makefile in the delta runs" false "$P" "$Rmk" DELTA_POLICY=docs-only +check "deny: Makefile in the delta may skip" true "$P" "$Rmk" +reset; commit "$Rgo" true "$BOT" web-flow "$P" "$Bgo"; runs "$P" success +check "docs-only: .go file in the delta runs" false "$P" "$Rgo" DELTA_POLICY=docs-only +reset; commit "$Rpkg" true "$BOT" web-flow "$P" "$Bpkg"; runs "$P" success +check "deny: nested package-lock.json in the delta runs" false "$P" "$Rpkg" +check "unknown policy runs" false "$P" "$Rpkg" DELTA_POLICY=bogus + +exit "$fail" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f3a6a8533c..67bc9b8d1f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -137,6 +137,12 @@ jobs: ref_name: ${{ steps.decide.outputs.ref_name }} cross_org: ${{ steps.registries.outputs.cross_org }} cross_secret: ${{ steps.registries.outputs.cross_secret }} + refresh_only: ${{ steps.refresh.outputs.refresh_only }} + # Job-level permissions replace the workflow's: gate pushes nothing, and + # the refresh-only check below reads commits and this workflow's runs. + permissions: + contents: read + actions: read steps: - name: Decide push policy id: decide @@ -245,6 +251,44 @@ jobs: echo "cross_secret=${cross_secret}" >> "$GITHUB_OUTPUT" echo "Native org: ${OWNER}; mirroring publishes to ghcr.io/${cross_org} using secret ${cross_secret}." + # Refresh-only heads (src/docs/ci-refresh-only-heads.md): when the new + # PR head is nothing but a verified hive App merge of the base whose + # three-dot diff is unchanged, this workflow already passed on the + # previous head, and every file the base delta brings in is docs or + # changelog only, the multi-arch build-* jobs below skip it. `gate` + # itself still runs and reports success, so the required context + # attaches as before. continue-on-error on both steps: a failure here + # leaves refresh_only empty, which means "build everything", and can + # never fail the required `gate`. + - name: Checkout refresh-only helper + if: github.event_name == 'pull_request' + continue-on-error: true + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + path: .refresh-check + persist-credentials: false + sparse-checkout: .github/scripts/refresh-only-head.sh + sparse-checkout-cone-mode: false + + - name: Detect refresh-only head + id: refresh + if: github.event_name == 'pull_request' + continue-on-error: true + env: + EVENT_NAME: ${{ github.event_name }} + EVENT_ACTION: ${{ github.event.action }} + BEFORE: ${{ github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + REPO: ${{ github.repository }} + WORKFLOW_PATH: .github/workflows/docker.yml + # Allowlist: skip only when every pulled-in base file is docs or + # changelog; any other file in the base delta builds the images. + DELTA_POLICY: docs-only + GH_TOKEN: ${{ github.token }} + run: bash .refresh-check/.github/scripts/refresh-only-head.sh + build: needs: gate # Runs when `gate` decided push=true, i.e. a long-lived branch or a @@ -272,7 +316,8 @@ jobs: if: >- (needs.gate.outputs.push == 'true' || (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository)) && + github.event.pull_request.head.repo.full_name == github.repository && + needs.gate.outputs.refresh_only != 'true')) && !startsWith(github.ref, 'refs/heads/release-gate/') strategy: matrix: @@ -729,7 +774,8 @@ jobs: if: >- (needs.gate.outputs.push == 'true' || (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository)) && + github.event.pull_request.head.repo.full_name == github.repository && + needs.gate.outputs.refresh_only != 'true')) && !startsWith(github.ref, 'refs/heads/release-gate/') strategy: matrix: @@ -941,7 +987,8 @@ jobs: if: >- (needs.gate.outputs.push == 'true' || (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository)) && + github.event.pull_request.head.repo.full_name == github.repository && + needs.gate.outputs.refresh_only != 'true')) && !startsWith(github.ref, 'refs/heads/release-gate/') strategy: matrix: @@ -1172,7 +1219,8 @@ jobs: if: >- (needs.gate.outputs.push == 'true' || (github.event_name == 'pull_request' && - github.event.pull_request.head.repo.full_name == github.repository)) && + github.event.pull_request.head.repo.full_name == github.repository && + needs.gate.outputs.refresh_only != 'true')) && !startsWith(github.ref, 'refs/heads/release-gate/') strategy: matrix: diff --git a/.github/workflows/fossa.yml b/.github/workflows/fossa.yml index c74f85f83e..c8528fc726 100644 --- a/.github/workflows/fossa.yml +++ b/.github/workflows/fossa.yml @@ -15,6 +15,8 @@ concurrency: permissions: contents: read + # The refresh-only check reads this workflow's runs on the previous head. + actions: read jobs: fossa: @@ -27,10 +29,33 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Refresh-only heads (src/docs/ci-refresh-only-heads.md): a verified + # hive App merge of the base that leaves the PR's three-dot diff + # unchanged, on a PR whose previous head already passed this scan, and + # whose base delta touches no dependency manifest, Dockerfile, deploy or + # CI path, has nothing new to scan. The scan steps below skip and the + # job reports success. Any failure here leaves refresh_only empty, + # which means "scan". + - name: Detect refresh-only head + id: refresh + if: github.event_name == 'pull_request' + continue-on-error: true + env: + EVENT_NAME: ${{ github.event_name }} + EVENT_ACTION: ${{ github.event.action }} + BEFORE: ${{ github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + REPO: ${{ github.repository }} + WORKFLOW_PATH: .github/workflows/fossa.yml + GH_TOKEN: ${{ github.token }} + run: bash .github/scripts/refresh-only-head.sh + # Forks and repos without the secret skip the scan instead of failing # the check on a missing input; the scan activates once FOSSA_API_KEY is set. - name: Skip when FOSSA_API_KEY is not configured - if: env.FOSSA_API_KEY == '' + if: env.FOSSA_API_KEY == '' && steps.refresh.outputs.refresh_only != 'true' run: echo "FOSSA_API_KEY secret not set; skipping FOSSA scan." >> "$GITHUB_STEP_SUMMARY" # The key must be a push-only token from the *CNCF* FOSSA organization so @@ -41,7 +66,7 @@ jobs: # and forwards `team`/`branch` to it, which `fossa test` rejects. So: # always analyze (upload), then on PRs run the policy test separately. - name: Run FOSSA scan - if: env.FOSSA_API_KEY != '' + if: env.FOSSA_API_KEY != '' && steps.refresh.outputs.refresh_only != 'true' uses: fossas/fossa-action@29693cc50323968e039056be419b32989fc5880c # v2.0.0 with: api-key: ${{ secrets.FOSSA_API_KEY }} @@ -53,7 +78,7 @@ jobs: # Upload the exact diff baseline here so the PR test does not race it. # Check it out after the PR scan to keep it out of that scan's discovery. - name: Checkout FOSSA diff baseline - if: env.FOSSA_API_KEY != '' && github.event_name == 'pull_request' + if: env.FOSSA_API_KEY != '' && github.event_name == 'pull_request' && steps.refresh.outputs.refresh_only != 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.base.sha }} @@ -61,7 +86,7 @@ jobs: persist-credentials: false - name: Analyze FOSSA diff baseline - if: env.FOSSA_API_KEY != '' && github.event_name == 'pull_request' + if: env.FOSSA_API_KEY != '' && github.event_name == 'pull_request' && steps.refresh.outputs.refresh_only != 'true' working-directory: .fossa-base env: FOSSA_BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -74,7 +99,7 @@ jobs: --team "Hive Commons" - name: Run FOSSA policy test - if: env.FOSSA_API_KEY != '' && github.event_name == 'pull_request' + if: env.FOSSA_API_KEY != '' && github.event_name == 'pull_request' && steps.refresh.outputs.refresh_only != 'true' uses: fossas/fossa-action@29693cc50323968e039056be419b32989fc5880c # v2.0.0 with: api-key: ${{ secrets.FOSSA_API_KEY }} diff --git a/.github/workflows/mutation-advisory.yml b/.github/workflows/mutation-advisory.yml index 6852a01dc2..9ac1ae5243 100644 --- a/.github/workflows/mutation-advisory.yml +++ b/.github/workflows/mutation-advisory.yml @@ -23,21 +23,47 @@ jobs: permissions: contents: read pull-requests: write + # The refresh-only check reads this workflow's runs on the previous head. + actions: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 + # Refresh-only heads (src/docs/ci-refresh-only-heads.md): a verified + # hive App merge of the base that leaves the PR's three-dot diff + # unchanged, on a PR whose previous head already passed this workflow, + # and whose base delta touches no dependency manifest, Dockerfile, + # deploy or CI path, skips the mutation run. The comment from the + # previous head stays as it is: it describes the same diff. Any failure + # here leaves refresh_only empty, which means "run". + - name: Detect refresh-only head + id: refresh + continue-on-error: true + env: + EVENT_NAME: ${{ github.event_name }} + EVENT_ACTION: ${{ github.event.action }} + BEFORE: ${{ github.event.before }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + REPO: ${{ github.repository }} + WORKFLOW_PATH: .github/workflows/mutation-advisory.yml + GH_TOKEN: ${{ github.token }} + run: bash .github/scripts/refresh-only-head.sh - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + if: steps.refresh.outputs.refresh_only != 'true' with: cache: false go-version-file: src/go.mod - name: Install gremlins + if: steps.refresh.outputs.refresh_only != 'true' run: go install github.com/go-gremlins/gremlins/cmd/gremlins@v0.6.0 # The mutation run executes PR-controlled test code, so it gets no token: # it only writes mutation-out/comment.md. The token is scoped to the # separate comment-upsert step below. - name: Run mutation report + if: steps.refresh.outputs.refresh_only != 'true' env: BASE_REF: ${{ github.base_ref }} # Must stay under timeout-minutes (20m = 1200s): a job killed by that @@ -47,7 +73,7 @@ jobs: export PATH="$(go env GOPATH)/bin:$PATH" bash src/scripts/mutation-report.sh run - name: Upsert mutation comment - if: always() + if: always() && steps.refresh.outputs.refresh_only != 'true' env: GH_TOKEN: ${{ github.token }} PR_NUMBER: ${{ github.event.pull_request.number }} diff --git a/.github/workflows/pr-auto-update-test.yml b/.github/workflows/pr-auto-update-test.yml index 58bd258e5c..7e26209fd2 100644 --- a/.github/workflows/pr-auto-update-test.yml +++ b/.github/workflows/pr-auto-update-test.yml @@ -10,6 +10,8 @@ on: - '.github/scripts/diagnose-dco-provider*.cjs' - '.github/scripts/pr-verifier*.cjs' - '.github/scripts/integration-governance.cjs' + - '.github/scripts/refresh-only-head.sh' + - '.github/scripts/test-refresh-only-head.sh' - 'src/pkg/dashboard/static-src/js/config-helpers.js' - 'src/pkg/dashboard/static-src/js/config-features.js' - 'src/pkg/dashboard/static-src/js/tracker-native-queue.js' @@ -32,6 +34,8 @@ on: - '.github/scripts/diagnose-dco-provider*.cjs' - '.github/scripts/pr-verifier*.cjs' - '.github/scripts/integration-governance.cjs' + - '.github/scripts/refresh-only-head.sh' + - '.github/scripts/test-refresh-only-head.sh' - 'src/pkg/dashboard/static-src/js/config-helpers.js' - 'src/pkg/dashboard/static-src/js/config-features.js' - 'src/pkg/dashboard/static-src/js/tracker-native-queue.js' @@ -60,6 +64,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Exercise push and shared-CI incident recovery with mocked APIs run: bash .github/scripts/test_pr-auto-update.sh + - name: Refresh-only head detector answers false on every doubt + run: bash .github/scripts/test-refresh-only-head.sh - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '22' diff --git a/changelog.d/changed-19276-ci-refresh-only-heads.md b/changelog.d/changed-19276-ci-refresh-only-heads.md new file mode 100644 index 0000000000..17ac6304f6 --- /dev/null +++ b/changelog.d/changed-19276-ci-refresh-only-heads.md @@ -0,0 +1 @@ +- CI no longer reruns the heaviest jobs on a pull request head that is only the hive App's base refresh. When the new head is a GitHub-verified App merge of the base whose three-dot diff is unchanged, the same workflow already passed on an earlier head of the refresh chain, and the base delta passes that workflow's policy, the Docker multi-arch build jobs, the FOSSA scan and the Mutation run skip with a `skipped: refresh-only head` step summary. Required contexts (`gate` included) still report success, and `build-and-test`, the test shards and `coverage` always run. Docker uses an allowlist: it skips only when every pulled-in base file is docs or changelog, and `v5` pushes still build every image after merge. FOSSA and Mutation skip unless the base delta touches a dependency manifest or lock file anywhere, `.fossa.yml`, a Dockerfile, `src/deploy/` or a CI definition. Any doubt runs everything. Each refresh used to cost about 38 runner-minutes across 16 workflows. See `src/docs/ci-refresh-only-heads.md` (#19276). diff --git a/src/docs/ci-refresh-only-heads.md b/src/docs/ci-refresh-only-heads.md new file mode 100644 index 0000000000..d9e27644f9 --- /dev/null +++ b/src/docs/ci-refresh-only-heads.md @@ -0,0 +1,100 @@ +# Refresh-only heads: which CI jobs skip a base refresh + +Repositories governed by `auto_merge.require_current_base` (see +[Final base preparation](hive-merge.md#final-base-preparation-one-refresh-owner-no-ready-work-blockade)) +get their behind pull requests refreshed by the hive App: an update-branch +merge commit, "Merge branch 'v5' into ...", authored by the App and signed by +GitHub. Each refresh is a new head, so every pull_request workflow runs again. +Measured on this repository: about 16 workflows and 38 runner-minutes per +refresh, and 133 refreshes in 48 hours. The heaviest jobs on a refresh are the +Docker multi-arch build (about 8.7 minutes), FOSSA (about 6.8 minutes) and +Mutation (about 3.2 minutes), and none of them can learn anything new from a +head whose PR content did not change. + +## The rule + +On a pull_request `synchronize`, `.github/scripts/refresh-only-head.sh` +answers `refresh_only=true` only when **all** of these hold: + +1. The pull request's head branch is in this repository (fork heads always + run everything). +2. The new head is a refresh of the previous head: a two-parent merge commit + whose signature GitHub verified, authored by the hive App + (`hivecommons-hive[bot]`) and committed by `web-flow` (or the App itself), + with the previous head as one parent. The other parent is a commit on the + base branch and is the merge-base of the new head with the base. +3. The three-dot diff of the new head against the base is identical, blob for + blob, to the previous head's. A merge that resolved a conflict or changed + any file the PR touches fails this. +4. The same workflow already concluded `success` on the previous head. If that + run was cancelled (a refresh often cancels the run in flight), the check + walks back through earlier refreshes, at most 5, each held to rules 2 and + 3, until it finds a head that passed. A head that is not a refresh stops + the walk. +5. The base delta brought in since that passing head passes the workflow's + delta policy (`DELTA_POLICY`): + - **`docs-only` (Docker, an allowlist).** Every file in the delta is + `CHANGELOG.md`, under `changelog.d/`, `docs/` or `src/docs/`, or any + `*.md` file. Any other file, a `Makefile` or a `.go` file included, + means the images build. + - **`deny` (FOSSA and Mutation, the default).** No file in the delta is a + dependency manifest or lock file anywhere in the tree (`go.mod`, + `go.sum`, `go.work`, `package*.json`, `npm-shrinkwrap.json`, + `yarn.lock`, `pnpm-lock.yaml`, `requirements*`, `pyproject.toml`, + `poetry.lock`, `Cargo.*`), `.fossa.yml`, a Dockerfile or + `.dockerignore`, under `src/deploy/`, `.github/workflows/` or + `.github/actions/`, or the detector script itself. + +This is the workflow form of the proof the hive uses to carry a recorded +decision across its own refresh, `HeadIsBaseRefreshOf` in +`src/pkg/github/pr_decision_base_refresh.go`. The script works on a blobless +clone (`--filter=blob:none`, about 3 seconds for this repository) and compares +`git diff --raw --no-renames` output, so it needs trees only. + +**Fail safe.** Every lookup failure, unexpected shape or doubt answers +`false`, and `false` means run everything. The detector steps are +`continue-on-error: true`, so a broken detector leaves the answer empty, which +the jobs also treat as "run everything". Each run writes one line to the step +summary: either `skipped: refresh-only head.` with the evidence, or +`refresh-only check: running everything ()`. + +## What skips, what still runs + +| Workflow | On a refresh-only head | +| --- | --- | +| `docker.yml` (`docs-only`) | `gate` runs and reports success (it is a required context). The `build`, `build-contributor`, `build-hub` and `build-mx` multi-arch jobs are skipped. | +| `fossa.yml` (`deny`) | The `FOSSA license scan` job runs and reports success; its scan, baseline and policy-test steps are skipped. | +| `mutation-advisory.yml` (`deny`) | The job runs and reports success; setup, the mutation run and the comment upsert are skipped, so the previous head's comment stays (it describes the same diff). | +| everything else | Unchanged. `build-and-test`, the test shards and `coverage` always run, as do every other required context. | + +**Post-merge backstop.** The skip only ever applies to `pull_request` runs. +Every push to `v5` still runs the full multi-arch build: the `push` trigger +matches every branch (`.github/workflows/docker.yml:4-15`), `v5` is in +`LONG_LIVED` (`.github/workflows/docker.yml:157`), so `gate` decides +`push=true` and each `build-*` job runs on it (`.github/workflows/docker.yml:317`). +A refresh-only PR that skipped its images is built again the moment it +merges. + +The skip is decided **inside** the jobs, never with trigger `paths:` filters: +a required context must attach to every head, and a filtered workflow never +reports on the heads it does not match. + +## Permissions and pinning + +The detector reads the head commit and this workflow's runs on earlier heads +with the job's `GITHUB_TOKEN`, so `docker.yml`'s `gate` job, `fossa.yml` and +`mutation-advisory.yml` add `actions: read`. The script lives under +`.github/scripts/`, not `src/scripts/`, so it is outside the +`TRUSTED_SCRIPTS_TREE` pin in `.github/scripts/integration-governance.cjs`; +changes to it are covered by the `.github` tree that integration candidates +already record. A change to the script is itself in the sensitive list, so a +base refresh that brings one in always runs everything. + +`.github/scripts/test-refresh-only-head.sh` exercises the script against a +local git fixture and a stubbed `gh` (clean refresh, multi-hop walk, and every +negative case); it runs in the `PR Auto-update Regression Tests` workflow. + +## Related + +- [Final base preparation](hive-merge.md#final-base-preparation-one-refresh-owner-no-ready-work-blockade): who refreshes a behind PR and when. +- [CI runner labels](ci-runner-labels.md): where these jobs run. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index fb797fd5ab..7ffefd8ebe 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -252,6 +252,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [CNCF reference architecture](cncf-reference-architecture.md) — CNCF submission/reference template. - [Podman CI runner map](podman-ci-runner-map.md) — measured hosted-runner capabilities and which Podman lane goes where; SELinux is the only lane needing non-hosted infrastructure. - [CI runner labels](ci-runner-labels.md) — how `runs-on:` picks the self-hosted fleet, why a fork must degrade to a GitHub-hosted runner, and the one variable `hivecommons` sets. +- [Refresh-only heads](ci-refresh-only-heads.md) - which heavy CI jobs (Docker multi-arch, FOSSA, Mutation) skip a verified hive App base refresh that leaves the PR diff unchanged, and the fail-safe proof behind the skip. - [Session taint design](design/session-taint.md) - **design/proposal:** a session-level record of what an agent has read and the rules that would hold or deny later pushes, comments and PRs against it, off by default with an observe-only first phase (RFC #18987). - [Gate-integrity invariants for agent lanes](design/gate-integrity-invariants.md) — proposed write-gate rules for agent lanes: no history rewrites on branches a lane did not create, no sign-off on other authors' commits, and hold-gated demotion for gate manipulation. - [Crux: the agent runtime boundary](design/crux.md) — **Design record:** runtime separation, module boundary, staged rollout, and naming decision. diff --git a/src/docs/hive-merge.md b/src/docs/hive-merge.md index 72d71874cd..5748065f0f 100644 --- a/src/docs/hive-merge.md +++ b/src/docs/hive-merge.md @@ -559,6 +559,11 @@ Train sibling refreshes and client-side follow-up/relay/supersession updates defer to preparation on these repositories; opted-out repositories retain their previous behavior. +A refresh the App makes this way is cheap for CI on this repository: the +heavy Docker multi-arch, FOSSA and Mutation jobs skip a head whose only +change is that verified base merge, while every required context still +reports. See [Refresh-only heads](ci-refresh-only-heads.md). + Ownership and first-admission FIFO tickets live in `/data/base-preparation/`, with per-destination file locking and atomic durable snapshots. An update identity (PR/head/base SHA) is recorded before the request, across lanes,