From 7a83946a12ec046ff3ec2bf443b41f4c1cc5865e Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 31 Oct 2026 07:46:09 -0400 Subject: [PATCH 1/7] =?UTF-8?q?=F0=9F=8C=B1=20ci:=20remove=20redundant=20s?= =?UTF-8?q?cans=20and=20parallelize=20build=20contracts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .github/workflows/go-security-analysis.yml | 13 +++- ...gration-technical-go-security-analysis.yml | 11 +++- .../workflows/integration-technical-v2-ci.yml | 63 ++++++++++++++++++- .github/workflows/v2-ci.yml | 63 ++++++++++++++++++- bin/test_hive_podman_cleanup.sh | 46 +++++++++----- docs/development.md | 22 +++++++ 6 files changed, 197 insertions(+), 21 deletions(-) diff --git a/.github/workflows/go-security-analysis.yml b/.github/workflows/go-security-analysis.yml index f134fffa8f..f55ebe9361 100644 --- a/.github/workflows/go-security-analysis.yml +++ b/.github/workflows/go-security-analysis.yml @@ -232,7 +232,16 @@ jobs: # v0.25.0, which does not COMPILE under Go 1.25 ("invalid array length"). # The failure looks like a scan failure but is a build failure in the # tool itself — do not downgrade to "fix" a finding. + - name: Restore pinned gosec binary + id: gosec-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/hive-ci-tools/gosec + key: hive-gosec-${{ runner.os }}-${{ runner.arch }}-v2.29.0-${{ hashFiles('src/go.mod') }} - name: Install gosec + if: steps.gosec-cache.outputs.cache-hit != 'true' + env: + GOBIN: ${{ runner.temp }}/hive-ci-tools run: go install github.com/securego/gosec/v2/cmd/gosec@v2.29.0 # -exclude-generated: generated files are not hand-maintained, so a @@ -255,7 +264,9 @@ jobs: - name: Scan working-directory: src continue-on-error: true - run: gosec -exclude-generated ./... ./crux/... + # Unlike go test/vet, gosec recursively discovers nested modules. + # Listing ./crux/... too scans every Crux directory twice. + run: '"${{ runner.temp }}/hive-ci-tools/gosec" -exclude-generated ./...' golangci-lint: needs: required-context-scope diff --git a/.github/workflows/integration-technical-go-security-analysis.yml b/.github/workflows/integration-technical-go-security-analysis.yml index 0710fed719..5b9f28384c 100644 --- a/.github/workflows/integration-technical-go-security-analysis.yml +++ b/.github/workflows/integration-technical-go-security-analysis.yml @@ -114,12 +114,21 @@ jobs: run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" - name: Download Go modules (retry transient proxy resets) uses: ./.github/actions/go-mod-download-retry + - name: Restore pinned gosec binary + id: gosec-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ runner.temp }}/hive-ci-tools/gosec + key: hive-gosec-${{ runner.os }}-${{ runner.arch }}-v2.29.0-${{ hashFiles('src/go.mod') }} - name: Install gosec + if: steps.gosec-cache.outputs.cache-hit != 'true' + env: + GOBIN: ${{ runner.temp }}/hive-ci-tools run: go install github.com/securego/gosec/v2/cmd/gosec@v2.29.0 - name: Scan working-directory: src continue-on-error: true - run: gosec -exclude-generated ./... ./crux/... + run: '"${{ runner.temp }}/hive-ci-tools/gosec" -exclude-generated ./...' golangci-lint: needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} diff --git a/.github/workflows/integration-technical-v2-ci.yml b/.github/workflows/integration-technical-v2-ci.yml index c2354fbfe7..8dc21ec57e 100644 --- a/.github/workflows/integration-technical-v2-ci.yml +++ b/.github/workflows/integration-technical-v2-ci.yml @@ -70,7 +70,7 @@ jobs: else echo "No v2 CI paths changed; build-and-test will be reported as skipped/success." fi - build-and-test: + build-go: needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} runs-on: ubuntu-latest @@ -144,6 +144,25 @@ jobs: run: go vet ./... ./crux/... - name: Price table staleness guard run: go run ./internal/tools/pricetable --check-age 60 + contract-tests: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } - name: Entrypoint runtime-config migration tests run: bash deploy/test_entrypoint_runtime_config.sh - name: Entrypoint NET_ADMIN ambient-cap tests (#3760) @@ -304,6 +323,25 @@ jobs: - name: Every bin/ test suite is wired into something (#4363) working-directory: . run: bash bin/test_bin_suites_wired.sh + runtime-tests: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 with: node-version: '22' @@ -454,6 +492,29 @@ jobs: - name: Runner labels are fork-portable (#6083) working-directory: . run: bash src/deploy/test_runner_labels_portable.sh + build-and-test: + needs: + - required-context-scope + - build-go + - contract-tests + - runtime-tests + if: ${{ always() && !cancelled() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Require every build and contract lane + env: + SCOPE_RESULT: ${{ needs.required-context-scope.result }} + GO_RESULT: ${{ needs.build-go.result }} + CONTRACT_RESULT: ${{ needs.contract-tests.result }} + RUNTIME_RESULT: ${{ needs.runtime-tests.result }} + run: | + for result in "$SCOPE_RESULT" "$GO_RESULT" "$CONTRACT_RESULT" "$RUNTIME_RESULT"; do + if [ "$result" != success ]; then + echo "::error::build-and-test requires successful scope, Go, contract and runtime lanes (scope=$SCOPE_RESULT go=$GO_RESULT contracts=$CONTRACT_RESULT runtime=$RUNTIME_RESULT)" + exit 1 + fi + done docker: name: docker (${{ matrix.name }}) needs: required-context-scope diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index b42d8910bb..50a7006956 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -120,7 +120,7 @@ jobs: echo "No v2 CI paths changed; build-and-test will be reported as skipped/success." fi - build-and-test: + build-go: needs: required-context-scope # release-gate/* is created only by tagged-release.yml and carries only CHANGELOG.md/changelog.d changes; report the required context immediately (#10863). if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} @@ -241,6 +241,26 @@ jobs: - name: Price table staleness guard run: go run ./internal/tools/pricetable --check-age 60 + contract-tests: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } + # The entrypoint decides which file a hive boots its config from. The # hive.yaml.bak -> hive.yaml.runtime migration must keep reading the # legacy name, or hives that have not saved since upgrading lose their @@ -872,6 +892,25 @@ jobs: working-directory: . run: bash bin/test_bin_suites_wired.sh + runtime-tests: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '22' @@ -1257,6 +1296,28 @@ jobs: - name: Runner labels are fork-portable (#6083) working-directory: . run: bash src/deploy/test_runner_labels_portable.sh + + # Preserve the protected check name: every original step still gates it. + build-and-test: + needs: [required-context-scope, build-go, contract-tests, runtime-tests] + if: ${{ always() && !cancelled() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + timeout-minutes: 10 + steps: + - name: Require every build and contract lane + env: + SCOPE_RESULT: ${{ needs.required-context-scope.result }} + GO_RESULT: ${{ needs.build-go.result }} + CONTRACT_RESULT: ${{ needs.contract-tests.result }} + RUNTIME_RESULT: ${{ needs.runtime-tests.result }} + run: | + for result in "$SCOPE_RESULT" "$GO_RESULT" "$CONTRACT_RESULT" "$RUNTIME_RESULT"; do + if [ "$result" != success ]; then + echo "::error::build-and-test requires successful scope, Go, contract and runtime lanes (scope=$SCOPE_RESULT go=$GO_RESULT contracts=$CONTRACT_RESULT runtime=$RUNTIME_RESULT)" + exit 1 + fi + done + create-issue-on-failure: if: "!inputs.integration_candidate && failure() && github.event_name == 'push' && github.ref == 'refs/heads/v2'" runs-on: ${{ fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') }} diff --git a/bin/test_hive_podman_cleanup.sh b/bin/test_hive_podman_cleanup.sh index a628f6889a..d5fab10fc1 100755 --- a/bin/test_hive_podman_cleanup.sh +++ b/bin/test_hive_podman_cleanup.sh @@ -237,23 +237,35 @@ exempt=( ) if command -v git >/dev/null 2>&1 && git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then - offenders="" - while IFS= read -r tracked; do - skip="" - for exempt_path in "${exempt[@]}"; do - [[ "$tracked" == "$exempt_path" ]] && skip="yes" - done - [[ -n "$skip" ]] && continue - [[ -f "${ROOT}/${tracked}" ]] || continue - - for pattern in "${broad_patterns[@]}"; do - if grep -qF -- "$pattern" "${ROOT}/${tracked}" 2>/dev/null; then - offenders+="${tracked}: ${pattern}"$'\n' - fi - done - done < <(git -C "$ROOT" ls-files) - - [[ -z "$offenders" ]] || fail "broad Podman/Buildah cleanup found:"$'\n'"${offenders}" + # Read each tracked file once instead of spawning one grep per pattern/file + # (over 100,000 processes on the current tree). Keep literal byte matching, + # symlink handling, and all existing exemptions, including binary files. + python3 - "$ROOT" "${#broad_patterns[@]}" "${broad_patterns[@]}" "${exempt[@]}" <<'PYSCAN' +import os +import subprocess +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +pattern_count = int(sys.argv[2]) +patterns = [pattern.encode() for pattern in sys.argv[3:3 + pattern_count]] +exempt = set(sys.argv[3 + pattern_count:]) +tracked = subprocess.check_output(["git", "-C", str(root), "ls-files", "-z"]) +offenders = [] +for name in tracked.split(b"\0"): + if not name: + continue + relative = os.fsdecode(name) + path = root / relative + if relative in exempt or not path.is_file(): + continue + contents = path.read_bytes() + for pattern in patterns: + if pattern in contents: + offenders.append(f"{relative}: {pattern.decode()}") +if offenders: + sys.exit("FAIL: broad Podman/Buildah cleanup found:\n" + "\n".join(offenders)) +PYSCAN pass_count=$((pass_count + 1)) fi diff --git a/docs/development.md b/docs/development.md index 47159e4970..6b19a7dc45 100644 --- a/docs/development.md +++ b/docs/development.md @@ -152,6 +152,28 @@ Two flags that appear in CI but are *not* part of the PR gate: report. Coverage is scored, but it is not the required merge gate; a failing test is. +### CI wall clock and required checks + +The `v2 CI` workflow runs Go build/vet, deployment and shell contracts, and +Node/runtime contracts as three independent lanes. The protected +`build-and-test` check aggregates **all three** and fails if any lane fails, +is cancelled, or unexpectedly skips. This changes scheduling, not which +contracts run. The separate race/shuffle matrix and per-package coverage +checks remain required and unchanged. + +The Podman cleanup contract reads every tracked file once, checking all +forbidden literal patterns together rather than spawning a grep for each +pattern in each file. Go security CI caches the version-pinned gosec binary; +the scan itself still runs every time. Gosec's recursive discovery includes +nested modules, so its `./...` scan covers Crux without scanning it twice. +Other Go tools still need both `./...` and `./crux/...`. + +Measure PR turnaround from workflow creation to the **last required check** +completing, including runner queues and aggregate/coverage jobs, not just +the slowest test shard's execution. Cache misses and shared runner load +can change this substantially; a short shard alone is not a five-minute +pipeline. + ### Affected-package selection on pull requests On a pull request the gate does not always run every package. The From 7d7d48b758f356abb2e51c99c6688e97c748e7c7 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 31 Oct 2026 07:53:58 -0400 Subject: [PATCH 2/7] =?UTF-8?q?=F0=9F=8C=B1=20ci:=20bound=20scanner=20fano?= =?UTF-8?q?ut=20and=20remove=20coverage=20cache=20post-save?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .github/workflows/go-security-analysis.yml | 4 +++- .../integration-technical-go-security-analysis.yml | 2 +- .github/workflows/integration-technical-v2-tests.yml | 4 +--- .github/workflows/v2-tests.yml | 7 ++++--- docs/development.md | 5 +++++ 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/go-security-analysis.yml b/.github/workflows/go-security-analysis.yml index f55ebe9361..568524063e 100644 --- a/.github/workflows/go-security-analysis.yml +++ b/.github/workflows/go-security-analysis.yml @@ -266,7 +266,9 @@ jobs: continue-on-error: true # Unlike go test/vet, gosec recursively discovers nested modules. # Listing ./crux/... too scans every Crux directory twice. - run: '"${{ runner.temp }}/hive-ci-tools/gosec" -exclude-generated ./...' + # Its default concurrency is host NumCPU, not the runner's CPU quota; + # each worker also launches a Go package loader. Bound that fanout. + run: '"${{ runner.temp }}/hive-ci-tools/gosec" -concurrency "${HIVE_CI_GOSEC_CONCURRENCY:-2}" -exclude-generated ./...' golangci-lint: needs: required-context-scope diff --git a/.github/workflows/integration-technical-go-security-analysis.yml b/.github/workflows/integration-technical-go-security-analysis.yml index 5b9f28384c..d35a6ea99e 100644 --- a/.github/workflows/integration-technical-go-security-analysis.yml +++ b/.github/workflows/integration-technical-go-security-analysis.yml @@ -128,7 +128,7 @@ jobs: - name: Scan working-directory: src continue-on-error: true - run: '"${{ runner.temp }}/hive-ci-tools/gosec" -exclude-generated ./...' + run: '"${{ runner.temp }}/hive-ci-tools/gosec" -concurrency "${HIVE_CI_GOSEC_CONCURRENCY:-2}" -exclude-generated ./...' golangci-lint: needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} diff --git a/.github/workflows/integration-technical-v2-tests.yml b/.github/workflows/integration-technical-v2-tests.yml index 5c5ef59cc6..e8e640b4b3 100644 --- a/.github/workflows/integration-technical-v2-tests.yml +++ b/.github/workflows/integration-technical-v2-tests.yml @@ -954,10 +954,8 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e if: ${{ steps.cov-scope.outputs.score == 'true' }} with: + cache: false go-version-file: src/go.mod - cache-dependency-path: | - src/go.sum - src/crux/go.sum - name: Download Go modules (retry transient proxy resets) if: ${{ steps.cov-scope.outputs.score == 'true' }} uses: ./.github/actions/go-mod-download-retry diff --git a/.github/workflows/v2-tests.yml b/.github/workflows/v2-tests.yml index 2140960609..ab695843ce 100644 --- a/.github/workflows/v2-tests.yml +++ b/.github/workflows/v2-tests.yml @@ -2084,10 +2084,11 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 if: ${{ steps.cov-scope.outputs.score == 'true' }} with: + # This job only lists packages and scores uploaded profiles. Saving + # the runners' shared multi-GB module/build caches adds a serial + # post-step to the required coverage verdict with no compile reuse. + cache: false go-version-file: src/go.mod - cache-dependency-path: | - src/go.sum - src/crux/go.sum # Fill GOMODCACHE up front with bounded retries so a transient # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the # job's first module-fetching step; see .github/actions/go-mod-download-retry. diff --git a/docs/development.md b/docs/development.md index 6b19a7dc45..1007047dad 100644 --- a/docs/development.md +++ b/docs/development.md @@ -167,6 +167,11 @@ pattern in each file. Go security CI caches the version-pinned gosec binary; the scan itself still runs every time. Gosec's recursive discovery includes nested modules, so its `./...` scan covers Crux without scanning it twice. Other Go tools still need both `./...` and `./crux/...`. +Gosec uses two package-loading workers by default rather than the host CPU +count (which can exceed a runner container's CPU quota); operators can set +`HIVE_CI_GOSEC_CONCURRENCY` in the runner environment to tune it. +The coverage scorer only lists packages and reads shard profiles; it does not +archive shared Go caches in a serial post-step after scoring. Measure PR turnaround from workflow creation to the **last required check** completing, including runner queues and aggregate/coverage jobs, not just From 32fe2056576fce6ddf93a51f1dd17431575679d7 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 31 Oct 2026 07:58:03 -0400 Subject: [PATCH 3/7] =?UTF-8?q?=F0=9F=8C=B1=20ci:=20overlap=20independent?= =?UTF-8?q?=20scope=20and=20package=20discovery=20preflights?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .github/workflows/integration-technical-v2-tests.yml | 3 +-- .github/workflows/v2-tests.yml | 6 ++++-- docs/development.md | 3 +++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-technical-v2-tests.yml b/.github/workflows/integration-technical-v2-tests.yml index e8e640b4b3..463518acf0 100644 --- a/.github/workflows/integration-technical-v2-tests.yml +++ b/.github/workflows/integration-technical-v2-tests.yml @@ -526,8 +526,7 @@ jobs: if-no-files-found: warn retention-days: 1 list-packages: - needs: required-context-scope - if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + if: ${{ !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} runs-on: ubuntu-latest timeout-minutes: 15 outputs: diff --git a/.github/workflows/v2-tests.yml b/.github/workflows/v2-tests.yml index ab695843ce..5c7ecf94fc 100644 --- a/.github/workflows/v2-tests.yml +++ b/.github/workflows/v2-tests.yml @@ -1106,8 +1106,10 @@ jobs: # are all ≤ 5s). The hints only affect BALANCE, never MEMBERSHIP: a stale # weight makes a bucket slower, not a package untested. list-packages: - needs: required-context-scope - if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + # Discovery uses only the checkout/event, not the scope verdict. Start it + # alongside scope instead of paying another runner-queue round trip. + # Shards and both required aggregates still depend on successful scope. + if: ${{ !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 15 outputs: diff --git a/docs/development.md b/docs/development.md index 1007047dad..78dbdfcac3 100644 --- a/docs/development.md +++ b/docs/development.md @@ -172,6 +172,9 @@ count (which can exceed a runner container's CPU quota); operators can set `HIVE_CI_GOSEC_CONCURRENCY` in the runner environment to tune it. The coverage scorer only lists packages and reads shard profiles; it does not archive shared Go caches in a serial post-step after scoring. +Package discovery runs alongside the scope preflight; test shards still wait +for both, and neither test nor coverage can pass a failed scope verdict. +Out-of-scope PRs may perform discovery but still execute no test shards. Measure PR turnaround from workflow creation to the **last required check** completing, including runner queues and aggregate/coverage jobs, not just From 6b79e3fc8a0670a378e8688e47840ace796dc96b Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 31 Oct 2026 08:17:33 -0400 Subject: [PATCH 4/7] =?UTF-8?q?=F0=9F=90=9B=20ci:=20keep=20the=20matrix=20?= =?UTF-8?q?platform=20in=20PR=20image=20freshness=20probes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .github/workflows/docker.yml | 1 + docs/development.md | 3 +++ src/deploy/test_arm64_lane_probes_pr_code.sh | 15 ++++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f1daf7d209..83be9ec4c7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -439,6 +439,7 @@ jobs: else IMG=hive-freshness-probe:local docker buildx build --load \ + --platform "${{ matrix.platform }}" \ -f src/Dockerfile \ --build-arg BASE_REGISTRY="$BASE_REGISTRY" \ --build-arg GIT_HASH="$EXPECTED_SHA" \ diff --git a/docs/development.md b/docs/development.md index 78dbdfcac3..8ba78f9ac1 100644 --- a/docs/development.md +++ b/docs/development.md @@ -175,6 +175,9 @@ archive shared Go caches in a serial post-step after scoring. Package discovery runs alongside the scope preflight; test shards still wait for both, and neither test nor coverage can pass a failed scope verdict. Out-of-scope PRs may perform discovery but still execute no test shards. +The Docker PR freshness reload also keeps the matrix platform, so an ARM64 +job reuses and probes its ARM64 build instead of rebuilding a host-architecture +image. Measure PR turnaround from workflow creation to the **last required check** completing, including runner queues and aggregate/coverage jobs, not just diff --git a/src/deploy/test_arm64_lane_probes_pr_code.sh b/src/deploy/test_arm64_lane_probes_pr_code.sh index e86fd772e8..cf4e688b7e 100755 --- a/src/deploy/test_arm64_lane_probes_pr_code.sh +++ b/src/deploy/test_arm64_lane_probes_pr_code.sh @@ -24,6 +24,7 @@ FAIL=0 HERE="$(cd "$(dirname "$0")" && pwd)" LANE="$(cd "$HERE/../../.github/workflows" && pwd)/podman-arm64-lane.yml" +DOCKER="$(dirname "$LANE")/docker.yml" PROBE="$HERE/probe_arm64_image_startup.sh" ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } @@ -31,10 +32,22 @@ bad() { echo " FAIL: $1"; [ -n "${2:-}" ] && echo " $2"; FAIL=$((FAIL + echo "=== #5370: the arm64 lane probes PR code, not the published image ===" -for f in "$LANE" "$PROBE"; do +for f in "$LANE" "$DOCKER" "$PROBE"; do [ -f "$f" ] || { bad "missing file: $f"; echo; echo "=== $PASS passed, $FAIL failed ==="; exit 1; } done +# The matrix build and its local freshness reload must select the same +# architecture. Otherwise an arm64 job rebuilds/probes amd64 under the host +# default instead of checking the image it just built. +freshness_block="$(awk '/- name: Assert embedded commit matches build/,/^[[:space:]]+# SECURITY/' "$DOCKER")" +if grep -E '^[[:space:]]+--platform ' <<<"$freshness_block" \ + | grep -qF '${{ matrix.platform }}'; then + ok "the Docker freshness reload keeps the matrix platform" +else + bad "the Docker freshness reload does not select the matrix platform" \ + "an arm64 job would rebuild and probe the host architecture instead of its own image" +fi + # ── The lane must build on a pull request ──────────────────────────────── if grep -q "if: github.event_name == 'pull_request'" "$LANE" \ && grep -qE '^\s+podman build' "$LANE"; then From c916e1aee0c095224fa5e04861ba26a7834240c8 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 31 Oct 2026 08:24:03 -0400 Subject: [PATCH 5/7] =?UTF-8?q?=F0=9F=8C=B1=20ci:=20use=20free=20native=20?= =?UTF-8?q?ARM64=20runners=20for=20public=20PR=20images?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .github/workflows/docker.yml | 14 ++++++++------ docs/development.md | 4 ++++ src/deploy/test_arm64_lane_probes_pr_code.sh | 11 +++++++++++ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 83be9ec4c7..d893f44ac8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -279,7 +279,9 @@ jobs: self_hosted: true qemu: true timeout_minutes: 75 - runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} + # Public PRs use the same free native ARM64 VM as the Podman arm64 lane. + # Explicit ARM64 overrides and all release/push runner selection stay intact. + runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || (github.event_name == 'pull_request' && github.event.repository.private == false && '["ubuntu-24.04-arm"]' || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]')) || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} timeout-minutes: ${{ matrix.timeout_minutes }} steps: - name: Checkout repository @@ -293,7 +295,7 @@ jobs: fetch-tags: true - name: Set up QEMU for arm64 emulation - if: matrix.qemu + if: matrix.qemu && runner.arch != 'ARM64' uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: arm64 @@ -682,7 +684,7 @@ jobs: self_hosted: true qemu: true timeout_minutes: 75 - runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} + runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || (github.event_name == 'pull_request' && github.event.repository.private == false && '["ubuntu-24.04-arm"]' || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]')) || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} timeout-minutes: ${{ matrix.timeout_minutes }} steps: - name: Checkout repository @@ -691,7 +693,7 @@ jobs: ref: ${{ needs.gate.outputs.sha }} - name: Set up QEMU for arm64 emulation - if: matrix.qemu + if: matrix.qemu && runner.arch != 'ARM64' uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: arm64 @@ -891,7 +893,7 @@ jobs: self_hosted: true qemu: true timeout_minutes: 75 - runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} + runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || (github.event_name == 'pull_request' && github.event.repository.private == false && '["ubuntu-24.04-arm"]' || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]')) || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} timeout-minutes: ${{ matrix.timeout_minutes }} steps: - name: Checkout repository @@ -905,7 +907,7 @@ jobs: fetch-tags: true - name: Set up QEMU for arm64 emulation - if: matrix.qemu + if: matrix.qemu && runner.arch != 'ARM64' uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: arm64 diff --git a/docs/development.md b/docs/development.md index 8ba78f9ac1..dbe565df2a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -178,6 +178,10 @@ Out-of-scope PRs may perform discovery but still execute no test shards. The Docker PR freshness reload also keeps the matrix platform, so an ARM64 job reuses and probes its ARM64 build instead of rebuilding a host-architecture image. +Public PR image builds default to the free `ubuntu-24.04-arm` native runner +already used by the Podman ARM64 lane; an explicit `HIVE_ARM64_RUNNER` override +still wins. Private repositories and release/push events retain their existing +runner selection, and QEMU is installed only when the selected host needs it. Measure PR turnaround from workflow creation to the **last required check** completing, including runner queues and aggregate/coverage jobs, not just diff --git a/src/deploy/test_arm64_lane_probes_pr_code.sh b/src/deploy/test_arm64_lane_probes_pr_code.sh index cf4e688b7e..5fbd8d04d2 100755 --- a/src/deploy/test_arm64_lane_probes_pr_code.sh +++ b/src/deploy/test_arm64_lane_probes_pr_code.sh @@ -48,6 +48,17 @@ else "an arm64 job would rebuild and probe the host architecture instead of its own image" fi +EXPECTED_DOCKER_ARM64_JOBS=3 +native_pr_runners="$(grep -F 'runs-on:' "$DOCKER" \ + | grep -F "vars.HIVE_ARM64_RUNNER || (github.event_name == 'pull_request' && github.event.repository.private == false" \ + | grep -cF '["ubuntu-24.04-arm"]')" +if [ "$native_pr_runners" -eq "$EXPECTED_DOCKER_ARM64_JOBS" ]; then + ok "all three Docker ARM64 builds default to free native runners only on public PRs" +else + bad "Docker ARM64 native PR runner selection is incomplete" \ + "keep the explicit ARM64 override and existing private/release runner fallback in all three image jobs" +fi + # ── The lane must build on a pull request ──────────────────────────────── if grep -q "if: github.event_name == 'pull_request'" "$LANE" \ && grep -qE '^\s+podman build' "$LANE"; then From fe6beb935be7b93570b5747be3ac5e0cce1c62a5 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 31 Oct 2026 10:52:17 -0400 Subject: [PATCH 6/7] =?UTF-8?q?=F0=9F=90=9B=20ci:=20preserve=20full=20buil?= =?UTF-8?q?d-lane=20fallback=20after=20scope=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .../workflows/integration-technical-v2-ci.yml | 7 +++-- .github/workflows/v2-ci.yml | 8 ++++-- docs/development.md | 4 +++ src/deploy/test_ci_build_lane_aggregate.sh | 26 +++++++++++++++++++ 4 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 src/deploy/test_ci_build_lane_aggregate.sh diff --git a/.github/workflows/integration-technical-v2-ci.yml b/.github/workflows/integration-technical-v2-ci.yml index 8dc21ec57e..b1f041113b 100644 --- a/.github/workflows/integration-technical-v2-ci.yml +++ b/.github/workflows/integration-technical-v2-ci.yml @@ -492,6 +492,9 @@ jobs: - name: Runner labels are fork-portable (#6083) working-directory: . run: bash src/deploy/test_runner_labels_portable.sh + - name: Build aggregate preserves scope fallback and rejects skipped lanes + working-directory: . + run: bash src/deploy/test_ci_build_lane_aggregate.sh build-and-test: needs: - required-context-scope @@ -509,9 +512,9 @@ jobs: CONTRACT_RESULT: ${{ needs.contract-tests.result }} RUNTIME_RESULT: ${{ needs.runtime-tests.result }} run: | - for result in "$SCOPE_RESULT" "$GO_RESULT" "$CONTRACT_RESULT" "$RUNTIME_RESULT"; do + for result in "$GO_RESULT" "$CONTRACT_RESULT" "$RUNTIME_RESULT"; do if [ "$result" != success ]; then - echo "::error::build-and-test requires successful scope, Go, contract and runtime lanes (scope=$SCOPE_RESULT go=$GO_RESULT contracts=$CONTRACT_RESULT runtime=$RUNTIME_RESULT)" + echo "::error::build-and-test requires successful Go, contract and runtime lanes (scope=$SCOPE_RESULT go=$GO_RESULT contracts=$CONTRACT_RESULT runtime=$RUNTIME_RESULT)" exit 1 fi done diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index 50a7006956..ba7d8ea8f5 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -1297,6 +1297,10 @@ jobs: working-directory: . run: bash src/deploy/test_runner_labels_portable.sh + - name: Build aggregate preserves scope fallback and rejects skipped lanes + working-directory: . + run: bash src/deploy/test_ci_build_lane_aggregate.sh + # Preserve the protected check name: every original step still gates it. build-and-test: needs: [required-context-scope, build-go, contract-tests, runtime-tests] @@ -1311,9 +1315,9 @@ jobs: CONTRACT_RESULT: ${{ needs.contract-tests.result }} RUNTIME_RESULT: ${{ needs.runtime-tests.result }} run: | - for result in "$SCOPE_RESULT" "$GO_RESULT" "$CONTRACT_RESULT" "$RUNTIME_RESULT"; do + for result in "$GO_RESULT" "$CONTRACT_RESULT" "$RUNTIME_RESULT"; do if [ "$result" != success ]; then - echo "::error::build-and-test requires successful scope, Go, contract and runtime lanes (scope=$SCOPE_RESULT go=$GO_RESULT contracts=$CONTRACT_RESULT runtime=$RUNTIME_RESULT)" + echo "::error::build-and-test requires successful Go, contract and runtime lanes (scope=$SCOPE_RESULT go=$GO_RESULT contracts=$CONTRACT_RESULT runtime=$RUNTIME_RESULT)" exit 1 fi done diff --git a/docs/development.md b/docs/development.md index dbe565df2a..8b6ed07abb 100644 --- a/docs/development.md +++ b/docs/development.md @@ -160,6 +160,10 @@ Node/runtime contracts as three independent lanes. The protected is cancelled, or unexpectedly skips. This changes scheduling, not which contracts run. The separate race/shuffle matrix and per-package coverage checks remain required and unchanged. +As before, a transient build scope-preflight failure falls back to executing +the full actual build/contract/runtime lanes. Scope failure is not itself a +new build gate: all three actual lanes must succeed, and skipped or cancelled +lanes cannot create a green aggregate. The Podman cleanup contract reads every tracked file once, checking all forbidden literal patterns together rather than spawning a grep for each diff --git a/src/deploy/test_ci_build_lane_aggregate.sh b/src/deploy/test_ci_build_lane_aggregate.sh new file mode 100644 index 0000000000..a5c23b75e7 --- /dev/null +++ b/src/deploy/test_ci_build_lane_aggregate.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +python3 - <<'PY' +from pathlib import Path +import os +import re +import subprocess + +workflow = Path(".github/workflows/v2-ci.yml").read_text() +aggregate = workflow.split("\n build-and-test:\n", 1)[1].split("\n create-issue-on-failure:", 1)[0] +assert "!cancelled()" in aggregate, "cancelled workflows must not become green" +body = aggregate.split(" run: |\n", 1)[1] +body = "\n".join(line[10:] for line in body.splitlines() if line.strip()) +lanes = ("GO_RESULT", "CONTRACT_RESULT", "RUNTIME_RESULT") +for scope in ("success", "failure"): + env = dict(os.environ, SCOPE_RESULT=scope, **dict.fromkeys(lanes, "success")) + assert subprocess.run(["bash", "-c", body], env=env, capture_output=True).returncode == 0 + for lane in lanes: + for state in ("failure", "skipped", "cancelled"): + failing = dict(env, **{lane: state}) + assert subprocess.run(["bash", "-c", body], env=failing, capture_output=True).returncode != 0, (scope, lane, state) +for lane in ("build-go", "contract-tests", "runtime-tests"): + job = re.split(r"\n (?=\S)", workflow.split(f"\n {lane}:\n", 1)[1])[0] + assert "needs.required-context-scope.result != 'success'" in job, lane +print("build aggregate: full-lane scope fallback; failed/skipped/cancelled lanes rejected") +PY From ca4290aa8d8f25f031f3419c0f5a285bb91f082d Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Sat, 31 Oct 2026 11:11:42 -0400 Subject: [PATCH 7/7] =?UTF-8?q?=F0=9F=90=9B=20fix:=20exercise=20checkout?= =?UTF-8?q?=20provenance=20on=20actual=20build=20producer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- .github/scripts/test-generate-integration-ci.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/test-generate-integration-ci.py b/.github/scripts/test-generate-integration-ci.py index ca0e2b05f4..58dabe55ca 100644 --- a/.github/scripts/test-generate-integration-ci.py +++ b/.github/scripts/test-generate-integration-ci.py @@ -48,6 +48,7 @@ def test_unqualified_producer_permissions_or_context_drift_cannot_generate_green elif mutation == "missing": job["name"] = "not-the-required-context" elif mutation == "checkout": + job = documents["v2-ci.yml"]["jobs"]["build-go"] checkout = next(step for step in job["steps"] if str(step.get("uses", "")).startswith("actions/checkout@")) checkout.setdefault("with", {})["ref"] = "other-head" else: