diff --git a/.github/scripts/test-generate-integration-ci.py b/.github/scripts/test-generate-integration-ci.py index ca0e2b05f4..58dabe55ca 100644 --- a/.github/scripts/test-generate-integration-ci.py +++ b/.github/scripts/test-generate-integration-ci.py @@ -48,6 +48,7 @@ def test_unqualified_producer_permissions_or_context_drift_cannot_generate_green elif mutation == "missing": job["name"] = "not-the-required-context" elif mutation == "checkout": + job = documents["v2-ci.yml"]["jobs"]["build-go"] checkout = next(step for step in job["steps"] if str(step.get("uses", "")).startswith("actions/checkout@")) checkout.setdefault("with", {})["ref"] = "other-head" else: diff --git a/.github/scripts/test-native-queue-workflows.py b/.github/scripts/test-native-queue-workflows.py index e48c2c83a1..1268630f98 100644 --- a/.github/scripts/test-native-queue-workflows.py +++ b/.github/scripts/test-native-queue-workflows.py @@ -74,20 +74,22 @@ def test_gosec_concurrency_fits_the_runner_cpu_limit(self): # gosec defaults -concurrency to runtime.NumCPU(), which inside a # runner pod is the NODE's vCPU count, not the pod's cgroup limit. # Unbounded, every worker type-checks and builds SSA for its own - # package at once, and the pod is OOM-killed or evicted mid-Scan: - # "runner lost communication", hivecommons/hive#16040. The pod CPU - # limit is a ceiling, not a memory proof: it only guarantees gosec - # never runs more workers than the pod can schedule. + # package at once, a plausible source of runner memory pressure + # in hivecommons/hive#16040. The pod CPU limit is a ceiling, not an + # OOM diagnosis or memory proof: it only bounds worker fanout. values = yaml.safe_load( (ROOT / "src/deploy/ci-runners/lke/hive-runners-lke-values.yaml").read_text()) runner = next(c for c in values["template"]["spec"]["containers"] if c["name"] == "runner") cpu = str(runner["resources"]["limits"]["cpu"]) cpu_limit = int(cpu[:-1]) // 1000 if cpu.endswith("m") else int(float(cpu)) - for file in ("go-security-analysis.yml", - "integration-technical-go-security-analysis.yml"): + for file, job in ( + ("go-security-analysis.yml", "gosec"), + ("integration-technical-go-security-analysis.yml", "gosec"), + ("integration-candidate.yml", "security-gosec"), + ): with self.subTest(file=file): - steps = workflow(file)["jobs"]["gosec"]["steps"] + steps = workflow(file)["jobs"][job]["steps"] scan = next(s for s in steps if s.get("name") == "Scan")["run"] match = re.search(r"\bgosec\b.*\s-concurrency[ =](\d+)\b", scan) self.assertIsNotNone(match, f"gosec must pass an explicit -concurrency: {scan!r}") diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f1daf7d209..d893f44ac8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -279,7 +279,9 @@ jobs: self_hosted: true qemu: true timeout_minutes: 75 - runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} + # Public PRs use the same free native ARM64 VM as the Podman arm64 lane. + # Explicit ARM64 overrides and all release/push runner selection stay intact. + runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || (github.event_name == 'pull_request' && github.event.repository.private == false && '["ubuntu-24.04-arm"]' || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]')) || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} timeout-minutes: ${{ matrix.timeout_minutes }} steps: - name: Checkout repository @@ -293,7 +295,7 @@ jobs: fetch-tags: true - name: Set up QEMU for arm64 emulation - if: matrix.qemu + if: matrix.qemu && runner.arch != 'ARM64' uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: arm64 @@ -439,6 +441,7 @@ jobs: else IMG=hive-freshness-probe:local docker buildx build --load \ + --platform "${{ matrix.platform }}" \ -f src/Dockerfile \ --build-arg BASE_REGISTRY="$BASE_REGISTRY" \ --build-arg GIT_HASH="$EXPECTED_SHA" \ @@ -681,7 +684,7 @@ jobs: self_hosted: true qemu: true timeout_minutes: 75 - runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} + runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || (github.event_name == 'pull_request' && github.event.repository.private == false && '["ubuntu-24.04-arm"]' || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]')) || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} timeout-minutes: ${{ matrix.timeout_minutes }} steps: - name: Checkout repository @@ -690,7 +693,7 @@ jobs: ref: ${{ needs.gate.outputs.sha }} - name: Set up QEMU for arm64 emulation - if: matrix.qemu + if: matrix.qemu && runner.arch != 'ARM64' uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: arm64 @@ -890,7 +893,7 @@ jobs: self_hosted: true qemu: true timeout_minutes: 75 - runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} + runs-on: ${{ matrix.qemu && fromJSON(vars.HIVE_ARM64_RUNNER || (github.event_name == 'pull_request' && github.event.repository.private == false && '["ubuntu-24.04-arm"]' || vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]')) || ((github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest') }} timeout-minutes: ${{ matrix.timeout_minutes }} steps: - name: Checkout repository @@ -904,7 +907,7 @@ jobs: fetch-tags: true - name: Set up QEMU for arm64 emulation - if: matrix.qemu + if: matrix.qemu && runner.arch != 'ARM64' uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 with: platforms: arm64 diff --git a/.github/workflows/go-security-analysis.yml b/.github/workflows/go-security-analysis.yml index 3aa7507434..2b7294d1eb 100644 --- a/.github/workflows/go-security-analysis.yml +++ b/.github/workflows/go-security-analysis.yml @@ -232,7 +232,16 @@ jobs: # v0.25.0, which does not COMPILE under Go 1.25 ("invalid array length"). # The failure looks like a scan failure but is a build failure in the # tool itself — do not downgrade to "fix" a finding. + - name: Restore pinned gosec binary + id: gosec-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/hive-ci-tools/gosec + key: hive-gosec-${{ runner.os }}-${{ runner.arch }}-v2.29.0-${{ hashFiles('src/go.mod') }} - name: Install gosec + if: steps.gosec-cache.outputs.cache-hit != 'true' + env: + GOBIN: ${{ runner.temp }}/hive-ci-tools run: go install github.com/securego/gosec/v2/cmd/gosec@v2.29.0 # -exclude-generated: generated files are not hand-maintained, so a @@ -256,14 +265,17 @@ jobs: # -concurrency 4: gosec defaults to runtime.NumCPU(), which in a runner # pod is the NODE's 32 vCPUs, not the pod's 6-CPU limit # (src/deploy/ci-runners/lke/hive-runners-lke-values.yaml). Each worker - # type-checks and builds SSA for its own package, so 32 at once pushed - # the pod far past its 3Gi request and it was killed mid-Scan — the - # "runner lost communication" failures of #16040. Keep this at or below - # the runner CPU limit (test-native-queue-workflows.py enforces it). + # type-checks and builds SSA for its own package, so 32 at once can + # create memory pressure. This bounds a plausible contributor to the + # "runner lost communication" failures of #16040, not a proven OOM + # diagnosis. Keep this at or below the runner CPU limit + # (test-native-queue-workflows.py enforces it). - name: Scan working-directory: src continue-on-error: true - run: gosec -concurrency 4 -exclude-generated ./... ./crux/... + # Unlike go test/vet, gosec recursively discovers nested modules. + # Listing ./crux/... too scans every Crux directory twice. + run: '"${{ runner.temp }}/hive-ci-tools/gosec" -concurrency 4 -exclude-generated ./...' golangci-lint: needs: required-context-scope diff --git a/.github/workflows/integration-candidate.yml b/.github/workflows/integration-candidate.yml index abfa20723b..2241179826 100644 --- a/.github/workflows/integration-candidate.yml +++ b/.github/workflows/integration-candidate.yml @@ -202,7 +202,7 @@ jobs: env: ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} - ci-build-and-test: + ci-build-go: needs: - subject - ci-required-context-scope @@ -280,6 +280,31 @@ jobs: run: go vet ./... ./crux/... - name: Price table staleness guard run: go run ./internal/tools/pricetable --check-age 60 + name: build-go + env: + ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' + GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} + ci-contract-tests: + needs: + - subject + - ci-required-context-scope + if: ${{ always() && needs.subject.result == 'success' && (always() && (needs.ci-required-context-scope.result != 'success' || needs.ci-required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/'))) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } - name: Entrypoint runtime-config migration tests run: bash deploy/test_entrypoint_runtime_config.sh - name: Entrypoint NET_ADMIN ambient-cap tests (#3760) @@ -440,6 +465,31 @@ jobs: - name: Every bin/ test suite is wired into something (#4363) working-directory: . run: bash bin/test_bin_suites_wired.sh + name: contract-tests + env: + ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' + GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} + ci-runtime-tests: + needs: + - subject + - ci-required-context-scope + if: ${{ always() && needs.subject.result == 'success' && (always() && (needs.ci-required-context-scope.result != 'success' || needs.ci-required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/'))) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 with: node-version: '22' @@ -590,6 +640,37 @@ jobs: - name: Runner labels are fork-portable (#6083) working-directory: . run: bash src/deploy/test_runner_labels_portable.sh + - name: Build aggregate preserves scope fallback and rejects skipped lanes + working-directory: . + run: bash src/deploy/test_ci_build_lane_aggregate.sh + name: runtime-tests + env: + ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' + GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} + ci-build-and-test: + needs: + - subject + - ci-required-context-scope + - ci-build-go + - ci-contract-tests + - ci-runtime-tests + if: ${{ always() && needs.subject.result == 'success' && (always() && !cancelled() && (needs.ci-required-context-scope.result != 'success' || needs.ci-required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/'))) }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Require every build and contract lane + env: + SCOPE_RESULT: ${{ needs.ci-required-context-scope.result }} + GO_RESULT: ${{ needs.ci-build-go.result }} + CONTRACT_RESULT: ${{ needs.ci-contract-tests.result }} + RUNTIME_RESULT: ${{ needs.ci-runtime-tests.result }} + run: | + for result in "$GO_RESULT" "$CONTRACT_RESULT" "$RUNTIME_RESULT"; do + if [ "$result" != success ]; then + echo "::error::build-and-test requires successful Go, contract and runtime lanes (scope=$SCOPE_RESULT go=$GO_RESULT contracts=$CONTRACT_RESULT runtime=$RUNTIME_RESULT)" + exit 1 + fi + done name: build-and-test env: ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' @@ -1224,10 +1305,7 @@ jobs: ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} tests-list-packages: - needs: - - subject - - tests-required-context-scope - if: ${{ always() && needs.subject.result == 'success' && ((needs.tests-required-context-scope.result != 'success' || needs.tests-required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/'))) }} + if: ${{ always() && needs.subject.result == 'success' && (!(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/'))) }} runs-on: ubuntu-latest timeout-minutes: 15 outputs: @@ -1300,6 +1378,8 @@ jobs: - name: Partition packages into balanced buckets id: partition run: "R=4\n# Drop only the monolithic packages that have function-sliced jobs\n# of their own: pkg/hub (test-hub), pkg/dashboard and pkg/scheduler\n# (test-sliced). All their subpackages belong in the rest buckets\n# so coverage floors and the no-test-files gate apply to them too.\n# pkg/hubbackup, pkg/dashboard/webstatic et al. stay here.\n# ./internal/... carries the shared test helpers (internal/testutil)\n# and their contract tests, such as the time.Sleep ratchet in\n# sleep_ratchet_test.go. ./crux/... is the nested Crux module\n# (#13536), reachable from src/ through src/go.work; its manager\n# package crux/agent has the dedicated agent slices.\ngo list ./pkg/... ./cmd/... ./internal/... ./crux/... \\\n | awk '$0 == \"github.com/hivecommons/hive/pkg/hub\" { next }\n $0 == \"github.com/hivecommons/hive/pkg/dashboard\" { next }\n $0 == \"github.com/hivecommons/hive/pkg/scheduler\" { next }\n { print }' \\\n | grep -Ev '^github\\.com/hivecommons/hive/crux/agent$' \\\n | LC_ALL=C sort > rest-pkgs.txt\nif [ ! -s rest-pkgs.txt ]; then\n echo \"::error::go list produced no rest packages \u2014 failing closed\"\n exit 1\nfi\n# Affected-package selection narrows membership only after the\n# fail-closed check above ran on the full list. The bucket count\n# stays R, so every `test (rest i/R)` context still reports; a\n# bucket left empty succeeds without running (see test-rest).\nif [ \"$(head -n1 ../affected-pkgs.txt)\" != all ]; then\n grep -Fx -f ../affected-pkgs.txt rest-pkgs.txt > rest-selected.txt || true\n echo \"affected-package selection: $(wc -l < rest-selected.txt) of $(wc -l < rest-pkgs.txt) rest packages\"\n mv rest-selected.txt rest-pkgs.txt\nfi\n\n# Weights = measured -short -race package wall seconds on the\n# hive-runners fleet, mean of two v5 push runs (40383615331 and\n# 40382244011, 2026-10-26, rounded). Re-measure with: for each\n# `test (rest i/R)` job log, grep the\n# `ok github.com/hivecommons/hive/ s` lines. Anything\n# under ~5s is noise and takes the 1s default. Packages inside a\n# bucket run concurrently under one go test, so a bucket's wall\n# clock is its heaviest package plus compile time \u2014 the table\n# mainly has to keep the big ones apart, and the sum-based greedy\n# below does that while spreading the compile load. pkg/hub/spoke\n# is the floor now that pkg/dashboard and pkg/scheduler are sliced\n# (test-sliced above): the widest bucket can never be faster than\n# it. pkg/dashboard and pkg/scheduler are listed so a stale\n# exclusion above still yields a sane partition. pkg/github is\n# ~1s with ~1% coverage on push runs but 70s on pull requests (PR\n# 14407's run 40387365403), where it is weighed \u2014 measure it from a\n# PR run, not a push run.\nweight() {\n case \"$1\" in\n */pkg/dashboard) echo 219 ;;\n */pkg/scheduler) echo 199 ;;\n */pkg/hub/spoke) echo 85 ;;\n */pkg/config) echo 82 ;;\n */pkg/github) echo 70 ;;\n */cmd/hive) echo 69 ;;\n */crux/proxy) echo 56 ;;\n */pkg/dashboard/webstatic) echo 35 ;;\n */pkg/hubbackup) echo 30 ;;\n */pkg/advisory) echo 29 ;;\n */internal/testutil) echo 22 ;;\n */pkg/ioscan) echo 16 ;;\n */pkg/hivectl) echo 15 ;;\n */pkg/tui) echo 14 ;;\n */pkg/discord) echo 12 ;;\n */pkg/fleetreport) echo 11 ;;\n */pkg/hivectl/commands) echo 10 ;;\n */pkg/stageengine) echo 10 ;;\n */pkg/apphealth) echo 9 ;;\n */pkg/rotation) echo 9 ;;\n */pkg/beads) echo 8 ;;\n */pkg/github/automerge) echo 8 ;;\n */crux/api/callback) echo 6 ;;\n */crux/inference) echo 6 ;;\n */cmd/hive-backup) echo 5 ;;\n */pkg/mint) echo 5 ;;\n */pkg/policies) echo 5 ;;\n *) echo 1 ;;\n esac\n}\n\n# Deterministic greedy LPT: stable-sort by weight desc (name asc on\n# ties), assign each package to the currently lightest bucket\n# (lowest index on ties).\nwhile read -r p; do\n echo \"$(weight \"$p\") $p\"\ndone < rest-pkgs.txt | sort -k1,1nr -k2,2 > weighted.txt\n\ndeclare -a LOAD BUCKET\nfor ((i=0; i> \"$GITHUB_OUTPUT\"\n" + needs: + - subject name: list-packages env: CGO_ENABLED: '1' @@ -1684,10 +1764,8 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e if: ${{ steps.cov-scope.outputs.score == 'true' }} with: + cache: false go-version-file: src/go.mod - cache-dependency-path: | - src/go.sum - src/crux/go.sum - name: Download Go modules (retry transient proxy resets) if: ${{ steps.cov-scope.outputs.score == 'true' }} uses: ./.github/actions/go-mod-download-retry @@ -1851,12 +1929,21 @@ jobs: run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" - name: Download Go modules (retry transient proxy resets) uses: ./.github/actions/go-mod-download-retry + - name: Restore pinned gosec binary + id: gosec-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ runner.temp }}/hive-ci-tools/gosec + key: hive-gosec-${{ runner.os }}-${{ runner.arch }}-v2.29.0-${{ hashFiles('src/go.mod') }} - name: Install gosec + if: steps.gosec-cache.outputs.cache-hit != 'true' + env: + GOBIN: ${{ runner.temp }}/hive-ci-tools run: go install github.com/securego/gosec/v2/cmd/gosec@v2.29.0 - name: Scan working-directory: src continue-on-error: true - run: gosec -concurrency 4 -exclude-generated ./... ./crux/... + run: '"${{ runner.temp }}/hive-ci-tools/gosec" -concurrency 4 -exclude-generated ./...' env: GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} security-golangci-lint: @@ -2081,6 +2168,9 @@ jobs: needs: - subject - ci-required-context-scope + - ci-build-go + - ci-contract-tests + - ci-runtime-tests - ci-build-and-test - ci-docker - ci-overlayfs-exec-guard diff --git a/.github/workflows/integration-technical-go-security-analysis.yml b/.github/workflows/integration-technical-go-security-analysis.yml index c78faa8b7c..20faf4537a 100644 --- a/.github/workflows/integration-technical-go-security-analysis.yml +++ b/.github/workflows/integration-technical-go-security-analysis.yml @@ -114,12 +114,21 @@ jobs: run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" - name: Download Go modules (retry transient proxy resets) uses: ./.github/actions/go-mod-download-retry + - name: Restore pinned gosec binary + id: gosec-cache + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ${{ runner.temp }}/hive-ci-tools/gosec + key: hive-gosec-${{ runner.os }}-${{ runner.arch }}-v2.29.0-${{ hashFiles('src/go.mod') }} - name: Install gosec + if: steps.gosec-cache.outputs.cache-hit != 'true' + env: + GOBIN: ${{ runner.temp }}/hive-ci-tools run: go install github.com/securego/gosec/v2/cmd/gosec@v2.29.0 - name: Scan working-directory: src continue-on-error: true - run: gosec -concurrency 4 -exclude-generated ./... ./crux/... + run: '"${{ runner.temp }}/hive-ci-tools/gosec" -concurrency 4 -exclude-generated ./...' golangci-lint: needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} diff --git a/.github/workflows/integration-technical-v2-ci.yml b/.github/workflows/integration-technical-v2-ci.yml index f63aed8f28..44fe2bdd66 100644 --- a/.github/workflows/integration-technical-v2-ci.yml +++ b/.github/workflows/integration-technical-v2-ci.yml @@ -70,7 +70,7 @@ jobs: else echo "No v2 CI paths changed; build-and-test will be reported as skipped/success." fi - build-and-test: + build-go: needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} runs-on: ubuntu-latest @@ -146,6 +146,25 @@ jobs: run: go vet ./... ./crux/... - name: Price table staleness guard run: go run ./internal/tools/pricetable --check-age 60 + contract-tests: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } - name: Entrypoint runtime-config migration tests run: bash deploy/test_entrypoint_runtime_config.sh - name: Entrypoint NET_ADMIN ambient-cap tests (#3760) @@ -306,6 +325,25 @@ jobs: - name: Every bin/ test suite is wired into something (#4363) working-directory: . run: bash bin/test_bin_suites_wired.sh + runtime-tests: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 with: node-version: '22' @@ -456,6 +494,32 @@ jobs: - name: Runner labels are fork-portable (#6083) working-directory: . run: bash src/deploy/test_runner_labels_portable.sh + - name: Build aggregate preserves scope fallback and rejects skipped lanes + working-directory: . + run: bash src/deploy/test_ci_build_lane_aggregate.sh + build-and-test: + needs: + - required-context-scope + - build-go + - contract-tests + - runtime-tests + if: ${{ always() && !cancelled() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Require every build and contract lane + env: + SCOPE_RESULT: ${{ needs.required-context-scope.result }} + GO_RESULT: ${{ needs.build-go.result }} + CONTRACT_RESULT: ${{ needs.contract-tests.result }} + RUNTIME_RESULT: ${{ needs.runtime-tests.result }} + run: | + for result in "$GO_RESULT" "$CONTRACT_RESULT" "$RUNTIME_RESULT"; do + if [ "$result" != success ]; then + echo "::error::build-and-test requires successful Go, contract and runtime lanes (scope=$SCOPE_RESULT go=$GO_RESULT contracts=$CONTRACT_RESULT runtime=$RUNTIME_RESULT)" + exit 1 + fi + done docker: name: docker (${{ matrix.name }}) needs: required-context-scope diff --git a/.github/workflows/integration-technical-v2-tests.yml b/.github/workflows/integration-technical-v2-tests.yml index 5c5ef59cc6..463518acf0 100644 --- a/.github/workflows/integration-technical-v2-tests.yml +++ b/.github/workflows/integration-technical-v2-tests.yml @@ -526,8 +526,7 @@ jobs: if-no-files-found: warn retention-days: 1 list-packages: - needs: required-context-scope - if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + if: ${{ !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} runs-on: ubuntu-latest timeout-minutes: 15 outputs: @@ -954,10 +953,8 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e if: ${{ steps.cov-scope.outputs.score == 'true' }} with: + cache: false go-version-file: src/go.mod - cache-dependency-path: | - src/go.sum - src/crux/go.sum - name: Download Go modules (retry transient proxy resets) if: ${{ steps.cov-scope.outputs.score == 'true' }} uses: ./.github/actions/go-mod-download-retry diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index ffab7567b9..31cd94ba3e 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -120,7 +120,7 @@ jobs: echo "No v2 CI paths changed; build-and-test will be reported as skipped/success." fi - build-and-test: + build-go: needs: required-context-scope # release-gate/* is created only by tagged-release.yml and carries only CHANGELOG.md/changelog.d changes; report the required context immediately (#10863). if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} @@ -243,6 +243,26 @@ jobs: - name: Price table staleness guard run: go run ./internal/tools/pricetable --check-age 60 + contract-tests: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } + # The entrypoint decides which file a hive boots its config from. The # hive.yaml.bak -> hive.yaml.runtime migration must keep reading the # legacy name, or hives that have not saved since upgrading lose their @@ -874,6 +894,25 @@ jobs: working-directory: . run: bash bin/test_bin_suites_wired.sh + runtime-tests: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} + - name: Ensure PyYAML on self-hosted runners + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 with: node-version: '22' @@ -1259,6 +1298,32 @@ jobs: - name: Runner labels are fork-portable (#6083) working-directory: . run: bash src/deploy/test_runner_labels_portable.sh + + - name: Build aggregate preserves scope fallback and rejects skipped lanes + working-directory: . + run: bash src/deploy/test_ci_build_lane_aggregate.sh + + # Preserve the protected check name: every original step still gates it. + build-and-test: + needs: [required-context-scope, build-go, contract-tests, runtime-tests] + if: ${{ always() && !cancelled() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + timeout-minutes: 10 + steps: + - name: Require every build and contract lane + env: + SCOPE_RESULT: ${{ needs.required-context-scope.result }} + GO_RESULT: ${{ needs.build-go.result }} + CONTRACT_RESULT: ${{ needs.contract-tests.result }} + RUNTIME_RESULT: ${{ needs.runtime-tests.result }} + run: | + for result in "$GO_RESULT" "$CONTRACT_RESULT" "$RUNTIME_RESULT"; do + if [ "$result" != success ]; then + echo "::error::build-and-test requires successful Go, contract and runtime lanes (scope=$SCOPE_RESULT go=$GO_RESULT contracts=$CONTRACT_RESULT runtime=$RUNTIME_RESULT)" + exit 1 + fi + done + create-issue-on-failure: if: "!inputs.integration_candidate && failure() && github.event_name == 'push' && github.ref == 'refs/heads/v2'" runs-on: ${{ fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') }} diff --git a/.github/workflows/v2-tests.yml b/.github/workflows/v2-tests.yml index 2140960609..5c7ecf94fc 100644 --- a/.github/workflows/v2-tests.yml +++ b/.github/workflows/v2-tests.yml @@ -1106,8 +1106,10 @@ jobs: # are all ≤ 5s). The hints only affect BALANCE, never MEMBERSHIP: a stale # weight makes a bucket slower, not a package untested. list-packages: - needs: required-context-scope - if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + # Discovery uses only the checkout/event, not the scope verdict. Start it + # alongside scope instead of paying another runner-queue round trip. + # Shards and both required aggregates still depend on successful scope. + if: ${{ !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 15 outputs: @@ -2084,10 +2086,11 @@ jobs: - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 if: ${{ steps.cov-scope.outputs.score == 'true' }} with: + # This job only lists packages and scores uploaded profiles. Saving + # the runners' shared multi-GB module/build caches adds a serial + # post-step to the required coverage verdict with no compile reuse. + cache: false go-version-file: src/go.mod - cache-dependency-path: | - src/go.sum - src/crux/go.sum # Fill GOMODCACHE up front with bounded retries so a transient # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the # job's first module-fetching step; see .github/actions/go-mod-download-retry. diff --git a/bin/test_hive_podman_cleanup.sh b/bin/test_hive_podman_cleanup.sh index a628f6889a..d5fab10fc1 100755 --- a/bin/test_hive_podman_cleanup.sh +++ b/bin/test_hive_podman_cleanup.sh @@ -237,23 +237,35 @@ exempt=( ) if command -v git >/dev/null 2>&1 && git -C "$ROOT" rev-parse --git-dir >/dev/null 2>&1; then - offenders="" - while IFS= read -r tracked; do - skip="" - for exempt_path in "${exempt[@]}"; do - [[ "$tracked" == "$exempt_path" ]] && skip="yes" - done - [[ -n "$skip" ]] && continue - [[ -f "${ROOT}/${tracked}" ]] || continue - - for pattern in "${broad_patterns[@]}"; do - if grep -qF -- "$pattern" "${ROOT}/${tracked}" 2>/dev/null; then - offenders+="${tracked}: ${pattern}"$'\n' - fi - done - done < <(git -C "$ROOT" ls-files) - - [[ -z "$offenders" ]] || fail "broad Podman/Buildah cleanup found:"$'\n'"${offenders}" + # Read each tracked file once instead of spawning one grep per pattern/file + # (over 100,000 processes on the current tree). Keep literal byte matching, + # symlink handling, and all existing exemptions, including binary files. + python3 - "$ROOT" "${#broad_patterns[@]}" "${broad_patterns[@]}" "${exempt[@]}" <<'PYSCAN' +import os +import subprocess +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +pattern_count = int(sys.argv[2]) +patterns = [pattern.encode() for pattern in sys.argv[3:3 + pattern_count]] +exempt = set(sys.argv[3 + pattern_count:]) +tracked = subprocess.check_output(["git", "-C", str(root), "ls-files", "-z"]) +offenders = [] +for name in tracked.split(b"\0"): + if not name: + continue + relative = os.fsdecode(name) + path = root / relative + if relative in exempt or not path.is_file(): + continue + contents = path.read_bytes() + for pattern in patterns: + if pattern in contents: + offenders.append(f"{relative}: {pattern.decode()}") +if offenders: + sys.exit("FAIL: broad Podman/Buildah cleanup found:\n" + "\n".join(offenders)) +PYSCAN pass_count=$((pass_count + 1)) fi diff --git a/docs/development.md b/docs/development.md index 47159e4970..4001e7a546 100644 --- a/docs/development.md +++ b/docs/development.md @@ -152,6 +152,47 @@ Two flags that appear in CI but are *not* part of the PR gate: report. Coverage is scored, but it is not the required merge gate; a failing test is. +### CI wall clock and required checks + +The `v2 CI` workflow runs Go build/vet, deployment and shell contracts, and +Node/runtime contracts as three independent lanes. The protected +`build-and-test` check aggregates **all three** and fails if any lane fails, +is cancelled, or unexpectedly skips. This changes scheduling, not which +contracts run. The separate race/shuffle matrix and per-package coverage +checks remain required and unchanged. +As before, a transient build scope-preflight failure falls back to executing +the full actual build/contract/runtime lanes. Scope failure is not itself a +new build gate: all three actual lanes must succeed, and skipped or cancelled +lanes cannot create a green aggregate. + +The Podman cleanup contract reads every tracked file once, checking all +forbidden literal patterns together rather than spawning a grep for each +pattern in each file. Go security CI caches the version-pinned gosec binary; +the scan itself still runs every time. Gosec's recursive discovery includes +nested modules, so its `./...` scan covers Crux without scanning it twice. +Other Go tools still need both `./...` and `./crux/...`. +Gosec retains the four package-loading workers established by #16045 rather +than the host CPU count (which can exceed a runner container's CPU quota). +The structural contract checks this explicit bound against the runner limit. +The coverage scorer only lists packages and reads shard profiles; it does not +archive shared Go caches in a serial post-step after scoring. +Package discovery runs alongside the scope preflight; test shards still wait +for both, and neither test nor coverage can pass a failed scope verdict. +Out-of-scope PRs may perform discovery but still execute no test shards. +The Docker PR freshness reload also keeps the matrix platform, so an ARM64 +job reuses and probes its ARM64 build instead of rebuilding a host-architecture +image. +Public PR image builds default to the free `ubuntu-24.04-arm` native runner +already used by the Podman ARM64 lane; an explicit `HIVE_ARM64_RUNNER` override +still wins. Private repositories and release/push events retain their existing +runner selection, and QEMU is installed only when the selected host needs it. + +Measure PR turnaround from workflow creation to the **last required check** +completing, including runner queues and aggregate/coverage jobs, not just +the slowest test shard's execution. Cache misses and shared runner load +can change this substantially; a short shard alone is not a five-minute +pipeline. + ### Affected-package selection on pull requests On a pull request the gate does not always run every package. The diff --git a/src/deploy/test_arm64_lane_probes_pr_code.sh b/src/deploy/test_arm64_lane_probes_pr_code.sh index e86fd772e8..5fbd8d04d2 100755 --- a/src/deploy/test_arm64_lane_probes_pr_code.sh +++ b/src/deploy/test_arm64_lane_probes_pr_code.sh @@ -24,6 +24,7 @@ FAIL=0 HERE="$(cd "$(dirname "$0")" && pwd)" LANE="$(cd "$HERE/../../.github/workflows" && pwd)/podman-arm64-lane.yml" +DOCKER="$(dirname "$LANE")/docker.yml" PROBE="$HERE/probe_arm64_image_startup.sh" ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } @@ -31,10 +32,33 @@ bad() { echo " FAIL: $1"; [ -n "${2:-}" ] && echo " $2"; FAIL=$((FAIL + echo "=== #5370: the arm64 lane probes PR code, not the published image ===" -for f in "$LANE" "$PROBE"; do +for f in "$LANE" "$DOCKER" "$PROBE"; do [ -f "$f" ] || { bad "missing file: $f"; echo; echo "=== $PASS passed, $FAIL failed ==="; exit 1; } done +# The matrix build and its local freshness reload must select the same +# architecture. Otherwise an arm64 job rebuilds/probes amd64 under the host +# default instead of checking the image it just built. +freshness_block="$(awk '/- name: Assert embedded commit matches build/,/^[[:space:]]+# SECURITY/' "$DOCKER")" +if grep -E '^[[:space:]]+--platform ' <<<"$freshness_block" \ + | grep -qF '${{ matrix.platform }}'; then + ok "the Docker freshness reload keeps the matrix platform" +else + bad "the Docker freshness reload does not select the matrix platform" \ + "an arm64 job would rebuild and probe the host architecture instead of its own image" +fi + +EXPECTED_DOCKER_ARM64_JOBS=3 +native_pr_runners="$(grep -F 'runs-on:' "$DOCKER" \ + | grep -F "vars.HIVE_ARM64_RUNNER || (github.event_name == 'pull_request' && github.event.repository.private == false" \ + | grep -cF '["ubuntu-24.04-arm"]')" +if [ "$native_pr_runners" -eq "$EXPECTED_DOCKER_ARM64_JOBS" ]; then + ok "all three Docker ARM64 builds default to free native runners only on public PRs" +else + bad "Docker ARM64 native PR runner selection is incomplete" \ + "keep the explicit ARM64 override and existing private/release runner fallback in all three image jobs" +fi + # ── The lane must build on a pull request ──────────────────────────────── if grep -q "if: github.event_name == 'pull_request'" "$LANE" \ && grep -qE '^\s+podman build' "$LANE"; then diff --git a/src/deploy/test_ci_build_lane_aggregate.sh b/src/deploy/test_ci_build_lane_aggregate.sh new file mode 100644 index 0000000000..a5c23b75e7 --- /dev/null +++ b/src/deploy/test_ci_build_lane_aggregate.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +python3 - <<'PY' +from pathlib import Path +import os +import re +import subprocess + +workflow = Path(".github/workflows/v2-ci.yml").read_text() +aggregate = workflow.split("\n build-and-test:\n", 1)[1].split("\n create-issue-on-failure:", 1)[0] +assert "!cancelled()" in aggregate, "cancelled workflows must not become green" +body = aggregate.split(" run: |\n", 1)[1] +body = "\n".join(line[10:] for line in body.splitlines() if line.strip()) +lanes = ("GO_RESULT", "CONTRACT_RESULT", "RUNTIME_RESULT") +for scope in ("success", "failure"): + env = dict(os.environ, SCOPE_RESULT=scope, **dict.fromkeys(lanes, "success")) + assert subprocess.run(["bash", "-c", body], env=env, capture_output=True).returncode == 0 + for lane in lanes: + for state in ("failure", "skipped", "cancelled"): + failing = dict(env, **{lane: state}) + assert subprocess.run(["bash", "-c", body], env=failing, capture_output=True).returncode != 0, (scope, lane, state) +for lane in ("build-go", "contract-tests", "runtime-tests"): + job = re.split(r"\n (?=\S)", workflow.split(f"\n {lane}:\n", 1)[1])[0] + assert "needs.required-context-scope.result != 'success'" in job, lane +print("build aggregate: full-lane scope fallback; failed/skipped/cancelled lanes rejected") +PY