diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 000000000..74135f65e --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,8 @@ +# actionlint ships a hardcoded table of GitHub-hosted runner labels. ubuntu-26.04 +# went GA on 2026-09-17 (actions/runner-images#14747) and is newer than any +# actionlint release, including v1.7.12, so the linter reports it as unknown. +# Declaring it here tells actionlint the label exists; it is not a self-hosted +# runner. Drop this block once an actionlint release knows the label natively. +self-hosted-runner: + labels: + - ubuntu-26.04 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e0f4889ad..d2d61e5ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,7 @@ name: Build Utah on: pull_request: + merge_group: push: branches: [testing] # Nightly scheduling lives in sync-main-to-testing so signatures and E2E @@ -12,13 +13,14 @@ permissions: {} concurrency: group: utah-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: # Resolve the complete transaction against the same pinned inputs as the # image before starting the flavor matrix, including extension build tools. contract: runs-on: ubuntu-26.04 + timeout-minutes: 15 permissions: contents: read outputs: @@ -73,10 +75,12 @@ jobs: # the three flavors that need it skip roughly half an hour of compiling each. kernel_cache: needs: contract - # Only the OGC and NVIDIA flavors unpack this. While the matrix is main-only + # Only non-PR builds (merge queue, testing push, dispatch) perform image builds. + # Furthermore, only the OGC and NVIDIA flavors unpack this. While the matrix is main-only # it is 45 minutes spent on an image nothing consumes. - if: needs.contract.outputs.needs_kernel == 'true' + if: github.event_name != 'pull_request' && needs.contract.outputs.needs_kernel == 'true' runs-on: ubuntu-26.04 + timeout-minutes: 60 permissions: contents: read packages: write @@ -110,6 +114,7 @@ jobs: || github.event.pull_request.head.repo.full_name == github.repository }} run: | set -euo pipefail + mkdir -p logs ref="ghcr.io/${REPO_ORGANIZATION,,}/utah-kernel-cache:$(./scripts/kernel-cache-tag.sh)" if podman pull -q "$ref" >/dev/null 2>&1; then echo "Cache hit: $ref is already published" @@ -117,7 +122,7 @@ jobs: exit 0 fi echo "Cache miss: building $ref" - podman build --tag "$ref" --file Containerfile.kernel . + podman build --tag "$ref" --file Containerfile.kernel . 2>&1 | tee logs/kernel-cache-build.log # A pull request from a fork gets a read-only GITHUB_TOKEN, so the # push fails no matter how well the kernel built: # @@ -133,12 +138,29 @@ jobs: echo "available=false" >> "$GITHUB_OUTPUT" exit 0 fi - podman push "$ref" + PUSH_OK=false + for attempt in 1 2 3; do + podman push "$ref" && { PUSH_OK=true; break; } + echo "Push attempt ${attempt} failed for ${ref}, retrying in 5s..." + [ "$attempt" -lt 3 ] && sleep 5 + done + $PUSH_OK || { echo "ERROR: all push attempts failed for ${ref}"; exit 1; } echo "available=true" >> "$GITHUB_OUTPUT" + - name: Upload kernel cache build logs + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: kernel-cache-build-logs + path: logs/ + retention-days: 7 + if-no-files-found: ignore # main starts the moment the contract check passes. It never pulls the kernel # cache, so there is no reason for it to wait on that job; before this split # a cache miss cost main the same twenty minutes it cost nvidia-gaming. + # + # build_main runs on PRs, merge queues, and testing pushes to ensure core + # package transactions and container composition are validated on all PRs. build_main: needs: contract if: needs.contract.outputs.main_flavors != '[]' @@ -166,6 +188,7 @@ jobs: needs.build_main.result == 'success' && (needs.build_kernel.result == 'success' || needs.build_kernel.result == 'skipped') runs-on: ubuntu-26.04 + timeout-minutes: 10 permissions: actions: write steps: @@ -185,9 +208,14 @@ jobs: # other. Utah's Justfile ignores base_name when naming images (image names # come from the flavor alone), so the only visible effect is the name of the # digest artifact, which post-testing-e2e matches by glob. + # + # Secondary-flavor policy: kernel flavors (nvidia, gaming, nvidia-gaming) are + # secondary variants that build on the kernel cache. PRs skip building them. + # On merge queues and testing pushes, they build in their own matrix so that + # kernel cache construction does not delay the primary flavor. build_kernel: needs: [contract, kernel_cache] - if: ${{ !cancelled() && needs.contract.result == 'success' && needs.kernel_cache.result == 'success' && needs.kernel_cache.outputs.available == 'true' && needs.contract.outputs.kernel_flavors != '[]' }} + if: ${{ github.event_name != 'pull_request' && !cancelled() && needs.contract.result == 'success' && needs.kernel_cache.result == 'success' && needs.kernel_cache.outputs.available == 'true' && needs.contract.outputs.kernel_flavors != '[]' }} uses: projectbluefin/actions/.github/workflows/reusable-build.yml@8895d09342174927b950abc75dccd6da2c8ba36f # v1 permissions: contents: read diff --git a/.github/workflows/execute-release.yml b/.github/workflows/execute-release.yml index 4f72078bb..0ec3261e4 100644 --- a/.github/workflows/execute-release.yml +++ b/.github/workflows/execute-release.yml @@ -8,12 +8,17 @@ on: permissions: id-token: write +concurrency: + group: utah-execute-release + cancel-in-progress: false + jobs: # config/flavors.json is the single source for which images exist. Promoting # or releasing an image the build matrix no longer produces fails late and # confusingly, so this reads the same list the build does. variants: runs-on: ubuntu-26.04 + timeout-minutes: 10 permissions: contents: read outputs: @@ -25,6 +30,7 @@ jobs: check-trigger: runs-on: ubuntu-latest + timeout-minutes: 10 outputs: is-promotion: ${{ steps.check.outputs.is-promotion }} steps: diff --git a/.github/workflows/post-testing-e2e.yml b/.github/workflows/post-testing-e2e.yml index 206e1a9e5..81564334e 100644 --- a/.github/workflows/post-testing-e2e.yml +++ b/.github/workflows/post-testing-e2e.yml @@ -17,6 +17,7 @@ concurrency: jobs: resolve: runs-on: ubuntu-26.04 + timeout-minutes: 15 permissions: contents: read actions: read @@ -144,6 +145,7 @@ jobs: promote-to-testing: needs: [resolve, luks] runs-on: ubuntu-26.04 + timeout-minutes: 15 permissions: contents: read packages: write @@ -164,12 +166,19 @@ jobs: sudo apt-get update && sudo apt-get install -y skopeo echo "$GH_TOKEN" | skopeo login ghcr.io -u "$GITHUB_ACTOR" --password-stdin while IFS= read -r ref; do - skopeo copy --all --preserve-digests "docker://$ref" "docker://${ref%@*}:testing" + PUSH_OK=false + for attempt in 1 2 3; do + skopeo copy --all --preserve-digests "docker://$ref" "docker://${ref%@*}:testing" && { PUSH_OK=true; break; } + echo "skopeo copy attempt ${attempt} failed for ${ref}, retrying in 5s..." + [ "$attempt" -lt 3 ] && sleep 5 + done + $PUSH_OK || { echo "ERROR: all skopeo copy attempts failed for ${ref}"; exit 1; } done < <(jq -r '.include[].ref' <<< "$MATRIX") documentation: needs: [resolve, luks] runs-on: ubuntu-26.04 + timeout-minutes: 10 permissions: contents: write pull-requests: write diff --git a/.github/workflows/promote-testing-to-main.yml b/.github/workflows/promote-testing-to-main.yml index a9cb90ba7..f54bcb912 100644 --- a/.github/workflows/promote-testing-to-main.yml +++ b/.github/workflows/promote-testing-to-main.yml @@ -15,12 +15,17 @@ permissions: actions: read statuses: write +concurrency: + group: utah-promote-testing + cancel-in-progress: false + jobs: # config/flavors.json is the single source for which images exist. Promoting # or releasing an image the build matrix no longer produces fails late and # confusingly, so this reads the same list the build does. variants: runs-on: ubuntu-26.04 + timeout-minutes: 10 permissions: contents: read outputs: diff --git a/.github/workflows/sync-main-to-testing.yml b/.github/workflows/sync-main-to-testing.yml index 750bba2f4..67d9f28e4 100644 --- a/.github/workflows/sync-main-to-testing.yml +++ b/.github/workflows/sync-main-to-testing.yml @@ -14,6 +14,10 @@ permissions: contents: write actions: write +concurrency: + group: utah-sync-main-to-testing + cancel-in-progress: false + jobs: sync: uses: projectbluefin/actions/.github/workflows/reusable-sync-branches.yml@8895d09342174927b950abc75dccd6da2c8ba36f # v1 @@ -23,6 +27,7 @@ jobs: build: needs: sync runs-on: ubuntu-26.04 + timeout-minutes: 10 permissions: actions: write steps: diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index 017b713d9..5ac53341d 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -1,7 +1,7 @@ --- name: ci-workflows -version: "1.0" -last_updated: "2026-09-05" +version: "1.1" +last_updated: "2026-09-18" id: ci-workflows one_line_purpose: Navigate Utah's build, promote, and sync workflow topology. entry_point: docs/skills/ci-workflows.md @@ -12,8 +12,8 @@ status: active dependencies: [] tags: [ci, workflows, actions, promotion] description: >- - build.yml contract gate, kernel-cache job, main/kernel matrix split, - promote-testing-to-main and sync-main-to-testing, actions@v1 delegation. + build.yml contract gate, Dakota PR and merge-queue build topology, + kernel-cache job, matrix split, promote and sync, actions@v1 delegation. Use when changing .github/workflows/ or debugging a red run. metadata: type: reference @@ -21,24 +21,53 @@ metadata: # CI Workflows -Three workflows, all thin callers into `projectbluefin/actions@v1` reusables, -each pinned to a SHA tagged `v1`: - -- `.github/workflows/build.yml` -- pull requests, pushes to `testing`, a - manual dispatch. Top-level `permissions: {}`; each job - grants its own. Cancels in-progress runs per workflow and ref. -- `.github/workflows/promote-testing-to-main.yml` -- pushes to `testing`, a - nightly cron, and manual dispatch. -- `.github/workflows/sync-main-to-testing.yml` -- source pushes to `main`, - nightly cron, and manual dispatch; explicitly dispatches the testing build - after syncing. Token-authenticated branch pushes alone do not start CI. -- `.github/workflows/post-testing-e2e.yml` -- successful non-PR testing builds - explicitly dispatch this, or manually supply a successful testing build run ID. +Workflows are thin callers into `projectbluefin/actions@v1` reusables, each +pinned to a SHA tagged `v1`: + +- `.github/workflows/build.yml` -- pull requests, merge queue (`merge_group`), + pushes to `testing`, and manual dispatch. Top-level `permissions: {}`; each + job grants its own. Cancels in-progress runs for pull requests only + (`cancel-in-progress: ${{ github.event_name == 'pull_request' }}`), ensuring + merge queue, scheduled, and release builds queue safely without dropping work. +- `.github/workflows/promote-testing-to-main.yml` -- pushes to `testing`, + nightly cron, manual dispatch. Serialized via `utah-promote-testing`. +- `.github/workflows/sync-main-to-testing.yml` -- pushes to `main`, nightly + cron, manual dispatch; dispatches testing build. Serialized via `utah-sync-main-to-testing`. +- `.github/workflows/post-testing-e2e.yml` -- testing build verification. + Serialized via `utah-iso-e2e`. +- `.github/workflows/execute-release.yml` -- promotion and release dispatch. + Serialized via `utah-execute-release`. CI delegates builds, vulnerability reporting, SBOMs, keyless signatures, provenance, caching, and rechunking to `projectbluefin/actions@v1` (originated as a `docs/building.md` design bullet; now lives in this skill). +## PR validation and merge-queue build topology + +In accordance with Dakota's CI topology: + +1. **PR validation protects core image builds without kernel compile overhead:** + PRs run the cheap `contract` job (`just check`, `just check-parity`, `just check-repos`, and + flavor resolution) and primary `build_main` image build, validating container + composition and package transactions without waiting for the 45-minute kernel cache compile. + PRs skip the secondary-flavor kernel builds (`kernel_cache`, `build_kernel`). +2. **Merge queue performs full release-candidate builds:** When a PR enters the + merge queue, GitHub fires `merge_group`. Merge queue builds execute the full + matrix including secondary flavors: `contract`, `build_main`, `kernel_cache`, and `build_kernel`. +3. **Release-oriented concurrency:** `cancel-in-progress` is active only for + pull requests (`${{ github.event_name == 'pull_request' }}`). Merge queue, + testing pushes, scheduled runs, and promotion workflows queue safely instead + of being superseded and discarded mid-run. +4. **Explicit timeouts and log preservation:** Every standalone runner job defines an explicit + `timeout-minutes` (reusable workflows inherit timeout configuration from the reusable workflow). + Long-running tasks capture build output and upload logs as artifacts on completion or failure + (e.g. `kernel-cache-build-logs`). +5. **Transient push retries and explicit secondary-flavor policy:** Registry + pushes (`podman push` for kernel cache and `skopeo copy` for testing tag + promotion) retry transient failures up to three times with backoff. Secondary + flavors (`nvidia`, `gaming`, `nvidia-gaming`) build in `build_kernel` on top + of the kernel cache; their configuration is owned strictly by `config/flavors.json`. + ## contract: the cheap gate `build.yml` opens with a container-based gate so a contract package that none of diff --git a/docs/skills/index.json b/docs/skills/index.json index 93d6db3e7..3eacbb37d 100644 --- a/docs/skills/index.json +++ b/docs/skills/index.json @@ -15,9 +15,9 @@ "actions", "promotion" ], - "description": "build.yml contract gate, kernel-cache job, main/kernel matrix split, promote-testing-to-main and sync-main-to-testing, actions@v1 delegation. Use when changing .github/workflows/ or debugging a red run.", - "version": "1.0", - "last_updated": "2026-09-05", + "description": "build.yml contract gate, Dakota PR and merge-queue build topology, kernel-cache job, matrix split, promote and sync, actions@v1 delegation. Use when changing .github/workflows/ or debugging a red run.", + "version": "1.1", + "last_updated": "2026-09-18", "doc_type": "reference" }, { diff --git a/tests/test_flavors.py b/tests/test_flavors.py index c7976ab0d..b99adadae 100644 --- a/tests/test_flavors.py +++ b/tests/test_flavors.py @@ -152,4 +152,4 @@ def test_shipped_config_agrees_across_all_queries(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main()