Skip to content
107 changes: 4 additions & 103 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -149,100 +149,6 @@ jobs:
- name: Validate Element Graph
run: just validate

# FSDK's kernel is most of a cold build
# (docs/skills/ci-tooling.md#build-time-and-caches), and no upstream cache
# holds it (our module certificate and kernel config patches change its
# key). These jobs build it (and Go) with no signing secrets, each against
# one committed module certificate, and push the key-free artifacts to the
# project CAS unless a remote already holds them; every build pulls them
# through project.conf (scripts/kernel-cache.sh):
# - kernel-cache: the release certificate, for release builds;
# - kernel-cache-dev: the public INSECURE dev certificate that
# `just gen-dev-keys` uses (files/dev-keys/), for pull requests, the
# nightly build and local builds. No job needs it, so it never delays
# a release.
# The cache only saves time: a failed seed must not hold back the release,
# which checks build's and boot-test-result's own results.
kernel-cache:
needs: changes
if: needs.changes.outputs.release == 'true'
# CASD_CLIENT_KEY lives in this environment, which only main may use.
environment: bst-cache
runs-on: ubuntu-26.04
timeout-minutes: 240
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
with:
tool: just@1.59.0

- name: Set up runner disk space
run: bash scripts/ci-runner-disk.sh

- name: Build and push the kernel cache unless it exists
id: seed
continue-on-error: true
env:
CASD_CLIENT_CERT: ${{ vars.CASD_CLIENT_CERT }}
CASD_CLIENT_KEY: ${{ secrets.CASD_CLIENT_KEY }}
run: bash scripts/kernel-cache.sh seed release

# continue-on-error turns the job green; say so, or a cache that never
# seeds goes unnoticed while every build compiles the kernel.
- name: Report a failed seed
if: steps.seed.outcome == 'failure'
run: |
msg="The kernel cache seed failed; this release builds the kernel itself. See the step above."
echo "::warning title=Kernel cache not seeded::${msg}"
echo "${msg}" >> "$GITHUB_STEP_SUMMARY"

kernel-cache-dev:
needs: changes
if: needs.changes.outputs.release == 'true'
# CASD_CLIENT_KEY lives in this environment, which only main may use.
environment: bst-cache
runs-on: ubuntu-26.04
timeout-minutes: 240
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
with:
tool: just@1.59.0

- name: Set up runner disk space
run: bash scripts/ci-runner-disk.sh

- name: Build and push the dev kernel cache unless it exists
id: seed
continue-on-error: true
env:
CASD_CLIENT_CERT: ${{ vars.CASD_CLIENT_CERT }}
CASD_CLIENT_KEY: ${{ secrets.CASD_CLIENT_KEY }}
run: bash scripts/kernel-cache.sh seed dev

# continue-on-error turns the job green; say so, or a cache that never
# seeds goes unnoticed while every build compiles the kernel.
- name: Report a failed seed
if: steps.seed.outcome == 'failure'
run: |
msg="The dev kernel cache seed failed; pull requests and the nightly build compile the kernel until a release seeds it. See the step above."
echo "::warning title=Kernel cache not seeded::${msg}"
echo "${msg}" >> "$GITHUB_STEP_SUMMARY"

# Installs what the boot test runs and checks every host tool its dogfood
# scripts call is there (scripts/ci/dogfood-preflight.sh), in about a
# minute, so a missing one fails before the build instead of after it.
Expand All @@ -269,9 +175,9 @@ jobs:
run: bash scripts/ci/dogfood-preflight.sh

build:
needs: [changes, kernel-cache, boot-test-preflight]
# Runs when kernel-cache is skipped (non-release builds) or failed: the
# cache only saves time. A failed preflight stops it.
needs: [changes, boot-test-preflight]
# Cache warming is advisory, never a release startup dependency.
# A failed boot-test preflight still stops the image build.
if: ${{ !cancelled() && needs.changes.outputs.image == 'true' && needs.boot-test-preflight.result == 'success' }}
# The signing secrets live in the `release` environment, which only main
# may use. Every other build runs with no environment (an empty name):
Expand Down Expand Up @@ -539,9 +445,6 @@ jobs:
# parallel, one runner each; boot-test-result is their one gate.
boot-test:
needs: [changes, build]
# build needs kernel-cache, which is skipped outside release builds; a
# skipped job anywhere upstream skips this one unless it checks build's
# own result.
if: ${{ !cancelled() && needs.build.result == 'success' }}
runs-on: ubuntu-26.04
timeout-minutes: 60
Expand Down Expand Up @@ -754,9 +657,7 @@ jobs:
# command is scripts/publish-release.sh, which release-dry-run rehearses on
# each pull request that builds.
release:
# Explicit results, not !failure(): a failed kernel-cache (only a cache)
# must not hold back a release, and nothing but a built, boot-tested and
# image-checked set may publish.
# Publish only a built, boot-tested and image-checked set.
if: ${{ !cancelled() && needs.changes.outputs.release == 'true' && needs.build.result == 'success' && needs.boot-test-result.result == 'success' && needs.image-check.result == 'success' }}
needs: [changes, build, boot-test-result, image-check]
runs-on: ubuntu-26.04
Expand Down
75 changes: 75 additions & 0 deletions .github/workflows/cache-warm.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
name: Warm BuildStream cache

on:
push:
branches: [main]
workflow_dispatch:

# A seed must not occupy build.yml's serialized release queue.
# Finish the current warming run; GitHub keeps only the latest pending one.
concurrency:
group: cache-warm-${{ github.ref }}
cancel-in-progress: false

permissions:
contents: read

jobs:
kernel-cache:
if: github.ref == 'refs/heads/main'
# CASD_CLIENT_KEY lives in this environment, which only main may use.
environment: bst-cache
runs-on: ubuntu-26.04
timeout-minutes: 240
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
with:
tool: just@1.59.0

- name: Set up runner disk space
run: bash scripts/ci-runner-disk.sh

- name: Build and push the kernel cache unless it exists
id: seed
env:
CASD_CLIENT_CERT: ${{ vars.CASD_CLIENT_CERT }}
CASD_CLIENT_KEY: ${{ secrets.CASD_CLIENT_KEY }}
run: bash scripts/kernel-cache.sh seed release

kernel-cache-dev:
if: github.ref == 'refs/heads/main'
# CASD_CLIENT_KEY lives in this environment, which only main may use.
environment: bst-cache
runs-on: ubuntu-26.04
timeout-minutes: 240
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@cfdb7ed4ccc7c7dce37759eae89dc5674b469b39 # v2.90.3
with:
tool: just@1.59.0

- name: Set up runner disk space
run: bash scripts/ci-runner-disk.sh

- name: Build and push the dev kernel cache unless it exists
id: seed
env:
CASD_CLIENT_CERT: ${{ vars.CASD_CLIENT_CERT }}
CASD_CLIENT_KEY: ${{ secrets.CASD_CLIENT_KEY }}
run: bash scripts/kernel-cache.sh seed dev

33 changes: 19 additions & 14 deletions docs/skills/ci-tooling.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ one of these environments is refused before its first step.
| Environment | Secrets | Jobs |
|---|---|---|
| `release` | `BOOT_KEYS_TARBALL`, `SYSUPDATE_SIGNING_KEY` | `build`, only when `changes` says `release=true` |
| `bst-cache` | `CASD_CLIENT_KEY` (`CASD_CLIENT_CERT` is a repository variable) | `kernel-cache`, `kernel-cache-dev` (releases only) |
| `bst-cache` | `CASD_CLIENT_KEY` (`CASD_CLIENT_CERT` is a repository variable) | `kernel-cache`, `kernel-cache-dev` in `cache-warm.yml` (main only) |
| `trackers` | `MERGERAPTOR_APP_ID`, `MERGERAPTOR_PRIVATE_KEY` | `track-junctions`, `track-binaries`' `propose` |

`build` serves releases and every other build, so it names the environment
Expand Down Expand Up @@ -222,6 +222,7 @@ use: remove only this repository from their repository access.
| `validate` | `build.yml` | `pull_request`, `merge_group` | `just validate` with throwaway keys: resolves every shipped element graph and runs the version-invariant checks, in minutes. Read-only token. |
| `boot-test-preflight` | `build.yml` | when `changes` says `image=true` | Installs the boot test's tools with the same `scripts/ci/install-qemu-ovmf.sh` as `boot-test`, then runs `scripts/ci/dogfood-preflight.sh`: every host tool the dogfood scripts `boot-test` runs must be on the `PATH`, the OVMF files present and `/dev/kvm` read-writable. Each name its tool pattern matches in those scripts must be listed there, as a host tool or as one the runner never runs, so a new tool fails this job until it is. About a minute; `build` waits for it, so a missing tool (#486: `mcopy`) fails before the build instead of after it. Read-only token. |
| `build` | `build.yml` | when `changes` says `image=true` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/kubeadm/OpenZFS/NVIDIA sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. For releases it runs in the `release` environment and installs its `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there. Other builds use no environment. Every other build uses throwaway keys and also exports two higher-versioned sets (`1.<run>.1`, `1.<run>.2`) for the update test. It uploads each set in parts by what its consumers read: `dist/diskless/` as `image-set-netboot` (OS DDI, netboot UKI and ESP), `-installer`, `-sysupdate` (usr, usr-verity, disk UKI) and `-sysexts` (`*.raw.zst`), plus the catch-all `image-set-core` that excludes exactly what the others include, so no file is ever dropped; `1.<run>.1` as `image-next-boot` (catch-all) and `image-next-update` (sysupdate sources and sysexts); `1.<run>.2` as `image-broken`; the Ignition test signatures as `ignition-sigs`. The update sets leave out the installer. `image-check`, `release` and `release-dry-run` merge every `image-set-*` part back into `dist/diskless/`; `tests/unit/test_ci_image_artifacts.py` keeps the split exhaustive. Read-only token. |
| `kernel-cache`, `kernel-cache-dev` | `cache-warm.yml` | `push/main`, `workflow_dispatch` on main | Warms the key-free kernel/Go/GCC/Brew closure in the project CAS. Separate workflow and concurrency group: never occupies the serialized release queue. Read-only token; mTLS credentials in `bst-cache` only. |
| `boot-test` | `build.yml` | after `build` | Runs the Secure Boot QEMU checks on the exported sets (see Core Process step 4), with the harness checked out at the commit `build` built. A `fail-fast: false` matrix over the buckets of `scripts/ci/boot-test.sh`, one runner each, so its checks are `boot-test (<bucket>)`; each bucket downloads only the `build` artifacts its scenarios read, as listed in its matrix entry. Each bucket's wall time, passed or failed, is a notice and a row of its step summary. On failure each bucket uploads its logs as `boot-test-logs-<bucket>`. Read-only token. |
| `boot-test-result` | `build.yml` | after `boot-test`, when `build` succeeded | The one result of every `boot-test` bucket: fails unless `boot-test` succeeded. `release` and `nightly-status` gate on it, and it is skipped like `boot-test` when nothing was built, so it is the boot test's required check. No checkout, `permissions: {}`. |
| `image-check` | `build.yml` | after `build` | Runs `.github/scripts/check-image.py` on the built image set: no missing libraries, required dlopen libraries, or unit/udev programs in `/usr`, the disk UKI's initrd, or the sysexts (resolved over `/usr`); the step summary diffs `/usr` paths against the latest release. Then `scripts/ci/image-sizes.py` tables the size of every file, the disk UKI's `.linux` and `.initrd` and the unpacked initrd against the latest release's `bluefin-server_<ver>.size.json` (or its `*.usr.raw` alone), with a `::warning::` for each that grew by more than 5 % or 5 MiB (never a failure), and uploads this set's report as the `image-size` artifact that `release` and `release-dry-run` publish with the set, unsigned (see "Publishing" in `systemd-sysupdate-verification.md`). `release` needs it. Read-only token. |
Expand Down Expand Up @@ -396,22 +397,26 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about
(`elements/`, `files/`, `include/`, `patches/`, `plugins/`, `scripts/`,
`tests/fixtures/`, `project.conf`, `Justfile`, `build.yml`) would skip.
A skipped job reports as passing, so its required check does not hold it.
- **Kernel cache in the project CAS, key-free by construction.** Two jobs,
on every push to `main` (`release=true`), run `scripts/kernel-cache.sh seed`, one per module
certificate the kernel can trust (it is part of the kernel's cache key):
- **Key-free build cache in the project CAS.** Two jobs in `cache-warm.yml`
run on pushes to `main` or a manual dispatch on `main`:
- `kernel-cache` runs `seed release` with the committed release certificate
(`files/release-keys/linux-module-cert.crt`); `build` waits for it.
(`files/release-keys/linux-module-cert.crt`).
- `kernel-cache-dev` runs `seed dev` with the public INSECURE dev
certificate (`files/dev-keys/INSECURE-dev-module-key.crt`) that
`just gen-dev-keys` uses for every non-release build. No job waits for
it, so it never delays a release.
`just gen-dev-keys` uses for every non-release build.
Separate `cache-warm-*` concurrency keeps a slow seed out of the release
workflow and its queue; removing only `build.needs` is insufficient.
Cold image builds may compile concurrently with warming.

With no signing secrets and that one certificate staged as the only file of
`files/boot-keys/modules/` (`seed` refuses anything else there: that
directory's import is pushed with the kernel), each builds FSDK's kernel,
Go and `bootstrap/build/gcc-stage2.bst`, plus the Brew machine. BuildStream
pulls or rebuilds payloads, then `bst artifact push --deps all` replicates
upstream artifacts into `cache.projectbluefin.io:11002`; metadata never skips seeding.
Go, GCC stage 2, Brew and unsigned ZFS/NVIDIA outputs. Their full closure
includes Ruby and systemd's base build, but excludes module-signing stages.
`bst build --deps all` materializes their whole key-free closure; cached
targets otherwise leave build-only dependencies uncached. Then
`bst artifact push --deps all` replicates upstream artifacts into
`cache.projectbluefin.io:11002`; metadata never skips seeding.
The push endpoint takes mTLS: its client certificate is the repository variable
`CASD_CLIENT_CERT` and its key the `bst-cache` environment secret
`CASD_CLIENT_KEY`, given to that step only and written to a gitignored `.casd.*/` directory that the script
Expand All @@ -428,11 +433,11 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about
The CAS is publicly readable: `seed` refuses if
`bluefin-server/keys/boot-keys.bst` is anywhere in the graph it builds,
and the job holds no signing secret. A content grep is no guard here: FSDK
sources (Go's TLS test data and others) carry 307 PEM private keys. The
seed step is `continue-on-error`, so a failed seed only costs time; the
next step then posts a warning and a step summary line.
sources (Go's TLS test data and others) carry 307 PEM private keys.
The independent warming workflow fails visibly on seed errors; its result
does not gate image building or publication.
The former kernel/Go-only seed took 44 min and 15 GB on 16+ CPUs.
Kernel, Go, GCC, Brew or their dependency changes reseed the affected targets.
Changes to these targets or their dependencies reseed the affected artifacts.
- **No other cache push.** `bluefin-server/keys/boot-keys.bst` imports
`files/boot-keys/`, which on `main` holds the Secure Boot, module-signing
and sysupdate private keys, and the image, UKIs, `kernel-modules.bst`,
Expand Down
9 changes: 6 additions & 3 deletions scripts/kernel-cache.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Key-free BuildStream cache of FSDK's kernel, Go, GCC and the Brew machine.
# Key-free BuildStream cache of the SDK, Brew and unsigned ZFS/NVIDIA builds.
#
# CASD_CLIENT_CERT=<pem> CASD_CLIENT_KEY=<pem> kernel-cache.sh seed release|dev
#
Expand Down Expand Up @@ -40,6 +40,8 @@ ELEMENTS=(
freedesktop-sdk.bst:components/go.bst
freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst
fsdk-containers.bst:oci/brew-nspawn.bst
zfs/openzfs.bst
nvidia/nvidia-open-595.bst
)
KEYS_ELEMENT=bluefin-server/keys/boot-keys.bst
PUSH_URL=https://cache.projectbluefin.io:11002
Expand Down Expand Up @@ -129,8 +131,9 @@ seed() {
echo "kernel-cache: cannot reach ${PUSH_URL}; refusing to build without pushing" >&2
exit 1
fi
just bst build "${ELEMENTS[@]}"
# Builds need not push dependencies pulled from upstream caches.
# Cached targets can leave build-only dependencies uncached; push --deps
# all requires those artifacts too, including the certificate import.
just bst build --deps all "${ELEMENTS[@]}"
just bst artifact push --deps all "${ELEMENTS[@]}"
echo "kernel-cache: ${variant}: pushed ${ELEMENTS[*]} to ${PUSH_URL}"
}
Expand Down
6 changes: 0 additions & 6 deletions tests/unit/test_build_modes.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,12 +97,6 @@ def test_signing_secrets_and_publishing_are_release_only() -> None:
for step in job.get("steps", []):
for key, value in (step.get("env") or {}).items():
if "secrets." in str(value) and "GITHUB_TOKEN" not in str(value):
if name in ("kernel-cache", "kernel-cache-dev"):
# The CAS push key, in a job that only runs for
# releases (test_dev_module_key.py) and holds no
# signing secret.
assert (key, value) == ("CASD_CLIENT_KEY", "${{ secrets.CASD_CLIENT_KEY }}")
continue
assert name == "build", (name, key)
assert str(value).startswith("${{ needs.changes.outputs.release == 'true' && secrets."), key
assert "needs.changes.outputs.release == 'true'" in JOBS["release"]["if"]
2 changes: 1 addition & 1 deletion tests/unit/test_ci_runner_disk.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def test_runner_disk_setup_is_not_inlined(workflow: str) -> None:


def test_every_build_job_sets_up_the_runner_disk() -> None:
for workflow, job in (("build.yml", "kernel-cache"), ("build.yml", "kernel-cache-dev"), ("build.yml", "build"),
for workflow, job in (("cache-warm.yml", "kernel-cache"), ("cache-warm.yml", "kernel-cache-dev"), ("build.yml", "build"),
("reproducibility.yml", "reproducibility")):
steps = yaml.safe_load((WORKFLOWS / workflow).read_text())["jobs"][job]["steps"]
assert any(s.get("run") == "bash scripts/ci-runner-disk.sh" for s in steps), f"{workflow}:{job}"
Expand Down
Loading
Loading