From a9915a811052771c7a0ed96f1049b2df3d62e2bb Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:08:01 -0500 Subject: [PATCH 01/16] test(ignition): fixture that writes a file and enables a unit apply-marker.ign writes /etc/dogfood-ignition-marker and enables a oneshot unit through Ignition's preset; apply-marker.probe reports both from inside the booted guest, so a dogfood boot can assert that Ignition applied the config rather than only that the node booted. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tests/fixtures/ignition/apply-marker.ign | 26 ++++++++++++++++++++++ tests/fixtures/ignition/apply-marker.probe | 7 ++++++ 2 files changed, 33 insertions(+) create mode 100644 tests/fixtures/ignition/apply-marker.ign create mode 100644 tests/fixtures/ignition/apply-marker.probe diff --git a/tests/fixtures/ignition/apply-marker.ign b/tests/fixtures/ignition/apply-marker.ign new file mode 100644 index 00000000..e0c534e2 --- /dev/null +++ b/tests/fixtures/ignition/apply-marker.ign @@ -0,0 +1,26 @@ +{ + "ignition": { + "version": "3.6.0" + }, + "storage": { + "files": [ + { + "path": "/etc/dogfood-ignition-marker", + "mode": 420, + "overwrite": true, + "contents": { + "source": "data:,applied%0A" + } + } + ] + }, + "systemd": { + "units": [ + { + "name": "dogfood-ignition-marker.service", + "enabled": true, + "contents": "[Unit]\nDescription=Dogfood check that Ignition enabled this unit\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/bin/touch /run/dogfood-ignition-marker\n\n[Install]\nWantedBy=multi-user.target\n" + } + ] + } +} diff --git a/tests/fixtures/ignition/apply-marker.probe b/tests/fixtures/ignition/apply-marker.probe new file mode 100644 index 00000000..ba9c9568 --- /dev/null +++ b/tests/fixtures/ignition/apply-marker.probe @@ -0,0 +1,7 @@ +# DOGFOOD_EXTRA_PROBE for apply-marker.ign: Ignition wrote the file and the +# unit, the preset enabled the unit, and PID 1 ran it on this boot. +marker="$(cat /etc/dogfood-ignition-marker 2>/dev/null || echo missing)" +unit="$(systemctl is-active dogfood-ignition-marker.service 2>/dev/null || true)" +enabled="$(systemctl is-enabled dogfood-ignition-marker.service 2>/dev/null || true)" +ran="$([ -e /run/dogfood-ignition-marker ] && echo yes || echo no)" +echo "PROBE ignition marker=${marker} unit=${unit} enabled=${enabled} ran=${ran}" From 8fd798cf427b1e876ad7cbea38cec20e4601aed1 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:08:01 -0500 Subject: [PATCH 02/16] ci: classify pull requests that cannot change the image set image-build-needed.py reads a PR's changed paths and answers false only for docs, Markdown outside the build inputs, unit/e2e tests and workflows that do not build the image. The unit test fails if any tracked file under a build input root (elements/, files/, include/, patches/, plugins/, scripts/, tests/fixtures/, project.conf, Justfile, build.yml) would skip the build. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/image-build-needed.py | 68 +++++++++++++++++++ tests/unit/test_image_build_needed.py | 97 +++++++++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 .github/scripts/image-build-needed.py create mode 100644 tests/unit/test_image_build_needed.py diff --git a/.github/scripts/image-build-needed.py b/.github/scripts/image-build-needed.py new file mode 100644 index 00000000..9d9c572e --- /dev/null +++ b/.github/scripts/image-build-needed.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Decide whether a pull request needs the image build and boot test. + +Reads the PR's changed paths (one per line, renames listed under both names) +on stdin and prints ``true`` or ``false`` for the build workflow's `changes` +job. Only changes that cannot reach the image set or the boot test skip the +~2 h build: docs, Markdown outside the build inputs, unit/e2e tests and the +workflows that run them. Anything else, and anything under a build input +root, builds; an empty or truncated list builds too. +""" + +import sys + +# Paths the build or the boot test consume. Always build, even for *.md. +BUILD_PREFIXES = ( + "elements/", + "files/", + "include/", + "patches/", + "plugins/", + "scripts/", + "tests/fixtures/", +) + +SKIP_PREFIXES = ( + "docs/", + "tests/unit/", + "tests/e2e/", +) + +SKIP_FILES = frozenset( + { + ".github/scripts/docs-checks.py", + ".github/workflows/docs-checks.yml", + ".github/workflows/unit-tests.yml", + ".github/workflows/track-junctions.yml", + ".pre-commit-config.yaml", + "cliff.toml", + "LICENSE", + } +) + +# GitHub's "list pull request files" API stops at 3000 files. +API_FILE_LIMIT = 3000 + + +def needs_build(path: str) -> bool: + if path.startswith(BUILD_PREFIXES): + return True + if path.startswith(SKIP_PREFIXES) or path in SKIP_FILES or path.endswith(".md"): + return False + return True + + +def image_build_needed(paths: list[str]) -> bool: + if not paths or len(paths) >= API_FILE_LIMIT: + return True + return any(needs_build(p) for p in paths) + + +def main() -> int: + paths = [line.strip() for line in sys.stdin if line.strip()] + print("true" if image_build_needed(paths) else "false") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_image_build_needed.py b/tests/unit/test_image_build_needed.py new file mode 100644 index 00000000..274f0175 --- /dev/null +++ b/tests/unit/test_image_build_needed.py @@ -0,0 +1,97 @@ +"""Unit coverage for .github/scripts/image-build-needed.py. + +The build workflow skips the ~2 h image build and the boot test for pull +requests the script classifies as not touching the image. A wrong `false` +merges an untested image change, so the invariant guarded here is that every +tracked file under a build input root, and the build workflow itself, always +builds. +""" + +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".github" / "scripts" / "image-build-needed.py" + +_spec = importlib.util.spec_from_file_location("image_build_needed", SCRIPT) +image_build_needed = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(image_build_needed) + + +def tracked_files(): + out = subprocess.run( + ["git", "ls-files"], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout + return [line for line in out.splitlines() if line] + + +def test_every_build_input_builds(): + inputs = [ + p + for p in tracked_files() + if p.startswith(image_build_needed.BUILD_PREFIXES) + or p in {"project.conf", "Justfile", ".github/workflows/build.yml"} + or p.startswith(".github/scripts/check-") + ] + assert inputs, "git ls-files returned no build inputs" + skipped = [p for p in inputs if not image_build_needed.needs_build(p)] + assert skipped == [] + + +def test_the_classifier_itself_builds(): + assert image_build_needed.needs_build(".github/scripts/image-build-needed.py") + + +@pytest.mark.parametrize( + "path", + [ + "docs/skills/ci-tooling.md", + "README.md", + ".github/copilot-instructions.md", + "tests/unit/test_repart_layout.py", + "tests/unit/os-justfile_test.bats", + ".github/scripts/docs-checks.py", + ".github/workflows/unit-tests.yml", + ], +) +def test_docs_and_unit_test_changes_skip(path): + assert not image_build_needed.needs_build(path) + + +@pytest.mark.parametrize( + "path", + [ + "files/os/README.md", + "elements/notes.md", + "tests/fixtures/ignition/apply-marker.ign", + "scripts/dogfood-diskless.sh", + "renovate.json", + "a-new-top-level-file", + ], +) +def test_build_inputs_and_unknown_paths_build(path): + assert image_build_needed.needs_build(path) + + +def test_one_build_input_among_docs_builds(): + assert image_build_needed.image_build_needed(["docs/a.md", "elements/x.bst"]) + + +def test_empty_or_truncated_lists_build(): + assert image_build_needed.image_build_needed([]) + many = ["docs/x.md"] * image_build_needed.API_FILE_LIMIT + assert image_build_needed.image_build_needed(many) + + +def test_cli_reads_stdin(): + def run(text): + return subprocess.run( + [sys.executable, str(SCRIPT)], input=text, capture_output=True, text=True, check=True + ).stdout.strip() + + assert run("docs/skills/index.md\nREADME.md\n") == "false" + assert run("docs/skills/index.md\nfiles/os/x\n") == "true" From 3d83ce258e53af6368df2fed93e315ec591a9989 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:12:33 -0500 Subject: [PATCH 03/16] fix(dogfood): honour DOGFOOD_PORT in dogfood-install.sh The update probe rewrote the sysupdate transfers to a hardcoded 10.0.2.2:8765, so any DOGFOOD_PORT other than the default served the image on one port while the guest updated from another. Export the port once and substitute it into the probe. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/dogfood-install.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/dogfood-install.sh b/scripts/dogfood-install.sh index 91944222..3a55eb48 100644 --- a/scripts/dogfood-install.sh +++ b/scripts/dogfood-install.sh @@ -10,10 +10,12 @@ # sysupdate feature enabled, so the ZFS sysext follows the OS in lock-step # and must still be active after the rollback. # Usage: dogfood-install.sh [ []] +# and are image sets with increasingly higher versions. +# DOGFOOD_PORT (default 8765) is shared with dogfood-diskless.sh's server. set -euo pipefail here="$(cd "$(dirname "$0")" && pwd)" -dir="$(realpath "${1:?usage: $0 []}")" +dir="$(realpath "${1:?usage: $0 [ []]}")" next="${2:+$(realpath "$2")}" broken="${3:+$(realpath "$3")}" state="$(realpath "${DOGFOOD_STATE:-dist/dogfood-install}")" @@ -21,6 +23,7 @@ rm -rf "${state}" mkdir -p "${state}" truncate -s 16G "${state}/disk.raw" +export DOGFOOD_PORT="${DOGFOOD_PORT:-8765}" export DOGFOOD_STATE_DISK="${state}/disk.raw" export DOGFOOD_VARS="${state}/vars.fd" run() { DOGFOOD_EXTRA_PROBE="$2" bash "${here}/dogfood-diskless.sh" "$1" --check; } @@ -50,7 +53,7 @@ cat > "${state}/update.probe" <<'EOF' mkdir -p /etc/sysupdate.d/zfs.feature.d printf '[Feature]\nEnabled=true\n' > /etc/sysupdate.d/zfs.feature.d/enable.conf for f in /usr/lib/sysupdate.d/*.transfer; do - sed -e 's|^Path=https://.*|Path=http://10.0.2.2:8765/|' \ + sed -e 's|^Path=https://.*|Path=http://10.0.2.2:@DOGFOOD_PORT@/|' \ "${f}" > "/etc/sysupdate.d/${f##*/}" done rc=0 @@ -59,6 +62,7 @@ echo "PROBE update=${rc}" tail -n 15 /run/sysupdate.log | sed 's/^/PROBE-LOG /' timeout 60 systemd-sysupdate list --no-pager 2>&1 | sed 's/^/PROBE-LOG /' EOF +sed -i "s|@DOGFOOD_PORT@|${DOGFOOD_PORT}|" "${state}/update.probe" echo "==> 1/4 diskless boot + systemd-sysinstall" run "${dir}" "${state}/install.probe" | tee "${state}/1-install.log" From 8c1564edc6b5278cb1a81fee791213a9cbb90629 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:12:33 -0500 Subject: [PATCH 04/16] feat(dogfood): assert tamper refusal and probe expectations With DOGFOOD_TAMPER, --check now passes only if the initrd's pull fails after the image, SHA256SUMS and SHA256SUMS.gpg were served and nothing booted; a systemd.unit-dropin credential copies the pull's own messages to the serial console so the log says why. DOGFOOD_EXPECT= makes --check also require a probe line (e.g. Ignition's marker). Robustness: key enrollment for HTTP boot waits (bounded) for the firmware to reset after systemd-boot enrolls instead of a fixed sleep, and keeps its log; the EXIT trap kills QEMU too, so an aborted run no longer leaves an orphaned guest holding the port and the caller's locks. The trap only kills PIDs that are set, since kill 0 signals the whole process group. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/dogfood-diskless.sh | 96 ++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 18 deletions(-) diff --git a/scripts/dogfood-diskless.sh b/scripts/dogfood-diskless.sh index e12cd66f..6c7714a1 100644 --- a/scripts/dogfood-diskless.sh +++ b/scripts/dogfood-diskless.sh @@ -9,8 +9,13 @@ # is handed over as the import.pull system credential via SMBIOS. # # Usage: dogfood-diskless.sh [--check] -# --check headless: boot, run a probe, exit 0 if no unit failed +# --check headless: boot, run a probe, exit 0 if no unit failed. With +# DOGFOOD_TAMPER set, exit 0 only if the initrd refuses the image. # Environment: +# DOGFOOD_PORT= HTTP port for the built-in server (default 8765) +# DOGFOOD_MEM= guest memory (default 4096) +# DOGFOOD_TIMEOUT= --check deadline (default 600) +# DOGFOOD_EXPECT= --check also requires the probe output to match # DOGFOOD_IGNITION= pass an Ignition config as the ignition.config credential # DOGFOOD_STATE_DISK= attach a persistent second disk (/dev/vdb), created if missing # DOGFOOD_EXTRA_PROBE= shell snippet appended to the in-guest probe @@ -21,7 +26,8 @@ # DOGFOOD_BOOT_URL= HTTP boot from another server (e.g. Booty) instead # DOGFOOD_NODE_IGN= serve it as bluefin-node.ign next to the UKI (HTTP boot) # DOGFOOD_SERVE_EXTRA= also serve the files in -# DOGFOOD_TAMPER=raw|sums serve a corrupted image or a re-hashed, unsigned manifest +# DOGFOOD_TAMPER=raw|sums serve a corrupted image or a re-hashed, unsigned manifest; +# --check then passes only if the initrd refuses it set -euo pipefail dir="$(realpath "${1:?usage: $0 [--check]}")" @@ -48,7 +54,9 @@ vars_tmpl="${OVMF_VARS:-$(first_existing \ /usr/share/edk2/x64/OVMF_VARS.4m.fd)}" || { echo "ERROR: no blank OVMF_VARS found (set OVMF_VARS)" >&2; exit 1; } work="$(mktemp -d /tmp/bluefin-dogfood.XXXXXX)" -trap 'kill "${http_pid:-0}" 2>/dev/null || true; rm -rf "${work}"' EXIT +# Never leave a guest behind: an orphaned QEMU keeps its port and the caller's +# locks. Only kill PIDs that are set; `kill 0` signals the whole process group. +trap 'for p in ${qemu_pid:-} ${enroll_pid:-} ${http_pid:-}; do kill "${p}" 2>/dev/null || true; done; rm -rf "${work}"' EXIT vars="${DOGFOOD_VARS:-${work}/vars.fd}" [ -f "${vars}" ] || cp "${vars_tmpl}" "${vars}" cp "${esp}" "${work}/esp.raw" @@ -73,6 +81,19 @@ http_pid=$! cred() { printf 'type=11,value=io.systemd.credential.binary:%s=%s' "$1" "$(base64 -w0 < "$2")"; } +# wait_for : poll the command once a second until +# it succeeds (0), or the process exits or the time runs out (1). +wait_for() { + local deadline=$(( $(date +%s) + $1 )) pid="$2" + shift 2 + while [ "$(date +%s)" -lt "${deadline}" ]; do + "$@" && return 0 + kill -0 "${pid}" 2>/dev/null || { "$@"; return; } + sleep 1 + done + return 1 +} + printf 'raw,machine,verify=signature,blockdev:rootdisk:http://10.0.2.2:%s/%s\n' "${port}" "${image}" > "${work}/import.pull" qemu=(qemu-system-x86_64 -machine q35,smm=on,accel=kvm -cpu host -m "${mem}" -smp 2 @@ -89,14 +110,15 @@ if [ "${boot}" = http ] && [ ! -s "${vars}.enrolled" ]; then -display none -monitor none \ -serial "file:${work}/enroll.log" /dev/null 2>&1 & enroll_pid=$! - for _ in $(seq 1 60); do - grep -aq 'successfully enrolled' "${work}/enroll.log" 2>/dev/null && break - sleep 1 - done - sleep 2 + # systemd-boot resets the machine after enrolling; the firmware loading a + # boot option again proves the variable store has the keys. + enrolled_and_reset() { sed -n '/successfully enrolled/,$p' "${work}/enroll.log" 2>/dev/null | grep -aq 'BdsDxe: loading'; } + rc=0 + wait_for 90 "${enroll_pid}" enrolled_and_reset || rc=$? kill "${enroll_pid}" 2>/dev/null || true wait "${enroll_pid}" 2>/dev/null || true - grep -aq 'successfully enrolled' "${work}/enroll.log" || { echo "ERROR: key enrollment failed" >&2; exit 1; } + cp "${work}/enroll.log" "${dir}/dogfood-enroll.log" 2>/dev/null || true + [ "${rc}" = 0 ] || { echo "ERROR: key enrollment failed (log: ${dir}/dogfood-enroll.log)" >&2; exit 1; } echo enrolled > "${vars}.enrolled" fi if [ "${boot}" = http ]; then @@ -119,6 +141,13 @@ fi if [ -n "${DOGFOOD_IGNITION:-}" ]; then qemu+=(-smbios "$(cred ignition.config "${DOGFOOD_IGNITION}")") fi +tamper="${DOGFOOD_TAMPER:-}" +if [ -n "${tamper}" ]; then + # The initrd's console shows only that the download unit failed; copy the + # pull's own messages there so the log names why the image was refused. + printf '[Service]\nStandardOutput=journal+console\nStandardError=journal+console\n' > "${work}/import-console.conf" + qemu+=(-smbios "$(cred systemd.unit-dropin.systemd-import@.service "${work}/import-console.conf")") +fi echo "Serving ${dir} on :${port}; ${boot} boot of ${ver} (Secure Boot)" if [ "${mode}" != "--check" ]; then @@ -165,23 +194,54 @@ UNIT -smbios "$(cred systemd.extra-unit.dogfood-probe.service "${work}/probe.service")" \ /dev/null 2>&1 & qemu_pid=$! -deadline=$(( $(date +%s) + timeout_s )) +probe_done() { grep -aq 'PROBE failed=' "${work}/probe.log" 2>/dev/null; } +refused() { sed 's/\x1b\[[0-9;?]*[a-zA-Z]//g' "${work}/serial.log" 2>/dev/null | grep -aqE "${refusal_re}"; } +refusal_re='Failed to start Download of .*/'"${image//./\\.}" status=1 -while kill -0 "${qemu_pid}" 2>/dev/null && [ "$(date +%s)" -lt "${deadline}" ]; do - if grep -aq 'PROBE failed=' "${work}/probe.log" 2>/dev/null; then - status=0 - break - fi - sleep 2 -done +if [ -n "${tamper}" ]; then + stop() { probe_done || refused; } +else + stop() { probe_done; } +fi +wait_for "${timeout_s}" "${qemu_pid}" stop && status=0 kill "${qemu_pid}" 2>/dev/null || true wait "${qemu_pid}" 2>/dev/null || true cat "${work}/serial.log" "${work}/probe.log" 2>/dev/null \ | sed 's/\x1b\[[0-9;?]*[a-zA-Z]//g; s/\x1bP[^\x1b]*\x1b\\//g' | tr -d '\r' > "${dir}/dogfood-serial.log" grep -a 'GET ' "${work}/http.log" > "${dir}/dogfood-http.log" || true grep -aoE 'PROBE[ -].*' "${dir}/dogfood-serial.log" || true + +if [ -n "${tamper}" ]; then + served() { grep -aq "\"GET /$1 HTTP/1.1\" 200" "${dir}/dogfood-http.log"; } + if probe_done; then + echo "FAIL: tamper=${tamper}: the tampered image booted (serial log: ${dir}/dogfood-serial.log)" >&2 + status=1 + elif ! refused; then + echo "FAIL: tamper=${tamper}: no refusal within ${timeout_s}s (serial log: ${dir}/dogfood-serial.log)" >&2 + tail -n 40 "${dir}/dogfood-serial.log" >&2 + status=1 + elif ! served "${image}" || ! served SHA256SUMS || ! served SHA256SUMS.gpg; then + # Refused before it had the image and the signed manifest: that is a + # transport failure, not a verification failure. + echo "FAIL: tamper=${tamper}: the pull failed before fetching ${image}, SHA256SUMS and SHA256SUMS.gpg" >&2 + status=1 + else + grep -aE "${refusal_re}" "${dir}/dogfood-serial.log" | sed 's/^/REFUSED /' | head -n 5 + echo "PASS: tamper=${tamper}: ${image} refused after fetching the signed manifest (serial log: ${dir}/dogfood-serial.log)" + status=0 + fi + if [ "${status}" = 0 ] && [ -n "${DOGFOOD_EXPECT:-}" ] && ! grep -aqE -- "${DOGFOOD_EXPECT}" "${dir}/dogfood-serial.log"; then + echo "FAIL: tamper=${tamper}: serial log does not match DOGFOOD_EXPECT=${DOGFOOD_EXPECT}" >&2 + status=1 + fi + exit "${status}" +fi + failed="$(grep -a '\[FAILED\]' "${dir}/dogfood-serial.log" || true)" -if [ "${status}" = 0 ] && [ -z "${failed}" ] && grep -aq 'PROBE failed=0' "${dir}/dogfood-serial.log"; then +if [ "${status}" = 0 ] && [ -n "${DOGFOOD_EXPECT:-}" ] && ! grep -aqE -- "${DOGFOOD_EXPECT}" "${dir}/dogfood-serial.log"; then + echo "FAIL: booted, but the probe output does not match DOGFOOD_EXPECT=${DOGFOOD_EXPECT}" >&2 + status=1 +elif [ "${status}" = 0 ] && [ -z "${failed}" ] && grep -aq 'PROBE failed=0' "${dir}/dogfood-serial.log"; then echo "PASS: ${boot} boot with no failed units (serial log: ${dir}/dogfood-serial.log)" elif [ "${status}" = 0 ]; then echo "FAIL: booted, but units failed:" >&2 From 32a3286be7d40805f14b8f7b8337c042ec695b55 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:12:47 -0500 Subject: [PATCH 05/16] ci(build): boot-test tamper refusal, HTTP boot, Ignition and A/B rollback; skip docs-only builds boot-test now also runs DOGFOOD_TAMPER=raw and =sums (must be refused), Ignition from the ignition.config credential and UEFI HTTP boot with bluefin-node.ign (both must show the fixture's file and enabled unit), and off main the full install -> sysupdate A->B -> boot-counted rollback. The build job exports the two higher-versioned sets it needs (.1, .2) from the warm cache; main skips them so no unpublished release-signed versions exist. Failed boot tests upload their serial logs. A changes job skips build and boot-test for pull requests image-build-needed.py classifies as docs/tests-only; skipped jobs report as passing. On main, an opt-in step pushes all artifacts to cache.projectbluefin.io:11002 when the CASD_CLIENT_CERT variable and CASD_CLIENT_KEY secret exist (the dakota identity), so main builds stop rebuilding the kernel; it is best effort and a no-op without them. The release job is unchanged. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 165 +++++++++++++++++++++++++++++++++++- 1 file changed, 163 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5bcf73f0..b863b8ee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,46 @@ permissions: contents: read jobs: + # Pull requests that cannot change the image set or the boot test (docs, + # unit tests, other workflows; see .github/scripts/image-build-needed.py) + # skip the ~2 h build. A skipped job reports as passing, so `build` and + # `boot-test` can be required checks without blocking those PRs. + changes: + runs-on: ubuntu-26.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read # lists the pull request's changed files + outputs: + image: ${{ steps.decide.outputs.image }} + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.ref || github.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + sparse-checkout: .github/scripts + + - name: Decide whether to build the image set + id: decide + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + if [ "${GITHUB_EVENT_NAME}" != pull_request ]; then + echo "image=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --jq '.[] | .filename, (.previous_filename // empty)' > "${RUNNER_TEMP}/changed.txt" + image="$(python3 .github/scripts/image-build-needed.py < "${RUNNER_TEMP}/changed.txt")" + echo "image=${image}" >> "$GITHUB_OUTPUT" + echo "Image build needed: ${image}" + build: + needs: changes + if: needs.changes.outputs.image == 'true' runs-on: ubuntu-26.04 timeout-minutes: 300 outputs: @@ -102,6 +141,37 @@ jobs: mkdir -p ~/.cache ln -sf /mnt/buildstream ~/.cache/buildstream + # Opt-in: with the CASD client identity configured (repository variable + # CASD_CLIENT_CERT, secret CASD_CLIENT_KEY), builds on main push their + # artifacts to the shared BuildStream cache so later builds pull them. + # Without it, and always on pull requests, the caches stay pull-only. + - name: Configure BuildStream cache push + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + env: + CASD_CLIENT_CERT: ${{ vars.CASD_CLIENT_CERT }} + CASD_CLIENT_KEY: ${{ secrets.CASD_CLIENT_KEY }} + run: | + set -euo pipefail + if [ -z "${CASD_CLIENT_CERT}" ] || [ -z "${CASD_CLIENT_KEY}" ]; then + echo "No CASD client identity configured; not pushing to the BuildStream cache." + exit 0 + fi + umask 077 + mkdir -p .bst-push + printf '%s\n' "${CASD_CLIENT_CERT}" > .bst-push/client.crt + printf '%s\n' "${CASD_CLIENT_KEY}" > .bst-push/client.key + cat > .bst-push/buildstream.conf <<'EOF' + artifacts: + override-project-caches: false + servers: + - url: https://cache.projectbluefin.io:11002 + push: true + auth: + client-cert: /src/.bst-push/client.crt + client-key: /src/.bst-push/client.key + EOF + echo "BST_PUSH_CONFIG=/src/.bst-push/buildstream.conf" >> "$GITHUB_ENV" + - name: Build and export the image set run: just export-image @@ -115,12 +185,55 @@ jobs: if-no-files-found: error retention-days: 1 + # Best effort: a cache outage must not fail a release build. --deps all + # reaches the kernel, which FSDK's caches never hold for our module + # certificate and which dominates the build time. + - name: Push artifacts to the BuildStream cache + if: env.BST_PUSH_CONFIG != '' + continue-on-error: true + env: + BST_FLAGS: --config ${{ env.BST_PUSH_CONFIG }} + run: just bst artifact push --deps all oci/bluefin-server-image.bst + + - name: Remove the BuildStream cache identity + if: always() + run: rm -rf .bst-push + + # The A/B update and boot-counted rollback test needs two more image + # sets that sort above this one under strverscmp(). Only the version + # differs, so each is a warm rebuild of the version-stamped elements. + # Not on main: these would be release-signed versions nobody publishes. + - name: Build the next and broken image sets + if: github.ref != 'refs/heads/main' + env: + VERSION: ${{ steps.version.outputs.version }} + run: | + set -euo pipefail + just set-version "${VERSION}.1" + just export-image dist/diskless-next + just set-version "${VERSION}.2" + just export-image dist/diskless-broken + + - name: Upload update test image sets + if: github.ref != 'refs/heads/main' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: image-update + path: | + dist/diskless-next/ + dist/diskless-broken/ + !dist/*/bluefin-server-installer_*.raw + if-no-files-found: error + retention-days: 1 + # Secure Boot QEMU boots of the built images: diskless (network pull, - # dm-verity /usr, tmpfs root) and diskless -> systemd-sysinstall -> disk. + # dm-verity /usr, tmpfs root), refusal of a tampered image or manifest, + # UEFI HTTP boot, Ignition applying a config, and diskless -> + # systemd-sysinstall -> disk; off main also A/B update and rollback. boot-test: needs: build runs-on: ubuntu-26.04 - timeout-minutes: 60 + timeout-minutes: 90 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -134,6 +247,13 @@ jobs: name: image path: dist/diskless + - name: Download update test image sets + if: github.ref != 'refs/heads/main' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: image-update + path: dist + - name: Install QEMU and OVMF run: | set -euo pipefail @@ -144,9 +264,50 @@ jobs: - name: Diskless boot run: bash scripts/dogfood-diskless.sh dist/diskless --check + - name: Refuse a tampered image + env: + DOGFOOD_TAMPER: raw + run: bash scripts/dogfood-diskless.sh dist/diskless --check + + - name: Refuse an unsigned manifest + env: + DOGFOOD_TAMPER: sums + run: bash scripts/dogfood-diskless.sh dist/diskless --check + + - name: Ignition config from a credential + env: + DOGFOOD_IGNITION: tests/fixtures/ignition/apply-marker.ign + DOGFOOD_EXTRA_PROBE: tests/fixtures/ignition/apply-marker.probe + DOGFOOD_EXPECT: "PROBE ignition marker=applied unit=active enabled=enabled ran=yes" + run: bash scripts/dogfood-diskless.sh dist/diskless --check + + - name: UEFI HTTP boot with bluefin-node.ign + env: + DOGFOOD_BOOT: http + DOGFOOD_NODE_IGN: tests/fixtures/ignition/apply-marker.ign + DOGFOOD_EXTRA_PROBE: tests/fixtures/ignition/apply-marker.probe + DOGFOOD_EXPECT: "PROBE ignition marker=applied unit=active enabled=enabled ran=yes" + run: bash scripts/dogfood-diskless.sh dist/diskless --check + - name: Install to disk and boot it + if: github.ref == 'refs/heads/main' run: bash scripts/dogfood-install.sh dist/diskless + - name: Install, A/B update and boot-counted rollback + if: github.ref != 'refs/heads/main' + run: bash scripts/dogfood-install.sh dist/diskless dist/diskless-next dist/diskless-broken + + - name: Upload boot logs + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: boot-test-logs + path: | + dist/*/dogfood-*.log + dist/dogfood-install/*.log + if-no-files-found: ignore + retention-days: 7 + # Publishes the signed image set as a new, immutable GitHub Release and as # an ORAS OCI artifact (one layer per file, tags and latest). # Only runs on pushes/dispatches to main, where the code is already From 37edf30d9c5fb01586ad93295192af087666b831 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:12:47 -0500 Subject: [PATCH 06/16] docs(skills): CI boot coverage, build time and cache setup ci-tooling.md is canonical for what boot-test covers, the measured build-time breakdown (kernel on the critical path), the docs-only skip, the CASD_CLIENT_CERT / CASD_CLIENT_KEY cache push, and why PR kernels and actions/cache are out of scope. ddi-installer-build.md documents DOGFOOD_EXPECT/PORT/MEM/TIMEOUT, the tamper pass criterion, how to make next/broken sets, and points at ci-tooling.md for CI coverage. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/ci-tooling.md | 84 ++++++++++++++++++++++++++---- docs/skills/ddi-installer-build.md | 21 +++++--- 2 files changed, 90 insertions(+), 15 deletions(-) diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index 4eb28c2e..02f6fcbf 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -4,7 +4,7 @@ description: CI workflow conventions for Bluefin Server. Use when writing or edi metadata: type: reference status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" context7-sources: - /websites/github_en_actions - /websites/cli_github_manual @@ -83,8 +83,9 @@ freedesktop-sdk and gnome-build-meta bumps. It now lives in `track-junctions.yml` on a schedule, opening its own PR on its own branch. The `build` job (validation, compile, signing) runs with the read-only default -on every event. If a new job needs additional permissions, keep them as narrow -as possible and document why. +on every event. `changes` adds only `pull-requests: read`, to list a pull +request's files. If a new job needs additional permissions, keep them as +narrow as possible and document why. ### `sudo` scope @@ -116,8 +117,9 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e | Job | Workflow | Trigger | Purpose | |-----|----------|---------|---------| | `track-junctions` | `track-junctions.yml` | `schedule` (08:00 UTC), `workflow_dispatch` | Resolves the `freedesktop-sdk.bst` + `gnome-build-meta.bst` junction refs, syncs `project.conf`'s `installer-version`, and opens/updates its own PR on `auto/track-junctions`. `contents: write` + `pull-requests: write`, never on `pull_request`. | -| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there. Read-only token. | -| `boot-test` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Downloads the build job's exported image set and runs the Secure Boot QEMU checks: `scripts/dogfood-diskless.sh --check` (diskless boot) and `scripts/dogfood-install.sh` (install to disk and boot it). Read-only token. | +| `changes` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Decides whether `build` and `boot-test` run: always outside pull requests; on a pull request only if `.github/scripts/image-build-needed.py` finds a changed path that can reach the image set or the boot test (see [Build time and caches](#build-time-and-caches)). `contents: read` + `pull-requests: read`. | +| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there, and it pushes to the BuildStream cache when the CASD identity is configured. Off `main` it also exports two higher-versioned sets (`.1`, `.2`) for the update test. Read-only token. | +| `boot-test` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Runs the Secure Boot QEMU checks on the exported sets (see Core Process step 4). Read-only token. | | `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`) (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write` + `packages: write`. | | `docs` | `docs-checks.yml` | `pull_request`, `push/main` | Runs markdown and skill metadata checks via `docs-checks.py`. Read-only token. | | `unit` | `unit-tests.yml` | `pull_request`, `push/main` | Runs pytest and BATS unit test suites. Read-only token. | @@ -140,10 +142,22 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). and push to `main`, and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst` (gpg sign plus a `gpgv` proof against the shipped keyring). - 4. **Boot test:** Downloads the exported image set and runs - `scripts/dogfood-diskless.sh --check` (diskless Secure Boot boot) and - `scripts/dogfood-install.sh` (diskless boot, `systemd-sysinstall` to disk, - boot the installed disk) in QEMU with OVMF. + 4. **Boot test:** Downloads the exported image sets and runs, in QEMU with + Secure Boot OVMF, each as one `scripts/dogfood-diskless.sh --check` or + `scripts/dogfood-install.sh` call: + - diskless netboot, no failed units; + - `DOGFOOD_TAMPER=raw` and `=sums`: the initrd must refuse a corrupted DDI + and a re-hashed, unsigned `SHA256SUMS` after fetching the image and the + signed manifest (the pull's own messages are copied to the serial log); + - Ignition from the `ignition.config` credential, and UEFI HTTP boot with + `bluefin-node.ign` served next to the UKI, both with + `tests/fixtures/ignition/apply-marker.ign`: the probe must see the + written file and the Ignition-enabled unit active (`DOGFOOD_EXPECT`); + - on `main`: diskless boot, `systemd-sysinstall` to disk, boot the disk; + - off `main`: the same, then `systemd-sysupdate` A->B to `.1` and a + boot-counted rollback from a corrupted `.2`. Main skips this + because its extra sets would be release-signed versions nobody + publishes; every change reaches main through a pull request that ran it. 5. **Version Derivation:** The release version is set per build with `just set-version`: `YY.MM.` on main, `0.` on pull requests so a PR build can never sort above a release. @@ -153,6 +167,56 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). `ghcr.io//bluefin-server:,latest`. Nodes verify updates against the `SHA256SUMS` / `SHA256SUMS.gpg` already in that set. +## Build time and caches + +Measured on run 36499270842 (pull request, cold runner): `Build and export the +image set` took 2 h. BuildStream pulled 239 artifacts and built 85 (about 3.9 h +of build time over 4 cores). The critical path is FSDK's +`components/linux.bst`: 6 min to fetch its source (not in any source cache) +and 1 h 43 min to build. FSDK's caches never hold it for us, because the +`components/linux-module-cert.bst` junction override (our module certificate) +and `patches/freedesktop-sdk/0006-linux-*.patch` change its cache key. Next are +about 40 FSDK elements our FSDK patches or their reverse dependencies change +(glib-stage1, gobject-introspection, harfbuzz, go, vala, ...; about 70 min of +build time in parallel with the kernel). Changing only `image-version` +rebuilds 13 version-stamped elements (os-release to `oci/bluefin-server-image.bst` +and the sysexts), 2 min locally on a warm cache; CI's +per-set cost is estimated at under 10 min. + +- **Docs-only pull requests skip the build.** The `changes` job feeds the PR's + changed paths (renames under both names) to + `.github/scripts/image-build-needed.py`. It answers `false` only when every + path is docs, Markdown outside the build inputs, `tests/unit/`, + `tests/e2e/`, or a workflow or script that does not build the image; + `tests/unit/test_image_build_needed.py` fails if any tracked build input + (`elements/`, `files/`, `include/`, `patches/`, `plugins/`, `scripts/`, + `tests/fixtures/`, `project.conf`, `Justfile`, `build.yml`) would skip. No + status check is required on `main` today; a skipped job reports as passing, + so they can be made required without `paths-ignore` leaving them pending. +- **Cache push from main (opt-in).** `project.conf` recommends the GNOME and + Bluefin caches pull-only. With the repository variable `CASD_CLIENT_CERT` + (PEM client certificate) and secret `CASD_CLIENT_KEY` (its key), the same + mTLS identity `projectbluefin/dakota` uses for + `https://cache.projectbluefin.io:11002`, builds on `main` write a global + `artifacts` remote (`push: true`, `override-project-caches: false`) and run + `bst artifact push --deps all oci/bluefin-server-image.bst` after the + upload, as a best-effort step. Without them the step is a no-op. Main builds + then pull the kernel instead of rebuilding it while its inputs and the + release module certificate are unchanged. +- **Pull requests still build the kernel.** `just gen-dev-keys` makes a new + module certificate on every PR run, so the PR kernel's cache key never + matches anything cached. Caching it needs a stable, non-release PR module + key pair that PR builds can use (fork PRs get no secrets) and a job that + pushes the kernel built with it; that is a maintainer decision about key + handling, not a workflow change. +- **No `actions/cache` for sources or artifacts.** The kernel source fetch + (6 min, 1 of the 85 fetches) is the only one on the critical path, and it + only matters when the kernel is rebuilt anyway. The runner's BuildStream + cache after a build holds every pulled and built artifact of the graph, + several times the 2.1 GB image set, and would churn the 10 GB repository + cache quota; BuildStream 2 cannot import a single artifact from a tarball, + so the kernel alone cannot be cached that way. + ## Common Rationalizations | Rationalization | Reality | @@ -175,6 +239,8 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). - [ ] Every `uses:` line has a full 40-character SHA and a `# vX` comment. - [ ] Every `install-action` `tool:` names an explicit version (`just@1.58.0`). - [ ] `just validate` passes after workflow changes. +- [ ] A new directory the build or boot test reads is listed in + `BUILD_PREFIXES` in `.github/scripts/image-build-needed.py`. - [ ] No new mutable action refs introduced. - [ ] Release signing happens in `oci/bluefin-server-image.bst`; there is no separate CI signing step, and the release job publishes `dist/diskless/` diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 0d8e1067..cc7e2ac2 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -4,7 +4,7 @@ description: Build, export, and dogfood the Bluefin Server image set (OS DDI, si metadata: type: how-to status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" context7-sources: - /systemd/systemd - /apache/buildstream @@ -99,8 +99,15 @@ Useful environment variables: (picked up by HTTP-booted nodes with no Ignition credential). - `DOGFOOD_SERVE_EXTRA=` — also serve the files in ``. - `DOGFOOD_TAMPER=raw|sums` — serve a corrupted DDI or a re-hashed, unsigned - `SHA256SUMS`; the boot must fail, proving the signature check. + `SHA256SUMS`. `--check` then passes only if the initrd's pull fails after + the image, `SHA256SUMS` and `SHA256SUMS.gpg` were served, and nothing + booted; the pull's own messages are copied to the serial log. - `DOGFOOD_EXTRA_PROBE=` — shell snippet appended to the in-guest probe. +- `DOGFOOD_EXPECT=` — `--check` also requires the probe output to match, + e.g. with `tests/fixtures/ignition/apply-marker.ign` and its `.probe`: + `PROBE ignition marker=applied unit=active enabled=enabled ran=yes`. +- `DOGFOOD_PORT`, `DOGFOOD_MEM`, `DOGFOOD_TIMEOUT` — HTTP port (8765), guest + memory in MiB (4096), `--check` deadline in seconds (600). `scripts/dogfood-install.sh [ []]` is the full end-to-end check: install from a diskless boot, boot the installed disk, @@ -108,10 +115,12 @@ end-to-end check: install from a diskless boot, boot the installed disk, the signed manifest (with the `zfs` feature enabled, so the ZFS sysext follows the OS in lock-step), and with `` corrupt the updated slot and confirm boot counting rolls the node back to `` on its own, with the -matching ZFS sysext still merged. CI runs the first two stages -(`dogfood-diskless.sh --check`, `dogfood-install.sh dist/diskless`) as the -`boot-test` job in `.github/workflows/build.yml` on every pull request and -push to main. +matching ZFS sysext still merged. `` and `` are +ordinary image sets with higher versions, e.g. +`just set-version .1 && just export-image dist/diskless-next` (and +`.2` into `dist/diskless-broken`); the script corrupts the broken slot +itself. Which of these scenarios CI runs is listed in +[ci-tooling.md](ci-tooling.md) (the `boot-test` job). ## Local builds with a remote cache From e8eb21da689a92787dc892c436cc821472dd15fc Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 23:27:44 -0500 Subject: [PATCH 07/16] fix(dogfood): make DOGFOOD_TAMPER=sums test the signature, assert the reason The sums tamper overwrote every hash in SHA256SUMS, so the pull failed on the checksum and the signature check was never exercised. It now serves the corrupted image with SHA256SUMS re-hashed to match it, which only SHA256SUMS.gpg can expose. --check requires the pull's own verdict on the serial console: 'DOWNLOAD INVALID: Checksum of file did not check out' for raw, 'DOWNLOAD INVALID: Signature verification failed' for sums. Drop-ins for systemd-import@.service and systemd-importd.service (SYSTEMD_LOG_TARGET=console, journal+console) put those messages there. The EXIT cleanup moves into a function (shellcheck SC2154 on the inline trap). Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/dogfood-diskless.sh | 42 +++++++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/scripts/dogfood-diskless.sh b/scripts/dogfood-diskless.sh index 6c7714a1..ff2f9dc4 100644 --- a/scripts/dogfood-diskless.sh +++ b/scripts/dogfood-diskless.sh @@ -26,8 +26,10 @@ # DOGFOOD_BOOT_URL= HTTP boot from another server (e.g. Booty) instead # DOGFOOD_NODE_IGN= serve it as bluefin-node.ign next to the UKI (HTTP boot) # DOGFOOD_SERVE_EXTRA= also serve the files in -# DOGFOOD_TAMPER=raw|sums serve a corrupted image or a re-hashed, unsigned manifest; -# --check then passes only if the initrd refuses it +# DOGFOOD_TAMPER=raw|sums serve a corrupted image (raw), or a corrupted image with +# SHA256SUMS re-hashed to match it but no longer matching +# SHA256SUMS.gpg (sums); --check then passes only if the +# initrd's pull refuses it for that reason set -euo pipefail dir="$(realpath "${1:?usage: $0 [--check]}")" @@ -56,7 +58,12 @@ vars_tmpl="${OVMF_VARS:-$(first_existing \ work="$(mktemp -d /tmp/bluefin-dogfood.XXXXXX)" # Never leave a guest behind: an orphaned QEMU keeps its port and the caller's # locks. Only kill PIDs that are set; `kill 0` signals the whole process group. -trap 'for p in ${qemu_pid:-} ${enroll_pid:-} ${http_pid:-}; do kill "${p}" 2>/dev/null || true; done; rm -rf "${work}"' EXIT +cleanup() { + local pid + for pid in ${qemu_pid:-} ${enroll_pid:-} ${http_pid:-}; do kill "${pid}" 2>/dev/null || true; done + rm -rf "${work}" +} +trap cleanup EXIT vars="${DOGFOOD_VARS:-${work}/vars.fd}" [ -f "${vars}" ] || cp "${vars_tmpl}" "${vars}" cp "${esp}" "${work}/esp.raw" @@ -68,13 +75,23 @@ mkdir -p "${srv}" for f in "${dir}"/*; do ln -s "${f}" "${srv}/"; done [ -n "${DOGFOOD_NODE_IGN:-}" ] && cp "${DOGFOOD_NODE_IGN}" "${srv}/bluefin-node.ign" if [ -n "${DOGFOOD_SERVE_EXTRA:-}" ]; then for f in "${DOGFOOD_SERVE_EXTRA}"/*; do ln -sf "$(realpath "${f}")" "${srv}/"; done; fi +image_re="${image//./\\.}" case "${DOGFOOD_TAMPER:-}" in - raw) + raw|sums) rm "${srv}/${image}"; cp "${dir}/${image}" "${srv}/${image}" - printf 'X' | dd of="${srv}/${image}" bs=1 seek=4096 conv=notrunc status=none ;; + printf 'X' | dd of="${srv}/${image}" bs=1 seek=4096 conv=notrunc status=none ;;& + raw) + refusal_re="DOWNLOAD INVALID: Checksum of ${image_re} file did not check out" ;; sums) + # What an attacker without the signing key can do: make the manifest + # match the modified image. Only the signature check can catch it. rm "${srv}/SHA256SUMS" - sed 's/^[0-9a-f]\{8\}/deadbeef/' "${dir}/SHA256SUMS" > "${srv}/SHA256SUMS" ;; + sum="$(sha256sum "${srv}/${image}" | cut -d' ' -f1)" + sed -E "s/^[0-9a-f]{64}( [ *]${image_re})$/${sum}\1/" "${dir}/SHA256SUMS" > "${srv}/SHA256SUMS" + grep -q "^${sum} " "${srv}/SHA256SUMS" || { echo "ERROR: ${image} not in SHA256SUMS" >&2; exit 1; } + refusal_re="DOWNLOAD INVALID: Signature verification failed" ;; + '') ;; + *) echo "ERROR: DOGFOOD_TAMPER must be raw or sums" >&2; exit 1 ;; esac (cd "${srv}" && exec python3 -m http.server --bind 127.0.0.1 "${port}" >"${work}/http.log" 2>&1) & http_pid=$! @@ -144,9 +161,13 @@ fi tamper="${DOGFOOD_TAMPER:-}" if [ -n "${tamper}" ]; then # The initrd's console shows only that the download unit failed; copy the - # pull's own messages there so the log names why the image was refused. - printf '[Service]\nStandardOutput=journal+console\nStandardError=journal+console\n' > "${work}/import-console.conf" - qemu+=(-smbios "$(cred systemd.unit-dropin.systemd-import@.service "${work}/import-console.conf")") + # pull's own messages (from the unit and from systemd-importd, which runs + # the transfer) there so the log names why the image was refused. systemd + # tools log natively to the journal when they can, bypassing the stream + # journald forwards, so make them write to stderr. + printf '[Service]\nEnvironment=SYSTEMD_LOG_TARGET=console\nStandardOutput=journal+console\nStandardError=journal+console\n' > "${work}/import-console.conf" + qemu+=(-smbios "$(cred systemd.unit-dropin.systemd-import@.service "${work}/import-console.conf")" + -smbios "$(cred systemd.unit-dropin.systemd-importd.service "${work}/import-console.conf")") fi echo "Serving ${dir} on :${port}; ${boot} boot of ${ver} (Secure Boot)" @@ -196,7 +217,6 @@ UNIT qemu_pid=$! probe_done() { grep -aq 'PROBE failed=' "${work}/probe.log" 2>/dev/null; } refused() { sed 's/\x1b\[[0-9;?]*[a-zA-Z]//g' "${work}/serial.log" 2>/dev/null | grep -aqE "${refusal_re}"; } -refusal_re='Failed to start Download of .*/'"${image//./\\.}" status=1 if [ -n "${tamper}" ]; then stop() { probe_done || refused; } @@ -226,7 +246,7 @@ if [ -n "${tamper}" ]; then echo "FAIL: tamper=${tamper}: the pull failed before fetching ${image}, SHA256SUMS and SHA256SUMS.gpg" >&2 status=1 else - grep -aE "${refusal_re}" "${dir}/dogfood-serial.log" | sed 's/^/REFUSED /' | head -n 5 + grep -aE "${refusal_re}|Failed to start Download of " "${dir}/dogfood-serial.log" | sed 's/^/REFUSED /' | head -n 5 echo "PASS: tamper=${tamper}: ${image} refused after fetching the signed manifest (serial log: ${dir}/dogfood-serial.log)" status=0 fi From 5b5ba95a19117d4e4bbeab00f84b99a75ace10d3 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 23:27:44 -0500 Subject: [PATCH 08/16] ci(build): version the update test sets 1..N systemd-sysinstall writes the installed image's boot entry as a Type #1 entry with 'version 1.' (bluefin-server-commit_1..conf). systemd-boot sorts it above a 0..1 update, so the updated node kept booting 0. and the A/B step failed (reproduced locally with 0.900 -> 0.900.1). 1..N sorts above both and below every release; with it the full install, A/B update and rollback pass locally. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b863b8ee..cdcabe8e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -200,18 +200,20 @@ jobs: run: rm -rf .bst-push # The A/B update and boot-counted rollback test needs two more image - # sets that sort above this one under strverscmp(). Only the version - # differs, so each is a warm rebuild of the version-stamped elements. - # Not on main: these would be release-signed versions nobody publishes. + # sets that sort above this one. Only the version differs, so each is a + # warm rebuild of the version-stamped elements. Not on main: these would + # be release-signed versions nobody publishes. + # 1..N, not 0..N: systemd-sysinstall's boot entry for the + # installed 0. carries `version 1.0.`, and systemd-boot would + # keep defaulting to it over a 0..N update. 1..N sorts above + # both and still below every release (YY.MM.). - name: Build the next and broken image sets if: github.ref != 'refs/heads/main' - env: - VERSION: ${{ steps.version.outputs.version }} run: | set -euo pipefail - just set-version "${VERSION}.1" + just set-version "1.${GITHUB_RUN_NUMBER}.1" just export-image dist/diskless-next - just set-version "${VERSION}.2" + just set-version "1.${GITHUB_RUN_NUMBER}.2" just export-image dist/diskless-broken - name: Upload update test image sets From 0f718b3b9dc28f623f1c4831c8b17ce4f6367fef Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 23:27:44 -0500 Subject: [PATCH 09/16] docs(skills): tamper reasons and update test versions Document which refusal each DOGFOOD_TAMPER mode asserts, and that next/broken sets must sort above the installed entry's 'version 1.'. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/ci-tooling.md | 17 +++++++++++------ docs/skills/ddi-installer-build.md | 20 +++++++++++++------- 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index 02f6fcbf..e3ef2eab 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -118,7 +118,7 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e |-----|----------|---------|---------| | `track-junctions` | `track-junctions.yml` | `schedule` (08:00 UTC), `workflow_dispatch` | Resolves the `freedesktop-sdk.bst` + `gnome-build-meta.bst` junction refs, syncs `project.conf`'s `installer-version`, and opens/updates its own PR on `auto/track-junctions`. `contents: write` + `pull-requests: write`, never on `pull_request`. | | `changes` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Decides whether `build` and `boot-test` run: always outside pull requests; on a pull request only if `.github/scripts/image-build-needed.py` finds a changed path that can reach the image set or the boot test (see [Build time and caches](#build-time-and-caches)). `contents: read` + `pull-requests: read`. | -| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there, and it pushes to the BuildStream cache when the CASD identity is configured. Off `main` it also exports two higher-versioned sets (`.1`, `.2`) for the update test. Read-only token. | +| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there, and it pushes to the BuildStream cache when the CASD identity is configured. Off `main` it also exports two higher-versioned sets (`1..1`, `1..2`) for the update test. Read-only token. | | `boot-test` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Runs the Secure Boot QEMU checks on the exported sets (see Core Process step 4). Read-only token. | | `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`) (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write` + `packages: write`. | | `docs` | `docs-checks.yml` | `pull_request`, `push/main` | Runs markdown and skill metadata checks via `docs-checks.py`. Read-only token. | @@ -146,16 +146,21 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). Secure Boot OVMF, each as one `scripts/dogfood-diskless.sh --check` or `scripts/dogfood-install.sh` call: - diskless netboot, no failed units; - - `DOGFOOD_TAMPER=raw` and `=sums`: the initrd must refuse a corrupted DDI - and a re-hashed, unsigned `SHA256SUMS` after fetching the image and the - signed manifest (the pull's own messages are copied to the serial log); + - `DOGFOOD_TAMPER=raw`: a corrupted DDI must be refused by the manifest + check (`DOWNLOAD INVALID: Checksum of ... did not check out`); + `DOGFOOD_TAMPER=sums`: the same DDI with `SHA256SUMS` re-hashed to match + it must be refused by the signature check (`DOWNLOAD INVALID: Signature + verification failed`). Both only after the image, `SHA256SUMS` and + `SHA256SUMS.gpg` were served, and nothing may boot; - Ignition from the `ignition.config` credential, and UEFI HTTP boot with `bluefin-node.ign` served next to the UKI, both with `tests/fixtures/ignition/apply-marker.ign`: the probe must see the written file and the Ignition-enabled unit active (`DOGFOOD_EXPECT`); - on `main`: diskless boot, `systemd-sysinstall` to disk, boot the disk; - - off `main`: the same, then `systemd-sysupdate` A->B to `.1` and a - boot-counted rollback from a corrupted `.2`. Main skips this + - off `main`: the same, then `systemd-sysupdate` A->B to `1..1` and + a boot-counted rollback from a corrupted `1..2` (see + [ddi-installer-build.md](ddi-installer-build.md) for why not + `0..N`). Main skips this because its extra sets would be release-signed versions nobody publishes; every change reaches main through a pull request that ran it. 5. **Version Derivation:** The release version is set per build with diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index cc7e2ac2..9814c06d 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -98,10 +98,12 @@ Useful environment variables: - `DOGFOOD_NODE_IGN=` — serve it as `bluefin-node.ign` next to the UKI (picked up by HTTP-booted nodes with no Ignition credential). - `DOGFOOD_SERVE_EXTRA=` — also serve the files in ``. -- `DOGFOOD_TAMPER=raw|sums` — serve a corrupted DDI or a re-hashed, unsigned - `SHA256SUMS`. `--check` then passes only if the initrd's pull fails after - the image, `SHA256SUMS` and `SHA256SUMS.gpg` were served, and nothing - booted; the pull's own messages are copied to the serial log. +- `DOGFOOD_TAMPER=raw|sums` — serve a corrupted DDI (`raw`), or the corrupted + DDI with `SHA256SUMS` re-hashed to match it, so only `SHA256SUMS.gpg` no + longer fits (`sums`). `--check` then passes only if the initrd's pull + refuses it for that reason (checksum mismatch, bad signature) after the + image and both manifest files were served, and nothing booted. Credential + drop-ins copy `systemd-importd`'s messages to the serial console. - `DOGFOOD_EXTRA_PROBE=` — shell snippet appended to the in-guest probe. - `DOGFOOD_EXPECT=` — `--check` also requires the probe output to match, e.g. with `tests/fixtures/ignition/apply-marker.ign` and its `.probe`: @@ -117,9 +119,13 @@ the OS in lock-step), and with `` corrupt the updated slot and confirm boot counting rolls the node back to `` on its own, with the matching ZFS sysext still merged. `` and `` are ordinary image sets with higher versions, e.g. -`just set-version .1 && just export-image dist/diskless-next` (and -`.2` into `dist/diskless-broken`); the script corrupts the broken slot -itself. Which of these scenarios CI runs is listed in +`just set-version && just export-image dist/diskless-next` (and a +higher one into `dist/diskless-broken`); the script corrupts the broken slot +itself. The versions must also sort above `1.` for systemd-boot: the +Type #1 entry `systemd-sysinstall` writes for the installed image carries +`version 1.` (`bluefin-server-commit_1..conf`), so after +installing `0.674` an update to `0.674.1` still boots `0.674`; use `1.674.1`. +Release versions (`YY.MM.`) sort above it. Which of these scenarios CI runs is listed in [ci-tooling.md](ci-tooling.md) (the `boot-test` job). ## Local builds with a remote cache From 5c29f02674adfa7f006853f9e0cb24255109649c Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 23:45:15 -0500 Subject: [PATCH 10/16] fix(dogfood): fail --check when Secure Boot is not enforced Firmware that refuses systemd-boot's key enrollment boots on in setup mode, where nothing is verified, and --check still passed. CI's boot-test did exactly that on Ubuntu's OVMF: every probe reported 'secureboot=disabled (setup)' (e.g. run 36499270842). --check now requires 'PROBE secureboot=enabled', and a tamper run requires the kernel's 'Secure boot enabled' line. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/dogfood-diskless.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/dogfood-diskless.sh b/scripts/dogfood-diskless.sh index ff2f9dc4..70b1f28e 100644 --- a/scripts/dogfood-diskless.sh +++ b/scripts/dogfood-diskless.sh @@ -9,8 +9,9 @@ # is handed over as the import.pull system credential via SMBIOS. # # Usage: dogfood-diskless.sh [--check] -# --check headless: boot, run a probe, exit 0 if no unit failed. With -# DOGFOOD_TAMPER set, exit 0 only if the initrd refuses the image. +# --check headless: boot, run a probe, exit 0 if Secure Boot was enforced and no +# unit failed. With DOGFOOD_TAMPER set, exit 0 only if the initrd +# refuses the image. # Environment: # DOGFOOD_PORT= HTTP port for the built-in server (default 8765) # DOGFOOD_MEM= guest memory (default 4096) @@ -245,6 +246,9 @@ if [ -n "${tamper}" ]; then # transport failure, not a verification failure. echo "FAIL: tamper=${tamper}: the pull failed before fetching ${image}, SHA256SUMS and SHA256SUMS.gpg" >&2 status=1 + elif ! grep -aq 'Secure boot enabled' "${dir}/dogfood-serial.log"; then + echo "FAIL: tamper=${tamper}: refused, but the kernel did not run with Secure Boot enabled" >&2 + status=1 else grep -aE "${refusal_re}|Failed to start Download of " "${dir}/dogfood-serial.log" | sed 's/^/REFUSED /' | head -n 5 echo "PASS: tamper=${tamper}: ${image} refused after fetching the signed manifest (serial log: ${dir}/dogfood-serial.log)" @@ -261,6 +265,11 @@ failed="$(grep -a '\[FAILED\]' "${dir}/dogfood-serial.log" || true)" if [ "${status}" = 0 ] && [ -n "${DOGFOOD_EXPECT:-}" ] && ! grep -aqE -- "${DOGFOOD_EXPECT}" "${dir}/dogfood-serial.log"; then echo "FAIL: booted, but the probe output does not match DOGFOOD_EXPECT=${DOGFOOD_EXPECT}" >&2 status=1 +elif [ "${status}" = 0 ] && ! grep -aq 'PROBE secureboot=enabled' "${dir}/dogfood-serial.log"; then + # Firmware that refuses the enrollment payloads boots on in setup mode, + # where nothing is verified; that is not a Secure Boot boot. + echo "FAIL: booted without Secure Boot ($(grep -aoE 'PROBE secureboot=.*' "${dir}/dogfood-serial.log" | head -n1)); see the enrollment messages in the serial log" >&2 + status=1 elif [ "${status}" = 0 ] && [ -z "${failed}" ] && grep -aq 'PROBE failed=0' "${dir}/dogfood-serial.log"; then echo "PASS: ${boot} boot with no failed units (serial log: ${dir}/dogfood-serial.log)" elif [ "${status}" = 0 ]; then From 02b55b3f017912d1ba992daf4793283b63e0b545 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 23:45:15 -0500 Subject: [PATCH 11/16] ci(build): boot-test on Fedora's edk2 OVMF so Secure Boot is enforced Ubuntu 26.04's OVMF 2025.11 rejects systemd-boot's PK enrollment ('Failed to write PK secure boot variable: Security violation'), so boot-test never ran with Secure Boot on, and the new HTTP boot step, which enrolls explicitly, fails outright. Fetch Fedora's edk2-ovmf 20260508-6.fc44 from Koji, check its SHA-256, extract OVMF_CODE.secboot.fd / OVMF_VARS.fd with bsdtar and point the harness at them. It is the build dogfooding uses locally, where enrollment works. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cdcabe8e..e274fd84 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -256,12 +256,26 @@ jobs: name: image-update path: dist + # Ubuntu's OVMF (2025.11) rejects systemd-boot's PK enrollment with + # "Security violation", so the guests kept booting in setup mode with + # nothing verified. Use Fedora's edk2 build, pinned by checksum, which + # enrolls the keys; dogfood-diskless.sh now fails if Secure Boot is off. - name: Install QEMU and OVMF + env: + OVMF_RPM_URL: https://kojipkgs.fedoraproject.org/packages/edk2/20260508/6.fc44/noarch/edk2-ovmf-20260508-6.fc44.noarch.rpm + OVMF_RPM_SHA256: 8b157ab2aa0cccccf021f43011308349147c76ae59550f52239ea50678dd0a5f run: | set -euo pipefail sudo apt-get update - sudo apt-get install -y --no-install-recommends qemu-system-x86 ovmf + sudo apt-get install -y --no-install-recommends qemu-system-x86 libarchive-tools sudo chmod 666 /dev/kvm + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/edk2-ovmf.rpm" "${OVMF_RPM_URL}" + echo "${OVMF_RPM_SHA256} ${RUNNER_TEMP}/edk2-ovmf.rpm" | sha256sum -c - + mkdir -p "${RUNNER_TEMP}/ovmf" + bsdtar -xf "${RUNNER_TEMP}/edk2-ovmf.rpm" -C "${RUNNER_TEMP}/ovmf" \ + ./usr/share/edk2/ovmf/OVMF_CODE.secboot.fd ./usr/share/edk2/ovmf/OVMF_VARS.fd + echo "OVMF_CODE=${RUNNER_TEMP}/ovmf/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd" >> "$GITHUB_ENV" + echo "OVMF_VARS=${RUNNER_TEMP}/ovmf/usr/share/edk2/ovmf/OVMF_VARS.fd" >> "$GITHUB_ENV" - name: Diskless boot run: bash scripts/dogfood-diskless.sh dist/diskless --check From 57d94a4ab6474dc151e04d447d0bb88abcc4e1cd Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 23:45:15 -0500 Subject: [PATCH 12/16] docs(skills): Secure Boot enforcement in boot-test and the OVMF pin Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/ci-tooling.md | 9 ++++++++- docs/skills/ddi-installer-build.md | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index e3ef2eab..dc8286f3 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -144,7 +144,14 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). shipped keyring). 4. **Boot test:** Downloads the exported image sets and runs, in QEMU with Secure Boot OVMF, each as one `scripts/dogfood-diskless.sh --check` or - `scripts/dogfood-install.sh` call: + `scripts/dogfood-install.sh` call. The firmware is Fedora's + `edk2-ovmf` (Koji URL + SHA-256 in `build.yml`), not Ubuntu's `ovmf`: + Ubuntu 26.04's OVMF 2025.11 rejects systemd-boot's PK enrollment + (`Failed to write PK secure boot variable: Security violation`), and + until this was caught every CI boot ran in setup mode with Secure Boot + off. `--check` now fails unless the probe reports + `secureboot=enabled` (or, for tamper runs, the kernel logs + `Secure boot enabled`). Bump the pin by hand; Renovate does not track it. - diskless netboot, no failed units; - `DOGFOOD_TAMPER=raw`: a corrupted DDI must be refused by the manifest check (`DOWNLOAD INVALID: Checksum of ... did not check out`); diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 9814c06d..903dbcfa 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -69,6 +69,10 @@ their images are never published. All dogfood paths boot with Secure Boot firmware (OVMF secboot). The firmware starts in setup mode; systemd-boot enrolls the dev keys from the ESP (`secure-boot-enroll if-safe`) and reboots, so every later boot is verified. +`--check` fails if the guest did not boot with Secure Boot enabled: some OVMF +builds (Ubuntu 26.04's 2025.11) refuse the enrollment and would otherwise boot +on in setup mode, verifying nothing. Point `OVMF_CODE` / `OVMF_VARS` at +another build (Fedora's `edk2-ovmf` works) if yours does. ```bash just dogfood # interactive diskless boot of dist/diskless/ From 1e37fa086f9b15781fa4330b84bb031f5bfab935 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 01:43:18 -0500 Subject: [PATCH 13/16] ci(build): drop the BuildStream cache push bst artifact push --deps all oci/bluefin-server-image.bst would upload bluefin-server/keys/boot-keys.bst, an import of files/boot-keys that on main holds the Secure Boot db, module-signing and sysupdate private keys, to a cache every build pulls from. Remove the push steps and the CASD_CLIENT_CERT/CASD_CLIENT_KEY configuration; ci-tooling.md records that pushing needs an element allow-list proven not to depend on boot-keys.bst. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 45 ------------------------------------- docs/skills/ci-tooling.md | 23 ++++++++++--------- 2 files changed, 12 insertions(+), 56 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e274fd84..c295bc7b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -141,37 +141,6 @@ jobs: mkdir -p ~/.cache ln -sf /mnt/buildstream ~/.cache/buildstream - # Opt-in: with the CASD client identity configured (repository variable - # CASD_CLIENT_CERT, secret CASD_CLIENT_KEY), builds on main push their - # artifacts to the shared BuildStream cache so later builds pull them. - # Without it, and always on pull requests, the caches stay pull-only. - - name: Configure BuildStream cache push - if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' - env: - CASD_CLIENT_CERT: ${{ vars.CASD_CLIENT_CERT }} - CASD_CLIENT_KEY: ${{ secrets.CASD_CLIENT_KEY }} - run: | - set -euo pipefail - if [ -z "${CASD_CLIENT_CERT}" ] || [ -z "${CASD_CLIENT_KEY}" ]; then - echo "No CASD client identity configured; not pushing to the BuildStream cache." - exit 0 - fi - umask 077 - mkdir -p .bst-push - printf '%s\n' "${CASD_CLIENT_CERT}" > .bst-push/client.crt - printf '%s\n' "${CASD_CLIENT_KEY}" > .bst-push/client.key - cat > .bst-push/buildstream.conf <<'EOF' - artifacts: - override-project-caches: false - servers: - - url: https://cache.projectbluefin.io:11002 - push: true - auth: - client-cert: /src/.bst-push/client.crt - client-key: /src/.bst-push/client.key - EOF - echo "BST_PUSH_CONFIG=/src/.bst-push/buildstream.conf" >> "$GITHUB_ENV" - - name: Build and export the image set run: just export-image @@ -185,20 +154,6 @@ jobs: if-no-files-found: error retention-days: 1 - # Best effort: a cache outage must not fail a release build. --deps all - # reaches the kernel, which FSDK's caches never hold for our module - # certificate and which dominates the build time. - - name: Push artifacts to the BuildStream cache - if: env.BST_PUSH_CONFIG != '' - continue-on-error: true - env: - BST_FLAGS: --config ${{ env.BST_PUSH_CONFIG }} - run: just bst artifact push --deps all oci/bluefin-server-image.bst - - - name: Remove the BuildStream cache identity - if: always() - run: rm -rf .bst-push - # The A/B update and boot-counted rollback test needs two more image # sets that sort above this one. Only the version differs, so each is a # warm rebuild of the version-stamped elements. Not on main: these would diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index dc8286f3..c5c07bd6 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -118,7 +118,7 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e |-----|----------|---------|---------| | `track-junctions` | `track-junctions.yml` | `schedule` (08:00 UTC), `workflow_dispatch` | Resolves the `freedesktop-sdk.bst` + `gnome-build-meta.bst` junction refs, syncs `project.conf`'s `installer-version`, and opens/updates its own PR on `auto/track-junctions`. `contents: write` + `pull-requests: write`, never on `pull_request`. | | `changes` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Decides whether `build` and `boot-test` run: always outside pull requests; on a pull request only if `.github/scripts/image-build-needed.py` finds a changed path that can reach the image set or the boot test (see [Build time and caches](#build-time-and-caches)). `contents: read` + `pull-requests: read`. | -| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there, and it pushes to the BuildStream cache when the CASD identity is configured. Off `main` it also exports two higher-versioned sets (`1..1`, `1..2`) for the update test. Read-only token. | +| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there. Off `main` it also exports two higher-versioned sets (`1..1`, `1..2`) for the update test. Read-only token. | | `boot-test` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Runs the Secure Boot QEMU checks on the exported sets (see Core Process step 4). Read-only token. | | `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`) (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write` + `packages: write`. | | `docs` | `docs-checks.yml` | `pull_request`, `push/main` | Runs markdown and skill metadata checks via `docs-checks.py`. Read-only token. | @@ -205,16 +205,17 @@ per-set cost is estimated at under 10 min. `tests/fixtures/`, `project.conf`, `Justfile`, `build.yml`) would skip. No status check is required on `main` today; a skipped job reports as passing, so they can be made required without `paths-ignore` leaving them pending. -- **Cache push from main (opt-in).** `project.conf` recommends the GNOME and - Bluefin caches pull-only. With the repository variable `CASD_CLIENT_CERT` - (PEM client certificate) and secret `CASD_CLIENT_KEY` (its key), the same - mTLS identity `projectbluefin/dakota` uses for - `https://cache.projectbluefin.io:11002`, builds on `main` write a global - `artifacts` remote (`push: true`, `override-project-caches: false`) and run - `bst artifact push --deps all oci/bluefin-server-image.bst` after the - upload, as a best-effort step. Without them the step is a no-op. Main builds - then pull the kernel instead of rebuilding it while its inputs and the - release module certificate are unchanged. +- **No cache push.** CI only pulls from the caches `project.conf` lists. + `bluefin-server/keys/boot-keys.bst` imports `files/boot-keys/`, which on + `main` holds the Secure Boot, module-signing and sysupdate private keys, and + the image, UKIs, `kernel-modules.bst`, `efi-keys.bst`, + `os-sd-boot-signed.bst` and `openzfs-signed.bst` build-depend on it, so + `bst artifact push --deps all` of the image would upload the keys to a cache + every build pulls from. Pushing needs an explicit element allow-list, with a + unit test proving no listed element is, or build-depends on, + `boot-keys.bst`. FSDK's `components/linux.bst` only stages the public + certificate (`bluefin-server/keys/linux-module-cert.bst`), so the kernel is + the candidate worth listing. - **Pull requests still build the kernel.** `just gen-dev-keys` makes a new module certificate on every PR run, so the PR kernel's cache key never matches anything cached. Caching it needs a stable, non-release PR module From b93e88ab2a9915f2d58ee06d1aae5917b66aa7ed Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 01:43:28 -0500 Subject: [PATCH 14/16] ci(build): run the build classifier from the PR's base revision The changes job checked out .github/scripts from the PR head, so a PR could edit image-build-needed.py to answer false and skip build and boot-test. Check out the base SHA instead, feed it the PR's changed files, and build when the base has no classifier yet. The classifier now lists itself and build.yml as gate files that always build; the unit test asserts that and that the changes job checks out the base. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/image-build-needed.py | 21 ++++++++++++++++----- .github/workflows/build.yml | 13 +++++++++---- tests/unit/test_image_build_needed.py | 27 ++++++++++++++++++++++++--- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/.github/scripts/image-build-needed.py b/.github/scripts/image-build-needed.py index 9d9c572e..82343a18 100644 --- a/.github/scripts/image-build-needed.py +++ b/.github/scripts/image-build-needed.py @@ -3,14 +3,25 @@ Reads the PR's changed paths (one per line, renames listed under both names) on stdin and prints ``true`` or ``false`` for the build workflow's `changes` -job. Only changes that cannot reach the image set or the boot test skip the -~2 h build: docs, Markdown outside the build inputs, unit/e2e tests and the -workflows that run them. Anything else, and anything under a build input -root, builds; an empty or truncated list builds too. +job, which runs this script from the PR's base revision. Only changes that +cannot reach the image set or the boot test skip the ~2 h build: docs, +Markdown outside the build inputs, unit/e2e tests and the workflows that run +them. Anything else, and anything under a build input root, builds; so does +any change to this classifier or to the build workflow, and an empty or +truncated list. """ import sys +# The gate itself: a change to either must be built and boot-tested before +# the base branch's classifier trusts it. +GATE_FILES = frozenset( + { + ".github/scripts/image-build-needed.py", + ".github/workflows/build.yml", + } +) + # Paths the build or the boot test consume. Always build, even for *.md. BUILD_PREFIXES = ( "elements/", @@ -45,7 +56,7 @@ def needs_build(path: str) -> bool: - if path.startswith(BUILD_PREFIXES): + if path in GATE_FILES or path.startswith(BUILD_PREFIXES): return True if path.startswith(SKIP_PREFIXES) or path in SKIP_FILES or path.endswith(".md"): return False diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c295bc7b..77192d66 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,6 +22,9 @@ jobs: # unit tests, other workflows; see .github/scripts/image-build-needed.py) # skip the ~2 h build. A skipped job reports as passing, so `build` and # `boot-test` can be required checks without blocking those PRs. + # The classifier runs from the base revision, never the PR head, so a PR + # cannot edit it to skip its own build; changes to it or to this workflow + # always build. changes: runs-on: ubuntu-26.04 timeout-minutes: 10 @@ -31,12 +34,13 @@ jobs: outputs: image: ${{ steps.decide.outputs.image }} steps: - - name: Checkout repository + - name: Checkout the base revision's scripts + if: github.event_name == 'pull_request' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.pull_request.head.ref || github.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.base.sha }} sparse-checkout: .github/scripts + persist-credentials: false - name: Decide whether to build the image set id: decide @@ -45,7 +49,8 @@ jobs: PR_NUMBER: ${{ github.event.pull_request.number }} run: | set -euo pipefail - if [ "${GITHUB_EVENT_NAME}" != pull_request ]; then + # Before the classifier exists on the base branch, build. + if [ "${GITHUB_EVENT_NAME}" != pull_request ] || [ ! -f .github/scripts/image-build-needed.py ]; then echo "image=true" >> "$GITHUB_OUTPUT" exit 0 fi diff --git a/tests/unit/test_image_build_needed.py b/tests/unit/test_image_build_needed.py index 274f0175..cc8f40eb 100644 --- a/tests/unit/test_image_build_needed.py +++ b/tests/unit/test_image_build_needed.py @@ -2,9 +2,9 @@ The build workflow skips the ~2 h image build and the boot test for pull requests the script classifies as not touching the image. A wrong `false` -merges an untested image change, so the invariant guarded here is that every -tracked file under a build input root, and the build workflow itself, always -builds. +merges an untested image change, so the invariants guarded here are that every +tracked file under a build input root, the build workflow and the classifier +always build, and that the workflow runs the base revision's classifier. """ import importlib.util @@ -13,6 +13,7 @@ from pathlib import Path import pytest +import yaml ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / ".github" / "scripts" / "image-build-needed.py" @@ -46,6 +47,26 @@ def test_the_classifier_itself_builds(): assert image_build_needed.needs_build(".github/scripts/image-build-needed.py") +@pytest.mark.parametrize("path", sorted(image_build_needed.GATE_FILES)) +def test_gate_changes_build_even_among_docs(path): + assert (ROOT / path).is_file(), f"{path} moved; update GATE_FILES" + assert image_build_needed.needs_build(path) + assert image_build_needed.image_build_needed(["docs/a.md", path]) + + +def test_changes_job_runs_the_base_revision_classifier(): + workflow = yaml.safe_load((ROOT / ".github" / "workflows" / "build.yml").read_text()) + steps = workflow["jobs"]["changes"]["steps"] + checkouts = [s for s in steps if str(s.get("uses", "")).startswith("actions/checkout@")] + assert len(checkouts) == 1 + ref = checkouts[0]["with"]["ref"] + assert ref == "${{ github.event.pull_request.base.sha }}" + assert "repository" not in checkouts[0]["with"] + run = "\n".join(s.get("run", "") for s in steps) + assert "python3 .github/scripts/image-build-needed.py" in run + assert "[ ! -f .github/scripts/image-build-needed.py ]" in run + + @pytest.mark.parametrize( "path", [ From 1dd8d09da1bf08305779131a7ee75aa83e7c957e Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 01:43:28 -0500 Subject: [PATCH 15/16] docs(skills): the build classifier runs from the base revision Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/ci-tooling.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index c5c07bd6..08d1a76b 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -117,7 +117,7 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e | Job | Workflow | Trigger | Purpose | |-----|----------|---------|---------| | `track-junctions` | `track-junctions.yml` | `schedule` (08:00 UTC), `workflow_dispatch` | Resolves the `freedesktop-sdk.bst` + `gnome-build-meta.bst` junction refs, syncs `project.conf`'s `installer-version`, and opens/updates its own PR on `auto/track-junctions`. `contents: write` + `pull-requests: write`, never on `pull_request`. | -| `changes` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Decides whether `build` and `boot-test` run: always outside pull requests; on a pull request only if `.github/scripts/image-build-needed.py` finds a changed path that can reach the image set or the boot test (see [Build time and caches](#build-time-and-caches)). `contents: read` + `pull-requests: read`. | +| `changes` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Decides whether `build` and `boot-test` run: always outside pull requests; on a pull request only if `.github/scripts/image-build-needed.py`, checked out from the PR's base revision, finds a changed path that can reach the image set or the boot test (see [Build time and caches](#build-time-and-caches)). `contents: read` + `pull-requests: read`. | | `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there. Off `main` it also exports two higher-versioned sets (`1..1`, `1..2`) for the update test. Read-only token. | | `boot-test` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Runs the Secure Boot QEMU checks on the exported sets (see Core Process step 4). Read-only token. | | `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`) (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write` + `packages: write`. | @@ -197,7 +197,9 @@ per-set cost is estimated at under 10 min. - **Docs-only pull requests skip the build.** The `changes` job feeds the PR's changed paths (renames under both names) to - `.github/scripts/image-build-needed.py`. It answers `false` only when every + `.github/scripts/image-build-needed.py` from the base revision, so a PR + cannot edit the classifier to skip its own build; a change to the + classifier or `build.yml` always builds. It answers `false` only when every path is docs, Markdown outside the build inputs, `tests/unit/`, `tests/e2e/`, or a workflow or script that does not build the image; `tests/unit/test_image_build_needed.py` fails if any tracked build input From 007548941545748866b2e3482fd8701492148bbc Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 08:45:21 -0500 Subject: [PATCH 16/16] fix(dogfood): strip OSC and DCS sequences before matching the tamper refusal Since #273 the initrd writes its failure summary to the console while systemd-importd logs the refusal, so DCS/OSC sequences can land inside the refusal line and refused() missed it (boot-test run 36560436141). Match against the same cleaned log the harness saves. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/dogfood-diskless.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/dogfood-diskless.sh b/scripts/dogfood-diskless.sh index 732162ab..5c3a6e29 100644 --- a/scripts/dogfood-diskless.sh +++ b/scripts/dogfood-diskless.sh @@ -230,7 +230,11 @@ UNIT /dev/null 2>&1 & qemu_pid=$! probe_done() { grep -aq 'PROBE failed=' "${work}/probe.log" 2>/dev/null; } -refused() { sed 's/\x1b\[[0-9;?]*[a-zA-Z]//g' "${work}/serial.log" 2>/dev/null | grep -aqE "${refusal_re}"; } +# Console output interleaves CSI, OSC and DCS escape sequences, sometimes in the +# middle of a line (the initrd's failure summary writes to the console while +# systemd-importd logs), so strip all three before matching. +clean_log() { sed -E 's/\x1b\][^\x07\x1b]*(\x07|\x1b\\)//g; s/\x1bP[^\x1b]*\x1b\\//g; s/\x1b\[[0-9;?]*[a-zA-Z]//g' | tr -d '\r'; } +refused() { clean_log < "${work}/serial.log" 2>/dev/null | grep -aqE "${refusal_re}"; } status=1 if [ -n "${tamper}" ]; then stop() { probe_done || refused; } @@ -241,7 +245,7 @@ wait_for "${timeout_s}" "${qemu_pid}" stop && status=0 kill "${qemu_pid}" 2>/dev/null || true wait "${qemu_pid}" 2>/dev/null || true cat "${work}/serial.log" "${work}/probe.log" 2>/dev/null \ - | sed 's/\x1b\[[0-9;?]*[a-zA-Z]//g; s/\x1bP[^\x1b]*\x1b\\//g' | tr -d '\r' > "${dir}/dogfood-serial.log" + | clean_log > "${dir}/dogfood-serial.log" grep -a 'GET ' "${work}/http.log" > "${dir}/dogfood-http.log" || true grep -aoE 'PROBE[ -].*' "${dir}/dogfood-serial.log" || true