diff --git a/.github/scripts/image-build-needed.py b/.github/scripts/image-build-needed.py new file mode 100644 index 00000000..82343a18 --- /dev/null +++ b/.github/scripts/image-build-needed.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Decide whether a pull request needs the image build and boot test. + +Reads the PR's changed paths (one per line, renames listed under both names) +on stdin and prints ``true`` or ``false`` for the build workflow's `changes` +job, which runs this script from the PR's base revision. Only changes that +cannot reach the image set or the boot test skip the ~2 h build: docs, +Markdown outside the build inputs, unit/e2e tests and the workflows that run +them. Anything else, and anything under a build input root, builds; so does +any change to this classifier or to the build workflow, and an empty or +truncated list. +""" + +import sys + +# The gate itself: a change to either must be built and boot-tested before +# the base branch's classifier trusts it. +GATE_FILES = frozenset( + { + ".github/scripts/image-build-needed.py", + ".github/workflows/build.yml", + } +) + +# Paths the build or the boot test consume. Always build, even for *.md. +BUILD_PREFIXES = ( + "elements/", + "files/", + "include/", + "patches/", + "plugins/", + "scripts/", + "tests/fixtures/", +) + +SKIP_PREFIXES = ( + "docs/", + "tests/unit/", + "tests/e2e/", +) + +SKIP_FILES = frozenset( + { + ".github/scripts/docs-checks.py", + ".github/workflows/docs-checks.yml", + ".github/workflows/unit-tests.yml", + ".github/workflows/track-junctions.yml", + ".pre-commit-config.yaml", + "cliff.toml", + "LICENSE", + } +) + +# GitHub's "list pull request files" API stops at 3000 files. +API_FILE_LIMIT = 3000 + + +def needs_build(path: str) -> bool: + if path in GATE_FILES or path.startswith(BUILD_PREFIXES): + return True + if path.startswith(SKIP_PREFIXES) or path in SKIP_FILES or path.endswith(".md"): + return False + return True + + +def image_build_needed(paths: list[str]) -> bool: + if not paths or len(paths) >= API_FILE_LIMIT: + return True + return any(needs_build(p) for p in paths) + + +def main() -> int: + paths = [line.strip() for line in sys.stdin if line.strip()] + print("true" if image_build_needed(paths) else "false") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5bcf73f0..77192d66 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,51 @@ permissions: contents: read jobs: + # Pull requests that cannot change the image set or the boot test (docs, + # unit tests, other workflows; see .github/scripts/image-build-needed.py) + # skip the ~2 h build. A skipped job reports as passing, so `build` and + # `boot-test` can be required checks without blocking those PRs. + # The classifier runs from the base revision, never the PR head, so a PR + # cannot edit it to skip its own build; changes to it or to this workflow + # always build. + changes: + runs-on: ubuntu-26.04 + timeout-minutes: 10 + permissions: + contents: read + pull-requests: read # lists the pull request's changed files + outputs: + image: ${{ steps.decide.outputs.image }} + steps: + - name: Checkout the base revision's scripts + if: github.event_name == 'pull_request' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.base.sha }} + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Decide whether to build the image set + id: decide + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + # Before the classifier exists on the base branch, build. + if [ "${GITHUB_EVENT_NAME}" != pull_request ] || [ ! -f .github/scripts/image-build-needed.py ]; then + echo "image=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files" \ + --jq '.[] | .filename, (.previous_filename // empty)' > "${RUNNER_TEMP}/changed.txt" + image="$(python3 .github/scripts/image-build-needed.py < "${RUNNER_TEMP}/changed.txt")" + echo "image=${image}" >> "$GITHUB_OUTPUT" + echo "Image build needed: ${image}" + build: + needs: changes + if: needs.changes.outputs.image == 'true' runs-on: ubuntu-26.04 timeout-minutes: 300 outputs: @@ -115,12 +159,43 @@ jobs: if-no-files-found: error retention-days: 1 + # The A/B update and boot-counted rollback test needs two more image + # sets that sort above this one. Only the version differs, so each is a + # warm rebuild of the version-stamped elements. Not on main: these would + # be release-signed versions nobody publishes. + # 1..N, not 0..N: systemd-sysinstall's boot entry for the + # installed 0. carries `version 1.0.`, and systemd-boot would + # keep defaulting to it over a 0..N update. 1..N sorts above + # both and still below every release (YY.MM.). + - name: Build the next and broken image sets + if: github.ref != 'refs/heads/main' + run: | + set -euo pipefail + just set-version "1.${GITHUB_RUN_NUMBER}.1" + just export-image dist/diskless-next + just set-version "1.${GITHUB_RUN_NUMBER}.2" + just export-image dist/diskless-broken + + - name: Upload update test image sets + if: github.ref != 'refs/heads/main' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: image-update + path: | + dist/diskless-next/ + dist/diskless-broken/ + !dist/*/bluefin-server-installer_*.raw + if-no-files-found: error + retention-days: 1 + # Secure Boot QEMU boots of the built images: diskless (network pull, - # dm-verity /usr, tmpfs root) and diskless -> systemd-sysinstall -> disk. + # dm-verity /usr, tmpfs root), refusal of a tampered image or manifest, + # UEFI HTTP boot, Ignition applying a config, and diskless -> + # systemd-sysinstall -> disk; off main also A/B update and rollback. boot-test: needs: build runs-on: ubuntu-26.04 - timeout-minutes: 60 + timeout-minutes: 90 steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -134,19 +209,81 @@ jobs: name: image path: dist/diskless + - name: Download update test image sets + if: github.ref != 'refs/heads/main' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: image-update + path: dist + + # Ubuntu's OVMF (2025.11) rejects systemd-boot's PK enrollment with + # "Security violation", so the guests kept booting in setup mode with + # nothing verified. Use Fedora's edk2 build, pinned by checksum, which + # enrolls the keys; dogfood-diskless.sh now fails if Secure Boot is off. - name: Install QEMU and OVMF + env: + OVMF_RPM_URL: https://kojipkgs.fedoraproject.org/packages/edk2/20260508/6.fc44/noarch/edk2-ovmf-20260508-6.fc44.noarch.rpm + OVMF_RPM_SHA256: 8b157ab2aa0cccccf021f43011308349147c76ae59550f52239ea50678dd0a5f run: | set -euo pipefail sudo apt-get update - sudo apt-get install -y --no-install-recommends qemu-system-x86 ovmf + sudo apt-get install -y --no-install-recommends qemu-system-x86 libarchive-tools sudo chmod 666 /dev/kvm + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/edk2-ovmf.rpm" "${OVMF_RPM_URL}" + echo "${OVMF_RPM_SHA256} ${RUNNER_TEMP}/edk2-ovmf.rpm" | sha256sum -c - + mkdir -p "${RUNNER_TEMP}/ovmf" + bsdtar -xf "${RUNNER_TEMP}/edk2-ovmf.rpm" -C "${RUNNER_TEMP}/ovmf" \ + ./usr/share/edk2/ovmf/OVMF_CODE.secboot.fd ./usr/share/edk2/ovmf/OVMF_VARS.fd + echo "OVMF_CODE=${RUNNER_TEMP}/ovmf/usr/share/edk2/ovmf/OVMF_CODE.secboot.fd" >> "$GITHUB_ENV" + echo "OVMF_VARS=${RUNNER_TEMP}/ovmf/usr/share/edk2/ovmf/OVMF_VARS.fd" >> "$GITHUB_ENV" - name: Diskless boot run: bash scripts/dogfood-diskless.sh dist/diskless --check + - name: Refuse a tampered image + env: + DOGFOOD_TAMPER: raw + run: bash scripts/dogfood-diskless.sh dist/diskless --check + + - name: Refuse an unsigned manifest + env: + DOGFOOD_TAMPER: sums + run: bash scripts/dogfood-diskless.sh dist/diskless --check + + - name: Ignition config from a credential + env: + DOGFOOD_IGNITION: tests/fixtures/ignition/apply-marker.ign + DOGFOOD_EXTRA_PROBE: tests/fixtures/ignition/apply-marker.probe + DOGFOOD_EXPECT: "PROBE ignition marker=applied unit=active enabled=enabled ran=yes" + run: bash scripts/dogfood-diskless.sh dist/diskless --check + + - name: UEFI HTTP boot with bluefin-node.ign + env: + DOGFOOD_BOOT: http + DOGFOOD_NODE_IGN: tests/fixtures/ignition/apply-marker.ign + DOGFOOD_EXTRA_PROBE: tests/fixtures/ignition/apply-marker.probe + DOGFOOD_EXPECT: "PROBE ignition marker=applied unit=active enabled=enabled ran=yes" + run: bash scripts/dogfood-diskless.sh dist/diskless --check + - name: Install to disk and boot it + if: github.ref == 'refs/heads/main' run: bash scripts/dogfood-install.sh dist/diskless + - name: Install, A/B update and boot-counted rollback + if: github.ref != 'refs/heads/main' + run: bash scripts/dogfood-install.sh dist/diskless dist/diskless-next dist/diskless-broken + + - name: Upload boot logs + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: boot-test-logs + path: | + dist/*/dogfood-*.log + dist/dogfood-install/*.log + if-no-files-found: ignore + retention-days: 7 + # Publishes the signed image set as a new, immutable GitHub Release and as # an ORAS OCI artifact (one layer per file, tags and latest). # Only runs on pushes/dispatches to main, where the code is already diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index b4c6d063..425762a6 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -83,8 +83,9 @@ freedesktop-sdk and gnome-build-meta bumps. It now lives in `track-junctions.yml` on a schedule, opening its own PR on its own branch. The `build` job (validation, compile, signing) runs with the read-only default -on every event. If a new job needs additional permissions, keep them as narrow -as possible and document why. +on every event. `changes` adds only `pull-requests: read`, to list a pull +request's files. If a new job needs additional permissions, keep them as +narrow as possible and document why. ### `sudo` scope @@ -116,8 +117,9 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e | Job | Workflow | Trigger | Purpose | |-----|----------|---------|---------| | `track-junctions` | `track-junctions.yml` | `schedule` (08:00 UTC), `workflow_dispatch` | Resolves the `freedesktop-sdk.bst` + `gnome-build-meta.bst` junction refs, syncs `project.conf`'s `installer-version`, and opens/updates its own PR on `auto/track-junctions`. `contents: write` + `pull-requests: write`, never on `pull_request`. | -| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there. Read-only token. | -| `boot-test` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Downloads the build job's exported image set and runs the Secure Boot QEMU checks: `scripts/dogfood-diskless.sh --check` (diskless boot) and `scripts/dogfood-install.sh` (install to disk and boot it). Read-only token. | +| `changes` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Decides whether `build` and `boot-test` run: always outside pull requests; on a pull request only if `.github/scripts/image-build-needed.py`, checked out from the PR's base revision, finds a changed path that can reach the image set or the boot test (see [Build time and caches](#build-time-and-caches)). `contents: read` + `pull-requests: read`. | +| `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there. Off `main` it also exports two higher-versioned sets (`1..1`, `1..2`) for the update test. Read-only token. | +| `boot-test` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Runs the Secure Boot QEMU checks on the exported sets (see Core Process step 4). Read-only token. | | `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`) (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write` + `packages: write`. | | `docs` | `docs-checks.yml` | `pull_request`, `push/main` | Runs markdown and skill metadata checks via `docs-checks.py`. Read-only token. | | `unit` | `unit-tests.yml` | `pull_request`, `push/main` | Runs pytest and BATS unit test suites. Read-only token. | @@ -141,10 +143,34 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). and push to `main`, and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst` (gpg sign plus a `gpgv` proof against the shipped keyring). - 4. **Boot test:** Downloads the exported image set and runs - `scripts/dogfood-diskless.sh --check` (diskless Secure Boot boot) and - `scripts/dogfood-install.sh` (diskless boot, `systemd-sysinstall` to disk, - boot the installed disk) in QEMU with OVMF. + 4. **Boot test:** Downloads the exported image sets and runs, in QEMU with + Secure Boot OVMF, each as one `scripts/dogfood-diskless.sh --check` or + `scripts/dogfood-install.sh` call. The firmware is Fedora's + `edk2-ovmf` (Koji URL + SHA-256 in `build.yml`), not Ubuntu's `ovmf`: + Ubuntu 26.04's OVMF 2025.11 rejects systemd-boot's PK enrollment + (`Failed to write PK secure boot variable: Security violation`), and + until this was caught every CI boot ran in setup mode with Secure Boot + off. `--check` now fails unless the probe reports + `secureboot=enabled` (or, for tamper runs, the kernel logs + `Secure boot enabled`). Bump the pin by hand; Renovate does not track it. + - diskless netboot, no failed units; + - `DOGFOOD_TAMPER=raw`: a corrupted DDI must be refused by the manifest + check (`DOWNLOAD INVALID: Checksum of ... did not check out`); + `DOGFOOD_TAMPER=sums`: the same DDI with `SHA256SUMS` re-hashed to match + it must be refused by the signature check (`DOWNLOAD INVALID: Signature + verification failed`). Both only after the image, `SHA256SUMS` and + `SHA256SUMS.gpg` were served, and nothing may boot; + - Ignition from the `ignition.config` credential, and UEFI HTTP boot with + `bluefin-node.ign` served next to the UKI, both with + `tests/fixtures/ignition/apply-marker.ign`: the probe must see the + written file and the Ignition-enabled unit active (`DOGFOOD_EXPECT`); + - on `main`: diskless boot, `systemd-sysinstall` to disk, boot the disk; + - off `main`: the same, then `systemd-sysupdate` A->B to `1..1` and + a boot-counted rollback from a corrupted `1..2` (see + [ddi-installer-build.md](ddi-installer-build.md) for why not + `0..N`). Main skips this + because its extra sets would be release-signed versions nobody + publishes; every change reaches main through a pull request that ran it. 5. **Version Derivation:** The release version is set per build with `just set-version`: `YY.MM.` on main, `0.` on pull requests so a PR build can never sort above a release. @@ -154,6 +180,59 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). `ghcr.io//bluefin-server:,latest`. Nodes verify updates against the `SHA256SUMS` / `SHA256SUMS.gpg` already in that set. +## Build time and caches + +Measured on run 36499270842 (pull request, cold runner): `Build and export the +image set` took 2 h. BuildStream pulled 239 artifacts and built 85 (about 3.9 h +of build time over 4 cores). The critical path is FSDK's +`components/linux.bst`: 6 min to fetch its source (not in any source cache) +and 1 h 43 min to build. FSDK's caches never hold it for us, because the +`components/linux-module-cert.bst` junction override (our module certificate) +and `patches/freedesktop-sdk/0006-linux-*.patch` change its cache key. Next are +about 40 FSDK elements our FSDK patches or their reverse dependencies change +(glib-stage1, gobject-introspection, harfbuzz, go, vala, ...; about 70 min of +build time in parallel with the kernel). Changing only `image-version` +rebuilds 13 version-stamped elements (os-release to `oci/bluefin-server-image.bst` +and the sysexts), 2 min locally on a warm cache; CI's +per-set cost is estimated at under 10 min. + +- **Docs-only pull requests skip the build.** The `changes` job feeds the PR's + changed paths (renames under both names) to + `.github/scripts/image-build-needed.py` from the base revision, so a PR + cannot edit the classifier to skip its own build; a change to the + classifier or `build.yml` always builds. It answers `false` only when every + path is docs, Markdown outside the build inputs, `tests/unit/`, + `tests/e2e/`, or a workflow or script that does not build the image; + `tests/unit/test_image_build_needed.py` fails if any tracked build input + (`elements/`, `files/`, `include/`, `patches/`, `plugins/`, `scripts/`, + `tests/fixtures/`, `project.conf`, `Justfile`, `build.yml`) would skip. No + status check is required on `main` today; a skipped job reports as passing, + so they can be made required without `paths-ignore` leaving them pending. +- **No cache push.** CI only pulls from the caches `project.conf` lists. + `bluefin-server/keys/boot-keys.bst` imports `files/boot-keys/`, which on + `main` holds the Secure Boot, module-signing and sysupdate private keys, and + the image, UKIs, `kernel-modules.bst`, `efi-keys.bst`, + `os-sd-boot-signed.bst` and `openzfs-signed.bst` build-depend on it, so + `bst artifact push --deps all` of the image would upload the keys to a cache + every build pulls from. Pushing needs an explicit element allow-list, with a + unit test proving no listed element is, or build-depends on, + `boot-keys.bst`. FSDK's `components/linux.bst` only stages the public + certificate (`bluefin-server/keys/linux-module-cert.bst`), so the kernel is + the candidate worth listing. +- **Pull requests still build the kernel.** `just gen-dev-keys` makes a new + module certificate on every PR run, so the PR kernel's cache key never + matches anything cached. Caching it needs a stable, non-release PR module + key pair that PR builds can use (fork PRs get no secrets) and a job that + pushes the kernel built with it; that is a maintainer decision about key + handling, not a workflow change. +- **No `actions/cache` for sources or artifacts.** The kernel source fetch + (6 min, 1 of the 85 fetches) is the only one on the critical path, and it + only matters when the kernel is rebuilt anyway. The runner's BuildStream + cache after a build holds every pulled and built artifact of the graph, + several times the 2.1 GB image set, and would churn the 10 GB repository + cache quota; BuildStream 2 cannot import a single artifact from a tarball, + so the kernel alone cannot be cached that way. + ## Common Rationalizations | Rationalization | Reality | @@ -176,6 +255,8 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). - [ ] Every `uses:` line has a full 40-character SHA and a `# vX` comment. - [ ] Every `install-action` `tool:` names an explicit version (`just@1.58.0`). - [ ] `just validate` passes after workflow changes. +- [ ] A new directory the build or boot test reads is listed in + `BUILD_PREFIXES` in `.github/scripts/image-build-needed.py`. - [ ] No new mutable action refs introduced. - [ ] Release signing happens in `oci/bluefin-server-image.bst`; there is no separate CI signing step, and the release job publishes `dist/diskless/` diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 35ce263c..85fc3ffb 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -69,6 +69,10 @@ their images are never published. All dogfood paths boot with Secure Boot firmware (OVMF secboot). The firmware starts in setup mode; systemd-boot enrolls the dev keys from the ESP (`secure-boot-enroll if-safe`) and reboots, so every later boot is verified. +`--check` fails if the guest did not boot with Secure Boot enabled: some OVMF +builds (Ubuntu 26.04's 2025.11) refuse the enrollment and would otherwise boot +on in setup mode, verifying nothing. Point `OVMF_CODE` / `OVMF_VARS` at +another build (Fedora's `edk2-ovmf` works) if yours does. ```bash just dogfood # interactive diskless boot of dist/diskless/ @@ -100,11 +104,20 @@ Useful environment variables: - `DOGFOOD_NODE_IGN=` — serve it as `bluefin-node.ign` next to the UKI (picked up by HTTP-booted nodes with no Ignition credential). - `DOGFOOD_SERVE_EXTRA=` — also serve the files in ``. -- `DOGFOOD_TAMPER=raw|sums` — serve a corrupted DDI or a re-hashed, unsigned - `SHA256SUMS`; the boot must fail, proving the signature check. +- `DOGFOOD_TAMPER=raw|sums` — serve a corrupted DDI (`raw`), or the corrupted + DDI with `SHA256SUMS` re-hashed to match it, so only `SHA256SUMS.gpg` no + longer fits (`sums`). `--check` then passes only if the initrd's pull + refuses it for that reason (checksum mismatch, bad signature) after the + image and both manifest files were served, and nothing booted. Credential + drop-ins copy `systemd-importd`'s messages to the serial console. - `DOGFOOD_MEM=` — guest RAM (default 4096); below the diskless minimum the boot must fail with the RAM message (see `diskless-troubleshooting.md`). - `DOGFOOD_EXTRA_PROBE=` — shell snippet appended to the in-guest probe. +- `DOGFOOD_EXPECT=` — `--check` also requires the probe output to match, + e.g. with `tests/fixtures/ignition/apply-marker.ign` and its `.probe`: + `PROBE ignition marker=applied unit=active enabled=enabled ran=yes`. +- `DOGFOOD_PORT`, `DOGFOOD_MEM`, `DOGFOOD_TIMEOUT` — HTTP port (8765), guest + memory in MiB (4096), `--check` deadline in seconds (600). Every diskless `--check` boot also runs `bluefin-diskless-update-check` once and reports `PROBE update-check= flag=`. Its origin is the @@ -128,10 +141,16 @@ only and shortens the boot deadline (`DOGFOOD_DEADLINE`, default `2min`): each counted boot reaches `multi-user.target`, misses `boot-complete.target`, and `bluefin-boot-deadline` reboots it. The run asserts three such boots, the fallback boot with the deadline timer inactive, and then no kured flag and a -skipped `systemd-sysupdate-reboot.service` for the failed version. CI runs the first two stages -(`dogfood-diskless.sh --check`, `dogfood-install.sh dist/diskless`) as the -`boot-test` job in `.github/workflows/build.yml` on every pull request and -push to main. +skipped `systemd-sysupdate-reboot.service` for the failed version. `` and `` are +ordinary image sets with higher versions, e.g. +`just set-version && just export-image dist/diskless-next` (and a +higher one into `dist/diskless-broken`); the script breaks the +broken set itself. The versions must also sort above `1.` for systemd-boot: the +Type #1 entry `systemd-sysinstall` writes for the installed image carries +`version 1.` (`bluefin-server-commit_1..conf`), so after +installing `0.674` an update to `0.674.1` still boots `0.674`; use `1.674.1`. +Release versions (`YY.MM.`) sort above it. Which of these scenarios CI runs is listed in +[ci-tooling.md](ci-tooling.md) (the `boot-test` job). ## Local builds with a remote cache diff --git a/scripts/dogfood-diskless.sh b/scripts/dogfood-diskless.sh index 8e46de17..5c3a6e29 100644 --- a/scripts/dogfood-diskless.sh +++ b/scripts/dogfood-diskless.sh @@ -9,8 +9,14 @@ # is handed over as the import.pull system credential via SMBIOS. # # Usage: dogfood-diskless.sh [--check] -# --check headless: boot, run a probe, exit 0 if no unit failed +# --check headless: boot, run a probe, exit 0 if Secure Boot was enforced and no +# unit failed. With DOGFOOD_TAMPER set, exit 0 only if the initrd +# refuses the image. # Environment: +# DOGFOOD_PORT= HTTP port for the built-in server (default 8765) +# DOGFOOD_MEM= guest memory (default 4096) +# DOGFOOD_TIMEOUT= --check deadline (default 600) +# DOGFOOD_EXPECT= --check also requires the probe output to match # DOGFOOD_IGNITION= pass an Ignition config as the ignition.config credential # DOGFOOD_CREDS= pass every file in as a system credential named after it # DOGFOOD_STATE_DISK= attach a persistent second disk (/dev/vdb), created if missing @@ -22,7 +28,10 @@ # DOGFOOD_BOOT_URL= HTTP boot from another server (e.g. Booty) instead # DOGFOOD_NODE_IGN= serve it as bluefin-node.ign next to the UKI (HTTP boot) # DOGFOOD_SERVE_EXTRA= also serve the files in -# DOGFOOD_TAMPER=raw|sums serve a corrupted image or a re-hashed, unsigned manifest +# DOGFOOD_TAMPER=raw|sums serve a corrupted image (raw), or a corrupted image with +# SHA256SUMS re-hashed to match it but no longer matching +# SHA256SUMS.gpg (sums); --check then passes only if the +# initrd's pull refuses it for that reason set -euo pipefail dir="$(realpath "${1:?usage: $0 [--check]}")" @@ -49,7 +58,14 @@ vars_tmpl="${OVMF_VARS:-$(first_existing \ /usr/share/edk2/x64/OVMF_VARS.4m.fd)}" || { echo "ERROR: no blank OVMF_VARS found (set OVMF_VARS)" >&2; exit 1; } work="$(mktemp -d /tmp/bluefin-dogfood.XXXXXX)" -trap 'kill "${http_pid:-0}" 2>/dev/null || true; rm -rf "${work}"' EXIT +# Never leave a guest behind: an orphaned QEMU keeps its port and the caller's +# locks. Only kill PIDs that are set; `kill 0` signals the whole process group. +cleanup() { + local pid + for pid in ${qemu_pid:-} ${enroll_pid:-} ${http_pid:-}; do kill "${pid}" 2>/dev/null || true; done + rm -rf "${work}" +} +trap cleanup EXIT vars="${DOGFOOD_VARS:-${work}/vars.fd}" [ -f "${vars}" ] || cp "${vars_tmpl}" "${vars}" cp "${esp}" "${work}/esp.raw" @@ -61,19 +77,42 @@ mkdir -p "${srv}" for f in "${dir}"/*; do ln -s "${f}" "${srv}/"; done [ -n "${DOGFOOD_NODE_IGN:-}" ] && cp "${DOGFOOD_NODE_IGN}" "${srv}/bluefin-node.ign" if [ -n "${DOGFOOD_SERVE_EXTRA:-}" ]; then for f in "${DOGFOOD_SERVE_EXTRA}"/*; do ln -sf "$(realpath "${f}")" "${srv}/"; done; fi +image_re="${image//./\\.}" case "${DOGFOOD_TAMPER:-}" in - raw) + raw|sums) rm "${srv}/${image}"; cp "${dir}/${image}" "${srv}/${image}" - printf 'X' | dd of="${srv}/${image}" bs=1 seek=4096 conv=notrunc status=none ;; + printf 'X' | dd of="${srv}/${image}" bs=1 seek=4096 conv=notrunc status=none ;;& + raw) + refusal_re="DOWNLOAD INVALID: Checksum of ${image_re} file did not check out" ;; sums) + # What an attacker without the signing key can do: make the manifest + # match the modified image. Only the signature check can catch it. rm "${srv}/SHA256SUMS" - sed 's/^[0-9a-f]\{8\}/deadbeef/' "${dir}/SHA256SUMS" > "${srv}/SHA256SUMS" ;; + sum="$(sha256sum "${srv}/${image}" | cut -d' ' -f1)" + sed -E "s/^[0-9a-f]{64}( [ *]${image_re})$/${sum}\1/" "${dir}/SHA256SUMS" > "${srv}/SHA256SUMS" + grep -q "^${sum} " "${srv}/SHA256SUMS" || { echo "ERROR: ${image} not in SHA256SUMS" >&2; exit 1; } + refusal_re="DOWNLOAD INVALID: Signature verification failed" ;; + '') ;; + *) echo "ERROR: DOGFOOD_TAMPER must be raw or sums" >&2; exit 1 ;; esac (cd "${srv}" && exec python3 -m http.server --bind 127.0.0.1 "${port}" >"${work}/http.log" 2>&1) & http_pid=$! cred() { printf 'type=11,value=io.systemd.credential.binary:%s=%s' "$1" "$(base64 -w0 < "$2")"; } +# wait_for : poll the command once a second until +# it succeeds (0), or the process exits or the time runs out (1). +wait_for() { + local deadline=$(( $(date +%s) + $1 )) pid="$2" + shift 2 + while [ "$(date +%s)" -lt "${deadline}" ]; do + "$@" && return 0 + kill -0 "${pid}" 2>/dev/null || { "$@"; return; } + sleep 1 + done + return 1 +} + printf 'raw,machine,verify=signature,blockdev:rootdisk:http://10.0.2.2:%s/%s\n' "${port}" "${image}" > "${work}/import.pull" qemu=(qemu-system-x86_64 -machine q35,smm=on,accel=kvm -cpu host -m "${mem}" -smp 2 @@ -90,14 +129,15 @@ if [ "${boot}" = http ] && [ ! -s "${vars}.enrolled" ]; then -display none -monitor none \ -serial "file:${work}/enroll.log" /dev/null 2>&1 & enroll_pid=$! - for _ in $(seq 1 60); do - grep -aq 'successfully enrolled' "${work}/enroll.log" 2>/dev/null && break - sleep 1 - done - sleep 2 + # systemd-boot resets the machine after enrolling; the firmware loading a + # boot option again proves the variable store has the keys. + enrolled_and_reset() { sed -n '/successfully enrolled/,$p' "${work}/enroll.log" 2>/dev/null | grep -aq 'BdsDxe: loading'; } + rc=0 + wait_for 90 "${enroll_pid}" enrolled_and_reset || rc=$? kill "${enroll_pid}" 2>/dev/null || true wait "${enroll_pid}" 2>/dev/null || true - grep -aq 'successfully enrolled' "${work}/enroll.log" || { echo "ERROR: key enrollment failed" >&2; exit 1; } + cp "${work}/enroll.log" "${dir}/dogfood-enroll.log" 2>/dev/null || true + [ "${rc}" = 0 ] || { echo "ERROR: key enrollment failed (log: ${dir}/dogfood-enroll.log)" >&2; exit 1; } echo enrolled > "${vars}.enrolled" fi if [ "${boot}" = http ]; then @@ -120,6 +160,17 @@ fi if [ -n "${DOGFOOD_IGNITION:-}" ]; then qemu+=(-smbios "$(cred ignition.config "${DOGFOOD_IGNITION}")") fi +tamper="${DOGFOOD_TAMPER:-}" +if [ -n "${tamper}" ]; then + # The initrd's console shows only that the download unit failed; copy the + # pull's own messages (from the unit and from systemd-importd, which runs + # the transfer) there so the log names why the image was refused. systemd + # tools log natively to the journal when they can, bypassing the stream + # journald forwards, so make them write to stderr. + printf '[Service]\nEnvironment=SYSTEMD_LOG_TARGET=console\nStandardOutput=journal+console\nStandardError=journal+console\n' > "${work}/import-console.conf" + qemu+=(-smbios "$(cred systemd.unit-dropin.systemd-import@.service "${work}/import-console.conf")" + -smbios "$(cred systemd.unit-dropin.systemd-importd.service "${work}/import-console.conf")") +fi if [ -n "${DOGFOOD_CREDS:-}" ]; then for f in "${DOGFOOD_CREDS}"/*; do if [ -f "${f}" ]; then qemu+=(-smbios "$(cred "${f##*/}" "${f}")"); fi @@ -178,23 +229,65 @@ UNIT -smbios "$(cred systemd.extra-unit.dogfood-probe.service "${work}/probe.service")" \ /dev/null 2>&1 & qemu_pid=$! -deadline=$(( $(date +%s) + timeout_s )) +probe_done() { grep -aq 'PROBE failed=' "${work}/probe.log" 2>/dev/null; } +# Console output interleaves CSI, OSC and DCS escape sequences, sometimes in the +# middle of a line (the initrd's failure summary writes to the console while +# systemd-importd logs), so strip all three before matching. +clean_log() { sed -E 's/\x1b\][^\x07\x1b]*(\x07|\x1b\\)//g; s/\x1bP[^\x1b]*\x1b\\//g; s/\x1b\[[0-9;?]*[a-zA-Z]//g' | tr -d '\r'; } +refused() { clean_log < "${work}/serial.log" 2>/dev/null | grep -aqE "${refusal_re}"; } status=1 -while kill -0 "${qemu_pid}" 2>/dev/null && [ "$(date +%s)" -lt "${deadline}" ]; do - if grep -aq 'PROBE failed=' "${work}/probe.log" 2>/dev/null; then - status=0 - break - fi - sleep 2 -done +if [ -n "${tamper}" ]; then + stop() { probe_done || refused; } +else + stop() { probe_done; } +fi +wait_for "${timeout_s}" "${qemu_pid}" stop && status=0 kill "${qemu_pid}" 2>/dev/null || true wait "${qemu_pid}" 2>/dev/null || true cat "${work}/serial.log" "${work}/probe.log" 2>/dev/null \ - | sed 's/\x1b\[[0-9;?]*[a-zA-Z]//g; s/\x1bP[^\x1b]*\x1b\\//g' | tr -d '\r' > "${dir}/dogfood-serial.log" + | clean_log > "${dir}/dogfood-serial.log" grep -a 'GET ' "${work}/http.log" > "${dir}/dogfood-http.log" || true grep -aoE 'PROBE[ -].*' "${dir}/dogfood-serial.log" || true + +if [ -n "${tamper}" ]; then + served() { grep -aq "\"GET /$1 HTTP/1.1\" 200" "${dir}/dogfood-http.log"; } + if probe_done; then + echo "FAIL: tamper=${tamper}: the tampered image booted (serial log: ${dir}/dogfood-serial.log)" >&2 + status=1 + elif ! refused; then + echo "FAIL: tamper=${tamper}: no refusal within ${timeout_s}s (serial log: ${dir}/dogfood-serial.log)" >&2 + tail -n 40 "${dir}/dogfood-serial.log" >&2 + status=1 + elif ! served "${image}" || ! served SHA256SUMS || ! served SHA256SUMS.gpg; then + # Refused before it had the image and the signed manifest: that is a + # transport failure, not a verification failure. + echo "FAIL: tamper=${tamper}: the pull failed before fetching ${image}, SHA256SUMS and SHA256SUMS.gpg" >&2 + status=1 + elif ! grep -aq 'Secure boot enabled' "${dir}/dogfood-serial.log"; then + echo "FAIL: tamper=${tamper}: refused, but the kernel did not run with Secure Boot enabled" >&2 + status=1 + else + grep -aE "${refusal_re}|Failed to start Download of " "${dir}/dogfood-serial.log" | sed 's/^/REFUSED /' | head -n 5 + echo "PASS: tamper=${tamper}: ${image} refused after fetching the signed manifest (serial log: ${dir}/dogfood-serial.log)" + status=0 + fi + if [ "${status}" = 0 ] && [ -n "${DOGFOOD_EXPECT:-}" ] && ! grep -aqE -- "${DOGFOOD_EXPECT}" "${dir}/dogfood-serial.log"; then + echo "FAIL: tamper=${tamper}: serial log does not match DOGFOOD_EXPECT=${DOGFOOD_EXPECT}" >&2 + status=1 + fi + exit "${status}" +fi + failed="$(grep -a '\[FAILED\]' "${dir}/dogfood-serial.log" || true)" -if [ "${status}" = 0 ] && [ -z "${failed}" ] && grep -aq 'PROBE failed=0' "${dir}/dogfood-serial.log"; then +if [ "${status}" = 0 ] && [ -n "${DOGFOOD_EXPECT:-}" ] && ! grep -aqE -- "${DOGFOOD_EXPECT}" "${dir}/dogfood-serial.log"; then + echo "FAIL: booted, but the probe output does not match DOGFOOD_EXPECT=${DOGFOOD_EXPECT}" >&2 + status=1 +elif [ "${status}" = 0 ] && ! grep -aq 'PROBE secureboot=enabled' "${dir}/dogfood-serial.log"; then + # Firmware that refuses the enrollment payloads boots on in setup mode, + # where nothing is verified; that is not a Secure Boot boot. + echo "FAIL: booted without Secure Boot ($(grep -aoE 'PROBE secureboot=.*' "${dir}/dogfood-serial.log" | head -n1)); see the enrollment messages in the serial log" >&2 + status=1 +elif [ "${status}" = 0 ] && [ -z "${failed}" ] && grep -aq 'PROBE failed=0' "${dir}/dogfood-serial.log"; then echo "PASS: ${boot} boot with no failed units (serial log: ${dir}/dogfood-serial.log)" elif [ "${status}" = 0 ]; then echo "FAIL: booted, but units failed:" >&2 diff --git a/scripts/dogfood-install.sh b/scripts/dogfood-install.sh index 009813a1..1dfad556 100644 --- a/scripts/dogfood-install.sh +++ b/scripts/dogfood-install.sh @@ -22,10 +22,12 @@ # enabled; a boot-counted UKI is blessed only after boot-complete.target, # which requires that no unit failed. # Usage: [DOGFOOD_BROKEN=slot|unit] dogfood-install.sh [ []] +# and are image sets with increasingly higher versions. +# DOGFOOD_PORT (default 8765) is shared with dogfood-diskless.sh's server. set -euo pipefail here="$(cd "$(dirname "$0")" && pwd)" -dir="$(realpath "${1:?usage: $0 []}")" +dir="$(realpath "${1:?usage: $0 [ []]}")" next="${2:+$(realpath "$2")}" broken="${3:+$(realpath "$3")}" state="$(realpath "${DOGFOOD_STATE:-dist/dogfood-install}")" @@ -33,6 +35,7 @@ rm -rf "${state}" mkdir -p "${state}" truncate -s 16G "${state}/disk.raw" +export DOGFOOD_PORT="${DOGFOOD_PORT:-8765}" export DOGFOOD_STATE_DISK="${state}/disk.raw" export DOGFOOD_VARS="${state}/vars.fd" run() { DOGFOOD_EXTRA_PROBE="$2" bash "${here}/dogfood-diskless.sh" "$1" --check; } @@ -66,7 +69,7 @@ cat > "${state}/update.probe" <<'EOF' mkdir -p /etc/sysupdate.d/zfs.feature.d printf '[Feature]\nEnabled=true\n' > /etc/sysupdate.d/zfs.feature.d/enable.conf for f in /usr/lib/sysupdate.d/*.transfer; do - sed -e 's|^Path=https://.*|Path=http://10.0.2.2:8765/|' \ + sed -e 's|^Path=https://.*|Path=http://10.0.2.2:@DOGFOOD_PORT@/|' \ "${f}" > "/etc/sysupdate.d/${f##*/}" done rm -f /run/reboot-required @@ -81,6 +84,7 @@ systemctl start systemd-sysupdate-reboot.service echo "PROBE interlock=$(systemctl show -P Result systemd-sysupdate-reboot.service)" systemctl stop kubelet.service EOF +sed -i "s|@DOGFOOD_PORT@|${DOGFOOD_PORT}|" "${state}/update.probe" echo "==> 1/4 diskless boot + systemd-sysinstall" run "${dir}" "${state}/install.probe" | tee "${state}/1-install.log" diff --git a/tests/fixtures/ignition/apply-marker.ign b/tests/fixtures/ignition/apply-marker.ign new file mode 100644 index 00000000..e0c534e2 --- /dev/null +++ b/tests/fixtures/ignition/apply-marker.ign @@ -0,0 +1,26 @@ +{ + "ignition": { + "version": "3.6.0" + }, + "storage": { + "files": [ + { + "path": "/etc/dogfood-ignition-marker", + "mode": 420, + "overwrite": true, + "contents": { + "source": "data:,applied%0A" + } + } + ] + }, + "systemd": { + "units": [ + { + "name": "dogfood-ignition-marker.service", + "enabled": true, + "contents": "[Unit]\nDescription=Dogfood check that Ignition enabled this unit\n\n[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/bin/touch /run/dogfood-ignition-marker\n\n[Install]\nWantedBy=multi-user.target\n" + } + ] + } +} diff --git a/tests/fixtures/ignition/apply-marker.probe b/tests/fixtures/ignition/apply-marker.probe new file mode 100644 index 00000000..ba9c9568 --- /dev/null +++ b/tests/fixtures/ignition/apply-marker.probe @@ -0,0 +1,7 @@ +# DOGFOOD_EXTRA_PROBE for apply-marker.ign: Ignition wrote the file and the +# unit, the preset enabled the unit, and PID 1 ran it on this boot. +marker="$(cat /etc/dogfood-ignition-marker 2>/dev/null || echo missing)" +unit="$(systemctl is-active dogfood-ignition-marker.service 2>/dev/null || true)" +enabled="$(systemctl is-enabled dogfood-ignition-marker.service 2>/dev/null || true)" +ran="$([ -e /run/dogfood-ignition-marker ] && echo yes || echo no)" +echo "PROBE ignition marker=${marker} unit=${unit} enabled=${enabled} ran=${ran}" diff --git a/tests/unit/test_image_build_needed.py b/tests/unit/test_image_build_needed.py new file mode 100644 index 00000000..cc8f40eb --- /dev/null +++ b/tests/unit/test_image_build_needed.py @@ -0,0 +1,118 @@ +"""Unit coverage for .github/scripts/image-build-needed.py. + +The build workflow skips the ~2 h image build and the boot test for pull +requests the script classifies as not touching the image. A wrong `false` +merges an untested image change, so the invariants guarded here are that every +tracked file under a build input root, the build workflow and the classifier +always build, and that the workflow runs the base revision's classifier. +""" + +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".github" / "scripts" / "image-build-needed.py" + +_spec = importlib.util.spec_from_file_location("image_build_needed", SCRIPT) +image_build_needed = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(image_build_needed) + + +def tracked_files(): + out = subprocess.run( + ["git", "ls-files"], cwd=ROOT, check=True, capture_output=True, text=True + ).stdout + return [line for line in out.splitlines() if line] + + +def test_every_build_input_builds(): + inputs = [ + p + for p in tracked_files() + if p.startswith(image_build_needed.BUILD_PREFIXES) + or p in {"project.conf", "Justfile", ".github/workflows/build.yml"} + or p.startswith(".github/scripts/check-") + ] + assert inputs, "git ls-files returned no build inputs" + skipped = [p for p in inputs if not image_build_needed.needs_build(p)] + assert skipped == [] + + +def test_the_classifier_itself_builds(): + assert image_build_needed.needs_build(".github/scripts/image-build-needed.py") + + +@pytest.mark.parametrize("path", sorted(image_build_needed.GATE_FILES)) +def test_gate_changes_build_even_among_docs(path): + assert (ROOT / path).is_file(), f"{path} moved; update GATE_FILES" + assert image_build_needed.needs_build(path) + assert image_build_needed.image_build_needed(["docs/a.md", path]) + + +def test_changes_job_runs_the_base_revision_classifier(): + workflow = yaml.safe_load((ROOT / ".github" / "workflows" / "build.yml").read_text()) + steps = workflow["jobs"]["changes"]["steps"] + checkouts = [s for s in steps if str(s.get("uses", "")).startswith("actions/checkout@")] + assert len(checkouts) == 1 + ref = checkouts[0]["with"]["ref"] + assert ref == "${{ github.event.pull_request.base.sha }}" + assert "repository" not in checkouts[0]["with"] + run = "\n".join(s.get("run", "") for s in steps) + assert "python3 .github/scripts/image-build-needed.py" in run + assert "[ ! -f .github/scripts/image-build-needed.py ]" in run + + +@pytest.mark.parametrize( + "path", + [ + "docs/skills/ci-tooling.md", + "README.md", + ".github/copilot-instructions.md", + "tests/unit/test_repart_layout.py", + "tests/unit/os-justfile_test.bats", + ".github/scripts/docs-checks.py", + ".github/workflows/unit-tests.yml", + ], +) +def test_docs_and_unit_test_changes_skip(path): + assert not image_build_needed.needs_build(path) + + +@pytest.mark.parametrize( + "path", + [ + "files/os/README.md", + "elements/notes.md", + "tests/fixtures/ignition/apply-marker.ign", + "scripts/dogfood-diskless.sh", + "renovate.json", + "a-new-top-level-file", + ], +) +def test_build_inputs_and_unknown_paths_build(path): + assert image_build_needed.needs_build(path) + + +def test_one_build_input_among_docs_builds(): + assert image_build_needed.image_build_needed(["docs/a.md", "elements/x.bst"]) + + +def test_empty_or_truncated_lists_build(): + assert image_build_needed.image_build_needed([]) + many = ["docs/x.md"] * image_build_needed.API_FILE_LIMIT + assert image_build_needed.image_build_needed(many) + + +def test_cli_reads_stdin(): + def run(text): + return subprocess.run( + [sys.executable, str(SCRIPT)], input=text, capture_output=True, text=True, check=True + ).stdout.strip() + + assert run("docs/skills/index.md\nREADME.md\n") == "false" + assert run("docs/skills/index.md\nfiles/os/x\n") == "true"