From da48e810dfcf54bb52ad94b6ccbfa711f95c4182 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 00:10:50 -0500 Subject: [PATCH 1/5] docs: replace stale Flatcar narrative with FSDK 26.08 facts The OS payload is now pure FSDK 26.08 (systemd v261). Update comments in shipped elements, scripts, and tests to remove Flatcar references and explain why each workaround still exists: - /etc/resolv.conf: FSDK ships no rule (upstream systemd leaves the symlink to the distro), and /etc is empty on every boot, so the tmpfiles.d rule is still required for kubelet pod sandboxes. - firstboot credentials: the helper exists because Bluefin applies firstboot.hostname unconditionally and sets the live kernel hostname, which systemd-firstboot does not do; it is not because systemd-firstboot is missing from the image. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/bluefin-server/os-resolv-conf.bst | 10 +++++----- files/os/libexec/bluefin-firstboot-credentials | 15 +++++++++------ files/os/tmpfiles.d/20-bluefin-resolv.conf | 13 +++++-------- tests/unit/test_credentials_provisioning.py | 6 ++++-- tests/unit/test_k0s_first_boot.py | 5 ++--- 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/elements/bluefin-server/os-resolv-conf.bst b/elements/bluefin-server/os-resolv-conf.bst index 79361cc1..f088390a 100644 --- a/elements/bluefin-server/os-resolv-conf.bst +++ b/elements/bluefin-server/os-resolv-conf.bst @@ -1,10 +1,10 @@ kind: import description: | - Install the tmpfiles.d rule that recreates Flatcar's /etc/resolv.conf - symlink to systemd-resolved's resolv.conf. The Flatcar /usr payload does - not carry it (Flatcar ships the symlink in the image /etc, which this DDI - does not import), and kubelet requires /etc/resolv.conf to create any pod - sandbox. + Install the tmpfiles.d rule that creates /etc/resolv.conf as a symlink to + systemd-resolved's resolv.conf. FSDK 26.08 ships no /etc/resolv.conf rule + (upstream systemd leaves the symlink to the distro), and this DDI boots + with an empty /etc, so nothing ever creates the symlink and kubelet fails + every pod sandbox with "open /etc/resolv.conf: no such file or directory". sources: - kind: local diff --git a/files/os/libexec/bluefin-firstboot-credentials b/files/os/libexec/bluefin-firstboot-credentials index 5b90d941..845b5744 100755 --- a/files/os/libexec/bluefin-firstboot-credentials +++ b/files/os/libexec/bluefin-firstboot-credentials @@ -5,14 +5,17 @@ # firstboot.locale, firstboot.locale-messages -> /etc/locale.conf # firstboot.keymap -> /etc/vconsole.conf # firstboot.timezone -> /etc/localtime -# and additionally (systemd <= v259, Flatcar's, has no firstboot.hostname; -# systemd v260 added it as a static hostname applied only when none is set. -# Bluefin applies it unconditionally and also sets the live kernel hostname -# so the first DHCP request carries the name): +# and additionally: # firstboot.hostname -> /etc/hostname + kernel hostname # from $CREDENTIALS_DIRECTORY, as populated by ImportCredential= in -# bluefin-firstboot-credentials.service. The OS payload is Flatcar's /usr, -# which is built without systemd-firstboot, so the unit cannot call it. +# bluefin-firstboot-credentials.service. +# +# This helper exists rather than calling systemd-firstboot because Bluefin +# needs firstboot.hostname applied unconditionally (systemd v260+ applies it +# only as a static hostname when none is set) and needs the live kernel +# hostname set so the first DHCP request carries the name. It also validates +# every credential before writing anything, so a bad value fails the unit +# instead of leaving a partial configuration. # # Known differences from systemd-firstboot: trailing newlines in a credential # are stripped rather than rejected; LC_MESSAGES= is written even when it diff --git a/files/os/tmpfiles.d/20-bluefin-resolv.conf b/files/os/tmpfiles.d/20-bluefin-resolv.conf index 085ac2ab..6678502a 100644 --- a/files/os/tmpfiles.d/20-bluefin-resolv.conf +++ b/files/os/tmpfiles.d/20-bluefin-resolv.conf @@ -1,11 +1,8 @@ -# Flatcar's image build populates /etc/resolv.conf in the image's /etc and -# strips /etc-populating lines from /usr/lib/tmpfiles.d (see -# flatcar/scripts build_library/build_image_util.sh). This DDI imports only -# Flatcar's /usr and boots with an empty /etc, so nothing ever creates the -# symlink and kubelet fails every pod sandbox with +# FSDK 26.08 ships no /etc/resolv.conf rule (upstream systemd leaves the +# symlink to the distro), and this DDI boots with an empty /etc, so nothing +# ever creates the symlink and kubelet fails every pod sandbox with # "open /etc/resolv.conf: no such file or directory". # -# Recreate Flatcar's stock symlink: /etc/resolv.conf points at -# systemd-resolved's full resolver list (not the 127.0.0.53 stub), matching -# Flatcar >= 3510.3.0 behaviour. +# Point /etc/resolv.conf at systemd-resolved's full resolver list (not the +# 127.0.0.53 stub), matching standard server distro behaviour. L /etc/resolv.conf - - - - ../run/systemd/resolve/resolv.conf diff --git a/tests/unit/test_credentials_provisioning.py b/tests/unit/test_credentials_provisioning.py index 38b7664e..1c4366f9 100644 --- a/tests/unit/test_credentials_provisioning.py +++ b/tests/unit/test_credentials_provisioning.py @@ -85,8 +85,10 @@ def test_firstboot_credentials_are_noninteractive_and_presence_gated() -> None: assert f"ConditionCredential=|{credential}" in unit assert f"ImportCredential={credential}" in unit - # Flatcar's /usr ships no systemd-firstboot: the unit must not call it, or - # it fails at step EXEC and every firstboot.* credential is a silent no-op. + # The unit must not call systemd-firstboot: Bluefin's helper applies + # firstboot.hostname unconditionally and sets the live kernel hostname, + # which systemd-firstboot does not do. It also runs before sysinit.target + # with minimal dependencies (bash + coreutils only). assert "systemd-firstboot" not in unit assert "ExecStart=/usr/libexec/bluefin-firstboot-credentials" in unit assert unit.index("ExecStart=/usr/libexec/bluefin-firstboot-credentials") < unit.index( diff --git a/tests/unit/test_k0s_first_boot.py b/tests/unit/test_k0s_first_boot.py index 58dac21e..6cfab0e4 100644 --- a/tests/unit/test_k0s_first_boot.py +++ b/tests/unit/test_k0s_first_boot.py @@ -163,9 +163,8 @@ def test_presets_sort_between_ignition_and_fsdk_defaults() -> None: def test_etc_resolv_conf_symlink_is_seeded_for_kubelet() -> None: - # Flatcar ships /etc/resolv.conf inside the image /etc and strips - # /etc-populating lines from /usr/lib/tmpfiles.d at image build. This DDI - # imports only Flatcar's /usr and first-boots with an empty /etc, so + # FSDK 26.08 ships no /etc/resolv.conf rule (upstream systemd leaves the + # symlink to the distro), and this DDI boots with an empty /etc, so # without an explicit tmpfiles.d rule the symlink never exists and # kubelet fails every pod sandbox with # "open /etc/resolv.conf: no such file or directory". From d116bdf6ac865b71fb18c86a95c9107e4ea3d81a Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 00:10:59 -0500 Subject: [PATCH 2/5] docs: correct patch count in bump-fsdk-version.md patches/freedesktop-sdk/ carries six patches (0001-0006), not one. 0006 adds the Cilium/Kubernetes kernel options (VXLAN, GENEVE, tc BPF, conntrack/ss diagnostics); dropping it breaks the kubeadm and k0s sysexts. Update the verification checklist and the notes so a bumper does not drop it. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/bump-fsdk-version.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/skills/bump-fsdk-version.md b/docs/skills/bump-fsdk-version.md index 541f2df1..5164ca07 100644 --- a/docs/skills/bump-fsdk-version.md +++ b/docs/skills/bump-fsdk-version.md @@ -62,7 +62,7 @@ Before merging a bump: - [ ] `just validate` passes (element graph resolves with new ref) - [ ] `just tags` output matches the expected `latest / YY.MM / YY.MM.PP` triple -- [ ] The `patches/freedesktop-sdk/0001-project.conf-Add-GNOME-CAS-servers.patch` applied cleanly (no patch failure in `just validate`) +- [ ] All patches in `patches/freedesktop-sdk/` apply cleanly (no patch failure in `just validate`) - [ ] `just build-image` completes without error - [ ] The built image's `/usr/lib/os-release` carries the new `image-version` @@ -71,7 +71,12 @@ Before merging a bump: - `components/systemd-base.bst` was dropped in FSDK 26.08, and FSDK now ships its own systemd directly, so the previous `gnome-build-meta` systemd overrides were removed from `elements/freedesktop-sdk.bst`. - A point-release tag is immutable: once a GitHub Release for a given `image-version` is published, never republish different bits under it. -- **Only one CAS-config patch remains.** FSDK 26.08 absorbed the old `0001` CAS-limits patch (upstream `project.conf` sets `retry-limit`/`retry-delay`/`request-timeout` itself), so `patches/freedesktop-sdk/` carries only `0001-project.conf-Add-GNOME-CAS-servers.patch`. +- **Six patches live under `patches/freedesktop-sdk/`.** `0001` adds the GNOME + CAS servers to `project.conf`; `0002`–`0005` are FSDK build fixes; **`0006` + carries the Cilium/Kubernetes kernel options** (VXLAN, GENEVE, tc BPF, + conntrack/ss diagnostics). A bumper must not drop `0006` or the kubeadm and + k0s sysexts lose their datapath. If a release changed a patched file, + refresh the patch in place; never delete `0006` because "it looks small". - Junction overrides are only meaningful for components your local elements reference directly. The 25 GNOME sdk/* overrides (cairo, gtk3, pango, glib, gdk-pixbuf…) were dead weight — none of our `base-stack`, `brew-deps` etc. ever From 6b67f7cf1436c6f2a88f1cc7f821e27cdf30f4d3 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 00:11:08 -0500 Subject: [PATCH 3/5] fix: remove unused Justfile build alias The build recipe was a forwarding alias to export-image. No doc, CI workflow, or script calls bare 'just build'. Remove it to match the avoid-over-engineering skill's rule against forwarding aliases. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Justfile b/Justfile index ce2c3e7d..0eb280ac 100644 --- a/Justfile +++ b/Justfile @@ -75,11 +75,6 @@ test-unit: bats tests/unit # ── Build ───────────────────────────────────────────────────────────── -# Build and export the release image set (alias for export-image). -[group('build')] -build: - just export-image - # Build the k0s and KubeStellar systemd-sysext images. [group('sysext')] build-sysext: From 5643ae53f01fe54e145b0fba58d3e79b0f4d60a3 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 00:11:18 -0500 Subject: [PATCH 4/5] docs: add booty-integration and secure-boot-keys skills booty-integration.md: how Booty serves releases (HTTP Boot, iPXE chainload, per-node bluefin-node.ign, doInstall, kubeadm workers). secure-boot-keys.md: what each key signs (PK/KEK/DB, module cert, sysupdate signing), where keys live, how CI gets them via BOOT_KEYS_TARBALL and SYSUPDATE_SIGNING_KEY, and rotation mechanics. Index rows are deferred until #273 merges (it owns docs/skills/index.md). Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/booty-integration.md | 61 ++++++++++++++++++++++++++ docs/skills/secure-boot-keys.md | 75 ++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100644 docs/skills/booty-integration.md create mode 100644 docs/skills/secure-boot-keys.md diff --git a/docs/skills/booty-integration.md b/docs/skills/booty-integration.md new file mode 100644 index 00000000..de2f0b0a --- /dev/null +++ b/docs/skills/booty-integration.md @@ -0,0 +1,61 @@ +--- +name: booty-integration +description: How Booty serves Bluefin Server releases to nodes. Load when working on HTTP boot, iPXE chainloading, per-node Ignition, or kubeadm worker provisioning. +metadata: + type: reference + status: stable + last_updated: "2026-09-29" +--- +# Booty Integration + +[Booty](https://github.com/jeefy/booty) is the network boot server for Bluefin +Server at scale. It syncs a release and answers boot requests with the right +artifacts per node. + +## What Booty serves + +A release `v` (from GitHub Releases or the OCI artifact) contains the +netboot UKI, the OS DDI `bluefin-server_.raw`, `SHA256SUMS(.gpg)`, and a +netboot ESP image. Booty syncs these and serves them over HTTP. + +## Boot paths + +| Firmware | Path | +|---|---| +| UEFI HTTP Boot | Firmware fetches `http:///bluefin//bluefin-server-netboot.efi`; Secure Boot on. | +| iPXE chainload | Legacy or custom NICs chainload iPXE, which then HTTP-boots the UKI; Secure Boot off. | +| Legacy BIOS | Not supported for diskless; use the USB installer or disk install instead. | + +## Per-node configuration + +Booty writes a `bluefin-node.ign` per MAC address next to the UKI. The initrd +picks it up as the `ignition.config.url` system credential. Fields include +hostname, SSH keys, state disk, extensions, and k0s token. + +## Install to disk + +Setting `doInstall` in the node's Booty config boots it into +`booty-install.service`, which runs `systemd-sysinstall` against the local +disk. Without Booty, write `bluefin-server-netboot_.esp.raw` to a USB +stick and place an `import.pull.cred` credential in `/loader/credentials/` +specifying the URL to pull. + +## kubeadm workers + +Nodes that should join a kubeadm cluster get the kubeadm sysext via their +`bluefin-node.ign` (extension list) plus the cluster token. The sysext carries +kubelet and containerd; the node joins on first boot. + +## Read more + +- [Booty README](https://github.com/jeefy/booty) — flags, config file layout, + and the `feat/bluefin-http-boot` branch status. +- [ddi-installer.md](ddi-installer.md) — boot flow, DDI layout, and sysinstall + contract. +- [kubeadm-sysext.md](kubeadm-sysext.md) — worker sysext build and runtime + contract. + +## See also + +- [index.md](index.md) — lazy-load routing manifest. +- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary. diff --git a/docs/skills/secure-boot-keys.md b/docs/skills/secure-boot-keys.md new file mode 100644 index 00000000..e7c57475 --- /dev/null +++ b/docs/skills/secure-boot-keys.md @@ -0,0 +1,75 @@ +--- +name: secure-boot-keys +description: Secure Boot and image signing key management for Bluefin Server. Load when rotating keys, setting up CI secrets, or debugging signature verification. +metadata: + type: reference + status: stable + last_updated: "2026-09-29" +--- +# Secure Boot Key Management + +Bluefin Server signs the boot chain, kernel modules, and the release manifest. +This skill covers what each key signs, where it lives, and how CI gets it. + +## Key inventory + +| Key | Signs | Private location | Public location | +|---|---|---|---| +| PK (Platform Key) | KEK updates | `files/boot-keys/PK.key` | `files/boot-keys/PK.crt` | +| KEK (Key Exchange Key) | db updates | `files/boot-keys/KEK.key` | `files/boot-keys/KEK.crt` | +| DB (Signature Database) | systemd-boot, UKIs | `files/boot-keys/DB.key` | `files/boot-keys/DB.crt` | +| linux-module-cert | kernel modules | `files/boot-keys/linux-module-cert.key` | `files/boot-keys/modules/linux-module-cert.crt` | +| sysupdate-signing | `SHA256SUMS` | `files/boot-keys/sysupdate-signing.asc` | `files/boot-keys/import-pubring.pgp` | + +The module certificate is baked into the kernel's trusted keyring via +`SYSTEM_TRUSTED_KEYS`; changing it forces a kernel rebuild. + +## Local development + +```bash +just gen-dev-keys # throwaway keys in files/boot-keys/ (gitignored) +just gen-dev-keys --force # rotate (rebuilds the kernel) +``` + +`just validate` and `just build-image` depend on `gen-dev-keys`, so a fresh +clone builds out of the box. + +## CI secrets + +On `main`, the workflow unpacks two secrets: + +- `BOOT_KEYS_TARBALL` — gzipped tar of `files/boot-keys/` with the real + PK/KEK/DB, module key, and sysupdate signing key. +- `SYSUPDATE_SIGNING_KEY` — the ASCII-armored secret key written to + `files/boot-keys/sysupdate-signing.asc`. + +Pull requests get throwaway keys from `just gen-dev-keys`; they never see the +real secrets. + +## Key rotation + +1. Generate new keys (`just gen-dev-keys --force` locally, or the equivalent + in a secure environment for production). +2. Update the CI secrets (`BOOT_KEYS_TARBALL`, `SYSUPDATE_SIGNING_KEY`). +3. Bump `image-version` in `include/image.yml` (a new build is required; the + old image still trusts the old keys). +4. Nodes re-enroll Secure Boot keys on next boot if the firmware is in Setup + Mode, or the operator enrolls manually via the systemd-boot menu. + +The module key is the hardest to rotate: it is baked into the kernel, so a +new kernel build is mandatory. Plan kernel rebuilds into the rotation window. + +## Verification + +- [ ] `just validate` passes after key generation. +- [ ] `sbverify --cert files/boot-keys/DB.crt ` passes on built UKIs. +- [ ] `gpgv --keyring files/boot-keys/import-pubring.pgp SHA256SUMS.gpg SHA256SUMS` passes. +- [ ] No private keys are committed outside the gitignored `files/boot-keys/`. + +## See also + +- [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md) — + manifest signing and sysupdate trust model. +- [ddi-installer-build.md](ddi-installer-build.md) — local build and dogfood + workflow. +- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary. From 6f45178bb433fea82f0b0b2bf57c5d8bf7a53722 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 00:11:27 -0500 Subject: [PATCH 5/5] fix: stage system-container helper in the base image files/bin/system-container was documented as shipped at /usr/bin/system-container but no element staged it. Add os-system-container.bst to install it into /usr/bin and wire it into os-stack.bst. Remove the KNOWN_UNSTAGED waiver. machinectl and systemd-nspawn ship in the base image via FSDK's systemd component, so the helper's dependencies are present. Closes #154 Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/bluefin-server/os-stack.bst | 3 +++ elements/bluefin-server/os-system-container.bst | 11 +++++++++++ tests/unit/test_files_payload_reachability.py | 12 +----------- 3 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 elements/bluefin-server/os-system-container.bst diff --git a/elements/bluefin-server/os-stack.bst b/elements/bluefin-server/os-stack.bst index 1a9e54c8..e48b0b68 100644 --- a/elements/bluefin-server/os-stack.bst +++ b/elements/bluefin-server/os-stack.bst @@ -45,5 +45,8 @@ depends: # System Management Justfile - bluefin-server/os-justfile.bst + # system-container helper (machinectl wrapper) + - bluefin-server/os-system-container.bst + # Console issue banner with KubeStellar dashboard URL - bluefin-server/os-issue.bst diff --git a/elements/bluefin-server/os-system-container.bst b/elements/bluefin-server/os-system-container.bst new file mode 100644 index 00000000..f3d5dc10 --- /dev/null +++ b/elements/bluefin-server/os-system-container.bst @@ -0,0 +1,11 @@ +kind: import +description: | + Install the system-container helper into the target OS image. The helper + wraps machinectl to start, stop, enter, and reset systemd-nspawn machines. + +sources: + - kind: local + path: files/bin/system-container + +config: + target: /usr/bin diff --git a/tests/unit/test_files_payload_reachability.py b/tests/unit/test_files_payload_reachability.py index 4150210e..c082fda0 100644 --- a/tests/unit/test_files_payload_reachability.py +++ b/tests/unit/test_files_payload_reachability.py @@ -13,11 +13,6 @@ also invisible to the unit suite, because bats tests execute helper scripts straight out of the checkout whether or not they ship. -That gap is not hypothetical: ``files/bin/system-container`` is documented in -``docs/skills/system-containers.md`` as "shipped in the OS image" at -``/usr/bin/system-container`` and has bats coverage, yet no element stages it — -see issue #154. - This module enforces both directions of the contract: * every ``path:`` an element declares still exists on disk, so renaming a @@ -63,12 +58,7 @@ # Payload that claims to ship but is staged by no element. Each entry must name # the issue tracking its resolution. Shrink this set; never grow it. -KNOWN_UNSTAGED = { - # docs/skills/system-containers.md promises /usr/bin/system-container in the - # OS image. No element stages it. Tracked by issue #154 — fixing that issue - # (staging it, or correcting the docs) must remove this entry. - "files/bin/system-container", -} +KNOWN_UNSTAGED = set() def _element_files():