diff --git a/Justfile b/Justfile index ce2c3e7d..0eb280ac 100644 --- a/Justfile +++ b/Justfile @@ -75,11 +75,6 @@ test-unit: bats tests/unit # ── Build ───────────────────────────────────────────────────────────── -# Build and export the release image set (alias for export-image). -[group('build')] -build: - just export-image - # Build the k0s and KubeStellar systemd-sysext images. [group('sysext')] build-sysext: diff --git a/docs/skills/booty-integration.md b/docs/skills/booty-integration.md new file mode 100644 index 00000000..de2f0b0a --- /dev/null +++ b/docs/skills/booty-integration.md @@ -0,0 +1,61 @@ +--- +name: booty-integration +description: How Booty serves Bluefin Server releases to nodes. Load when working on HTTP boot, iPXE chainloading, per-node Ignition, or kubeadm worker provisioning. +metadata: + type: reference + status: stable + last_updated: "2026-09-29" +--- +# Booty Integration + +[Booty](https://github.com/jeefy/booty) is the network boot server for Bluefin +Server at scale. It syncs a release and answers boot requests with the right +artifacts per node. + +## What Booty serves + +A release `v` (from GitHub Releases or the OCI artifact) contains the +netboot UKI, the OS DDI `bluefin-server_.raw`, `SHA256SUMS(.gpg)`, and a +netboot ESP image. Booty syncs these and serves them over HTTP. + +## Boot paths + +| Firmware | Path | +|---|---| +| UEFI HTTP Boot | Firmware fetches `http:///bluefin//bluefin-server-netboot.efi`; Secure Boot on. | +| iPXE chainload | Legacy or custom NICs chainload iPXE, which then HTTP-boots the UKI; Secure Boot off. | +| Legacy BIOS | Not supported for diskless; use the USB installer or disk install instead. | + +## Per-node configuration + +Booty writes a `bluefin-node.ign` per MAC address next to the UKI. The initrd +picks it up as the `ignition.config.url` system credential. Fields include +hostname, SSH keys, state disk, extensions, and k0s token. + +## Install to disk + +Setting `doInstall` in the node's Booty config boots it into +`booty-install.service`, which runs `systemd-sysinstall` against the local +disk. Without Booty, write `bluefin-server-netboot_.esp.raw` to a USB +stick and place an `import.pull.cred` credential in `/loader/credentials/` +specifying the URL to pull. + +## kubeadm workers + +Nodes that should join a kubeadm cluster get the kubeadm sysext via their +`bluefin-node.ign` (extension list) plus the cluster token. The sysext carries +kubelet and containerd; the node joins on first boot. + +## Read more + +- [Booty README](https://github.com/jeefy/booty) — flags, config file layout, + and the `feat/bluefin-http-boot` branch status. +- [ddi-installer.md](ddi-installer.md) — boot flow, DDI layout, and sysinstall + contract. +- [kubeadm-sysext.md](kubeadm-sysext.md) — worker sysext build and runtime + contract. + +## See also + +- [index.md](index.md) — lazy-load routing manifest. +- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary. diff --git a/docs/skills/bump-fsdk-version.md b/docs/skills/bump-fsdk-version.md index 541f2df1..5164ca07 100644 --- a/docs/skills/bump-fsdk-version.md +++ b/docs/skills/bump-fsdk-version.md @@ -62,7 +62,7 @@ Before merging a bump: - [ ] `just validate` passes (element graph resolves with new ref) - [ ] `just tags` output matches the expected `latest / YY.MM / YY.MM.PP` triple -- [ ] The `patches/freedesktop-sdk/0001-project.conf-Add-GNOME-CAS-servers.patch` applied cleanly (no patch failure in `just validate`) +- [ ] All patches in `patches/freedesktop-sdk/` apply cleanly (no patch failure in `just validate`) - [ ] `just build-image` completes without error - [ ] The built image's `/usr/lib/os-release` carries the new `image-version` @@ -71,7 +71,12 @@ Before merging a bump: - `components/systemd-base.bst` was dropped in FSDK 26.08, and FSDK now ships its own systemd directly, so the previous `gnome-build-meta` systemd overrides were removed from `elements/freedesktop-sdk.bst`. - A point-release tag is immutable: once a GitHub Release for a given `image-version` is published, never republish different bits under it. -- **Only one CAS-config patch remains.** FSDK 26.08 absorbed the old `0001` CAS-limits patch (upstream `project.conf` sets `retry-limit`/`retry-delay`/`request-timeout` itself), so `patches/freedesktop-sdk/` carries only `0001-project.conf-Add-GNOME-CAS-servers.patch`. +- **Six patches live under `patches/freedesktop-sdk/`.** `0001` adds the GNOME + CAS servers to `project.conf`; `0002`–`0005` are FSDK build fixes; **`0006` + carries the Cilium/Kubernetes kernel options** (VXLAN, GENEVE, tc BPF, + conntrack/ss diagnostics). A bumper must not drop `0006` or the kubeadm and + k0s sysexts lose their datapath. If a release changed a patched file, + refresh the patch in place; never delete `0006` because "it looks small". - Junction overrides are only meaningful for components your local elements reference directly. The 25 GNOME sdk/* overrides (cairo, gtk3, pango, glib, gdk-pixbuf…) were dead weight — none of our `base-stack`, `brew-deps` etc. ever diff --git a/docs/skills/secure-boot-keys.md b/docs/skills/secure-boot-keys.md new file mode 100644 index 00000000..e7c57475 --- /dev/null +++ b/docs/skills/secure-boot-keys.md @@ -0,0 +1,75 @@ +--- +name: secure-boot-keys +description: Secure Boot and image signing key management for Bluefin Server. Load when rotating keys, setting up CI secrets, or debugging signature verification. +metadata: + type: reference + status: stable + last_updated: "2026-09-29" +--- +# Secure Boot Key Management + +Bluefin Server signs the boot chain, kernel modules, and the release manifest. +This skill covers what each key signs, where it lives, and how CI gets it. + +## Key inventory + +| Key | Signs | Private location | Public location | +|---|---|---|---| +| PK (Platform Key) | KEK updates | `files/boot-keys/PK.key` | `files/boot-keys/PK.crt` | +| KEK (Key Exchange Key) | db updates | `files/boot-keys/KEK.key` | `files/boot-keys/KEK.crt` | +| DB (Signature Database) | systemd-boot, UKIs | `files/boot-keys/DB.key` | `files/boot-keys/DB.crt` | +| linux-module-cert | kernel modules | `files/boot-keys/linux-module-cert.key` | `files/boot-keys/modules/linux-module-cert.crt` | +| sysupdate-signing | `SHA256SUMS` | `files/boot-keys/sysupdate-signing.asc` | `files/boot-keys/import-pubring.pgp` | + +The module certificate is baked into the kernel's trusted keyring via +`SYSTEM_TRUSTED_KEYS`; changing it forces a kernel rebuild. + +## Local development + +```bash +just gen-dev-keys # throwaway keys in files/boot-keys/ (gitignored) +just gen-dev-keys --force # rotate (rebuilds the kernel) +``` + +`just validate` and `just build-image` depend on `gen-dev-keys`, so a fresh +clone builds out of the box. + +## CI secrets + +On `main`, the workflow unpacks two secrets: + +- `BOOT_KEYS_TARBALL` — gzipped tar of `files/boot-keys/` with the real + PK/KEK/DB, module key, and sysupdate signing key. +- `SYSUPDATE_SIGNING_KEY` — the ASCII-armored secret key written to + `files/boot-keys/sysupdate-signing.asc`. + +Pull requests get throwaway keys from `just gen-dev-keys`; they never see the +real secrets. + +## Key rotation + +1. Generate new keys (`just gen-dev-keys --force` locally, or the equivalent + in a secure environment for production). +2. Update the CI secrets (`BOOT_KEYS_TARBALL`, `SYSUPDATE_SIGNING_KEY`). +3. Bump `image-version` in `include/image.yml` (a new build is required; the + old image still trusts the old keys). +4. Nodes re-enroll Secure Boot keys on next boot if the firmware is in Setup + Mode, or the operator enrolls manually via the systemd-boot menu. + +The module key is the hardest to rotate: it is baked into the kernel, so a +new kernel build is mandatory. Plan kernel rebuilds into the rotation window. + +## Verification + +- [ ] `just validate` passes after key generation. +- [ ] `sbverify --cert files/boot-keys/DB.crt ` passes on built UKIs. +- [ ] `gpgv --keyring files/boot-keys/import-pubring.pgp SHA256SUMS.gpg SHA256SUMS` passes. +- [ ] No private keys are committed outside the gitignored `files/boot-keys/`. + +## See also + +- [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md) — + manifest signing and sysupdate trust model. +- [ddi-installer-build.md](ddi-installer-build.md) — local build and dogfood + workflow. +- [CONTEXT.md](../../CONTEXT.md) — canonical project domain glossary. diff --git a/elements/bluefin-server/os-resolv-conf.bst b/elements/bluefin-server/os-resolv-conf.bst index 79361cc1..f088390a 100644 --- a/elements/bluefin-server/os-resolv-conf.bst +++ b/elements/bluefin-server/os-resolv-conf.bst @@ -1,10 +1,10 @@ kind: import description: | - Install the tmpfiles.d rule that recreates Flatcar's /etc/resolv.conf - symlink to systemd-resolved's resolv.conf. The Flatcar /usr payload does - not carry it (Flatcar ships the symlink in the image /etc, which this DDI - does not import), and kubelet requires /etc/resolv.conf to create any pod - sandbox. + Install the tmpfiles.d rule that creates /etc/resolv.conf as a symlink to + systemd-resolved's resolv.conf. FSDK 26.08 ships no /etc/resolv.conf rule + (upstream systemd leaves the symlink to the distro), and this DDI boots + with an empty /etc, so nothing ever creates the symlink and kubelet fails + every pod sandbox with "open /etc/resolv.conf: no such file or directory". sources: - kind: local diff --git a/elements/bluefin-server/os-stack.bst b/elements/bluefin-server/os-stack.bst index 1a9e54c8..e48b0b68 100644 --- a/elements/bluefin-server/os-stack.bst +++ b/elements/bluefin-server/os-stack.bst @@ -45,5 +45,8 @@ depends: # System Management Justfile - bluefin-server/os-justfile.bst + # system-container helper (machinectl wrapper) + - bluefin-server/os-system-container.bst + # Console issue banner with KubeStellar dashboard URL - bluefin-server/os-issue.bst diff --git a/elements/bluefin-server/os-system-container.bst b/elements/bluefin-server/os-system-container.bst new file mode 100644 index 00000000..f3d5dc10 --- /dev/null +++ b/elements/bluefin-server/os-system-container.bst @@ -0,0 +1,11 @@ +kind: import +description: | + Install the system-container helper into the target OS image. The helper + wraps machinectl to start, stop, enter, and reset systemd-nspawn machines. + +sources: + - kind: local + path: files/bin/system-container + +config: + target: /usr/bin diff --git a/files/os/libexec/bluefin-firstboot-credentials b/files/os/libexec/bluefin-firstboot-credentials index 5b90d941..845b5744 100755 --- a/files/os/libexec/bluefin-firstboot-credentials +++ b/files/os/libexec/bluefin-firstboot-credentials @@ -5,14 +5,17 @@ # firstboot.locale, firstboot.locale-messages -> /etc/locale.conf # firstboot.keymap -> /etc/vconsole.conf # firstboot.timezone -> /etc/localtime -# and additionally (systemd <= v259, Flatcar's, has no firstboot.hostname; -# systemd v260 added it as a static hostname applied only when none is set. -# Bluefin applies it unconditionally and also sets the live kernel hostname -# so the first DHCP request carries the name): +# and additionally: # firstboot.hostname -> /etc/hostname + kernel hostname # from $CREDENTIALS_DIRECTORY, as populated by ImportCredential= in -# bluefin-firstboot-credentials.service. The OS payload is Flatcar's /usr, -# which is built without systemd-firstboot, so the unit cannot call it. +# bluefin-firstboot-credentials.service. +# +# This helper exists rather than calling systemd-firstboot because Bluefin +# needs firstboot.hostname applied unconditionally (systemd v260+ applies it +# only as a static hostname when none is set) and needs the live kernel +# hostname set so the first DHCP request carries the name. It also validates +# every credential before writing anything, so a bad value fails the unit +# instead of leaving a partial configuration. # # Known differences from systemd-firstboot: trailing newlines in a credential # are stripped rather than rejected; LC_MESSAGES= is written even when it diff --git a/files/os/tmpfiles.d/20-bluefin-resolv.conf b/files/os/tmpfiles.d/20-bluefin-resolv.conf index 085ac2ab..6678502a 100644 --- a/files/os/tmpfiles.d/20-bluefin-resolv.conf +++ b/files/os/tmpfiles.d/20-bluefin-resolv.conf @@ -1,11 +1,8 @@ -# Flatcar's image build populates /etc/resolv.conf in the image's /etc and -# strips /etc-populating lines from /usr/lib/tmpfiles.d (see -# flatcar/scripts build_library/build_image_util.sh). This DDI imports only -# Flatcar's /usr and boots with an empty /etc, so nothing ever creates the -# symlink and kubelet fails every pod sandbox with +# FSDK 26.08 ships no /etc/resolv.conf rule (upstream systemd leaves the +# symlink to the distro), and this DDI boots with an empty /etc, so nothing +# ever creates the symlink and kubelet fails every pod sandbox with # "open /etc/resolv.conf: no such file or directory". # -# Recreate Flatcar's stock symlink: /etc/resolv.conf points at -# systemd-resolved's full resolver list (not the 127.0.0.53 stub), matching -# Flatcar >= 3510.3.0 behaviour. +# Point /etc/resolv.conf at systemd-resolved's full resolver list (not the +# 127.0.0.53 stub), matching standard server distro behaviour. L /etc/resolv.conf - - - - ../run/systemd/resolve/resolv.conf diff --git a/tests/unit/test_credentials_provisioning.py b/tests/unit/test_credentials_provisioning.py index 38b7664e..1c4366f9 100644 --- a/tests/unit/test_credentials_provisioning.py +++ b/tests/unit/test_credentials_provisioning.py @@ -85,8 +85,10 @@ def test_firstboot_credentials_are_noninteractive_and_presence_gated() -> None: assert f"ConditionCredential=|{credential}" in unit assert f"ImportCredential={credential}" in unit - # Flatcar's /usr ships no systemd-firstboot: the unit must not call it, or - # it fails at step EXEC and every firstboot.* credential is a silent no-op. + # The unit must not call systemd-firstboot: Bluefin's helper applies + # firstboot.hostname unconditionally and sets the live kernel hostname, + # which systemd-firstboot does not do. It also runs before sysinit.target + # with minimal dependencies (bash + coreutils only). assert "systemd-firstboot" not in unit assert "ExecStart=/usr/libexec/bluefin-firstboot-credentials" in unit assert unit.index("ExecStart=/usr/libexec/bluefin-firstboot-credentials") < unit.index( diff --git a/tests/unit/test_files_payload_reachability.py b/tests/unit/test_files_payload_reachability.py index 4150210e..c082fda0 100644 --- a/tests/unit/test_files_payload_reachability.py +++ b/tests/unit/test_files_payload_reachability.py @@ -13,11 +13,6 @@ also invisible to the unit suite, because bats tests execute helper scripts straight out of the checkout whether or not they ship. -That gap is not hypothetical: ``files/bin/system-container`` is documented in -``docs/skills/system-containers.md`` as "shipped in the OS image" at -``/usr/bin/system-container`` and has bats coverage, yet no element stages it — -see issue #154. - This module enforces both directions of the contract: * every ``path:`` an element declares still exists on disk, so renaming a @@ -63,12 +58,7 @@ # Payload that claims to ship but is staged by no element. Each entry must name # the issue tracking its resolution. Shrink this set; never grow it. -KNOWN_UNSTAGED = { - # docs/skills/system-containers.md promises /usr/bin/system-container in the - # OS image. No element stages it. Tracked by issue #154 — fixing that issue - # (staging it, or correcting the docs) must remove this entry. - "files/bin/system-container", -} +KNOWN_UNSTAGED = set() def _element_files(): diff --git a/tests/unit/test_k0s_first_boot.py b/tests/unit/test_k0s_first_boot.py index 58dac21e..6cfab0e4 100644 --- a/tests/unit/test_k0s_first_boot.py +++ b/tests/unit/test_k0s_first_boot.py @@ -163,9 +163,8 @@ def test_presets_sort_between_ignition_and_fsdk_defaults() -> None: def test_etc_resolv_conf_symlink_is_seeded_for_kubelet() -> None: - # Flatcar ships /etc/resolv.conf inside the image /etc and strips - # /etc-populating lines from /usr/lib/tmpfiles.d at image build. This DDI - # imports only Flatcar's /usr and first-boots with an empty /etc, so + # FSDK 26.08 ships no /etc/resolv.conf rule (upstream systemd leaves the + # symlink to the distro), and this DDI boots with an empty /etc, so # without an explicit tmpfiles.d rule the symlink never exists and # kubelet fails every pod sandbox with # "open /etc/resolv.conf: no such file or directory".