From 845a13302cdc3aeb19f000ff504ac50501863c12 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Thu, 19 Nov 2026 21:20:34 -0600 Subject: [PATCH 1/2] fix(ssh): accept a provisioned key while root is still locked Root ships locked ("!unprovisioned"). Ignition users entries and the ssh.authorized_keys.root credential install a key without touching the password, so root stays locked, and sshd without PAM refuses a locked account even for keys ("User root not allowed because account is locked"). The documented ssh root@ path was refused on netboot, Booty and credential-provisioned nodes. UsePAM yes hands the account check to FSDK's /etc/pam.d/sshd, which ignores the password field. PasswordAuthentication and KbdInteractiveAuthentication stay off, so PAM is never asked for a password. The dogfood SSH login no longer unlocks root first, and asserts root is still locked when the key login succeeds. Assisted-by: Claude via Claude Code Co-Authored-By: Claude Code --- docs/skills/systemd-sysupdate-verification.md | 2 +- docs/skills/usb-installer.md | 4 ++-- files/os/ssh/sshd_config.d/bluefin-server.conf | 6 ++++++ files/os/tmpfiles.d/40-bluefin-motd.conf | 2 +- scripts/dogfood-installer.sh | 9 ++++++--- tests/unit/test_automatic_updates.py | 2 +- tests/unit/test_root_credential_contract.py | 12 ++++++++++++ 7 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/skills/systemd-sysupdate-verification.md b/docs/skills/systemd-sysupdate-verification.md index 5ad86f44..761a0607 100644 --- a/docs/skills/systemd-sysupdate-verification.md +++ b/docs/skills/systemd-sysupdate-verification.md @@ -159,7 +159,7 @@ last success, the last error and the run it last recorded in `/var/lib/bluefin-update-status/state`, so they survive reboots and updates, and writes `/run/issue.d/40-bluefin-update.issue` (agetty, next to `/usr/lib/issue.d/30-bluefin.issue`) and `/run/motd`. login (`MOTD_FILE` in -`login.defs`) and sshd (`UsePAM no`) only print `/etc/motd`, so +`login.defs`) and sshd (`PrintMotd`) only print `/etc/motd`, so `tmpfiles.d/40-bluefin-motd.conf` links it to `/run/motd` unless the operator has an `/etc/motd` of their own. It only reads local state and contacts nothing. A failed check also stays a failed unit, `systemd-sysupdate.service`, diff --git a/docs/skills/usb-installer.md b/docs/skills/usb-installer.md index 780a9a9c..fec18d2e 100644 --- a/docs/skills/usb-installer.md +++ b/docs/skills/usb-installer.md @@ -343,8 +343,8 @@ printf 'd /home/%s 0700 %s %s -\nd /home/%s/.ssh 0700 %s %s -\nf+~ /home/%s/.ssh # ssh.listen: systemd-ssh-generator takes the binary form only (v261 does # not unbase64); services take the base64 form systemd-creds writes printf 22 | enc ssh.listen - | base64 -d > ssh.listen.cred -# "*": no password login, but not locked like sysusers' default "!*", which -# sshd (UsePAM no) refuses even for keys +# "*": no password login; sshd takes the key either way (UsePAM yes), but +# "*" is not reported locked like sysusers' default "!*" printf '*' | enc "passwd.hashed-password.$user" "passwd.hashed-password.$user.cred" printf '!*' | enc passwd.hashed-password.root passwd.hashed-password.root.cred # copy the five .cred files into /loader/credentials/ on the stick's ESP diff --git a/files/os/ssh/sshd_config.d/bluefin-server.conf b/files/os/ssh/sshd_config.d/bluefin-server.conf index 34d9ade8..c9db7d7d 100644 --- a/files/os/ssh/sshd_config.d/bluefin-server.conf +++ b/files/os/ssh/sshd_config.d/bluefin-server.conf @@ -9,6 +9,12 @@ PermitRootLogin prohibit-password PubkeyAuthentication yes PasswordAuthentication no KbdInteractiveAuthentication no +# Without PAM, sshd refuses a locked account ("!" password) even for a key, +# and root stays locked when Ignition or the ssh.authorized_keys.root +# credential gives it only a key. With PAM, FSDK's /etc/pam.d/sshd account +# stack decides instead; it ignores the password field. The two lines above +# keep PAM out of authentication, so this opens no password path. +UsePAM yes # Keep standard SSH file transfer available without another server process. Subsystem sftp internal-sftp diff --git a/files/os/tmpfiles.d/40-bluefin-motd.conf b/files/os/tmpfiles.d/40-bluefin-motd.conf index 9fa33d91..1f8d3188 100644 --- a/files/os/tmpfiles.d/40-bluefin-motd.conf +++ b/files/os/tmpfiles.d/40-bluefin-motd.conf @@ -1,4 +1,4 @@ -# login (MOTD_FILE in login.defs) and sshd (UsePAM no) print /etc/motd only. +# login (MOTD_FILE in login.defs) and sshd (PrintMotd) print /etc/motd only. # Point it at the runtime message bluefin-update-status writes. Without "+", # an /etc/motd the operator wrote stays. L /etc/motd - - - - ../run/motd diff --git a/scripts/dogfood-installer.sh b/scripts/dogfood-installer.sh index c367f505..78d45ad8 100644 --- a/scripts/dogfood-installer.sh +++ b/scripts/dogfood-installer.sh @@ -873,10 +873,11 @@ sync echo "PROBE update-done" UPDATE # Appended to the probe of step 6: an SSH login through systemd-ssh-generator's -# local socket must print the banner (sshd reads /etc/motd). sshd refuses a -# locked root even for keys, so root gets the unlocked "*" first. +# local socket must print the banner (sshd reads /etc/motd). Root keeps its +# locked password, as with an Ignition- or credential-provisioned key: the +# login proves sshd (UsePAM yes) takes a key for a locked root. cat > "${state}/ssh.sh" <<'SSH' -usermod -p '*' root +echo "PROBE ssh-root-locked=$(passwd -S root | cut -d' ' -f2)" ssh-keygen -q -t ed25519 -N '' -f /run/dogfood-ssh install -d -m 0700 /root/.ssh cat /run/dogfood-ssh.pub >> /root/.ssh/authorized_keys @@ -1113,6 +1114,8 @@ banner_lacks 6-updated boot 'FAILED' banner_lacks 6-updated boot 'staged' grep -aqF "PROBE ssh-motd=Bluefin Server ${next_ver}, automatic updates on" "${state}/6-updated.ttyS1.log" \ || fail "6-updated: an SSH login does not show the banner: $(grep -a 'ssh' "${state}/6-updated.ttyS1.log" | tail -n 5)" +grep -aq 'PROBE ssh-root-locked=L' "${state}/6-updated.ttyS1.log" \ + || fail "6-updated: root was not locked for the SSH key login: $(grep -ao 'PROBE ssh-root-locked=.*' "${state}/6-updated.ttyS1.log")" checked="a local server after reporting an unreachable and a foreign-signed source as failed checks on its banner" [ "${next}" != release ] || checked="the image's own source" diff --git a/tests/unit/test_automatic_updates.py b/tests/unit/test_automatic_updates.py index 17cfa04d..d2f12e8f 100644 --- a/tests/unit/test_automatic_updates.py +++ b/tests/unit/test_automatic_updates.py @@ -248,7 +248,7 @@ def test_the_banners_are_the_files_agetty_login_and_sshd_read() -> None: # agetty merges /run/issue.d with the image's /usr/lib/issue.d banner. assert '"/run/issue.d/40-bluefin-update.issue"' in script assert (ROOT / "files" / "os" / "issue.d" / "30-bluefin.issue").name < "40-bluefin-update.issue" - # shadow's login (MOTD_FILE /etc/motd) and sshd (UsePAM no) read only + # shadow's login (MOTD_FILE /etc/motd) and sshd (PrintMotd) read only # /etc/motd; it links to the runtime file, never over an operator's own. assert '"/run/motd"' in script lines = [line.split() for line in MOTD_LINK.read_text(encoding="utf-8").splitlines() if line and not line.startswith("#")] diff --git a/tests/unit/test_root_credential_contract.py b/tests/unit/test_root_credential_contract.py index b9be6c6e..ebcf127b 100644 --- a/tests/unit/test_root_credential_contract.py +++ b/tests/unit/test_root_credential_contract.py @@ -130,6 +130,18 @@ def test_sshd_dropin_never_permits_password_authentication() -> None: assert not re.search(r"^PasswordAuthentication\s+yes$", sshd, re.MULTILINE) +def test_sshd_accepts_keys_for_a_root_that_has_no_password() -> None: + # Root ships locked ("!unprovisioned"); Ignition and the + # ssh.authorized_keys.root credential give it a key without unlocking it. + # Without PAM, sshd refuses a locked account even for keys. + sshd = SSHD_DROPIN.read_text(encoding="utf-8") + + assert re.search(r"^UsePAM\s+yes$", sshd, re.MULTILINE) + # The dogfood SSH login must prove it on a locked root, not unlock it. + dogfood = (REPO_ROOT / "scripts" / "dogfood-installer.sh").read_text(encoding="utf-8") + assert "usermod -p '*' root" not in dogfood + + def test_generated_sshd_gets_per_device_host_keys() -> None: dropin = GENERATED_SSHD_DROPIN.read_text(encoding="utf-8") From 132d56d92b908adf3fb03a1136288c7e25f6713a Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Thu, 19 Nov 2026 22:36:35 -0600 Subject: [PATCH 2/2] fix(os): ship the sshd, sysctl and journald drop-ins in /usr These three drop-ins lived in the factory /etc. systemd-tmpfiles copies factory files with C, which never overwrites, and a disk install keeps /etc across A/B updates, so an installed node kept the copy it first booted. No change to them, including the UsePAM fix, could reach it. resolved, system.conf and the prompt already moved to /usr for this. - sysctl and journald: written to /usr/lib/sysctl.d and /usr/lib/systemd/journald.conf.d, which both read directly. - sshd: installed in /usr/lib/bluefin/sshd_config.d and linked into /etc/ssh/sshd_config.d by a tmpfiles L line. sshd has no /usr config directory, and L leaves an operator's own file of that name alone. The main sshd_config, which only includes sshd_config.d, stays in the factory /etc. - bluefin-migrate-etc-vendor (bluefin-host) runs once on installed nodes, before systemd-tmpfiles-setup, and removes each old copy only while it is byte-identical to a version a release shipped (hashes from every v26.09+ tag). An edited copy stays and keeps winning. Assisted-by: Claude via Claude Code Co-Authored-By: Claude Code --- docs/skills/ddi-installer.md | 6 +- docs/skills/tpm2-credential-sealing.md | 2 +- elements/bluefin-server/os-host.bst | 8 +- elements/bluefin-server/os-sshd-config.bst | 27 ++++- elements/oci/bluefin-server-usr.bst | 17 ++- .../homelab/cluster/cmd/bluefin-host/main.go | 2 + .../internal/host/etcvendor/etcvendor.go | 97 +++++++++++++++++ .../internal/host/etcvendor/etcvendor_test.go | 78 ++++++++++++++ .../sshd_config.d/bluefin-server.conf | 0 .../system/bluefin-migrate-etc-vendor.service | 23 ++++ files/os/tmpfiles.d/50-bluefin-sshd.conf | 6 ++ tests/unit/conftest.py | 1 + tests/unit/test_etc_vendor_dropins.py | 102 ++++++++++++++++++ tests/unit/test_root_credential_contract.py | 2 +- 14 files changed, 358 insertions(+), 13 deletions(-) create mode 100644 files/homelab/cluster/internal/host/etcvendor/etcvendor.go create mode 100644 files/homelab/cluster/internal/host/etcvendor/etcvendor_test.go rename files/os/{ssh => }/sshd_config.d/bluefin-server.conf (100%) create mode 100644 files/os/systemd/system/bluefin-migrate-etc-vendor.service create mode 100644 files/os/tmpfiles.d/50-bluefin-sshd.conf create mode 100644 tests/unit/test_etc_vendor_dropins.py diff --git a/docs/skills/ddi-installer.md b/docs/skills/ddi-installer.md index 0fb3e6af..6f8f11e3 100644 --- a/docs/skills/ddi-installer.md +++ b/docs/skills/ddi-installer.md @@ -57,7 +57,11 @@ hash partition (`bluefin_usr_verity_`), and the root hash is recorded in live in `/usr/share/factory/etc` and are copied in by `systemd-tmpfiles`. A diskless node keeps `/etc` on tmpfs, so it starts empty on every boot; a disk install keeps it on the root partition (`files/os/repart.d/50-root.conf`), -where it persists across A/B updates. +where it persists across A/B updates. `systemd-tmpfiles` copies a factory +file only when it is missing, so a vendor setting that must follow updates +ships in `/usr` instead: read from `/usr/lib` (sysctl, journald, resolved, +`system.conf`) or linked into `/etc` with an `L` line (the sshd drop-in). +`bluefin-migrate-etc-vendor` removes the unmodified copies older images left. ## The boot chain diff --git a/docs/skills/tpm2-credential-sealing.md b/docs/skills/tpm2-credential-sealing.md index 6248f894..f85b946e 100644 --- a/docs/skills/tpm2-credential-sealing.md +++ b/docs/skills/tpm2-credential-sealing.md @@ -40,7 +40,7 @@ The image ships no password. Root's `/etc/shadow` entry is `!unprovisioned` (`elements/oci/bluefin-server-usr.bst`), systemd's marker for a root account nobody has configured: `passwd -S root` reports `L`, console login, `su` and `sulogin` refuse it, and sshd accepts keys only -(`files/os/ssh/sshd_config.d/bluefin-server.conf`). Access is provisioned per +(`files/os/sshd_config.d/bluefin-server.conf`). Access is provisioned per node: - **SSH key:** Ignition `passwd.users` with `name: root` and diff --git a/elements/bluefin-server/os-host.bst b/elements/bluefin-server/os-host.bst index 03ae4bcd..c0efc965 100644 --- a/elements/bluefin-server/os-host.bst +++ b/elements/bluefin-server/os-host.bst @@ -11,7 +11,7 @@ description: | bluefin-boot-deadline, bluefin-diskless-update-check, bluefin-sysext-fetch, bluefin-boot-origin; - first boot: bluefin-hostname, bluefin-firstboot-credentials, - bluefin-repair-etc-modes; + bluefin-repair-etc-modes, bluefin-migrate-etc-vendor; - bluefin-sysext-modules, which module sysexts' load units run. Unit drop-ins call `bluefin-host kube-interlock`, `kured-flag`, `mask-uki-transfer` and `boot-is-good` directly. @@ -57,8 +57,8 @@ config: install -D -m 0755 bluefin-host "%{install-root}/usr/bin/bluefin-host" mkdir -p "%{install-root}/usr/libexec" for helper in boot-deadline boot-origin diskless-update-check \ - firstboot-credentials hostname reboot-lock repair-etc-modes \ - sysext-fetch sysext-modules update-pending update-status \ - update-status-migrate; do + firstboot-credentials hostname migrate-etc-vendor reboot-lock \ + repair-etc-modes sysext-fetch sysext-modules update-pending \ + update-status update-status-migrate; do ln -s ../bin/bluefin-host "%{install-root}/usr/libexec/bluefin-${helper}" done diff --git a/elements/bluefin-server/os-sshd-config.bst b/elements/bluefin-server/os-sshd-config.bst index d99a61fe..b6df87c0 100644 --- a/elements/bluefin-server/os-sshd-config.bst +++ b/elements/bluefin-server/os-sshd-config.bst @@ -1,9 +1,30 @@ -kind: import -description: Install sshd configuration and drop-ins. +kind: manual +description: | + sshd configuration. The main sshd_config, which only includes + /etc/ssh/sshd_config.d/*.conf, goes to the factory /etc. The vendor + drop-in goes to /usr/lib/bluefin/sshd_config.d, and + files/os/tmpfiles.d/50-bluefin-sshd.conf (os-resolv-conf.bst) links it into + /etc/ssh/sshd_config.d: a disk install keeps /etc across A/B updates, so a + copy would stay at the version the node first booted, while the link + follows /usr. L leaves an operator's own file of that name alone. + +build-depends: + - base/base-stack.bst sources: - kind: local path: files/os/ssh + directory: ssh-src + - kind: local + path: files/os/sshd_config.d + directory: dropin-src + +variables: + strip-binaries: "" config: - target: /etc/ssh + install-commands: + - | + set -eu + install -Dm0644 ssh-src/sshd_config "%{install-root}/etc/ssh/sshd_config" + install -Dm0644 dropin-src/*.conf -t "%{install-root}/usr/lib/bluefin/sshd_config.d/" diff --git a/elements/oci/bluefin-server-usr.bst b/elements/oci/bluefin-server-usr.bst index d580c7ef..5c3ebb78 100644 --- a/elements/oci/bluefin-server-usr.bst +++ b/elements/oci/bluefin-server-usr.bst @@ -72,7 +72,7 @@ config: grep -q '^root:' /sysroot/etc/passwd chmod 0400 /sysroot/etc/shadow /sysroot/etc/gshadow - mkdir -p /sysroot/etc/systemd/system /sysroot/etc/systemd/journald.conf.d /sysroot/etc/sysctl.d + mkdir -p /sysroot/etc/systemd/system # Headless: homed-firstboot would block the console asking for a user. ln -sf /dev/null /sysroot/etc/systemd/system/systemd-homed-firstboot.service # The initrd's import.pull credential (local name "rootdisk") stays in @@ -80,8 +80,13 @@ config: # /etc is the only unit directory that outranks generator output. ln -sf /dev/null /sysroot/etc/systemd/system/systemd-import@var-lib-machines-rootdisk.raw.service ln -sf /dev/null /sysroot/etc/systemd/system/systemd-loop@var-lib-machines-rootdisk.raw.service - printf '[Journal]\nForwardToConsole=no\nMaxLevelConsole=warning\n' > /sysroot/etc/systemd/journald.conf.d/10-console.conf - printf 'kernel.printk = 3 4 1 3\n' > /sysroot/etc/sysctl.d/10-console.conf + # Vendor drop-ins go to /usr/lib, not the factory /etc: tmpfiles' C + # never overwrites, so a copy in /etc would stay at the version an + # installed node first booted. An /etc file of the same name overrides. + # bluefin-migrate-etc-vendor removes the copies older images left. + mkdir -p /sysroot/usr/lib/systemd/journald.conf.d /sysroot/usr/lib/sysctl.d + printf '[Journal]\nForwardToConsole=no\nMaxLevelConsole=warning\n' > /sysroot/usr/lib/systemd/journald.conf.d/10-console.conf + printf 'kernel.printk = 3 4 1 3\n' > /sysroot/usr/lib/sysctl.d/10-console.conf printf 'LANG=C.UTF-8\n' > /sysroot/etc/locale.conf ln -sf ../usr/share/zoneinfo/UTC /sysroot/etc/localtime rm -f /sysroot/etc/machine-id @@ -160,6 +165,12 @@ config: [Install] WantedBy=multi-user.target EOF + # Enabled in /usr, not by preset, so nodes that updated into it run it. + # Before systemd-tmpfiles-setup, so the same boot links the /usr copy in + # where it removed an unmodified one. + mkdir -p /sysroot/usr/lib/systemd/system/sysinit.target.wants + ln -sf ../bluefin-migrate-etc-vendor.service \ + /sysroot/usr/lib/systemd/system/sysinit.target.wants/bluefin-migrate-etc-vendor.service mkdir -p /sysroot/usr/lib/systemd/system/multi-user.target.wants ln -sf ../bluefin-repair-etc-modes.service \ /sysroot/usr/lib/systemd/system/multi-user.target.wants/bluefin-repair-etc-modes.service diff --git a/files/homelab/cluster/cmd/bluefin-host/main.go b/files/homelab/cluster/cmd/bluefin-host/main.go index 0aba8b64..dcf1c1cb 100644 --- a/files/homelab/cluster/cmd/bluefin-host/main.go +++ b/files/homelab/cluster/cmd/bluefin-host/main.go @@ -15,6 +15,7 @@ import ( "github.com/projectbluefin/server/files/homelab/cluster/internal/host/bootorigin" "github.com/projectbluefin/server/files/homelab/cluster/internal/host/diskless" "github.com/projectbluefin/server/files/homelab/cluster/internal/host/etcmodes" + "github.com/projectbluefin/server/files/homelab/cluster/internal/host/etcvendor" "github.com/projectbluefin/server/files/homelab/cluster/internal/host/firstboot" "github.com/projectbluefin/server/files/homelab/cluster/internal/host/fleetlock" "github.com/projectbluefin/server/files/homelab/cluster/internal/host/hostname" @@ -45,6 +46,7 @@ var helpers = map[string]command{ "diskless-update-check": diskless.Check, "firstboot-credentials": firstboot.Main, "hostname": hostname.Main, + "migrate-etc-vendor": etcvendor.Main, "reboot-lock": fleetlock.Main, "repair-etc-modes": etcmodes.Main, "sysext-fetch": sysextfetch.Main, diff --git a/files/homelab/cluster/internal/host/etcvendor/etcvendor.go b/files/homelab/cluster/internal/host/etcvendor/etcvendor.go new file mode 100644 index 00000000..f15511d7 --- /dev/null +++ b/files/homelab/cluster/internal/host/etcvendor/etcvendor.go @@ -0,0 +1,97 @@ +// Package etcvendor is bluefin-migrate-etc-vendor: one-shot removal of the +// vendor drop-ins older images copied into /etc, now that they ship in /usr. +// +// Images up to this one put the sshd, sysctl and journald drop-ins in the +// factory /etc, which systemd-tmpfiles copies (C) only when missing. A disk +// install keeps /etc across A/B updates, so those copies stayed at the +// version the node first booted. The drop-ins now live in /usr, linked (sshd) +// or read directly (sysctl, journald). This removes a copy only while its +// content is byte-identical to a version an image shipped, so a file an +// operator edited stays, and tmpfiles then links the current one in. It runs +// once; a stamp in /var suppresses later boots. +package etcvendor + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "io/fs" + "os" + "slices" + + "github.com/projectbluefin/server/files/homelab/cluster/internal/host/sys" +) + +const Stamp = "/var/lib/bluefin/etc-vendor-migrate.stamp" + +// Shipped maps each /etc path an older image copied in to the sha256 of +// every version of it a release carried (from v26.09.673, the first FSDK +// image, on), and of the sshd drop-in as #700 left it. Only an exact match +// is removed. +var Shipped = map[string][]string{ + "ssh/sshd_config.d/bluefin-server.conf": { + "d6db893a293261e9bc1a7ec183378f3199140039258e42ff7f231bcb0f2fc5b2", + "cd0d2c8586c7fb9825f398990ed46ac32454b54704333485e79e31d6fd03051f", + "b1c4085814e5b17a9ae70715a822421ebe57e485fd4b77dcb88b6e07558ef217", + "0e04923abd1f70d8012dd95292ec7f343f17a6e6e41277517072fbe323b19989", + }, + "sysctl.d/10-console.conf": { + "815e73d09439ab850feafddda67cb818025d12211ebf6ab28293cacaafa7552e", + }, + "systemd/journald.conf.d/10-console.conf": { + "f4ef13fbdd83470bb0a462f525c6fa00d75428ecbafb4346f45bbdbeaee17db3", + }, +} + +// Migrate removes etc/ for every path in shipped that is a regular +// file with one of its listed hashes, then creates stamp. It returns the +// paths it removed. +func Migrate(etc string, shipped map[string][]string, stamp string) ([]string, error) { + var removed []string + for rel, sums := range shipped { + p := etc + "/" + rel + fi, err := os.Lstat(p) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + return removed, err + } + // A link (the new layout) or anything else an operator put there stays. + if !fi.Mode().IsRegular() { + continue + } + data, err := os.ReadFile(p) + if err != nil { + return removed, err + } + sum := sha256.Sum256(data) + if !slices.Contains(sums, hex.EncodeToString(sum[:])) { + continue + } + if err := os.Remove(p); err != nil { + return removed, err + } + removed = append(removed, rel) + } + if err := sys.MkdirP(sys.Dir(stamp)); err != nil { + return removed, err + } + return removed, sys.Truncate(stamp) +} + +// Main is bluefin-migrate-etc-vendor. +// Test overrides: BLUEFIN_ETC, BLUEFIN_ETC_VENDOR_STAMP. +func Main(h *sys.Host, _ []string) int { + etc := h.Env("BLUEFIN_ETC", "/etc") + stamp := h.Env("BLUEFIN_ETC_VENDOR_STAMP", Stamp) + removed, err := Migrate(etc, Shipped, stamp) + for _, rel := range removed { + h.Printf("bluefin-migrate-etc-vendor: removed the unmodified %s/%s; /usr now ships it\n", etc, rel) + } + if err != nil { + h.Eprintf("bluefin-migrate-etc-vendor: %v\n", err) + return 1 + } + return 0 +} diff --git a/files/homelab/cluster/internal/host/etcvendor/etcvendor_test.go b/files/homelab/cluster/internal/host/etcvendor/etcvendor_test.go new file mode 100644 index 00000000..b76c2365 --- /dev/null +++ b/files/homelab/cluster/internal/host/etcvendor/etcvendor_test.go @@ -0,0 +1,78 @@ +package etcvendor + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "sort" + "testing" +) + +func sum(s string) string { + b := sha256.Sum256([]byte(s)) + return hex.EncodeToString(b[:]) +} + +func write(t *testing.T, p, data string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(data), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestMigrateRemovesOnlyUnmodifiedShippedCopies(t *testing.T) { + dir := t.TempDir() + etc := filepath.Join(dir, "etc") + shipped := map[string][]string{ + "a.conf": {sum("old\n"), sum("older\n")}, + "edited.conf": {sum("old\n")}, + "linked.conf": {sum("old\n")}, + "missing.conf": {sum("old\n")}, + "d/b.conf": {sum("b\n")}, + } + write(t, filepath.Join(etc, "a.conf"), "older\n") + write(t, filepath.Join(etc, "edited.conf"), "old\n# mine\n") + write(t, filepath.Join(etc, "d", "b.conf"), "b\n") + write(t, filepath.Join(dir, "usr.conf"), "old\n") + if err := os.Symlink(filepath.Join(dir, "usr.conf"), filepath.Join(etc, "linked.conf")); err != nil { + t.Fatal(err) + } + stamp := filepath.Join(dir, "var", "lib", "bluefin", "stamp") + + removed, err := Migrate(etc, shipped, stamp) + if err != nil { + t.Fatal(err) + } + sort.Strings(removed) + if len(removed) != 2 || removed[0] != "a.conf" || removed[1] != "d/b.conf" { + t.Fatalf("removed %q; want [a.conf d/b.conf]", removed) + } + for _, kept := range []string{"edited.conf", "linked.conf"} { + if _, err := os.Lstat(filepath.Join(etc, kept)); err != nil { + t.Errorf("%s: %v; want it kept", kept, err) + } + } + if _, err := os.Stat(filepath.Join(dir, "usr.conf")); err != nil { + t.Errorf("the link target was touched: %v", err) + } + if _, err := os.Stat(stamp); err != nil { + t.Errorf("stamp: %v", err) + } +} + +func TestShippedHashesAreFullSHA256(t *testing.T) { + for rel, sums := range Shipped { + if len(sums) == 0 { + t.Errorf("%s: no hashes", rel) + } + for _, s := range sums { + if b, err := hex.DecodeString(s); err != nil || len(b) != sha256.Size { + t.Errorf("%s: %q is not a sha256", rel, s) + } + } + } +} diff --git a/files/os/ssh/sshd_config.d/bluefin-server.conf b/files/os/sshd_config.d/bluefin-server.conf similarity index 100% rename from files/os/ssh/sshd_config.d/bluefin-server.conf rename to files/os/sshd_config.d/bluefin-server.conf diff --git a/files/os/systemd/system/bluefin-migrate-etc-vendor.service b/files/os/systemd/system/bluefin-migrate-etc-vendor.service new file mode 100644 index 00000000..d45fc1f0 --- /dev/null +++ b/files/os/systemd/system/bluefin-migrate-etc-vendor.service @@ -0,0 +1,23 @@ +[Unit] +Description=Remove Vendor Drop-ins Older Images Copied into /etc +Documentation=file:/usr/libexec/bluefin-migrate-etc-vendor +# Older images shipped the sshd, sysctl and journald drop-ins in the factory +# /etc, which tmpfiles copies only when missing, so an installed node kept +# the copy it first booted. They ship in /usr now; this removes each copy +# that is still exactly as an image shipped it, and tmpfiles-setup then +# links the /usr one in. Enabled in /usr (elements/oci/bluefin-server-usr.bst) +# so updated nodes run it; once per node. Installed nodes only: a diskless +# node's /etc starts empty every boot. +ConditionPathExists=!/var/lib/bluefin/etc-vendor-migrate.stamp +ConditionPathExists=!/run/machines/rootdisk.raw +ConditionKernelCommandLine=!root=tmpfs +DefaultDependencies=no +After=local-fs.target +Before=systemd-tmpfiles-setup.service sysinit.target +Conflicts=shutdown.target +Before=shutdown.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/usr/libexec/bluefin-migrate-etc-vendor diff --git a/files/os/tmpfiles.d/50-bluefin-sshd.conf b/files/os/tmpfiles.d/50-bluefin-sshd.conf new file mode 100644 index 00000000..cc5c4df9 --- /dev/null +++ b/files/os/tmpfiles.d/50-bluefin-sshd.conf @@ -0,0 +1,6 @@ +# sshd's vendor drop-in (/usr/lib/bluefin/sshd_config.d, os-sshd-config.bst). +# A link, not a copy in the factory /etc, so installed nodes, which keep /etc +# across A/B updates, get its changes too. L leaves an operator's own file +# alone; bluefin-migrate-etc-vendor removes the unmodified copies older +# images left, before this runs. +L /etc/ssh/sshd_config.d/bluefin-server.conf - - - - ../../../usr/lib/bluefin/sshd_config.d/bluefin-server.conf diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 9abd0b1f..ae45a2f0 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -70,6 +70,7 @@ def pytest_runtest_setup(item: pytest.Item) -> None: "diskless-update-check", "firstboot-credentials", "hostname", + "migrate-etc-vendor", "reboot-lock", "repair-etc-modes", "sysext-fetch", diff --git a/tests/unit/test_etc_vendor_dropins.py b/tests/unit/test_etc_vendor_dropins.py new file mode 100644 index 00000000..0b7f2e40 --- /dev/null +++ b/tests/unit/test_etc_vendor_dropins.py @@ -0,0 +1,102 @@ +"""Vendor drop-ins ship in /usr, so A/B updates reach installed nodes. + +A disk install keeps /etc across A/B updates, and the factory /etc reaches it +through tmpfiles ``C``, which never overwrites. A drop-in shipped there stays +at the version the node first booted. The sshd drop-in is linked in from +/usr instead, and the sysctl and journald ones are read from /usr/lib +directly. ``bluefin-migrate-etc-vendor`` removes the copies older images left, +but only while they are byte-identical to a version an image shipped. +""" + + +import hashlib +import os +import posixpath +import subprocess +from pathlib import Path + +import yaml + +from _systemd import SystemdFile + +ROOT = Path(__file__).resolve().parents[2] +ELEMENTS = ROOT / "elements" +USR_ELEMENT = (ELEMENTS / "oci" / "bluefin-server-usr.bst").read_text(encoding="utf-8") +SSHD_ELEMENT = yaml.safe_load((ELEMENTS / "bluefin-server" / "os-sshd-config.bst").read_text(encoding="utf-8")) +DROPIN = ROOT / "files" / "os" / "sshd_config.d" / "bluefin-server.conf" +LINK_RULE = ROOT / "files" / "os" / "tmpfiles.d" / "50-bluefin-sshd.conf" +UNIT = ROOT / "files" / "os" / "systemd" / "system" / "bluefin-migrate-etc-vendor.service" +GO_SOURCE = ROOT / "files" / "homelab" / "cluster" / "internal" / "host" / "etcvendor" / "etcvendor.go" + + +def test_no_vendor_drop_in_is_written_to_the_factory_etc() -> None: + assert "/sysroot/etc/sysctl.d" not in USR_ELEMENT + assert "/sysroot/etc/systemd/journald.conf.d" not in USR_ELEMENT + assert "> /sysroot/usr/lib/sysctl.d/10-console.conf" in USR_ELEMENT + assert "> /sysroot/usr/lib/systemd/journald.conf.d/10-console.conf" in USR_ELEMENT + install = "\n".join(SSHD_ELEMENT["config"]["install-commands"]) + assert '"%{install-root}/usr/lib/bluefin/sshd_config.d/"' in install + # Only the main sshd_config, which just includes sshd_config.d, stays in /etc. + assert '"%{install-root}/etc/ssh/sshd_config"' in install + assert "sshd_config.d" not in install.split("/etc/ssh/sshd_config", 1)[1].split("\n", 1)[0] + + +def test_the_sshd_drop_in_is_linked_from_usr() -> None: + rules = [line.split() for line in LINK_RULE.read_text(encoding="utf-8").splitlines() if line and not line.startswith("#")] + assert rules == [ + ["L", "/etc/ssh/sshd_config.d/bluefin-server.conf", "-", "-", "-", "-", + "../../../usr/lib/bluefin/sshd_config.d/bluefin-server.conf"], + ] + target = posixpath.normpath(posixpath.join("/etc/ssh/sshd_config.d", rules[0][6])) + assert target == "/usr/lib/bluefin/sshd_config.d/bluefin-server.conf" + + +def test_the_migration_runs_once_on_installed_nodes_before_tmpfiles() -> None: + unit = SystemdFile(UNIT) + assert unit.value("Service", "ExecStart") == "/usr/libexec/bluefin-migrate-etc-vendor" + assert unit.value("Service", "RemainAfterExit") == "yes" + conditions = unit.values("Unit", "ConditionPathExists") + assert "!/var/lib/bluefin/etc-vendor-migrate.stamp" in conditions + assert "!/run/machines/rootdisk.raw" in conditions + assert unit.value("Unit", "ConditionKernelCommandLine") == "!root=tmpfs" + assert "systemd-tmpfiles-setup.service" in unit.words("Unit", "Before") + assert "local-fs.target" in unit.words("Unit", "After") + assert "sysinit.target.wants/bluefin-migrate-etc-vendor.service" in USR_ELEMENT + + +def test_the_current_drop_ins_are_among_the_shipped_hashes() -> None: + # A node that first booted this image gets links, never copies, so the + # current content needs no hash; the sshd one is listed anyway, because + # the image before this one shipped it as a copy. + go = GO_SOURCE.read_text(encoding="utf-8") + assert hashlib.sha256(DROPIN.read_bytes()).hexdigest() in go + printk = hashlib.sha256(b"kernel.printk = 3 4 1 3\n").hexdigest() + journald = hashlib.sha256(b"[Journal]\nForwardToConsole=no\nMaxLevelConsole=warning\n").hexdigest() + assert printk in go and journald in go + + +def run(host_bin: Path, etc: Path, stamp: Path) -> subprocess.CompletedProcess[str]: + env = dict(os.environ, BLUEFIN_ETC=str(etc), BLUEFIN_ETC_VENDOR_STAMP=str(stamp)) + return subprocess.run([str(host_bin / "bluefin-migrate-etc-vendor")], env=env, capture_output=True, text=True, check=False) + + +def test_the_helper_removes_unmodified_copies_and_keeps_edits(tmp_path: Path, host_bin: Path) -> None: + etc = tmp_path / "etc" + sshd = etc / "ssh" / "sshd_config.d" / "bluefin-server.conf" + sysctl = etc / "sysctl.d" / "10-console.conf" + journald = etc / "systemd" / "journald.conf.d" / "10-console.conf" + for path in (sshd, sysctl, journald): + path.parent.mkdir(parents=True) + sshd.write_bytes(DROPIN.read_bytes()) + sysctl.write_text("kernel.printk = 3 4 1 3\n", encoding="utf-8") + journald.write_text("[Journal]\nForwardToConsole=no\nMaxLevelConsole=debug\n", encoding="utf-8") + stamp = tmp_path / "var" / "lib" / "bluefin" / "etc-vendor-migrate.stamp" + + result = run(host_bin, etc, stamp) + + assert result.returncode == 0, result.stderr + assert not sshd.exists() + assert not sysctl.exists() + assert journald.exists(), "an operator-edited copy must stay" + assert stamp.exists() + assert "ssh/sshd_config.d/bluefin-server.conf" in result.stdout diff --git a/tests/unit/test_root_credential_contract.py b/tests/unit/test_root_credential_contract.py index ebcf127b..92f2ec65 100644 --- a/tests/unit/test_root_credential_contract.py +++ b/tests/unit/test_root_credential_contract.py @@ -30,7 +30,7 @@ / "systemd-firstboot.service.d" / "10-bluefin-no-root-prompt.conf" ) -SSHD_DROPIN = REPO_ROOT / "files" / "os" / "ssh" / "sshd_config.d" / "bluefin-server.conf" +SSHD_DROPIN = REPO_ROOT / "files" / "os" / "sshd_config.d" / "bluefin-server.conf" GENERATED_SSHD_DROPIN = ( REPO_ROOT / "files"