From 263c713cfb5a1002b85a3cb3a80974b50c2237ac Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Sun, 20 Sep 2026 18:48:07 +0000 Subject: [PATCH 1/2] fix(boot): resolve QEMU installer boot deadlock and console readiness probe In `installer-test / Install and boot in QEMU`, target boot times out waiting for `/dev/disk/by-partlabel/var` and eventually fails KubeStellar Console readiness. Root causes resolved: 1. `elements/flatcar/flatcar-zfs.bst`: The extracted Flatcar ZFS system extension ships `usr/lib/systemd/system/systemd-udevd.service.d/10-zfs.conf` (`After=systemd-sysext.service`) and a tmpfiles copy rule in `usr/lib/tmpfiles.d/10-zfs.conf`. Because ZFS is extracted directly into the OS rootfs, this ordering caused systemd-udevd to wait for systemd-sysext.service, which waited for local-fs.target, which waited for var.mount, creating a circular boot deadlock where udevd could never start and device nodes were never created. Remove the drop-in, strip the tmpfiles rule, and add `sed.bst` build dependency. 2. `elements/oci/bluefin-server-ddi.bst` & `elements/oci/bluefin-server-installer.bst`: Flatcar kernel module directories reside under `/usr/lib/modules`; point `depmod -b` to `/layer/usr` and `/target-root/usr` instead of root. 3. `elements/oci/bluefin-server-ddi.bst` & `Justfile`: Mask `systemd-homed-firstboot.service` alongside `systemd-firstboot.service` on the target image and kernel cmdline to prevent headless hangs on interactive user creation prompt. 4. `Justfile` & `41-kubestellar-kiosk-proxy.yaml`: Move console readiness probe inside the guest using an injected test unit over TLS to correctly reach loopback-bound kiosk proxy and announce readiness on serial console. 5. `tests/unit/test_installer_contract.py`: Add unit contract assertions for the fixes. Fixes #218 Assisted-by: Gemini 3.8 Flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mrbobbytables --- Justfile | 36 ++++++++++++-- docs/skills/ddi-installer.md | 6 ++- elements/flatcar/flatcar-zfs.bst | 12 +++++ elements/oci/bluefin-server-ddi.bst | 11 ++++- elements/oci/bluefin-server-installer.bst | 4 +- .../41-kubestellar-kiosk-proxy.yaml | 11 +++-- tests/unit/test_installer_contract.py | 49 ++++++++++++++++++- 7 files changed, 115 insertions(+), 14 deletions(-) diff --git a/Justfile b/Justfile index 1bae6d3f..d029e9f9 100644 --- a/Justfile +++ b/Justfile @@ -318,6 +318,30 @@ test-installer-artifact: } trap cleanup EXIT INT TERM + # The kiosk proxy publishes hostPort 8080 on the guest's loopback only + # (files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml), and a + # QEMU hostfwd with an unspecified guest address is delivered to the guest's + # DHCP address, never to 127.0.0.1 — so no host-side probe can reach it + # without exposing the console on the guest's external interface. Probe from + # inside the guest instead and report the verdict on the system console, + # which is already captured in $SERIAL_LOG. The proxy terminates TLS with the + # self-signed certificate baked into the k0s sysext, so the probe speaks + # https and skips certificate verification, matching tests/e2e. + READY_MARKER="KIOSK_CONSOLE_READY" + READY_UNIT=$(base64 -w0 <<'UNIT' + [Unit] + Description=Report KubeStellar Console readiness on the system console + ConditionPathExists=!/etc/initrd-release + After=k0s-first-boot.service + + [Service] + Type=oneshot + TimeoutStartSec=infinity + StandardOutput=journal+console + ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' + UNIT + ) + echo "==> Booting the installed server in QEMU (background)..." qemu-system-x86_64 \ -enable-kvm \ @@ -327,9 +351,10 @@ test-installer-artifact: -drive file="$WORKDIR/target.raw",format=raw,if=virtio \ -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ - -nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:8080-:8080 \ + -nic user,model=virtio-net-pci \ -smbios "type=11,value=io.systemd.credential.binary:fstab.extra=L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" \ - -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service" \ + -smbios "type=11,value=io.systemd.credential.binary:systemd.extra-unit.bluefin-kiosk-ready.service=${READY_UNIT}" \ + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service systemd.mask=systemd-homed-firstboot.service systemd.wants=bluefin-kiosk-ready.service" \ -nographic \ -serial file:"$SERIAL_LOG" \ -monitor none & @@ -337,7 +362,7 @@ test-installer-artifact: DEADLINE_SECS="${SHOW_ME_THE_FUTURE_DEADLINE:-${SHOW_ME_THE_FUTURE_TIMEOUT:-600}}" START_TIME=$(date +%s) - echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..." + echo "==> Polling KubeStellar Console readiness from inside the guest (deadline: ${DEADLINE_SECS}s)..." while true; do if ! kill -0 "$TARGET_QEMU_PID" 2>/dev/null; then @@ -349,6 +374,11 @@ test-installer-artifact: exit 1 fi + if grep -q "$READY_MARKER" "$SERIAL_LOG" 2>/dev/null; then + echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" + break + fi + HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) diff --git a/docs/skills/ddi-installer.md b/docs/skills/ddi-installer.md index a511f943..d073014f 100644 --- a/docs/skills/ddi-installer.md +++ b/docs/skills/ddi-installer.md @@ -48,8 +48,10 @@ terminal-based interactive installation that: User provisioning is handled on the target system's first boot via systemd system credentials (`systemd-sysusers`, `systemd-tmpfiles`) so the base image -remains stateless. `systemd-firstboot.service` is masked on the target image -(`/etc/systemd/system/systemd-firstboot.service -> /dev/null`) to guarantee +remains stateless. `systemd-firstboot.service` and +`systemd-homed-firstboot.service` are masked on the target image +(`/etc/systemd/system/systemd-firstboot.service -> /dev/null` and +`/etc/systemd/system/systemd-homed-firstboot.service -> /dev/null`) to guarantee unattended, prompt-free startup. The target DDI also pre-stages the extracted CA certificate bundle (`/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem` and `/etc/ssl/certs/ca-certificates.crt`) and a standard `/etc/hosts` file for diff --git a/elements/flatcar/flatcar-zfs.bst b/elements/flatcar/flatcar-zfs.bst index 82832969..98fc3c3c 100644 --- a/elements/flatcar/flatcar-zfs.bst +++ b/elements/flatcar/flatcar-zfs.bst @@ -10,6 +10,7 @@ description: | build-depends: - base/base-stack.bst + - freedesktop-sdk.bst:components/sed.bst - freedesktop-sdk.bst:components/squashfs-tools.bst variables: @@ -31,3 +32,14 @@ config: # Also preserve the raw sysext image under /usr/lib/sysexts for runtime sysext usage mkdir -p "%{install-root}/usr/lib/sysexts" install -D -m 0644 flatcar-zfs.raw "%{install-root}/usr/lib/sysexts/zfs.raw" + + # Drop Flatcar's udevd sysext ordering override. In Flatcar, ZFS is loaded as + # a runtime sysext, so udevd was ordered After=systemd-sysext.service. + # In Bluefin Server, ZFS is extracted directly into the OS rootfs at build time. + # Leaving this drop-in causes systemd-udevd to wait for systemd-sysext.service, + # which waits for local-fs.target, which waits for var.mount, which waits for + # the udev device unit — creating a boot deadlock. + rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d" + if [ -f "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf" ]; then + sed -i "/systemd-udevd\.service\.d/d" "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf" + fi diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index 9f801570..ab161db0 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -38,7 +38,7 @@ config: exit 1 fi rm -f "/layer/usr/lib/modules/${KVER}/vmlinux" "/layer/usr/lib/modules/${KVER}/System.map" - depmod -b /layer "${KVER}" + depmod -b /layer/usr "${KVER}" # Stage extracted CA certificate bundle into target DDI rootfs mkdir -p /layer/etc/pki/ca-trust/extracted /layer/etc/ssl/certs /layer/etc/pki/tls/certs @@ -47,9 +47,16 @@ config: ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/certs/ca-bundle.crt ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/cert.pem - # Mask systemd-firstboot to prevent interactive prompts on headless first boot + # Mask both first-boot wizards to prevent interactive prompts on headless + # first boot: systemd-firstboot.service prompts for locale/keymap/timezone/ + # root password, and systemd-homed-firstboot.service runs + # `homectl firstboot --prompt-new-user`. Both are ConditionFirstBoot=yes, + # read StandardInput=tty, and order themselves before + # first-boot-complete.target, so either one blocks the boot transaction + # forever on a console nobody is attached to. mkdir -p /layer/etc/systemd/system ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service + ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service # Mask audit-rules to prevent failure on shell-free OS (augenrules requires /bin/sh) ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 44d9f300..ab95bc43 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -103,7 +103,7 @@ config: mkdir -p /target-root/boot ln -sf "../usr/lib/modules/${TARGET_KVER}/vmlinuz" /target-root/boot/vmlinuz - depmod -b /target-root "${TARGET_KVER}" || true + depmod -b /target-root/usr "${TARGET_KVER}" || true # Configure dynamic linker search paths so ldconfig indexes FSDK libraries # (e.g. libc.so.6, libgcc_s.so.1, libkmod.so) for dracut-install @@ -360,7 +360,7 @@ config: rm -rf "${MOD_DIR}/drivers/infiniband" fi # Re-run depmod to update module dependencies after stripping unneeded modules - depmod -b /layer "${KVER}" || true + depmod -b /layer/usr "${KVER}" || true fi ( cd /layer && find . -print0 \ diff --git a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml index 80e38d5a..8ade0861 100644 --- a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml +++ b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml @@ -29,10 +29,13 @@ spec: - /etc/kubestellar-kiosk/nginx.conf ports: - containerPort: 8080 - # Kiosk-only proxy: bind loopback so the console is not exposed on - # LAN interfaces over plaintext HTTP. The VM flow reaches it via the - # existing hostfwd of 127.0.0.1:8080; LAN access, if ever wanted, - # must go through a TLS-terminating, authenticated front end. + # Kiosk-only proxy: bind loopback so the console is never published on + # the node's external interfaces. A QEMU hostfwd cannot reach this + # binding, because libslirp delivers a forward with an unspecified + # guest address to the guest's DHCP address rather than to 127.0.0.1; + # the installer smoke test therefore probes the proxy from inside the + # guest. LAN access, if ever wanted, must go through an + # authenticated front end. hostIP: 127.0.0.1 hostPort: 8080 name: http diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index d423c685..73a261c8 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -14,6 +14,7 @@ ) JUSTFILE = REPO_ROOT / "Justfile" DDI_ELEMENT = REPO_ROOT / "elements" / "oci" / "bluefin-server-ddi.bst" +FLATCAR_ZFS_ELEMENT = REPO_ROOT / "elements" / "flatcar" / "flatcar-zfs.bst" def _published_uki_cmdline(installer_element: str) -> str: @@ -92,10 +93,11 @@ def test_ddi_generates_module_indexes_for_runtime_filesystem_drivers() -> None: ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") assert "freedesktop-sdk.bst:components/kmod.bst" in ddi_element - assert 'depmod -b /layer "${KVER}"' in ddi_element + assert 'depmod -b /layer/usr "${KVER}"' in ddi_element assert "cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/" in ddi_element assert "tls-ca-bundle.pem" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service" in ddi_element + assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service" in ddi_element assert "printf '127.0.0.1 localhost" in ddi_element assert "> /layer/etc/hosts" in ddi_element @@ -135,3 +137,48 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: assert "find /layer -type f -name '*.a' -delete" in installer_element assert 'rm -f "/layer/usr/lib/modules/${KVER}/vmlinux"' in ddi_element assert "find /layer -type f -name '*.a' -delete" in ddi_element + + +def test_flatcar_zfs_removes_udevd_sysext_ordering_dropin() -> None: + flatcar_zfs = FLATCAR_ZFS_ELEMENT.read_text(encoding="utf-8") + + assert ( + 'rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d"' + in flatcar_zfs + ) + assert ( + 'sed -i "/systemd-udevd\\.service\\.d/d" "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf"' + in flatcar_zfs + ) + assert "freedesktop-sdk.bst:components/sed.bst" in flatcar_zfs + + +def test_var_partition_contracts_use_consistent_partlabel() -> None: + import base64 + + repart_var = ( + REPO_ROOT / "files" / "installer" / "repart.d" / "30-var.conf" + ).read_text(encoding="utf-8") + ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") + var_mount = ( + REPO_ROOT / "files" / "os" / "systemd" / "system" / "var.mount" + ).read_text(encoding="utf-8") + justfile = JUSTFILE.read_text(encoding="utf-8") + + assert "Label=var" in repart_var + assert "/dev/disk/by-partlabel/var /var xfs defaults 0 0" in ddi_element + assert "What=/dev/disk/by-partlabel/var" in var_mount + expected_b64 = base64.b64encode( + b"/dev/disk/by-partlabel/var /var xfs defaults 0 0\n" + ).decode("ascii") + assert expected_b64 in justfile + + +def test_installer_smoke_probes_the_kiosk_over_tls_from_inside_the_guest() -> None: + justfile = JUSTFILE.read_text(encoding="utf-8") + assert "systemd.mask=systemd-homed-firstboot.service" in justfile + assert "https://127.0.0.1:8080/healthz" in justfile + assert "--insecure" in justfile + assert "systemd.extra-unit.bluefin-kiosk-ready.service" in justfile + assert "systemd.wants=bluefin-kiosk-ready.service" in justfile + From 8e279aa1c38b5c7dfcdd6ba8005d12a57629fbf3 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Sun, 20 Sep 2026 16:07:21 -0400 Subject: [PATCH 2/2] fix(boot): drop dead host-side kiosk probe and rebuild cleaned zfs sysext Address review findings on #220: 1. `Justfile`: remove the host-side poll of `http://127.0.0.1:8080/healthz` from the `installer-test` smoke recipe. The PR removed the QEMU hostfwd and the kiosk proxy terminates TLS (`files/k0s/kiosk/nginx.conf`, `listen 8080 ssl`), so this probe can never reach the guest. It could only ever hit an unrelated host process on port 8080 and false-pass the smoke test. The in-guest serial `KIOSK_CONSOLE_READY` marker is now the sole readiness check. 2. `elements/flatcar/flatcar-zfs.bst`: the preserved `/usr/lib/sysexts/zfs.raw` was the untouched upstream image and still carried the `After=systemd-sysext.service` udevd drop-in this PR removes from the rootfs. Inert today, but the deadlock ordering would return if that raw were ever merged for its stated runtime sysext usage. Rebuild the raw with mksquashfs from the cleaned install root after the drop-in and tmpfiles cleanup. 3. `tests/unit/test_installer_contract.py`: assert the host-side probe is gone from the smoke recipe and that the preserved raw is rebuilt post-cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 9 --------- elements/flatcar/flatcar-zfs.bst | 11 +++++++---- tests/unit/test_installer_contract.py | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/Justfile b/Justfile index d029e9f9..ea47448f 100644 --- a/Justfile +++ b/Justfile @@ -379,15 +379,6 @@ test-installer-artifact: break fi - HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) - if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then - ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) - if [ "$ROOT_CODE" = "200" ]; then - echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" - break - fi - fi - NOW=$(date +%s) ELAPSED=$((NOW - START_TIME)) if [ "$ELAPSED" -ge "$DEADLINE_SECS" ]; then diff --git a/elements/flatcar/flatcar-zfs.bst b/elements/flatcar/flatcar-zfs.bst index 98fc3c3c..eabdcf8e 100644 --- a/elements/flatcar/flatcar-zfs.bst +++ b/elements/flatcar/flatcar-zfs.bst @@ -29,10 +29,6 @@ config: # Extract the SquashFS sysext directly into the element install root unsquashfs -no-xattrs -f -d "%{install-root}" flatcar-zfs.raw - # Also preserve the raw sysext image under /usr/lib/sysexts for runtime sysext usage - mkdir -p "%{install-root}/usr/lib/sysexts" - install -D -m 0644 flatcar-zfs.raw "%{install-root}/usr/lib/sysexts/zfs.raw" - # Drop Flatcar's udevd sysext ordering override. In Flatcar, ZFS is loaded as # a runtime sysext, so udevd was ordered After=systemd-sysext.service. # In Bluefin Server, ZFS is extracted directly into the OS rootfs at build time. @@ -43,3 +39,10 @@ config: if [ -f "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf" ]; then sed -i "/systemd-udevd\.service\.d/d" "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf" fi + + # Preserve a raw sysext image under /usr/lib/sysexts for runtime sysext usage. + # Rebuild it from the cleaned tree rather than shipping the upstream image, so + # the udevd ordering override above cannot reappear if the raw is ever merged. + mksquashfs "%{install-root}" zfs-cleaned.raw -noappend -no-xattrs -all-root + mkdir -p "%{install-root}/usr/lib/sysexts" + install -D -m 0644 zfs-cleaned.raw "%{install-root}/usr/lib/sysexts/zfs.raw" diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 73a261c8..73a3d78d 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -152,6 +152,18 @@ def test_flatcar_zfs_removes_udevd_sysext_ordering_dropin() -> None: ) assert "freedesktop-sdk.bst:components/sed.bst" in flatcar_zfs + # The preserved raw sysext must be rebuilt from the cleaned tree, not the + # untouched upstream image, so the ordering override cannot come back if the + # raw is ever merged at runtime. + assert "install -D -m 0644 flatcar-zfs.raw" not in flatcar_zfs + assert ( + 'mksquashfs "%{install-root}" zfs-cleaned.raw -noappend -no-xattrs -all-root' + in flatcar_zfs + ) + assert flatcar_zfs.index("mksquashfs") > flatcar_zfs.index( + 'rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d"' + ) + def test_var_partition_contracts_use_consistent_partlabel() -> None: import base64 @@ -181,4 +193,10 @@ def test_installer_smoke_probes_the_kiosk_over_tls_from_inside_the_guest() -> No assert "--insecure" in justfile assert "systemd.extra-unit.bluefin-kiosk-ready.service" in justfile assert "systemd.wants=bluefin-kiosk-ready.service" in justfile + # The host cannot reach the guest's loopback-bound kiosk proxy, so no + # host-side probe may remain in the smoke test — a stray host process on + # port 8080 would otherwise let it pass without the guest being ready. + smoke = justfile.split("install-vm:")[0] + assert "http://127.0.0.1:8080/healthz" not in smoke + assert "KIOSK_CONSOLE_READY" in smoke