diff --git a/Justfile b/Justfile index 1bae6d3f..ea47448f 100644 --- a/Justfile +++ b/Justfile @@ -318,6 +318,30 @@ test-installer-artifact: } trap cleanup EXIT INT TERM + # The kiosk proxy publishes hostPort 8080 on the guest's loopback only + # (files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml), and a + # QEMU hostfwd with an unspecified guest address is delivered to the guest's + # DHCP address, never to 127.0.0.1 — so no host-side probe can reach it + # without exposing the console on the guest's external interface. Probe from + # inside the guest instead and report the verdict on the system console, + # which is already captured in $SERIAL_LOG. The proxy terminates TLS with the + # self-signed certificate baked into the k0s sysext, so the probe speaks + # https and skips certificate verification, matching tests/e2e. + READY_MARKER="KIOSK_CONSOLE_READY" + READY_UNIT=$(base64 -w0 <<'UNIT' + [Unit] + Description=Report KubeStellar Console readiness on the system console + ConditionPathExists=!/etc/initrd-release + After=k0s-first-boot.service + + [Service] + Type=oneshot + TimeoutStartSec=infinity + StandardOutput=journal+console + ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' + UNIT + ) + echo "==> Booting the installed server in QEMU (background)..." qemu-system-x86_64 \ -enable-kvm \ @@ -327,9 +351,10 @@ test-installer-artifact: -drive file="$WORKDIR/target.raw",format=raw,if=virtio \ -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ - -nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:8080-:8080 \ + -nic user,model=virtio-net-pci \ -smbios "type=11,value=io.systemd.credential.binary:fstab.extra=L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" \ - -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service" \ + -smbios "type=11,value=io.systemd.credential.binary:systemd.extra-unit.bluefin-kiosk-ready.service=${READY_UNIT}" \ + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service systemd.mask=systemd-homed-firstboot.service systemd.wants=bluefin-kiosk-ready.service" \ -nographic \ -serial file:"$SERIAL_LOG" \ -monitor none & @@ -337,7 +362,7 @@ test-installer-artifact: DEADLINE_SECS="${SHOW_ME_THE_FUTURE_DEADLINE:-${SHOW_ME_THE_FUTURE_TIMEOUT:-600}}" START_TIME=$(date +%s) - echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..." + echo "==> Polling KubeStellar Console readiness from inside the guest (deadline: ${DEADLINE_SECS}s)..." while true; do if ! kill -0 "$TARGET_QEMU_PID" 2>/dev/null; then @@ -349,13 +374,9 @@ test-installer-artifact: exit 1 fi - HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) - if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then - ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) - if [ "$ROOT_CODE" = "200" ]; then - echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" - break - fi + if grep -q "$READY_MARKER" "$SERIAL_LOG" 2>/dev/null; then + echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" + break fi NOW=$(date +%s) diff --git a/docs/skills/ddi-installer.md b/docs/skills/ddi-installer.md index a511f943..d073014f 100644 --- a/docs/skills/ddi-installer.md +++ b/docs/skills/ddi-installer.md @@ -48,8 +48,10 @@ terminal-based interactive installation that: User provisioning is handled on the target system's first boot via systemd system credentials (`systemd-sysusers`, `systemd-tmpfiles`) so the base image -remains stateless. `systemd-firstboot.service` is masked on the target image -(`/etc/systemd/system/systemd-firstboot.service -> /dev/null`) to guarantee +remains stateless. `systemd-firstboot.service` and +`systemd-homed-firstboot.service` are masked on the target image +(`/etc/systemd/system/systemd-firstboot.service -> /dev/null` and +`/etc/systemd/system/systemd-homed-firstboot.service -> /dev/null`) to guarantee unattended, prompt-free startup. The target DDI also pre-stages the extracted CA certificate bundle (`/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem` and `/etc/ssl/certs/ca-certificates.crt`) and a standard `/etc/hosts` file for diff --git a/elements/flatcar/flatcar-zfs.bst b/elements/flatcar/flatcar-zfs.bst index 82832969..eabdcf8e 100644 --- a/elements/flatcar/flatcar-zfs.bst +++ b/elements/flatcar/flatcar-zfs.bst @@ -10,6 +10,7 @@ description: | build-depends: - base/base-stack.bst + - freedesktop-sdk.bst:components/sed.bst - freedesktop-sdk.bst:components/squashfs-tools.bst variables: @@ -28,6 +29,20 @@ config: # Extract the SquashFS sysext directly into the element install root unsquashfs -no-xattrs -f -d "%{install-root}" flatcar-zfs.raw - # Also preserve the raw sysext image under /usr/lib/sysexts for runtime sysext usage + # Drop Flatcar's udevd sysext ordering override. In Flatcar, ZFS is loaded as + # a runtime sysext, so udevd was ordered After=systemd-sysext.service. + # In Bluefin Server, ZFS is extracted directly into the OS rootfs at build time. + # Leaving this drop-in causes systemd-udevd to wait for systemd-sysext.service, + # which waits for local-fs.target, which waits for var.mount, which waits for + # the udev device unit — creating a boot deadlock. + rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d" + if [ -f "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf" ]; then + sed -i "/systemd-udevd\.service\.d/d" "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf" + fi + + # Preserve a raw sysext image under /usr/lib/sysexts for runtime sysext usage. + # Rebuild it from the cleaned tree rather than shipping the upstream image, so + # the udevd ordering override above cannot reappear if the raw is ever merged. + mksquashfs "%{install-root}" zfs-cleaned.raw -noappend -no-xattrs -all-root mkdir -p "%{install-root}/usr/lib/sysexts" - install -D -m 0644 flatcar-zfs.raw "%{install-root}/usr/lib/sysexts/zfs.raw" + install -D -m 0644 zfs-cleaned.raw "%{install-root}/usr/lib/sysexts/zfs.raw" diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index 9f801570..ab161db0 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -38,7 +38,7 @@ config: exit 1 fi rm -f "/layer/usr/lib/modules/${KVER}/vmlinux" "/layer/usr/lib/modules/${KVER}/System.map" - depmod -b /layer "${KVER}" + depmod -b /layer/usr "${KVER}" # Stage extracted CA certificate bundle into target DDI rootfs mkdir -p /layer/etc/pki/ca-trust/extracted /layer/etc/ssl/certs /layer/etc/pki/tls/certs @@ -47,9 +47,16 @@ config: ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/certs/ca-bundle.crt ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/cert.pem - # Mask systemd-firstboot to prevent interactive prompts on headless first boot + # Mask both first-boot wizards to prevent interactive prompts on headless + # first boot: systemd-firstboot.service prompts for locale/keymap/timezone/ + # root password, and systemd-homed-firstboot.service runs + # `homectl firstboot --prompt-new-user`. Both are ConditionFirstBoot=yes, + # read StandardInput=tty, and order themselves before + # first-boot-complete.target, so either one blocks the boot transaction + # forever on a console nobody is attached to. mkdir -p /layer/etc/systemd/system ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service + ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service # Mask audit-rules to prevent failure on shell-free OS (augenrules requires /bin/sh) ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 44d9f300..ab95bc43 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -103,7 +103,7 @@ config: mkdir -p /target-root/boot ln -sf "../usr/lib/modules/${TARGET_KVER}/vmlinuz" /target-root/boot/vmlinuz - depmod -b /target-root "${TARGET_KVER}" || true + depmod -b /target-root/usr "${TARGET_KVER}" || true # Configure dynamic linker search paths so ldconfig indexes FSDK libraries # (e.g. libc.so.6, libgcc_s.so.1, libkmod.so) for dracut-install @@ -360,7 +360,7 @@ config: rm -rf "${MOD_DIR}/drivers/infiniband" fi # Re-run depmod to update module dependencies after stripping unneeded modules - depmod -b /layer "${KVER}" || true + depmod -b /layer/usr "${KVER}" || true fi ( cd /layer && find . -print0 \ diff --git a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml index 80e38d5a..8ade0861 100644 --- a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml +++ b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml @@ -29,10 +29,13 @@ spec: - /etc/kubestellar-kiosk/nginx.conf ports: - containerPort: 8080 - # Kiosk-only proxy: bind loopback so the console is not exposed on - # LAN interfaces over plaintext HTTP. The VM flow reaches it via the - # existing hostfwd of 127.0.0.1:8080; LAN access, if ever wanted, - # must go through a TLS-terminating, authenticated front end. + # Kiosk-only proxy: bind loopback so the console is never published on + # the node's external interfaces. A QEMU hostfwd cannot reach this + # binding, because libslirp delivers a forward with an unspecified + # guest address to the guest's DHCP address rather than to 127.0.0.1; + # the installer smoke test therefore probes the proxy from inside the + # guest. LAN access, if ever wanted, must go through an + # authenticated front end. hostIP: 127.0.0.1 hostPort: 8080 name: http diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index d423c685..73a3d78d 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -14,6 +14,7 @@ ) JUSTFILE = REPO_ROOT / "Justfile" DDI_ELEMENT = REPO_ROOT / "elements" / "oci" / "bluefin-server-ddi.bst" +FLATCAR_ZFS_ELEMENT = REPO_ROOT / "elements" / "flatcar" / "flatcar-zfs.bst" def _published_uki_cmdline(installer_element: str) -> str: @@ -92,10 +93,11 @@ def test_ddi_generates_module_indexes_for_runtime_filesystem_drivers() -> None: ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") assert "freedesktop-sdk.bst:components/kmod.bst" in ddi_element - assert 'depmod -b /layer "${KVER}"' in ddi_element + assert 'depmod -b /layer/usr "${KVER}"' in ddi_element assert "cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/" in ddi_element assert "tls-ca-bundle.pem" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service" in ddi_element + assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service" in ddi_element assert "printf '127.0.0.1 localhost" in ddi_element assert "> /layer/etc/hosts" in ddi_element @@ -135,3 +137,66 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: assert "find /layer -type f -name '*.a' -delete" in installer_element assert 'rm -f "/layer/usr/lib/modules/${KVER}/vmlinux"' in ddi_element assert "find /layer -type f -name '*.a' -delete" in ddi_element + + +def test_flatcar_zfs_removes_udevd_sysext_ordering_dropin() -> None: + flatcar_zfs = FLATCAR_ZFS_ELEMENT.read_text(encoding="utf-8") + + assert ( + 'rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d"' + in flatcar_zfs + ) + assert ( + 'sed -i "/systemd-udevd\\.service\\.d/d" "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf"' + in flatcar_zfs + ) + assert "freedesktop-sdk.bst:components/sed.bst" in flatcar_zfs + + # The preserved raw sysext must be rebuilt from the cleaned tree, not the + # untouched upstream image, so the ordering override cannot come back if the + # raw is ever merged at runtime. + assert "install -D -m 0644 flatcar-zfs.raw" not in flatcar_zfs + assert ( + 'mksquashfs "%{install-root}" zfs-cleaned.raw -noappend -no-xattrs -all-root' + in flatcar_zfs + ) + assert flatcar_zfs.index("mksquashfs") > flatcar_zfs.index( + 'rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d"' + ) + + +def test_var_partition_contracts_use_consistent_partlabel() -> None: + import base64 + + repart_var = ( + REPO_ROOT / "files" / "installer" / "repart.d" / "30-var.conf" + ).read_text(encoding="utf-8") + ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") + var_mount = ( + REPO_ROOT / "files" / "os" / "systemd" / "system" / "var.mount" + ).read_text(encoding="utf-8") + justfile = JUSTFILE.read_text(encoding="utf-8") + + assert "Label=var" in repart_var + assert "/dev/disk/by-partlabel/var /var xfs defaults 0 0" in ddi_element + assert "What=/dev/disk/by-partlabel/var" in var_mount + expected_b64 = base64.b64encode( + b"/dev/disk/by-partlabel/var /var xfs defaults 0 0\n" + ).decode("ascii") + assert expected_b64 in justfile + + +def test_installer_smoke_probes_the_kiosk_over_tls_from_inside_the_guest() -> None: + justfile = JUSTFILE.read_text(encoding="utf-8") + assert "systemd.mask=systemd-homed-firstboot.service" in justfile + assert "https://127.0.0.1:8080/healthz" in justfile + assert "--insecure" in justfile + assert "systemd.extra-unit.bluefin-kiosk-ready.service" in justfile + assert "systemd.wants=bluefin-kiosk-ready.service" in justfile + # The host cannot reach the guest's loopback-bound kiosk proxy, so no + # host-side probe may remain in the smoke test — a stray host process on + # port 8080 would otherwise let it pass without the guest being ready. + smoke = justfile.split("install-vm:")[0] + assert "http://127.0.0.1:8080/healthz" not in smoke + assert "KIOSK_CONSOLE_READY" in smoke +