Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 31 additions & 10 deletions Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,30 @@ test-installer-artifact:
}
trap cleanup EXIT INT TERM

# The kiosk proxy publishes hostPort 8080 on the guest's loopback only
# (files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml), and a
# QEMU hostfwd with an unspecified guest address is delivered to the guest's
# DHCP address, never to 127.0.0.1 — so no host-side probe can reach it
# without exposing the console on the guest's external interface. Probe from
# inside the guest instead and report the verdict on the system console,
# which is already captured in $SERIAL_LOG. The proxy terminates TLS with the
# self-signed certificate baked into the k0s sysext, so the probe speaks
# https and skips certificate verification, matching tests/e2e.
READY_MARKER="KIOSK_CONSOLE_READY"
READY_UNIT=$(base64 -w0 <<'UNIT'
[Unit]
Description=Report KubeStellar Console readiness on the system console
ConditionPathExists=!/etc/initrd-release
After=k0s-first-boot.service

[Service]
Type=oneshot
TimeoutStartSec=infinity
StandardOutput=journal+console
ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"'
UNIT
)

echo "==> Booting the installed server in QEMU (background)..."
qemu-system-x86_64 \
-enable-kvm \
Expand All @@ -327,17 +351,18 @@ test-installer-artifact:
-drive file="$WORKDIR/target.raw",format=raw,if=virtio \
-drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \
-drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \
-nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:8080-:8080 \
-nic user,model=virtio-net-pci \
-smbios "type=11,value=io.systemd.credential.binary:fstab.extra=L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" \
-smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service" \
-smbios "type=11,value=io.systemd.credential.binary:systemd.extra-unit.bluefin-kiosk-ready.service=${READY_UNIT}" \
-smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service systemd.mask=systemd-homed-firstboot.service systemd.wants=bluefin-kiosk-ready.service" \
-nographic \
-serial file:"$SERIAL_LOG" \
-monitor none &
TARGET_QEMU_PID=$!

DEADLINE_SECS="${SHOW_ME_THE_FUTURE_DEADLINE:-${SHOW_ME_THE_FUTURE_TIMEOUT:-600}}"
START_TIME=$(date +%s)
echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..."
echo "==> Polling KubeStellar Console readiness from inside the guest (deadline: ${DEADLINE_SECS}s)..."

while true; do
if ! kill -0 "$TARGET_QEMU_PID" 2>/dev/null; then
Expand All @@ -349,13 +374,9 @@ test-installer-artifact:
exit 1
fi

HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true)
if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then
ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true)
if [ "$ROOT_CODE" = "200" ]; then
echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200"
break
fi
if grep -q "$READY_MARKER" "$SERIAL_LOG" 2>/dev/null; then
echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200"
break
fi

NOW=$(date +%s)
Expand Down
6 changes: 4 additions & 2 deletions docs/skills/ddi-installer.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,10 @@ terminal-based interactive installation that:

User provisioning is handled on the target system's first boot via systemd
system credentials (`systemd-sysusers`, `systemd-tmpfiles`) so the base image
remains stateless. `systemd-firstboot.service` is masked on the target image
(`/etc/systemd/system/systemd-firstboot.service -> /dev/null`) to guarantee
remains stateless. `systemd-firstboot.service` and
`systemd-homed-firstboot.service` are masked on the target image
(`/etc/systemd/system/systemd-firstboot.service -> /dev/null` and
`/etc/systemd/system/systemd-homed-firstboot.service -> /dev/null`) to guarantee
unattended, prompt-free startup. The target DDI also pre-stages the extracted CA
certificate bundle (`/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem` and
`/etc/ssl/certs/ca-certificates.crt`) and a standard `/etc/hosts` file for
Expand Down
19 changes: 17 additions & 2 deletions elements/flatcar/flatcar-zfs.bst
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ description: |

build-depends:
- base/base-stack.bst
- freedesktop-sdk.bst:components/sed.bst
- freedesktop-sdk.bst:components/squashfs-tools.bst

variables:
Expand All @@ -28,6 +29,20 @@ config:
# Extract the SquashFS sysext directly into the element install root
unsquashfs -no-xattrs -f -d "%{install-root}" flatcar-zfs.raw

# Also preserve the raw sysext image under /usr/lib/sysexts for runtime sysext usage
# Drop Flatcar's udevd sysext ordering override. In Flatcar, ZFS is loaded as
# a runtime sysext, so udevd was ordered After=systemd-sysext.service.
# In Bluefin Server, ZFS is extracted directly into the OS rootfs at build time.
# Leaving this drop-in causes systemd-udevd to wait for systemd-sysext.service,
# which waits for local-fs.target, which waits for var.mount, which waits for
# the udev device unit — creating a boot deadlock.
rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d"
if [ -f "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf" ]; then
sed -i "/systemd-udevd\.service\.d/d" "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf"
fi

# Preserve a raw sysext image under /usr/lib/sysexts for runtime sysext usage.
# Rebuild it from the cleaned tree rather than shipping the upstream image, so
# the udevd ordering override above cannot reappear if the raw is ever merged.
mksquashfs "%{install-root}" zfs-cleaned.raw -noappend -no-xattrs -all-root
mkdir -p "%{install-root}/usr/lib/sysexts"
install -D -m 0644 flatcar-zfs.raw "%{install-root}/usr/lib/sysexts/zfs.raw"
install -D -m 0644 zfs-cleaned.raw "%{install-root}/usr/lib/sysexts/zfs.raw"
11 changes: 9 additions & 2 deletions elements/oci/bluefin-server-ddi.bst
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ config:
exit 1
fi
rm -f "/layer/usr/lib/modules/${KVER}/vmlinux" "/layer/usr/lib/modules/${KVER}/System.map"
depmod -b /layer "${KVER}"
depmod -b /layer/usr "${KVER}"

# Stage extracted CA certificate bundle into target DDI rootfs
mkdir -p /layer/etc/pki/ca-trust/extracted /layer/etc/ssl/certs /layer/etc/pki/tls/certs
Expand All @@ -47,9 +47,16 @@ config:
ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/certs/ca-bundle.crt
ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/pki/tls/cert.pem

# Mask systemd-firstboot to prevent interactive prompts on headless first boot
# Mask both first-boot wizards to prevent interactive prompts on headless
# first boot: systemd-firstboot.service prompts for locale/keymap/timezone/
# root password, and systemd-homed-firstboot.service runs
# `homectl firstboot --prompt-new-user`. Both are ConditionFirstBoot=yes,
# read StandardInput=tty, and order themselves before
# first-boot-complete.target, so either one blocks the boot transaction
# forever on a console nobody is attached to.
mkdir -p /layer/etc/systemd/system
ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service
ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service

# Mask audit-rules to prevent failure on shell-free OS (augenrules requires /bin/sh)
ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service
Expand Down
4 changes: 2 additions & 2 deletions elements/oci/bluefin-server-installer.bst
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ config:
mkdir -p /target-root/boot
ln -sf "../usr/lib/modules/${TARGET_KVER}/vmlinuz" /target-root/boot/vmlinuz

depmod -b /target-root "${TARGET_KVER}" || true
depmod -b /target-root/usr "${TARGET_KVER}" || true

# Configure dynamic linker search paths so ldconfig indexes FSDK libraries
# (e.g. libc.so.6, libgcc_s.so.1, libkmod.so) for dracut-install
Expand Down Expand Up @@ -360,7 +360,7 @@ config:
rm -rf "${MOD_DIR}/drivers/infiniband"
fi
# Re-run depmod to update module dependencies after stripping unneeded modules
depmod -b /layer "${KVER}" || true
depmod -b /layer/usr "${KVER}" || true
fi

( cd /layer && find . -print0 \
Expand Down
11 changes: 7 additions & 4 deletions files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,13 @@ spec:
- /etc/kubestellar-kiosk/nginx.conf
ports:
- containerPort: 8080
# Kiosk-only proxy: bind loopback so the console is not exposed on
# LAN interfaces over plaintext HTTP. The VM flow reaches it via the
# existing hostfwd of 127.0.0.1:8080; LAN access, if ever wanted,
# must go through a TLS-terminating, authenticated front end.
# Kiosk-only proxy: bind loopback so the console is never published on
# the node's external interfaces. A QEMU hostfwd cannot reach this
# binding, because libslirp delivers a forward with an unspecified
# guest address to the guest's DHCP address rather than to 127.0.0.1;
# the installer smoke test therefore probes the proxy from inside the
# guest. LAN access, if ever wanted, must go through an
# authenticated front end.
hostIP: 127.0.0.1
hostPort: 8080
name: http
Expand Down
67 changes: 66 additions & 1 deletion tests/unit/test_installer_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
)
JUSTFILE = REPO_ROOT / "Justfile"
DDI_ELEMENT = REPO_ROOT / "elements" / "oci" / "bluefin-server-ddi.bst"
FLATCAR_ZFS_ELEMENT = REPO_ROOT / "elements" / "flatcar" / "flatcar-zfs.bst"


def _published_uki_cmdline(installer_element: str) -> str:
Expand Down Expand Up @@ -92,10 +93,11 @@ def test_ddi_generates_module_indexes_for_runtime_filesystem_drivers() -> None:
ddi_element = DDI_ELEMENT.read_text(encoding="utf-8")

assert "freedesktop-sdk.bst:components/kmod.bst" in ddi_element
assert 'depmod -b /layer "${KVER}"' in ddi_element
assert 'depmod -b /layer/usr "${KVER}"' in ddi_element
assert "cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/" in ddi_element
assert "tls-ca-bundle.pem" in ddi_element
assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service" in ddi_element
assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service" in ddi_element
assert "ln -sf /dev/null /layer/etc/systemd/system/audit-rules.service" in ddi_element
assert "printf '127.0.0.1 localhost" in ddi_element
assert "> /layer/etc/hosts" in ddi_element
Expand Down Expand Up @@ -135,3 +137,66 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None:
assert "find /layer -type f -name '*.a' -delete" in installer_element
assert 'rm -f "/layer/usr/lib/modules/${KVER}/vmlinux"' in ddi_element
assert "find /layer -type f -name '*.a' -delete" in ddi_element


def test_flatcar_zfs_removes_udevd_sysext_ordering_dropin() -> None:
flatcar_zfs = FLATCAR_ZFS_ELEMENT.read_text(encoding="utf-8")

assert (
'rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d"'
in flatcar_zfs
)
assert (
'sed -i "/systemd-udevd\\.service\\.d/d" "%{install-root}/usr/lib/tmpfiles.d/10-zfs.conf"'
in flatcar_zfs
)
assert "freedesktop-sdk.bst:components/sed.bst" in flatcar_zfs

# The preserved raw sysext must be rebuilt from the cleaned tree, not the
# untouched upstream image, so the ordering override cannot come back if the
# raw is ever merged at runtime.
assert "install -D -m 0644 flatcar-zfs.raw" not in flatcar_zfs
assert (
'mksquashfs "%{install-root}" zfs-cleaned.raw -noappend -no-xattrs -all-root'
in flatcar_zfs
)
assert flatcar_zfs.index("mksquashfs") > flatcar_zfs.index(
'rm -rf "%{install-root}/usr/lib/systemd/system/systemd-udevd.service.d"'
)


def test_var_partition_contracts_use_consistent_partlabel() -> None:
import base64

repart_var = (
REPO_ROOT / "files" / "installer" / "repart.d" / "30-var.conf"
).read_text(encoding="utf-8")
ddi_element = DDI_ELEMENT.read_text(encoding="utf-8")
var_mount = (
REPO_ROOT / "files" / "os" / "systemd" / "system" / "var.mount"
).read_text(encoding="utf-8")
justfile = JUSTFILE.read_text(encoding="utf-8")

assert "Label=var" in repart_var
assert "/dev/disk/by-partlabel/var /var xfs defaults 0 0" in ddi_element
assert "What=/dev/disk/by-partlabel/var" in var_mount
expected_b64 = base64.b64encode(
b"/dev/disk/by-partlabel/var /var xfs defaults 0 0\n"
).decode("ascii")
assert expected_b64 in justfile


def test_installer_smoke_probes_the_kiosk_over_tls_from_inside_the_guest() -> None:
justfile = JUSTFILE.read_text(encoding="utf-8")
assert "systemd.mask=systemd-homed-firstboot.service" in justfile
assert "https://127.0.0.1:8080/healthz" in justfile
assert "--insecure" in justfile
assert "systemd.extra-unit.bluefin-kiosk-ready.service" in justfile
assert "systemd.wants=bluefin-kiosk-ready.service" in justfile
# The host cannot reach the guest's loopback-bound kiosk proxy, so no
# host-side probe may remain in the smoke test — a stray host process on
# port 8080 would otherwise let it pass without the guest being ready.
smoke = justfile.split("install-vm:")[0]
assert "http://127.0.0.1:8080/healthz" not in smoke
assert "KIOSK_CONSOLE_READY" in smoke

Loading