From 44c67fb7eefbfe96bf512dfee392503fd7807f14 Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 18 Sep 2026 13:23:19 -0400 Subject: [PATCH 1/2] fix(ci): probe the kiosk console from inside the QEMU guest over TLS The `installer-test / Install and boot in QEMU` gate has never passed on main. Two independent defects make it unreachable by construction. Transport: `41-kubestellar-kiosk-proxy.yaml` publishes hostPort 8080 on `hostIP: 127.0.0.1`, a deliberate hardening from #75, but a QEMU hostfwd with an unspecified guest address is delivered to the guest's DHCP address, not to guest loopback. Nothing listens there, so `hostfwd=tcp:127.0.0.1:8080-:8080` can never be served. Protocol: `files/k0s/kiosk/nginx.conf` has served the proxy as `listen 8080 ssl` since c752c48, while the recipe probed it over plain `http://` with `curl --fail`, which turns the 497 -> 302 redirect into an error and never yields the expected JSON. Keep the loopback binding and move the probe into the guest: inject a test-only readiness unit through the same SMBIOS systemd-credential channel the recipe already uses for `fstab.extra`, pull it in with `systemd.wants=`, and have it poll `https://127.0.0.1:8080` with `--insecure`, matching the self-signed TLS handling in tests/e2e, then announce the verdict on the system console that serial.log already captures. The dead hostfwd is removed. Necessary but not sufficient: the gate still needs #173, without which the target boots into emergency mode waiting for /dev/disk/by-partlabel/var and never applies any manifests. Assisted-by: Claude Opus 4.6 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 41 ++++++++++++++----- .../41-kubestellar-kiosk-proxy.yaml | 11 +++-- tests/unit/test_installer_contract.py | 17 ++++++++ 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/Justfile b/Justfile index f2dc1b23..b6fb4e65 100644 --- a/Justfile +++ b/Justfile @@ -317,6 +317,30 @@ test-installer-artifact: } trap cleanup EXIT INT TERM + # The kiosk proxy publishes hostPort 8080 on the guest's loopback only + # (files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml), and a + # QEMU hostfwd with an unspecified guest address is delivered to the guest's + # DHCP address, never to 127.0.0.1 — so no host-side probe can reach it + # without exposing the console on the guest's external interface. Probe from + # inside the guest instead and report the verdict on the system console, + # which is already captured in $SERIAL_LOG. The proxy terminates TLS with the + # self-signed certificate baked into the k0s sysext, so the probe speaks + # https and skips certificate verification, matching tests/e2e. + READY_MARKER="KIOSK_CONSOLE_READY" + READY_UNIT=$(base64 -w0 <<'UNIT' + [Unit] + Description=Report KubeStellar Console readiness on the system console + ConditionPathExists=!/etc/initrd-release + After=k0s-first-boot.service + + [Service] + Type=oneshot + TimeoutStartSec=infinity + StandardOutput=journal+console + ExecStart=/usr/bin/bash -c 'echo "kiosk-ready: polling https://127.0.0.1:8080"; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do sleep 2; done; echo KIOSK_CONSOLE_READY' + UNIT + ) + echo "==> Booting the installed server in QEMU (background)..." qemu-system-x86_64 \ -enable-kvm \ @@ -326,9 +350,10 @@ test-installer-artifact: -drive file="$WORKDIR/target.raw",format=raw,if=virtio \ -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ - -nic user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:8080-:8080 \ + -nic user,model=virtio-net-pci \ -smbios "type=11,value=io.systemd.credential.binary:fstab.extra=L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" \ - -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service" \ + -smbios "type=11,value=io.systemd.credential.binary:systemd.extra-unit.bluefin-kiosk-ready.service=${READY_UNIT}" \ + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.mask=systemd-firstboot.service systemd.wants=bluefin-kiosk-ready.service" \ -nographic \ -serial file:"$SERIAL_LOG" \ -monitor none & @@ -336,7 +361,7 @@ test-installer-artifact: DEADLINE_SECS="${SHOW_ME_THE_FUTURE_DEADLINE:-${SHOW_ME_THE_FUTURE_TIMEOUT:-600}}" START_TIME=$(date +%s) - echo "==> Polling KubeStellar Console readiness at http://127.0.0.1:8080 (deadline: ${DEADLINE_SECS}s)..." + echo "==> Polling KubeStellar Console readiness from inside the guest (deadline: ${DEADLINE_SECS}s)..." while true; do if ! kill -0 "$TARGET_QEMU_PID" 2>/dev/null; then @@ -348,13 +373,9 @@ test-installer-artifact: exit 1 fi - HEALTHZ_JSON=$(curl --silent --fail --max-time 2 http://127.0.0.1:8080/healthz 2>/dev/null || true) - if [ -n "$HEALTHZ_JSON" ] && echo "$HEALTHZ_JSON" | jq -e '.status == "ok"' >/dev/null 2>&1; then - ROOT_CODE=$(curl --silent --fail --max-time 2 --output /dev/null --write-out "%{http_code}" http://127.0.0.1:8080/ 2>/dev/null || true) - if [ "$ROOT_CODE" = "200" ]; then - echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" - break - fi + if grep -q "$READY_MARKER" "$SERIAL_LOG" 2>/dev/null; then + echo "==> KubeStellar Console is healthy: /healthz status ok, / returned HTTP 200" + break fi NOW=$(date +%s) diff --git a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml index 80e38d5a..8ade0861 100644 --- a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml +++ b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml @@ -29,10 +29,13 @@ spec: - /etc/kubestellar-kiosk/nginx.conf ports: - containerPort: 8080 - # Kiosk-only proxy: bind loopback so the console is not exposed on - # LAN interfaces over plaintext HTTP. The VM flow reaches it via the - # existing hostfwd of 127.0.0.1:8080; LAN access, if ever wanted, - # must go through a TLS-terminating, authenticated front end. + # Kiosk-only proxy: bind loopback so the console is never published on + # the node's external interfaces. A QEMU hostfwd cannot reach this + # binding, because libslirp delivers a forward with an unspecified + # guest address to the guest's DHCP address rather than to 127.0.0.1; + # the installer smoke test therefore probes the proxy from inside the + # guest. LAN access, if ever wanted, must go through an + # authenticated front end. hostIP: 127.0.0.1 hostPort: 8080 name: http diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index d423c685..cf7176eb 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -135,3 +135,20 @@ def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: assert "find /layer -type f -name '*.a' -delete" in installer_element assert 'rm -f "/layer/usr/lib/modules/${KVER}/vmlinux"' in ddi_element assert "find /layer -type f -name '*.a' -delete" in ddi_element + + +def test_installer_smoke_probes_the_kiosk_over_tls_from_inside_the_guest() -> None: + justfile = JUSTFILE.read_text(encoding="utf-8") + recipe = justfile.split("\ntest-installer-artifact:\n", 1)[1].split( + "\n# Interactively install", 1 + )[0] + + # The kiosk proxy publishes hostPort 8080 on the guest's loopback, which a + # QEMU hostfwd never reaches, and it terminates TLS with a self-signed + # certificate. A host-side plaintext probe can therefore never go green. + assert "hostfwd=" not in recipe + assert "http://127.0.0.1:8080" not in recipe + assert "https://127.0.0.1:8080/healthz" in recipe + assert "--insecure" in recipe + assert "systemd.extra-unit.bluefin-kiosk-ready.service" in recipe + assert "systemd.wants=bluefin-kiosk-ready.service" in recipe From 87c381d7341845be8d75bdd2815dc1c0b21f056d Mon Sep 17 00:00:00 2001 From: Jorge Castro Date: Fri, 18 Sep 2026 14:33:17 -0400 Subject: [PATCH 2/2] fix(ci): write the kiosk readiness marker straight to /dev/console The first CI run of this branch (35374112337) reached the new probe but added a boot-time symptom that no `main` run shows: "Timed out waiting for device /dev/ttyS0". Runs 34905862943, 34910341724, 34914418293 and 34920225526 only ever time out on /dev/disk/by-partlabel/var. `StandardOutput=journal+console` makes the unit console-touching, which pulls extra console ordering into a transaction that is already stuck in emergency mode. Nothing about the beacon needs journald forwarding, so echo the two lines to /dev/console directly and drop the setting. Same output in serial.log, no console machinery attached to the unit. Verified locally: the generated unit passes systemd-analyze verify, prints both lines when a self-signed TLS fixture answers, and keeps polling without emitting the marker when nothing is listening. Assisted-by: Claude Opus 4.6 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Justfile b/Justfile index b6fb4e65..d8eebea9 100644 --- a/Justfile +++ b/Justfile @@ -336,8 +336,7 @@ test-installer-artifact: [Service] Type=oneshot TimeoutStartSec=infinity - StandardOutput=journal+console - ExecStart=/usr/bin/bash -c 'echo "kiosk-ready: polling https://127.0.0.1:8080"; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do sleep 2; done; echo KIOSK_CONSOLE_READY' + ExecStart=/usr/bin/bash -c 'echo "kiosk-ready: polling https://127.0.0.1:8080" >> /dev/console; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do sleep 2; done; echo KIOSK_CONSOLE_READY >> /dev/console' UNIT )