diff --git a/Justfile b/Justfile index ea47448f..724c30ce 100644 --- a/Justfile +++ b/Justfile @@ -465,10 +465,66 @@ install-vm: -drive file="$TARGET_RAW",format=raw,if=virtio \ -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ -drive if=pflash,format=raw,file="$OVMF_VARS" + # The interactive installer is driven by hand, so it can be aborted or + # fail without a non-zero exit here. Marking the install complete in that + # case caches a broken target that every later run boots forever. Gate + # the marker on the partition labels files/repart.d creates; an aborted + # install now fails loudly and re-runs the installer next invocation + # instead of silently booting an unusable disk. + echo "==> Verifying target partition layout before marking installation complete..." + TARGET_PARTS=$(sfdisk --json "$TARGET_RAW" 2>/dev/null | jq -r '.partitiontable.partitions[]?.name // empty' || true) + if ! echo "$TARGET_PARTS" | grep -Fxq 'bluefin-server-root-a' || \ + ! echo "$TARGET_PARTS" | grep -Fxq 'var'; then + echo "ERROR: Target disk $TARGET_RAW does not contain expected partitions (bluefin-server-root-a, var)!" >&2 + echo "The installation did not complete; re-run 'just install-vm' to install again." >&2 + echo "Observed partitions:" >&2 + echo "$TARGET_PARTS" >&2 + sfdisk -l "$TARGET_RAW" >&2 || true + exit 1 + fi touch "$INSTALL_COMPLETE" fi echo "==> Booting the installed kiosk..." + SERIAL_LOG="$STATE_DIR/kiosk-serial.log" + : > "$SERIAL_LOG" + + # The kiosk proxy binds hostPort 8080 on the guest's loopback interface only + # (files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml) and + # terminates TLS (files/k0s/kiosk/nginx.conf, `listen 8080 ssl`). A + # QEMU/libslirp hostfwd with an unspecified guest address is delivered to + # the guest's DHCP address, never to 127.0.0.1, so a host-side forward of + # 8080 can never reach the proxy, and a plaintext probe would fail against + # its TLS listener regardless. Probe readiness from inside the guest + # instead and report the verdict on the system console, which is captured + # in $SERIAL_LOG — the same in-guest-probe approach proposed for + # test-installer-artifact in #220. Host access, when wanted, goes over + # the existing SSH hostfwd rather than a direct (non-working) 8080 + # forward. The readiness check deliberately does not parse the /healthz + # response body: curl --fail already fails on a non-2xx response, which + # is exactly "healthz ok and / returns 200", and it avoids embedding + # quotes inside a systemd ExecStart= line, where escaping rules differ + # from a plain shell. + # The installed image carries no fstab entry for /var and gpt-auto cannot + # mount it, so /var (which holds k0s state and therefore the kiosk proxy) + # stays empty unless the same SMBIOS fstab.extra credential + # test-installer-artifact passes is supplied here too. Without it k0s never + # starts and the readiness marker never appears. + READY_MARKER="KIOSK_CONSOLE_READY" + READY_UNIT=$(base64 -w0 <<'UNIT' + [Unit] + Description=Report KubeStellar Console readiness on the system console + ConditionPathExists=!/etc/initrd-release + After=k0s-first-boot.service + + [Service] + Type=oneshot + TimeoutStartSec=infinity + StandardOutput=journal+console + ExecStart=/usr/bin/bash -c 'until curl --silent --fail --insecure --max-time 2 --output /dev/null https://127.0.0.1:8080/healthz && curl --silent --fail --insecure --max-time 2 --output /dev/null https://127.0.0.1:8080/; do sleep 2; done; echo KIOSK_CONSOLE_READY' + UNIT + ) + qemu-system-x86_64 \ -enable-kvm \ -m "${MEM_SIZE}" \ @@ -477,7 +533,11 @@ install-vm: -drive file="$TARGET_RAW",format=raw,if=virtio \ -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ -drive if=pflash,format=raw,file="$OVMF_VARS" \ - -nic user,model=virtio-net-pci,hostfwd=tcp::8080-:8080,hostfwd=tcp::2222-:22,hostfwd=tcp::6443-:6443 & + -nic user,model=virtio-net-pci,hostfwd=tcp::2222-:22,hostfwd=tcp::6443-:6443 \ + -smbios "type=11,value=io.systemd.credential.binary:fstab.extra=L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" \ + -smbios "type=11,value=io.systemd.credential.binary:systemd.extra-unit.bluefin-kiosk-ready.service=${READY_UNIT}" \ + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.wants=bluefin-kiosk-ready.service" \ + -serial file:"$SERIAL_LOG" & QEMU_PID=$! cleanup() { if kill -0 "$QEMU_PID" 2>/dev/null; then @@ -487,19 +547,42 @@ install-vm: } trap cleanup INT TERM - until curl --silent --show-error --max-time 2 --output /dev/null http://127.0.0.1:8080/; do + READY_DEADLINE_SECS="${INSTALL_VM_READY_DEADLINE:-600}" + READY_START_TIME=$(date +%s) + echo "==> Waiting for the KubeStellar Console to become ready inside the guest (deadline: ${READY_DEADLINE_SECS}s)..." + until grep -q "$READY_MARKER" "$SERIAL_LOG" 2>/dev/null; do if ! kill -0 "$QEMU_PID" 2>/dev/null; then + echo "ERROR: QEMU process ($QEMU_PID) died before the kiosk became ready!" >&2 + if [ -f "$SERIAL_LOG" ]; then + echo "==> Serial log tail (last 100 lines):" >&2 + tail -n 100 "$SERIAL_LOG" >&2 + fi wait "$QEMU_PID" exit 1 fi + + NOW=$(date +%s) + ELAPSED=$((NOW - READY_START_TIME)) + if [ "$ELAPSED" -ge "$READY_DEADLINE_SECS" ]; then + echo "ERROR: Timed out after ${READY_DEADLINE_SECS}s waiting for KubeStellar Console readiness!" >&2 + if [ -f "$SERIAL_LOG" ]; then + echo "==> Serial log tail (last 100 lines):" >&2 + tail -n 100 "$SERIAL_LOG" >&2 + fi + cleanup + exit 1 + fi + sleep 2 done - HOST_IP="$(ip -4 -o addr show scope global | awk '{print $4}' | cut -d/ -f1 | head -n1)" echo "==> KubeStellar Console is ready!" - echo "==> Access URL (LAN): http://${HOST_IP:-localhost}:8080/" - echo "==> Access URL (Local): http://localhost:8080/" - xdg-open "http://${HOST_IP:-localhost}:8080/" || xdg-open http://localhost:8080/ || true + echo "==> The kiosk proxy binds the guest's loopback interface only and terminates" + echo "==> TLS with a self-signed certificate, so it is not directly reachable from" + echo "==> the host. To view it in a host browser, start sshd inside the VM" + echo "==> (systemctl start sshd) and open a tunnel over the forwarded SSH port:" + echo "==> ssh -p 2222 -L 8080:127.0.0.1:8080 @127.0.0.1" + echo "==> then browse https://localhost:8080/ and accept the self-signed certificate." wait "$QEMU_PID" # Set up KubeStellar kc-agent for the user in ONE command. diff --git a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml index 8ade0861..588288ae 100644 --- a/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml +++ b/files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml @@ -30,11 +30,11 @@ spec: ports: - containerPort: 8080 # Kiosk-only proxy: bind loopback so the console is never published on - # the node's external interfaces. A QEMU hostfwd cannot reach this - # binding, because libslirp delivers a forward with an unspecified - # guest address to the guest's DHCP address rather than to 127.0.0.1; - # the installer smoke test therefore probes the proxy from inside the - # guest. LAN access, if ever wanted, must go through an + # the node's external interfaces. A QEMU/libslirp hostfwd with an + # unspecified guest address is delivered to the guest's DHCP address, + # not to 127.0.0.1, so no QEMU hostfwd can reach this binding; + # `just install-vm` and the installer smoke test probe the proxy from + # inside the guest instead. LAN access, if ever wanted, must go through an # authenticated front end. hostIP: 127.0.0.1 hostPort: 8080 diff --git a/files/os/issue.d/40-kubestellar.issue b/files/os/issue.d/40-kubestellar.issue index 61319dc5..862fc3b5 100644 --- a/files/os/issue.d/40-kubestellar.issue +++ b/files/os/issue.d/40-kubestellar.issue @@ -1,5 +1,6 @@ Bluefin Server (\l) Default login: root / bluefin -KubeStellar Console: http://\4:8080/ (or http://127.0.0.1:8080 from host) +KubeStellar Console: https://127.0.0.1:8080/ (loopback, TLS only) +Remote access: ssh -L 8080:127.0.0.1:8080 @ diff --git a/tests/unit/test_console_issue.py b/tests/unit/test_console_issue.py index af83ead1..809fc946 100644 --- a/tests/unit/test_console_issue.py +++ b/tests/unit/test_console_issue.py @@ -15,7 +15,8 @@ def test_issue_file_exists_and_contains_kubestellar_url() -> None: assert ISSUE_FILE.is_file(), f"{ISSUE_FILE} must exist" content = ISSUE_FILE.read_text(encoding="utf-8") assert "Bluefin Server" in content - assert "KubeStellar Console: http://\\4:8080/" in content + assert "KubeStellar Console: https://127.0.0.1:8080/" in content + assert "ssh -L 8080:127.0.0.1:8080" in content def test_os_issue_element_target_usr_lib_issue_d() -> None: diff --git a/tests/unit/test_install_vm_kiosk_readiness.py b/tests/unit/test_install_vm_kiosk_readiness.py new file mode 100644 index 00000000..67a875a5 --- /dev/null +++ b/tests/unit/test_install_vm_kiosk_readiness.py @@ -0,0 +1,122 @@ +"""Contracts for `just install-vm` reaching the loopback-bound, TLS-terminating +kiosk proxy (files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml, +files/k0s/kiosk/nginx.conf).""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +JUSTFILE = ROOT / "Justfile" +KIOSK_PROXY_MANIFEST = ( + ROOT / "files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml" +) + + +def _install_vm_recipe() -> str: + justfile = JUSTFILE.read_text(encoding="utf-8") + start = justfile.index("install-vm:") + end = justfile.index("setup-kubestellar", start) + return justfile[start:end] + + +def test_install_vm_drops_the_unreachable_direct_kiosk_forward() -> None: + recipe = _install_vm_recipe() + + # A QEMU/libslirp hostfwd with an unspecified guest address is delivered + # to the guest's DHCP address, never to 127.0.0.1, so it can never reach + # a loopback-bound service. This forward must be gone, and the host must + # not claim a URL it cannot reach. + assert "hostfwd=tcp::8080-:8080" not in recipe + assert "http://127.0.0.1:8080/" not in recipe + assert "http://${HOST_IP" not in recipe + assert "xdg-open" not in recipe + + # Unrelated forwards used elsewhere in the recipe must survive. + assert "hostfwd=tcp::2222-:22" in recipe + assert "hostfwd=tcp::6443-:6443" in recipe + + +def test_install_vm_probes_readiness_inside_the_guest_over_tls() -> None: + recipe = _install_vm_recipe() + + # Readiness is probed from inside the guest, over TLS (the proxy + # terminates TLS with a self-signed certificate), and reported on the + # system console, which is captured to a host-side serial log file. + assert "https://127.0.0.1:8080/healthz" in recipe + assert "--insecure" in recipe + assert "systemd.extra-unit.bluefin-kiosk-ready.service" in recipe + assert "systemd.wants=bluefin-kiosk-ready.service" in recipe + assert "-serial file:" in recipe + assert 'SERIAL_LOG="$STATE_DIR/kiosk-serial.log"' in recipe + assert 'grep -q "$READY_MARKER" "$SERIAL_LOG"' in recipe + + # The in-guest probe deliberately avoids parsing the /healthz response + # body (with jq, grep, or otherwise): curl --fail already fails on a + # non-2xx response, which is exactly "healthz ok and / returns 200", + # and it avoids embedding quotes inside a systemd ExecStart= line, whose + # escaping rules differ from a plain shell. + exec_start_line = next( + line for line in recipe.splitlines() if line.strip().startswith("ExecStart=") + ) + assert exec_start_line.strip().startswith("ExecStart=/usr/bin/bash -c 'until curl") + assert "jq" not in exec_start_line + + # ttyS0 must be the last (and therefore primary) console so /dev/console + # resolves to the serial device the host captures, matching the ordering + # test-installer-artifact already relies on for the same reason. + assert "console=tty0 console=ttyS0,,115200 systemd.wants=bluefin-kiosk-ready.service" in recipe + + # Dies-while-waiting must still fail fast instead of polling forever. + assert 'if ! kill -0 "$QEMU_PID" 2>/dev/null; then' in recipe + + +def test_install_vm_mounts_var_from_the_partlabel_credential() -> None: + recipe = _install_vm_recipe() + + # The installed image has no fstab entry for /var and gpt-auto cannot + # mount it, so the boot needs the same SMBIOS fstab.extra credential + # test-installer-artifact passes. Without /var, k0s (and the kiosk proxy + # it runs) never starts and the readiness marker never appears. + assert ( + "io.systemd.credential.binary:fstab.extra=" + "L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" + ) in recipe + + +def test_install_vm_readiness_wait_has_a_deadline_and_diagnostics() -> None: + recipe = _install_vm_recipe() + + # A broken guest boot must not hang forever silently: bound the wait and + # dump the serial log on both timeout and premature QEMU exit, matching + # test-installer-artifact. + assert 'READY_DEADLINE_SECS="${INSTALL_VM_READY_DEADLINE:-600}"' in recipe + assert 'if [ "$ELAPSED" -ge "$READY_DEADLINE_SECS" ]; then' in recipe + assert recipe.count('tail -n 100 "$SERIAL_LOG" >&2') == 2 + + +def test_install_vm_documents_ssh_tunnel_for_host_access() -> None: + recipe = _install_vm_recipe() + + # Host access, when wanted, goes over the existing SSH hostfwd instead of + # a direct forward the loopback-bound proxy can never receive. + assert "ssh -p 2222" in recipe + assert "127.0.0.1:8080" in recipe + assert "self-signed certificate" in recipe + + +def test_kiosk_proxy_manifest_comment_matches_actual_reachability() -> None: + manifest = KIOSK_PROXY_MANIFEST.read_text(encoding="utf-8") + + # The comment must not claim a hostfwd can reach a loopback-only bind; + # a QEMU/libslirp hostfwd with an unspecified guest address never lands + # on the guest's loopback interface. + assert "existing hostfwd of 127.0.0.1:8080" not in manifest + assert "hostIP: 127.0.0.1" in manifest + +def test_install_vm_verifies_target_partition_layout_before_marking_complete() -> None: + recipe = _install_vm_recipe() + + assert "bluefin-server-root-a" in recipe + assert 'sfdisk --json "$TARGET_RAW"' in recipe + assert 'grep -Fxq \'bluefin-server-root-a\'' in recipe + assert 'grep -Fxq \'var\'' in recipe + assert 'touch "$INSTALL_COMPLETE"' in recipe