Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 89 additions & 6 deletions Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -465,10 +465,66 @@ install-vm:
-drive file="$TARGET_RAW",format=raw,if=virtio \
-drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \
-drive if=pflash,format=raw,file="$OVMF_VARS"
# The interactive installer is driven by hand, so it can be aborted or
# fail without a non-zero exit here. Marking the install complete in that
# case caches a broken target that every later run boots forever. Gate
# the marker on the partition labels files/repart.d creates; an aborted
# install now fails loudly and re-runs the installer next invocation
# instead of silently booting an unusable disk.
echo "==> Verifying target partition layout before marking installation complete..."
TARGET_PARTS=$(sfdisk --json "$TARGET_RAW" 2>/dev/null | jq -r '.partitiontable.partitions[]?.name // empty' || true)
if ! echo "$TARGET_PARTS" | grep -Fxq 'bluefin-server-root-a' || \
! echo "$TARGET_PARTS" | grep -Fxq 'var'; then
echo "ERROR: Target disk $TARGET_RAW does not contain expected partitions (bluefin-server-root-a, var)!" >&2
echo "The installation did not complete; re-run 'just install-vm' to install again." >&2
echo "Observed partitions:" >&2
echo "$TARGET_PARTS" >&2
sfdisk -l "$TARGET_RAW" >&2 || true
exit 1
fi
touch "$INSTALL_COMPLETE"
fi

echo "==> Booting the installed kiosk..."
SERIAL_LOG="$STATE_DIR/kiosk-serial.log"
: > "$SERIAL_LOG"

# The kiosk proxy binds hostPort 8080 on the guest's loopback interface only
# (files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml) and
# terminates TLS (files/k0s/kiosk/nginx.conf, `listen 8080 ssl`). A
# QEMU/libslirp hostfwd with an unspecified guest address is delivered to
# the guest's DHCP address, never to 127.0.0.1, so a host-side forward of
# 8080 can never reach the proxy, and a plaintext probe would fail against
# its TLS listener regardless. Probe readiness from inside the guest
# instead and report the verdict on the system console, which is captured
# in $SERIAL_LOG — the same in-guest-probe approach proposed for
# test-installer-artifact in #220. Host access, when wanted, goes over
# the existing SSH hostfwd rather than a direct (non-working) 8080
# forward. The readiness check deliberately does not parse the /healthz
# response body: curl --fail already fails on a non-2xx response, which
# is exactly "healthz ok and / returns 200", and it avoids embedding
# quotes inside a systemd ExecStart= line, where escaping rules differ
# from a plain shell.
# The installed image carries no fstab entry for /var and gpt-auto cannot
# mount it, so /var (which holds k0s state and therefore the kiosk proxy)
# stays empty unless the same SMBIOS fstab.extra credential
# test-installer-artifact passes is supplied here too. Without it k0s never
# starts and the readiness marker never appears.
READY_MARKER="KIOSK_CONSOLE_READY"
READY_UNIT=$(base64 -w0 <<'UNIT'
[Unit]
Description=Report KubeStellar Console readiness on the system console
ConditionPathExists=!/etc/initrd-release
After=k0s-first-boot.service

[Service]
Type=oneshot
TimeoutStartSec=infinity
StandardOutput=journal+console
ExecStart=/usr/bin/bash -c 'until curl --silent --fail --insecure --max-time 2 --output /dev/null https://127.0.0.1:8080/healthz && curl --silent --fail --insecure --max-time 2 --output /dev/null https://127.0.0.1:8080/; do sleep 2; done; echo KIOSK_CONSOLE_READY'
UNIT
)

qemu-system-x86_64 \
-enable-kvm \
-m "${MEM_SIZE}" \
Expand All @@ -477,7 +533,11 @@ install-vm:
-drive file="$TARGET_RAW",format=raw,if=virtio \
-drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \
-drive if=pflash,format=raw,file="$OVMF_VARS" \
-nic user,model=virtio-net-pci,hostfwd=tcp::8080-:8080,hostfwd=tcp::2222-:22,hostfwd=tcp::6443-:6443 &
-nic user,model=virtio-net-pci,hostfwd=tcp::2222-:22,hostfwd=tcp::6443-:6443 \
-smbios "type=11,value=io.systemd.credential.binary:fstab.extra=L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg==" \
-smbios "type=11,value=io.systemd.credential.binary:systemd.extra-unit.bluefin-kiosk-ready.service=${READY_UNIT}" \
-smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 systemd.wants=bluefin-kiosk-ready.service" \
-serial file:"$SERIAL_LOG" &
QEMU_PID=$!
cleanup() {
if kill -0 "$QEMU_PID" 2>/dev/null; then
Expand All @@ -487,19 +547,42 @@ install-vm:
}
trap cleanup INT TERM

until curl --silent --show-error --max-time 2 --output /dev/null http://127.0.0.1:8080/; do
READY_DEADLINE_SECS="${INSTALL_VM_READY_DEADLINE:-600}"
READY_START_TIME=$(date +%s)
echo "==> Waiting for the KubeStellar Console to become ready inside the guest (deadline: ${READY_DEADLINE_SECS}s)..."
until grep -q "$READY_MARKER" "$SERIAL_LOG" 2>/dev/null; do
if ! kill -0 "$QEMU_PID" 2>/dev/null; then
echo "ERROR: QEMU process ($QEMU_PID) died before the kiosk became ready!" >&2
if [ -f "$SERIAL_LOG" ]; then
echo "==> Serial log tail (last 100 lines):" >&2
tail -n 100 "$SERIAL_LOG" >&2
fi
wait "$QEMU_PID"
exit 1
fi

NOW=$(date +%s)
ELAPSED=$((NOW - READY_START_TIME))
if [ "$ELAPSED" -ge "$READY_DEADLINE_SECS" ]; then
echo "ERROR: Timed out after ${READY_DEADLINE_SECS}s waiting for KubeStellar Console readiness!" >&2
if [ -f "$SERIAL_LOG" ]; then
echo "==> Serial log tail (last 100 lines):" >&2
tail -n 100 "$SERIAL_LOG" >&2
fi
cleanup
exit 1
fi

sleep 2
done

HOST_IP="$(ip -4 -o addr show scope global | awk '{print $4}' | cut -d/ -f1 | head -n1)"
echo "==> KubeStellar Console is ready!"
echo "==> Access URL (LAN): http://${HOST_IP:-localhost}:8080/"
echo "==> Access URL (Local): http://localhost:8080/"
xdg-open "http://${HOST_IP:-localhost}:8080/" || xdg-open http://localhost:8080/ || true
echo "==> The kiosk proxy binds the guest's loopback interface only and terminates"
echo "==> TLS with a self-signed certificate, so it is not directly reachable from"
echo "==> the host. To view it in a host browser, start sshd inside the VM"
echo "==> (systemctl start sshd) and open a tunnel over the forwarded SSH port:"
echo "==> ssh -p 2222 -L 8080:127.0.0.1:8080 <your-user>@127.0.0.1"
echo "==> then browse https://localhost:8080/ and accept the self-signed certificate."
wait "$QEMU_PID"

# Set up KubeStellar kc-agent for the user in ONE command.
Expand Down
10 changes: 5 additions & 5 deletions files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ spec:
ports:
- containerPort: 8080
# Kiosk-only proxy: bind loopback so the console is never published on
# the node's external interfaces. A QEMU hostfwd cannot reach this
# binding, because libslirp delivers a forward with an unspecified
# guest address to the guest's DHCP address rather than to 127.0.0.1;
# the installer smoke test therefore probes the proxy from inside the
# guest. LAN access, if ever wanted, must go through an
# the node's external interfaces. A QEMU/libslirp hostfwd with an
# unspecified guest address is delivered to the guest's DHCP address,
# not to 127.0.0.1, so no QEMU hostfwd can reach this binding;
# `just install-vm` and the installer smoke test probe the proxy from
# inside the guest instead. LAN access, if ever wanted, must go through an
# authenticated front end.
hostIP: 127.0.0.1
hostPort: 8080
Expand Down
3 changes: 2 additions & 1 deletion files/os/issue.d/40-kubestellar.issue
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
Bluefin Server (\l)
Default login: root / bluefin
KubeStellar Console: http://\4:8080/ (or http://127.0.0.1:8080 from host)
KubeStellar Console: https://127.0.0.1:8080/ (loopback, TLS only)
Remote access: ssh -L 8080:127.0.0.1:8080 <user>@<host>


3 changes: 2 additions & 1 deletion tests/unit/test_console_issue.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ def test_issue_file_exists_and_contains_kubestellar_url() -> None:
assert ISSUE_FILE.is_file(), f"{ISSUE_FILE} must exist"
content = ISSUE_FILE.read_text(encoding="utf-8")
assert "Bluefin Server" in content
assert "KubeStellar Console: http://\\4:8080/" in content
assert "KubeStellar Console: https://127.0.0.1:8080/" in content
assert "ssh -L 8080:127.0.0.1:8080" in content


def test_os_issue_element_target_usr_lib_issue_d() -> None:
Expand Down
122 changes: 122 additions & 0 deletions tests/unit/test_install_vm_kiosk_readiness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
"""Contracts for `just install-vm` reaching the loopback-bound, TLS-terminating
kiosk proxy (files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml,
files/k0s/kiosk/nginx.conf)."""

from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]
JUSTFILE = ROOT / "Justfile"
KIOSK_PROXY_MANIFEST = (
ROOT / "files/k0s/manifests/kubestellar/41-kubestellar-kiosk-proxy.yaml"
)


def _install_vm_recipe() -> str:
justfile = JUSTFILE.read_text(encoding="utf-8")
start = justfile.index("install-vm:")
end = justfile.index("setup-kubestellar", start)
return justfile[start:end]


def test_install_vm_drops_the_unreachable_direct_kiosk_forward() -> None:
recipe = _install_vm_recipe()

# A QEMU/libslirp hostfwd with an unspecified guest address is delivered
# to the guest's DHCP address, never to 127.0.0.1, so it can never reach
# a loopback-bound service. This forward must be gone, and the host must
# not claim a URL it cannot reach.
assert "hostfwd=tcp::8080-:8080" not in recipe
assert "http://127.0.0.1:8080/" not in recipe
assert "http://${HOST_IP" not in recipe
assert "xdg-open" not in recipe

# Unrelated forwards used elsewhere in the recipe must survive.
assert "hostfwd=tcp::2222-:22" in recipe
assert "hostfwd=tcp::6443-:6443" in recipe


def test_install_vm_probes_readiness_inside_the_guest_over_tls() -> None:
recipe = _install_vm_recipe()

# Readiness is probed from inside the guest, over TLS (the proxy
# terminates TLS with a self-signed certificate), and reported on the
# system console, which is captured to a host-side serial log file.
assert "https://127.0.0.1:8080/healthz" in recipe
assert "--insecure" in recipe
assert "systemd.extra-unit.bluefin-kiosk-ready.service" in recipe
assert "systemd.wants=bluefin-kiosk-ready.service" in recipe
assert "-serial file:" in recipe
assert 'SERIAL_LOG="$STATE_DIR/kiosk-serial.log"' in recipe
assert 'grep -q "$READY_MARKER" "$SERIAL_LOG"' in recipe

# The in-guest probe deliberately avoids parsing the /healthz response
# body (with jq, grep, or otherwise): curl --fail already fails on a
# non-2xx response, which is exactly "healthz ok and / returns 200",
# and it avoids embedding quotes inside a systemd ExecStart= line, whose
# escaping rules differ from a plain shell.
exec_start_line = next(
line for line in recipe.splitlines() if line.strip().startswith("ExecStart=")
)
assert exec_start_line.strip().startswith("ExecStart=/usr/bin/bash -c 'until curl")
assert "jq" not in exec_start_line

# ttyS0 must be the last (and therefore primary) console so /dev/console
# resolves to the serial device the host captures, matching the ordering
# test-installer-artifact already relies on for the same reason.
assert "console=tty0 console=ttyS0,,115200 systemd.wants=bluefin-kiosk-ready.service" in recipe

# Dies-while-waiting must still fail fast instead of polling forever.
assert 'if ! kill -0 "$QEMU_PID" 2>/dev/null; then' in recipe


def test_install_vm_mounts_var_from_the_partlabel_credential() -> None:
recipe = _install_vm_recipe()

# The installed image has no fstab entry for /var and gpt-auto cannot
# mount it, so the boot needs the same SMBIOS fstab.extra credential
# test-installer-artifact passes. Without /var, k0s (and the kiosk proxy
# it runs) never starts and the readiness marker never appears.
assert (
"io.systemd.credential.binary:fstab.extra="
"L2Rldi9kaXNrL2J5LXBhcnRsYWJlbC92YXIgL3ZhciB4ZnMgZGVmYXVsdHMgMCAwCg=="
) in recipe


def test_install_vm_readiness_wait_has_a_deadline_and_diagnostics() -> None:
recipe = _install_vm_recipe()

# A broken guest boot must not hang forever silently: bound the wait and
# dump the serial log on both timeout and premature QEMU exit, matching
# test-installer-artifact.
assert 'READY_DEADLINE_SECS="${INSTALL_VM_READY_DEADLINE:-600}"' in recipe
assert 'if [ "$ELAPSED" -ge "$READY_DEADLINE_SECS" ]; then' in recipe
assert recipe.count('tail -n 100 "$SERIAL_LOG" >&2') == 2


def test_install_vm_documents_ssh_tunnel_for_host_access() -> None:
recipe = _install_vm_recipe()

# Host access, when wanted, goes over the existing SSH hostfwd instead of
# a direct forward the loopback-bound proxy can never receive.
assert "ssh -p 2222" in recipe
assert "127.0.0.1:8080" in recipe
assert "self-signed certificate" in recipe


def test_kiosk_proxy_manifest_comment_matches_actual_reachability() -> None:
manifest = KIOSK_PROXY_MANIFEST.read_text(encoding="utf-8")

# The comment must not claim a hostfwd can reach a loopback-only bind;
# a QEMU/libslirp hostfwd with an unspecified guest address never lands
# on the guest's loopback interface.
assert "existing hostfwd of 127.0.0.1:8080" not in manifest
assert "hostIP: 127.0.0.1" in manifest

def test_install_vm_verifies_target_partition_layout_before_marking_complete() -> None:
recipe = _install_vm_recipe()

assert "bluefin-server-root-a" in recipe
assert 'sfdisk --json "$TARGET_RAW"' in recipe
assert 'grep -Fxq \'bluefin-server-root-a\'' in recipe
assert 'grep -Fxq \'var\'' in recipe
assert 'touch "$INSTALL_COMPLETE"' in recipe