diff --git a/Justfile b/Justfile index c3cf9bb3..86d76970 100644 --- a/Justfile +++ b/Justfile @@ -350,20 +350,46 @@ show-me-the-future: just export-installer just test-installer-artifact +# INSTALLER_BOOT_MODE selects how the installer itself is started: +# usb (default) — firmware (OVMF) boots the raw image off an emulated xHCI USB +# drive, the bare-metal USB install path. +# pxe — QEMU boots the exported PXE vmlinuz/initrd pair directly with +# -kernel/-initrd/-append, the network install path. +# Everything after the install (target partition check, booting the installed +# system, kiosk readiness probe) is identical for both modes. # Install and reboot already-exported server artifacts in QEMU. [group('test')] test-installer-artifact: #!/usr/bin/env bash set -euo pipefail + BOOT_MODE="${INSTALLER_BOOT_MODE:-usb}" + case "${BOOT_MODE}" in + usb|pxe) ;; + *) echo "ERROR: INSTALLER_BOOT_MODE must be 'usb' or 'pxe', got '${BOOT_MODE}'." >&2; exit 1 ;; + esac + CACHE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}" mkdir -p "$CACHE_DIR" WORKDIR="$(mktemp -d "${CACHE_DIR}/bluefin-show-future.XXXXXX")" trap 'rm -rf "$WORKDIR"' EXIT + # The raw installer image is always needed; usb mode boots it through firmware + # so it consumes nothing else. The PXE vmlinuz/initrd pair is only copied in pxe + # mode, because copying it unconditionally would fail the usb test on hosts that + # have the installer image but no PXE artifacts. cp dist/bluefin-server-installer-*.raw.zst "$WORKDIR/installer.raw.zst" - cp dist/bluefin-server-pxe-vmlinuz-* "$WORKDIR/installer.vmlinuz" - cp dist/bluefin-server-pxe-initrd-*.cpio.gz "$WORKDIR/installer.initrd" + if [ "${BOOT_MODE}" = "pxe" ]; then + PXE_KERNEL=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-pxe-vmlinuz-*' | head -n1) + PXE_INITRD=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-pxe-initrd-*.cpio.gz' | head -n1) + if [ -z "${PXE_KERNEL}" ] || [ -z "${PXE_INITRD}" ]; then + echo "ERROR: INSTALLER_BOOT_MODE=pxe needs an exported PXE kernel and initrd in dist/." >&2 + echo "Please run: just build-installer && just export-pxe" >&2 + exit 1 + fi + cp "${PXE_KERNEL}" "$WORKDIR/installer.vmlinuz" + cp "${PXE_INITRD}" "$WORKDIR/installer.initrd" + fi zstd -d "$WORKDIR/installer.raw.zst" -o "$WORKDIR/installer.raw" TARGET_SIZE="${SHOW_ME_THE_FUTURE_DISK_SIZE:-16G}" truncate -s "${TARGET_SIZE}" "$WORKDIR/target.raw" @@ -393,42 +419,102 @@ test-installer-artifact: OVMF_VARS=$(first_existing \ /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-vars.fd \ + /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-i386-vars.fd \ /usr/share/edk2/ovmf/OVMF_VARS.fd \ /usr/share/OVMF/OVMF_VARS.fd \ /usr/share/OVMF/OVMF_VARS_4M.fd \ /usr/share/edk2/x64/OVMF_VARS.4m.fd \ /usr/share/qemu/edk2-x86_64-vars.fd \ + /usr/share/qemu/edk2-i386-vars.fd \ /usr/share/qemu/OVMF_VARS.fd) \ || true - if [ -n "$OVMF_VARS" ]; then + if [ -n "${OVMF_VARS}" ]; then cp "$OVMF_VARS" "$WORKDIR/ovmf-vars.fd" else + # Hosts that ship only OVMF_CODE still boot: firmware initialises a blank + # variable store on first use, it just has no preseeded boot entries. + echo "WARNING: no OVMF_VARS template found; using a blank variable store sized to $OVMF_CODE" >&2 truncate -s "$(stat -c '%s' "$OVMF_CODE")" "$WORKDIR/ovmf-vars.fd" fi SMP_CPUS="${SHOW_ME_THE_FUTURE_SMP:-$(nproc)}" MEM_SIZE="${SHOW_ME_THE_FUTURE_MEM:-8192}" - echo "==> Booting installer media in QEMU..." + echo "==> Booting installer media in QEMU (mode: ${BOOT_MODE})..." # ponytail: we want QEMU to exit cleanly after install. Since QEMU's -no-reboot # suspends/halts on reboot signals, we override systemd-sysinstall.service SuccessAction/FailureAction # to poweroff. When the installer triggers poweroff, QEMU terminates, and we boot into the newly installed OS. - qemu-system-x86_64 \ - -enable-kvm \ - -m "${MEM_SIZE}" \ - -cpu host \ - -smp "${SMP_CPUS}" \ - -drive file="$WORKDIR/installer.raw",format=raw,if=virtio,readonly=on \ - -drive file="$WORKDIR/target.raw",format=raw,if=virtio \ - -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ - -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ - -kernel "$WORKDIR/installer.vmlinuz" \ - -initrd "$WORKDIR/installer.initrd" \ - -append "systemd.unit=system-install.target console=tty0 console=ttyS0,115200 rw unattended" \ + INSTALLER_TIMEOUT="${SHOW_ME_THE_FUTURE_INSTALL_TIMEOUT:-600}" + INSTALL_ARGS=( + -enable-kvm + -m "${MEM_SIZE}" + -cpu host + -smp "${SMP_CPUS}" + -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" + -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" + ) + if [ "${BOOT_MODE}" = "usb" ]; then + # Firmware boots the image's own bootloader, so the unattended flag has to + # reach the UKI through systemd-stub rather than QEMU's -append. + # + # NOTE: systemd-stub deliberately ignores io.systemd.stub.kernel-cmdline-extra + # when Secure Boot is enabled, because the SMBIOS string is not covered by the + # signature. Point $OVMF_CODE at a secure-boot firmware build and this run + # boots interactive instead of unattended, then sits there until + # SHOW_ME_THE_FUTURE_INSTALL_TIMEOUT expires with a misleading "install did + # not finish" failure. Use INSTALLER_BOOT_MODE=pxe (-append is always honoured) + # to exercise unattended installs under Secure Boot firmware. + INSTALL_ARGS+=( + -device qemu-xhci,id=xhci + -drive file="$WORKDIR/installer.raw",format=raw,if=none,id=installer-disk,readonly=on + -device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1 + -drive file="$WORKDIR/target.raw",format=raw,if=none,id=target-disk + -device virtio-blk-pci,drive=target-disk,bootindex=2 + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 rw unattended" + ) + else + INSTALL_ARGS+=( + -drive file="$WORKDIR/installer.raw",format=raw,if=virtio,readonly=on + -drive file="$WORKDIR/target.raw",format=raw,if=virtio + -kernel "$WORKDIR/installer.vmlinuz" + -initrd "$WORKDIR/installer.initrd" + -append "systemd.unit=system-install.target console=tty0 console=ttyS0,115200 rw unattended" + ) + fi + timeout "${INSTALLER_TIMEOUT}" qemu-system-x86_64 \ + "${INSTALL_ARGS[@]}" \ -nographic \ -serial mon:stdio \ -no-reboot < /dev/null + echo "==> Validating target disk partitions after installer execution..." + for tool in sfdisk jq; do + command -v "$tool" >/dev/null 2>&1 \ + || { echo "ERROR: '$tool' is required to validate the target disk layout but is not installed." >&2; exit 1; } + done + + # Keep sfdisk and jq failures distinguishable from "the installer did not + # partition the disk", otherwise a broken host tool is misreported as a + # failed install. + if ! TARGET_TABLE=$(sfdisk --json "$WORKDIR/target.raw"); then + echo "ERROR: sfdisk could not read a partition table from $WORKDIR/target.raw!" >&2 + sfdisk -l "$WORKDIR/target.raw" >&2 || true + exit 1 + fi + if ! TARGET_PARTS=$(jq -r '.partitiontable.partitions[]?.name // empty' <<<"$TARGET_TABLE"); then + echo "ERROR: jq failed to parse the sfdisk JSON output!" >&2 + echo "$TARGET_TABLE" >&2 + exit 1 + fi + if ! echo "$TARGET_PARTS" | grep -Fxq 'bluefin-server-root-a' || \ + ! echo "$TARGET_PARTS" | grep -Fxq 'var'; then + echo "ERROR: Target disk did not receive expected partition layout from installer!" >&2 + echo "Observed partitions:" >&2 + echo "$TARGET_PARTS" >&2 + sfdisk -l "$WORKDIR/target.raw" >&2 || true + exit 1 + fi + echo "==> Target disk successfully partitioned and populated!" SERIAL_LOG="$WORKDIR/serial.log" TARGET_QEMU_PID="" cleanup() { @@ -466,7 +552,7 @@ test-installer-artifact: Type=oneshot TimeoutStartSec=infinity StandardOutput=journal+console - ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' + ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; k0s kubectl get pods -A --no-headers 2>/dev/null | sed "s/^/<4>kiosk-pods: /"; k0s kubectl get nodes -o wide --no-headers 2>/dev/null | sed "s/^/<4>kiosk-node: /"; k0s kubectl get events -A --no-headers 2>/dev/null | grep -v FailedScheduling | tail -n 8 | sed "s/^/<4>kiosk-events: /"; journalctl -u k0scontroller --no-pager -n 800 2>/dev/null | grep -iE "error|fail|taint" | grep -vE "FailedScheduling|apply.*retrying" | tail -n 8 | sed "s/^/<4>kiosk-k0s: /"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' UNIT ) @@ -521,6 +607,20 @@ test-installer-artifact: sleep 2 done +# This is the default mode of test-installer-artifact; the recipe exists so the USB +# boot path is discoverable by name. +# Boot the exported installer media via firmware (OVMF) as an xHCI USB drive. +[group('test')] +test-installer-boot-usb: + INSTALLER_BOOT_MODE=usb just test-installer-artifact + +# `just export-pxe` only checks that the two files exist, so this recipe is the only +# thing that proves they still boot and install. +# Boot the exported PXE kernel/initrd pair directly (QEMU -kernel/-initrd/-append). +[group('test')] +test-installer-boot-pxe: + INSTALLER_BOOT_MODE=pxe just test-installer-artifact + # Interactively install and boot a persistent local KubeStellar kiosk VM. [group('test')] install-vm: @@ -559,16 +659,21 @@ install-vm: if [ ! -f "$OVMF_VARS" ]; then OVMF_TEMPLATE=$(first_existing \ /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-vars.fd \ + /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-i386-vars.fd \ /usr/share/edk2/ovmf/OVMF_VARS.fd \ /usr/share/OVMF/OVMF_VARS.fd \ /usr/share/OVMF/OVMF_VARS_4M.fd \ /usr/share/edk2/x64/OVMF_VARS.4m.fd \ /usr/share/qemu/edk2-x86_64-vars.fd \ + /usr/share/qemu/edk2-i386-vars.fd \ /usr/share/qemu/OVMF_VARS.fd) \ || true - if [ -n "$OVMF_TEMPLATE" ]; then + if [ -n "${OVMF_TEMPLATE}" ]; then cp "$OVMF_TEMPLATE" "$OVMF_VARS" else + # Hosts that ship only OVMF_CODE still boot: firmware initialises a blank + # variable store on first use, it just has no preseeded boot entries. + echo "WARNING: no OVMF_VARS template found; using a blank variable store sized to $OVMF_CODE" >&2 truncate -s "$(stat -c '%s' "$OVMF_CODE")" "$OVMF_VARS" fi fi diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index c4deaf91..57346063 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -137,7 +137,7 @@ uploaded to a GitHub Release tagged `installer-v`. result as its own pull request against `main`. 3. **Full Compilation:** Builds the standalone DDI OS image, live installer, and k0s systemd-sysext on every pull request and push to `main`. -4. **Installer boot test:** Uploads the already-built installer and PXE artifacts and calls `projectbluefin/actions/.github/workflows/server-installer-test.yml`. The shared workflow prepares QEMU/KVM and runs this repository's `just test-installer-artifact`; Lima remains local-only. +4. **Installer boot test:** Uploads the already-built installer and PXE artifacts and calls `projectbluefin/actions/.github/workflows/server-installer-test.yml`. The shared workflow prepares QEMU/KVM and runs this repository's `just test-installer-artifact`, which defaults to the USB/firmware boot path and consumes only the raw installer image; the uploaded PXE pair is exercised by `just test-installer-boot-pxe` (`INSTALLER_BOOT_MODE=pxe`). Lima remains local-only. 5. **Version Derivation:** The release tag is derived with `just version`, which parses the pinned FSDK point release from `elements/freedesktop-sdk.bst`. 6. **Automated Publishing:** For pushes to `main` (including Renovate PR merges), diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 167bd780..8a4b26b7 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -31,9 +31,33 @@ just export-sysext # export sysext artifacts to dist/sysext/ just flash-installer # write the installer image to a USB device just show-me-the-future # end-to-end QEMU installer smoke test just test-installer-artifact # test already-exported artifacts in QEMU without rebuilding +just test-installer-boot-usb # test-installer-artifact in USB/firmware boot mode +just test-installer-boot-pxe # test-installer-artifact in PXE -kernel/-initrd boot mode just tags # show FSDK-derived version tags ``` +`just test-installer-artifact` boots the exported raw installer image through OVMF +firmware as an emulated xHCI USB drive (`-device qemu-xhci` plus `-device usb-storage`), +with the target disk on virtio-blk. This is the bare-metal USB install path: firmware +picks the bootloader off the image itself rather than QEMU injecting a kernel through +`-kernel`/`-initrd`. The boot method is selected by `INSTALLER_BOOT_MODE` (`usb`, the +default, or `pxe`); everything after the install is identical for both. `just +test-installer-boot-usb` and `just test-installer-boot-pxe` are the named entry points. + +Use `pxe` mode to exercise the exported `bluefin-server-pxe-vmlinuz-*` / +`bluefin-server-pxe-initrd-*.cpio.gz` pair, which is booted with `-kernel`/`-initrd` +and an explicit `-append` command line. `just export-pxe` only asserts that those two +files exist, so the PXE recipe is what proves they still boot and install. PXE mode is +also the way to test an unattended install under Secure Boot firmware: in USB mode the +unattended flag travels via `-smbios type=11 io.systemd.stub.kernel-cmdline-extra`, and +systemd-stub ignores that credential when Secure Boot is enabled. + +The recipe prefers a real `OVMF_VARS` template but falls back to a blank variable store +sized to `OVMF_CODE` on hosts that ship CODE only, so a missing template is a warning +rather than a hard failure. It needs `qemu-system-x86_64`, `zstd`, `sfdisk` and `jq` on +the host; the last two are used to assert that the installer actually partitioned the +target disk, and the recipe fails with a named error if either is missing. + ### Payload base `build-ddi` takes an optional `os-base` argument selecting which payload it @@ -144,7 +168,7 @@ The release process is driven by `.github/workflows/build.yml`: - Boot cmdline uses a hardcoded device path like `root=/dev/vda2`. - DDI decompression is placed before the cpio step. - The installer data partition uses FAT32/vfat instead of XFS. -- The repart recipe is missing `GrowFileSystem=yes` for the copied rootfs. +- The repart recipe is missing `GrowFileSystem=no` for the copied rootfs. - Unattended target-disk discovery is not filtered and can select empty devices. ## Verification @@ -159,7 +183,7 @@ The release process is driven by `.github/workflows/build.yml`: - [ ] The live installer does not bake hardcoded SSH keys or pre-hashed root passwords. - [ ] The DDI is decompressed after the cpio step. -- [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=yes`. +- [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=no`. ## See also diff --git a/docs/skills/ddi-installer.md b/docs/skills/ddi-installer.md index 7550e484..4699417a 100644 --- a/docs/skills/ddi-installer.md +++ b/docs/skills/ddi-installer.md @@ -87,9 +87,8 @@ container runtime pod sandboxes. 6. `20-root-a.conf` copies the DDI block-for-block from `/dev/disk/by-partlabel/bluefin-installer-data` (the embedded DDI data partition on the installer media). -7. Target OS volume expansion is handled by `systemd-growfs`; the target OS - stack includes `xfsprogs` so the root and `/var` filesystems can grow to fill - their partitions on first boot. +7. The immutable root DDI is copied without filesystem growth; `/var` uses + `GrowFileSystem=yes` to fill the remaining target disk. ## Partition Layout @@ -231,7 +230,7 @@ in tmpfs, so plan roughly `live-env + DDI` of free RAM (~8 GiB guidance). - [ ] The interactive installer service sets `TTYPath=/dev/tty0` so the TUI appears on the attached display even when serial is the primary console. - [ ] `bluefin-server-installer.bst` decompresses the DDI after the cpio step. -- [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=yes`. +- [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=no`. ## See also diff --git a/elements/bluefin-server/os-resolv-conf.bst b/elements/bluefin-server/os-resolv-conf.bst new file mode 100644 index 00000000..79361cc1 --- /dev/null +++ b/elements/bluefin-server/os-resolv-conf.bst @@ -0,0 +1,14 @@ +kind: import +description: | + Install the tmpfiles.d rule that recreates Flatcar's /etc/resolv.conf + symlink to systemd-resolved's resolv.conf. The Flatcar /usr payload does + not carry it (Flatcar ships the symlink in the image /etc, which this DDI + does not import), and kubelet requires /etc/resolv.conf to create any pod + sandbox. + +sources: + - kind: local + path: files/os/tmpfiles.d + +config: + target: /usr/lib/tmpfiles.d diff --git a/elements/bluefin-server/os-stack.bst b/elements/bluefin-server/os-stack.bst index 44f80576..5aa65158 100644 --- a/elements/bluefin-server/os-stack.bst +++ b/elements/bluefin-server/os-stack.bst @@ -44,6 +44,9 @@ depends: # Wired DHCP configuration for the installed OS - bluefin-server/os-networkd.bst + # /etc/resolv.conf symlink to systemd-resolved (tmpfiles.d) + - bluefin-server/os-resolv-conf.bst + # Activate the optional k0s sysext after installation - bluefin-server/os-k0s-first-boot.bst diff --git a/elements/flatcar/flatcar-usr.bst b/elements/flatcar/flatcar-usr.bst index df8d299a..0c6ca19d 100644 --- a/elements/flatcar/flatcar-usr.bst +++ b/elements/flatcar/flatcar-usr.bst @@ -91,3 +91,9 @@ config: # ensure-sysext.service: replaced by Bluefin's systemd-sysext.service and sysupdate integration rm "%{install-root}/usr/lib/systemd/system/ensure-sysext.service" rm "%{install-root}/usr/lib/systemd/system/sysinit.target.wants/ensure-sysext.service" + + # oem.mount: replaced by Bluefin's standard GPT partition layout (no OEM partition) + rm "%{install-root}/usr/lib/systemd/system/oem.mount" + rm "%{install-root}/usr/lib/systemd/system/local-fs.target.wants/oem.mount" + rm "%{install-root}/usr/lib/systemd/system/dev-disk-by\\x2dlabel-OEM.device" + rm -r "%{install-root}/usr/lib/systemd/system/systemd-fsck@dev-disk-by\\x2dlabel-OEM.service.d" diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index c58f9d8e..bee1ed53 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -41,6 +41,9 @@ config: depmod -b /layer/usr "${KVER}" # Stage extracted CA certificate bundle into target DDI rootfs + # Remote execution may hard-link the staged CA files to the build root; + # replace the target tree before copying rather than copying onto itself. + rm -rf /layer/etc/pki/ca-trust/extracted mkdir -p /layer/etc/pki/ca-trust/extracted /layer/etc/ssl/certs /layer/etc/pki/tls/certs cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/ ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/ssl/certs/ca-certificates.crt @@ -76,9 +79,14 @@ config: # Ensure all standard virtual, serial, and hypervisor consoles allow root login printf 'tty0\ntty1\ntty2\ntty3\ntty4\ntty5\ntty6\nttyS0\nttyS1\nconsole\nhvc0\n' > /layer/etc/securetty - # Seed default root password 'bluefin' into /etc/shadow for console and ssh login + # Seed default root password 'bluefin' for local console bring-up only. + # The last-change field is 0 (equivalent to `chage -d 0 root`), so the + # password is expired at first login and the console forces a per-install + # rotation before any shell is granted. The seeded credential must never + # be usable over the network: the sshd drop-in sets PermitRootLogin + # prohibit-password and PasswordAuthentication no. mkdir -p /layer/etc - printf 'root:$6$bluefin123$o2pQjPN1n3ZCnYUROlA01xAdnGO7mhVRAVw98x4xc8LAUIJv7b44bBNoER6fM4P.rrj4ePYFfkx9WvZICU8nt/:19700:0:99999:7:::\n' > /layer/etc/shadow + printf 'root:$6$bluefin123$o2pQjPN1n3ZCnYUROlA01xAdnGO7mhVRAVw98x4xc8LAUIJv7b44bBNoER6fM4P.rrj4ePYFfkx9WvZICU8nt/:0:0:99999:7:::\n' > /layer/etc/shadow chmod 0600 /layer/etc/shadow printf 'root:x:0:0:root:/root:/bin/bash\n' > /layer/etc/passwd chmod 0644 /layer/etc/passwd @@ -95,9 +103,8 @@ config: ROOT_BYTES=$(du -sb /layer | cut -f1) echo "Computed root filesystem size: ${ROOT_BYTES} bytes" - # 25% overhead and 100MB margin for XFS metadata, journal, and structures. - # ponytail: 25% overhead + 100MB margin for XFS. - TARGET_BYTES=$(( ROOT_BYTES + ROOT_BYTES / 4 + 100 * 1024 * 1024 )) + # 30% overhead and 250MB margin for XFS metadata, journal, and structures. + TARGET_BYTES=$(( ROOT_BYTES + ROOT_BYTES * 3 / 10 + 250 * 1024 * 1024 )) # Align TARGET_BYTES to 4096-byte boundary (multiple of 512/4096) for sector compatibility # ponytail: sector alignment is required by systemd-repart CopyBlocks=. @@ -108,7 +115,7 @@ config: truncate -s "${TARGET_BYTES}" "${OUT}/${FNAME}" echo "Allocating XFS image of ${TARGET_BYTES} bytes" - mkfs.xfs -f -L bluefin-root -p /layer "${OUT}/${FNAME}" + mkfs.xfs -f -L bluefin-root -m reflink=0,rmapbt=0 -n parent=0 -i exchange=0 -p /layer "${OUT}/${FNAME}" zstd --rm -T0 -19 -q "${OUT}/${FNAME}" -o "${OUT}/${FNAME}.zst" ( diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 5999abe2..9a6a84ff 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -188,6 +188,25 @@ config: #!/bin/bash set -euo pipefail + CMDLINE="$(< /proc/cmdline)" + + # The service drop-in uses StandardOutput=journal+console so unattended runs + # land on the serial console that automated tests capture. /dev/console follows + # the last console= argument, which is ttyS0, so an interactive bare-metal + # install would otherwise print to serial rather than the attached display. + # Keep the interactive installer on the attached display by redirecting this + # script's own output to /dev/tty0 — but only when the kernel actually drives + # tty0 as a console. On serial-only machines the VT device still exists and is + # writable, so a -w test alone would send the whole install to a display nobody + # is watching; /sys/class/tty/console/active lists the consoles the kernel + # really registered, and omits tty0 when the boot was serial-only. If that file + # is unreadable the redirect is skipped and output stays on journal+console. + if [[ " ${CMDLINE} " != *" unattended "* ]] \ + && [ -w /dev/tty0 ] \ + && grep -qw tty0 /sys/class/tty/console/active 2>/dev/null; then + exec > /dev/tty0 2>&1 + fi + # Ensure storage controller modules and filesystems are loaded before probing devices modprobe -q nvme || true modprobe -q nvme_core || true @@ -261,13 +280,27 @@ config: > /usr/lib/repart.sysinstall.d/20-root-a.conf fi + INSTALLER_PART="" # Hard preflight: the embedded installer-media path needs the # bluefin-installer-data partition. Under PXE (inst.ddi_url=) there is no # installer media, so the check is skipped when inst.ddi_url is set. if [ -z "${DDI_URL}" ]; then + # Hard preflight: wait up to 30s for storage settle and installer data partition to become available INSTALLER_PART_PATH="/dev/disk/by-partlabel/bluefin-installer-data" + DEADLINE=$((SECONDS + 30)) + udevadm trigger --subsystem-match=block || true + udevadm settle --timeout=5 || true + while [ ! -b "${INSTALLER_PART_PATH}" ] && [ "${SECONDS}" -lt "${DEADLINE}" ]; do + sleep 1 + udevadm trigger --subsystem-match=block || true + REMAINING=$((DEADLINE - SECONDS)) + [ "${REMAINING}" -gt 0 ] || break + SETTLE_TIMEOUT=$(( REMAINING < 5 ? REMAINING : 5 )) + udevadm settle --timeout="${SETTLE_TIMEOUT}" || true + done + if [ ! -b "${INSTALLER_PART_PATH}" ]; then - echo "ERROR: Installer data partition '${INSTALLER_PART_PATH}' is missing or not a block device!" >&2 + echo "ERROR: Installer data partition '${INSTALLER_PART_PATH}' is missing or not a block device after 30s deadline!" >&2 echo "==> Block devices visible at failure:" >&2 lsblk -p -o NAME,TYPE,PARTLABEL,PKNAME,SIZE,FSTYPE >&2 || true echo "==> Partition label symlinks visible under /dev/disk/by-partlabel/:" >&2 @@ -275,10 +308,10 @@ config: exit 1 fi INSTALLER_PART="$(readlink -f "${INSTALLER_PART_PATH}")" + echo "==> Found installer partition at ${INSTALLER_PART_PATH} -> ${INSTALLER_PART}" fi - # Auto-detect target disk for unattended install (any raw disk of type "disk" that is not the installer, is not read-only, and has size > 0) - TARGET_DISK="" + TARGET_DISKS=() while read -r name type ro size; do [ "${type}" = "disk" ] || continue [ "${ro}" = "0" ] || continue @@ -289,9 +322,9 @@ config: continue fi - TARGET_DISK="${name}" - break + TARGET_DISKS+=("${name}") done < <(lsblk -p -d -n -o NAME,TYPE,RO,SIZE -b) + TARGET_DISK="${TARGET_DISKS[0]:-}" # inst.target_disk= overrides auto-detection (explicit target under PXE). if [ -n "${TARGET_DISK_OVERRIDE}" ]; then @@ -343,6 +376,7 @@ config: fi } + # Check for unattended in kernel command line if [[ " ${CMDLINE} " == *" unattended "* ]]; then echo "==> Running in UNATTENDED mode..." if [ -n "${TARGET_DISK}" ]; then @@ -363,6 +397,17 @@ config: fi else echo "==> Running in INTERACTIVE mode..." + # If exactly one non-installer target disk exists, pass it directly to systemd-sysinstall + # so the user isn't prompted with device serial/path queries. + # If multiple disks exist, systemd-sysinstall prompts with its native numbered list; + # print a hint reminding users they can simply type the item number (1, 2, ...). + EXTRA_ARGS=() + if [ "${#TARGET_DISKS[@]}" -eq 1 ]; then + echo "==> Automatically selected only available installation disk: ${TARGET_DISKS[0]}" + EXTRA_ARGS+=("${TARGET_DISKS[0]}") + elif [ "${#TARGET_DISKS[@]}" -gt 1 ]; then + echo "==> Multiple disks detected. You can type the menu item number (1, 2, ...) to choose." + fi /usr/bin/systemd-sysinstall \ --kernel=/usr/lib/bluefin-server/bluefin-server.efi \ --variables=yes \ @@ -370,7 +415,8 @@ config: --mute-console=yes \ --copy-locale=yes \ --copy-keymap=yes \ - --copy-timezone=yes + --copy-timezone=yes \ + "${EXTRA_ARGS[@]}" stage_fallback_bootloader echo "==> Installation complete! Rebooting..." systemctl reboot @@ -387,8 +433,13 @@ config: FailureAction=poweroff [Service] - # Keep the interactive installer on the attached display. The kernel - # still uses the final serial console for boot diagnostics. + # Mirror installer service output to journal+console so progress and diagnostics + # are captured on the serial console in headless/automated test runs. TTYPath is + # only honoured for Standard*=tty, so it does not steer this output; the wrapper + # keeps the interactive installer on the attached display by redirecting itself + # to /dev/tty0 when 'unattended' is absent from the kernel command line. + StandardOutput=journal+console + StandardError=journal+console TTYPath=/dev/tty0 ExecStart= ExecStart=/usr/bin/bluefin-sysinstall diff --git a/elements/oci/k0s-sysext.bst b/elements/oci/k0s-sysext.bst index dc5bcdbc..658f7f93 100644 --- a/elements/oci/k0s-sysext.bst +++ b/elements/oci/k0s-sysext.bst @@ -89,6 +89,7 @@ config: mkdir -p sysext/usr/share/k0s/kubeflex cp -a kubeflex-src/. sysext/usr/share/k0s/kubeflex/ chmod 0755 sysext/usr/share/k0s/kubeflex/generate-postgres-secret.sh + chmod 0755 sysext/usr/share/k0s/kubeflex/generate-console-secret.sh # Generate self-signed TLS certificate for local kiosk proxy openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \ diff --git a/files/installer/repart.d/20-root-a.conf b/files/installer/repart.d/20-root-a.conf index e843b30a..94072ccc 100644 --- a/files/installer/repart.d/20-root-a.conf +++ b/files/installer/repart.d/20-root-a.conf @@ -8,4 +8,4 @@ Label=bluefin-server-root-a CopyBlocks=/dev/disk/by-partlabel/bluefin-installer-data SizeMinBytes=4G SizeMaxBytes=8G -GrowFileSystem=yes +GrowFileSystem=no diff --git a/files/k0s/kubeflex/generate-console-secret.sh b/files/k0s/kubeflex/generate-console-secret.sh new file mode 100755 index 00000000..9798a4fa --- /dev/null +++ b/files/k0s/kubeflex/generate-console-secret.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# First-boot generator for the KubeStellar console session-signing secret. +# +# The console Deployment (40-kubestellar-console.yaml) reads JWT_SECRET from +# the kubestellar-console-github-oauth Secret via secretKeyRef. This script +# creates that Secret once, with a random jwt-secret, so no usable signing key +# is committed to git (see projectbluefin/server#72). The GitHub OAuth keys +# are created empty: the console consumes them with `optional: true`, and an +# operator enabling OAuth edits the generated file with real credentials. +# +# Idempotent: the secret is generated only if the file does not already exist, +# so operator-supplied OAuth credentials and issued sessions survive re-boots. +# Run once per boot by k0s-first-boot.service, before k0s applies the +# manifests (its 02- name sorts before 40-kubestellar-console.yaml). +set -euo pipefail + +manifest_dir="${KUBESTELLAR_MANIFEST_DIR:-/var/lib/k0s/manifests/kubestellar}" +secret_file="$manifest_dir/02-kubestellar-console-secret.yaml" + +if [ -e "$secret_file" ]; then + exit 0 +fi + +mkdir -p "$manifest_dir" +jwt_secret="$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n')" + +cat > "$secret_file" <@ diff --git a/files/os/systemd/system-preset/zz-enable-bluefin-firstboot-credentials.preset b/files/os/systemd/system-preset/20-bluefin-firstboot-credentials.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-bluefin-firstboot-credentials.preset rename to files/os/systemd/system-preset/20-bluefin-firstboot-credentials.preset diff --git a/files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset b/files/os/systemd/system-preset/20-bluefin-k0s-first-boot.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset rename to files/os/systemd/system-preset/20-bluefin-k0s-first-boot.preset diff --git a/files/os/systemd/system-preset/zz-enable-networkd.preset b/files/os/systemd/system-preset/20-bluefin-networkd.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-networkd.preset rename to files/os/systemd/system-preset/20-bluefin-networkd.preset diff --git a/files/os/systemd/system-preset/zz-enable-sshd.preset b/files/os/systemd/system-preset/20-bluefin-sshd.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-sshd.preset rename to files/os/systemd/system-preset/20-bluefin-sshd.preset diff --git a/files/os/systemd/system-preset/zz-enable-systemd-network-generator.preset b/files/os/systemd/system-preset/20-bluefin-systemd-network-generator.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-systemd-network-generator.preset rename to files/os/systemd/system-preset/20-bluefin-systemd-network-generator.preset diff --git a/files/os/systemd/system-preset/zz-enable-var-mount.preset b/files/os/systemd/system-preset/20-bluefin-var-mount.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-var-mount.preset rename to files/os/systemd/system-preset/20-bluefin-var-mount.preset diff --git a/files/os/systemd/system/k0s-first-boot.service b/files/os/systemd/system/k0s-first-boot.service index 6e61f470..97493654 100644 --- a/files/os/systemd/system/k0s-first-boot.service +++ b/files/os/systemd/system/k0s-first-boot.service @@ -16,6 +16,7 @@ ExecStart=/usr/bin/install -D -m 0644 /var/lib/k0s/k0s.raw /run/extensions/k0s.r ExecStart=/usr/bin/systemd-sysext merge ExecStart=/usr/bin/systemd-tmpfiles --create /usr/lib/tmpfiles.d/k0s-manifests.conf ExecStart=/usr/share/k0s/kubeflex/generate-postgres-secret.sh +ExecStart=/usr/share/k0s/kubeflex/generate-console-secret.sh ExecStart=/usr/bin/systemctl daemon-reload ExecStart=/usr/bin/systemctl enable --now k0scontroller.service [Install] diff --git a/files/os/tmpfiles.d/20-bluefin-resolv.conf b/files/os/tmpfiles.d/20-bluefin-resolv.conf new file mode 100644 index 00000000..085ac2ab --- /dev/null +++ b/files/os/tmpfiles.d/20-bluefin-resolv.conf @@ -0,0 +1,11 @@ +# Flatcar's image build populates /etc/resolv.conf in the image's /etc and +# strips /etc-populating lines from /usr/lib/tmpfiles.d (see +# flatcar/scripts build_library/build_image_util.sh). This DDI imports only +# Flatcar's /usr and boots with an empty /etc, so nothing ever creates the +# symlink and kubelet fails every pod sandbox with +# "open /etc/resolv.conf: no such file or directory". +# +# Recreate Flatcar's stock symlink: /etc/resolv.conf points at +# systemd-resolved's full resolver list (not the 127.0.0.53 stub), matching +# Flatcar >= 3510.3.0 behaviour. +L /etc/resolv.conf - - - - ../run/systemd/resolve/resolv.conf diff --git a/tests/unit/test-installer-artifact_test.bats b/tests/unit/test-installer-artifact_test.bats index 173a7925..34398fd3 100755 --- a/tests/unit/test-installer-artifact_test.bats +++ b/tests/unit/test-installer-artifact_test.bats @@ -86,6 +86,7 @@ setup() { make_zstd_stub make_qemu_stub make_curl_stub + make_sfdisk_stub } # The OVMF_CODE lookup (`Justfile:254`, duplicated verbatim at the same @@ -135,6 +136,35 @@ exit 0 EOF chmod +x "${STUB_DIR}/zstd" } +make_sfdisk_stub() { + cat > "${STUB_DIR}/sfdisk" <> "${LOG}" +for ((i = 1; i <= \$#; i++)); do + arg="\${!i}" + if [ "\$arg" = "--json" ]; then + cat <<'JSON' +{ + "partitiontable": { + "label": "gpt", + "id": "12345678-1234-1234-1234-123456789abc", + "device": "target.raw", + "unit": "sectors", + "partitions": [ + {"node": "target.raw1", "start": 2048, "size": 204800, "type": "C12A7328-F81F-11D2-BA4B-00A0C93EC93B", "name": "ESP"}, + {"node": "target.raw2", "start": 206848, "size": 8388608, "type": "4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709", "name": "bluefin-server-root-a"}, + {"node": "target.raw3", "start": 8595456, "size": 8388608, "type": "4D21B016-B534-45C2-A9FB-5C16E091FD2D", "name": "var"} + ] + } +} +JSON + exit 0 + fi +done +exit 0 +EOF + chmod +x "${STUB_DIR}/sfdisk" +} # make_qemu_stub # diff --git a/tests/unit/test_credentials_provisioning.py b/tests/unit/test_credentials_provisioning.py index f9c36654..852207e1 100644 --- a/tests/unit/test_credentials_provisioning.py +++ b/tests/unit/test_credentials_provisioning.py @@ -24,7 +24,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-bluefin-firstboot-credentials.preset" + / "20-bluefin-firstboot-credentials.preset" ) NETWORK_GENERATOR_PRESET = ( REPO_ROOT @@ -32,7 +32,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-systemd-network-generator.preset" + / "20-bluefin-systemd-network-generator.preset" ) NETWORK_GENERATOR_DROPIN = ( REPO_ROOT diff --git a/tests/unit/test_flatcar_usr.py b/tests/unit/test_flatcar_usr.py index b7d2f70a..b84e2f43 100644 --- a/tests/unit/test_flatcar_usr.py +++ b/tests/unit/test_flatcar_usr.py @@ -50,6 +50,10 @@ "usr/lib/systemd/system/oem-cloudinit.service", "usr/lib/systemd/system/ensure-sysext.service", "usr/lib/systemd/system/sysinit.target.wants/ensure-sysext.service", + "usr/lib/systemd/system/oem.mount", + "usr/lib/systemd/system/local-fs.target.wants/oem.mount", + "usr/lib/systemd/system/dev-disk-by\\x2dlabel-OEM.device", + "usr/lib/systemd/system/systemd-fsck@dev-disk-by\\x2dlabel-OEM.service.d", ] REMOVED_EXTRA_FILES = [ @@ -143,6 +147,7 @@ def test_flatcar_usr_explicit_removals_and_replacements() -> None: "flatcar-update", "download_sysext", "ensure-sysext", + "oem.mount", ]: pattern = rf"#\s*{component}.*?:.*?replaced by" assert re.search(pattern, content, re.IGNORECASE), ( @@ -224,6 +229,10 @@ def test_flatcar_usr_contract_execution(tmp_path: Path) -> None: (systemd_dir / "enable-oem-cloudinit.service").write_text("mock-unit", encoding="utf-8") (systemd_dir / "oem-cloudinit.service").write_text("mock-unit", encoding="utf-8") (systemd_dir / "ensure-sysext.service").write_text("mock-unit", encoding="utf-8") + (systemd_dir / "oem.mount").write_text("mock-unit", encoding="utf-8") + (systemd_dir / "dev-disk-by\\x2dlabel-OEM.device").write_text("mock-unit", encoding="utf-8") + (systemd_dir / "systemd-fsck@dev-disk-by\\x2dlabel-OEM.service.d").mkdir(parents=True) + (systemd_dir / "systemd-fsck@dev-disk-by\\x2dlabel-OEM.service.d" / "10-mock.conf").write_text("mock", encoding="utf-8") wants_dir1 = systemd_dir / "multi-user.target.wants" wants_dir1.mkdir(parents=True) @@ -232,6 +241,9 @@ def test_flatcar_usr_contract_execution(tmp_path: Path) -> None: (wants_dir1 / "locksmithd.service").symlink_to("../locksmithd.service") (wants_dir1 / "enable-oem-cloudinit.service").symlink_to("../enable-oem-cloudinit.service") + local_fs_dir = systemd_dir / "local-fs.target.wants" + local_fs_dir.mkdir(parents=True) + (local_fs_dir / "oem.mount").symlink_to("../oem.mount") wants_dir2 = systemd_dir / "sysinit.target.wants" wants_dir2.mkdir(parents=True) (wants_dir2 / "ignition-delete-config.service").symlink_to("../ignition-delete-config.service") diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 114df0b3..42cd524f 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -50,6 +50,13 @@ def test_installer_runtime_and_boot_contracts() -> None: assert "console=ttyS0,115200 rw" in installer_element assert "unattended" not in published_uki_cmdline assert target_uki_cmdline == "rw console=ttyS0,115200 console=tty0 quiet loglevel=3 audit=0" + assert "test-installer-artifact:" in justfile + assert "-device qemu-xhci,id=xhci" in justfile + assert "-device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1" in justfile + assert "-device virtio-blk-pci,drive=target-disk,bootindex=2" in justfile + assert 'sfdisk --json "$WORKDIR/target.raw"' in justfile + assert "bluefin-server-root-a" in justfile + # The PXE pair is still exported, so the direct-kernel boot path stays covered. assert ( '-append "systemd.unit=system-install.target ' 'console=tty0 console=ttyS0,115200 rw unattended"' @@ -94,7 +101,6 @@ def test_ddi_generates_module_indexes_for_runtime_filesystem_drivers() -> None: assert "freedesktop-sdk.bst:components/kmod.bst" in ddi_element assert 'depmod -b /layer/usr "${KVER}"' in ddi_element - assert "cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/" in ddi_element assert "tls-ca-bundle.pem" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service" in ddi_element @@ -119,7 +125,7 @@ def test_installer_loads_storage_drivers_and_settles_udev() -> None: assert "modprobe -q nvme_core || true" in installer_element assert "modprobe -q usb-storage || true" in installer_element assert "modprobe -q uas || true" in installer_element - assert "udevadm settle --timeout=15 || true" in installer_element + assert 'udevadm settle --timeout="${SETTLE_TIMEOUT}" || true' in installer_element assert "After=systemd-udev-settle.service" in installer_element assert "Wants=systemd-udev-settle.service" in installer_element @@ -127,14 +133,29 @@ def test_installer_loads_storage_drivers_and_settles_udev() -> None: def test_installer_hard_preflight_aborts_on_missing_installer_data_part() -> None: installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") + assert 'DEADLINE=$((SECONDS + 30))' in installer_element + assert 'while [ ! -b "${INSTALLER_PART_PATH}" ] && [ "${SECONDS}" -lt "${DEADLINE}" ]; do' in installer_element assert '[ ! -b "${INSTALLER_PART_PATH}" ]' in installer_element assert "/dev/disk/by-partlabel/bluefin-installer-data" in installer_element assert "lsblk -p -o NAME,TYPE,PARTLABEL,PKNAME,SIZE,FSTYPE" in installer_element + def test_interactive_installer_uses_local_virtual_console() -> None: installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") assert "TTYPath=/dev/tty0" in installer_element + assert "StandardOutput=journal+console" in installer_element + assert "StandardError=journal+console" in installer_element + # journal+console follows the last console= argument (ttyS0), so the wrapper has + # to put interactive runs back on the attached display itself — but only when the + # kernel actually registered tty0 as a console, otherwise serial-only machines + # lose the install output to a VT nobody is watching. + assert ( + 'if [[ " ${CMDLINE} " != *" unattended "* ]] \\\n' + " && [ -w /dev/tty0 ] \\\n" + " && grep -qw tty0 /sys/class/tty/console/active 2>/dev/null; then\n" + " exec > /dev/tty0 2>&1" + ) in installer_element def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: @@ -208,3 +229,22 @@ def test_installer_smoke_probes_the_kiosk_over_tls_from_inside_the_guest() -> No assert "http://127.0.0.1:8080/healthz" not in smoke assert "KIOSK_CONSOLE_READY" in smoke + +def test_test_installer_boot_usb_contract() -> None: + justfile = JUSTFILE.read_text(encoding="utf-8") + start = justfile.index("test-installer-boot-usb:") + end = justfile.index("test-installer-boot-pxe:", start) + recipe = justfile[start:end] + assert "INSTALLER_BOOT_MODE=usb just test-installer-artifact" in recipe + + +def test_test_installer_boot_pxe_recipe_exercises_direct_kernel_boot() -> None: + justfile = JUSTFILE.read_text(encoding="utf-8") + start = justfile.index("test-installer-boot-pxe:") + end = justfile.index("install-vm:", start) + recipe = justfile[start:end] + assert "INSTALLER_BOOT_MODE=pxe just test-installer-artifact" in recipe + assert '-kernel "$WORKDIR/installer.vmlinuz"' in justfile + assert '-initrd "$WORKDIR/installer.initrd"' in justfile + assert "bluefin-server-pxe-vmlinuz-*" in justfile + assert "bluefin-server-pxe-initrd-*.cpio.gz" in justfile diff --git a/tests/unit/test_k0s_first_boot.py b/tests/unit/test_k0s_first_boot.py index a185a611..1f89861e 100644 --- a/tests/unit/test_k0s_first_boot.py +++ b/tests/unit/test_k0s_first_boot.py @@ -20,7 +20,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-k0s-first-boot.preset" + / "20-bluefin-k0s-first-boot.preset" ) NETWORK = ROOT / "files" / "os" / "systemd" / "network" / "20-wired.network" NETWORK_PRESET = ( @@ -29,7 +29,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-networkd.preset" + / "20-bluefin-networkd.preset" ) ELEMENT = ROOT / "elements" / "bluefin-server" / "os-k0s-first-boot.bst" NETWORK_ELEMENT = ROOT / "elements" / "bluefin-server" / "os-networkd.bst" @@ -146,3 +146,49 @@ def test_k0s_sysupdate_transfer_is_packaged_as_a_component() -> None: "bluefin-server/os-k0s-sysupdate.bst" in STACK.read_text(encoding="utf-8") ) + + +def test_presets_sort_before_flatcar_disable_all() -> None: + # Flatcar's /usr ships /usr/lib/systemd/system-preset/99-default.preset + # containing a catch-all "disable *". systemd applies preset files in + # lexicographic filename order and the first matching line wins, so any + # Bluefin preset named after 99-default.preset is silently ignored at + # first-boot preset-all. + preset_dir = PRESET.parent + presets = sorted(p.name for p in preset_dir.glob("*.preset")) + assert presets, f"no presets found in {preset_dir}" + for name in presets: + assert name < "99-default.preset", ( + f"{name} sorts after Flatcar's 99-default.preset; its " + "'disable *' catch-all would win and the preset would be ignored" + ) + + +def test_etc_resolv_conf_symlink_is_seeded_for_kubelet() -> None: + # Flatcar ships /etc/resolv.conf inside the image /etc and strips + # /etc-populating lines from /usr/lib/tmpfiles.d at image build. This DDI + # imports only Flatcar's /usr and first-boots with an empty /etc, so + # without an explicit tmpfiles.d rule the symlink never exists and + # kubelet fails every pod sandbox with + # "open /etc/resolv.conf: no such file or directory". + conf = ( + ROOT / "files" / "os" / "tmpfiles.d" / "20-bluefin-resolv.conf" + ) + assert conf.is_file(), "resolv.conf tmpfiles rule is missing" + lines = [ + line + for line in conf.read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") + ] + assert lines == [ + "L /etc/resolv.conf - - - - ../run/systemd/resolve/resolv.conf" + ] + + element = ( + ROOT / "elements" / "bluefin-server" / "os-resolv-conf.bst" + ).read_text(encoding="utf-8") + assert "path: files/os/tmpfiles.d" in element + assert "target: /usr/lib/tmpfiles.d" in element + assert ( + "bluefin-server/os-resolv-conf.bst" in STACK.read_text(encoding="utf-8") + ) diff --git a/tests/unit/test_k0s_manifests.py b/tests/unit/test_k0s_manifests.py index 36d65c73..51b3e799 100644 --- a/tests/unit/test_k0s_manifests.py +++ b/tests/unit/test_k0s_manifests.py @@ -91,3 +91,53 @@ def test_generate_postgres_secret_is_idempotent(tmp_path): first = secret_file.read_text() run() assert secret_file.read_text() == first, "password changed on re-run; DB would lose access" + + +def test_console_jwt_secret_not_hardcoded(): + # #72: the console JWT signing secret must not be committed to git. The + # 01- manifest only creates the namespace; the Secret itself is generated + # at first boot with a random jwt-secret. + manifest = ROOT / "files" / "k0s" / "manifests" / "kubestellar" / "01-kubestellar-console-github-oauth.yaml" + text = manifest.read_text() + assert "kind: Secret" not in text + assert "jwt-secret" not in yaml_values(text) + docs = [d for d in yaml.safe_load_all(text) if d] + assert [d["kind"] for d in docs] == ["Namespace"] + + +def yaml_values(text): + # Collapse to non-comment lines so doc comments may still mention keys. + return "\n".join(l for l in text.splitlines() if not l.lstrip().startswith("#")) + + +def test_k0s_first_boot_generates_console_secret_before_k0s(): + unit = ROOT / "files" / "os" / "systemd" / "system" / "k0s-first-boot.service" + lines = [l for l in unit.read_text().splitlines() if l.startswith("ExecStart")] + gen = next((i for i, l in enumerate(lines) if "generate-console-secret.sh" in l), None) + k0s = next((i for i, l in enumerate(lines) if "k0scontroller.service" in l), None) + assert gen is not None, "first-boot service never runs the console secret generator" + assert k0s is not None + assert gen < k0s, "console secret generator must run before k0s applies manifests" + + +def test_generate_console_secret_is_idempotent(tmp_path): + # Re-running must not rotate the jwt-secret (sessions would break) and + # must not clobber operator-supplied OAuth credentials. + script = ROOT / "files" / "k0s" / "kubeflex" / "generate-console-secret.sh" + env = dict(os.environ, KUBESTELLAR_MANIFEST_DIR=str(tmp_path)) + run = lambda: subprocess.run(["/bin/bash", str(script)], env=env, check=True, capture_output=True, text=True) + run() + secret_file = tmp_path / "02-kubestellar-console-secret.yaml" + assert secret_file.is_file() + assert (secret_file.stat().st_mode & 0o777) == 0o600 + secret = yaml.safe_load(secret_file.read_text()) + assert secret["kind"] == "Secret" + assert secret["metadata"]["name"] == "kubestellar-console-github-oauth" + assert secret["metadata"]["namespace"] == "kubestellar-console" + jwt = secret["stringData"]["jwt-secret"] + assert len(jwt) == 64 and all(c in "0123456789abcdef" for c in jwt) + assert secret["stringData"]["client-id"] == "" + assert secret["stringData"]["client-secret"] == "" + first = secret_file.read_text() + run() + assert secret_file.read_text() == first, "jwt-secret changed on re-run" diff --git a/tests/unit/test_repart_layout.py b/tests/unit/test_repart_layout.py index e21e5b46..48684707 100644 --- a/tests/unit/test_repart_layout.py +++ b/tests/unit/test_repart_layout.py @@ -171,11 +171,11 @@ def test_root_slot_copies_blocks_from_a_label_the_installer_media_stamps(): ) -def test_root_slot_grows_and_is_bounded_below_the_var_partition(): +def test_root_slot_is_bounded_below_the_var_partition(): root = next(s for s in partitions().values() if s["Type"] == "root") - assert root.get("GrowFileSystem") == "yes", ( - "the root filesystem must grow to its partition, the DDI payload is " - "smaller than SizeMinBytes" + assert root.get("GrowFileSystem") == "no", ( + "the immutable DDI payload must not auto-grow at boot, which risks " + "interrupted first-boot metadata corruption" ) assert "SizeMaxBytes" in root, ( "the root slot must be capped, otherwise /var gets no space on small disks" diff --git a/tests/unit/test_root_credential_contract.py b/tests/unit/test_root_credential_contract.py new file mode 100644 index 00000000..8ee2ba95 --- /dev/null +++ b/tests/unit/test_root_credential_contract.py @@ -0,0 +1,71 @@ +"""Contracts for the seeded root credential and SSH login policy. + +The shipped DDI seeds a well-known default root password for local console +bring-up only. This file pins the two safety contracts around it: + +- the password is expired at first login (shadow last-change field 0, the + `chage -d 0` equivalent), so every install is forced onto a per-install + secret before any shell is granted; and +- the effective SSH policy never allows the seeded password (or any password) + over the network. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +DDI_ELEMENT = REPO_ROOT / "elements" / "oci" / "bluefin-server-ddi.bst" +SSHD_DROPIN = REPO_ROOT / "files" / "os" / "ssh" / "sshd_config.d" / "bluefin-server.conf" +ISSUE_FILE = REPO_ROOT / "files" / "os" / "issue.d" / "40-kubestellar.issue" + + +def _root_shadow_fields(ddi_element: str) -> list[str]: + match = re.search( + r"printf '(root:[^']*)", ddi_element + ) + assert match, "DDI element must seed a root account into /etc/shadow" + shadow_line = match.group(1).split("\\n")[0] + fields = shadow_line.split(":") + assert fields[0] == "root", "seeded shadow line must be the root account" + assert len(fields) == 9, "seeded shadow line must have all 9 shadow fields" + return fields + + +def test_root_password_is_expired_for_first_login_rotation() -> None: + ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") + fields = _root_shadow_fields(ddi_element) + + # shadow field 2 is the last-change date: 0 means expired, so login + # (console included) forces a password rotation before any shell. + assert fields[2] == "0", ( + "seeded root password must be expired at first login (last-change 0); " + "a fixed date leaves the well-known default usable indefinitely" + ) + + +def test_root_password_uses_no_fixed_build_date() -> None: + ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") + fields = _root_shadow_fields(ddi_element) + + # Regression guard: a fixed last-change date (e.g. 19700) ships the same + # usable credential in every image with no rotation enforced. + assert fields[2] != "19700" + + +def test_sshd_dropin_never_permits_password_authentication() -> None: + sshd = SSHD_DROPIN.read_text(encoding="utf-8") + + assert re.search(r"^PermitRootLogin\s+prohibit-password$", sshd, re.MULTILINE) + assert re.search(r"^PasswordAuthentication\s+no$", sshd, re.MULTILINE) + assert re.search(r"^KbdInteractiveAuthentication\s+no$", sshd, re.MULTILINE) + assert not re.search(r"^PermitRootLogin\s+yes$", sshd, re.MULTILINE) + assert not re.search(r"^PasswordAuthentication\s+yes$", sshd, re.MULTILINE) + + +def test_console_banner_discloses_forced_password_change() -> None: + banner = ISSUE_FILE.read_text(encoding="utf-8") + + assert "Default login: root / bluefin" in banner + assert "password change forced at first login" in banner