From f1e17eff2da1923e2c22c7af697593e19ce0d90a Mon Sep 17 00:00:00 2001 From: Jorge Castro Date: Mon, 21 Sep 2026 10:38:06 -0400 Subject: [PATCH 01/15] test(installer): add test-installer-boot-usb recipe for bare-metal USB OVMF testing Attach installer media via emulated xHCI USB drive in QEMU and boot via OVMF firmware. Verify target GPT partition table using sfdisk --json and jq for bluefin-server-root-a and var partitions. Add contract unit tests. Assisted-by: github-copilot/gemini-3.8-flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 88 +++++++++++++++++++++++++++ tests/unit/test_installer_contract.py | 9 +++ 2 files changed, 97 insertions(+) diff --git a/Justfile b/Justfile index 724c30ce..d1698699 100644 --- a/Justfile +++ b/Justfile @@ -393,6 +393,94 @@ test-installer-artifact: sleep 2 done + +# Boot the exported installer media via firmware (OVMF) attached as an xHCI USB drive. +# Verifies UEFI boot and bare-metal USB storage driver discovery into target disk. +[group('test')] +test-installer-boot-usb: + #!/usr/bin/env bash + set -euo pipefail + + CACHE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}" + mkdir -p "$CACHE_DIR" + WORKDIR="$(mktemp -d "${CACHE_DIR}/bluefin-usb-boot.XXXXXX")" + trap 'rm -rf "$WORKDIR"' EXIT + + INSTALLER_ARCHIVE=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | sort -V | tail -n1) + [ -n "$INSTALLER_ARCHIVE" ] || { echo "ERROR: No exported installer found in dist/." >&2; exit 1; } + zstd -d "$INSTALLER_ARCHIVE" -o "$WORKDIR/installer.raw" + + TARGET_SIZE="${SHOW_ME_THE_FUTURE_DISK_SIZE:-16G}" + truncate -s "${TARGET_SIZE}" "$WORKDIR/target.raw" + + first_existing() { + for candidate in "$@"; do + if [ -f "$candidate" ]; then + echo "$candidate" + return 0 + fi + done + return 1 + } + + OVMF_CODE=$(first_existing \ + /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-code.fd \ + /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-secure-code.fd \ + /usr/share/edk2/ovmf/OVMF_CODE.fd \ + /usr/share/OVMF/OVMF_CODE.fd \ + /usr/share/OVMF/OVMF_CODE_4M.fd \ + /usr/share/edk2/x64/OVMF_CODE.4m.fd \ + /usr/share/qemu/edk2-x86_64-code.fd \ + /usr/share/qemu/edk2-x86_64-secure-code.fd \ + /usr/share/qemu/OVMF_CODE.fd) \ + || { echo "ERROR: OVMF_CODE not found"; exit 1; } + + OVMF_VARS=$(first_existing \ + /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-vars.fd \ + /usr/share/edk2/ovmf/OVMF_VARS.fd \ + /usr/share/OVMF/OVMF_VARS.fd \ + /usr/share/OVMF/OVMF_VARS_4M.fd \ + /usr/share/edk2/x64/OVMF_VARS.4m.fd \ + /usr/share/qemu/edk2-x86_64-vars.fd \ + /usr/share/qemu/OVMF_VARS.fd) \ + || true + if [ -n "$OVMF_VARS" ]; then + cp "$OVMF_VARS" "$WORKDIR/ovmf-vars.fd" + else + truncate -s "$(stat -c '%s' "$OVMF_CODE")" "$WORKDIR/ovmf-vars.fd" + fi + + SMP_CPUS="${SHOW_ME_THE_FUTURE_SMP:-$(nproc)}" + MEM_SIZE="${SHOW_ME_THE_FUTURE_MEM:-8192}" + + echo "==> Booting installer media via OVMF over emulated xHCI USB in QEMU..." + qemu-system-x86_64 \ + -enable-kvm \ + -m "${MEM_SIZE}" \ + -cpu host \ + -smp "${SMP_CPUS}" \ + -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ + -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ + -device qemu-xhci,id=xhci \ + -drive file="$WORKDIR/installer.raw",format=raw,if=none,id=installer-disk,readonly=on \ + -device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1 \ + -drive file="$WORKDIR/target.raw",format=raw,if=virtio,id=target-disk,bootindex=2 \ + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 rw unattended" \ + -nographic \ + -serial mon:stdio \ + -no-reboot < /dev/null + + echo "==> Validating target disk partitions after USB installation..." + TARGET_PARTS=$(sfdisk --json "$WORKDIR/target.raw" 2>/dev/null | jq -r '.partitiontable.partitions[]?.name // empty' || true) + if ! echo "$TARGET_PARTS" | grep -Fxq 'bluefin-server-root-a' || \ + ! echo "$TARGET_PARTS" | grep -Fxq 'var'; then + echo "ERROR: Target disk did not receive expected partition layout from USB installer!" >&2 + echo "Observed partitions:" >&2 + echo "$TARGET_PARTS" >&2 + sfdisk -l "$WORKDIR/target.raw" >&2 || true + exit 1 + fi + echo "==> Successfully installed from emulated USB media!" # Interactively install and boot a persistent local KubeStellar kiosk VM. [group('test')] install-vm: diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 114df0b3..394458c6 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -208,3 +208,12 @@ def test_installer_smoke_probes_the_kiosk_over_tls_from_inside_the_guest() -> No assert "http://127.0.0.1:8080/healthz" not in smoke assert "KIOSK_CONSOLE_READY" in smoke + +def test_test_installer_boot_usb_contract() -> None: + justfile = JUSTFILE.read_text(encoding="utf-8") + assert "test-installer-boot-usb:" in justfile + assert "-device qemu-xhci,id=xhci" in justfile + assert "-device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1" in justfile + assert 'sfdisk --json "$WORKDIR/target.raw"' in justfile + assert "bluefin-server-root-a" in justfile + assert "var" in justfile From b82b3a1f68ba8a26c4a6896f05166aff268b8704 Mon Sep 17 00:00:00 2001 From: Jorge Castro Date: Mon, 21 Sep 2026 10:44:10 -0400 Subject: [PATCH 02/15] test(installer): attach target disk as virtio-blk-pci device with bootindex=2 and enforce timeout Enforce INSTALLER_TIMEOUT via timeout utility in test-installer-artifact stage 1. Define target disk using -drive if=none and attach device virtio-blk-pci with bootindex=2 so boot priorities are valid for QEMU. Update contract tests. Assisted-by: github-copilot/gemini-3.8-flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 30 ++++++++++++++++++++------- tests/unit/test_installer_contract.py | 11 ++++++---- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/Justfile b/Justfile index d1698699..bb68c959 100644 --- a/Justfile +++ b/Justfile @@ -281,26 +281,39 @@ test-installer-artifact: SMP_CPUS="${SHOW_ME_THE_FUTURE_SMP:-$(nproc)}" MEM_SIZE="${SHOW_ME_THE_FUTURE_MEM:-8192}" - echo "==> Booting installer media in QEMU..." + echo "==> Booting installer media via OVMF over emulated xHCI USB in QEMU..." # ponytail: we want QEMU to exit cleanly after install. Since QEMU's -no-reboot # suspends/halts on reboot signals, we override systemd-sysinstall.service SuccessAction/FailureAction # to poweroff. When the installer triggers poweroff, QEMU terminates, and we boot into the newly installed OS. - qemu-system-x86_64 \ + INSTALLER_TIMEOUT="${SHOW_ME_THE_FUTURE_INSTALL_TIMEOUT:-600}" + timeout "${INSTALLER_TIMEOUT}" qemu-system-x86_64 \ -enable-kvm \ -m "${MEM_SIZE}" \ -cpu host \ -smp "${SMP_CPUS}" \ - -drive file="$WORKDIR/installer.raw",format=raw,if=virtio,readonly=on \ - -drive file="$WORKDIR/target.raw",format=raw,if=virtio \ -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ - -kernel "$WORKDIR/installer.vmlinuz" \ - -initrd "$WORKDIR/installer.initrd" \ - -append "systemd.unit=system-install.target console=tty0 console=ttyS0,115200 rw unattended" \ + -device qemu-xhci,id=xhci \ + -drive file="$WORKDIR/installer.raw",format=raw,if=none,id=installer-disk,readonly=on \ + -device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1 \ + -drive file="$WORKDIR/target.raw",format=raw,if=none,id=target-disk \ + -device virtio-blk-pci,drive=target-disk,bootindex=2 \ + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 rw unattended" \ -nographic \ -serial mon:stdio \ -no-reboot < /dev/null + echo "==> Validating target disk partitions after installer execution..." + TARGET_PARTS=$(sfdisk --json "$WORKDIR/target.raw" 2>/dev/null | jq -r '.partitiontable.partitions[]?.name // empty' || true) + if ! echo "$TARGET_PARTS" | grep -Fxq 'bluefin-server-root-a' || \ + ! echo "$TARGET_PARTS" | grep -Fxq 'var'; then + echo "ERROR: Target disk did not receive expected partition layout from installer!" >&2 + echo "Observed partitions:" >&2 + echo "$TARGET_PARTS" >&2 + sfdisk -l "$WORKDIR/target.raw" >&2 || true + exit 1 + fi + echo "==> Target disk successfully partitioned and populated!" SERIAL_LOG="$WORKDIR/serial.log" TARGET_QEMU_PID="" cleanup() { @@ -464,7 +477,8 @@ test-installer-boot-usb: -device qemu-xhci,id=xhci \ -drive file="$WORKDIR/installer.raw",format=raw,if=none,id=installer-disk,readonly=on \ -device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1 \ - -drive file="$WORKDIR/target.raw",format=raw,if=virtio,id=target-disk,bootindex=2 \ + -drive file="$WORKDIR/target.raw",format=raw,if=none,id=target-disk \ + -device virtio-blk-pci,drive=target-disk,bootindex=2 \ -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 rw unattended" \ -nographic \ -serial mon:stdio \ diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 394458c6..dc43f748 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -50,10 +50,12 @@ def test_installer_runtime_and_boot_contracts() -> None: assert "console=ttyS0,115200 rw" in installer_element assert "unattended" not in published_uki_cmdline assert target_uki_cmdline == "rw console=ttyS0,115200 console=tty0 quiet loglevel=3 audit=0" - assert ( - '-append "systemd.unit=system-install.target ' - 'console=tty0 console=ttyS0,115200 rw unattended"' - ) in justfile + assert "test-installer-artifact:" in justfile + assert "-device qemu-xhci,id=xhci" in justfile + assert "-device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1" in justfile + assert "-device virtio-blk-pci,drive=target-disk,bootindex=2" in justfile + assert 'sfdisk --json "$WORKDIR/target.raw"' in justfile + assert "bluefin-server-root-a" in justfile def test_installer_wrapper_reads_kernel_command_line_without_cat() -> None: @@ -214,6 +216,7 @@ def test_test_installer_boot_usb_contract() -> None: assert "test-installer-boot-usb:" in justfile assert "-device qemu-xhci,id=xhci" in justfile assert "-device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1" in justfile + assert "-device virtio-blk-pci,drive=target-disk,bootindex=2" in justfile assert 'sfdisk --json "$WORKDIR/target.raw"' in justfile assert "bluefin-server-root-a" in justfile assert "var" in justfile From 84cedb5b5bd65f72acac209817be263c75d9736e Mon Sep 17 00:00:00 2001 From: Jorge Castro Date: Tue, 22 Sep 2026 00:26:33 -0400 Subject: [PATCH 03/15] fix(installer): add 30s bounded wait for installer partition, mirror service output to serial console, and add edk2-i386-vars pairing - In bluefin-sysinstall, wait up to 30s using an absolute SECONDS deadline with periodic udevadm trigger and settle for bluefin-installer-data to be enumerated on slower USB controllers before failing. - Mirror systemd-sysinstall.service output to journal+console so installer execution progress and error diagnostics are captured on serial console. - Deduplicate test-installer-boot-usb by having it delegate directly to test-installer-artifact. - Add Homebrew and system edk2-i386-vars.fd to OVMF vars candidates for x86_64 firmware code pairing with fatal fallback. - Strictly scope test-installer-boot-usb unit contract assertions and assert exact dynamic settle timeout. Assisted-by: github-copilot/gemini-3.8-flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 107 ++-------------------- elements/oci/bluefin-server-installer.bst | 26 ++++-- tests/unit/test_installer_contract.py | 18 ++-- 3 files changed, 38 insertions(+), 113 deletions(-) diff --git a/Justfile b/Justfile index bb68c959..1ebe86de 100644 --- a/Justfile +++ b/Justfile @@ -265,18 +265,16 @@ test-installer-artifact: OVMF_VARS=$(first_existing \ /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-vars.fd \ + /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-i386-vars.fd \ /usr/share/edk2/ovmf/OVMF_VARS.fd \ /usr/share/OVMF/OVMF_VARS.fd \ /usr/share/OVMF/OVMF_VARS_4M.fd \ /usr/share/edk2/x64/OVMF_VARS.4m.fd \ /usr/share/qemu/edk2-x86_64-vars.fd \ + /usr/share/qemu/edk2-i386-vars.fd \ /usr/share/qemu/OVMF_VARS.fd) \ - || true - if [ -n "$OVMF_VARS" ]; then - cp "$OVMF_VARS" "$WORKDIR/ovmf-vars.fd" - else - truncate -s "$(stat -c '%s' "$OVMF_CODE")" "$WORKDIR/ovmf-vars.fd" - fi + || { echo "ERROR: OVMF_VARS template not found"; exit 1; } + cp "$OVMF_VARS" "$WORKDIR/ovmf-vars.fd" SMP_CPUS="${SHOW_ME_THE_FUTURE_SMP:-$(nproc)}" MEM_SIZE="${SHOW_ME_THE_FUTURE_MEM:-8192}" @@ -408,93 +406,10 @@ test-installer-artifact: # Boot the exported installer media via firmware (OVMF) attached as an xHCI USB drive. -# Verifies UEFI boot and bare-metal USB storage driver discovery into target disk. +# Delegates to test-installer-artifact, which runs the authoritative OVMF/xHCI installer test. [group('test')] test-installer-boot-usb: - #!/usr/bin/env bash - set -euo pipefail - - CACHE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}" - mkdir -p "$CACHE_DIR" - WORKDIR="$(mktemp -d "${CACHE_DIR}/bluefin-usb-boot.XXXXXX")" - trap 'rm -rf "$WORKDIR"' EXIT - - INSTALLER_ARCHIVE=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-installer-*.raw.zst' | sort -V | tail -n1) - [ -n "$INSTALLER_ARCHIVE" ] || { echo "ERROR: No exported installer found in dist/." >&2; exit 1; } - zstd -d "$INSTALLER_ARCHIVE" -o "$WORKDIR/installer.raw" - - TARGET_SIZE="${SHOW_ME_THE_FUTURE_DISK_SIZE:-16G}" - truncate -s "${TARGET_SIZE}" "$WORKDIR/target.raw" - - first_existing() { - for candidate in "$@"; do - if [ -f "$candidate" ]; then - echo "$candidate" - return 0 - fi - done - return 1 - } - - OVMF_CODE=$(first_existing \ - /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-code.fd \ - /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-secure-code.fd \ - /usr/share/edk2/ovmf/OVMF_CODE.fd \ - /usr/share/OVMF/OVMF_CODE.fd \ - /usr/share/OVMF/OVMF_CODE_4M.fd \ - /usr/share/edk2/x64/OVMF_CODE.4m.fd \ - /usr/share/qemu/edk2-x86_64-code.fd \ - /usr/share/qemu/edk2-x86_64-secure-code.fd \ - /usr/share/qemu/OVMF_CODE.fd) \ - || { echo "ERROR: OVMF_CODE not found"; exit 1; } - - OVMF_VARS=$(first_existing \ - /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-vars.fd \ - /usr/share/edk2/ovmf/OVMF_VARS.fd \ - /usr/share/OVMF/OVMF_VARS.fd \ - /usr/share/OVMF/OVMF_VARS_4M.fd \ - /usr/share/edk2/x64/OVMF_VARS.4m.fd \ - /usr/share/qemu/edk2-x86_64-vars.fd \ - /usr/share/qemu/OVMF_VARS.fd) \ - || true - if [ -n "$OVMF_VARS" ]; then - cp "$OVMF_VARS" "$WORKDIR/ovmf-vars.fd" - else - truncate -s "$(stat -c '%s' "$OVMF_CODE")" "$WORKDIR/ovmf-vars.fd" - fi - - SMP_CPUS="${SHOW_ME_THE_FUTURE_SMP:-$(nproc)}" - MEM_SIZE="${SHOW_ME_THE_FUTURE_MEM:-8192}" - - echo "==> Booting installer media via OVMF over emulated xHCI USB in QEMU..." - qemu-system-x86_64 \ - -enable-kvm \ - -m "${MEM_SIZE}" \ - -cpu host \ - -smp "${SMP_CPUS}" \ - -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ - -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ - -device qemu-xhci,id=xhci \ - -drive file="$WORKDIR/installer.raw",format=raw,if=none,id=installer-disk,readonly=on \ - -device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1 \ - -drive file="$WORKDIR/target.raw",format=raw,if=none,id=target-disk \ - -device virtio-blk-pci,drive=target-disk,bootindex=2 \ - -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 rw unattended" \ - -nographic \ - -serial mon:stdio \ - -no-reboot < /dev/null - - echo "==> Validating target disk partitions after USB installation..." - TARGET_PARTS=$(sfdisk --json "$WORKDIR/target.raw" 2>/dev/null | jq -r '.partitiontable.partitions[]?.name // empty' || true) - if ! echo "$TARGET_PARTS" | grep -Fxq 'bluefin-server-root-a' || \ - ! echo "$TARGET_PARTS" | grep -Fxq 'var'; then - echo "ERROR: Target disk did not receive expected partition layout from USB installer!" >&2 - echo "Observed partitions:" >&2 - echo "$TARGET_PARTS" >&2 - sfdisk -l "$WORKDIR/target.raw" >&2 || true - exit 1 - fi - echo "==> Successfully installed from emulated USB media!" + just test-installer-artifact # Interactively install and boot a persistent local KubeStellar kiosk VM. [group('test')] install-vm: @@ -533,18 +448,16 @@ install-vm: if [ ! -f "$OVMF_VARS" ]; then OVMF_TEMPLATE=$(first_existing \ /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-x86_64-vars.fd \ + /home/linuxbrew/.linuxbrew/Cellar/qemu/*/share/qemu/edk2-i386-vars.fd \ /usr/share/edk2/ovmf/OVMF_VARS.fd \ /usr/share/OVMF/OVMF_VARS.fd \ /usr/share/OVMF/OVMF_VARS_4M.fd \ /usr/share/edk2/x64/OVMF_VARS.4m.fd \ /usr/share/qemu/edk2-x86_64-vars.fd \ + /usr/share/qemu/edk2-i386-vars.fd \ /usr/share/qemu/OVMF_VARS.fd) \ - || true - if [ -n "$OVMF_TEMPLATE" ]; then - cp "$OVMF_TEMPLATE" "$OVMF_VARS" - else - truncate -s "$(stat -c '%s' "$OVMF_CODE")" "$OVMF_VARS" - fi + || { echo "ERROR: OVMF_VARS template not found"; exit 1; } + cp "$OVMF_TEMPLATE" "$OVMF_VARS" fi SMP_CPUS="${INSTALL_VM_SMP:-$(nproc)}" diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 646308ea..a0d95e34 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -192,12 +192,22 @@ config: modprobe -q ahci || true modprobe -q sd_mod || true modprobe -q xfs || true - udevadm trigger --subsystem-match=block || true - udevadm settle --timeout=15 || true - # Hard preflight: abort with diagnostics if installer data partition is absent or not a block device + # Hard preflight: wait up to 30s for storage settle and installer data partition to become available INSTALLER_PART_PATH="/dev/disk/by-partlabel/bluefin-installer-data" + DEADLINE=$((SECONDS + 30)) + udevadm trigger --subsystem-match=block || true + udevadm settle --timeout=5 || true + while [ ! -b "${INSTALLER_PART_PATH}" ] && [ "${SECONDS}" -lt "${DEADLINE}" ]; do + sleep 1 + udevadm trigger --subsystem-match=block || true + REMAINING=$((DEADLINE - SECONDS)) + [ "${REMAINING}" -gt 0 ] || break + SETTLE_TIMEOUT=$(( REMAINING < 5 ? REMAINING : 5 )) + udevadm settle --timeout="${SETTLE_TIMEOUT}" || true + done + if [ ! -b "${INSTALLER_PART_PATH}" ]; then - echo "ERROR: Installer data partition '${INSTALLER_PART_PATH}' is missing or not a block device!" >&2 + echo "ERROR: Installer data partition '${INSTALLER_PART_PATH}' is missing or not a block device after 30s deadline!" >&2 echo "==> Block devices visible at failure:" >&2 lsblk -p -o NAME,TYPE,PARTLABEL,PKNAME,SIZE,FSTYPE >&2 || true echo "==> Partition label symlinks visible under /dev/disk/by-partlabel/:" >&2 @@ -205,7 +215,7 @@ config: exit 1 fi INSTALLER_PART="$(readlink -f "${INSTALLER_PART_PATH}")" - + echo "==> Found installer partition at ${INSTALLER_PART_PATH} -> ${INSTALLER_PART}" # Auto-detect target disk for unattended install (any raw disk of type "disk" that is not the installer, is not read-only, and has size > 0) TARGET_DISK="" while read -r name type ro size; do @@ -309,8 +319,10 @@ config: FailureAction=poweroff [Service] - # Keep the interactive installer on the attached display. The kernel - # still uses the final serial console for boot diagnostics. + # Mirror installer service output to journal+console so progress and diagnostics + # are captured on serial console in headless/automated test runs. + StandardOutput=journal+console + StandardError=journal+console TTYPath=/dev/tty0 ExecStart= ExecStart=/usr/bin/bluefin-sysinstall diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index dc43f748..157c17fa 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -121,7 +121,7 @@ def test_installer_loads_storage_drivers_and_settles_udev() -> None: assert "modprobe -q nvme_core || true" in installer_element assert "modprobe -q usb-storage || true" in installer_element assert "modprobe -q uas || true" in installer_element - assert "udevadm settle --timeout=15 || true" in installer_element + assert 'udevadm settle --timeout="${SETTLE_TIMEOUT}" || true' in installer_element assert "After=systemd-udev-settle.service" in installer_element assert "Wants=systemd-udev-settle.service" in installer_element @@ -129,14 +129,17 @@ def test_installer_loads_storage_drivers_and_settles_udev() -> None: def test_installer_hard_preflight_aborts_on_missing_installer_data_part() -> None: installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") + assert 'DEADLINE=$((SECONDS + 30))' in installer_element + assert 'while [ ! -b "${INSTALLER_PART_PATH}" ] && [ "${SECONDS}" -lt "${DEADLINE}" ]; do' in installer_element assert '[ ! -b "${INSTALLER_PART_PATH}" ]' in installer_element assert "/dev/disk/by-partlabel/bluefin-installer-data" in installer_element assert "lsblk -p -o NAME,TYPE,PARTLABEL,PKNAME,SIZE,FSTYPE" in installer_element - def test_interactive_installer_uses_local_virtual_console() -> None: installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") assert "TTYPath=/dev/tty0" in installer_element + assert "StandardOutput=journal+console" in installer_element + assert "StandardError=journal+console" in installer_element def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: @@ -213,10 +216,7 @@ def test_installer_smoke_probes_the_kiosk_over_tls_from_inside_the_guest() -> No def test_test_installer_boot_usb_contract() -> None: justfile = JUSTFILE.read_text(encoding="utf-8") - assert "test-installer-boot-usb:" in justfile - assert "-device qemu-xhci,id=xhci" in justfile - assert "-device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1" in justfile - assert "-device virtio-blk-pci,drive=target-disk,bootindex=2" in justfile - assert 'sfdisk --json "$WORKDIR/target.raw"' in justfile - assert "bluefin-server-root-a" in justfile - assert "var" in justfile + start = justfile.index("test-installer-boot-usb:") + end = justfile.index("install-vm:", start) + recipe = justfile[start:end] + assert "just test-installer-artifact" in recipe From d3ce3468a97830d42dcf1f2991b63b6ff433fb0f Mon Sep 17 00:00:00 2001 From: ci-maintainer <223556219+Copilot@users.noreply.github.com> Date: Tue, 22 Sep 2026 00:56:05 -0400 Subject: [PATCH 04/15] fix(installer): drop dead PXE copies, restore OVMF_VARS fallback Three review findings from #228: - test-installer-artifact still copied dist/bluefin-server-pxe-vmlinuz-* and -initrd-*.cpio.gz into $WORKDIR, but the recipe no longer passes -kernel/-initrd, so nothing consumed them. Under set -euo pipefail the copies made the test fail on hosts that have the installer image but no PXE artifacts. export-installer already asserts the PXE pair exists, so that coverage is not lost. - Replacing the truncate-based blank variable store with exit 1 meant hosts shipping only OVMF_CODE could no longer run the test or install-vm. Restore the fallback in both recipes and log a warning, keeping the edk2-i386-vars.fd candidates this PR added. - docs/skills/ddi-installer-build.md did not list test-installer-boot-usb and still implied the old direct-kernel behaviour of test-installer-artifact. Document the OVMF/xHCI USB boot path, the alias, and the OVMF_VARS fallback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 29 +++++++++++++++++++++++------ docs/skills/ddi-installer-build.md | 12 ++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/Justfile b/Justfile index 1ebe86de..19eef2a7 100644 --- a/Justfile +++ b/Justfile @@ -233,9 +233,12 @@ test-installer-artifact: WORKDIR="$(mktemp -d "${CACHE_DIR}/bluefin-show-future.XXXXXX")" trap 'rm -rf "$WORKDIR"' EXIT + # Only the raw installer image is needed: this recipe boots it through OVMF as + # a USB drive rather than via QEMU -kernel/-initrd, so the PXE vmlinuz/initrd + # are not consumed here. Copying them would make the test fail on hosts that + # have the installer image but no PXE artifacts. `just export-installer` + # already asserts the PXE pair was exported. cp dist/bluefin-server-installer-*.raw.zst "$WORKDIR/installer.raw.zst" - cp dist/bluefin-server-pxe-vmlinuz-* "$WORKDIR/installer.vmlinuz" - cp dist/bluefin-server-pxe-initrd-*.cpio.gz "$WORKDIR/installer.initrd" zstd -d "$WORKDIR/installer.raw.zst" -o "$WORKDIR/installer.raw" TARGET_SIZE="${SHOW_ME_THE_FUTURE_DISK_SIZE:-16G}" truncate -s "${TARGET_SIZE}" "$WORKDIR/target.raw" @@ -273,8 +276,15 @@ test-installer-artifact: /usr/share/qemu/edk2-x86_64-vars.fd \ /usr/share/qemu/edk2-i386-vars.fd \ /usr/share/qemu/OVMF_VARS.fd) \ - || { echo "ERROR: OVMF_VARS template not found"; exit 1; } - cp "$OVMF_VARS" "$WORKDIR/ovmf-vars.fd" + || true + if [ -n "${OVMF_VARS}" ]; then + cp "$OVMF_VARS" "$WORKDIR/ovmf-vars.fd" + else + # Hosts that ship only OVMF_CODE still boot: firmware initialises a blank + # variable store on first use, it just has no preseeded boot entries. + echo "WARNING: no OVMF_VARS template found; using a blank variable store sized to $OVMF_CODE" >&2 + truncate -s "$(stat -c '%s' "$OVMF_CODE")" "$WORKDIR/ovmf-vars.fd" + fi SMP_CPUS="${SHOW_ME_THE_FUTURE_SMP:-$(nproc)}" MEM_SIZE="${SHOW_ME_THE_FUTURE_MEM:-8192}" @@ -456,8 +466,15 @@ install-vm: /usr/share/qemu/edk2-x86_64-vars.fd \ /usr/share/qemu/edk2-i386-vars.fd \ /usr/share/qemu/OVMF_VARS.fd) \ - || { echo "ERROR: OVMF_VARS template not found"; exit 1; } - cp "$OVMF_TEMPLATE" "$OVMF_VARS" + || true + if [ -n "${OVMF_TEMPLATE}" ]; then + cp "$OVMF_TEMPLATE" "$OVMF_VARS" + else + # Hosts that ship only OVMF_CODE still boot: firmware initialises a blank + # variable store on first use, it just has no preseeded boot entries. + echo "WARNING: no OVMF_VARS template found; using a blank variable store sized to $OVMF_CODE" >&2 + truncate -s "$(stat -c '%s' "$OVMF_CODE")" "$OVMF_VARS" + fi fi SMP_CPUS="${INSTALL_VM_SMP:-$(nproc)}" diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 0916d10f..d4029cab 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -31,9 +31,21 @@ just export-sysext # export sysext artifacts to dist/sysext/ just flash-installer # write the installer image to a USB device just show-me-the-future # end-to-end QEMU installer smoke test just test-installer-artifact # test already-exported artifacts in QEMU without rebuilding +just test-installer-boot-usb # alias for test-installer-artifact, named for the USB boot path just tags # show FSDK-derived version tags ``` +`just test-installer-artifact` boots the exported raw installer image through OVMF +firmware as an emulated xHCI USB drive (`-device qemu-xhci` plus `-device usb-storage`), +with the target disk on virtio-blk. This is the bare-metal USB install path: firmware +picks the bootloader off the image itself rather than QEMU injecting a kernel through +`-kernel`/`-initrd`. `just test-installer-boot-usb` is an alias for the same recipe, +provided so the USB boot path is discoverable by name. + +The recipe prefers a real `OVMF_VARS` template but falls back to a blank variable store +sized to `OVMF_CODE` on hosts that ship CODE only, so a missing template is a warning +rather than a hard failure. + ## Mandatory build path: ghost cluster This project MUST always build on the ghost cluster using distributed BuildStream: From fa1c93b31f5f0101ed423e0d89ac1dad40df2797 Mon Sep 17 00:00:00 2001 From: ci-maintainer <223556219+Copilot@users.noreply.github.com> Date: Tue, 22 Sep 2026 01:20:39 -0400 Subject: [PATCH 05/15] fix(installer): keep interactive installs on tty0, harden partition check Four review findings from #228: - StandardOutput/StandardError=journal+console routes installer output to /dev/console, which follows the last console= argument (ttyS0), so an interactive bare-metal install printed to serial instead of the attached display. TTYPath= is only honoured for Standard*=tty, so it did not counteract this. The wrapper now redirects its own output to /dev/tty0 when 'unattended' is absent from the kernel command line and a local VT is writable, which restores the display for interactive installs while unattended/automated runs keep the serial capture that the drop-in was added for. Headless serial-only interactive installs fall through to journal+console unchanged. - The target-disk partition check piped sfdisk into jq with 2>/dev/null and || true, so a missing or broken tool was reported as "Target disk did not receive expected partition layout". sfdisk and jq are now checked up front and their failures reported separately from a genuinely unpartitioned disk. jq was an undeclared host dependency of this recipe; it is now named in the error path and in docs/skills/ddi-installer-build.md. - The comment added in the previous commit credited export-installer with asserting the PXE pair was exported; those assertions are in export-pxe. - Restore the blank line before test_interactive_installer_uses_local_virtual_console (PEP8 E302) and separate the test-installer-boot-usb recipe from the install-vm comment block. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 25 +++++++++++++++++++---- docs/skills/ddi-installer-build.md | 4 +++- elements/oci/bluefin-server-installer.bst | 19 +++++++++++++++-- tests/unit/test_installer_contract.py | 8 ++++++++ 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/Justfile b/Justfile index 19eef2a7..2cef7a31 100644 --- a/Justfile +++ b/Justfile @@ -236,8 +236,8 @@ test-installer-artifact: # Only the raw installer image is needed: this recipe boots it through OVMF as # a USB drive rather than via QEMU -kernel/-initrd, so the PXE vmlinuz/initrd # are not consumed here. Copying them would make the test fail on hosts that - # have the installer image but no PXE artifacts. `just export-installer` - # already asserts the PXE pair was exported. + # have the installer image but no PXE artifacts. `just export-pxe` is where a + # missing PXE pair is caught. cp dist/bluefin-server-installer-*.raw.zst "$WORKDIR/installer.raw.zst" zstd -d "$WORKDIR/installer.raw.zst" -o "$WORKDIR/installer.raw" TARGET_SIZE="${SHOW_ME_THE_FUTURE_DISK_SIZE:-16G}" @@ -312,7 +312,24 @@ test-installer-artifact: -no-reboot < /dev/null echo "==> Validating target disk partitions after installer execution..." - TARGET_PARTS=$(sfdisk --json "$WORKDIR/target.raw" 2>/dev/null | jq -r '.partitiontable.partitions[]?.name // empty' || true) + for tool in sfdisk jq; do + command -v "$tool" >/dev/null 2>&1 \ + || { echo "ERROR: '$tool' is required to validate the target disk layout but is not installed." >&2; exit 1; } + done + + # Keep sfdisk and jq failures distinguishable from "the installer did not + # partition the disk", otherwise a broken host tool is misreported as a + # failed install. + if ! TARGET_TABLE=$(sfdisk --json "$WORKDIR/target.raw"); then + echo "ERROR: sfdisk could not read a partition table from $WORKDIR/target.raw!" >&2 + sfdisk -l "$WORKDIR/target.raw" >&2 || true + exit 1 + fi + if ! TARGET_PARTS=$(jq -r '.partitiontable.partitions[]?.name // empty' <<<"$TARGET_TABLE"); then + echo "ERROR: jq failed to parse the sfdisk JSON output!" >&2 + echo "$TARGET_TABLE" >&2 + exit 1 + fi if ! echo "$TARGET_PARTS" | grep -Fxq 'bluefin-server-root-a' || \ ! echo "$TARGET_PARTS" | grep -Fxq 'var'; then echo "ERROR: Target disk did not receive expected partition layout from installer!" >&2 @@ -414,12 +431,12 @@ test-installer-artifact: sleep 2 done - # Boot the exported installer media via firmware (OVMF) attached as an xHCI USB drive. # Delegates to test-installer-artifact, which runs the authoritative OVMF/xHCI installer test. [group('test')] test-installer-boot-usb: just test-installer-artifact + # Interactively install and boot a persistent local KubeStellar kiosk VM. [group('test')] install-vm: diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index d4029cab..bb9d2a83 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -44,7 +44,9 @@ provided so the USB boot path is discoverable by name. The recipe prefers a real `OVMF_VARS` template but falls back to a blank variable store sized to `OVMF_CODE` on hosts that ship CODE only, so a missing template is a warning -rather than a hard failure. +rather than a hard failure. It needs `qemu-system-x86_64`, `zstd`, `sfdisk` and `jq` on +the host; the last two are used to assert that the installer actually partitioned the +target disk, and the recipe fails with a named error if either is missing. ## Mandatory build path: ghost cluster diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index a0d95e34..790b7bc9 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -184,6 +184,19 @@ config: #!/bin/bash set -euo pipefail + CMDLINE="$(< /proc/cmdline)" + + # The service drop-in uses StandardOutput=journal+console so unattended runs + # land on the serial console that automated tests capture. /dev/console follows + # the last console= argument, which is ttyS0, so an interactive bare-metal + # install would otherwise print to serial rather than the attached display. + # Keep the interactive installer on the attached display by redirecting this + # script's own output to /dev/tty0, falling back to journal+console when there + # is no local VT (headless serial-only installs). + if [[ " ${CMDLINE} " != *" unattended "* ]] && [ -w /dev/tty0 ]; then + exec > /dev/tty0 2>&1 + fi + # Ensure storage controller modules and filesystems are loaded before probing devices modprobe -q nvme || true modprobe -q nvme_core || true @@ -274,7 +287,6 @@ config: } # Check for unattended in kernel command line - CMDLINE="$(< /proc/cmdline)" if [[ " ${CMDLINE} " == *" unattended "* ]]; then echo "==> Running in UNATTENDED mode..." if [ -n "${TARGET_DISK}" ]; then @@ -320,7 +332,10 @@ config: [Service] # Mirror installer service output to journal+console so progress and diagnostics - # are captured on serial console in headless/automated test runs. + # are captured on the serial console in headless/automated test runs. TTYPath is + # only honoured for Standard*=tty, so it does not steer this output; the wrapper + # keeps the interactive installer on the attached display by redirecting itself + # to /dev/tty0 when 'unattended' is absent from the kernel command line. StandardOutput=journal+console StandardError=journal+console TTYPath=/dev/tty0 diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index 157c17fa..e599bc36 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -134,12 +134,20 @@ def test_installer_hard_preflight_aborts_on_missing_installer_data_part() -> Non assert '[ ! -b "${INSTALLER_PART_PATH}" ]' in installer_element assert "/dev/disk/by-partlabel/bluefin-installer-data" in installer_element assert "lsblk -p -o NAME,TYPE,PARTLABEL,PKNAME,SIZE,FSTYPE" in installer_element + + def test_interactive_installer_uses_local_virtual_console() -> None: installer_element = INSTALLER_ELEMENT.read_text(encoding="utf-8") assert "TTYPath=/dev/tty0" in installer_element assert "StandardOutput=journal+console" in installer_element assert "StandardError=journal+console" in installer_element + # journal+console follows the last console= argument (ttyS0), so the wrapper has + # to put interactive runs back on the attached display itself. + assert ( + 'if [[ " ${CMDLINE} " != *" unattended "* ]] && [ -w /dev/tty0 ]; then\n' + " exec > /dev/tty0 2>&1" + ) in installer_element def test_installer_and_ddi_strip_vmlinux_and_static_archives() -> None: From 1af7b47ac739f93af565cf8aa13c47acf5c2a7e7 Mon Sep 17 00:00:00 2001 From: ci-maintainer Date: Tue, 22 Sep 2026 15:11:02 -0400 Subject: [PATCH 06/15] test(installer): keep the PXE boot path under test, tighten the tty0 guard test-installer-artifact now takes an INSTALLER_BOOT_MODE of usb (default, firmware boots the raw image off an emulated xHCI USB drive) or pxe (QEMU boots the exported vmlinuz/initrd pair with -kernel/-initrd/-append). Everything after the install is shared. test-installer-boot-usb and test-installer-boot-pxe are the named entry points, so the direct-kernel path the previous revision dropped is exercised again rather than merely existence-checked by export-pxe. Document next to the -smbios flag that systemd-stub ignores io.systemd.stub.kernel-cmdline-extra under Secure Boot, so a secure-boot OVMF build boots interactive and hangs until the install timeout, and point at pxe mode as the way to test unattended installs on such firmware. Restrict the interactive /dev/tty0 redirect to boots where the kernel actually registered tty0 as a console. A writable VT device exists on serial-only bare metal too, so the -w test alone sent the whole interactive install to a display nobody was watching. Correct the ci-tooling sentence that still claimed the uploaded PXE artifacts feed test-installer-artifact. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Justfile | 98 ++++++++++++++++++----- docs/skills/ci-tooling.md | 2 +- docs/skills/ddi-installer-build.md | 16 +++- elements/oci/bluefin-server-installer.bst | 12 ++- tests/unit/test_installer_contract.py | 27 ++++++- 5 files changed, 124 insertions(+), 31 deletions(-) diff --git a/Justfile b/Justfile index 2cef7a31..47093fa5 100644 --- a/Justfile +++ b/Justfile @@ -222,23 +222,46 @@ show-me-the-future: just export-installer just test-installer-artifact +# INSTALLER_BOOT_MODE selects how the installer itself is started: +# usb (default) — firmware (OVMF) boots the raw image off an emulated xHCI USB +# drive, the bare-metal USB install path. +# pxe — QEMU boots the exported PXE vmlinuz/initrd pair directly with +# -kernel/-initrd/-append, the network install path. +# Everything after the install (target partition check, booting the installed +# system, kiosk readiness probe) is identical for both modes. # Install and reboot already-exported server artifacts in QEMU. [group('test')] test-installer-artifact: #!/usr/bin/env bash set -euo pipefail + BOOT_MODE="${INSTALLER_BOOT_MODE:-usb}" + case "${BOOT_MODE}" in + usb|pxe) ;; + *) echo "ERROR: INSTALLER_BOOT_MODE must be 'usb' or 'pxe', got '${BOOT_MODE}'." >&2; exit 1 ;; + esac + CACHE_DIR="${XDG_CACHE_HOME:-$HOME/.cache}" mkdir -p "$CACHE_DIR" WORKDIR="$(mktemp -d "${CACHE_DIR}/bluefin-show-future.XXXXXX")" trap 'rm -rf "$WORKDIR"' EXIT - # Only the raw installer image is needed: this recipe boots it through OVMF as - # a USB drive rather than via QEMU -kernel/-initrd, so the PXE vmlinuz/initrd - # are not consumed here. Copying them would make the test fail on hosts that - # have the installer image but no PXE artifacts. `just export-pxe` is where a - # missing PXE pair is caught. + # The raw installer image is always needed; usb mode boots it through firmware + # so it consumes nothing else. The PXE vmlinuz/initrd pair is only copied in pxe + # mode, because copying it unconditionally would fail the usb test on hosts that + # have the installer image but no PXE artifacts. cp dist/bluefin-server-installer-*.raw.zst "$WORKDIR/installer.raw.zst" + if [ "${BOOT_MODE}" = "pxe" ]; then + PXE_KERNEL=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-pxe-vmlinuz-*' | head -n1) + PXE_INITRD=$(find dist/ -maxdepth 1 -type f -name 'bluefin-server-pxe-initrd-*.cpio.gz' | head -n1) + if [ -z "${PXE_KERNEL}" ] || [ -z "${PXE_INITRD}" ]; then + echo "ERROR: INSTALLER_BOOT_MODE=pxe needs an exported PXE kernel and initrd in dist/." >&2 + echo "Please run: just build-installer && just export-pxe" >&2 + exit 1 + fi + cp "${PXE_KERNEL}" "$WORKDIR/installer.vmlinuz" + cp "${PXE_INITRD}" "$WORKDIR/installer.initrd" + fi zstd -d "$WORKDIR/installer.raw.zst" -o "$WORKDIR/installer.raw" TARGET_SIZE="${SHOW_ME_THE_FUTURE_DISK_SIZE:-16G}" truncate -s "${TARGET_SIZE}" "$WORKDIR/target.raw" @@ -289,24 +312,49 @@ test-installer-artifact: SMP_CPUS="${SHOW_ME_THE_FUTURE_SMP:-$(nproc)}" MEM_SIZE="${SHOW_ME_THE_FUTURE_MEM:-8192}" - echo "==> Booting installer media via OVMF over emulated xHCI USB in QEMU..." + echo "==> Booting installer media in QEMU (mode: ${BOOT_MODE})..." # ponytail: we want QEMU to exit cleanly after install. Since QEMU's -no-reboot # suspends/halts on reboot signals, we override systemd-sysinstall.service SuccessAction/FailureAction # to poweroff. When the installer triggers poweroff, QEMU terminates, and we boot into the newly installed OS. INSTALLER_TIMEOUT="${SHOW_ME_THE_FUTURE_INSTALL_TIMEOUT:-600}" + INSTALL_ARGS=( + -enable-kvm + -m "${MEM_SIZE}" + -cpu host + -smp "${SMP_CPUS}" + -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" + -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" + ) + if [ "${BOOT_MODE}" = "usb" ]; then + # Firmware boots the image's own bootloader, so the unattended flag has to + # reach the UKI through systemd-stub rather than QEMU's -append. + # + # NOTE: systemd-stub deliberately ignores io.systemd.stub.kernel-cmdline-extra + # when Secure Boot is enabled, because the SMBIOS string is not covered by the + # signature. Point $OVMF_CODE at a secure-boot firmware build and this run + # boots interactive instead of unattended, then sits there until + # SHOW_ME_THE_FUTURE_INSTALL_TIMEOUT expires with a misleading "install did + # not finish" failure. Use INSTALLER_BOOT_MODE=pxe (-append is always honoured) + # to exercise unattended installs under Secure Boot firmware. + INSTALL_ARGS+=( + -device qemu-xhci,id=xhci + -drive file="$WORKDIR/installer.raw",format=raw,if=none,id=installer-disk,readonly=on + -device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1 + -drive file="$WORKDIR/target.raw",format=raw,if=none,id=target-disk + -device virtio-blk-pci,drive=target-disk,bootindex=2 + -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 rw unattended" + ) + else + INSTALL_ARGS+=( + -drive file="$WORKDIR/installer.raw",format=raw,if=virtio,readonly=on + -drive file="$WORKDIR/target.raw",format=raw,if=virtio + -kernel "$WORKDIR/installer.vmlinuz" + -initrd "$WORKDIR/installer.initrd" + -append "systemd.unit=system-install.target console=tty0 console=ttyS0,115200 rw unattended" + ) + fi timeout "${INSTALLER_TIMEOUT}" qemu-system-x86_64 \ - -enable-kvm \ - -m "${MEM_SIZE}" \ - -cpu host \ - -smp "${SMP_CPUS}" \ - -drive if=pflash,format=raw,readonly=on,file="$OVMF_CODE" \ - -drive if=pflash,format=raw,file="$WORKDIR/ovmf-vars.fd" \ - -device qemu-xhci,id=xhci \ - -drive file="$WORKDIR/installer.raw",format=raw,if=none,id=installer-disk,readonly=on \ - -device usb-storage,bus=xhci.0,drive=installer-disk,bootindex=1 \ - -drive file="$WORKDIR/target.raw",format=raw,if=none,id=target-disk \ - -device virtio-blk-pci,drive=target-disk,bootindex=2 \ - -smbios "type=11,value=io.systemd.stub.kernel-cmdline-extra=console=tty0 console=ttyS0,,115200 rw unattended" \ + "${INSTALL_ARGS[@]}" \ -nographic \ -serial mon:stdio \ -no-reboot < /dev/null @@ -431,11 +479,19 @@ test-installer-artifact: sleep 2 done -# Boot the exported installer media via firmware (OVMF) attached as an xHCI USB drive. -# Delegates to test-installer-artifact, which runs the authoritative OVMF/xHCI installer test. +# This is the default mode of test-installer-artifact; the recipe exists so the USB +# boot path is discoverable by name. +# Boot the exported installer media via firmware (OVMF) as an xHCI USB drive. [group('test')] test-installer-boot-usb: - just test-installer-artifact + INSTALLER_BOOT_MODE=usb just test-installer-artifact + +# `just export-pxe` only checks that the two files exist, so this recipe is the only +# thing that proves they still boot and install. +# Boot the exported PXE kernel/initrd pair directly (QEMU -kernel/-initrd/-append). +[group('test')] +test-installer-boot-pxe: + INSTALLER_BOOT_MODE=pxe just test-installer-artifact # Interactively install and boot a persistent local KubeStellar kiosk VM. [group('test')] diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index c4deaf91..57346063 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -137,7 +137,7 @@ uploaded to a GitHub Release tagged `installer-v`. result as its own pull request against `main`. 3. **Full Compilation:** Builds the standalone DDI OS image, live installer, and k0s systemd-sysext on every pull request and push to `main`. -4. **Installer boot test:** Uploads the already-built installer and PXE artifacts and calls `projectbluefin/actions/.github/workflows/server-installer-test.yml`. The shared workflow prepares QEMU/KVM and runs this repository's `just test-installer-artifact`; Lima remains local-only. +4. **Installer boot test:** Uploads the already-built installer and PXE artifacts and calls `projectbluefin/actions/.github/workflows/server-installer-test.yml`. The shared workflow prepares QEMU/KVM and runs this repository's `just test-installer-artifact`, which defaults to the USB/firmware boot path and consumes only the raw installer image; the uploaded PXE pair is exercised by `just test-installer-boot-pxe` (`INSTALLER_BOOT_MODE=pxe`). Lima remains local-only. 5. **Version Derivation:** The release tag is derived with `just version`, which parses the pinned FSDK point release from `elements/freedesktop-sdk.bst`. 6. **Automated Publishing:** For pushes to `main` (including Renovate PR merges), diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index bb9d2a83..bd73be40 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -31,7 +31,8 @@ just export-sysext # export sysext artifacts to dist/sysext/ just flash-installer # write the installer image to a USB device just show-me-the-future # end-to-end QEMU installer smoke test just test-installer-artifact # test already-exported artifacts in QEMU without rebuilding -just test-installer-boot-usb # alias for test-installer-artifact, named for the USB boot path +just test-installer-boot-usb # test-installer-artifact in USB/firmware boot mode +just test-installer-boot-pxe # test-installer-artifact in PXE -kernel/-initrd boot mode just tags # show FSDK-derived version tags ``` @@ -39,8 +40,17 @@ just tags # show FSDK-derived version tags firmware as an emulated xHCI USB drive (`-device qemu-xhci` plus `-device usb-storage`), with the target disk on virtio-blk. This is the bare-metal USB install path: firmware picks the bootloader off the image itself rather than QEMU injecting a kernel through -`-kernel`/`-initrd`. `just test-installer-boot-usb` is an alias for the same recipe, -provided so the USB boot path is discoverable by name. +`-kernel`/`-initrd`. The boot method is selected by `INSTALLER_BOOT_MODE` (`usb`, the +default, or `pxe`); everything after the install is identical for both. `just +test-installer-boot-usb` and `just test-installer-boot-pxe` are the named entry points. + +Use `pxe` mode to exercise the exported `bluefin-server-pxe-vmlinuz-*` / +`bluefin-server-pxe-initrd-*.cpio.gz` pair, which is booted with `-kernel`/`-initrd` +and an explicit `-append` command line. `just export-pxe` only asserts that those two +files exist, so the PXE recipe is what proves they still boot and install. PXE mode is +also the way to test an unattended install under Secure Boot firmware: in USB mode the +unattended flag travels via `-smbios type=11 io.systemd.stub.kernel-cmdline-extra`, and +systemd-stub ignores that credential when Secure Boot is enabled. The recipe prefers a real `OVMF_VARS` template but falls back to a blank variable store sized to `OVMF_CODE` on hosts that ship CODE only, so a missing template is a warning diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 790b7bc9..40e90bed 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -191,9 +191,15 @@ config: # the last console= argument, which is ttyS0, so an interactive bare-metal # install would otherwise print to serial rather than the attached display. # Keep the interactive installer on the attached display by redirecting this - # script's own output to /dev/tty0, falling back to journal+console when there - # is no local VT (headless serial-only installs). - if [[ " ${CMDLINE} " != *" unattended "* ]] && [ -w /dev/tty0 ]; then + # script's own output to /dev/tty0 — but only when the kernel actually drives + # tty0 as a console. On serial-only machines the VT device still exists and is + # writable, so a -w test alone would send the whole install to a display nobody + # is watching; /sys/class/tty/console/active lists the consoles the kernel + # really registered, and omits tty0 when the boot was serial-only. If that file + # is unreadable the redirect is skipped and output stays on journal+console. + if [[ " ${CMDLINE} " != *" unattended "* ]] \ + && [ -w /dev/tty0 ] \ + && grep -qw tty0 /sys/class/tty/console/active 2>/dev/null; then exec > /dev/tty0 2>&1 fi diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index e599bc36..e1f9eeab 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -56,6 +56,11 @@ def test_installer_runtime_and_boot_contracts() -> None: assert "-device virtio-blk-pci,drive=target-disk,bootindex=2" in justfile assert 'sfdisk --json "$WORKDIR/target.raw"' in justfile assert "bluefin-server-root-a" in justfile + # The PXE pair is still exported, so the direct-kernel boot path stays covered. + assert ( + '-append "systemd.unit=system-install.target ' + 'console=tty0 console=ttyS0,115200 rw unattended"' + ) in justfile def test_installer_wrapper_reads_kernel_command_line_without_cat() -> None: @@ -143,9 +148,13 @@ def test_interactive_installer_uses_local_virtual_console() -> None: assert "StandardOutput=journal+console" in installer_element assert "StandardError=journal+console" in installer_element # journal+console follows the last console= argument (ttyS0), so the wrapper has - # to put interactive runs back on the attached display itself. + # to put interactive runs back on the attached display itself — but only when the + # kernel actually registered tty0 as a console, otherwise serial-only machines + # lose the install output to a VT nobody is watching. assert ( - 'if [[ " ${CMDLINE} " != *" unattended "* ]] && [ -w /dev/tty0 ]; then\n' + 'if [[ " ${CMDLINE} " != *" unattended "* ]] \\\n' + " && [ -w /dev/tty0 ] \\\n" + " && grep -qw tty0 /sys/class/tty/console/active 2>/dev/null; then\n" " exec > /dev/tty0 2>&1" ) in installer_element @@ -225,6 +234,18 @@ def test_installer_smoke_probes_the_kiosk_over_tls_from_inside_the_guest() -> No def test_test_installer_boot_usb_contract() -> None: justfile = JUSTFILE.read_text(encoding="utf-8") start = justfile.index("test-installer-boot-usb:") + end = justfile.index("test-installer-boot-pxe:", start) + recipe = justfile[start:end] + assert "INSTALLER_BOOT_MODE=usb just test-installer-artifact" in recipe + + +def test_test_installer_boot_pxe_recipe_exercises_direct_kernel_boot() -> None: + justfile = JUSTFILE.read_text(encoding="utf-8") + start = justfile.index("test-installer-boot-pxe:") end = justfile.index("install-vm:", start) recipe = justfile[start:end] - assert "just test-installer-artifact" in recipe + assert "INSTALLER_BOOT_MODE=pxe just test-installer-artifact" in recipe + assert '-kernel "$WORKDIR/installer.vmlinuz"' in justfile + assert '-initrd "$WORKDIR/installer.initrd"' in justfile + assert "bluefin-server-pxe-vmlinuz-*" in justfile + assert "bluefin-server-pxe-initrd-*.cpio.gz" in justfile From b2d44041e3455035d5bcec2ba9b70a94fd829348 Mon Sep 17 00:00:00 2001 From: "sec-check[bot]" Date: Wed, 23 Sep 2026 17:44:12 -0400 Subject: [PATCH 07/15] fix: generate console JWT secret at first boot, not in git The kubestellar-console-github-oauth Secret committed a usable jwt-secret (default-jwt-secret-123456789012345678901234). JWT_SECRET signs console session tokens, so a value published in git lets anyone forge valid sessions on every default install. Mirror the kubeflex-postgres pattern (#98): the committed 01- manifest now creates only the namespace, and a first-boot generator writes 02-kubestellar-console-secret.yaml with a random 256-bit jwt-secret and empty OAuth credentials (consumed with optional: true). Idempotent, 0600, runs from k0s-first-boot.service before k0scontroller starts. Existing installs rotate on upgrade: tmpfiles C+ replaces the stale 01- copy (dropping the published Secret) and the generator creates the 02- file fresh, invalidating any tokens forged with the public value. Closes #72 Assisted-by: Claude Fable 5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check[bot] --- elements/oci/k0s-sysext.bst | 1 + files/k0s/kubeflex/generate-console-secret.sh | 40 +++++++++++++++ .../01-kubestellar-console-github-oauth.yaml | 20 ++++---- .../os/systemd/system/k0s-first-boot.service | 1 + tests/unit/test_k0s_manifests.py | 50 +++++++++++++++++++ 5 files changed, 101 insertions(+), 11 deletions(-) create mode 100755 files/k0s/kubeflex/generate-console-secret.sh diff --git a/elements/oci/k0s-sysext.bst b/elements/oci/k0s-sysext.bst index dc5bcdbc..658f7f93 100644 --- a/elements/oci/k0s-sysext.bst +++ b/elements/oci/k0s-sysext.bst @@ -89,6 +89,7 @@ config: mkdir -p sysext/usr/share/k0s/kubeflex cp -a kubeflex-src/. sysext/usr/share/k0s/kubeflex/ chmod 0755 sysext/usr/share/k0s/kubeflex/generate-postgres-secret.sh + chmod 0755 sysext/usr/share/k0s/kubeflex/generate-console-secret.sh # Generate self-signed TLS certificate for local kiosk proxy openssl req -x509 -nodes -days 3650 -newkey rsa:2048 \ diff --git a/files/k0s/kubeflex/generate-console-secret.sh b/files/k0s/kubeflex/generate-console-secret.sh new file mode 100755 index 00000000..9798a4fa --- /dev/null +++ b/files/k0s/kubeflex/generate-console-secret.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# First-boot generator for the KubeStellar console session-signing secret. +# +# The console Deployment (40-kubestellar-console.yaml) reads JWT_SECRET from +# the kubestellar-console-github-oauth Secret via secretKeyRef. This script +# creates that Secret once, with a random jwt-secret, so no usable signing key +# is committed to git (see projectbluefin/server#72). The GitHub OAuth keys +# are created empty: the console consumes them with `optional: true`, and an +# operator enabling OAuth edits the generated file with real credentials. +# +# Idempotent: the secret is generated only if the file does not already exist, +# so operator-supplied OAuth credentials and issued sessions survive re-boots. +# Run once per boot by k0s-first-boot.service, before k0s applies the +# manifests (its 02- name sorts before 40-kubestellar-console.yaml). +set -euo pipefail + +manifest_dir="${KUBESTELLAR_MANIFEST_DIR:-/var/lib/k0s/manifests/kubestellar}" +secret_file="$manifest_dir/02-kubestellar-console-secret.yaml" + +if [ -e "$secret_file" ]; then + exit 0 +fi + +mkdir -p "$manifest_dir" +jwt_secret="$(od -An -tx1 -N32 /dev/urandom | tr -d ' \n')" + +cat > "$secret_file" < Date: Thu, 24 Sep 2026 02:15:24 -0400 Subject: [PATCH 08/15] fix(boot): rename systemd presets to sort before Flatcar's 99-default 'disable *' Flatcar's /usr ships /usr/lib/systemd/system-preset/99-default.preset containing 'disable *'. systemd reads preset files in lexicographic filename order and the first matching line wins, so every Bluefin preset named zz-enable-*.preset sorted after it and was ignored at first-boot preset-all. k0s-first-boot.service was therefore never enabled: the QEMU smoke test sat at 'kiosk-ready: waiting k0s=inactive' for the full 900s window on every run since the Flatcar payload switch. Rename the presets to 20-bluefin-*.preset so they sort before 99-default.preset, and add a unit test asserting no staged preset can regress behind the Flatcar catch-all again. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check[bot] --- ...credentials.preset => 20-bluefin-firstboot-credentials.preset} | 0 ...ble-k0s-first-boot.preset => 20-bluefin-k0s-first-boot.preset} | 0 .../{zz-enable-networkd.preset => 20-bluefin-networkd.preset} | 0 .../{zz-enable-sshd.preset => 20-bluefin-sshd.preset} | 0 ...nerator.preset => 20-bluefin-systemd-network-generator.preset} | 0 .../{zz-enable-var-mount.preset => 20-bluefin-var-mount.preset} | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename files/os/systemd/system-preset/{zz-enable-bluefin-firstboot-credentials.preset => 20-bluefin-firstboot-credentials.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-k0s-first-boot.preset => 20-bluefin-k0s-first-boot.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-networkd.preset => 20-bluefin-networkd.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-sshd.preset => 20-bluefin-sshd.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-systemd-network-generator.preset => 20-bluefin-systemd-network-generator.preset} (100%) rename files/os/systemd/system-preset/{zz-enable-var-mount.preset => 20-bluefin-var-mount.preset} (100%) diff --git a/files/os/systemd/system-preset/zz-enable-bluefin-firstboot-credentials.preset b/files/os/systemd/system-preset/20-bluefin-firstboot-credentials.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-bluefin-firstboot-credentials.preset rename to files/os/systemd/system-preset/20-bluefin-firstboot-credentials.preset diff --git a/files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset b/files/os/systemd/system-preset/20-bluefin-k0s-first-boot.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-k0s-first-boot.preset rename to files/os/systemd/system-preset/20-bluefin-k0s-first-boot.preset diff --git a/files/os/systemd/system-preset/zz-enable-networkd.preset b/files/os/systemd/system-preset/20-bluefin-networkd.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-networkd.preset rename to files/os/systemd/system-preset/20-bluefin-networkd.preset diff --git a/files/os/systemd/system-preset/zz-enable-sshd.preset b/files/os/systemd/system-preset/20-bluefin-sshd.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-sshd.preset rename to files/os/systemd/system-preset/20-bluefin-sshd.preset diff --git a/files/os/systemd/system-preset/zz-enable-systemd-network-generator.preset b/files/os/systemd/system-preset/20-bluefin-systemd-network-generator.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-systemd-network-generator.preset rename to files/os/systemd/system-preset/20-bluefin-systemd-network-generator.preset diff --git a/files/os/systemd/system-preset/zz-enable-var-mount.preset b/files/os/systemd/system-preset/20-bluefin-var-mount.preset similarity index 100% rename from files/os/systemd/system-preset/zz-enable-var-mount.preset rename to files/os/systemd/system-preset/20-bluefin-var-mount.preset From b40d6f6a6f7ee3f8de841d7fa22748d4b69e3201 Mon Sep 17 00:00:00 2001 From: sec-check Date: Thu, 24 Sep 2026 06:26:35 -0400 Subject: [PATCH 09/15] test: update preset paths to 20-bluefin-*, add sort-order guard, log pod states in smoke probe - Update the two unit tests that still referenced the removed zz-enable-*.preset filenames (the rename commit shipped without them). - Add test_presets_sort_before_flatcar_disable_all so no future preset can sort behind Flatcar's 99-default.preset 'disable *' catch-all. - Teach the installer smoke readiness unit to print the pod table and the last Warning events alongside the existing counters, so a run that reaches k0s=active but never gets a pod to Running names the blocker in the serial log instead of only counting it. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check --- Justfile | 2 +- tests/unit/test_credentials_provisioning.py | 4 ++-- tests/unit/test_k0s_first_boot.py | 20 ++++++++++++++++++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/Justfile b/Justfile index c3cf9bb3..e163c421 100644 --- a/Justfile +++ b/Justfile @@ -466,7 +466,7 @@ test-installer-artifact: Type=oneshot TimeoutStartSec=infinity StandardOutput=journal+console - ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' + ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; k0s kubectl get pods -A --no-headers 2>/dev/null | sed "s/^/<4>kiosk-pods: /"; k0s kubectl get events -A --field-selector type=Warning --no-headers 2>/dev/null | tail -n 5 | sed "s/^/<4>kiosk-events: /"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' UNIT ) diff --git a/tests/unit/test_credentials_provisioning.py b/tests/unit/test_credentials_provisioning.py index f9c36654..852207e1 100644 --- a/tests/unit/test_credentials_provisioning.py +++ b/tests/unit/test_credentials_provisioning.py @@ -24,7 +24,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-bluefin-firstboot-credentials.preset" + / "20-bluefin-firstboot-credentials.preset" ) NETWORK_GENERATOR_PRESET = ( REPO_ROOT @@ -32,7 +32,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-systemd-network-generator.preset" + / "20-bluefin-systemd-network-generator.preset" ) NETWORK_GENERATOR_DROPIN = ( REPO_ROOT diff --git a/tests/unit/test_k0s_first_boot.py b/tests/unit/test_k0s_first_boot.py index a185a611..72e4a124 100644 --- a/tests/unit/test_k0s_first_boot.py +++ b/tests/unit/test_k0s_first_boot.py @@ -20,7 +20,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-k0s-first-boot.preset" + / "20-bluefin-k0s-first-boot.preset" ) NETWORK = ROOT / "files" / "os" / "systemd" / "network" / "20-wired.network" NETWORK_PRESET = ( @@ -29,7 +29,7 @@ / "os" / "systemd" / "system-preset" - / "zz-enable-networkd.preset" + / "20-bluefin-networkd.preset" ) ELEMENT = ROOT / "elements" / "bluefin-server" / "os-k0s-first-boot.bst" NETWORK_ELEMENT = ROOT / "elements" / "bluefin-server" / "os-networkd.bst" @@ -146,3 +146,19 @@ def test_k0s_sysupdate_transfer_is_packaged_as_a_component() -> None: "bluefin-server/os-k0s-sysupdate.bst" in STACK.read_text(encoding="utf-8") ) + + +def test_presets_sort_before_flatcar_disable_all() -> None: + # Flatcar's /usr ships /usr/lib/systemd/system-preset/99-default.preset + # containing a catch-all "disable *". systemd applies preset files in + # lexicographic filename order and the first matching line wins, so any + # Bluefin preset named after 99-default.preset is silently ignored at + # first-boot preset-all. + preset_dir = PRESET.parent + presets = sorted(p.name for p in preset_dir.glob("*.preset")) + assert presets, f"no presets found in {preset_dir}" + for name in presets: + assert name < "99-default.preset", ( + f"{name} sorts after Flatcar's 99-default.preset; its " + "'disable *' catch-all would win and the preset would be ignored" + ) From 6888c2fff690634b1ad0cf4546c3a4fff10bbd3e Mon Sep 17 00:00:00 2001 From: mendezr Date: Thu, 24 Sep 2026 10:33:06 +0000 Subject: [PATCH 10/15] fix(security): force rotation of seeded root password at first login MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DDI seeds a well-known root password ('bluefin', hash public in this repo) into every shipped image with shadow last-change 19700, so the credential never expires and is identical fleet-wide. PR #169 already restored the sshd drop-in hardening (PermitRootLogin prohibit-password, PasswordAuthentication no), which closes the remote root path; the remaining gap from #168 is that console bring-up never forces rotation. Set the shadow last-change field to 0 (the `chage -d 0 root` equivalent): every login as root — console included — is forced onto a per-install password before any shell is granted. Update the console banner to disclose the forced change, and add a unit contract locking in both the rotation and the SSH no-password-auth policy as a regression guard. The long-term removal of the baked password (core operator account, systemd-credentials key provisioning) is tracked by #80; this is the minimal interim hardening requested by the issue. Fixes: #168 Hive-Run: projectbluefin/server#168 Hive-Plan: Security remediation — interim hardening of seeded root credential (issue #168, recommendation item 2) Hive-Spec: docs/superpowers/specs/2026-09-09-core-login-design.md#interim-shadow-expiry Assisted-by: GLM (z-ai/glm-5.3-flash) via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: mendezr --- elements/oci/bluefin-server-ddi.bst | 9 ++- files/os/issue.d/40-kubestellar.issue | 2 +- tests/unit/test_root_credential_contract.py | 71 +++++++++++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_root_credential_contract.py diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index c58f9d8e..a6915963 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -76,9 +76,14 @@ config: # Ensure all standard virtual, serial, and hypervisor consoles allow root login printf 'tty0\ntty1\ntty2\ntty3\ntty4\ntty5\ntty6\nttyS0\nttyS1\nconsole\nhvc0\n' > /layer/etc/securetty - # Seed default root password 'bluefin' into /etc/shadow for console and ssh login + # Seed default root password 'bluefin' for local console bring-up only. + # The last-change field is 0 (equivalent to `chage -d 0 root`), so the + # password is expired at first login and the console forces a per-install + # rotation before any shell is granted. The seeded credential must never + # be usable over the network: the sshd drop-in sets PermitRootLogin + # prohibit-password and PasswordAuthentication no. mkdir -p /layer/etc - printf 'root:$6$bluefin123$o2pQjPN1n3ZCnYUROlA01xAdnGO7mhVRAVw98x4xc8LAUIJv7b44bBNoER6fM4P.rrj4ePYFfkx9WvZICU8nt/:19700:0:99999:7:::\n' > /layer/etc/shadow + printf 'root:$6$bluefin123$o2pQjPN1n3ZCnYUROlA01xAdnGO7mhVRAVw98x4xc8LAUIJv7b44bBNoER6fM4P.rrj4ePYFfkx9WvZICU8nt/:0:0:99999:7:::\n' > /layer/etc/shadow chmod 0600 /layer/etc/shadow printf 'root:x:0:0:root:/root:/bin/bash\n' > /layer/etc/passwd chmod 0644 /layer/etc/passwd diff --git a/files/os/issue.d/40-kubestellar.issue b/files/os/issue.d/40-kubestellar.issue index 862fc3b5..cef7eaaf 100644 --- a/files/os/issue.d/40-kubestellar.issue +++ b/files/os/issue.d/40-kubestellar.issue @@ -1,5 +1,5 @@ Bluefin Server (\l) -Default login: root / bluefin +Default login: root / bluefin (password change forced at first login) KubeStellar Console: https://127.0.0.1:8080/ (loopback, TLS only) Remote access: ssh -L 8080:127.0.0.1:8080 @ diff --git a/tests/unit/test_root_credential_contract.py b/tests/unit/test_root_credential_contract.py new file mode 100644 index 00000000..8ee2ba95 --- /dev/null +++ b/tests/unit/test_root_credential_contract.py @@ -0,0 +1,71 @@ +"""Contracts for the seeded root credential and SSH login policy. + +The shipped DDI seeds a well-known default root password for local console +bring-up only. This file pins the two safety contracts around it: + +- the password is expired at first login (shadow last-change field 0, the + `chage -d 0` equivalent), so every install is forced onto a per-install + secret before any shell is granted; and +- the effective SSH policy never allows the seeded password (or any password) + over the network. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +DDI_ELEMENT = REPO_ROOT / "elements" / "oci" / "bluefin-server-ddi.bst" +SSHD_DROPIN = REPO_ROOT / "files" / "os" / "ssh" / "sshd_config.d" / "bluefin-server.conf" +ISSUE_FILE = REPO_ROOT / "files" / "os" / "issue.d" / "40-kubestellar.issue" + + +def _root_shadow_fields(ddi_element: str) -> list[str]: + match = re.search( + r"printf '(root:[^']*)", ddi_element + ) + assert match, "DDI element must seed a root account into /etc/shadow" + shadow_line = match.group(1).split("\\n")[0] + fields = shadow_line.split(":") + assert fields[0] == "root", "seeded shadow line must be the root account" + assert len(fields) == 9, "seeded shadow line must have all 9 shadow fields" + return fields + + +def test_root_password_is_expired_for_first_login_rotation() -> None: + ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") + fields = _root_shadow_fields(ddi_element) + + # shadow field 2 is the last-change date: 0 means expired, so login + # (console included) forces a password rotation before any shell. + assert fields[2] == "0", ( + "seeded root password must be expired at first login (last-change 0); " + "a fixed date leaves the well-known default usable indefinitely" + ) + + +def test_root_password_uses_no_fixed_build_date() -> None: + ddi_element = DDI_ELEMENT.read_text(encoding="utf-8") + fields = _root_shadow_fields(ddi_element) + + # Regression guard: a fixed last-change date (e.g. 19700) ships the same + # usable credential in every image with no rotation enforced. + assert fields[2] != "19700" + + +def test_sshd_dropin_never_permits_password_authentication() -> None: + sshd = SSHD_DROPIN.read_text(encoding="utf-8") + + assert re.search(r"^PermitRootLogin\s+prohibit-password$", sshd, re.MULTILINE) + assert re.search(r"^PasswordAuthentication\s+no$", sshd, re.MULTILINE) + assert re.search(r"^KbdInteractiveAuthentication\s+no$", sshd, re.MULTILINE) + assert not re.search(r"^PermitRootLogin\s+yes$", sshd, re.MULTILINE) + assert not re.search(r"^PasswordAuthentication\s+yes$", sshd, re.MULTILINE) + + +def test_console_banner_discloses_forced_password_change() -> None: + banner = ISSUE_FILE.read_text(encoding="utf-8") + + assert "Default login: root / bluefin" in banner + assert "password change forced at first login" in banner From 2b9339efd25387d3ba31439ccc46e18e4d7577a5 Mon Sep 17 00:00:00 2001 From: sec-check Date: Thu, 24 Sep 2026 10:37:01 -0400 Subject: [PATCH 11/15] test(installer): surface node, pull, and k0s runtime errors in the kiosk waiter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The preset rename in this PR gets k0scontroller active, but the smoke gate still times out: every pod sits Pending/ContainerCreating for the full 900s window (kube-proxy ContainerCreating, kube-router Init:0/2, node tainted not-ready). The current 30s debug line only shows Warning events, and the FailedScheduling flood from ten Pending pods pushes the informative events (image pulls, sandbox creation, CNI) out of the tail -n 5 window. Extend the in-guest waiter's periodic dump so the serial log carries enough to diagnose the next layer: - node status line (kiosk-node) — shows Ready/NotReady and the CRI version - all event kinds minus the FailedScheduling flood (kiosk-events) - error/fail/taint lines from the k0scontroller journal (kiosk-k0s), where the embedded kubelet and containerd log Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index e163c421..788423f4 100644 --- a/Justfile +++ b/Justfile @@ -466,7 +466,7 @@ test-installer-artifact: Type=oneshot TimeoutStartSec=infinity StandardOutput=journal+console - ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; k0s kubectl get pods -A --no-headers 2>/dev/null | sed "s/^/<4>kiosk-pods: /"; k0s kubectl get events -A --field-selector type=Warning --no-headers 2>/dev/null | tail -n 5 | sed "s/^/<4>kiosk-events: /"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' + ExecStart=/usr/bin/bash -c 'echo "<4>kiosk-ready: polling https://127.0.0.1:8080"; i=0; until [ "$(curl --silent --fail --insecure --max-time 2 https://127.0.0.1:8080/healthz | jq -r .status)" = ok ] && [ "$(curl --silent --fail --insecure --max-time 2 --output /dev/null --write-out "%%{http_code}" https://127.0.0.1:8080/)" = 200 ]; do i=$((i+1)); if [ $((i %% 15)) -eq 0 ]; then echo "<4>kiosk-ready: waiting k0s=$(systemctl is-active k0scontroller.service) hz=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/healthz) root=$(curl -sk -o /dev/null -w "%%{http_code}" --max-time 2 https://127.0.0.1:8080/) pods=$(k0s kubectl get pods -A --no-headers 2>/dev/null | wc -l) running=$(k0s kubectl get pods -A --no-headers 2>/dev/null | grep -c Running)"; k0s kubectl get pods -A --no-headers 2>/dev/null | sed "s/^/<4>kiosk-pods: /"; k0s kubectl get nodes -o wide --no-headers 2>/dev/null | sed "s/^/<4>kiosk-node: /"; k0s kubectl get events -A --no-headers 2>/dev/null | grep -v FailedScheduling | tail -n 8 | sed "s/^/<4>kiosk-events: /"; journalctl -u k0scontroller --no-pager -n 800 2>/dev/null | grep -iE "error|fail|taint" | grep -vE "FailedScheduling|apply.*retrying" | tail -n 8 | sed "s/^/<4>kiosk-k0s: /"; fi; sleep 2; done; echo "<4>KIOSK_CONSOLE_READY"' UNIT ) From f1a52e5f56e1146c2418742fcbf69d963755e151 Mon Sep 17 00:00:00 2001 From: sec-check Date: Thu, 24 Sep 2026 14:50:29 -0400 Subject: [PATCH 12/15] fix(boot): seed /etc/resolv.conf symlink via tmpfiles.d so kubelet can create pod sandboxes The preset rename got k0s and kubelet running, but every pod sandbox then failed with 'open /etc/resolv.conf: no such file or directory' and the console readiness gate timed out (run 36014129412). Flatcar ships the /etc/resolv.conf symlink inside the image /etc and strips /etc-populating lines from /usr/lib/tmpfiles.d at image build; this DDI imports only Flatcar's /usr and first-boots with an empty /etc, so the symlink never exists. Recreate Flatcar's stock symlink (/etc/resolv.conf -> ../run/systemd/resolve/resolv.conf, the full resolver list, matching Flatcar >= 3510.3.0) with a tmpfiles.d rule installed from a new os-resolv-conf.bst element, and guard it with a unit test. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: sec-check --- elements/bluefin-server/os-resolv-conf.bst | 14 ++++++++++ elements/bluefin-server/os-stack.bst | 3 +++ files/os/tmpfiles.d/20-bluefin-resolv.conf | 11 ++++++++ tests/unit/test_k0s_first_boot.py | 30 ++++++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 elements/bluefin-server/os-resolv-conf.bst create mode 100644 files/os/tmpfiles.d/20-bluefin-resolv.conf diff --git a/elements/bluefin-server/os-resolv-conf.bst b/elements/bluefin-server/os-resolv-conf.bst new file mode 100644 index 00000000..79361cc1 --- /dev/null +++ b/elements/bluefin-server/os-resolv-conf.bst @@ -0,0 +1,14 @@ +kind: import +description: | + Install the tmpfiles.d rule that recreates Flatcar's /etc/resolv.conf + symlink to systemd-resolved's resolv.conf. The Flatcar /usr payload does + not carry it (Flatcar ships the symlink in the image /etc, which this DDI + does not import), and kubelet requires /etc/resolv.conf to create any pod + sandbox. + +sources: + - kind: local + path: files/os/tmpfiles.d + +config: + target: /usr/lib/tmpfiles.d diff --git a/elements/bluefin-server/os-stack.bst b/elements/bluefin-server/os-stack.bst index 44f80576..5aa65158 100644 --- a/elements/bluefin-server/os-stack.bst +++ b/elements/bluefin-server/os-stack.bst @@ -44,6 +44,9 @@ depends: # Wired DHCP configuration for the installed OS - bluefin-server/os-networkd.bst + # /etc/resolv.conf symlink to systemd-resolved (tmpfiles.d) + - bluefin-server/os-resolv-conf.bst + # Activate the optional k0s sysext after installation - bluefin-server/os-k0s-first-boot.bst diff --git a/files/os/tmpfiles.d/20-bluefin-resolv.conf b/files/os/tmpfiles.d/20-bluefin-resolv.conf new file mode 100644 index 00000000..085ac2ab --- /dev/null +++ b/files/os/tmpfiles.d/20-bluefin-resolv.conf @@ -0,0 +1,11 @@ +# Flatcar's image build populates /etc/resolv.conf in the image's /etc and +# strips /etc-populating lines from /usr/lib/tmpfiles.d (see +# flatcar/scripts build_library/build_image_util.sh). This DDI imports only +# Flatcar's /usr and boots with an empty /etc, so nothing ever creates the +# symlink and kubelet fails every pod sandbox with +# "open /etc/resolv.conf: no such file or directory". +# +# Recreate Flatcar's stock symlink: /etc/resolv.conf points at +# systemd-resolved's full resolver list (not the 127.0.0.53 stub), matching +# Flatcar >= 3510.3.0 behaviour. +L /etc/resolv.conf - - - - ../run/systemd/resolve/resolv.conf diff --git a/tests/unit/test_k0s_first_boot.py b/tests/unit/test_k0s_first_boot.py index 72e4a124..1f89861e 100644 --- a/tests/unit/test_k0s_first_boot.py +++ b/tests/unit/test_k0s_first_boot.py @@ -162,3 +162,33 @@ def test_presets_sort_before_flatcar_disable_all() -> None: f"{name} sorts after Flatcar's 99-default.preset; its " "'disable *' catch-all would win and the preset would be ignored" ) + + +def test_etc_resolv_conf_symlink_is_seeded_for_kubelet() -> None: + # Flatcar ships /etc/resolv.conf inside the image /etc and strips + # /etc-populating lines from /usr/lib/tmpfiles.d at image build. This DDI + # imports only Flatcar's /usr and first-boots with an empty /etc, so + # without an explicit tmpfiles.d rule the symlink never exists and + # kubelet fails every pod sandbox with + # "open /etc/resolv.conf: no such file or directory". + conf = ( + ROOT / "files" / "os" / "tmpfiles.d" / "20-bluefin-resolv.conf" + ) + assert conf.is_file(), "resolv.conf tmpfiles rule is missing" + lines = [ + line + for line in conf.read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") + ] + assert lines == [ + "L /etc/resolv.conf - - - - ../run/systemd/resolve/resolv.conf" + ] + + element = ( + ROOT / "elements" / "bluefin-server" / "os-resolv-conf.bst" + ).read_text(encoding="utf-8") + assert "path: files/os/tmpfiles.d" in element + assert "target: /usr/lib/tmpfiles.d" in element + assert ( + "bluefin-server/os-resolv-conf.bst" in STACK.read_text(encoding="utf-8") + ) From 1004614e2d8e466c0e512a1c6f12a45be0278826 Mon Sep 17 00:00:00 2001 From: Jorge Castro Date: Thu, 24 Sep 2026 18:00:43 -0400 Subject: [PATCH 13/15] fix(installer): bare-metal rootfs stability, OEM unit cleanup, and preflight hardening - Set GrowFileSystem=no on bluefin-server-root-a in 20-root-a.conf to prevent interrupted first-boot metadata corruption on the immutable root partition. - Disable experimental features (reflink, rmapbt, parent, exchange) and widen headroom in elements/oci/bluefin-server-ddi.bst mkfs.xfs to prevent ENOSPC metadata reservation failure during log replay. - Remove Flatcar oem.mount, dev-disk-by-label-OEM.device, and fsck drop-ins in elements/flatcar/flatcar-usr.bst to prevent 90s device timeout on first boot. - Default INSTALLER_PART="" before preflight in elements/oci/bluefin-server-installer.bst so PXE boots under `set -u` do not fail on unbound variable. - Update contract tests in test_flatcar_usr.py and test_repart_layout.py. Assisted-by: claude-3-5-sonnet via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/flatcar/flatcar-usr.bst | 6 ++++++ elements/oci/bluefin-server-ddi.bst | 7 +++---- elements/oci/bluefin-server-installer.bst | 1 + files/installer/repart.d/20-root-a.conf | 2 +- tests/unit/test_flatcar_usr.py | 12 ++++++++++++ tests/unit/test_repart_layout.py | 8 ++++---- 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/elements/flatcar/flatcar-usr.bst b/elements/flatcar/flatcar-usr.bst index df8d299a..0c6ca19d 100644 --- a/elements/flatcar/flatcar-usr.bst +++ b/elements/flatcar/flatcar-usr.bst @@ -91,3 +91,9 @@ config: # ensure-sysext.service: replaced by Bluefin's systemd-sysext.service and sysupdate integration rm "%{install-root}/usr/lib/systemd/system/ensure-sysext.service" rm "%{install-root}/usr/lib/systemd/system/sysinit.target.wants/ensure-sysext.service" + + # oem.mount: replaced by Bluefin's standard GPT partition layout (no OEM partition) + rm "%{install-root}/usr/lib/systemd/system/oem.mount" + rm "%{install-root}/usr/lib/systemd/system/local-fs.target.wants/oem.mount" + rm "%{install-root}/usr/lib/systemd/system/dev-disk-by\\x2dlabel-OEM.device" + rm -r "%{install-root}/usr/lib/systemd/system/systemd-fsck@dev-disk-by\\x2dlabel-OEM.service.d" diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index a6915963..d41549c7 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -100,9 +100,8 @@ config: ROOT_BYTES=$(du -sb /layer | cut -f1) echo "Computed root filesystem size: ${ROOT_BYTES} bytes" - # 25% overhead and 100MB margin for XFS metadata, journal, and structures. - # ponytail: 25% overhead + 100MB margin for XFS. - TARGET_BYTES=$(( ROOT_BYTES + ROOT_BYTES / 4 + 100 * 1024 * 1024 )) + # 30% overhead and 250MB margin for XFS metadata, journal, and structures. + TARGET_BYTES=$(( ROOT_BYTES + ROOT_BYTES * 3 / 10 + 250 * 1024 * 1024 )) # Align TARGET_BYTES to 4096-byte boundary (multiple of 512/4096) for sector compatibility # ponytail: sector alignment is required by systemd-repart CopyBlocks=. @@ -113,7 +112,7 @@ config: truncate -s "${TARGET_BYTES}" "${OUT}/${FNAME}" echo "Allocating XFS image of ${TARGET_BYTES} bytes" - mkfs.xfs -f -L bluefin-root -p /layer "${OUT}/${FNAME}" + mkfs.xfs -f -L bluefin-root -m reflink=0,rmapbt=0 -n parent=0 -i exchange=0 -p /layer "${OUT}/${FNAME}" zstd --rm -T0 -19 -q "${OUT}/${FNAME}" -o "${OUT}/${FNAME}.zst" ( diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 3373a1b3..994402be 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -280,6 +280,7 @@ config: > /usr/lib/repart.sysinstall.d/20-root-a.conf fi + INSTALLER_PART="" # Hard preflight: the embedded installer-media path needs the # bluefin-installer-data partition. Under PXE (inst.ddi_url=) there is no # installer media, so the check is skipped when inst.ddi_url is set. diff --git a/files/installer/repart.d/20-root-a.conf b/files/installer/repart.d/20-root-a.conf index e843b30a..94072ccc 100644 --- a/files/installer/repart.d/20-root-a.conf +++ b/files/installer/repart.d/20-root-a.conf @@ -8,4 +8,4 @@ Label=bluefin-server-root-a CopyBlocks=/dev/disk/by-partlabel/bluefin-installer-data SizeMinBytes=4G SizeMaxBytes=8G -GrowFileSystem=yes +GrowFileSystem=no diff --git a/tests/unit/test_flatcar_usr.py b/tests/unit/test_flatcar_usr.py index b7d2f70a..b84e2f43 100644 --- a/tests/unit/test_flatcar_usr.py +++ b/tests/unit/test_flatcar_usr.py @@ -50,6 +50,10 @@ "usr/lib/systemd/system/oem-cloudinit.service", "usr/lib/systemd/system/ensure-sysext.service", "usr/lib/systemd/system/sysinit.target.wants/ensure-sysext.service", + "usr/lib/systemd/system/oem.mount", + "usr/lib/systemd/system/local-fs.target.wants/oem.mount", + "usr/lib/systemd/system/dev-disk-by\\x2dlabel-OEM.device", + "usr/lib/systemd/system/systemd-fsck@dev-disk-by\\x2dlabel-OEM.service.d", ] REMOVED_EXTRA_FILES = [ @@ -143,6 +147,7 @@ def test_flatcar_usr_explicit_removals_and_replacements() -> None: "flatcar-update", "download_sysext", "ensure-sysext", + "oem.mount", ]: pattern = rf"#\s*{component}.*?:.*?replaced by" assert re.search(pattern, content, re.IGNORECASE), ( @@ -224,6 +229,10 @@ def test_flatcar_usr_contract_execution(tmp_path: Path) -> None: (systemd_dir / "enable-oem-cloudinit.service").write_text("mock-unit", encoding="utf-8") (systemd_dir / "oem-cloudinit.service").write_text("mock-unit", encoding="utf-8") (systemd_dir / "ensure-sysext.service").write_text("mock-unit", encoding="utf-8") + (systemd_dir / "oem.mount").write_text("mock-unit", encoding="utf-8") + (systemd_dir / "dev-disk-by\\x2dlabel-OEM.device").write_text("mock-unit", encoding="utf-8") + (systemd_dir / "systemd-fsck@dev-disk-by\\x2dlabel-OEM.service.d").mkdir(parents=True) + (systemd_dir / "systemd-fsck@dev-disk-by\\x2dlabel-OEM.service.d" / "10-mock.conf").write_text("mock", encoding="utf-8") wants_dir1 = systemd_dir / "multi-user.target.wants" wants_dir1.mkdir(parents=True) @@ -232,6 +241,9 @@ def test_flatcar_usr_contract_execution(tmp_path: Path) -> None: (wants_dir1 / "locksmithd.service").symlink_to("../locksmithd.service") (wants_dir1 / "enable-oem-cloudinit.service").symlink_to("../enable-oem-cloudinit.service") + local_fs_dir = systemd_dir / "local-fs.target.wants" + local_fs_dir.mkdir(parents=True) + (local_fs_dir / "oem.mount").symlink_to("../oem.mount") wants_dir2 = systemd_dir / "sysinit.target.wants" wants_dir2.mkdir(parents=True) (wants_dir2 / "ignition-delete-config.service").symlink_to("../ignition-delete-config.service") diff --git a/tests/unit/test_repart_layout.py b/tests/unit/test_repart_layout.py index e21e5b46..48684707 100644 --- a/tests/unit/test_repart_layout.py +++ b/tests/unit/test_repart_layout.py @@ -171,11 +171,11 @@ def test_root_slot_copies_blocks_from_a_label_the_installer_media_stamps(): ) -def test_root_slot_grows_and_is_bounded_below_the_var_partition(): +def test_root_slot_is_bounded_below_the_var_partition(): root = next(s for s in partitions().values() if s["Type"] == "root") - assert root.get("GrowFileSystem") == "yes", ( - "the root filesystem must grow to its partition, the DDI payload is " - "smaller than SizeMinBytes" + assert root.get("GrowFileSystem") == "no", ( + "the immutable DDI payload must not auto-grow at boot, which risks " + "interrupted first-boot metadata corruption" ) assert "SizeMaxBytes" in root, ( "the root slot must be capped, otherwise /var gets no space on small disks" From 51c31e04afb387026cc96bbacb9255e0b91a64d0 Mon Sep 17 00:00:00 2001 From: Jorge Castro Date: Thu, 24 Sep 2026 19:15:26 -0400 Subject: [PATCH 14/15] fix(installer): auto-pass target disk when singular and clarify numbered selection - When exactly one non-installer target disk is present, pass it directly to systemd-sysinstall so the user is not prompted for the device path. - When multiple target disks are present, print a hint that systemd-sysinstall's native interactive prompt accepts single-digit menu numbers (1, 2, ...). Assisted-by: claude-3-5-sonnet via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/oci/bluefin-server-installer.bst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/elements/oci/bluefin-server-installer.bst b/elements/oci/bluefin-server-installer.bst index 994402be..9a6a84ff 100644 --- a/elements/oci/bluefin-server-installer.bst +++ b/elements/oci/bluefin-server-installer.bst @@ -311,7 +311,7 @@ config: echo "==> Found installer partition at ${INSTALLER_PART_PATH} -> ${INSTALLER_PART}" fi # Auto-detect target disk for unattended install (any raw disk of type "disk" that is not the installer, is not read-only, and has size > 0) - TARGET_DISK="" + TARGET_DISKS=() while read -r name type ro size; do [ "${type}" = "disk" ] || continue [ "${ro}" = "0" ] || continue @@ -322,9 +322,9 @@ config: continue fi - TARGET_DISK="${name}" - break + TARGET_DISKS+=("${name}") done < <(lsblk -p -d -n -o NAME,TYPE,RO,SIZE -b) + TARGET_DISK="${TARGET_DISKS[0]:-}" # inst.target_disk= overrides auto-detection (explicit target under PXE). if [ -n "${TARGET_DISK_OVERRIDE}" ]; then @@ -397,6 +397,17 @@ config: fi else echo "==> Running in INTERACTIVE mode..." + # If exactly one non-installer target disk exists, pass it directly to systemd-sysinstall + # so the user isn't prompted with device serial/path queries. + # If multiple disks exist, systemd-sysinstall prompts with its native numbered list; + # print a hint reminding users they can simply type the item number (1, 2, ...). + EXTRA_ARGS=() + if [ "${#TARGET_DISKS[@]}" -eq 1 ]; then + echo "==> Automatically selected only available installation disk: ${TARGET_DISKS[0]}" + EXTRA_ARGS+=("${TARGET_DISKS[0]}") + elif [ "${#TARGET_DISKS[@]}" -gt 1 ]; then + echo "==> Multiple disks detected. You can type the menu item number (1, 2, ...) to choose." + fi /usr/bin/systemd-sysinstall \ --kernel=/usr/lib/bluefin-server/bluefin-server.efi \ --variables=yes \ @@ -404,7 +415,8 @@ config: --mute-console=yes \ --copy-locale=yes \ --copy-keymap=yes \ - --copy-timezone=yes + --copy-timezone=yes \ + "${EXTRA_ARGS[@]}" stage_fallback_bootloader echo "==> Installation complete! Rebooting..." systemctl reboot From f549a9e9febf66ebf9bb401c607d8b8dc218838c Mon Sep 17 00:00:00 2001 From: Jorge Castro Date: Thu, 24 Sep 2026 20:48:58 -0400 Subject: [PATCH 15/15] fix(installer): replace hardlinked CA tree before DDI copy Avoid copying BuildBarn CAS-hardlinked certificate files onto themselves when composing the OS DDI. Align installer documentation with the existing no-grow root recipe. Assisted-by: github-copilot/gpt-6-sol via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/ddi-installer-build.md | 4 ++-- docs/skills/ddi-installer.md | 7 +++---- elements/oci/bluefin-server-ddi.bst | 3 +++ tests/unit/test_installer_contract.py | 1 - 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index b3c11158..8a4b26b7 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -168,7 +168,7 @@ The release process is driven by `.github/workflows/build.yml`: - Boot cmdline uses a hardcoded device path like `root=/dev/vda2`. - DDI decompression is placed before the cpio step. - The installer data partition uses FAT32/vfat instead of XFS. -- The repart recipe is missing `GrowFileSystem=yes` for the copied rootfs. +- The repart recipe is missing `GrowFileSystem=no` for the copied rootfs. - Unattended target-disk discovery is not filtered and can select empty devices. ## Verification @@ -183,7 +183,7 @@ The release process is driven by `.github/workflows/build.yml`: - [ ] The live installer does not bake hardcoded SSH keys or pre-hashed root passwords. - [ ] The DDI is decompressed after the cpio step. -- [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=yes`. +- [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=no`. ## See also diff --git a/docs/skills/ddi-installer.md b/docs/skills/ddi-installer.md index 7550e484..4699417a 100644 --- a/docs/skills/ddi-installer.md +++ b/docs/skills/ddi-installer.md @@ -87,9 +87,8 @@ container runtime pod sandboxes. 6. `20-root-a.conf` copies the DDI block-for-block from `/dev/disk/by-partlabel/bluefin-installer-data` (the embedded DDI data partition on the installer media). -7. Target OS volume expansion is handled by `systemd-growfs`; the target OS - stack includes `xfsprogs` so the root and `/var` filesystems can grow to fill - their partitions on first boot. +7. The immutable root DDI is copied without filesystem growth; `/var` uses + `GrowFileSystem=yes` to fill the remaining target disk. ## Partition Layout @@ -231,7 +230,7 @@ in tmpfs, so plan roughly `live-env + DDI` of free RAM (~8 GiB guidance). - [ ] The interactive installer service sets `TTYPath=/dev/tty0` so the TUI appears on the attached display even when serial is the primary console. - [ ] `bluefin-server-installer.bst` decompresses the DDI after the cpio step. -- [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=yes`. +- [ ] `files/installer/repart.d/20-root-a.conf` has `GrowFileSystem=no`. ## See also diff --git a/elements/oci/bluefin-server-ddi.bst b/elements/oci/bluefin-server-ddi.bst index d41549c7..bee1ed53 100644 --- a/elements/oci/bluefin-server-ddi.bst +++ b/elements/oci/bluefin-server-ddi.bst @@ -41,6 +41,9 @@ config: depmod -b /layer/usr "${KVER}" # Stage extracted CA certificate bundle into target DDI rootfs + # Remote execution may hard-link the staged CA files to the build root; + # replace the target tree before copying rather than copying onto itself. + rm -rf /layer/etc/pki/ca-trust/extracted mkdir -p /layer/etc/pki/ca-trust/extracted /layer/etc/ssl/certs /layer/etc/pki/tls/certs cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/ ln -sf /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /layer/etc/ssl/certs/ca-certificates.crt diff --git a/tests/unit/test_installer_contract.py b/tests/unit/test_installer_contract.py index e1f9eeab..42cd524f 100644 --- a/tests/unit/test_installer_contract.py +++ b/tests/unit/test_installer_contract.py @@ -101,7 +101,6 @@ def test_ddi_generates_module_indexes_for_runtime_filesystem_drivers() -> None: assert "freedesktop-sdk.bst:components/kmod.bst" in ddi_element assert 'depmod -b /layer/usr "${KVER}"' in ddi_element - assert "cp -a /etc/pki/ca-trust/extracted/* /layer/etc/pki/ca-trust/extracted/" in ddi_element assert "tls-ca-bundle.pem" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-firstboot.service" in ddi_element assert "ln -sf /dev/null /layer/etc/systemd/system/systemd-homed-firstboot.service" in ddi_element