diff --git a/docs/skills/local-testing.md b/docs/skills/local-testing.md index c4c26f613..babc6c831 100644 --- a/docs/skills/local-testing.md +++ b/docs/skills/local-testing.md @@ -144,7 +144,17 @@ persists. `just luks-test` runs `iso/scripts/luks-e2e.sh` against a debug live ISO (`just iso testing 1`). It checks the live GNOME session, installs to a disposable LUKS2 disk from the embedded payload, boots without the ISO, -unlocks the disk, and checks graphical login and extension states. +unlocks the disk, confirms `bootc status` reports the offline embedded +payload (not a network pull) on a guest with no route out, and checks +graphical login and extension states. A trailing check, after login, +confirms every default Flatpak in the Brewfile contract is also present +offline -- deferred past login because it deploys asynchronously on first +boot. Both gates have escape hatches for unblocking a promotion when the +gate itself, rather than the image, is at fault: `UTAH_E2E_FLATPAKS` sets the +expected Flatpak set directly (empty skips the check), and +`UTAH_E2E_PAYLOAD_CHECK=""` skips the booted-image assertion. The booted-image +read falls back to `sudo` when `bootc status --json` returns nothing to the +unprivileged test user. Read the recipe and script prerequisites before running it: it creates test accounts and requires local QEMU/KVM access, not a production installation. diff --git a/iso/scripts/luks-e2e.sh b/iso/scripts/luks-e2e.sh index 3b69fc136..aaad9ef6e 100755 --- a/iso/scripts/luks-e2e.sh +++ b/iso/scripts/luks-e2e.sh @@ -1,15 +1,22 @@ #!/usr/bin/env bash # Install Utah from its live ISO onto an encrypted disk, then log in to it. # -# Six phases, each of which can fail the test on its own: +# Seven phases, each of which can fail the test on its own: # 1. boot the live ISO in QEMU with one blank disk attached # 2. prove the live session reached a graphical desktop, not just a shell # 3. run the installer with a LUKS passphrase and a user account # 4. boot the installed disk with no ISO, so it must come up on its own # 5. answer Plymouth's passphrase prompt -# 6. log in at the GDM greeter and prove a GNOME session is running +# 6. reach the graphical target and confirm the booted image is the +# offline embedded payload, with no network route out +# 7. log in at the GDM greeter and prove a GNOME session is running # -# Phases 2 and 6 are the point. An image can install perfectly, unlock +# A trailing check, after phase 7's desktop and fastfetch evidence, confirms +# the Brewfile's default Flatpak set is also present offline -- deferred +# because it deploys asynchronously on first boot and checking right after +# login undercounts it. +# +# Phases 2 and 7 are the point. An image can install perfectly, unlock # perfectly, and still be useless if the desktop never starts -- and a boot # that stops at a text console looks identical to a working one if all you # check is that the machine came up. @@ -167,7 +174,35 @@ send_keys() { } qemu_pids=() +# Never the disk itself: post-testing-e2e.yml explicitly never uploads +# install.qcow2 because it carries the test LUKS passphrase, the test +# account password, and host keys. qemu-img's own metadata is not guest +# data -- virtual/actual size and format are enough to tell a disk that +# never got written (an install that silently no-op'd) from one whose boot +# phases simply could not unlock or start it, without shipping the guest +# filesystem anywhere. Written under WORK, which the CI workflow already +# globs for "*.log" on failure. +preserve_failed_disk_diagnostics() { + [[ -f "${INSTALL_DISK}" ]] || return 0 + local out="${WORK}/failed-disk-info-$(date +%Y%m%d-%H%M%S).log" + # -U: this runs from the EXIT trap, before the kill loop below has stopped + # QEMU, so the image is still write-locked by the running guest. Without + # it the retained log holds a "Failed to get shared 'write' lock" error + # instead of the size and format the diagnostic exists to record. + { echo "=== qemu-img info: ${INSTALL_DISK} ==="; qemu-img info -U "${INSTALL_DISK}"; } \ + > "${out}" 2>&1 || true + echo " disk diagnostics kept (metadata only, never the disk itself): ${out}" >&2 +} + cleanup() { + # $? must be read before anything else runs in this function, or it + # stops reflecting the exit that triggered the trap. `fail()` funnels + # every deliberate failure through `exit 1`, but a bare command failing + # under `set -e` (the installer step in phase 3, for one) aborts here + # directly without ever calling `fail()` -- so diagnostics belong in the + # trap that always runs, not only in the helper that sometimes does. + local status=$? + (( status != 0 )) && preserve_failed_disk_diagnostics for pid in "${qemu_pids[@]:-}"; do [[ -n "${pid}" ]] && kill "${pid}" 2>/dev/null || true done @@ -207,7 +242,7 @@ if [[ -f "${WORK}/try.pid" ]] && kill -0 "$(cat "${WORK}/try.pid" 2>/dev/null)" fail "a VM from 'just try-installed' is running on this work directory (pid $(cat "${WORK}/try.pid")); stop it first: kill \$(cat ${WORK}/try.pid)" fi -echo "=== Phase 1/6: boot the live ISO ===" +echo "=== Phase 1/7: boot the live ISO ===" rm -f "${INSTALL_DISK}" "${MONITOR_LIVE}" "${MONITOR_INSTALLED}" \ "${SERIAL_LIVE}" "${SERIAL_INSTALLED}" qemu-img create -f qcow2 "${INSTALL_DISK}" 64G >/dev/null @@ -252,7 +287,7 @@ for i in $(seq 1 90); do sleep 5 done -echo "=== Phase 2/6: prove the live session reached a desktop ===" +echo "=== Phase 2/7: prove the live session reached a desktop ===" # The live ISO autologins liveuser into GNOME. If that silently degraded to a # text console the installer would still be reachable over SSH and every other # check here would pass, so assert the session explicitly. @@ -278,7 +313,7 @@ live_session_type="$(ssh_live "loginctl show-session \$(loginctl show-user liveu echo " live session type: ${live_session_type:-unknown}" shot live-desktop "${MONITOR_LIVE}" -echo "=== Phase 3/6: install onto an encrypted disk ===" +echo "=== Phase 3/7: install onto an encrypted disk ===" # An empty "image" with a "targetImgref" is the offline shape: fisherman then # installs from containers-storage rather than pulling, which is the only way # the ISO's embedded payload gets used. Naming the image directly makes it @@ -418,7 +453,7 @@ kill -0 "${live_pid}" 2>/dev/null && fail "the live VM would not exit; refusing echo " live VM exited cleanly" sync -echo "=== Phase 4/6: boot the installed disk ===" +echo "=== Phase 4/7: boot the installed disk ===" # Carry the live VM's firmware variables over rather than starting from a # pristine copy. The installer writes an NVRAM boot entry pointing at # \EFI\fedora\shimx64.efi, and this ESP has no \EFI\BOOT\BOOTX64.EFI @@ -445,7 +480,7 @@ qemu_pids+=("$(cat "${WORK}/installed.pid")") echo " watch the installed VM: vnc://127.0.0.1:$((5900 + VNC_INSTALLED))" sleep 5 -echo "=== Phase 5/6: answer the passphrase prompt ===" +echo "=== Phase 5/7: answer the passphrase prompt ===" status=0 python3 "${ROOT}/iso/scripts/luks-unlock.py" qemu \ "${MONITOR_INSTALLED}" "${PASSPHRASE}" "${SERIAL_INSTALLED}" || status=$? @@ -455,7 +490,7 @@ if [[ ${status} -ne 0 ]]; then fail "LUKS unlock or post-unlock boot failed (exit ${status})" fi -echo "=== Phase 6/6: log in and prove the desktop starts ===" +echo "=== Phase 6/7: reach the graphical target and verify the offline payload ===" echo "Waiting for the installed system to reach the graphical target..." emergency_seen="" for i in $(seq 1 90); do @@ -494,6 +529,38 @@ for i in $(seq 1 60); do done echo " ssh: logged in as ${TEST_USER}" +echo "Verifying the installed system booted the offline embedded payload..." +# bootc's own view of what is booted is the ground truth for "the offline +# payload survived the install": fisherman's recipe named targetImgref +# exactly ${PAYLOAD_IMAGE}, formatted from the ISO's embedded +# containers-storage with no network (phase 3), so the booted image +# reference matching it proves that payload -- not a substitute, and not one +# reached over a network this guest does not have -- is what is running. +# (/usr/share/utah/contract.txt, tried here in an earlier revision, is the +# resolved *RPM* contract written at image-build time by +# scripts/install-packages.py; its presence says nothing about which OCI +# image or digest actually got deployed.) +# +# Read unprivileged first, and retry under sudo (${TEST_USER} is in wheel, set +# up that way by the recipe in phase 3) if that returns nothing: whether bootc +# lets a normal user read its status varies by version, and an unreadable +# status is not evidence of a substituted image. Set UTAH_E2E_PAYLOAD_CHECK="" +# to skip this gate entirely, the same escape hatch UTAH_E2E_FLATPAKS gives the +# Flatpak assertion below. +if [[ -n "${UTAH_E2E_PAYLOAD_CHECK-x}" ]]; then + read_booted_image() { + ssh_target "$1" 2>/dev/null \ + | python3 -c 'import json, sys; print(json.load(sys.stdin)["status"]["booted"]["image"]["image"]["image"])' 2>/dev/null || true + } + booted_image="$(read_booted_image 'bootc status --json')" + [[ -n "${booted_image}" ]] || booted_image="$(read_booted_image \ + "printf '%s\\n' '${TEST_PASSWORD}' | sudo -S -p '' bootc status --json")" + [[ "${booted_image}" == "${PAYLOAD_IMAGE}" ]] \ + || fail "installed system booted '${booted_image:-}', not the offline payload '${PAYLOAD_IMAGE}'" + echo " booted image: ${booted_image}" +fi + +echo "=== Phase 7/7: log in and prove the desktop starts ===" ssh_target 'systemctl is-active gdm.service' 2>/dev/null | grep -qx active \ || fail "gdm is not running on the installed system" echo " gdm.service: active" @@ -664,6 +731,49 @@ if [[ "${UTAH_E2E_REQUIRE_SCREENSHOTS:-0}" == 1 ]]; then done fi +# The default Flatpak set is asserted last, not right after login: the +# fastfetch shot above showed 45 flatpaks present on the installed disk only +# after ~2 minutes of uptime (docs/verification, 2026-09-19 run), so checking +# immediately after SSH -- long before first-boot deployment finishes -- +# would misreport a set that has simply not arrived yet as missing. By here +# the system has already spent minutes reaching the graphical target, logging +# in, and (when required) proving fastfetch on screen, so deployment has had +# real time to complete; the loop below still tolerates a little more. +# +# Same Brewfile-to-list conversion iso/live/Containerfile uses to seed the +# ISO's own Flatpak repo, run here against the installed contract's copy so +# this checks the actual default set rather than one app (Ghostty) picked +# only because the terminal-automation phase above happens to need it -- +# install-flatpaks.sh keeps Ghostty out of the Brewfile-derived list on +# purpose, as Utah's own addition rather than the Bluefin parity contract. +# UTAH_E2E_FLATPAKS overrides the expected set directly; empty to skip. +if [[ -n "${UTAH_E2E_FLATPAKS-x}" ]]; then + if [[ -n "${UTAH_E2E_FLATPAKS-}" ]]; then + expected_flatpaks="${UTAH_E2E_FLATPAKS}" + else + expected_flatpaks="$(ssh_target "awk -F '\"' '/^flatpak / {print \$2}' /usr/share/ublue-os/homebrew/system-flatpaks.Brewfile" 2>/dev/null || true)" + [[ -n "${expected_flatpaks}" ]] || fail "could not read the default Flatpak Brewfile on the installed system" + fi + missing_flatpaks=() + for _ in $(seq 1 12); do + installed_flatpaks="$(ssh_target 'flatpak list --system --app --columns=application' 2>/dev/null || true)" + [[ -n "${installed_flatpaks}" ]] || { sleep 10; continue; } + missing_flatpaks=() + while IFS= read -r app; do + [[ -n "${app}" ]] || continue + grep -qxF "${app}" <<< "${installed_flatpaks}" || missing_flatpaks+=("${app}") + done <<< "${expected_flatpaks}" + (( ${#missing_flatpaks[@]} == 0 )) && break + sleep 10 + done + [[ -n "${installed_flatpaks}" ]] \ + || fail "'flatpak list --system' returned nothing on the installed system" + if (( ${#missing_flatpaks[@]} > 0 )); then + fail "default Flatpak(s) missing on the installed, network-isolated system: ${missing_flatpaks[*]}" + fi + echo " default Flatpaks present offline: ${expected_flatpaks//$'\n'/, }" +fi + echo echo "PASS: Utah installed to an encrypted disk, unlocked, and ${TEST_USER} logged" echo " in to a GNOME session on it." @@ -713,7 +823,12 @@ the check beside it passed. 3. The installed disk boots on its own, with no ISO attached. 4. Plymouth's passphrase prompt is answered and the root volume opens. 5. The system reaches the graphical target rather than an emergency shell. -6. The user logs in at the GDM greeter and gets a GNOME session. +6. \`bootc status\` on the installed system reports the booted image as the + offline embedded payload, not a substitute reached over a network this + guest does not have. +7. The user logs in at the GDM greeter and gets a GNOME session. +8. Every default Flatpak in the Brewfile contract is present and listed by + \`flatpak\` on the installed, network-isolated system. ## Screenshots diff --git a/tests/test_iso_ci.py b/tests/test_iso_ci.py index ae01fc097..7da5608cd 100644 --- a/tests/test_iso_ci.py +++ b/tests/test_iso_ci.py @@ -173,6 +173,69 @@ def test_harness_restricts_both_guests_and_requires_real_screenshots(self): self.assertIn("fastfetch output was not visible", script) self.assertIn("missing required screenshot", script) + def test_installed_system_asserts_the_booted_image_is_the_offline_payload(self): + script = (ROOT / "iso/scripts/luks-e2e.sh").read_text() + self.assertIn("bootc status --json", script) + self.assertIn('["status"]["booted"]["image"]["image"]["image"]', script) + self.assertIn('"${booted_image}" == "${PAYLOAD_IMAGE}"', script) + self.assertIn("not the offline payload", script) + # Must run before the login phase, on a guest that still has no route + # to a network that could otherwise mask a substituted image. + payload_at = script.index("bootc status --json") + graphical_at = script.index('echo "=== Phase 6/7') + login_at = script.index('echo "=== Phase 7/7') + self.assertLess(graphical_at, payload_at) + self.assertLess(payload_at, login_at) + + def test_default_flatpak_check_executes_against_stubbed_ssh(self): + # Extract the real block and drive it with stubbed ssh_target/sleep/ + # fail, the same way test_iso_budget_guard_fails_closed_above_ceiling + # drives the ISO size gate -- this is genuinely new set-difference + # logic, not a string in a shell script. + script = (ROOT / "iso/scripts/luks-e2e.sh").read_text() + start = script.index('if [[ -n "${UTAH_E2E_FLATPAKS-x}" ]]; then') + end = script.index("\nfi\n", script.index("present offline", start)) + len("\nfi\n") + block = script[start:end] + + def run(installed, flatpaks_env): + harness = ( + "sleep() { :; }\n" + "ssh_target() { [[ \"$1\" == *'flatpak list'* ]] && printf '%s\\n' \"$INSTALLED\"; }\n" + "fail() { echo \"FAIL: $*\" >&2; exit 1; }\n" + + block + ) + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "INSTALLED": installed, "UTAH_E2E_FLATPAKS": flatpaks_env} + return subprocess.run(["bash", "-eu", "-c", harness], + capture_output=True, text=True, env=env) + + complete = run("org.a\norg.b\norg.c", "org.a\norg.b") + self.assertEqual(complete.returncode, 0, complete.stderr) + self.assertIn("org.a, org.b", complete.stdout) + + missing = run("org.a", "org.a\norg.b") + self.assertNotEqual(missing.returncode, 0) + self.assertIn("org.b", missing.stderr) + self.assertNotIn("org.a\n", missing.stderr.split("missing")[-1]) + + skipped = run("org.a", "") + self.assertEqual(skipped.returncode, 0, skipped.stderr) + self.assertNotIn("present offline", skipped.stdout) + + def test_failure_diagnostics_run_from_the_exit_trap_not_only_fail(self): + script = (ROOT / "iso/scripts/luks-e2e.sh").read_text() + self.assertIn("preserve_failed_disk_diagnostics", script) + self.assertIn("qemu-img info", script) + self.assertIn("never the disk itself", script) + # A bare command failing under `set -e` (the installer step, for one) + # aborts straight to the EXIT trap without ever calling fail(), so + # the capture must live in cleanup(), not only in fail()'s body. + cleanup_at = script.index("cleanup() {") + trap_at = script.index("trap cleanup EXIT") + self.assertIn("preserve_failed_disk_diagnostics", script[cleanup_at:trap_at]) + fail_at = script.index('fail() { echo "FAIL: $*"') + self.assertNotIn("preserve_failed_disk_diagnostics", script[fail_at:fail_at + 80]) + class FastfetchOcrGateTests(unittest.TestCase): """The gate runs against tesseract output, which drops and mangles glyphs."""