diff --git a/.github/workflows/lifecycle-e2e.yml b/.github/workflows/lifecycle-e2e.yml new file mode 100644 index 000000000..2028093a7 --- /dev/null +++ b/.github/workflows/lifecycle-e2e.yml @@ -0,0 +1,87 @@ +name: Bootc Lifecycle E2E + +on: + workflow_dispatch: + inputs: + base_image: + description: "Initial Utah image ref or digest (default: ghcr.io/projectbluefin/utah:testing)" + required: false + type: string + default: "ghcr.io/projectbluefin/utah:testing" + candidate_image: + description: "Candidate Utah image ref or digest to upgrade to (optional)" + required: false + type: string + default: "" + workflow_run: + workflows: ["Post-Testing E2E"] + types: [completed] + branches: [testing] + +permissions: read-all + +concurrency: + group: utah-lifecycle-${{ github.event.workflow_run.head_sha || github.run_id }} + cancel-in-progress: false + +jobs: + lifecycle-test: + name: Validate bootc upgrade and rollback lifecycle + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'testing') + runs-on: ubuntu-24.04 + # Generous headroom for TCG (no KVM on hosted runners): base install, + # two guest reboots, an upgrade and a rollback. guest_ssh_bounded() + # in bootc-lifecycle-e2e.sh already fails the phase cleanly on a + # hung upgrade/rollback; this is only the outer backstop. + timeout-minutes: 90 + permissions: + contents: read + actions: read + steps: + - name: Checkout repository + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + submodules: recursive + + - name: Install QEMU and virtualization dependencies + run: | + set -euo pipefail + sudo apt-get update -qq + sudo apt-get install -y -qq \ + qemu-system-x86 \ + qemu-utils \ + ovmf \ + ffmpeg + + - name: Run bootc upgrade and rollback lifecycle E2E + env: + BASE_IMG: ${{ inputs.base_image || 'ghcr.io/projectbluefin/utah:testing' }} + CANDIDATE_IMG: ${{ inputs.candidate_image || '' }} + UTAH_LIFECYCLE_WORK: /tmp/utah-lifecycle-e2e + run: | + set -euo pipefail + bash scripts/bootc-lifecycle-e2e.sh "${BASE_IMG}" "${CANDIDATE_IMG}" + + - name: Publish lifecycle report to step summary + if: always() + run: | + if [[ -f /tmp/utah-lifecycle-e2e/lifecycle-report.md ]]; then + cat /tmp/utah-lifecycle-e2e/lifecycle-report.md >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload lifecycle test artifacts and diagnostics + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bootc-lifecycle-diagnostics + path: | + /tmp/utah-lifecycle-e2e/lifecycle-summary.json + /tmp/utah-lifecycle-e2e/lifecycle-report.md + /tmp/utah-lifecycle-e2e/serial.log + /tmp/utah-lifecycle-e2e/screenshots/*.png + /tmp/utah-lifecycle-e2e/diagnostics/ + if-no-files-found: warn diff --git a/AGENTS.md b/AGENTS.md index 92965df25..7e3022ebc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,7 @@ just check-desktop-contract # in-image verifiers against a built image just generate-bootable-image testing # bootc install to-disk -> output/bootable.raw just boot-vm # QEMU/noVNC; confirm GDM + GNOME Shell render just iso testing && just boot-iso # live ISO build + boot (see docs/skills/local-testing.md) +just lifecycle-e2e # bootc upgrade & rollback lifecycle validation in QEMU ``` `just check` is the gate CI runs first; a change that fails it fails the whole diff --git a/Justfile b/Justfile index 6706129fc..7774b53af 100644 --- a/Justfile +++ b/Justfile @@ -79,6 +79,11 @@ check: <(grep -m1 '^ARG BASE_IMAGE=' Containerfile.kernel) python3 -m py_compile scripts/flavors.py python3 scripts/flavors.py list >/dev/null + test -f scripts/verify-bootc-lifecycle.py + python3 -m py_compile scripts/verify-bootc-lifecycle.py + test -f scripts/bootc-lifecycle-e2e.sh + bash -n scripts/bootc-lifecycle-e2e.sh + python3 -m unittest tests/unit/test_bootc_lifecycle.py pip install --quiet pyyaml 2>/dev/null || true python3 scripts/check_workflow_outputs.py pip install --quiet jsonschema 2>/dev/null || true @@ -405,3 +410,12 @@ secureboot base_name default_tag flavor: set -euo pipefail image_name="$(just image_name '{{ base_name }}' '{{ default_tag }}' '{{ flavor }}')" podman run --rm --entrypoint /bin/sh "localhost/$image_name:{{ default_tag }}" -c 'test -e /usr/lib/modules || test -e /boot' + +# Run the bootc upgrade and rollback lifecycle test in QEMU between two +# immutable Utah digests. Asserts initial desktop boot, upgrade to candidate +# digest, upgraded graphical desktop, rollback to initial digest, and produces +# phase diagnostics and screenshots. +lifecycle-e2e base_image="localhost/utah:testing" candidate_image="": + #!/usr/bin/env bash + set -euo pipefail + bash scripts/bootc-lifecycle-e2e.sh "{{ base_image }}" "{{ candidate_image }}" diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index 24c8c643f..19fbb2b4f 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -122,6 +122,15 @@ Every push to `main` calls `reusable-sync-branches.yml@v1` (same SHA pin) with `contents: write` and nothing else -- there is no local logic to drift (workflow, `.github/workflows/sync-main-to-testing.yml`). +## lifecycle-e2e + +`.github/workflows/lifecycle-e2e.yml` runs the Dakota-style bootc upgrade and +rollback lifecycle gate. Triggered automatically following `Post-Testing E2E` or +manually via `workflow_dispatch`, it exercises QEMU boot, bootc/uupd update +staging, reboot into candidate digest with graphical validation, rollback, and +restoration of the original baseline digest, publishing structured failure +diagnostics and framebuffer screenshots. + ## Verification ```bash diff --git a/docs/skills/local-testing.md b/docs/skills/local-testing.md index 67755c90f..a2739bddc 100644 --- a/docs/skills/local-testing.md +++ b/docs/skills/local-testing.md @@ -127,6 +127,22 @@ QEMU-for-Docker and exposes the noVNC console at the printed URL (comment above `boot-iso` in `Justfile`), with TPM, UEFI, and `-snapshot` so nothing persists. +## Bootc Upgrade and Rollback Lifecycle Testing + +Validate that atomic updates work end-to-end across deployments: + +```bash +just lifecycle-e2e [candidate_image] +``` + +The test executes in QEMU: +1. Boots the initial deployment in QEMU, establishes SSH, asserts Utah identity (`ID=utah`, `IMAGE_ID=utah`) and graphical desktop (`graphical.target`, `gdm.service`), and captures the booted digest. +2. Stages an upgrade using bootc/uupd policy (`bootc switch ` or `bootc upgrade`) and asserts that `.status.staged` exists with the candidate digest. +3. Reboots the VM, asserts that `.status.booted` matches the candidate digest and `.status.rollback` matches the initial digest, and validates desktop smoke checks. +4. Stages a rollback (`bootc rollback`) and asserts that `.status.staged` references the initial digest. +5. Reboots the VM, asserts that `.status.booted` matches the initial digest, and confirms desktop health. +6. Writes structured diagnostics (`lifecycle-summary.json` and `lifecycle-report.md`) detailing active deployments and digests at every phase. + ## Verification ```bash @@ -141,4 +157,5 @@ just generate-bootable-image testing just boot-vm # success: GDM appears and GNOME Shell renders in noVNC just iso testing just boot-iso # success: live session renders; serial shows UTAH_LIVE_READY +just lifecycle-e2e output/bootable.raw ``` diff --git a/scripts/bootc-lifecycle-e2e.sh b/scripts/bootc-lifecycle-e2e.sh new file mode 100755 index 000000000..2f4da7b46 --- /dev/null +++ b/scripts/bootc-lifecycle-e2e.sh @@ -0,0 +1,506 @@ +#!/usr/bin/env bash +# Validate bootc upgrade and rollback lifecycle on Utah in QEMU. +# +# Exercises the complete atomic update journey: +# Phase 1: Boot initial Utah deployment -> assert identity, graphical target, and capture baseline digest. +# Phase 2: Upgrade to candidate digest using bootc/uupd policy -> assert staged deployment. +# Phase 3: Reboot into upgraded deployment -> assert candidate digest, graphical target, and desktop smoke. +# Phase 4: Rollback to previous deployment -> assert rollback staged. +# Phase 5: Reboot into rolled back deployment -> assert baseline digest and graphical desktop restored. +# +# Usage: +# bootc-lifecycle-e2e.sh [BASE_IMAGE_OR_DISK] [CANDIDATE_IMAGE_OR_DIGEST] +# +# Environment variables: +# UTAH_LIFECYCLE_WORK Working directory (default: /var/tmp/utah-lifecycle-e2e) +# SSH_PORT Host port for guest SSH (default: 2222) +# BOOT_TIMEOUT Maximum seconds to wait for boot / SSH (default: 600) +# UPGRADE_TIMEOUT Maximum seconds to wait for bootc upgrade/switch/rollback +# to complete (default: 1200; hosted runners fall back to +# TCG emulation, which is much slower than KVM) +# QEMU_MEMORY VM memory in MB (default: 8192) +# QEMU_CPUS VM vCPUs (default: 4) +# QEMU_BINARY QEMU binary (default: qemu-system-x86_64) +# QEMU_IMG_BINARY qemu-img binary (default: qemu-img) + +set -euo pipefail + +BASE_TARGET="${1:-localhost/utah:testing}" +CANDIDATE_TARGET="${2:-}" +WORK="${UTAH_LIFECYCLE_WORK:-/var/tmp/utah-lifecycle-e2e}" +SSH_PORT="${SSH_PORT:-2222}" +BOOT_TIMEOUT="${BOOT_TIMEOUT:-600}" +UPGRADE_TIMEOUT="${UPGRADE_TIMEOUT:-1200}" +QEMU_MEMORY="${QEMU_MEMORY:-8192}" +QEMU_CPUS="${QEMU_CPUS:-4}" + +CURRENT_PHASE="init" +QEMU_PID="" + +mkdir -p "${WORK}/screenshots" "${WORK}/diagnostics" +WORK="$(realpath "${WORK}")" +SHOTS="${WORK}/screenshots" +DIAGS="${WORK}/diagnostics" +DISK="${WORK}/bootable.raw" +VARS="${WORK}/ovmf-vars.fd" +MONITOR="${WORK}/monitor.sock" +SERIAL="${WORK}/serial.log" +PID_FILE="${WORK}/qemu.pid" +KEY_FILE="${WORK}/id_ed25519" + +fail() { + echo "FAIL [${CURRENT_PHASE}]: $*" >&2 + exit 1 +} + +# Find QEMU, qemu-img, and python3 +QEMU="${QEMU_BINARY:-$(command -v qemu-system-x86_64 || true)}" +QEMU_IMG="${QEMU_IMG_BINARY:-$(command -v qemu-img || true)}" +PYTHON="$(command -v python3 || true)" +SSH="$(command -v ssh || true)" + +[[ -n "${PYTHON}" ]] || fail "python3 is not installed" +[[ -n "${SSH}" ]] || fail "ssh is not installed" +[[ -n "${QEMU}" ]] || fail "qemu-system-x86_64 is not installed" +[[ -n "${QEMU_IMG}" ]] || fail "qemu-img is not installed" + +# Resolve OVMF firmware +OVMF_CODE="" +for candidate in \ + /usr/share/OVMF/OVMF_CODE_4M.fd \ + /usr/share/OVMF/OVMF_CODE.fd \ + /usr/share/edk2/ovmf/OVMF_CODE.fd \ + /usr/share/edk2/ovmf/OVMF_CODE_4M.fd; do + if [[ -f "${candidate}" ]]; then + OVMF_CODE="${candidate}" + break + fi +done + +OVMF_VARS_SOURCE="" +for candidate in \ + /usr/share/OVMF/OVMF_VARS_4M.fd \ + /usr/share/OVMF/OVMF_VARS.fd \ + /usr/share/edk2/ovmf/OVMF_VARS.fd \ + /usr/share/edk2/ovmf/OVMF_VARS_4M.fd; do + if [[ -f "${candidate}" ]]; then + OVMF_VARS_SOURCE="${candidate}" + break + fi +done + +[[ -n "${OVMF_CODE}" ]] || fail "OVMF code firmware is not installed" +[[ -n "${OVMF_VARS_SOURCE}" ]] || fail "OVMF variable store is not installed" + +# Check acceleration +if [[ -r /dev/kvm ]]; then + ACCEL=(-accel kvm) + echo "QEMU acceleration: KVM" +else + ACCEL=(-accel tcg,thread=multi) + echo "QEMU acceleration: TCG (no /dev/kvm)" +fi + +# Generate ephemeral SSH key if not already present +if [[ ! -f "${KEY_FILE}" ]]; then + ssh-keygen -t ed25519 -N "" -f "${KEY_FILE}" >/dev/null 2>&1 + chmod 0600 "${KEY_FILE}" +fi + +# SSH command options +SSH_OPTS=( + -i "${KEY_FILE}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o ConnectTimeout=5 + -o BatchMode=yes + -o ServerAliveInterval=15 + -o ServerAliveCountMax=4 + -p "${SSH_PORT}" +) + +guest_ssh() { + "${SSH}" "${SSH_OPTS[@]}" root@127.0.0.1 "$@" +} + +# Bounds a guest command to UPGRADE_TIMEOUT from the client side: hosted +# runners fall back to TCG (software) emulation, where `bootc upgrade`/ +# `bootc rollback` pulling and unpacking layers is much slower than under +# KVM, so this needs real headroom -- but it must still fail the phase +# instead of hanging until the job's default multi-hour timeout. +guest_ssh_bounded() { + timeout "${UPGRADE_TIMEOUT}" "${SSH}" "${SSH_OPTS[@]}" root@127.0.0.1 "$@" +} + +monitor_cmd() { + "${PYTHON}" - "${MONITOR}" "$1" <<'PY' +import socket +import sys +import time + +path, command = sys.argv[1:] +with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(10) + client.connect(path) + client.sendall(command.encode() + b"\n") + time.sleep(0.4) +PY +} + +capture_screen() { + local label="$1" + local ppm="${SHOTS}/${label}.ppm" + local png="${SHOTS}/${label}.png" + + [[ -S "${MONITOR}" ]] || return 0 + monitor_cmd "screendump ${ppm}" >/dev/null 2>&1 || return 0 + for _ in $(seq 1 10); do + [[ -s "${ppm}" ]] && break + sleep 0.5 + done + [[ -s "${ppm}" ]] || return 0 + if command -v ffmpeg >/dev/null 2>&1 && \ + ffmpeg -y -loglevel error -i "${ppm}" "${png}" >/dev/null 2>&1; then + rm -f "${ppm}" + echo "Saved screenshot: ${png}" + else + echo "Saved framebuffer: ${ppm}" + fi +} + +kill_qemu() { + local pid="$1" + [[ -n "${pid}" ]] || return 0 + if kill -0 "${pid}" 2>/dev/null; then + kill "${pid}" 2>/dev/null || true + for _ in $(seq 1 15); do + kill -0 "${pid}" 2>/dev/null || return 0 + sleep 1 + done + kill -KILL "${pid}" 2>/dev/null || true + fi +} + +cleanup() { + local status=$? + set +e + if [[ "${status}" -ne 0 ]]; then + echo "=== LIFECYCLE TEST FAILED AT PHASE '${CURRENT_PHASE}' ===" >&2 + capture_screen "failure-${CURRENT_PHASE}" + if [[ -f "${WORK}/${CURRENT_PHASE}-status.json" ]]; then + "${PYTHON}" scripts/verify-bootc-lifecycle.py record-phase \ + --work-dir "${WORK}" \ + --phase "${CURRENT_PHASE}" \ + --status-file "${WORK}/${CURRENT_PHASE}-status.json" \ + --verdict FAIL \ + --notes "Failure trapped during phase execution" || true + fi + "${PYTHON}" scripts/verify-bootc-lifecycle.py generate-report --work-dir "${WORK}" || true + if [[ -f "${WORK}/lifecycle-report.md" ]]; then + cat "${WORK}/lifecycle-report.md" >&2 || true + fi + echo "--- Serial console excerpt ---" >&2 + tail -n 80 "${SERIAL}" >&2 2>/dev/null || true + fi + + if [[ -n "${QEMU_PID}" ]]; then + kill_qemu "${QEMU_PID}" + elif [[ -f "${PID_FILE}" ]]; then + kill_qemu "$(cat "${PID_FILE}")" + fi + echo "Lifecycle artifacts retained at ${WORK}" + exit "${status}" +} +trap cleanup EXIT + +# Start QEMU instance +start_qemu() { + local name="$1" + rm -f "${MONITOR}" "${PID_FILE}" + echo "Starting QEMU VM [${name}]..." + "${QEMU}" \ + -name "utah-lifecycle-${name}" \ + -machine q35 \ + "${ACCEL[@]}" \ + -cpu max \ + -m "${QEMU_MEMORY}" \ + -smp "${QEMU_CPUS}" \ + -drive "if=pflash,format=raw,readonly=on,file=${OVMF_CODE}" \ + -drive "if=pflash,format=raw,file=${VARS}" \ + -drive "if=none,id=disk,file=${DISK},format=raw" \ + -device virtio-blk-pci,drive=disk \ + -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT}-:22" \ + -device virtio-net-pci,netdev=net0 \ + -display none \ + -vnc 127.0.0.1:2 \ + -monitor "unix:${MONITOR},server,nowait" \ + -serial "file:${SERIAL}" \ + -no-reboot \ + -daemonize \ + -pidfile "${PID_FILE}" + + QEMU_PID="$(cat "${PID_FILE}")" + echo "QEMU running with PID ${QEMU_PID}" +} + +wait_for_ssh() { + local timeout="${1:-${BOOT_TIMEOUT}}" + local label="$2" + echo "Waiting up to ${timeout}s for SSH on port ${SSH_PORT} [${label}]..." + local ready=0 + for _ in $(seq 1 "${timeout}"); do + if guest_ssh true >/dev/null 2>&1; then + ready=1 + break + fi + sleep 1 + done + (( ready == 1 )) || fail "VM failed to respond on SSH within ${timeout}s [${label}]" + echo "SSH control channel ready [${label}]." +} + +stop_qemu() { + local label="$1" + echo "Stopping QEMU [${label}]..." + guest_ssh 'systemctl poweroff --no-block' >/dev/null 2>&1 || true + if [[ -n "${QEMU_PID}" ]]; then + for _ in $(seq 1 60); do + kill -0 "${QEMU_PID}" 2>/dev/null || break + sleep 1 + done + kill_qemu "${QEMU_PID}" + QEMU_PID="" + fi +} + +reboot_guest() { + local label="$1" + echo "Rebooting guest VM [${label}]..." + guest_ssh 'systemctl reboot --no-block' >/dev/null 2>&1 || true + if [[ -n "${QEMU_PID}" ]]; then + for _ in $(seq 1 60); do + kill -0 "${QEMU_PID}" 2>/dev/null || break + sleep 1 + done + kill_qemu "${QEMU_PID}" + QEMU_PID="" + fi + start_qemu "${label}" + wait_for_ssh "${BOOT_TIMEOUT}" "${label}" +} + +verify_desktop_smoke() { + local label="$1" + echo "Verifying graphical target, GDM, and desktop health [${label}]..." + local desktop_active=0 + for _ in $(seq 1 120); do + if guest_ssh 'systemctl is-active graphical.target' 2>/dev/null | grep -qx active && \ + guest_ssh 'systemctl is-active gdm.service' 2>/dev/null | grep -qx active; then + desktop_active=1 + break + fi + sleep 2 + done + (( desktop_active == 1 )) || fail "Desktop target/GDM not active [${label}]" + + # Assert Utah identity + guest_ssh 'grep -q "^ID=utah" /etc/os-release && grep -q "^IMAGE_ID=utah" /etc/os-release' || \ + fail "Utah identity assertion failed in /etc/os-release [${label}]" + + echo "Desktop target, GDM, and Utah identity verified [${label}]." +} + +# --------------------------------------------------------------------------- +# Disk Setup +# --------------------------------------------------------------------------- +CURRENT_PHASE="disk_setup" +cp -f "${OVMF_VARS_SOURCE}" "${VARS}" + +if [[ -f "${BASE_TARGET}" && ( "${BASE_TARGET}" == *.raw || "${BASE_TARGET}" == *.qcow2 ) ]]; then + echo "Using existing disk image: ${BASE_TARGET}" + if [[ "${BASE_TARGET}" != "${DISK}" ]]; then + cp -f "${BASE_TARGET}" "${DISK}" + fi +else + echo "Installing base image '${BASE_TARGET}' to raw disk ${DISK} via bootc install to-disk..." + rm -f "${DISK}" + truncate -s 30G "${DISK}" + + mkdir -p "${WORK}/ssh" + cp -f "${KEY_FILE}.pub" "${WORK}/ssh/authorized_keys" + + sudo podman run --rm --privileged --pid=host \ + --security-opt label=type:unconfined_t \ + -v "${WORK}:/data:Z" \ + "${BASE_TARGET}" bootc install to-disk \ + --via-loopback /data/bootable.raw \ + --filesystem btrfs --wipe --generic-image \ + --karg console=ttyS0,115200 --karg utah.local \ + --root-ssh-authorized-keys /data/id_ed25519.pub + + # Install removable-media EFI fallback for QEMU firmware compatibility + loop="$(sudo losetup --find --show --partscan "${DISK}")" + esp="${WORK}/esp_mount" + mkdir -p "${esp}" + trap 'sudo umount "${esp}" 2>/dev/null || true; sudo losetup -d "${loop}" 2>/dev/null || true; rm -rf "${esp}"' EXIT + sudo mount "${loop}p2" "${esp}" + sudo install -d "${esp}/EFI/BOOT" + if [[ -f "${esp}/EFI/fedora/shimx64.efi" ]]; then + sudo install -m 0644 "${esp}/EFI/fedora/shimx64.efi" "${esp}/EFI/BOOT/BOOTX64.EFI" + fi + if [[ -f "${esp}/EFI/fedora/grubx64.efi" ]]; then + sudo install -m 0644 "${esp}/EFI/fedora/grubx64.efi" "${esp}/EFI/BOOT/grubx64.efi" + fi + sudo umount "${esp}" + sudo losetup -d "${loop}" + rm -rf "${esp}" + trap cleanup EXIT + sync +fi + +# --------------------------------------------------------------------------- +# Phase 1: Boot initial Utah deployment +# --------------------------------------------------------------------------- +CURRENT_PHASE="initial_boot" +echo "=== Phase 1/5: Boot initial Utah deployment ===" +start_qemu "initial" +wait_for_ssh "${BOOT_TIMEOUT}" "initial" +verify_desktop_smoke "initial" +capture_screen "initial-desktop" + +guest_ssh 'bootc status --format=json' > "${WORK}/initial-status.json" +guest_ssh 'cat /etc/os-release' > "${WORK}/initial-os-release.txt" + +"${PYTHON}" scripts/verify-bootc-lifecycle.py record-phase \ + --work-dir "${WORK}" \ + --phase "initial_boot" \ + --status-file "${WORK}/initial-status.json" \ + --os-release "${WORK}/initial-os-release.txt" \ + --screenshot "${SHOTS}/initial-desktop.png" \ + --notes "Initial deployment booted to graphical target" + +initial_digest="$("${PYTHON}" -c 'import json; print(json.load(open("'${WORK}/initial-status.json'"))["status"]["booted"]["image"]["imageDigest"])')" +echo "Active initial deployment digest: ${initial_digest}" + +# --------------------------------------------------------------------------- +# Phase 2: Upgrade to candidate digest using bootc/uupd policy +# --------------------------------------------------------------------------- +CURRENT_PHASE="upgrade_staged" +echo "=== Phase 2/5: Stage upgrade via bootc/uupd policy ===" + +# Check uupd update policy daemon +guest_ssh 'systemctl is-enabled uupd.timer' || true + +if [[ -n "${CANDIDATE_TARGET}" ]]; then + echo "Switching to candidate image: ${CANDIDATE_TARGET}" + guest_ssh_bounded "bootc switch '${CANDIDATE_TARGET}'" \ + || fail "bootc switch did not complete within UPGRADE_TIMEOUT=${UPGRADE_TIMEOUT}s" +else + echo "Executing bootc upgrade..." + guest_ssh_bounded 'bootc upgrade' \ + || fail "bootc upgrade did not complete within UPGRADE_TIMEOUT=${UPGRADE_TIMEOUT}s" +fi + +guest_ssh 'bootc status --format=json' > "${WORK}/upgrade-staged-status.json" + +"${PYTHON}" scripts/verify-bootc-lifecycle.py record-phase \ + --work-dir "${WORK}" \ + --phase "upgrade_staged" \ + --status-file "${WORK}/upgrade-staged-status.json" \ + --notes "Candidate upgrade staged" + +"${PYTHON}" scripts/verify-bootc-lifecycle.py assert-transition \ + --work-dir "${WORK}" \ + --from-phase "initial_boot" \ + --to-phase "upgrade_staged" \ + --relation "upgrade_staged" + +staged_digest="$("${PYTHON}" -c 'import json; print(json.load(open("'${WORK}/upgrade-staged-status.json'"))["status"]["staged"]["image"]["imageDigest"])')" +echo "Staged candidate digest: ${staged_digest}" + +# --------------------------------------------------------------------------- +# Phase 3: Reboot into upgraded deployment +# --------------------------------------------------------------------------- +CURRENT_PHASE="upgraded_boot" +echo "=== Phase 3/5: Reboot into upgraded candidate deployment ===" +reboot_guest "upgraded" +verify_desktop_smoke "upgraded" +capture_screen "upgraded-desktop" + +guest_ssh 'bootc status --format=json' > "${WORK}/upgraded-status.json" +guest_ssh 'cat /etc/os-release' > "${WORK}/upgraded-os-release.txt" + +"${PYTHON}" scripts/verify-bootc-lifecycle.py record-phase \ + --work-dir "${WORK}" \ + --phase "upgraded_boot" \ + --status-file "${WORK}/upgraded-status.json" \ + --os-release "${WORK}/upgraded-os-release.txt" \ + --screenshot "${SHOTS}/upgraded-desktop.png" \ + --notes "Upgraded deployment active with desktop target" + +"${PYTHON}" scripts/verify-bootc-lifecycle.py assert-transition \ + --work-dir "${WORK}" \ + --from-phase "upgrade_staged" \ + --to-phase "upgraded_boot" \ + --relation "upgraded_boot" + +# --------------------------------------------------------------------------- +# Phase 4: Stage rollback +# --------------------------------------------------------------------------- +CURRENT_PHASE="rollback_staged" +echo "=== Phase 4/5: Stage rollback via bootc rollback ===" +guest_ssh_bounded 'bootc rollback' \ + || fail "bootc rollback did not complete within UPGRADE_TIMEOUT=${UPGRADE_TIMEOUT}s" + +guest_ssh 'bootc status --format=json' > "${WORK}/rollback-staged-status.json" + +"${PYTHON}" scripts/verify-bootc-lifecycle.py record-phase \ + --work-dir "${WORK}" \ + --phase "rollback_staged" \ + --status-file "${WORK}/rollback-staged-status.json" \ + --notes "Rollback staged to previous deployment" + +"${PYTHON}" scripts/verify-bootc-lifecycle.py assert-transition \ + --work-dir "${WORK}" \ + --from-phase "upgraded_boot" \ + --to-phase "rollback_staged" \ + --relation "rollback_staged" + +# --------------------------------------------------------------------------- +# Phase 5: Reboot into rollback deployment +# --------------------------------------------------------------------------- +CURRENT_PHASE="rollback_boot" +echo "=== Phase 5/5: Reboot into rollback deployment ===" +reboot_guest "rollback" +verify_desktop_smoke "rollback" +capture_screen "rollback-desktop" + +guest_ssh 'bootc status --format=json' > "${WORK}/rollback-status.json" +guest_ssh 'cat /etc/os-release' > "${WORK}/rollback-os-release.txt" + +"${PYTHON}" scripts/verify-bootc-lifecycle.py record-phase \ + --work-dir "${WORK}" \ + --phase "rollback_boot" \ + --status-file "${WORK}/rollback-status.json" \ + --os-release "${WORK}/rollback-os-release.txt" \ + --screenshot "${SHOTS}/rollback-desktop.png" \ + --notes "Rollback deployment active and verified" + +"${PYTHON}" scripts/verify-bootc-lifecycle.py assert-transition \ + --work-dir "${WORK}" \ + --from-phase "rollback_staged" \ + --to-phase "rollback_boot" \ + --relation "rollback_boot" + +# --------------------------------------------------------------------------- +# Phase 6: Finalize report and diagnostics +# --------------------------------------------------------------------------- +CURRENT_PHASE="complete" +stop_qemu "complete" + +echo "=== Generating final lifecycle diagnostics report ===" +"${PYTHON}" scripts/verify-bootc-lifecycle.py generate-report --work-dir "${WORK}" +cat "${WORK}/lifecycle-report.md" + +echo "SUCCESS: Bootc upgrade and rollback lifecycle fully validated on Utah." diff --git a/scripts/verify-bootc-lifecycle.py b/scripts/verify-bootc-lifecycle.py new file mode 100755 index 000000000..c34af86c2 --- /dev/null +++ b/scripts/verify-bootc-lifecycle.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +"""Bootc upgrade and rollback lifecycle verification and diagnostics tool. + +Parses and validates 'bootc status --format=json' output, tracks state transitions +across the upgrade and rollback lifecycle, asserts graphical desktop health, and +generates structured diagnostics capturing the active deployment and digest at each phase. +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import os +import re +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + + +# `bootc status --format=json` is explicitly documented upstream as an +# unstable schema. Pin the versions we know this tool's field paths are +# compatible with and fail with a clear message on drift, rather than +# letting a renamed/restructured field raise a raw KeyError deep inside a +# phase handler. v1alpha1 was bootc's original schema name; current bootc +# (1.16+) reports "org.containers.bootc/v1" for the same field layout this +# tool reads (booted/staged/rollback.image.imageDigest) -- verified against +# `bootc status --format=json` on a live bootc-1.16.7 host. +KNOWN_API_VERSIONS = { + "org.containers.bootc/v1alpha1", + "org.containers.bootc/v1", +} + + +def parse_bootc_status(raw_json: str) -> Dict[str, Any]: + """Parse and validate bootc status JSON against known schema versions.""" + if not raw_json or not raw_json.strip(): + raise ValueError("bootc status output is empty") + + try: + data = json.loads(raw_json) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid bootc status JSON: {exc}") from exc + + if not isinstance(data, dict): + raise ValueError("bootc status root must be a JSON object") + + api_version = data.get("apiVersion") + if api_version not in KNOWN_API_VERSIONS: + raise ValueError( + f"bootc status schema mismatch: expected apiVersion to be one " + f"of {sorted(KNOWN_API_VERSIONS)}, got {api_version!r}. The " + f"bootc version on the guest may have changed its status " + f"schema; add the new version to KNOWN_API_VERSIONS and " + f"re-verify the field paths below before trusting this tool's " + f"output." + ) + + status = data.get("status") + if not isinstance(status, dict): + raise ValueError( + f"bootc status missing required 'status' dictionary. Keys found: {list(data.keys())}" + ) + + booted = status.get("booted") + if not isinstance(booted, dict): + raise ValueError("bootc status missing 'status.booted' deployment dictionary") + + # Extract image info + image_obj = booted.get("image", {}) + if not isinstance(image_obj, dict): + raise ValueError("status.booted missing 'image' object") + + digest = image_obj.get("imageDigest") + if not digest or not isinstance(digest, str) or not digest.startswith("sha256:"): + raise ValueError(f"status.booted.image missing valid sha256 imageDigest: {digest}") + + return status + + +def extract_deployment_info(deployment: Optional[Dict[str, Any]]) -> Dict[str, Optional[str]]: + """Extract digest, image reference, and pinned status from a deployment dict.""" + if not deployment or not isinstance(deployment, dict): + return {"digest": None, "image": None, "pinned": None} + + image_obj = deployment.get("image", {}) + digest = image_obj.get("imageDigest") if isinstance(image_obj, dict) else None + + # Image ref can be in image.image.image or image.image + img_ref = None + if isinstance(image_obj, dict): + inner = image_obj.get("image") + if isinstance(inner, dict): + img_ref = inner.get("image") + elif isinstance(inner, str): + img_ref = inner + + pinned = deployment.get("pinned", False) + return { + "digest": digest, + "image": img_ref, + "pinned": bool(pinned) if pinned is not None else False, + } + + +def parse_os_release(content: str) -> Dict[str, str]: + """Parse /etc/os-release key-value content.""" + data = {} + for line in content.splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, v = line.split("=", 1) + data[k.strip()] = v.strip().strip('"').strip("'") + return data + + +class LifecycleTracker: + """Tracks phases, verifies transitions, and generates failure diagnostics.""" + + def __init__(self, work_dir: Path): + self.work_dir = Path(work_dir) + self.work_dir.mkdir(parents=True, exist_ok=True) + self.state_file = self.work_dir / "lifecycle-state.json" + self.data = self._load() + + def _load(self) -> Dict[str, Any]: + if self.state_file.exists(): + try: + return json.loads(self.state_file.read_text()) + except Exception: + pass + return { + "created_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "overall_status": "in_progress", + "phases": [], + "diagnostics": [], + } + + def _save(self) -> None: + self.state_file.write_text(json.dumps(self.data, indent=2) + "\n") + + def record_phase( + self, + phase_name: str, + status_raw: str, + status_verdict: str, + desktop_ok: bool = True, + systemd_state: str = "running", + os_release_text: Optional[str] = None, + notes: str = "", + screenshot: Optional[str] = None, + ) -> Dict[str, Any]: + """Record the state of a lifecycle phase.""" + try: + status = parse_bootc_status(status_raw) + booted = extract_deployment_info(status.get("booted")) + staged = extract_deployment_info(status.get("staged")) + rollback = extract_deployment_info(status.get("rollback")) + parse_error = None + except Exception as exc: + booted = {"digest": "unknown", "image": "unknown", "pinned": False} + staged = {"digest": None, "image": None, "pinned": None} + rollback = {"digest": None, "image": None, "pinned": None} + parse_error = str(exc) + + os_info = parse_os_release(os_release_text) if os_release_text else {} + is_utah_identity = ( + os_info.get("ID") == "utah" and os_info.get("IMAGE_ID") == "utah" + if os_info + else None + ) + + phase_record = { + "phase": phase_name, + "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "status": status_verdict.upper(), + "booted": booted, + "staged": staged, + "rollback": rollback, + "desktop_ok": desktop_ok, + "systemd_state": systemd_state, + "os_identity_utah": is_utah_identity, + "os_version": os_info.get("VERSION_ID"), + "screenshot": screenshot, + "notes": notes, + "parse_error": parse_error, + } + + # Validate phase-specific requirements and add diagnostics if failed + errors = [] + if parse_error: + errors.append(f"Phase {phase_name}: Failed to parse bootc status: {parse_error}") + + if status_verdict.upper() == "FAIL": + errors.append( + f"Phase {phase_name} marked as FAIL. Active booted digest: {booted.get('digest')}" + ) + + if desktop_ok is False: + errors.append( + f"Phase {phase_name}: Graphical desktop check failed. Active booted digest: {booted.get('digest')}" + ) + + if is_utah_identity is False: + errors.append( + f"Phase {phase_name}: Utah identity check failed: ID={os_info.get('ID')}, IMAGE_ID={os_info.get('IMAGE_ID')}" + ) + + if errors: + for err in errors: + self.data["diagnostics"].append( + { + "phase": phase_name, + "timestamp": phase_record["timestamp"], + "active_booted_digest": booted.get("digest"), + "active_booted_image": booted.get("image"), + "staged_digest": staged.get("digest"), + "rollback_digest": rollback.get("digest"), + "message": err, + } + ) + + self.data["phases"].append(phase_record) + self._save() + return phase_record + + def assert_transition( + self, + from_phase: str, + to_phase: str, + expected_relationship: str, + ) -> bool: + """Assert relationship between two recorded phases. + + expected_relationship: + 'upgrade_staged': to_phase staged digest is different from from_phase booted digest. + 'upgraded_boot': to_phase booted digest == from_phase staged digest, and + to_phase rollback digest == from_phase booted digest. + 'rollback_staged': to_phase staged digest == from_phase rollback digest. + 'rollback_boot': to_phase booted digest == original booted digest, and + to_phase rollback digest == upgraded booted digest. + """ + phases_by_name = {p["phase"]: p for p in self.data["phases"]} + p1 = phases_by_name.get(from_phase) + p2 = phases_by_name.get(to_phase) + + if not p1: + raise AssertionError(f"Prior phase '{from_phase}' not found in lifecycle history") + if not p2: + raise AssertionError(f"Target phase '{to_phase}' not found in lifecycle history") + + p1_booted = p1["booted"]["digest"] + p1_staged = p1["staged"]["digest"] + p2_booted = p2["booted"]["digest"] + p2_staged = p2["staged"]["digest"] + p2_rollback = p2["rollback"]["digest"] + + if expected_relationship == "upgrade_staged": + if not p2_staged: + msg = f"Upgrade did not stage any deployment in '{to_phase}'. Active booted: {p2_booted}" + self._record_error(to_phase, msg, p2) + raise AssertionError(msg) + if p2_staged == p1_booted: + msg = f"Upgrade staged the same digest as currently booted ({p2_staged}) in '{to_phase}'" + self._record_error(to_phase, msg, p2) + raise AssertionError(msg) + + elif expected_relationship == "upgraded_boot": + expected_target = p1_staged + if p2_booted != expected_target: + msg = ( + f"Reboot into upgraded deployment failed in '{to_phase}'. " + f"Active booted digest: {p2_booted}, Expected candidate digest: {expected_target}" + ) + self._record_error(to_phase, msg, p2) + raise AssertionError(msg) + if p2_rollback != p1_booted: + msg = ( + f"Rollback deployment does not match previous deployment in '{to_phase}'. " + f"Active rollback digest: {p2_rollback}, Expected previous digest: {p1_booted}" + ) + self._record_error(to_phase, msg, p2) + raise AssertionError(msg) + + elif expected_relationship == "rollback_staged": + expected_target = p1["rollback"]["digest"] + if p2_staged != expected_target: + msg = ( + f"Rollback staging failed in '{to_phase}'. " + f"Staged digest: {p2_staged}, Expected previous digest: {expected_target}" + ) + self._record_error(to_phase, msg, p2) + raise AssertionError(msg) + + elif expected_relationship == "rollback_boot": + # Initial phase is usually the first recorded phase + initial_phase = self.data["phases"][0] + initial_digest = initial_phase["booted"]["digest"] + upgraded_phase = phases_by_name.get("upgraded_boot", p1) + upgraded_digest = upgraded_phase["booted"]["digest"] + + if p2_booted != initial_digest: + msg = ( + f"Rollback boot failed in '{to_phase}'. " + f"Active booted digest: {p2_booted}, Expected original digest: {initial_digest}" + ) + self._record_error(to_phase, msg, p2) + raise AssertionError(msg) + if p2_rollback != upgraded_digest: + msg = ( + f"Rollback deployment does not reference candidate deployment in '{to_phase}'. " + f"Active rollback digest: {p2_rollback}, Expected candidate digest: {upgraded_digest}" + ) + self._record_error(to_phase, msg, p2) + raise AssertionError(msg) + + self._save() + return True + + def _record_error(self, phase_name: str, message: str, phase_info: Dict[str, Any]) -> None: + self.data["diagnostics"].append( + { + "phase": phase_name, + "timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "active_booted_digest": phase_info["booted"]["digest"], + "active_booted_image": phase_info["booted"]["image"], + "staged_digest": phase_info["staged"]["digest"], + "rollback_digest": phase_info["rollback"]["digest"], + "message": message, + } + ) + self._save() + + def generate_report(self) -> Dict[str, Any]: + """Generate final JSON summary and Markdown report.""" + has_failures = bool(self.data["diagnostics"]) or any( + p["status"] != "PASS" for p in self.data["phases"] + ) + self.data["overall_status"] = "FAIL" if has_failures else "PASS" + self.data["finished_at"] = datetime.datetime.now(datetime.timezone.utc).isoformat() + self._save() + + # Write lifecycle-summary.json + summary_file = self.work_dir / "lifecycle-summary.json" + summary_file.write_text(json.dumps(self.data, indent=2) + "\n") + + # Write lifecycle-report.md + md_file = self.work_dir / "lifecycle-report.md" + lines = [ + f"# Bootc Upgrade & Rollback Lifecycle Report: {self.data['overall_status']}", + "", + f"- **Overall Status:** `{self.data['overall_status']}`", + f"- **Created:** {self.data.get('created_at', 'N/A')}", + f"- **Finished:** {self.data.get('finished_at', 'N/A')}", + "", + "## Lifecycle Phase Progression", + "", + "| Phase | Status | Booted Digest | Staged Digest | Rollback Digest | Desktop OK |", + "|---|---|---|---|---|---|", + ] + + for p in self.data["phases"]: + b_dig = (p["booted"]["digest"] or "")[:19] + "..." if p["booted"]["digest"] else "-" + s_dig = (p["staged"]["digest"] or "")[:19] + "..." if p["staged"]["digest"] else "-" + r_dig = (p["rollback"]["digest"] or "")[:19] + "..." if p["rollback"]["digest"] else "-" + d_ok = "✅" if p.get("desktop_ok") else "❌" + status_icon = "✅ PASS" if p["status"] == "PASS" else "❌ FAIL" + lines.append( + f"| `{p['phase']}` | {status_icon} | `{b_dig}` | `{s_dig}` | `{r_dig}` | {d_ok} |" + ) + + if self.data["diagnostics"]: + lines.extend([ + "", + "## Failure Diagnostics", + "", + ]) + for diag in self.data["diagnostics"]: + lines.extend([ + f"### Phase: `{diag.get('phase')}`", + f"- **Timestamp:** {diag.get('timestamp')}", + f"- **Active Booted Digest:** `{diag.get('active_booted_digest')}`", + f"- **Active Booted Image:** `{diag.get('active_booted_image')}`", + f"- **Staged Digest:** `{diag.get('staged_digest')}`", + f"- **Rollback Digest:** `{diag.get('rollback_digest')}`", + f"- **Error:** {diag.get('message')}", + "", + ]) + else: + lines.extend([ + "", + "## Summary", + "", + "All lifecycle phases passed: clean boot, upgrade staged, upgraded boot with active graphical desktop, rollback staged, and rollback boot to original immutable digest.", + ]) + + md_file.write_text("\n".join(lines) + "\n") + return self.data + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + + # Subcommand: parse-status + p_parse = subparsers.add_parser("parse-status", help="Parse bootc status JSON") + p_parse.add_argument("status_file", type=Path, help="Path to bootc status JSON file") + + # Subcommand: record-phase + p_rec = subparsers.add_parser("record-phase", help="Record a lifecycle phase") + p_rec.add_argument("--work-dir", type=Path, required=True, help="Lifecycle work directory") + p_rec.add_argument("--phase", required=True, help="Name of phase") + p_rec.add_argument("--status-file", type=Path, required=True, help="Path to status JSON") + p_rec.add_argument("--verdict", default="PASS", choices=["PASS", "FAIL"], help="Phase verdict") + p_rec.add_argument("--desktop-ok", action="store_true", default=True, help="Desktop status") + p_rec.add_argument("--desktop-fail", action="store_true", help="Mark desktop check failed") + p_rec.add_argument("--systemd-state", default="running", help="Systemd state") + p_rec.add_argument("--os-release", type=Path, help="Path to /etc/os-release dump") + p_rec.add_argument("--notes", default="", help="Optional notes") + p_rec.add_argument("--screenshot", help="Path to screenshot") + + # Subcommand: assert-transition + p_trans = subparsers.add_parser("assert-transition", help="Assert transition between phases") + p_trans.add_argument("--work-dir", type=Path, required=True, help="Lifecycle work directory") + p_trans.add_argument("--from-phase", required=True, help="Prior phase name") + p_trans.add_argument("--to-phase", required=True, help="Current phase name") + p_trans.add_argument( + "--relation", + required=True, + choices=["upgrade_staged", "upgraded_boot", "rollback_staged", "rollback_boot"], + help="Expected relationship", + ) + + # Subcommand: generate-report + p_rep = subparsers.add_parser("generate-report", help="Generate summary and report") + p_rep.add_argument("--work-dir", type=Path, required=True, help="Lifecycle work directory") + + # Subcommand: simulate + subparsers.add_parser("simulate", help="Run a self-contained simulated lifecycle validation") + + args = parser.parse_args() + + if args.command == "parse-status": + raw = args.status_file.read_text() + status = parse_bootc_status(raw) + booted = extract_deployment_info(status.get("booted")) + staged = extract_deployment_info(status.get("staged")) + rollback = extract_deployment_info(status.get("rollback")) + print(json.dumps({"booted": booted, "staged": staged, "rollback": rollback}, indent=2)) + return 0 + + elif args.command == "record-phase": + tracker = LifecycleTracker(args.work_dir) + raw_status = args.status_file.read_text() + os_text = args.os_release.read_text() if args.os_release and args.os_release.exists() else None + desktop_ok = False if args.desktop_fail else args.desktop_ok + record = tracker.record_phase( + phase_name=args.phase, + status_raw=raw_status, + status_verdict=args.verdict, + desktop_ok=desktop_ok, + systemd_state=args.systemd_state, + os_release_text=os_text, + notes=args.notes, + screenshot=args.screenshot, + ) + print(f"Recorded phase '{args.phase}': status={record['status']} booted={record['booted']['digest']}") + return 0 if record["status"] == "PASS" else 1 + + elif args.command == "assert-transition": + tracker = LifecycleTracker(args.work_dir) + try: + tracker.assert_transition(args.from_phase, args.to_phase, args.relation) + print(f"Transition assertion '{args.relation}' ({args.from_phase} -> {args.to_phase}) passed.") + return 0 + except AssertionError as err: + print(f"Transition assertion FAILED: {err}", file=sys.stderr) + return 1 + + elif args.command == "generate-report": + tracker = LifecycleTracker(args.work_dir) + summary = tracker.generate_report() + print(f"Lifecycle report generated: status={summary['overall_status']}") + return 0 if summary["overall_status"] == "PASS" else 1 + + elif args.command == "simulate": + import tempfile + with tempfile.TemporaryDirectory() as tmpdir: + td = Path(tmpdir) + t = LifecycleTracker(td) + + d1 = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + d2 = "sha256:2222222222222222222222222222222222222222222222222222222222222222" + img = "ghcr.io/projectbluefin/utah" + + os_txt = "ID=utah\nIMAGE_ID=utah\nVERSION_ID=51\n" + + s_initial = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": {"image": {"imageDigest": d1, "image": {"image": f"{img}:testing"}}, "pinned": False}, + "staged": None, + "rollback": None, + } + }) + t.record_phase("initial_boot", s_initial, "PASS", desktop_ok=True, os_release_text=os_txt) + + s_up_staged = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": {"image": {"imageDigest": d1, "image": {"image": f"{img}:testing"}}, "pinned": False}, + "staged": {"image": {"imageDigest": d2, "image": {"image": f"{img}@sha256:2222..."}}, "pinned": False}, + "rollback": None, + } + }) + t.record_phase("upgrade_staged", s_up_staged, "PASS", desktop_ok=True) + t.assert_transition("initial_boot", "upgrade_staged", "upgrade_staged") + + s_up_booted = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": {"image": {"imageDigest": d2, "image": {"image": f"{img}@sha256:2222..."}}, "pinned": False}, + "staged": None, + "rollback": {"image": {"imageDigest": d1, "image": {"image": f"{img}:testing"}}, "pinned": False}, + } + }) + t.record_phase("upgraded_boot", s_up_booted, "PASS", desktop_ok=True, os_release_text=os_txt) + t.assert_transition("upgrade_staged", "upgraded_boot", "upgraded_boot") + + s_rb_staged = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": {"image": {"imageDigest": d2, "image": {"image": f"{img}@sha256:2222..."}}, "pinned": False}, + "staged": {"image": {"imageDigest": d1, "image": {"image": f"{img}:testing"}}, "pinned": False}, + "rollback": {"image": {"imageDigest": d1, "image": {"image": f"{img}:testing"}}, "pinned": False}, + } + }) + t.record_phase("rollback_staged", s_rb_staged, "PASS", desktop_ok=True) + t.assert_transition("upgraded_boot", "rollback_staged", "rollback_staged") + + s_rb_booted = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": {"image": {"imageDigest": d1, "image": {"image": f"{img}:testing"}}, "pinned": False}, + "staged": None, + "rollback": {"image": {"imageDigest": d2, "image": {"image": f"{img}@sha256:2222..."}}, "pinned": False}, + } + }) + t.record_phase("rollback_boot", s_rb_booted, "PASS", desktop_ok=True, os_release_text=os_txt) + t.assert_transition("rollback_staged", "rollback_boot", "rollback_boot") + + summary = t.generate_report() + assert summary["overall_status"] == "PASS" + assert len(summary["phases"]) == 5 + print("Simulated bootc lifecycle test passed all 5 phases successfully!") + return 0 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_bootc_lifecycle.py b/tests/unit/test_bootc_lifecycle.py new file mode 100644 index 000000000..f359cc240 --- /dev/null +++ b/tests/unit/test_bootc_lifecycle.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Unit tests for the bootc upgrade and rollback lifecycle verifier and diagnostics.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +import importlib.util +import sys + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT_PATH = REPO_ROOT / "scripts" / "verify-bootc-lifecycle.py" + +_spec = importlib.util.spec_from_file_location("verify_bootc_lifecycle", SCRIPT_PATH) +if _spec is None or _spec.loader is None: + raise ImportError(f"Cannot load module from {SCRIPT_PATH}") +_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_mod) + +LifecycleTracker = _mod.LifecycleTracker +extract_deployment_info = _mod.extract_deployment_info +parse_bootc_status = _mod.parse_bootc_status +parse_os_release = _mod.parse_os_release + + +class BootcLifecycleTests(unittest.TestCase): + def setUp(self) -> None: + self.d1 = "sha256:1111111111111111111111111111111111111111111111111111111111111111" + self.d2 = "sha256:2222222222222222222222222222222222222222222222222222222222222222" + self.img = "ghcr.io/projectbluefin/utah" + self.utah_os_release = "ID=utah\nIMAGE_ID=utah\nVARIANT_ID=utah\nVERSION_ID=51\n" + + def test_parse_valid_bootc_status(self) -> None: + payload = { + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": { + "image": { + "imageDigest": self.d1, + "image": {"image": f"{self.img}:testing"}, + }, + "pinned": False, + }, + "staged": None, + "rollback": None, + } + } + status = parse_bootc_status(json.dumps(payload)) + self.assertIn("booted", status) + + info = extract_deployment_info(status["booted"]) + self.assertEqual(info["digest"], self.d1) + self.assertEqual(info["image"], f"{self.img}:testing") + self.assertFalse(info["pinned"]) + + def test_parse_invalid_bootc_status(self) -> None: + # Empty + with self.assertRaises(ValueError): + parse_bootc_status("") + + # Not JSON + with self.assertRaises(ValueError): + parse_bootc_status("not json") + + # Missing .status + with self.assertRaises(ValueError): + parse_bootc_status(json.dumps({"booted": {}})) + + # Missing .status.booted + with self.assertRaises(ValueError): + parse_bootc_status(json.dumps({"status": {}})) + + # Missing sha256 prefix + with self.assertRaises(ValueError): + parse_bootc_status(json.dumps({ + "status": { + "booted": { + "image": {"imageDigest": "invalid_digest"} + } + } + })) + + def test_parse_bootc_status_accepts_current_v1_schema(self) -> None: + # Verified live against bootc-1.16.7: `bootc status --format=json` + # reports apiVersion "org.containers.bootc/v1", not "v1alpha1". + payload = { + "apiVersion": "org.containers.bootc/v1", + "status": { + "booted": { + "image": { + "imageDigest": self.d1, + "image": {"image": f"{self.img}:testing"}, + }, + "pinned": False, + }, + "staged": None, + "rollback": None, + } + } + status = parse_bootc_status(json.dumps(payload)) + self.assertIn("booted", status) + + def test_parse_bootc_status_rejects_unknown_schema_version(self) -> None: + payload = { + "apiVersion": "org.containers.bootc/v2", + "status": { + "booted": { + "image": {"imageDigest": self.d1}, + } + }, + } + with self.assertRaises(ValueError) as ctx: + parse_bootc_status(json.dumps(payload)) + self.assertIn("schema mismatch", str(ctx.exception)) + + def test_parse_os_release_utah_identity(self) -> None: + data = parse_os_release(self.utah_os_release) + self.assertEqual(data.get("ID"), "utah") + self.assertEqual(data.get("IMAGE_ID"), "utah") + self.assertEqual(data.get("VERSION_ID"), "51") + + def test_full_successful_lifecycle_progression(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + work_dir = Path(tmpdir) + tracker = LifecycleTracker(work_dir) + + # Phase 1: Initial boot + s1 = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": { + "image": {"imageDigest": self.d1, "image": {"image": f"{self.img}:testing"}}, + "pinned": False, + }, + "staged": None, + "rollback": None, + } + }) + p1 = tracker.record_phase( + "initial_boot", s1, "PASS", desktop_ok=True, os_release_text=self.utah_os_release + ) + self.assertEqual(p1["status"], "PASS") + self.assertEqual(p1["booted"]["digest"], self.d1) + self.assertTrue(p1["os_identity_utah"]) + + # Phase 2: Upgrade staged + s2 = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": { + "image": {"imageDigest": self.d1, "image": {"image": f"{self.img}:testing"}}, + "pinned": False, + }, + "staged": { + "image": {"imageDigest": self.d2, "image": {"image": f"{self.img}@sha256:2222"}}, + "pinned": False, + }, + "rollback": None, + } + }) + p2 = tracker.record_phase("upgrade_staged", s2, "PASS", desktop_ok=True) + self.assertEqual(p2["staged"]["digest"], self.d2) + self.assertTrue(tracker.assert_transition("initial_boot", "upgrade_staged", "upgrade_staged")) + + # Phase 3: Upgraded boot + s3 = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": { + "image": {"imageDigest": self.d2, "image": {"image": f"{self.img}@sha256:2222"}}, + "pinned": False, + }, + "staged": None, + "rollback": { + "image": {"imageDigest": self.d1, "image": {"image": f"{self.img}:testing"}}, + "pinned": False, + }, + } + }) + p3 = tracker.record_phase( + "upgraded_boot", s3, "PASS", desktop_ok=True, os_release_text=self.utah_os_release + ) + self.assertEqual(p3["booted"]["digest"], self.d2) + self.assertEqual(p3["rollback"]["digest"], self.d1) + self.assertTrue(tracker.assert_transition("upgrade_staged", "upgraded_boot", "upgraded_boot")) + + # Phase 4: Rollback staged + s4 = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": { + "image": {"imageDigest": self.d2, "image": {"image": f"{self.img}@sha256:2222"}}, + "pinned": False, + }, + "staged": { + "image": {"imageDigest": self.d1, "image": {"image": f"{self.img}:testing"}}, + "pinned": False, + }, + "rollback": { + "image": {"imageDigest": self.d1, "image": {"image": f"{self.img}:testing"}}, + "pinned": False, + }, + } + }) + p4 = tracker.record_phase("rollback_staged", s4, "PASS", desktop_ok=True) + self.assertEqual(p4["staged"]["digest"], self.d1) + self.assertTrue(tracker.assert_transition("upgraded_boot", "rollback_staged", "rollback_staged")) + + # Phase 5: Rollback boot + s5 = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": { + "image": {"imageDigest": self.d1, "image": {"image": f"{self.img}:testing"}}, + "pinned": False, + }, + "staged": None, + "rollback": { + "image": {"imageDigest": self.d2, "image": {"image": f"{self.img}@sha256:2222"}}, + "pinned": False, + }, + } + }) + p5 = tracker.record_phase( + "rollback_boot", s5, "PASS", desktop_ok=True, os_release_text=self.utah_os_release + ) + self.assertEqual(p5["booted"]["digest"], self.d1) + self.assertEqual(p5["rollback"]["digest"], self.d2) + self.assertTrue(tracker.assert_transition("rollback_staged", "rollback_boot", "rollback_boot")) + + report = tracker.generate_report() + self.assertEqual(report["overall_status"], "PASS") + self.assertEqual(len(report["phases"]), 5) + self.assertEqual(len(report["diagnostics"]), 0) + + # Check files were written + self.assertTrue((work_dir / "lifecycle-summary.json").exists()) + self.assertTrue((work_dir / "lifecycle-report.md").exists()) + + def test_failure_diagnostics_on_desktop_failure(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + work_dir = Path(tmpdir) + tracker = LifecycleTracker(work_dir) + + s1 = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": { + "image": {"imageDigest": self.d1, "image": {"image": f"{self.img}:testing"}}, + } + } + }) + p1 = tracker.record_phase( + "upgraded_boot", s1, "FAIL", desktop_ok=False, notes="GDM failed to start" + ) + self.assertEqual(p1["status"], "FAIL") + self.assertFalse(p1["desktop_ok"]) + + report = tracker.generate_report() + self.assertEqual(report["overall_status"], "FAIL") + self.assertGreaterEqual(len(report["diagnostics"]), 1) + + # Verify diagnostic captured the active booted digest + diag = report["diagnostics"][0] + self.assertEqual(diag["phase"], "upgraded_boot") + self.assertEqual(diag["active_booted_digest"], self.d1) + self.assertIn("Active booted digest", diag["message"]) + + def test_failure_diagnostics_on_unexpected_reboot_digest(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + work_dir = Path(tmpdir) + tracker = LifecycleTracker(work_dir) + + s1 = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": {"image": {"imageDigest": self.d1, "image": {"image": self.img}}}, + "staged": {"image": {"imageDigest": self.d2, "image": {"image": self.img}}}, + } + }) + tracker.record_phase("upgrade_staged", s1, "PASS") + + # Reboot resulted in the same old d1 instead of d2! + s2 = json.dumps({ + "apiVersion": "org.containers.bootc/v1alpha1", + "status": { + "booted": {"image": {"imageDigest": self.d1, "image": {"image": self.img}}}, + "staged": None, + } + }) + tracker.record_phase("upgraded_boot", s2, "PASS") + + with self.assertRaises(AssertionError) as ctx: + tracker.assert_transition("upgrade_staged", "upgraded_boot", "upgraded_boot") + self.assertIn("Active booted digest", str(ctx.exception)) + self.assertIn("Expected candidate digest", str(ctx.exception)) + + +if __name__ == "__main__": + unittest.main()