From fe2ba4fc31865c99000a6b28b91fe7fd7797495f Mon Sep 17 00:00:00 2001 From: kylerankin Date: Sat, 14 Nov 2026 19:22:58 +0000 Subject: [PATCH] fix(lifecycle): install dakota with the composefs backend so upgrade and rollback reboots work (#1215) The lifecycle lane installed every image with bootc's default ostree backend, but dakota ships a composefs-native image. bootc refuses an ostree install on it ("bootupd is required for ostree-based installs"), so the VM fell back to direct kernel boot and upgrade/rollback reboots could never reach the staged deployment they exist to test. Detect the backend from the image's own markers instead of its name: ostree images ship /usr/lib/ostree/prepare-root.conf, composefs images never do, so its absence selects the composefs backend. Dakota now installs with --composefs-backend --bootloader systemd and firmware boot, so reboots reach the staged deployment. The lifecycle gate now keys scenarios on the detected backend (ostree vs composefs) rather than the image name, and utah's ostree path is unchanged. The detection choice and the backend gate are documented in the bootc lifecycle SKILL. Fixes #1215 Hive-Run: projectbluefin/testsuite#1215 Hive-Plan: issue-1215-implementation-plan#1-backend-detection Hive-Spec: issue-1215#acceptance-criteria Assisted-by: Claude Opus 5.5 via Claude Code Co-Authored-By: Claude Opus 5.5 Signed-off-by: kylerankin --- .github/workflows/e2e.yml | 147 ++++++++++++------ docs/skills/test-authoring/bootc/SKILL.md | 32 ++++ docs/skills/test-authoring/suite-map/SKILL.md | 4 +- tests/lifecycle/features/bootc.feature | 15 ++ tests/lifecycle/features/environment.py | 64 +++++++- tests/lifecycle/features/steps/steps.py | 17 ++ tests/shared/image_family.py | 32 ++++ .../unit/test_e2e_lifecycle_firmware_boot.py | 7 +- tests/unit/test_image_family.py | 56 +++++++ tests/unit/test_startup_service_inventory.py | 11 +- 10 files changed, 328 insertions(+), 57 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0e5b846fb..3c223a7f0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -423,6 +423,27 @@ jobs: # --karg is also not present in all bootc versions; omit it entirely since # systemd.firstboot=no is already included in the QEMU -append KERNEL_ARGS. BOOTLOADER_FLAG="" + + # Detect bootc's backend from the image's own markers, never its name + # (#1215). bootc ships exactly one backend marker per image: + # ostree backend -> /usr/lib/ostree/prepare-root.conf (always present) + # composefs backend -> /usr/lib/composefs/setup-root.toml (no ostree marker) + # The ostree marker is the reliable discriminator: composefs images + # never ship prepare-root.conf, so its absence selects the composefs + # backend. We key on the ostree marker rather than betting on the + # composefs marker's exact path, which varies across bootc versions. + # dakota (composefs) installs --composefs-backend --bootloader systemd; + # every ostree image keeps the default ostree backend + bootupd. + if test -f /usr/lib/ostree/prepare-root.conf \ + || test -f /etc/ostree/prepare-root.conf; then + COMPOSEFS_BACKEND="false" + COMPOSEFS_INSTALL_FLAG="" + else + COMPOSEFS_BACKEND="true" + COMPOSEFS_INSTALL_FLAG="--composefs-backend --bootloader systemd" + fi + echo "COMPOSEFS_BACKEND=${COMPOSEFS_BACKEND}" >> "$GITHUB_ENV" + echo "COMPOSEFS_INSTALL_FLAG=${COMPOSEFS_INSTALL_FLAG}" >> "$GITHUB_ENV" # The lifecycle suite reboots into staged deployments, which a # direct -kernel boot can't do: it would boot the first deployment's # kernel again (#1023). It boots through OVMF and the image's own @@ -436,10 +457,11 @@ jobs: # limited as before #1189. FIRMWARE_BOOT=false if [[ "${SUITE_DIR}" == "lifecycle" ]] \ - && sudo podman run --rm --quiet --entrypoint /bin/sh "${IMAGE}" -c 'command -v bootupctl' >/dev/null 2>&1; then + && { [[ "${COMPOSEFS_BACKEND}" == "true" ]] \ + || sudo podman run --rm --quiet --entrypoint /bin/sh "${IMAGE}" -c 'command -v bootupctl' >/dev/null 2>&1; }; then FIRMWARE_BOOT=true elif [[ "${SUITE_DIR}" == "lifecycle" ]]; then - echo "::warning title=lifecycle without firmware boot::${IMAGE} has no bootupctl, so the lifecycle VM boots the first deployment's kernel directly and reboots cannot reach a staged deployment (#1023)" + echo "::warning title=lifecycle without firmware boot::${IMAGE} has neither a composefs backend nor bootupctl, so the lifecycle VM boots the first deployment's kernel directly and reboots cannot reach a staged deployment (#1215)" fi if [[ "${FIRMWARE_BOOT}" != "true" ]] \ && sudo podman run --rm --quiet "${IMAGE}" bootc install to-disk --help 2>&1 | grep -q "\-\-bootloader"; then @@ -464,7 +486,7 @@ jobs: --via-loopback /data/disk.raw \ --filesystem ext4 \ --wipe \ - ${BOOTLOADER_FLAG} \ + ${COMPOSEFS_INSTALL_FLAG} ${BOOTLOADER_FLAG} \ 2>&1 | tee "${BOOTC_INSTALL_LOG}" BOOTC_INSTALL_RC=${PIPESTATUS[0]} set -e @@ -480,7 +502,7 @@ jobs: # instead of ending the step on a bare "mount: wrong fs type". if sudo blkid -s TYPE -o value "${ROOT_PART}" >/dev/null 2>&1 \ && sudo mount "${ROOT_PART}" /mnt/root; then - DEPLOY=$(sudo find /mnt/root/ostree/deploy/default/deploy/ -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | head -1) + DEPLOY=$(sudo find /mnt/root/sysroot/state/deploy /mnt/root/ostree/deploy/default/deploy -mindepth 1 -maxdepth 1 -type d -printf '%f\n' 2>/dev/null | head -1) if [[ -n "${DEPLOY}" ]]; then break fi @@ -516,8 +538,17 @@ jobs: fi ROOT_UUID=$(sudo blkid -s UUID -o value "${ROOT_PART}") echo "root=${ROOT_PART} uuid=${ROOT_UUID}" - D="/mnt/root/ostree/deploy/default/deploy/${DEPLOY}" - VAR="/mnt/root/ostree/deploy/default/var" + # composefs keeps its var tree inside the deployment root; ostree + # keeps it under deploy/default/var (#1215). + if [[ "${COMPOSEFS_BACKEND}" == "true" ]]; then + D="/mnt/root/sysroot/state/deploy/${DEPLOY}" + # composefs keeps its var tree inside the deployment root, so the + # runtime /var/home backing is ${D}/var (ostree shares one var). + VAR="${D}/var" + else + D="/mnt/root/ostree/deploy/default/deploy/${DEPLOY}" + VAR="/mnt/root/ostree/deploy/default/var" + fi # el10/LTS images may have multiple entries in usr/lib/modules/ (e.g., a # modules-only dir for the old kernel + a full dir with vmlinuz for the new # one). Pick the version that actually contains a vmlinuz; fall back to the @@ -568,46 +599,51 @@ jobs: fi sudo chown runner:runner ./vmlinuz ./initramfs.img - # Find the ostree boot entry symlink (boot.N/default//serial). - # ostree-prepare-root requires this to be a symlink pointing to the - # deployment directory. bootc may fail at the bootloader step (bootupd - # not in image) before creating it. If missing, create it ourselves. - BOOT_ENTRY=$(sudo find /mnt/root/ostree -maxdepth 4 -type l -name '0' 2>/dev/null \ - | grep -E '/boot\.[0-9]' | grep '/default/' | head -1) - if [[ -z "${BOOT_ENTRY}" ]]; then - echo "No boot.N symlink found — constructing from deployment hash" - DEPLOY_HASH="${DEPLOY%.*}" - DEPLOY_SERIAL="${DEPLOY##*.}" - BOOT_DIR="/mnt/root/ostree/boot.0/default/${DEPLOY_HASH}" - sudo mkdir -p "${BOOT_DIR}" - # symlink target is relative from the symlink's parent directory: - # boot.0/default// -> ../../../deploy/default/deploy/. - sudo ln -sfT "../../../deploy/default/deploy/${DEPLOY}" \ - "${BOOT_DIR}/${DEPLOY_SERIAL}" - BOOT_ENTRY="${BOOT_DIR}/${DEPLOY_SERIAL}" - echo "Created symlink: ${BOOT_ENTRY}" - fi + # ostree images need a boot.N/default// symlink for + # ostree-prepare-root. Composefs boots through systemd-boot, which + # has no such requirement, so this whole block is ostree-only (#1215). + if [[ "${COMPOSEFS_BACKEND}" != "true" ]]; then + # Find the ostree boot entry symlink (boot.N/default//serial). + # ostree-prepare-root requires this to be a symlink pointing to the + # deployment directory. bootc may fail at the bootloader step (bootupd + # not in image) before creating it. If missing, create it ourselves. + BOOT_ENTRY=$(sudo find /mnt/root/ostree -maxdepth 4 -type l -name '0' 2>/dev/null \ + | grep -E '/boot\.[0-9]' | grep '/default/' | head -1) + if [[ -z "${BOOT_ENTRY}" ]]; then + echo "No boot.N symlink found — constructing from deployment hash" + DEPLOY_HASH="${DEPLOY%.*}" + DEPLOY_SERIAL="${DEPLOY##*.}" + BOOT_DIR="/mnt/root/ostree/boot.0/default/${DEPLOY_HASH}" + sudo mkdir -p "${BOOT_DIR}" + # symlink target is relative from the symlink's parent directory: + # boot.0/default// -> ../../../deploy/default/deploy/. + sudo ln -sfT "../../../deploy/default/deploy/${DEPLOY}" \ + "${BOOT_DIR}/${DEPLOY_SERIAL}" + BOOT_ENTRY="${BOOT_DIR}/${DEPLOY_SERIAL}" + echo "Created symlink: ${BOOT_ENTRY}" + fi - OSTREE_PATH="${BOOT_ENTRY#/mnt/root}" - - # ostree-system-generator regex only accepts /ostree/boot.[01]/... - # Newer bootc (Fedora 44+) installs deployments under a versioned directory - # (e.g. boot.1.1) for atomic updates. The generator fails to parse this path - # → /var never bind-mounted → dbus-broker/logind/GDM cascade failure. - # Fix: create a canonical boot.N symlink pointing to the versioned dir so the - # generator gets a path it can parse, while composefs stays untouched. - if [[ "${OSTREE_PATH}" =~ /ostree/(boot\.[0-9]+)\.[0-9]+ ]]; then - VERSIONED_NAME=$(echo "${OSTREE_PATH}" | grep -oE 'boot\.[0-9]+\.[0-9]+') - CANONICAL_NAME="${BASH_REMATCH[1]}" # e.g. boot.1 - CANONICAL_ABS="/mnt/root/ostree/${CANONICAL_NAME}" - if [[ ! -e "${CANONICAL_ABS}" ]]; then - sudo ln -s "${VERSIONED_NAME}" "${CANONICAL_ABS}" - echo "Created canonical boot symlink: ${CANONICAL_ABS} -> ${VERSIONED_NAME}" + OSTREE_PATH="${BOOT_ENTRY#/mnt/root}" + + # ostree-system-generator regex only accepts /ostree/boot.[01]/... + # Newer bootc (Fedora 44+) installs deployments under a versioned directory + # (e.g. boot.1.1) for atomic updates. The generator fails to parse this path + # → /var never bind-mounted → dbus-broker/logind/GDM cascade failure. + # Fix: create a canonical boot.N symlink pointing to the versioned dir so the + # generator gets a path it can parse, while composefs stays untouched. + if [[ "${OSTREE_PATH}" =~ /ostree/(boot\.[0-9]+)\.[0-9]+ ]]; then + VERSIONED_NAME=$(echo "${OSTREE_PATH}" | grep -oE 'boot\.[0-9]+\.[0-9]+') + CANONICAL_NAME="${BASH_REMATCH[1]}" # e.g. boot.1 + CANONICAL_ABS="/mnt/root/ostree/${CANONICAL_NAME}" + if [[ ! -e "${CANONICAL_ABS}" ]]; then + sudo ln -s "${VERSIONED_NAME}" "${CANONICAL_ABS}" + echo "Created canonical boot symlink: ${CANONICAL_ABS} -> ${VERSIONED_NAME}" + fi + OSTREE_PATH="${OSTREE_PATH/${VERSIONED_NAME}/${CANONICAL_NAME}}" fi - OSTREE_PATH="${OSTREE_PATH/${VERSIONED_NAME}/${CANONICAL_NAME}}" - fi - echo "ostree boot path: ${OSTREE_PATH}" + echo "ostree boot path: ${OSTREE_PATH}" + fi # Mask services not needed in CI to shorten first-boot time. # NetworkManager-wait-online can block for up to 120s — most # expensive to leave unmasked on a SLIRP QEMU network. @@ -621,7 +657,16 @@ jobs: # # systemd.journald.forward_to_console=1 — forward full journal to serial for # diagnostics when SSH fails (output captured in vm-serial.log artifact). - KERNEL_ARGS="root=UUID=${ROOT_UUID} rw ostree=${OSTREE_PATH} systemd.firstboot=no console=ttyS0,115200 systemd.journald.forward_to_console=1 systemd.mask=brew-setup.service systemd.mask=NetworkManager-wait-online.service systemd.mask=firewalld.service systemd.mask=wpa_supplicant.service systemd.mask=flatpak-preinstall.service systemd.mask=podman-auto-update.service systemd.mask=podman-auto-update.timer systemd.mask=malcontent-webd.service systemd.mask=malcontent-webd-update.service systemd.mask=malcontent-control.service systemd.mask=speech-dispatcherd.service systemd.mask=avahi-daemon.service systemd.mask=avahi-daemon.socket systemd.mask=cups.service systemd.mask=cups.path systemd.mask=cups.socket systemd.mask=cups.browsed.service systemd.mask=blueman-mechanism.service systemd.mask=gnome-remote-desktop.service systemd.mask=bazzite-hardware-setup.service systemd.mask=greenboot-healthcheck.service systemd.mask=greenboot-set-rollback-trigger.service systemd.mask=systemd-udev-settle.service systemd.wants=sshd.service" + # The CI kargs every image boots with. ostree images add ostree= + # (the booted deployment); composefs images boot from bootc's own + # composefs.digest= loader entry and need no ostree= karg (#1215). + # root= and rw are required either way. + CI_BOOT_KARGS="systemd.firstboot=no console=ttyS0,115200 systemd.journald.forward_to_console=1 systemd.mask=brew-setup.service systemd.mask=NetworkManager-wait-online.service systemd.mask=firewalld.service systemd.mask=wpa_supplicant.service systemd.mask=flatpak-preinstall.service systemd.mask=podman-auto-update.service systemd.mask=podman-auto-update.timer systemd.mask=malcontent-webd.service systemd.mask=malcontent-webd-update.service systemd.mask=malcontent-control.service systemd.mask=speech-dispatcherd.service systemd.mask=avahi-daemon.service systemd.mask=avahi-daemon.socket systemd.mask=cups.service systemd.mask=cups.path systemd.mask=cups.socket systemd.mask=cups.browsed.service systemd.mask=blueman-mechanism.service systemd.mask=gnome-remote-desktop.service systemd.mask=bazzite-hardware-setup.service systemd.mask=greenboot-healthcheck.service systemd.mask=greenboot-set-rollback-trigger.service systemd.mask=systemd-udev-settle.service systemd.wants=sshd.service" + if [[ "${COMPOSEFS_BACKEND}" == "true" ]]; then + KERNEL_ARGS="root=UUID=${ROOT_UUID} rw ${CI_BOOT_KARGS}" + else + KERNEL_ARGS="root=UUID=${ROOT_UUID} rw ostree=${OSTREE_PATH} ${CI_BOOT_KARGS}" + fi # Enforcing mode is the default (inputs.selinux_enforcing); passing # false falls back to the legacy SELinux-off boot. if [[ "${SELINUX_ENFORCING}" != "true" ]]; then @@ -664,7 +709,13 @@ jobs: echo "SELinux: expected=${SELINUX_EXPECTED} policy=${SELINUX_POLICY}" echo "SELINUX_EXPECTED=${SELINUX_EXPECTED}" >> "$GITHUB_ENV" SELINUX_DIR="$(pwd)/results/selinux" - STATEROOT="/mnt/root/ostree/deploy/default" + # composefs sysroot is at /mnt/root/sysroot/state; ostree at + # /mnt/root/ostree/deploy/default (#1215). + if [[ "${COMPOSEFS_BACKEND}" == "true" ]]; then + STATEROOT="/mnt/root/sysroot/state" + else + STATEROOT="/mnt/root/ostree/deploy/default" + fi mkdir -p "${SELINUX_DIR}" if [[ "${SELINUX_EXPECTED}" == enforcing ]]; then for DEP in $(sudo find "${STATEROOT}/deploy" -maxdepth 1 -mindepth 1 -type d 2>/dev/null); do @@ -681,7 +732,7 @@ jobs: # bootc may deploy with a different checksum than what `ls|head-1` # returns, so we iterate to ensure the booted deployment gets set up. BFT_UID=1001 - for DEP in $(sudo find /mnt/root/ostree/deploy/default/deploy \ + for DEP in $(sudo find "${STATEROOT}/deploy" \ -maxdepth 1 -mindepth 1 -type d 2>/dev/null); do # CI VMs are not users: opt out of the daily countme ping # (projectbluefin-countme.service honours this marker). @@ -835,7 +886,7 @@ jobs: sudo chmod 700 "${VAR}/home/bluefin-test/.ssh" sudo chmod 600 "${VAR}/home/bluefin-test/.ssh/authorized_keys" # Write key to AuthorizedKeysCommand target in every deployment - for DEP in $(sudo find /mnt/root/ostree/deploy/default/deploy \ + for DEP in $(sudo find "${STATEROOT}/deploy" \ -maxdepth 1 -mindepth 1 -type d 2>/dev/null); do sudo cp /tmp/vm_key.pub "${DEP}/etc/ssh/ci-authorized-keys" sudo chown root:root "${DEP}/etc/ssh/ci-authorized-keys" @@ -890,7 +941,7 @@ jobs: fi # Pre-boot diagnostics: confirm key files and sshd drop-in are correct. echo "=== ci-authorized-keys ===" - for DEP in $(sudo find /mnt/root/ostree/deploy/default/deploy \ + for DEP in $(sudo find "${STATEROOT}/deploy" \ -maxdepth 1 -mindepth 1 -type d 2>/dev/null); do echo "--- ${DEP}/etc/ssh/ci-authorized-keys ---" sudo cat "${DEP}/etc/ssh/ci-authorized-keys" || echo "(missing)" diff --git a/docs/skills/test-authoring/bootc/SKILL.md b/docs/skills/test-authoring/bootc/SKILL.md index ae78e6f25..5a9c3cfd7 100644 --- a/docs/skills/test-authoring/bootc/SKILL.md +++ b/docs/skills/test-authoring/bootc/SKILL.md @@ -138,6 +138,38 @@ results, and on non-dakota images without an explicit `MIGRATION_TARGET` `If bootc upgrade output indicates image was staged, reboot VM and wait for SSH` shares the same staged markers (`_UPGRADE_STAGED_MARKERS`) as the gate. +## Detecting the bootc backend (ostree vs composefs) + +bootc selects its install backend from the image's install config, which is +unchanged at runtime. The two backends are mutually exclusive, so detect the +backend from a **marker in the image**, never from the image name: + +| Backend | Marker in the image | Never ships | +|------------|-------------------------------------------------------|-------------| +| ostree | `/usr/lib/ostree/prepare-root.conf` (always present) | — | +| composefs | `/usr/lib/composefs/setup-root.toml` | ostree marker | + +The ostree marker is the **reliable discriminator**: composefs images never +ship `prepare-root.conf`, so its absence selects the composefs backend. We key +on the ostree marker rather than betting on the composefs marker's exact path, +which varies across bootc versions. A single `test -f /usr/lib/ostree/ +prepare-root.conf` (falling back to `/etc/ostree/prepare-root.conf`) is enough: + +```bash +if test -f /usr/lib/ostree/prepare-root.conf || test -f /etc/ostree/prepare-root.conf; then + # ostree backend: default install + bootupd (unchanged behaviour) +else + # composefs backend: install with --composefs-backend --bootloader systemd +fi +``` + +This is how `.github/workflows/e2e.yml` chooses the install flags and how the +lifecycle `environment.py` gates scenarios (see `_ostree_backend`). When writing +a scenario, tag it with the backend it needs — `@ostree` for `ostree admin +status`/pin, `@composefs` for the composefs deployment-count check — and the +gate skips it on the wrong backend with a stated reason instead of erroring on +a backend-specific command. + ## ostree admin status parsing ``` diff --git a/docs/skills/test-authoring/suite-map/SKILL.md b/docs/skills/test-authoring/suite-map/SKILL.md index a19bd9fa8..6b9488257 100644 --- a/docs/skills/test-authoring/suite-map/SKILL.md +++ b/docs/skills/test-authoring/suite-map/SKILL.md @@ -199,7 +199,7 @@ that issue rather than file another, and close it when it no longer applies. Det -554 scenarios across 81 feature files: 439 active, 0 quarantined, 115 `@future`/`@pending`/`@hardware_blocked` +555 scenarios across 81 feature files: 434 active, 6 quarantined, 115 `@future`/`@pending`/`@hardware_blocked` | Suite | Scenarios | Active | Quarantined | Pending/Future | Notes | |---|---|---|---|---|---| @@ -213,7 +213,7 @@ that issue rather than file another, and close it when it no longer applies. Det | hardware | 13 | 13 | 0 | 0 | udev rules syntax validation (ZSA, Apple SuperDrive, Framework 16, AMD s2idle, Wooting, VIIA); emulated peripherals driven by shared SSH steps | | installer | 3 | 3 | 0 | 0 | post-boot assertions for installer-driven installs (UEFI, Flatpak exclusion, LUKS cmdline) | | kde-smoke | 13 | 13 | 0 | 0 | Plasma session, D-Bus services, AT-SPI tree, KWin output, one KCM, Dolphin, Konsole, Kickoff; all `@informational` | -| lifecycle | 24 | 22 | 0 | 2 | bootc upgrade / rollback / migration; switch is `@future` (needs a valid alternate image ref; pin activated with settled-deployment barrier) | +| lifecycle | 25 | 23 | 0 | 2 | bootc upgrade / rollback / migration; switch is `@future` (needs a valid alternate image ref; pin activated with settled-deployment barrier) | | nvidia | 12 | 0 | 0 | 12 | `@future` / `@hardware_blocked` until GPU passthrough exists in the lab | | security | 7 | 7 | 0 | 0 | cosign verify: projectbluefin (bluefin, lts, dakota) + ublue-os (latest, LTS, DX, nvidia, GTS, DX-nvidia, negative) | | smoke | 221 | 175 | 0 | 46 | 39 `@pending` flatpak-permission audits blocked on CI never seeding system Flatpaks; MIME handler coverage (Firefox/Papers/Loupe/Text Editor/video); GNOME accessibility (AT-SPI daemon, high-contrast toggle, a11y panel); display fractional/integer scaling via Mutter DisplayConfig; Bluefin desktop identity (Wayland, hardware accel, Dash to Dock); GNOME regression guards in gnome_regression.feature; Dakota sudo-rs privilege and PAM checks | diff --git a/tests/lifecycle/features/bootc.feature b/tests/lifecycle/features/bootc.feature index 761a5d0ad..a5098ba15 100644 --- a/tests/lifecycle/features/bootc.feature +++ b/tests/lifecycle/features/bootc.feature @@ -107,6 +107,21 @@ Feature: bootc upgrade and rollback lifecycle * SSH command return code is "0" * ostree status shows two deployments + @lifecycle @composefs + Scenario: bootc status reports at least two deployments after upgrade + # composefs equivalent of the ostree scenario: bootc status has no + # deployment list, but after an upgrade + reboot the booted and rollback + # deployments are both present, so counting non-null booted/staged/rollback + # entries gives >=2. Gated to the composefs backend; skipped on ostree. + * VM is restored to the pre-upgrade baseline deployment + * Run long SSH command: "sudo bootc upgrade" + * SSH command return code is "0" + * bootc upgrade output indicates image was staged + * Reboot VM and wait for SSH + * Run SSH command: "sudo bootc status --format=json" + * SSH command return code is "0" + * bootc status shows at least two deployments + @lifecycle @upgrade @version Scenario: os-release version changes are tracked after upgrade * VM is restored to the pre-upgrade baseline deployment diff --git a/tests/lifecycle/features/environment.py b/tests/lifecycle/features/environment.py index 0afe408a0..0d3af8bd5 100644 --- a/tests/lifecycle/features/environment.py +++ b/tests/lifecycle/features/environment.py @@ -6,7 +6,7 @@ """ import os -from tests.shared.ssh_steps import * # noqa: F401,F403 +from tests.shared.ssh_steps import run_ssh try: from tests.shared.timing import record_end, record_start @@ -49,6 +49,50 @@ def homed_migration_skip_reason(environ=None): return None +def _gate_backend_scenarios(context, scenario): + """Skip lifecycle scenarios whose backend does not match this run. + + The ostree-only scenarios (ostree admin pin/status) only apply to the + ostree backend, and the composefs deployment-count scenario only applies + to the composefs backend. Skip here, before any step runs, so a + backend-specific command never executes on the wrong backend (for example + `ostree admin status` on a composefs-native image, which errors because + the ostree tree is absent). Kept mutually exclusive with utah's ostree + path: utah has no composefs marker, so it keeps every ostree scenario and + skips the composefs one. + """ + tags = scenario.effective_tags + if "ostree" not in tags and "composefs" not in tags: + return + if "composefs" in tags and "ostree" not in tags and _ostree_backend(context): + # composefs-only scenario on an ostree image. + scenario.skip( + "bootc status deployment count is the composefs equivalent of " + "ostree admin status; this run boots the ostree backend" + ) + elif "ostree" in tags and "composefs" not in tags and not _ostree_backend(context): + # ostree-only scenario on a composefs-native image. + scenario.skip( + "ostree admin pin/status are ostree-only; this run boots the " + "composefs backend, which has no ostree tree" + ) + + +def _ostree_backend(context) -> bool: + """True when the running VM booted via bootc's ostree backend. + + bootc selects the backend from the image's install config, which is + unchanged at runtime: ostree images ship /usr/lib/ostree/prepare-root.conf, + composefs images never do. The ostree marker is the reliable discriminator + for the same reason the install step keys on it (see e2e.yml) -- the + composefs marker path varies across bootc versions, but prepare-root.conf is + always present for ostree and never for composefs. Detect from the running + image so the lifecycle scenario gating never keys on the image name. + """ + _, rc = run_ssh(context, "test -f /usr/lib/ostree/prepare-root.conf", timeout=15) + return rc == 0 + + def before_all(context): """Set up SSH connection parameters from environment.""" context.vm_ip = os.environ.get("VM_IP", "") @@ -76,6 +120,7 @@ def before_scenario(context, scenario): print(f"SKIP Scenario {scenario.name}: {reason}", flush=True) scenario.skip(reason) return + _gate_backend_scenarios(context, scenario) context.scenario = scenario context.command_stdout = "" context.ssh_rc = None @@ -91,6 +136,22 @@ def after_scenario(context, scenario): record_end(context, scenario) +def _is_backend_mismatch_skip(scenario) -> bool: + """True when a backend-tagged scenario was skipped for a backend mismatch. + + A @ostree or @composefs scenario only runs on its matching backend; + before_scenario skips it on the other backend, leaving status 'skipped' + with exactly one backend tag. This distinguishes that skip from a real + failure (status 'failed') so the qualification gate does not flag a + backend-incompatible scenario as missing evidence. + """ + tags = set(getattr(scenario, "effective_tags", getattr(scenario, "tags", ()))) + return ( + scenario.status.name == "skipped" + and ("ostree" in tags) != ("composefs" in tags) + ) + + def _assert_required_lifecycle_passed(features): """Reject missing or skipped evidence, including tag-filtered scenarios. @@ -108,6 +169,7 @@ def _assert_required_lifecycle_passed(features): for feature in features for scenario in feature.walk_scenarios() if not is_non_runnable(scenario) + and not _is_backend_mismatch_skip(scenario) and any( step.name == "bootc upgrade output indicates image was staged" for step in scenario.steps diff --git a/tests/lifecycle/features/steps/steps.py b/tests/lifecycle/features/steps/steps.py index b93e5bb1c..6f2cf3814 100644 --- a/tests/lifecycle/features/steps/steps.py +++ b/tests/lifecycle/features/steps/steps.py @@ -599,6 +599,23 @@ def ostree_two_deployments(context): ) +@step("bootc status shows at least two deployments") +def bootc_two_deployments(context): + """Verify bootc status reports at least two deployments after upgrade. + + bootc status --format=json has no deployment list, but after an upgrade + + reboot the booted and rollback deployments are both present. Count the + non-null booted/staged/rollback entries in .status; a healthy upgraded VM + has booted + rollback (>= 2). Composefs equivalent of the ostree step. + """ + status = _parse_bootc_status(context) + deployments = [status.get(key) for key in ("booted", "staged", "rollback")] + present = sum(1 for deployment in deployments if deployment is not None) + assert present >= 2, ( + f"Expected at least 2 bootc deployments, found {present}. status={status}" + ) + + # --------------------------------------------------------------------------- # systemd-homed migration steps (dakota:testing → dakota:next) # MIGRATION_TARGET env var — defaults to ghcr.io/projectbluefin/dakota:next diff --git a/tests/shared/image_family.py b/tests/shared/image_family.py index a7883593b..6517acf21 100644 --- a/tests/shared/image_family.py +++ b/tests/shared/image_family.py @@ -44,3 +44,35 @@ def is_dakota_family_image(image: str) -> bool: def is_utah_image(image: str) -> bool: return "utah" in image_name(image) + + +def composefs_backend_from_markers(ostree_prepare_root_present: bool) -> bool: + """Return True when an image installs with the composefs backend. + + bootc selects the backend from the image's install config, which is + unchanged at runtime. It ships exactly one backend marker per image: + * ostree backend -> /usr/lib/ostree/prepare-root.conf (always present) + * composefs backend -> /usr/lib/composefs/setup-root.toml (no ostree marker) + The ostree marker is the reliable discriminator: composefs images never + ship ``prepare-root.conf``, so its absence selects the composefs backend. + We key on the ostree marker rather than betting on the composefs marker's + exact path (which varies across bootc versions). Any image that ships + ``prepare-root.conf`` installs with the ostree backend. + """ + return not ostree_prepare_root_present + + +def is_composefs_image( + image: str, + *, + ostree_prepare_root_present: bool = False, +) -> bool: + """Return True when ``image`` should install with the composefs backend. + + The backend is a property of the image, not its name, so the marker is + supplied by the caller (a runtime ``podman run`` probe of the image in + ``e2e.yml``). Kept mutually exclusive with the ostree path: an ostree image + ships ``prepare-root.conf`` (``ostree_prepare_root_present`` is True), so + the predicate is False regardless of any composefs marker. + """ + return composefs_backend_from_markers(ostree_prepare_root_present) diff --git a/tests/unit/test_e2e_lifecycle_firmware_boot.py b/tests/unit/test_e2e_lifecycle_firmware_boot.py index 1682487f9..4c7513486 100644 --- a/tests/unit/test_e2e_lifecycle_firmware_boot.py +++ b/tests/unit/test_e2e_lifecycle_firmware_boot.py @@ -28,12 +28,13 @@ def test_lifecycle_install_keeps_the_images_own_loader_and_needs_it(): firmware = install.index("FIRMWARE_BOOT=true", lifecycle) override = install.index('BOOTLOADER_FLAG="--bootloader systemd"') # Firmware boot needs bootupd in the image; the systemd-boot override - # applies only when firmware boot is off (other suites, or an image - # without bootupd such as dakota, run 43525356388). + # applies only when firmware boot is off (other suites, or an ostree + # lifecycle image without bootupd, run 43525356388). Composefs images + # such as dakota get firmware boot automatically (#1215). assert lifecycle < probe < firmware < override guard = install.index('if [[ "${FIRMWARE_BOOT}" != "true" ]]') assert firmware < guard < override - assert "has no bootupctl" in install + assert "has neither a composefs backend nor bootupctl" in install fatal = install.index('if [[ ${BOOTC_INSTALL_RC} -ne 0 && "${FIRMWARE_BOOT}" == "true" ]]; then') assert fatal < install.index("continuing with direct kernel boot") assert "has no -kernel fallback" in install[fatal:fatal + 400] diff --git a/tests/unit/test_image_family.py b/tests/unit/test_image_family.py index 852ae4884..d44007139 100644 --- a/tests/unit/test_image_family.py +++ b/tests/unit/test_image_family.py @@ -5,9 +5,11 @@ import pytest from tests.shared.image_family import ( + composefs_backend_from_markers, image_name, image_under_test, is_bluefin_image, + is_composefs_image, is_dakota_family_image, is_dakota_image, is_utah_image, @@ -137,3 +139,57 @@ def test_is_utah_image_matches_image_name_only(image, expected): ) def test_dakota_family_follows_dakota_defaults_by_image_name(image, expected): assert is_dakota_family_image(image) is expected + + +# composefs backend detection (#1215): composefs-native when the image has no +# ostree prepare-root.conf. bootc's two backends are mutually exclusive, so the +# ostree marker's absence is the decisive discriminator (see image_family.py). +@pytest.mark.parametrize( + ("ostree_prepare_root_present", "expected"), + [ + (True, False), # ostree prepare-root present -> ostree (utah/bluefin) + (False, True), # ostree prepare-root absent -> composefs (dakota) + ], +) +def test_composefs_backend_from_markers( + ostree_prepare_root_present, + expected, +): + assert ( + composefs_backend_from_markers(ostree_prepare_root_present) + is expected + ) + + +@pytest.mark.parametrize( + ("ostree_prepare_root_present", "expected"), + [ + (True, False), + (False, True), + ], +) +def test_is_composefs_image_matches_markers( + ostree_prepare_root_present, + expected, +): + assert ( + is_composefs_image( + "ghcr.io/projectbluefin/dakota:testing", + ostree_prepare_root_present=ostree_prepare_root_present, + ) + is expected + ) + + +def test_composefs_and_ostree_are_mutually_exclusive(): + # An ostree image (utah) ships prepare-root.conf, so it is never composefs + # regardless of any composefs marker. + assert is_composefs_image( + "ghcr.io/projectbluefin/utah:testing", + ostree_prepare_root_present=True, + ) is False + # A composefs image (dakota) is composefs only when prepare-root is absent. + assert is_composefs_image( + "ghcr.io/projectbluefin/dakota:testing", + ostree_prepare_root_present=False, + ) is True diff --git a/tests/unit/test_startup_service_inventory.py b/tests/unit/test_startup_service_inventory.py index d089989d6..6deabeab1 100644 --- a/tests/unit/test_startup_service_inventory.py +++ b/tests/unit/test_startup_service_inventory.py @@ -22,9 +22,14 @@ def kernel_arg_masks(workflow: str) -> set[str]: - match = re.search(r'^\s*KERNEL_ARGS="([^"]*)"', workflow, re.MULTILINE) - assert match, "KERNEL_ARGS assignment not found in e2e.yml" - return set(re.findall(r"systemd\.mask=(\S+)", match.group(1))) + # The fast lane keeps the CI kargs in CI_BOOT_KARGS and combines them into + # KERNEL_ARGS per backend, so read every *_ARGS assignment and union the + # masks rather than only the literal KERNEL_ARGS line (#1215). + masks = set() + for match in re.finditer(r'^\s*[A-Z_]*ARGS="([^"]*)"', workflow, re.MULTILINE): + masks.update(re.findall(r"systemd\.mask=(\S+)", match.group(1))) + assert masks, "no systemd.mask= kargs found in e2e.yml" + return masks def symlink_loop_tokens(workflow: str) -> list[str]: