diff --git a/.github/workflows/iso-e2e-test.yml b/.github/workflows/iso-e2e-test.yml index a7d7234..97af75a 100644 --- a/.github/workflows/iso-e2e-test.yml +++ b/.github/workflows/iso-e2e-test.yml @@ -88,7 +88,17 @@ jobs: - name: Install QEMU and tools run: | sudo apt-get update - sudo apt-get install --yes qemu-system-x86 qemu-utils xorriso ffmpeg + sudo apt-get install --yes qemu-system-x86 qemu-utils xorriso squashfs-tools cpio ffmpeg + + # GitHub-hosted x64 Linux runners expose /dev/kvm but only to root:kvm; + # open it up so the harness's accel=kvm:tcg does not fall back to TCG + # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). + - name: Enable KVM for the runner user + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + ls -l /dev/kvm - name: Resolve ISO input id: context @@ -140,9 +150,10 @@ jobs: - name: Run the e2e harness env: + # The harness reads the kickstart's ostreecontainer --url from the + # image store embedded in the ISO; IMAGE_REF/IMAGE_TAG only choose + # between stored images if an ISO ever carries more than one. IMAGE_REF: ghcr.io/ublue-os/bluefin - # The kickstart's ostreecontainer ref must match the image embedded - # in the ISO's container storage, which follows the variant. IMAGE_TAG: ${{ startsWith(inputs.variant, 'lts') && inputs.variant || 'stable' }} E2E_LUKS: ${{ inputs.luks && '1' || '0' }} # LUKS boots pause at the passphrase prompt before the login marker. diff --git a/docs/qa.md b/docs/qa.md index 7bab36b..1151919 100644 --- a/docs/qa.md +++ b/docs/qa.md @@ -49,7 +49,13 @@ Review the workflow summary and downloaded proof artifacts. Record the prereleas `.github/workflows/iso-e2e-test.yml` invokes the canonical `tests/iso/e2e.sh` harness (with `hack/iso-e2e-test.sh` retained as a compatibility entry point) to exercise unattended installation under QEMU. Use this path when the change affects installation, disk setup, boot, or post-install behavior. -The harness writes logs, screenshots, summary data, and proof files to its output directory. It passes only after the installer reports completion and the installed system produces serial boot evidence. `E2E_INSTALL_TIMEOUT` (default 1800 seconds) and `E2E_POST_INSTALL_TIMEOUT` (default 180 seconds) can tune the two phases. Keep generated files out of source changes. +The harness writes logs, screenshots, summary data, and proof files to its output directory. It passes only after the installer reports completion and the installed system produces serial boot evidence. `E2E_INSTALL_TIMEOUT` (default 2700 seconds) and `E2E_POST_INSTALL_TIMEOUT` (default 180 seconds) can tune the two phases. Keep generated files out of source changes. + +The kickstart is delivered inside the initramfs, not over the network: the harness appends a gzip cpio overlay to the ISO's `initramfs.img` that carries the rendered kickstart plus a `e2e-kickstart.service` initrd unit, which copies it to `/run/install/ks.cfg` before switch-root. That is the only path anaconda reads for `inst.ks=`, and the titanoboa initramfs has no anaconda dracut module to fetch a URL into it. If the serial log ends with `Kickstart file /run/install/ks.cfg is missing.`, the overlay did not unpack or the unit did not run; check for `e2e-kickstart.service` in the initrd portion of `installer-serial.log`. + +The kickstart's `ostreecontainer --url` is read from the ISO itself: the harness pulls `var/lib/containers/storage/overlay-images/images.json` out of `/LiveOS/squashfs.img` (`squashfs-tools`) and uses the name the build stored the payload under, so the E2E cannot ask anaconda for an image the ISO does not carry (the nvidia-open ISO embeds `bluefin-nvidia-open`). `IMAGE_REF`/`IMAGE_TAG` only disambiguate if an ISO ever carries several images. The kickstart's `%pre` and `%pre-install` blocks print `e2e-pre:` and `e2e-pre-install:` lines to the serial log with the installer's own view of the store, the latter immediately before the payload step. + +Budget note: anaconda's optional Subscription DBus module fails to activate in this image and dbus waits its full 600-second `service_start_timeout` before installation starts (`Failed to activate service 'org.fedoraproject.Anaconda.Modules.Subscription': timed out`). The install timeout includes that dead time. ## Failure rules diff --git a/docs/skills/test-iso/SKILL.md b/docs/skills/test-iso/SKILL.md index da64cdc..937082f 100644 --- a/docs/skills/test-iso/SKILL.md +++ b/docs/skills/test-iso/SKILL.md @@ -49,6 +49,13 @@ The installer E2E phase passes only after the installer reports completion and t A promotion candidate requires a successful matching smoke run. A run for another tag or variant is not evidence. +## Harness internals worth knowing + +- The install VM boots the ISO's extracted `vmlinuz` and `initramfs.img` directly; the ISO is only the live root. That initramfs (titanoboa: `dracut --add "dmsquash-live ..."`) has no anaconda dracut module, so `inst.ks=` is never fetched. `e2e.sh` appends a gzip cpio overlay carrying `/etc/e2e/ks.cfg` and an initrd unit that copies it to `/run/install/ks.cfg` (the only path anaconda's `inst.ks=` option reads) before switch-root. Needs `cpio` and `gzip` on the runner. +- The kickstart's `ostreecontainer --url=… --transport=containers-storage` must name the image exactly as the ISO's embedded store knows it: titanoboa `podman pull`s the build's image ref into the live root's `/var/lib/containers/storage`, and anaconda resolves the URL against that store, not a registry. `e2e.sh` reads the stored names out of the ISO (`unsquashfs` on `/LiveOS/squashfs.img` → `var/lib/containers/storage/overlay-images/images.json`) and uses them; `IMAGE_REF`/`IMAGE_TAG` only choose between stored images. Needs `squashfs-tools`. `Using kickstart ostreecontainer --url=…` in the harness output and the `e2e-pre:` / `e2e-pre-install:` lines in `installer-serial.log` (`%pre` and `%pre-install` blocks that list `podman images` and run `skopeo inspect` on the chosen URL inside the installer, the latter right before the payload step) show what was asked for and what the installer saw. +- `installer-serial.log` is the first thing to read on a failure. `Kickstart file /run/install/ks.cfg is missing.` means the overlay was not applied. `Installer did not report completion` with anaconda logs present means the install itself was slow or wedged; the Subscription DBus module always burns 600 seconds before installation starts, which the default `E2E_INSTALL_TIMEOUT` already accounts for. +- Serial logs are dominated by `brltty` noise; filter it (`grep -v brltty`) before reading anaconda output. + ## Related documents - [QA](../../qa.md) diff --git a/tests/iso/data/bluefin-unattended.ks.tmpl b/tests/iso/data/bluefin-unattended.ks.tmpl index 0e14e8e..e941329 100644 --- a/tests/iso/data/bluefin-unattended.ks.tmpl +++ b/tests/iso/data/bluefin-unattended.ks.tmpl @@ -8,5 +8,20 @@ autopart --type=plain __AUTOPART_EXTRA__ rootpw --plaintext bluefin user --name=bluefin --password bluefin --groups=wheel bootloader --location=mbr --append="console=ttyS0" -ostreecontainer --url=__IMAGE_REF__:__IMAGE_TAG__ --transport=containers-storage --no-signature-verification +ostreecontainer --url=__IMAGE_URL__ --transport=containers-storage --no-signature-verification reboot + +# Triage evidence on the serial console: what the installer's own +# containers-storage holds, and whether the deploy's lookup resolves there. +# %pre runs right after kickstart parsing, %pre-install right before the +# payload step that runs `ostree container image deploy`. +%pre --log=/dev/ttyS0 +echo "e2e-pre: containers-storage images: $(podman images --format '{{.Repository}}:{{.Tag}} {{.ID}}' 2>&1 | tr '\n' ';')" +echo "e2e-pre: skopeo inspect containers-storage:__IMAGE_URL__: $(skopeo inspect --format '{{.Digest}}' containers-storage:__IMAGE_URL__ 2>&1)" +%end + +%pre-install --log=/dev/ttyS0 +echo "e2e-pre-install: containers-storage images: $(podman images --format '{{.Repository}}:{{.Tag}} {{.ID}}' 2>&1 | tr '\n' ';')" +echo "e2e-pre-install: skopeo inspect containers-storage:__IMAGE_URL__: $(skopeo inspect --format '{{.Digest}}' containers-storage:__IMAGE_URL__ 2>&1)" +echo "e2e-pre-install: storage mounts: $(findmnt -rno TARGET,SOURCE,FSTYPE / /var/lib/containers/storage/overlay 2>&1 | tr '\n' ';')" +%end diff --git a/tests/iso/e2e.sh b/tests/iso/e2e.sh index 48be9bb..0a6bfec 100755 --- a/tests/iso/e2e.sh +++ b/tests/iso/e2e.sh @@ -20,11 +20,10 @@ qemu_binary="${QEMU_BINARY:-qemu-system-x86_64}" qemu_img_binary="${QEMU_IMG_BINARY:-qemu-img}" qemu_machine="${QEMU_MACHINE:-q35}" qemu_accel="${QEMU_ACCEL:-kvm:tcg}" -http_port="${E2E_HTTP_PORT:-18080}" image_ref="${IMAGE_REF:-ghcr.io/ublue-os/bluefin}" image_tag="${IMAGE_TAG:-stable}" disk_size="${E2E_DISK_SIZE:-32G}" -install_timeout="${E2E_INSTALL_TIMEOUT:-1800}" +install_timeout="${E2E_INSTALL_TIMEOUT:-2700}" post_install_timeout="${E2E_POST_INSTALL_TIMEOUT:-180}" luks_enabled="${E2E_LUKS:-0}" luks_passphrase="${E2E_LUKS_PASSPHRASE:-bluefin-e2e-luks}" @@ -38,14 +37,17 @@ proof_file="$output_directory/e2e-proof.svg" final_screen_ppm="$output_directory/final-screen.ppm" final_screen_png="$output_directory/final-screen.png" install_disk="$output_directory/installed.qcow2" -http_root="$output_directory/http-root" +kickstart_file="$output_directory/kickstart.ks" work_dir="$output_directory/.work" kernel_path="$work_dir/vmlinuz" initrd_path="$work_dir/initramfs.img" +e2e_initrd_path="$work_dir/initramfs-e2e.img" +initrd_overlay_dir="$work_dir/initrd-overlay" +initrd_overlay_cpio="$work_dir/e2e-overlay.cpio" qmp_socket="$output_directory/.work/qmp.sock" post_qmp_socket="$output_directory/.work/post-qmp.sock" -mkdir -p "$http_root" "$work_dir" +mkdir -p "$work_dir" install_pid="" post_install_pid="" @@ -55,34 +57,73 @@ if [[ ! -f "$iso_path" ]]; then exit 1 fi -if ! command -v "$qemu_img_binary" >/dev/null 2>&1; then - echo "QEMU image tool not found: $qemu_img_binary" >&2 - exit 1 -fi - -if ! command -v "$qemu_binary" >/dev/null 2>&1; then - echo "QEMU binary not found: $qemu_binary" >&2 - exit 1 -fi +for tool in "$qemu_img_binary" "$qemu_binary" xorriso unsquashfs cpio gzip; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "Required tool not found: $tool" >&2 + exit 1 + fi +done autopart_extra="" if [[ "$luks_enabled" == "1" ]]; then autopart_extra="--encrypted --luks-version=luks2 --passphrase=${luks_passphrase}" fi -kickstart_file="$http_root/kickstart.ks" -python3 - "$work_dir/bluefin-unattended.ks" "$template_file" "$image_ref" "$image_tag" "$autopart_extra" <<'PY' +# anaconda's `ostree container image deploy --transport=containers-storage` +# resolves the kickstart's --url against the live root's container store, +# never against a registry. titanoboa fills that store with `podman pull +# ` (Justfile: rootfs-include-container), so the name the image is +# stored under is whatever the build pulled — the nvidia-open ISO embeds +# bluefin-nvidia-open, not bluefin. Read the stored names out of the ISO's +# squashfs instead of trusting IMAGE_REF/IMAGE_TAG so the harness cannot drift +# from the build; IMAGE_REF:IMAGE_TAG only picks between multiple stored images. +images_json_member="var/lib/containers/storage/overlay-images/images.json" +squashfs_root="$work_dir/squashfs-root" +rm -rf "$squashfs_root" +# unsquashfs can read the squashfs straight out of the ISO when its extents +# are contiguous (xorrisofs writes them that way); otherwise extract it first. +squashfs_offset="$(xorriso -indev "$iso_path" -find /LiveOS/squashfs.img -exec report_lba -- 2>/dev/null \ + | awk -F' *, *' 'BEGIN { ok = 1 } /^File data lba:/ { if (n > 0 && $2 != want) ok = 0; if (n == 0) start = $2; want = $2 + $3; n++ } END { if (n > 0 && ok) printf "%.0f", start * 2048 }')" +if [[ -n "$squashfs_offset" ]]; then + unsquashfs -n -q -no-xattrs -o "$squashfs_offset" -d "$squashfs_root" "$iso_path" "$images_json_member" >/dev/null +else + xorriso -indev "$iso_path" -osirrox on -extract /LiveOS/squashfs.img "$work_dir/squashfs.img" + unsquashfs -n -q -no-xattrs -d "$squashfs_root" "$work_dir/squashfs.img" "$images_json_member" >/dev/null + rm -f "$work_dir/squashfs.img" +fi +if [[ ! -f "$squashfs_root/$images_json_member" ]]; then + echo "ISO has no embedded container image store ($images_json_member missing from /LiveOS/squashfs.img)" >&2 + exit 1 +fi +image_url="$(python3 - "$squashfs_root/$images_json_member" "${image_ref}:${image_tag}" <<'PY' +import json +import sys + +images = json.load(open(sys.argv[1])) +names = [name for image in images for name in image.get('names') or []] +requested = sys.argv[2] +if requested in names: + print(requested) +elif len(images) == 1 and names: + print(f"note: ISO store has no {requested}; using its only image {names[0]}", file=sys.stderr) + print(names[0]) +else: + print(f"ISO store has no {requested} and {len(images)} images to choose from: {names}", file=sys.stderr) + sys.exit(1) +PY +)" +echo "Using kickstart ostreecontainer --url=${image_url} (from ISO ${images_json_member})" + +python3 - "$kickstart_file" "$template_file" "$image_url" "$autopart_extra" <<'PY' import pathlib import sys source = pathlib.Path(sys.argv[2]) text = source.read_text() -text = text.replace('__IMAGE_REF__', sys.argv[3]) -text = text.replace('__IMAGE_TAG__', sys.argv[4]) -text = text.replace('__AUTOPART_EXTRA__', sys.argv[5]) +text = text.replace('__IMAGE_URL__', sys.argv[3]) +text = text.replace('__AUTOPART_EXTRA__', sys.argv[4]) pathlib.Path(sys.argv[1]).write_text(text) PY -cp "$work_dir/bluefin-unattended.ks" "$kickstart_file" if [[ ! -f "$kernel_path" ]]; then xorriso -indev "$iso_path" -osirrox on -extract /boot/vmlinuz "$kernel_path" @@ -122,12 +163,56 @@ if [[ ! -f "$initrd_path" ]]; then xorriso -indev "$iso_path" -osirrox on -extract /boot/initramfs.img "$initrd_path" fi +# anaconda's inst.ks= boot option does not carry a location into stage 2: it +# is a store_const that only sets --ks=/run/install/ks.cfg +# (pyanaconda/argument_parsing.py). Fetching the kickstart into that path is +# the job of anaconda's dracut module (anaconda-lib.sh:parse_kickstart), which +# the ISO's initramfs does not carry (titanoboa runs dracut with only +# dmsquash-live added), so anaconda aborts with "Kickstart file +# /run/install/ks.cfg is missing." Do that step ourselves: ship the kickstart +# in an initramfs overlay and copy it into /run/install, which systemd carries +# across switch-root, before the pivot. +mkdir -p "$initrd_overlay_dir/etc/e2e" "$initrd_overlay_dir/etc/systemd/system/initrd.target.wants" +cp "$kickstart_file" "$initrd_overlay_dir/etc/e2e/ks.cfg" +cat > "$initrd_overlay_dir/etc/systemd/system/e2e-kickstart.service" <<'EOF' +[Unit] +Description=Stage the E2E kickstart at /run/install/ks.cfg for anaconda +DefaultDependencies=no +ConditionPathExists=/etc/initrd-release +Before=initrd-cleanup.service initrd-switch-root.target + +[Service] +Type=oneshot +RemainAfterExit=yes +ExecStart=/bin/sh -c 'mkdir -p /run/install && cp /etc/e2e/ks.cfg /run/install/ks.cfg' + +[Install] +WantedBy=initrd.target +EOF +ln -s ../e2e-kickstart.service "$initrd_overlay_dir/etc/systemd/system/initrd.target.wants/e2e-kickstart.service" +(cd "$initrd_overlay_dir" && find . -mindepth 1 -print0 | LC_ALL=C sort -z \ + | cpio --null --quiet -o -H newc -R 0:0) > "$initrd_overlay_cpio" +echo "Initramfs overlay contents:" +cpio -itv --quiet < "$initrd_overlay_cpio" +# The kernel unpacks concatenated initramfs segments one after another. A +# separately compressed trailer needs no 4-byte alignment against the ISO's +# zstd image, which is how virt-install --initrd-inject appends files too. +{ cat "$initrd_path"; gzip -n < "$initrd_overlay_cpio"; } > "$e2e_initrd_path" + if [[ ! -f "$install_disk" ]]; then "$qemu_img_binary" create -f qcow2 "$install_disk" "$disk_size" fi -python3 -m http.server "$http_port" --bind 0.0.0.0 --directory "$http_root" >/dev/null 2>&1 & -http_server_pid=$! +# accel=kvm:tcg falls back silently; say which one the VM will get so a +# 45-minute TCG install is not mistaken for a wedged installer. +if [[ "$qemu_accel" == kvm* ]]; then + if [[ -r /dev/kvm && -w /dev/kvm ]]; then + echo "QEMU accelerator: kvm (/dev/kvm usable)" + else + echo "QEMU accelerator: tcg (/dev/kvm not usable; expect a slow install)" + fi +fi + cleanup() { local exit_code=$? for pid in "$install_pid" "$post_install_pid"; do @@ -136,20 +221,11 @@ cleanup() { wait "$pid" 2>/dev/null || true fi done - kill "$http_server_pid" 2>/dev/null || true - wait "$http_server_pid" 2>/dev/null || true rm -rf "$work_dir" exit "$exit_code" } trap cleanup EXIT -for _ in $(seq 1 20); do - if curl --silent --show-error --fail "http://127.0.0.1:${http_port}/kickstart.ks" >/dev/null 2>&1; then - break - fi - sleep 1 -done - "$qemu_binary" \ -name bluefin-e2e \ -machine "${qemu_machine},accel=${qemu_accel}" \ @@ -159,7 +235,7 @@ done -drive file="$install_disk",format=qcow2,if=virtio,cache=none \ -cdrom "$iso_path" \ -kernel "$kernel_path" \ - -initrd "$initrd_path" \ + -initrd "$e2e_initrd_path" \ -netdev user,id=net0 \ -device virtio-net-pci,netdev=net0 \ -serial "file:$serial_file" \ @@ -167,7 +243,7 @@ done -display none \ -monitor none \ -qmp "unix:$qmp_socket,server=on,wait=off" \ - -append "${iso_kargs} console=ttyS0 rd.neednet=1 ip=dhcp inst.ks=http://10.0.2.2:${http_port}/kickstart.ks inst.text systemd.unit=anaconda.target systemd.journald.forward_to_console=1" \ + -append "${iso_kargs} console=ttyS0 rd.neednet=1 ip=dhcp inst.ks=file:/etc/e2e/ks.cfg inst.text systemd.unit=anaconda.target systemd.journald.forward_to_console=1" \ -no-reboot > /dev/null 2>&1 & install_pid=$!