From 5025f4d9dca1be6107cb5837188d31f7b9bd60b7 Mon Sep 17 00:00:00 2001 From: Danathar Date: Fri, 11 Sep 2026 10:50:36 -0400 Subject: [PATCH 1/2] ci: add gated offline ISO build and boot validation Build the exact testing image into UEFI media and exercise the live Fisherman installer from its embedded OCI store before uploading the ISO. Retain serial logs, screenshots, and the failed disk image when a QEMU gate fails. Assisted-by: GPT-5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Danathar --- .github/workflows/iso.yml | 272 +++++++++++++++++++ AGENTS.md | 5 +- Justfile | 17 +- README.md | 13 +- docs/skills/ci-workflows.md | 34 ++- docs/skills/index.json | 8 +- docs/skills/index.md | 2 +- docs/skills/local-testing.md | 46 +++- iso/live/Containerfile | 10 +- iso/live/src/configure-live.sh | 8 +- iso/scripts/build-iso.sh | 24 +- iso/scripts/offline-e2e.sh | 462 +++++++++++++++++++++++++++++++++ 12 files changed, 857 insertions(+), 44 deletions(-) create mode 100644 .github/workflows/iso.yml create mode 100755 iso/scripts/offline-e2e.sh diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml new file mode 100644 index 000000000..7ea36e275 --- /dev/null +++ b/.github/workflows/iso.yml @@ -0,0 +1,272 @@ +name: Build, test, and publish Utah ISO + +run-name: >- + Utah ISO validation for ${{ github.event.workflow_run.head_sha || inputs.image || 'manual' }} + +on: + # The image promotion workflow is the dependency for this pipeline. Its + # successful conclusion means the candidate passed the digest gate before + # this workflow starts composing release media. + workflow_run: + workflows: ["Post-Testing E2E"] + types: [completed] + branches: [testing] + workflow_dispatch: + inputs: + image: + description: "Exact x86_64 image reference to test (must use @sha256:...)" + required: false + type: string + source_run_id: + description: "Successful Build Utah run containing image digest artifacts" + required: false + type: string + +permissions: {} + +concurrency: + group: utah-iso-${{ github.event.workflow_run.head_sha || github.run_id }} + cancel-in-progress: false + +jobs: + resolve-image: + name: Resolve exact candidate image + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-24.04 + permissions: + actions: read + contents: read + packages: read + outputs: + image: ${{ steps.resolve.outputs.image }} + payload_ref: ${{ steps.resolve.outputs.payload_ref }} + source_digest: ${{ steps.resolve.outputs.source_digest }} + steps: + - name: Resolve exact x86_64 image digest + id: resolve + env: + GH_TOKEN: ${{ github.token }} + DISPATCH_IMAGE: ${{ inputs.image }} + DISPATCH_RUN_ID: ${{ inputs.source_run_id }} + EVENT_SHA: ${{ github.event.workflow_run.head_sha }} + REPOSITORY: ${{ github.repository }} + OWNER: ${{ github.repository_owner }} + run: | + set -euo pipefail + build_run_id=manual + image="${DISPATCH_IMAGE:-}" + source_digest="" + + if [[ -n "$image" ]]; then + if [[ ! "$image" =~ ^ghcr\.io/[^/]+/utah@sha256:[0-9a-f]{64}$ ]]; then + echo "image must be an exact ghcr.io//utah@sha256: reference" >&2 + exit 1 + fi + source_digest="${image##*@}" + else + # A workflow_run for Post-Testing E2E carries that workflow's run + # ID, not the Build Utah run that produced the digest artifacts. + # Resolve the build by the shared commit unless a manual caller + # supplied its build run explicitly. + build_run_id="${DISPATCH_RUN_ID:-}" + if [[ -z "$build_run_id" && -n "${EVENT_SHA:-}" ]]; then + build_run_id="$(gh run list \ + --repo "$REPOSITORY" \ + --workflow build.yml \ + --commit "$EVENT_SHA" \ + --event push \ + --status success \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId')" + fi + [[ -n "$build_run_id" ]] || { + echo "No successful Build Utah run was supplied or found for ${EVENT_SHA:-the manual run}" >&2 + exit 1 + } + + digest_dir="${RUNNER_TEMP}/utah-build-digests" + mkdir -p "$digest_dir" + gh run download "$build_run_id" \ + --repo "$REPOSITORY" \ + --pattern 'image-digest-testing-*' \ + --dir "$digest_dir" + + mapfile -t digests < <( + grep -RhsE '^utah=sha256:[0-9a-f]{64}$' "$digest_dir" \ + | cut -d= -f2- \ + | sort -u + ) + [[ "${#digests[@]}" -eq 1 ]] || { + printf 'Expected exactly one main utah digest, found %s\n' "${#digests[@]}" >&2 + printf '%s\n' "${digests[@]}" >&2 + exit 1 + } + source_digest="${digests[0]}" + image="ghcr.io/${OWNER,,}/utah@${source_digest}" + fi + + # The payload reference is deliberately a run-specific tag. It is + # written into the ISO's local containers-storage and can never + # accidentally turn into a registry pull during the offline test. + payload_ref="ghcr.io/${OWNER,,}/utah:ci-$GITHUB_RUN_ID" + { + echo "image=$image" + echo "payload_ref=$payload_ref" + echo "source_digest=$source_digest" + } >> "$GITHUB_OUTPUT" + echo "Candidate: ${image}" + echo "Payload reference inside ISO: ${payload_ref}" + echo "Build run: ${build_run_id}" + + iso-e2e: + name: Build, install, and boot x86_64 UEFI ISO + needs: resolve-image + if: needs.resolve-image.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 360 + permissions: + contents: read + packages: read + env: + IMAGE: ${{ needs.resolve-image.outputs.image }} + PAYLOAD_REF: ${{ needs.resolve-image.outputs.payload_ref }} + SOURCE_DIGEST: ${{ needs.resolve-image.outputs.source_digest }} + ISO_OUTPUT: ${{ github.workspace }}/.ci/utah-iso-output + ISO_PATH: ${{ github.workspace }}/.ci/utah-iso-output/utah-live.iso + E2E_WORK: ${{ github.workspace }}/.ci/utah-iso-output/e2e + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + + - name: Install ISO and VM tooling + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + podman skopeo squashfs-tools xorriso dosfstools mtools \ + qemu-system-x86 qemu-utils ovmf ffmpeg sshpass + # The ISO build stages a full uncompressed root before making the + # squashfs. Reclaim the hosted runner's preinstalled SDKs first. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + mkdir -p "$ISO_OUTPUT" + df -h / + + - name: Pull exact candidate image + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + echo "$GH_TOKEN" | podman login ghcr.io \ + --username "$GITHUB_ACTOR" --password-stdin + podman pull "$IMAGE" + architecture="$(podman image inspect --format '{{.Architecture}}' "$IMAGE")" + case "$architecture" in + amd64|x86_64) ;; + *) echo "Expected an x86_64 image, got ${architecture}" >&2; exit 1 ;; + esac + echo "Pulled ${IMAGE} (${architecture})" + + - name: Build UEFI ISO without publishing it + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/utah-iso-stage" + UTAH_ISO_WORKDIR="${RUNNER_TEMP}/utah-iso-stage" \ + bash iso/scripts/build-iso.sh \ + "$IMAGE" "$ISO_PATH" "Utah Live" 1 "$PAYLOAD_REF" "$IMAGE" + test -s "$ISO_PATH" + xorriso -indev "$ISO_PATH" -report_el_torito plain \ + | tee "${ISO_OUTPUT}/el-torito.txt" + grep -qi 'EFI' "${ISO_OUTPUT}/el-torito.txt" + sha256sum "$ISO_PATH" > "${ISO_OUTPUT}/utah-live.iso.sha256" + cat > "${ISO_OUTPUT}/candidate.txt" <&2 + exit 1 + fi + done + + - name: Write final ISO manifest + if: success() + env: + MANIFEST: ${{ github.workspace }}/.ci/utah-iso-output/manifest.json + run: | + set -euo pipefail + python3 - "$MANIFEST" "$ISO_PATH" "$IMAGE" "$SOURCE_DIGEST" "$PAYLOAD_REF" <<'PY' + import hashlib + import json + import pathlib + import sys + + manifest, iso, image, digest, payload = sys.argv[1:] + path = pathlib.Path(iso) + sha256 = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + sha256.update(chunk) + pathlib.Path(manifest).write_text(json.dumps({ + "artifact": path.name, + "sha256": sha256.hexdigest(), + "architecture": "x86_64", + "firmware": "UEFI", + "source_image": image, + "source_digest": digest, + "embedded_payload": payload, + "live_marker": "UTAH_LIVE_READY", + "installed_marker": "UTAH_INSTALLED_GRAPHICAL_OK", + "network": { + "live_control": "qemu-user-restrict-on", + "installed": "none", + }, + }, indent=2) + "\n") + PY + + - name: Publish production ISO and checksum + if: success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: utah-iso-x86_64-${{ github.run_id }} + path: | + ${{ github.workspace }}/.ci/utah-iso-output/utah-live.iso + ${{ github.workspace }}/.ci/utah-iso-output/utah-live.iso.sha256 + ${{ github.workspace }}/.ci/utah-iso-output/candidate.txt + ${{ github.workspace }}/.ci/utah-iso-output/el-torito.txt + ${{ github.workspace }}/.ci/utah-iso-output/manifest.json + ${{ github.workspace }}/.ci/utah-iso-output/e2e/live-serial.log + ${{ github.workspace }}/.ci/utah-iso-output/e2e/installed-serial.log + ${{ github.workspace }}/.ci/utah-iso-output/e2e/e2e-summary.txt + ${{ github.workspace }}/.ci/utah-iso-output/e2e/screenshots/ + if-no-files-found: error + retention-days: 30 + + - name: Retain failed ISO diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: utah-iso-failure-${{ github.run_id }} + path: ${{ github.workspace }}/.ci/utah-iso-output/ + if-no-files-found: warn + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index 92965df25..f6c76ac3b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,9 @@ # Utah — Agent Operating Contract Utah composes [Bluefin](https://projectbluefin.io) on the Fedora Hummingbird -`bootc-os` base. Experimental pre-alpha: it has a build, not users. Nothing is -published — no image, no ISO artifact, no installer. +`bootc-os` base. Experimental pre-alpha: it has a build, not users. No stable +image or ISO is released yet; testing ISO artifacts are produced only by the +gated CI path. ## Read order diff --git a/Justfile b/Justfile index 6706129fc..f77b3ab80 100644 --- a/Justfile +++ b/Justfile @@ -43,14 +43,19 @@ check: test -f iso/live/src/etc/bootc-installer/images.json test -f iso/live/src/etc/bootc-installer/recipe.json test -f iso/scripts/build-iso.sh + test -f iso/scripts/offline-e2e.sh bash -n iso/live/src/configure-live.sh bash -n iso/live/src/install-flatpaks.sh bash -n iso/scripts/build-iso.sh + bash -n iso/scripts/offline-e2e.sh python3 -m json.tool iso/live/src/etc/bootc-installer/images.json >/dev/null python3 -m json.tool iso/live/src/etc/bootc-installer/recipe.json >/dev/null grep -q 'org.bootcinstaller.Installer' iso/live/src/install-flatpaks.sh grep -q 'containers-storage' iso/scripts/build-iso.sh grep -q 'UTAH_LIVE' iso/scripts/build-iso.sh + grep -q 'UTAH_LIVE_READY' iso/scripts/offline-e2e.sh + grep -q 'restrict=on' iso/scripts/offline-e2e.sh + grep -q -- '-net none' iso/scripts/offline-e2e.sh grep -q 'ENABLE_SSHD' Containerfile grep -q 'ENABLE_SSHD="${ENABLE_SSHD:-0}"' Justfile grep -q 'ARG PACKAGE_IMAGE_SHA=' Containerfile @@ -338,9 +343,9 @@ generate-bootable-image stream="testing": sync echo "Bootable disk ready: $disk" -# Build a single-architecture UEFI live ISO. This first slice proves the -# Utah live boot path; installer payload integration is intentionally the next -# ISO milestone. +# Build a single-architecture UEFI live ISO with the installer and an embedded +# OCI payload. Use `iso-e2e` below to exercise the same offline install path CI +# uses. iso stream="testing" debug="0": #!/usr/bin/env bash set -euo pipefail @@ -349,6 +354,12 @@ iso stream="testing" debug="0": mkdir -p "{{ base_dir }}" bash iso/scripts/build-iso.sh "$ref" "$(realpath "{{ base_dir }}")/utah-live.iso" "Utah Live" "{{ debug }}" "ghcr.io/{{ repo_organization }}/{{ image }}:{{ stream }}" +# Run the CI-shaped live boot, offline Fisherman install, and installed GNOME +# boot test against a locally built ISO. The test leaves its serial logs, +# screenshots, and installed disk under BASE_DIR for diagnosis. +iso-e2e iso_path="output/utah-live.iso" payload_ref="ghcr.io/projectbluefin/utah:testing": + UTAH_E2E_WORK="{{ base_dir }}/utah-iso-e2e" bash iso/scripts/offline-e2e.sh "{{ iso_path }}" "{{ payload_ref }}" + # Boot the live ISO with QEMU-for-Docker and expose its noVNC console. boot-iso: #!/usr/bin/env bash diff --git a/README.md b/README.md index 1ffae678c..515dcfb9a 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,10 @@ Bluefin built on Fedora Hummingbird. The more ... civilized murder machine. ![alt](https://github.com/user-attachments/assets/56428338-54a0-4376-a53b-5f02f8b101a1) **Experimental pre-alpha** — the image builds, boots, and reaches GDM in local -QEMU validation. No image has been published to a registry, there is no -installer payload, and no ISO has been released. Nothing here is ready to run on -a machine you care about. [Filing +QEMU validation. No stable image or ISO has been released. The gated CI path +now builds a candidate ISO, installs its embedded payload offline, and retains +the boot evidence; nothing here is ready to run on a machine you care about. +[Filing issues](https://github.com/projectbluefin/utah/issues) is the whole point. ## What it is @@ -66,9 +67,9 @@ verify step asserts *that file*, so the two cannot disagree. This is the honest list, and it is why the label above says pre-alpha. -- **Nothing is published.** No image, no ISO artifact, no installer. A live - ISO builds locally (`just iso`); installer payload integration is the next - ISO milestone. +- **Nothing is released.** The testing pipeline produces an ISO artifact only + after the exact-image, live-boot, offline-install, and installed-graphical + gates pass. A stable release ISO and public download remain future work. - **The NVIDIA and gaming flavors are unproven.** The OGC kernel compiles with `sched_ext` and `binderfs` genuinely enabled, and the NVIDIA open module compiles for the base kernel. The module against the OGC kernel, the driver diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index 24c8c643f..c01213f65 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -1,7 +1,7 @@ --- name: ci-workflows version: "1.0" -last_updated: "2026-09-05" +last_updated: "2026-09-11" id: ci-workflows one_line_purpose: Navigate Utah's build, promote, and sync workflow topology. entry_point: docs/skills/ci-workflows.md @@ -13,7 +13,7 @@ dependencies: [] tags: [ci, workflows, actions, promotion] description: >- build.yml contract gate, kernel-cache job, main/kernel matrix split, - promote-testing-to-main and sync-main-to-testing, actions@v1 delegation. + post-testing digest gate, ISO build/install/boot, and actions@v1 delegation. Use when changing .github/workflows/ or debugging a red run. metadata: type: reference @@ -21,8 +21,10 @@ metadata: # CI Workflows -Three workflows, all thin callers into `projectbluefin/actions@v1` reusables, -each pinned to a SHA tagged `v1`: +The image workflows are thin callers into `projectbluefin/actions@v1` +reusables, each pinned to a SHA tagged `v1`. The ISO workflow is intentionally +local because its QEMU and Fisherman gates need to retain disk and console +evidence on the same runner: - `.github/workflows/build.yml` -- pull requests, pushes to `testing`, a nightly cron, and manual dispatch. Top-level `permissions: {}`; each job @@ -30,6 +32,10 @@ each pinned to a SHA tagged `v1`: - `.github/workflows/promote-testing-to-main.yml` -- pushes to `testing`, a nightly cron, and manual dispatch. - `.github/workflows/sync-main-to-testing.yml` -- every push to `main`. +- `.github/workflows/iso.yml` -- after a successful `Post-Testing E2E` run on + `testing`, resolves the exact `utah` x86_64 digest, builds the UEFI ISO, + boots the live desktop, installs the embedded OCI payload with Fisherman + under a restricted network, and boots the installed system with no network. CI delegates builds, vulnerability reporting, SBOMs, keyless signatures, provenance, caching, and rechunking to `projectbluefin/actions@v1` (originated @@ -59,6 +65,26 @@ loudly there, because an invalid matrix creates no image job at all and the only symptom is `build_container: failure` from the aggregator (comment, `.github/workflows/build.yml`). +## iso: the offline production gate + +`iso.yml` listens to `Post-Testing E2E`, so ISO composition starts only after +the image promotion/digest gate has succeeded. It finds the matching +`Build Utah` run by commit and downloads the `utah=sha256:...` digest artifact; +the workflow never builds release media from a moving `:testing` tag. + +The build and all gates stay in one job. `iso/scripts/build-iso.sh` creates the +single-architecture UEFI media and embeds a run-specific local payload tag. +`iso/scripts/offline-e2e.sh` then observes `UTAH_LIVE_READY`, captures the live +serial console and framebuffer, invokes Fisherman through +`containers-storage:`, checks the installed root's Utah identity, and boots +the disk without its ISO or a network. A test-only service emits +`UTAH_INSTALLED_GRAPHICAL_OK` only after `graphical.target`, GDM, and GNOME +Shell are all active. + +The ISO, SHA-256 file, and manifest are uploaded only in a success-conditioned +step after every gate. Failure uploads retain the staging directory, including +the live/installed serial logs, screenshots, and failed disk image. + ## kernel_cache: skipped unless needed, skipped when published The kernel cache job runs only when `needs_kernel` is `true` -- while the diff --git a/docs/skills/index.json b/docs/skills/index.json index 9a380aa0a..639460a38 100644 --- a/docs/skills/index.json +++ b/docs/skills/index.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-05", + "generated_at": "2026-09-11", "schema_version": "1.0", "skills": [ { @@ -15,9 +15,9 @@ "actions", "promotion" ], - "description": "build.yml contract gate, kernel-cache job, main/kernel matrix split, promote-testing-to-main and sync-main-to-testing, actions@v1 delegation. Use when changing .github/workflows/ or debugging a red run.", + "description": "build.yml contract gate, kernel-cache job, main/kernel matrix split, post-testing digest gate, ISO build/install/boot, and actions@v1 delegation. Use when changing .github/workflows/ or debugging a red run.", "version": "1.0", - "last_updated": "2026-09-05", + "last_updated": "2026-09-11", "doc_type": "reference" }, { @@ -107,7 +107,7 @@ ], "description": "Local validation loop: build-ghcr, bootc install to-disk, QEMU/noVNC boot, live ISO. Use when validating changes end-to-end or debugging boot, GDM, or live-session failures.", "version": "1.0", - "last_updated": "2026-09-05", + "last_updated": "2026-09-11", "doc_type": "runbook" }, { diff --git a/docs/skills/index.md b/docs/skills/index.md index 2864a0b10..32e7fa052 100644 --- a/docs/skills/index.md +++ b/docs/skills/index.md @@ -3,7 +3,7 @@ This file is a human-readable mirror of `index.json`. Both are generated by `scripts/generate_skill_index.py` — do not hand-edit either file. -Generated: 2026-09-05 · schema 1.0 · 8 skills +Generated: 2026-09-11 · schema 1.0 · 8 skills | id | category | status | one-line purpose | |---|---|---|---| diff --git a/docs/skills/local-testing.md b/docs/skills/local-testing.md index 67755c90f..6cc953b99 100644 --- a/docs/skills/local-testing.md +++ b/docs/skills/local-testing.md @@ -1,7 +1,7 @@ --- name: local-testing version: "1.0" -last_updated: "2026-09-05" +last_updated: "2026-09-11" id: local-testing one_line_purpose: Build, install, and boot Utah locally in a VM or live ISO. entry_point: docs/skills/local-testing.md @@ -106,12 +106,11 @@ The local OCI ref is not available from the guest's localhost registry, so unified-storage service is skipped instead of retrying its registry repull forever. Published images omit that argument and keep the service enabled. -## Live ISO (initial bring-up) +## Live ISO and offline installation -The first ISO slice reuses Utah's own kernel, dracut-live, and GNOME image. -`just iso` builds a single-architecture UEFI live ISO; this first slice proves -the Utah live boot path, and installer payload integration is intentionally -the next ISO milestone (comment above `iso` in `Justfile`): +The ISO reuses Utah's own kernel, dracut-live, and GNOME image, then embeds the +bootc-installer bundle, default Flatpaks, and the selected OCI image in the +live squashfs. `just iso` builds a single-architecture UEFI live ISO: ```bash just iso testing @@ -120,12 +119,32 @@ just iso testing 1 # optional live-session SSH diagnostics (debug=1) ``` The result is `output/utah-live.iso`, assembled with systemd-boot, a -`UTAH_LIVE` dmsquash-live root, and a serial `UTAH_LIVE_READY` marker. It is -intended to prove live desktop boot first; bootc-installer/offline payload -integration is the next ISO milestone. `just boot-iso` boots it with -QEMU-for-Docker and exposes the noVNC console at the printed URL (comment -above `boot-iso` in `Justfile`), with TPM, UEFI, and `-snapshot` so nothing -persists. +`UTAH_LIVE` dmsquash-live root, and a serial `UTAH_LIVE_READY` marker. `just +boot-iso` boots it with QEMU-for-Docker and exposes the noVNC console at the +printed URL (comment above `boot-iso` in `Justfile`), with TPM, UEFI, and +`-snapshot` so nothing persists. + +The CI-shaped installer test requires the debug ISO because it uses the live +SSH control channel. It keeps the test output under `BASE_DIR`: + +```bash +just iso testing 1 +just iso-e2e +``` + +`iso-e2e` gives the live VM a QEMU user network with `restrict=on` and only a +host-forwarded SSH control port. Fisherman must install through the embedded +`containers-storage` reference. The installed disk is then booted with no ISO +and `-net none`; its serial marker is emitted only after Utah identity, +`graphical.target`, GDM, and a GNOME Shell session are all present. Serial +logs, screenshots, and the disposable installed disk remain in +`output/utah-iso-e2e` after either success or failure. + +The automated production path is `.github/workflows/iso.yml`. It starts after +the testing digest gate, resolves the exact x86_64 image digest from the build +artifacts, and uploads the ISO plus SHA-256 manifest only after the same live, +offline-install, and installed-boot gates pass. A failed run uploads the ISO +staging directory, including serial logs, screenshots, and the failed disk. ## Verification @@ -139,6 +158,7 @@ Then the manual runbook, when the change touches the boot path: just build-ghcr utah testing main just generate-bootable-image testing just boot-vm # success: GDM appears and GNOME Shell renders in noVNC -just iso testing +just iso testing 1 +just iso-e2e # success: live boot, offline Fisherman install, GNOME boot just boot-iso # success: live session renders; serial shows UTAH_LIVE_READY ``` diff --git a/iso/live/Containerfile b/iso/live/Containerfile index 0c271b1b2..46bad4438 100644 --- a/iso/live/Containerfile +++ b/iso/live/Containerfile @@ -4,12 +4,14 @@ ARG SOURCE_IMAGE=localhost/utah:testing FROM ${SOURCE_IMAGE} -# TARGET_IMAGE is the published image name recorded in the installer recipe -# and embedded offline store. SOURCE_IMAGE may be localhost for development; -# it must never leak into the released recipe. +# TARGET_IMAGE is the tag imported into the embedded offline store. The +# separately supplied TRACKING_IMAGE is the exact remote ref recorded for +# updates. SOURCE_IMAGE may be localhost for development; it must never leak +# into the released recipe. ARG TARGET_IMAGE=ghcr.io/projectbluefin/utah:testing +ARG TRACKING_IMAGE=ghcr.io/projectbluefin/utah:testing ARG DEBUG=0 -ENV TARGET_IMAGE=${TARGET_IMAGE} DEBUG=${DEBUG} +ENV TARGET_IMAGE=${TARGET_IMAGE} TRACKING_IMAGE=${TRACKING_IMAGE} DEBUG=${DEBUG} # Utah already carries dracut-live; systemd-boot is added only to the live # environment because the installed image uses the Hummingbird bootloader. diff --git a/iso/live/src/configure-live.sh b/iso/live/src/configure-live.sh index bcba740d1..78ab4d1ae 100644 --- a/iso/live/src/configure-live.sh +++ b/iso/live/src/configure-live.sh @@ -5,7 +5,10 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" INSTALLER_APP_ID=org.bootcinstaller.Installer +# TARGET_IMAGE names the OCI object embedded in the live image. The tracking +# reference can remain the exact digest that was validated before composition. TARGET_IMAGE="${TARGET_IMAGE:-ghcr.io/projectbluefin/utah:testing}" +TRACKING_IMAGE="${TRACKING_IMAGE:-${TARGET_IMAGE}}" # Live media must not apply update or unified-storage policy intended for an # installed image. The payload embedded in the squashfs is the install source. @@ -97,7 +100,8 @@ cp "${SCRIPT_DIR}/etc/bootc-installer/images.json" /etc/bootc-installer/images.j python3 - </dev/null || rm -rf "${WORK}" 2>/dev/null || true +} +trap cleanup_work EXIT cd "${ROOT}" echo "Building live environment from ${IMAGE}" podman build --layers \ + --cap-add sys_admin --security-opt label=disable \ --build-arg SOURCE_IMAGE="${IMAGE}" \ --build-arg TARGET_IMAGE="${PUBLISHED_IMAGE}" \ + --build-arg TRACKING_IMAGE="${TRACKING_IMAGE}" \ --build-arg DEBUG="${DEBUG}" \ --tag "${LIVE_IMAGE}" \ --file iso/live/Containerfile iso/live diff --git a/iso/scripts/offline-e2e.sh b/iso/scripts/offline-e2e.sh new file mode 100755 index 000000000..128348ac8 --- /dev/null +++ b/iso/scripts/offline-e2e.sh @@ -0,0 +1,462 @@ +#!/usr/bin/env bash +# Boot a Utah live ISO, install its embedded OCI payload with Fisherman, and +# boot the installed system without the ISO or a network. +# Usage: offline-e2e.sh ISO PAYLOAD_REF [TARGET_IMGREF] +# +# The live VM has a QEMU user-mode network only for the SSH control channel; +# 'restrict=on' prevents the guest from reaching the host or the outside +# network. The installed VM has no network device at all. This is deliberate: +# a successful install must come from the containers-storage payload in the +# ISO, not from a registry pull that happens to be available in CI. +set -euo pipefail + +ISO="${1:?live ISO path is required}" +PAYLOAD_REF="${2:?embedded payload reference is required}" +TARGET_IMGREF="${3:-${PAYLOAD_REF}}" +WORK="${UTAH_E2E_WORK:-/var/tmp/utah-iso-e2e}" +TEST_USER="${UTAH_E2E_USER:-utahtest}" +TEST_PASSWORD="${UTAH_E2E_PASSWORD:-utahtest}" +LIVE_PASSWORD="${UTAH_E2E_LIVE_PASSWORD:-live}" +SSH_PORT="${UTAH_E2E_SSH_PORT:-2222}" +SSH_TIMEOUT="${UTAH_E2E_SSH_TIMEOUT:-900}" +BOOT_TIMEOUT="${UTAH_E2E_BOOT_TIMEOUT:-900}" +QEMU_MEMORY="${UTAH_E2E_MEMORY:-8192}" +QEMU_CPUS="${UTAH_E2E_CPUS:-4}" + +ISO="$(realpath "${ISO}")" +mkdir -p "${WORK}/screenshots" +WORK="$(realpath "${WORK}")" +SHOTS="${WORK}/screenshots" +INSTALL_DISK="${WORK}/installed.qcow2" +LIVE_VARS="${WORK}/ovmf-vars-live.fd" +INSTALLED_VARS="${WORK}/ovmf-vars-installed.fd" +LIVE_MONITOR="${WORK}/live-monitor.sock" +INSTALLED_MONITOR="${WORK}/installed-monitor.sock" +LIVE_SERIAL="${WORK}/live-serial.log" +INSTALLED_SERIAL="${WORK}/installed-serial.log" +LIVE_PID_FILE="${WORK}/live.pid" +INSTALLED_PID_FILE="${WORK}/installed.pid" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +[[ -f "${ISO}" ]] || fail "ISO does not exist: ${ISO}" +[[ "${PAYLOAD_REF}" =~ ^[A-Za-z0-9./_-]+:[A-Za-z0-9._-]+$ ]] || \ + fail "payload reference must be a tag-shaped OCI reference: ${PAYLOAD_REF}" +[[ "${TARGET_IMGREF}" =~ ^[A-Za-z0-9./_-]+(:[A-Za-z0-9._-]+|@sha256:[0-9a-f]{64})$ ]] || \ + fail "tracking image must be a tag or digest OCI reference: ${TARGET_IMGREF}" + +QEMU="${QEMU_BINARY:-$(command -v qemu-system-x86_64 || true)}" +QEMU_IMG="${QEMU_IMG_BINARY:-$(command -v qemu-img || true)}" +[[ -n "${QEMU}" ]] || fail "qemu-system-x86_64 is not installed" +[[ -n "${QEMU_IMG}" ]] || fail "qemu-img is not installed" +command -v sshpass >/dev/null 2>&1 || fail "sshpass is not installed" + +OVMF_CODE="" +for candidate in \ + /usr/share/OVMF/OVMF_CODE_4M.fd \ + /usr/share/OVMF/OVMF_CODE.fd \ + /usr/share/edk2/ovmf/OVMF_CODE.fd \ + /usr/share/edk2/ovmf/OVMF_CODE_4M.fd; do + if [[ -f "${candidate}" ]]; then + OVMF_CODE="${candidate}" + break + fi +done +OVMF_VARS_SOURCE="" +for candidate in \ + /usr/share/OVMF/OVMF_VARS_4M.fd \ + /usr/share/OVMF/OVMF_VARS.fd \ + /usr/share/edk2/ovmf/OVMF_VARS.fd \ + /usr/share/edk2/ovmf/OVMF_VARS_4M.fd; do + if [[ -f "${candidate}" ]]; then + OVMF_VARS_SOURCE="${candidate}" + break + fi +done +[[ -n "${OVMF_CODE}" ]] || fail "OVMF code firmware is not installed" +[[ -n "${OVMF_VARS_SOURCE}" ]] || fail "OVMF variable store is not installed" + +if [[ -r /dev/kvm ]]; then + ACCEL=(-accel kvm) + echo "QEMU acceleration: KVM" +else + ACCEL=(-accel tcg,thread=multi) + echo "QEMU acceleration: TCG (no /dev/kvm)" +fi + +SSH_OPTS=( + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o ConnectTimeout=5 + -o PreferredAuthentications=password + -o PubkeyAuthentication=no + -o PasswordAuthentication=yes + -o ServerAliveInterval=30 + -o ServerAliveCountMax=10 +) + +ssh_live() { + sshpass -p "${LIVE_PASSWORD}" ssh "${SSH_OPTS[@]}" \ + -p "${SSH_PORT}" liveuser@127.0.0.1 "$@" +} + +scp_live() { + sshpass -p "${LIVE_PASSWORD}" scp "${SSH_OPTS[@]}" \ + -P "${SSH_PORT}" "$@" +} + +monitor() { + python3 - "$1" "$2" <<'PY' +import socket +import sys +import time + +path, command = sys.argv[1:] +with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(10) + client.connect(path) + client.sendall(command.encode() + b"\n") + time.sleep(0.4) +PY +} + +capture_screen() { + local label="$1" + local socket_path="$2" + local ppm="${SHOTS}/${label}.ppm" + local png="${SHOTS}/${label}.png" + + [[ -S "${socket_path}" ]] || { + echo "screenshot monitor is unavailable: ${socket_path}" >&2 + return 1 + } + monitor "${socket_path}" "screendump ${ppm}" || { + echo "QEMU screendump failed for ${label}" >&2 + return 1 + } + for _ in $(seq 1 10); do + [[ -s "${ppm}" ]] && break + sleep 1 + done + [[ -s "${ppm}" ]] || { + echo "QEMU produced no framebuffer for ${label}" >&2 + return 1 + } + if command -v ffmpeg >/dev/null 2>&1 && \ + ffmpeg -y -loglevel error -i "${ppm}" "${png}" >/dev/null 2>&1; then + rm -f "${ppm}" + echo "screenshot: ${png}" + else + echo "screenshot: ${ppm}" + fi +} + +kill_qemu() { + local pid="$1" + [[ -n "${pid}" ]] || return 0 + if kill -0 "${pid}" 2>/dev/null; then + kill "${pid}" 2>/dev/null || true + for _ in $(seq 1 20); do + kill -0 "${pid}" 2>/dev/null || return 0 + sleep 1 + done + kill -KILL "${pid}" 2>/dev/null || true + fi +} + +cleanup() { + local status=$? + set +e + # Keep the work directory, especially the failed disk image. A last + # framebuffer capture makes a boot failure diagnosable even when the + # failure happened before the phase-specific capture below. + if [[ "${status}" -ne 0 ]]; then + capture_screen live-failure "${LIVE_MONITOR}" + capture_screen installed-failure "${INSTALLED_MONITOR}" + fi + if [[ -f "${LIVE_PID_FILE}" ]]; then + kill_qemu "$(cat "${LIVE_PID_FILE}")" + fi + if [[ -f "${INSTALLED_PID_FILE}" ]]; then + kill_qemu "$(cat "${INSTALLED_PID_FILE}")" + fi + echo "E2E output retained at ${WORK}" + exit "${status}" +} +trap cleanup EXIT + +rm -f "${INSTALL_DISK}" "${LIVE_VARS}" "${INSTALLED_VARS}" \ + "${LIVE_MONITOR}" "${INSTALLED_MONITOR}" "${LIVE_SERIAL}" \ + "${INSTALLED_SERIAL}" "${LIVE_PID_FILE}" "${INSTALLED_PID_FILE}" +"${QEMU_IMG}" create -f qcow2 "${INSTALL_DISK}" 64G >/dev/null +cp -f "${OVMF_VARS_SOURCE}" "${LIVE_VARS}" + +echo "=== Phase 1/4: boot the Utah live ISO ===" +"${QEMU}" \ + -name utah-live-e2e \ + -machine q35 \ + "${ACCEL[@]}" \ + -cpu max \ + -m "${QEMU_MEMORY}" \ + -smp "${QEMU_CPUS}" \ + -drive "if=pflash,format=raw,readonly=on,file=${OVMF_CODE}" \ + -drive "if=pflash,format=raw,file=${LIVE_VARS}" \ + -drive "if=none,id=iso,file=${ISO},media=cdrom,readonly=on,format=raw" \ + -device virtio-scsi-pci,id=scsi \ + -device scsi-cd,drive=iso \ + -drive "if=none,id=disk,file=${INSTALL_DISK},format=qcow2" \ + -device virtio-blk-pci,drive=disk \ + -netdev "user,id=net0,restrict=on,hostfwd=tcp::${SSH_PORT}-:22" \ + -device virtio-net-pci,netdev=net0 \ + -display none \ + -vnc 127.0.0.1:2 \ + -monitor "unix:${LIVE_MONITOR},server,nowait" \ + -serial "file:${LIVE_SERIAL}" \ + -no-reboot \ + -daemonize \ + -pidfile "${LIVE_PID_FILE}" + +echo "Live VM network: QEMU user mode with restrict=on; only host SSH forwarding is available." +echo "Waiting for the live SSH control channel and UTAH_LIVE_READY marker..." +live_ready=0 +for _ in $(seq 1 "${SSH_TIMEOUT}"); do + if grep -aFq UTAH_LIVE_READY "${LIVE_SERIAL}" 2>/dev/null && \ + ssh_live true >/dev/null 2>&1; then + live_ready=1 + break + fi + sleep 1 +done +(( live_ready == 1 )) || { + tail -120 "${LIVE_SERIAL}" >&2 2>/dev/null || true + fail "live VM did not expose SSH and UTAH_LIVE_READY within ${SSH_TIMEOUT}s" +} + +echo "=== Phase 2/4: prove the live graphical session ===" +live_graphical=0 +for _ in $(seq 1 180); do + if ssh_live 'systemctl is-active graphical.target' 2>/dev/null | grep -qx active && \ + ssh_live 'systemctl is-active gdm.service' 2>/dev/null | grep -qx active && \ + ssh_live 'pgrep -u liveuser -x gnome-shell >/dev/null' 2>/dev/null; then + live_graphical=1 + break + fi + sleep 5 +done +(( live_graphical == 1 )) || fail "live VM did not reach a GNOME graphical session" +echo "Live graphical.target, gdm.service, and gnome-shell are active." +capture_screen live-desktop "${LIVE_MONITOR}" + +echo "=== Phase 3/4: install the embedded payload with no registry access ===" +RECIPE="${WORK}/recipe.json" +python3 - "${RECIPE}" "${PAYLOAD_REF}" "${TARGET_IMGREF}" "${TEST_USER}" "${TEST_PASSWORD}" <<'PY' +import json +import pathlib +import sys + +path, payload, target, user, password = sys.argv[1:] +recipe = { + "disk": "/dev/vda", + "filesystem": "btrfs", + "btrfsSubvolumes": False, + "encryption": {"type": "none"}, + "image": f"containers-storage:{payload}", + "targetImgref": target, + "local_imgref": f"containers-storage:{payload}", + "selinuxDisabled": False, + "unifiedStorage": True, + "composeFsBackend": False, + "bootloader": "grub2", + "hostname": "utah-e2e", + "user": { + "username": user, + "fullname": "Utah E2E", + "password": password, + "groups": ["wheel"], + }, + "flatpaks": [], +} +pathlib.Path(path).write_text(json.dumps(recipe, indent=2) + "\n") +PY +scp_live "${RECIPE}" liveuser@127.0.0.1:/tmp/utah-e2e-recipe.json + +# The isolated network is part of the test contract. Fisherman must resolve +# the image through the embedded containers-storage graphroot; it cannot +# satisfy the install by pulling from the registry. +ssh_live 'sudo /usr/local/bin/fisherman /tmp/utah-e2e-recipe.json' || \ + fail "Fisherman could not install the embedded OCI payload" + +echo "Fisherman completed from containers-storage:${PAYLOAD_REF}; tracking ${TARGET_IMGREF}." +echo "Preparing boot entries and a test-only graphical-session assertion..." +ssh_live "sudo TARGET_USER='${TEST_USER}' bash -s" <<'TARGET_SETUP' +set -euo pipefail + +root_part="$(blkid -t TYPE=btrfs -o device | head -1)" +esp_part="$(blkid -t TYPE=vfat -o device | head -1)" +[[ -n "${root_part}" ]] || { echo "installed btrfs partition not found" >&2; exit 1; } +[[ -n "${esp_part}" ]] || { echo "installed EFI partition not found" >&2; exit 1; } + +root_mount=/mnt/utah-e2e-root +esp_mount=/mnt/utah-e2e-esp +mkdir -p "${root_mount}" "${esp_mount}" +mount "${root_part}" "${root_mount}" +mount "${esp_part}" "${esp_mount}" +cleanup() { + umount "${esp_mount}" 2>/dev/null || true + umount "${root_mount}" 2>/dev/null || true +} +trap cleanup EXIT + +grep -q '^ID=utah$' "${root_mount}/etc/os-release" +grep -q '^IMAGE_ID=utah$' "${root_mount}/etc/os-release" +test -s "${root_mount}/usr/share/utah/contract.txt" + +# The test disk is disposable. Auto-login lets the guest prove that a GNOME +# shell was created for the account Fisherman made, instead of treating a +# graphical target with only a greeter as a complete desktop boot. +mkdir -p "${root_mount}/etc/gdm" +cat >"${root_mount}/etc/gdm/custom.conf" <"${root_mount}/etc/utah-e2e-graphical.sh" <<'EOF' +#!/usr/bin/env bash +set -u +for _ in $(seq 1 120); do + if grep -q '^ID=utah$' /etc/os-release && \ + grep -q '^IMAGE_ID=utah$' /etc/os-release && \ + systemctl is-active --quiet graphical.target && \ + systemctl is-active --quiet gdm.service && \ + pgrep -u "${TARGET_USER}" -x gnome-shell >/dev/null 2>&1; then + echo UTAH_INSTALLED_GRAPHICAL_OK + exit 0 + fi + sleep 2 +done +echo UTAH_INSTALLED_GRAPHICAL_FAIL +exit 1 +EOF +chmod 0755 "${root_mount}/etc/utah-e2e-graphical.sh" +cat >"${root_mount}/etc/systemd/system/utah-e2e-graphical.service" </dev/null +) +for entry in "${entries[@]}"; do + grep -q '^options ' "${entry}" || continue + sed -i 's/ rhgb//g; s/ quiet//g' "${entry}" + grep -q 'console=ttyS0' "${entry}" || \ + sed -i 's|^options .*|& console=ttyS0,115200n8|' "${entry}" + grep -q 'forward_to_console' "${entry}" || \ + sed -i 's|^options .*|& systemd.journald.forward_to_console=yes|' "${entry}" +done + +if [[ -f "${esp_mount}/EFI/fedora/shimx64.efi" ]]; then + mkdir -p "${esp_mount}/EFI/BOOT" + cp -f "${esp_mount}/EFI/fedora/shimx64.efi" "${esp_mount}/EFI/BOOT/BOOTX64.EFI" +fi +if [[ -f "${esp_mount}/EFI/fedora/grubx64.efi" ]]; then + cp -f "${esp_mount}/EFI/fedora/grubx64.efi" "${esp_mount}/EFI/BOOT/grubx64.efi" +fi + +sync +blockdev --flushbufs "${root_part}" 2>/dev/null || true +blockdev --flushbufs "${esp_part}" 2>/dev/null || true +echo "Utah identity and installed graphical assertion staged." +TARGET_SETUP + +echo "Powering off the live VM before booting its installed disk..." +ssh_live 'sudo systemctl poweroff --no-block' >/dev/null 2>&1 || true +live_pid="$(cat "${LIVE_PID_FILE}")" +for _ in $(seq 1 180); do + kill -0 "${live_pid}" 2>/dev/null || break + sleep 1 +done +if kill -0 "${live_pid}" 2>/dev/null; then + echo "Live VM did not power off cleanly; stopping it after disk flush." >&2 + kill_qemu "${live_pid}" +fi +cp -f "${LIVE_VARS}" "${INSTALLED_VARS}" + +echo "=== Phase 4/4: boot the installed Utah system without ISO or network ===" +"${QEMU}" \ + -name utah-installed-e2e \ + -machine q35 \ + "${ACCEL[@]}" \ + -cpu max \ + -m "${QEMU_MEMORY}" \ + -smp "${QEMU_CPUS}" \ + -drive "if=pflash,format=raw,readonly=on,file=${OVMF_CODE}" \ + -drive "if=pflash,format=raw,file=${INSTALLED_VARS}" \ + -drive "if=none,id=disk,file=${INSTALL_DISK},format=qcow2" \ + -device virtio-blk-pci,drive=disk \ + -net none \ + -display none \ + -vnc 127.0.0.1:3 \ + -monitor "unix:${INSTALLED_MONITOR},server,nowait" \ + -serial "file:${INSTALLED_SERIAL}" \ + -no-reboot \ + -daemonize \ + -pidfile "${INSTALLED_PID_FILE}" + +installed_ready=0 +for _ in $(seq 1 "${BOOT_TIMEOUT}"); do + if grep -aFq UTAH_INSTALLED_GRAPHICAL_OK "${INSTALLED_SERIAL}" 2>/dev/null; then + installed_ready=1 + break + fi + if grep -aEiq 'kernel panic|UTAH_INSTALLED_GRAPHICAL_FAIL|emergency mode' \ + "${INSTALLED_SERIAL}" 2>/dev/null; then + break + fi + sleep 1 +done +(( installed_ready == 1 )) || { + echo "--- installed serial log ---" >&2 + tail -200 "${INSTALLED_SERIAL}" >&2 2>/dev/null || true + fail "installed disk did not reach Utah's graphical-session marker" +} +capture_screen installed-desktop "${INSTALLED_MONITOR}" + +cat >"${WORK}/e2e-summary.txt" < Date: Sat, 12 Sep 2026 00:10:40 +0000 Subject: [PATCH 2/2] ci(iso): don't publish the full install disk on e2e failure Narrow the failure-artifact upload to serial logs, screenshots and build metadata instead of the whole utah-iso-output tree. That tree included the 64G qcow2 install disk and OVMF pflash vars, which add nothing for triage beyond what the serial logs/screenshots already show and shouldn't be published as a public CI artifact. The graphical-boot assertion this PR adds already asserts on a positive serial-console marker (UTAH_INSTALLED_GRAPHICAL_OK, written only after systemctl confirms graphical.target/gdm.service and a gnome-shell process) rather than on framebuffer screenshot state, so no change was needed there -- screenshots are diagnostic-only. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01LNgJ8VMdoqzGequbxsP2Kc --- .github/workflows/iso.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml index 7ea36e275..63ef571a0 100644 --- a/.github/workflows/iso.yml +++ b/.github/workflows/iso.yml @@ -267,6 +267,18 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: utah-iso-failure-${{ github.run_id }} - path: ${{ github.workspace }}/.ci/utah-iso-output/ + # Logs, screenshots and the summary only -- not the qcow2 install + # disk or OVMF pflash vars. Those are multi-GB, add nothing a + # serial log/screenshot doesn't already show for triage, and + # there's no reason to publish a full guest filesystem image + # (test creds, host keys, etc.) as a public CI artifact. + path: | + ${{ github.workspace }}/.ci/utah-iso-output/candidate.txt + ${{ github.workspace }}/.ci/utah-iso-output/el-torito.txt + ${{ github.workspace }}/.ci/utah-iso-output/manifest.json + ${{ github.workspace }}/.ci/utah-iso-output/e2e/live-serial.log + ${{ github.workspace }}/.ci/utah-iso-output/e2e/installed-serial.log + ${{ github.workspace }}/.ci/utah-iso-output/e2e/e2e-summary.txt + ${{ github.workspace }}/.ci/utah-iso-output/e2e/screenshots/ if-no-files-found: warn retention-days: 14