diff --git a/.github/workflows/iso.yml b/.github/workflows/iso.yml new file mode 100644 index 000000000..63ef571a0 --- /dev/null +++ b/.github/workflows/iso.yml @@ -0,0 +1,284 @@ +name: Build, test, and publish Utah ISO + +run-name: >- + Utah ISO validation for ${{ github.event.workflow_run.head_sha || inputs.image || 'manual' }} + +on: + # The image promotion workflow is the dependency for this pipeline. Its + # successful conclusion means the candidate passed the digest gate before + # this workflow starts composing release media. + workflow_run: + workflows: ["Post-Testing E2E"] + types: [completed] + branches: [testing] + workflow_dispatch: + inputs: + image: + description: "Exact x86_64 image reference to test (must use @sha256:...)" + required: false + type: string + source_run_id: + description: "Successful Build Utah run containing image digest artifacts" + required: false + type: string + +permissions: {} + +concurrency: + group: utah-iso-${{ github.event.workflow_run.head_sha || github.run_id }} + cancel-in-progress: false + +jobs: + resolve-image: + name: Resolve exact candidate image + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-24.04 + permissions: + actions: read + contents: read + packages: read + outputs: + image: ${{ steps.resolve.outputs.image }} + payload_ref: ${{ steps.resolve.outputs.payload_ref }} + source_digest: ${{ steps.resolve.outputs.source_digest }} + steps: + - name: Resolve exact x86_64 image digest + id: resolve + env: + GH_TOKEN: ${{ github.token }} + DISPATCH_IMAGE: ${{ inputs.image }} + DISPATCH_RUN_ID: ${{ inputs.source_run_id }} + EVENT_SHA: ${{ github.event.workflow_run.head_sha }} + REPOSITORY: ${{ github.repository }} + OWNER: ${{ github.repository_owner }} + run: | + set -euo pipefail + build_run_id=manual + image="${DISPATCH_IMAGE:-}" + source_digest="" + + if [[ -n "$image" ]]; then + if [[ ! "$image" =~ ^ghcr\.io/[^/]+/utah@sha256:[0-9a-f]{64}$ ]]; then + echo "image must be an exact ghcr.io//utah@sha256: reference" >&2 + exit 1 + fi + source_digest="${image##*@}" + else + # A workflow_run for Post-Testing E2E carries that workflow's run + # ID, not the Build Utah run that produced the digest artifacts. + # Resolve the build by the shared commit unless a manual caller + # supplied its build run explicitly. + build_run_id="${DISPATCH_RUN_ID:-}" + if [[ -z "$build_run_id" && -n "${EVENT_SHA:-}" ]]; then + build_run_id="$(gh run list \ + --repo "$REPOSITORY" \ + --workflow build.yml \ + --commit "$EVENT_SHA" \ + --event push \ + --status success \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId')" + fi + [[ -n "$build_run_id" ]] || { + echo "No successful Build Utah run was supplied or found for ${EVENT_SHA:-the manual run}" >&2 + exit 1 + } + + digest_dir="${RUNNER_TEMP}/utah-build-digests" + mkdir -p "$digest_dir" + gh run download "$build_run_id" \ + --repo "$REPOSITORY" \ + --pattern 'image-digest-testing-*' \ + --dir "$digest_dir" + + mapfile -t digests < <( + grep -RhsE '^utah=sha256:[0-9a-f]{64}$' "$digest_dir" \ + | cut -d= -f2- \ + | sort -u + ) + [[ "${#digests[@]}" -eq 1 ]] || { + printf 'Expected exactly one main utah digest, found %s\n' "${#digests[@]}" >&2 + printf '%s\n' "${digests[@]}" >&2 + exit 1 + } + source_digest="${digests[0]}" + image="ghcr.io/${OWNER,,}/utah@${source_digest}" + fi + + # The payload reference is deliberately a run-specific tag. It is + # written into the ISO's local containers-storage and can never + # accidentally turn into a registry pull during the offline test. + payload_ref="ghcr.io/${OWNER,,}/utah:ci-$GITHUB_RUN_ID" + { + echo "image=$image" + echo "payload_ref=$payload_ref" + echo "source_digest=$source_digest" + } >> "$GITHUB_OUTPUT" + echo "Candidate: ${image}" + echo "Payload reference inside ISO: ${payload_ref}" + echo "Build run: ${build_run_id}" + + iso-e2e: + name: Build, install, and boot x86_64 UEFI ISO + needs: resolve-image + if: needs.resolve-image.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 360 + permissions: + contents: read + packages: read + env: + IMAGE: ${{ needs.resolve-image.outputs.image }} + PAYLOAD_REF: ${{ needs.resolve-image.outputs.payload_ref }} + SOURCE_DIGEST: ${{ needs.resolve-image.outputs.source_digest }} + ISO_OUTPUT: ${{ github.workspace }}/.ci/utah-iso-output + ISO_PATH: ${{ github.workspace }}/.ci/utah-iso-output/utah-live.iso + E2E_WORK: ${{ github.workspace }}/.ci/utah-iso-output/e2e + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + + - name: Install ISO and VM tooling + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + podman skopeo squashfs-tools xorriso dosfstools mtools \ + qemu-system-x86 qemu-utils ovmf ffmpeg sshpass + # The ISO build stages a full uncompressed root before making the + # squashfs. Reclaim the hosted runner's preinstalled SDKs first. + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + mkdir -p "$ISO_OUTPUT" + df -h / + + - name: Pull exact candidate image + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + echo "$GH_TOKEN" | podman login ghcr.io \ + --username "$GITHUB_ACTOR" --password-stdin + podman pull "$IMAGE" + architecture="$(podman image inspect --format '{{.Architecture}}' "$IMAGE")" + case "$architecture" in + amd64|x86_64) ;; + *) echo "Expected an x86_64 image, got ${architecture}" >&2; exit 1 ;; + esac + echo "Pulled ${IMAGE} (${architecture})" + + - name: Build UEFI ISO without publishing it + run: | + set -euo pipefail + mkdir -p "${RUNNER_TEMP}/utah-iso-stage" + UTAH_ISO_WORKDIR="${RUNNER_TEMP}/utah-iso-stage" \ + bash iso/scripts/build-iso.sh \ + "$IMAGE" "$ISO_PATH" "Utah Live" 1 "$PAYLOAD_REF" "$IMAGE" + test -s "$ISO_PATH" + xorriso -indev "$ISO_PATH" -report_el_torito plain \ + | tee "${ISO_OUTPUT}/el-torito.txt" + grep -qi 'EFI' "${ISO_OUTPUT}/el-torito.txt" + sha256sum "$ISO_PATH" > "${ISO_OUTPUT}/utah-live.iso.sha256" + cat > "${ISO_OUTPUT}/candidate.txt" <&2 + exit 1 + fi + done + + - name: Write final ISO manifest + if: success() + env: + MANIFEST: ${{ github.workspace }}/.ci/utah-iso-output/manifest.json + run: | + set -euo pipefail + python3 - "$MANIFEST" "$ISO_PATH" "$IMAGE" "$SOURCE_DIGEST" "$PAYLOAD_REF" <<'PY' + import hashlib + import json + import pathlib + import sys + + manifest, iso, image, digest, payload = sys.argv[1:] + path = pathlib.Path(iso) + sha256 = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + sha256.update(chunk) + pathlib.Path(manifest).write_text(json.dumps({ + "artifact": path.name, + "sha256": sha256.hexdigest(), + "architecture": "x86_64", + "firmware": "UEFI", + "source_image": image, + "source_digest": digest, + "embedded_payload": payload, + "live_marker": "UTAH_LIVE_READY", + "installed_marker": "UTAH_INSTALLED_GRAPHICAL_OK", + "network": { + "live_control": "qemu-user-restrict-on", + "installed": "none", + }, + }, indent=2) + "\n") + PY + + - name: Publish production ISO and checksum + if: success() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: utah-iso-x86_64-${{ github.run_id }} + path: | + ${{ github.workspace }}/.ci/utah-iso-output/utah-live.iso + ${{ github.workspace }}/.ci/utah-iso-output/utah-live.iso.sha256 + ${{ github.workspace }}/.ci/utah-iso-output/candidate.txt + ${{ github.workspace }}/.ci/utah-iso-output/el-torito.txt + ${{ github.workspace }}/.ci/utah-iso-output/manifest.json + ${{ github.workspace }}/.ci/utah-iso-output/e2e/live-serial.log + ${{ github.workspace }}/.ci/utah-iso-output/e2e/installed-serial.log + ${{ github.workspace }}/.ci/utah-iso-output/e2e/e2e-summary.txt + ${{ github.workspace }}/.ci/utah-iso-output/e2e/screenshots/ + if-no-files-found: error + retention-days: 30 + + - name: Retain failed ISO diagnostics + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: utah-iso-failure-${{ github.run_id }} + # Logs, screenshots and the summary only -- not the qcow2 install + # disk or OVMF pflash vars. Those are multi-GB, add nothing a + # serial log/screenshot doesn't already show for triage, and + # there's no reason to publish a full guest filesystem image + # (test creds, host keys, etc.) as a public CI artifact. + path: | + ${{ github.workspace }}/.ci/utah-iso-output/candidate.txt + ${{ github.workspace }}/.ci/utah-iso-output/el-torito.txt + ${{ github.workspace }}/.ci/utah-iso-output/manifest.json + ${{ github.workspace }}/.ci/utah-iso-output/e2e/live-serial.log + ${{ github.workspace }}/.ci/utah-iso-output/e2e/installed-serial.log + ${{ github.workspace }}/.ci/utah-iso-output/e2e/e2e-summary.txt + ${{ github.workspace }}/.ci/utah-iso-output/e2e/screenshots/ + if-no-files-found: warn + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index 92965df25..f6c76ac3b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,8 +1,9 @@ # Utah — Agent Operating Contract Utah composes [Bluefin](https://projectbluefin.io) on the Fedora Hummingbird -`bootc-os` base. Experimental pre-alpha: it has a build, not users. Nothing is -published — no image, no ISO artifact, no installer. +`bootc-os` base. Experimental pre-alpha: it has a build, not users. No stable +image or ISO is released yet; testing ISO artifacts are produced only by the +gated CI path. ## Read order diff --git a/Justfile b/Justfile index 6706129fc..f77b3ab80 100644 --- a/Justfile +++ b/Justfile @@ -43,14 +43,19 @@ check: test -f iso/live/src/etc/bootc-installer/images.json test -f iso/live/src/etc/bootc-installer/recipe.json test -f iso/scripts/build-iso.sh + test -f iso/scripts/offline-e2e.sh bash -n iso/live/src/configure-live.sh bash -n iso/live/src/install-flatpaks.sh bash -n iso/scripts/build-iso.sh + bash -n iso/scripts/offline-e2e.sh python3 -m json.tool iso/live/src/etc/bootc-installer/images.json >/dev/null python3 -m json.tool iso/live/src/etc/bootc-installer/recipe.json >/dev/null grep -q 'org.bootcinstaller.Installer' iso/live/src/install-flatpaks.sh grep -q 'containers-storage' iso/scripts/build-iso.sh grep -q 'UTAH_LIVE' iso/scripts/build-iso.sh + grep -q 'UTAH_LIVE_READY' iso/scripts/offline-e2e.sh + grep -q 'restrict=on' iso/scripts/offline-e2e.sh + grep -q -- '-net none' iso/scripts/offline-e2e.sh grep -q 'ENABLE_SSHD' Containerfile grep -q 'ENABLE_SSHD="${ENABLE_SSHD:-0}"' Justfile grep -q 'ARG PACKAGE_IMAGE_SHA=' Containerfile @@ -338,9 +343,9 @@ generate-bootable-image stream="testing": sync echo "Bootable disk ready: $disk" -# Build a single-architecture UEFI live ISO. This first slice proves the -# Utah live boot path; installer payload integration is intentionally the next -# ISO milestone. +# Build a single-architecture UEFI live ISO with the installer and an embedded +# OCI payload. Use `iso-e2e` below to exercise the same offline install path CI +# uses. iso stream="testing" debug="0": #!/usr/bin/env bash set -euo pipefail @@ -349,6 +354,12 @@ iso stream="testing" debug="0": mkdir -p "{{ base_dir }}" bash iso/scripts/build-iso.sh "$ref" "$(realpath "{{ base_dir }}")/utah-live.iso" "Utah Live" "{{ debug }}" "ghcr.io/{{ repo_organization }}/{{ image }}:{{ stream }}" +# Run the CI-shaped live boot, offline Fisherman install, and installed GNOME +# boot test against a locally built ISO. The test leaves its serial logs, +# screenshots, and installed disk under BASE_DIR for diagnosis. +iso-e2e iso_path="output/utah-live.iso" payload_ref="ghcr.io/projectbluefin/utah:testing": + UTAH_E2E_WORK="{{ base_dir }}/utah-iso-e2e" bash iso/scripts/offline-e2e.sh "{{ iso_path }}" "{{ payload_ref }}" + # Boot the live ISO with QEMU-for-Docker and expose its noVNC console. boot-iso: #!/usr/bin/env bash diff --git a/README.md b/README.md index 1ffae678c..515dcfb9a 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,10 @@ Bluefin built on Fedora Hummingbird. The more ... civilized murder machine. ![alt](https://github.com/user-attachments/assets/56428338-54a0-4376-a53b-5f02f8b101a1) **Experimental pre-alpha** — the image builds, boots, and reaches GDM in local -QEMU validation. No image has been published to a registry, there is no -installer payload, and no ISO has been released. Nothing here is ready to run on -a machine you care about. [Filing +QEMU validation. No stable image or ISO has been released. The gated CI path +now builds a candidate ISO, installs its embedded payload offline, and retains +the boot evidence; nothing here is ready to run on a machine you care about. +[Filing issues](https://github.com/projectbluefin/utah/issues) is the whole point. ## What it is @@ -66,9 +67,9 @@ verify step asserts *that file*, so the two cannot disagree. This is the honest list, and it is why the label above says pre-alpha. -- **Nothing is published.** No image, no ISO artifact, no installer. A live - ISO builds locally (`just iso`); installer payload integration is the next - ISO milestone. +- **Nothing is released.** The testing pipeline produces an ISO artifact only + after the exact-image, live-boot, offline-install, and installed-graphical + gates pass. A stable release ISO and public download remain future work. - **The NVIDIA and gaming flavors are unproven.** The OGC kernel compiles with `sched_ext` and `binderfs` genuinely enabled, and the NVIDIA open module compiles for the base kernel. The module against the OGC kernel, the driver diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index 24c8c643f..c01213f65 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -1,7 +1,7 @@ --- name: ci-workflows version: "1.0" -last_updated: "2026-09-05" +last_updated: "2026-09-11" id: ci-workflows one_line_purpose: Navigate Utah's build, promote, and sync workflow topology. entry_point: docs/skills/ci-workflows.md @@ -13,7 +13,7 @@ dependencies: [] tags: [ci, workflows, actions, promotion] description: >- build.yml contract gate, kernel-cache job, main/kernel matrix split, - promote-testing-to-main and sync-main-to-testing, actions@v1 delegation. + post-testing digest gate, ISO build/install/boot, and actions@v1 delegation. Use when changing .github/workflows/ or debugging a red run. metadata: type: reference @@ -21,8 +21,10 @@ metadata: # CI Workflows -Three workflows, all thin callers into `projectbluefin/actions@v1` reusables, -each pinned to a SHA tagged `v1`: +The image workflows are thin callers into `projectbluefin/actions@v1` +reusables, each pinned to a SHA tagged `v1`. The ISO workflow is intentionally +local because its QEMU and Fisherman gates need to retain disk and console +evidence on the same runner: - `.github/workflows/build.yml` -- pull requests, pushes to `testing`, a nightly cron, and manual dispatch. Top-level `permissions: {}`; each job @@ -30,6 +32,10 @@ each pinned to a SHA tagged `v1`: - `.github/workflows/promote-testing-to-main.yml` -- pushes to `testing`, a nightly cron, and manual dispatch. - `.github/workflows/sync-main-to-testing.yml` -- every push to `main`. +- `.github/workflows/iso.yml` -- after a successful `Post-Testing E2E` run on + `testing`, resolves the exact `utah` x86_64 digest, builds the UEFI ISO, + boots the live desktop, installs the embedded OCI payload with Fisherman + under a restricted network, and boots the installed system with no network. CI delegates builds, vulnerability reporting, SBOMs, keyless signatures, provenance, caching, and rechunking to `projectbluefin/actions@v1` (originated @@ -59,6 +65,26 @@ loudly there, because an invalid matrix creates no image job at all and the only symptom is `build_container: failure` from the aggregator (comment, `.github/workflows/build.yml`). +## iso: the offline production gate + +`iso.yml` listens to `Post-Testing E2E`, so ISO composition starts only after +the image promotion/digest gate has succeeded. It finds the matching +`Build Utah` run by commit and downloads the `utah=sha256:...` digest artifact; +the workflow never builds release media from a moving `:testing` tag. + +The build and all gates stay in one job. `iso/scripts/build-iso.sh` creates the +single-architecture UEFI media and embeds a run-specific local payload tag. +`iso/scripts/offline-e2e.sh` then observes `UTAH_LIVE_READY`, captures the live +serial console and framebuffer, invokes Fisherman through +`containers-storage:`, checks the installed root's Utah identity, and boots +the disk without its ISO or a network. A test-only service emits +`UTAH_INSTALLED_GRAPHICAL_OK` only after `graphical.target`, GDM, and GNOME +Shell are all active. + +The ISO, SHA-256 file, and manifest are uploaded only in a success-conditioned +step after every gate. Failure uploads retain the staging directory, including +the live/installed serial logs, screenshots, and failed disk image. + ## kernel_cache: skipped unless needed, skipped when published The kernel cache job runs only when `needs_kernel` is `true` -- while the diff --git a/docs/skills/index.json b/docs/skills/index.json index 9a380aa0a..639460a38 100644 --- a/docs/skills/index.json +++ b/docs/skills/index.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-05", + "generated_at": "2026-09-11", "schema_version": "1.0", "skills": [ { @@ -15,9 +15,9 @@ "actions", "promotion" ], - "description": "build.yml contract gate, kernel-cache job, main/kernel matrix split, promote-testing-to-main and sync-main-to-testing, actions@v1 delegation. Use when changing .github/workflows/ or debugging a red run.", + "description": "build.yml contract gate, kernel-cache job, main/kernel matrix split, post-testing digest gate, ISO build/install/boot, and actions@v1 delegation. Use when changing .github/workflows/ or debugging a red run.", "version": "1.0", - "last_updated": "2026-09-05", + "last_updated": "2026-09-11", "doc_type": "reference" }, { @@ -107,7 +107,7 @@ ], "description": "Local validation loop: build-ghcr, bootc install to-disk, QEMU/noVNC boot, live ISO. Use when validating changes end-to-end or debugging boot, GDM, or live-session failures.", "version": "1.0", - "last_updated": "2026-09-05", + "last_updated": "2026-09-11", "doc_type": "runbook" }, { diff --git a/docs/skills/index.md b/docs/skills/index.md index 2864a0b10..32e7fa052 100644 --- a/docs/skills/index.md +++ b/docs/skills/index.md @@ -3,7 +3,7 @@ This file is a human-readable mirror of `index.json`. Both are generated by `scripts/generate_skill_index.py` — do not hand-edit either file. -Generated: 2026-09-05 · schema 1.0 · 8 skills +Generated: 2026-09-11 · schema 1.0 · 8 skills | id | category | status | one-line purpose | |---|---|---|---| diff --git a/docs/skills/local-testing.md b/docs/skills/local-testing.md index 67755c90f..6cc953b99 100644 --- a/docs/skills/local-testing.md +++ b/docs/skills/local-testing.md @@ -1,7 +1,7 @@ --- name: local-testing version: "1.0" -last_updated: "2026-09-05" +last_updated: "2026-09-11" id: local-testing one_line_purpose: Build, install, and boot Utah locally in a VM or live ISO. entry_point: docs/skills/local-testing.md @@ -106,12 +106,11 @@ The local OCI ref is not available from the guest's localhost registry, so unified-storage service is skipped instead of retrying its registry repull forever. Published images omit that argument and keep the service enabled. -## Live ISO (initial bring-up) +## Live ISO and offline installation -The first ISO slice reuses Utah's own kernel, dracut-live, and GNOME image. -`just iso` builds a single-architecture UEFI live ISO; this first slice proves -the Utah live boot path, and installer payload integration is intentionally -the next ISO milestone (comment above `iso` in `Justfile`): +The ISO reuses Utah's own kernel, dracut-live, and GNOME image, then embeds the +bootc-installer bundle, default Flatpaks, and the selected OCI image in the +live squashfs. `just iso` builds a single-architecture UEFI live ISO: ```bash just iso testing @@ -120,12 +119,32 @@ just iso testing 1 # optional live-session SSH diagnostics (debug=1) ``` The result is `output/utah-live.iso`, assembled with systemd-boot, a -`UTAH_LIVE` dmsquash-live root, and a serial `UTAH_LIVE_READY` marker. It is -intended to prove live desktop boot first; bootc-installer/offline payload -integration is the next ISO milestone. `just boot-iso` boots it with -QEMU-for-Docker and exposes the noVNC console at the printed URL (comment -above `boot-iso` in `Justfile`), with TPM, UEFI, and `-snapshot` so nothing -persists. +`UTAH_LIVE` dmsquash-live root, and a serial `UTAH_LIVE_READY` marker. `just +boot-iso` boots it with QEMU-for-Docker and exposes the noVNC console at the +printed URL (comment above `boot-iso` in `Justfile`), with TPM, UEFI, and +`-snapshot` so nothing persists. + +The CI-shaped installer test requires the debug ISO because it uses the live +SSH control channel. It keeps the test output under `BASE_DIR`: + +```bash +just iso testing 1 +just iso-e2e +``` + +`iso-e2e` gives the live VM a QEMU user network with `restrict=on` and only a +host-forwarded SSH control port. Fisherman must install through the embedded +`containers-storage` reference. The installed disk is then booted with no ISO +and `-net none`; its serial marker is emitted only after Utah identity, +`graphical.target`, GDM, and a GNOME Shell session are all present. Serial +logs, screenshots, and the disposable installed disk remain in +`output/utah-iso-e2e` after either success or failure. + +The automated production path is `.github/workflows/iso.yml`. It starts after +the testing digest gate, resolves the exact x86_64 image digest from the build +artifacts, and uploads the ISO plus SHA-256 manifest only after the same live, +offline-install, and installed-boot gates pass. A failed run uploads the ISO +staging directory, including serial logs, screenshots, and the failed disk. ## Verification @@ -139,6 +158,7 @@ Then the manual runbook, when the change touches the boot path: just build-ghcr utah testing main just generate-bootable-image testing just boot-vm # success: GDM appears and GNOME Shell renders in noVNC -just iso testing +just iso testing 1 +just iso-e2e # success: live boot, offline Fisherman install, GNOME boot just boot-iso # success: live session renders; serial shows UTAH_LIVE_READY ``` diff --git a/iso/live/Containerfile b/iso/live/Containerfile index 0c271b1b2..46bad4438 100644 --- a/iso/live/Containerfile +++ b/iso/live/Containerfile @@ -4,12 +4,14 @@ ARG SOURCE_IMAGE=localhost/utah:testing FROM ${SOURCE_IMAGE} -# TARGET_IMAGE is the published image name recorded in the installer recipe -# and embedded offline store. SOURCE_IMAGE may be localhost for development; -# it must never leak into the released recipe. +# TARGET_IMAGE is the tag imported into the embedded offline store. The +# separately supplied TRACKING_IMAGE is the exact remote ref recorded for +# updates. SOURCE_IMAGE may be localhost for development; it must never leak +# into the released recipe. ARG TARGET_IMAGE=ghcr.io/projectbluefin/utah:testing +ARG TRACKING_IMAGE=ghcr.io/projectbluefin/utah:testing ARG DEBUG=0 -ENV TARGET_IMAGE=${TARGET_IMAGE} DEBUG=${DEBUG} +ENV TARGET_IMAGE=${TARGET_IMAGE} TRACKING_IMAGE=${TRACKING_IMAGE} DEBUG=${DEBUG} # Utah already carries dracut-live; systemd-boot is added only to the live # environment because the installed image uses the Hummingbird bootloader. diff --git a/iso/live/src/configure-live.sh b/iso/live/src/configure-live.sh index bcba740d1..78ab4d1ae 100644 --- a/iso/live/src/configure-live.sh +++ b/iso/live/src/configure-live.sh @@ -5,7 +5,10 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" INSTALLER_APP_ID=org.bootcinstaller.Installer +# TARGET_IMAGE names the OCI object embedded in the live image. The tracking +# reference can remain the exact digest that was validated before composition. TARGET_IMAGE="${TARGET_IMAGE:-ghcr.io/projectbluefin/utah:testing}" +TRACKING_IMAGE="${TRACKING_IMAGE:-${TARGET_IMAGE}}" # Live media must not apply update or unified-storage policy intended for an # installed image. The payload embedded in the squashfs is the install source. @@ -97,7 +100,8 @@ cp "${SCRIPT_DIR}/etc/bootc-installer/images.json" /etc/bootc-installer/images.j python3 - </dev/null || rm -rf "${WORK}" 2>/dev/null || true +} +trap cleanup_work EXIT cd "${ROOT}" echo "Building live environment from ${IMAGE}" podman build --layers \ + --cap-add sys_admin --security-opt label=disable \ --build-arg SOURCE_IMAGE="${IMAGE}" \ --build-arg TARGET_IMAGE="${PUBLISHED_IMAGE}" \ + --build-arg TRACKING_IMAGE="${TRACKING_IMAGE}" \ --build-arg DEBUG="${DEBUG}" \ --tag "${LIVE_IMAGE}" \ --file iso/live/Containerfile iso/live diff --git a/iso/scripts/offline-e2e.sh b/iso/scripts/offline-e2e.sh new file mode 100755 index 000000000..128348ac8 --- /dev/null +++ b/iso/scripts/offline-e2e.sh @@ -0,0 +1,462 @@ +#!/usr/bin/env bash +# Boot a Utah live ISO, install its embedded OCI payload with Fisherman, and +# boot the installed system without the ISO or a network. +# Usage: offline-e2e.sh ISO PAYLOAD_REF [TARGET_IMGREF] +# +# The live VM has a QEMU user-mode network only for the SSH control channel; +# 'restrict=on' prevents the guest from reaching the host or the outside +# network. The installed VM has no network device at all. This is deliberate: +# a successful install must come from the containers-storage payload in the +# ISO, not from a registry pull that happens to be available in CI. +set -euo pipefail + +ISO="${1:?live ISO path is required}" +PAYLOAD_REF="${2:?embedded payload reference is required}" +TARGET_IMGREF="${3:-${PAYLOAD_REF}}" +WORK="${UTAH_E2E_WORK:-/var/tmp/utah-iso-e2e}" +TEST_USER="${UTAH_E2E_USER:-utahtest}" +TEST_PASSWORD="${UTAH_E2E_PASSWORD:-utahtest}" +LIVE_PASSWORD="${UTAH_E2E_LIVE_PASSWORD:-live}" +SSH_PORT="${UTAH_E2E_SSH_PORT:-2222}" +SSH_TIMEOUT="${UTAH_E2E_SSH_TIMEOUT:-900}" +BOOT_TIMEOUT="${UTAH_E2E_BOOT_TIMEOUT:-900}" +QEMU_MEMORY="${UTAH_E2E_MEMORY:-8192}" +QEMU_CPUS="${UTAH_E2E_CPUS:-4}" + +ISO="$(realpath "${ISO}")" +mkdir -p "${WORK}/screenshots" +WORK="$(realpath "${WORK}")" +SHOTS="${WORK}/screenshots" +INSTALL_DISK="${WORK}/installed.qcow2" +LIVE_VARS="${WORK}/ovmf-vars-live.fd" +INSTALLED_VARS="${WORK}/ovmf-vars-installed.fd" +LIVE_MONITOR="${WORK}/live-monitor.sock" +INSTALLED_MONITOR="${WORK}/installed-monitor.sock" +LIVE_SERIAL="${WORK}/live-serial.log" +INSTALLED_SERIAL="${WORK}/installed-serial.log" +LIVE_PID_FILE="${WORK}/live.pid" +INSTALLED_PID_FILE="${WORK}/installed.pid" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +[[ -f "${ISO}" ]] || fail "ISO does not exist: ${ISO}" +[[ "${PAYLOAD_REF}" =~ ^[A-Za-z0-9./_-]+:[A-Za-z0-9._-]+$ ]] || \ + fail "payload reference must be a tag-shaped OCI reference: ${PAYLOAD_REF}" +[[ "${TARGET_IMGREF}" =~ ^[A-Za-z0-9./_-]+(:[A-Za-z0-9._-]+|@sha256:[0-9a-f]{64})$ ]] || \ + fail "tracking image must be a tag or digest OCI reference: ${TARGET_IMGREF}" + +QEMU="${QEMU_BINARY:-$(command -v qemu-system-x86_64 || true)}" +QEMU_IMG="${QEMU_IMG_BINARY:-$(command -v qemu-img || true)}" +[[ -n "${QEMU}" ]] || fail "qemu-system-x86_64 is not installed" +[[ -n "${QEMU_IMG}" ]] || fail "qemu-img is not installed" +command -v sshpass >/dev/null 2>&1 || fail "sshpass is not installed" + +OVMF_CODE="" +for candidate in \ + /usr/share/OVMF/OVMF_CODE_4M.fd \ + /usr/share/OVMF/OVMF_CODE.fd \ + /usr/share/edk2/ovmf/OVMF_CODE.fd \ + /usr/share/edk2/ovmf/OVMF_CODE_4M.fd; do + if [[ -f "${candidate}" ]]; then + OVMF_CODE="${candidate}" + break + fi +done +OVMF_VARS_SOURCE="" +for candidate in \ + /usr/share/OVMF/OVMF_VARS_4M.fd \ + /usr/share/OVMF/OVMF_VARS.fd \ + /usr/share/edk2/ovmf/OVMF_VARS.fd \ + /usr/share/edk2/ovmf/OVMF_VARS_4M.fd; do + if [[ -f "${candidate}" ]]; then + OVMF_VARS_SOURCE="${candidate}" + break + fi +done +[[ -n "${OVMF_CODE}" ]] || fail "OVMF code firmware is not installed" +[[ -n "${OVMF_VARS_SOURCE}" ]] || fail "OVMF variable store is not installed" + +if [[ -r /dev/kvm ]]; then + ACCEL=(-accel kvm) + echo "QEMU acceleration: KVM" +else + ACCEL=(-accel tcg,thread=multi) + echo "QEMU acceleration: TCG (no /dev/kvm)" +fi + +SSH_OPTS=( + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o ConnectTimeout=5 + -o PreferredAuthentications=password + -o PubkeyAuthentication=no + -o PasswordAuthentication=yes + -o ServerAliveInterval=30 + -o ServerAliveCountMax=10 +) + +ssh_live() { + sshpass -p "${LIVE_PASSWORD}" ssh "${SSH_OPTS[@]}" \ + -p "${SSH_PORT}" liveuser@127.0.0.1 "$@" +} + +scp_live() { + sshpass -p "${LIVE_PASSWORD}" scp "${SSH_OPTS[@]}" \ + -P "${SSH_PORT}" "$@" +} + +monitor() { + python3 - "$1" "$2" <<'PY' +import socket +import sys +import time + +path, command = sys.argv[1:] +with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(10) + client.connect(path) + client.sendall(command.encode() + b"\n") + time.sleep(0.4) +PY +} + +capture_screen() { + local label="$1" + local socket_path="$2" + local ppm="${SHOTS}/${label}.ppm" + local png="${SHOTS}/${label}.png" + + [[ -S "${socket_path}" ]] || { + echo "screenshot monitor is unavailable: ${socket_path}" >&2 + return 1 + } + monitor "${socket_path}" "screendump ${ppm}" || { + echo "QEMU screendump failed for ${label}" >&2 + return 1 + } + for _ in $(seq 1 10); do + [[ -s "${ppm}" ]] && break + sleep 1 + done + [[ -s "${ppm}" ]] || { + echo "QEMU produced no framebuffer for ${label}" >&2 + return 1 + } + if command -v ffmpeg >/dev/null 2>&1 && \ + ffmpeg -y -loglevel error -i "${ppm}" "${png}" >/dev/null 2>&1; then + rm -f "${ppm}" + echo "screenshot: ${png}" + else + echo "screenshot: ${ppm}" + fi +} + +kill_qemu() { + local pid="$1" + [[ -n "${pid}" ]] || return 0 + if kill -0 "${pid}" 2>/dev/null; then + kill "${pid}" 2>/dev/null || true + for _ in $(seq 1 20); do + kill -0 "${pid}" 2>/dev/null || return 0 + sleep 1 + done + kill -KILL "${pid}" 2>/dev/null || true + fi +} + +cleanup() { + local status=$? + set +e + # Keep the work directory, especially the failed disk image. A last + # framebuffer capture makes a boot failure diagnosable even when the + # failure happened before the phase-specific capture below. + if [[ "${status}" -ne 0 ]]; then + capture_screen live-failure "${LIVE_MONITOR}" + capture_screen installed-failure "${INSTALLED_MONITOR}" + fi + if [[ -f "${LIVE_PID_FILE}" ]]; then + kill_qemu "$(cat "${LIVE_PID_FILE}")" + fi + if [[ -f "${INSTALLED_PID_FILE}" ]]; then + kill_qemu "$(cat "${INSTALLED_PID_FILE}")" + fi + echo "E2E output retained at ${WORK}" + exit "${status}" +} +trap cleanup EXIT + +rm -f "${INSTALL_DISK}" "${LIVE_VARS}" "${INSTALLED_VARS}" \ + "${LIVE_MONITOR}" "${INSTALLED_MONITOR}" "${LIVE_SERIAL}" \ + "${INSTALLED_SERIAL}" "${LIVE_PID_FILE}" "${INSTALLED_PID_FILE}" +"${QEMU_IMG}" create -f qcow2 "${INSTALL_DISK}" 64G >/dev/null +cp -f "${OVMF_VARS_SOURCE}" "${LIVE_VARS}" + +echo "=== Phase 1/4: boot the Utah live ISO ===" +"${QEMU}" \ + -name utah-live-e2e \ + -machine q35 \ + "${ACCEL[@]}" \ + -cpu max \ + -m "${QEMU_MEMORY}" \ + -smp "${QEMU_CPUS}" \ + -drive "if=pflash,format=raw,readonly=on,file=${OVMF_CODE}" \ + -drive "if=pflash,format=raw,file=${LIVE_VARS}" \ + -drive "if=none,id=iso,file=${ISO},media=cdrom,readonly=on,format=raw" \ + -device virtio-scsi-pci,id=scsi \ + -device scsi-cd,drive=iso \ + -drive "if=none,id=disk,file=${INSTALL_DISK},format=qcow2" \ + -device virtio-blk-pci,drive=disk \ + -netdev "user,id=net0,restrict=on,hostfwd=tcp::${SSH_PORT}-:22" \ + -device virtio-net-pci,netdev=net0 \ + -display none \ + -vnc 127.0.0.1:2 \ + -monitor "unix:${LIVE_MONITOR},server,nowait" \ + -serial "file:${LIVE_SERIAL}" \ + -no-reboot \ + -daemonize \ + -pidfile "${LIVE_PID_FILE}" + +echo "Live VM network: QEMU user mode with restrict=on; only host SSH forwarding is available." +echo "Waiting for the live SSH control channel and UTAH_LIVE_READY marker..." +live_ready=0 +for _ in $(seq 1 "${SSH_TIMEOUT}"); do + if grep -aFq UTAH_LIVE_READY "${LIVE_SERIAL}" 2>/dev/null && \ + ssh_live true >/dev/null 2>&1; then + live_ready=1 + break + fi + sleep 1 +done +(( live_ready == 1 )) || { + tail -120 "${LIVE_SERIAL}" >&2 2>/dev/null || true + fail "live VM did not expose SSH and UTAH_LIVE_READY within ${SSH_TIMEOUT}s" +} + +echo "=== Phase 2/4: prove the live graphical session ===" +live_graphical=0 +for _ in $(seq 1 180); do + if ssh_live 'systemctl is-active graphical.target' 2>/dev/null | grep -qx active && \ + ssh_live 'systemctl is-active gdm.service' 2>/dev/null | grep -qx active && \ + ssh_live 'pgrep -u liveuser -x gnome-shell >/dev/null' 2>/dev/null; then + live_graphical=1 + break + fi + sleep 5 +done +(( live_graphical == 1 )) || fail "live VM did not reach a GNOME graphical session" +echo "Live graphical.target, gdm.service, and gnome-shell are active." +capture_screen live-desktop "${LIVE_MONITOR}" + +echo "=== Phase 3/4: install the embedded payload with no registry access ===" +RECIPE="${WORK}/recipe.json" +python3 - "${RECIPE}" "${PAYLOAD_REF}" "${TARGET_IMGREF}" "${TEST_USER}" "${TEST_PASSWORD}" <<'PY' +import json +import pathlib +import sys + +path, payload, target, user, password = sys.argv[1:] +recipe = { + "disk": "/dev/vda", + "filesystem": "btrfs", + "btrfsSubvolumes": False, + "encryption": {"type": "none"}, + "image": f"containers-storage:{payload}", + "targetImgref": target, + "local_imgref": f"containers-storage:{payload}", + "selinuxDisabled": False, + "unifiedStorage": True, + "composeFsBackend": False, + "bootloader": "grub2", + "hostname": "utah-e2e", + "user": { + "username": user, + "fullname": "Utah E2E", + "password": password, + "groups": ["wheel"], + }, + "flatpaks": [], +} +pathlib.Path(path).write_text(json.dumps(recipe, indent=2) + "\n") +PY +scp_live "${RECIPE}" liveuser@127.0.0.1:/tmp/utah-e2e-recipe.json + +# The isolated network is part of the test contract. Fisherman must resolve +# the image through the embedded containers-storage graphroot; it cannot +# satisfy the install by pulling from the registry. +ssh_live 'sudo /usr/local/bin/fisherman /tmp/utah-e2e-recipe.json' || \ + fail "Fisherman could not install the embedded OCI payload" + +echo "Fisherman completed from containers-storage:${PAYLOAD_REF}; tracking ${TARGET_IMGREF}." +echo "Preparing boot entries and a test-only graphical-session assertion..." +ssh_live "sudo TARGET_USER='${TEST_USER}' bash -s" <<'TARGET_SETUP' +set -euo pipefail + +root_part="$(blkid -t TYPE=btrfs -o device | head -1)" +esp_part="$(blkid -t TYPE=vfat -o device | head -1)" +[[ -n "${root_part}" ]] || { echo "installed btrfs partition not found" >&2; exit 1; } +[[ -n "${esp_part}" ]] || { echo "installed EFI partition not found" >&2; exit 1; } + +root_mount=/mnt/utah-e2e-root +esp_mount=/mnt/utah-e2e-esp +mkdir -p "${root_mount}" "${esp_mount}" +mount "${root_part}" "${root_mount}" +mount "${esp_part}" "${esp_mount}" +cleanup() { + umount "${esp_mount}" 2>/dev/null || true + umount "${root_mount}" 2>/dev/null || true +} +trap cleanup EXIT + +grep -q '^ID=utah$' "${root_mount}/etc/os-release" +grep -q '^IMAGE_ID=utah$' "${root_mount}/etc/os-release" +test -s "${root_mount}/usr/share/utah/contract.txt" + +# The test disk is disposable. Auto-login lets the guest prove that a GNOME +# shell was created for the account Fisherman made, instead of treating a +# graphical target with only a greeter as a complete desktop boot. +mkdir -p "${root_mount}/etc/gdm" +cat >"${root_mount}/etc/gdm/custom.conf" <"${root_mount}/etc/utah-e2e-graphical.sh" <<'EOF' +#!/usr/bin/env bash +set -u +for _ in $(seq 1 120); do + if grep -q '^ID=utah$' /etc/os-release && \ + grep -q '^IMAGE_ID=utah$' /etc/os-release && \ + systemctl is-active --quiet graphical.target && \ + systemctl is-active --quiet gdm.service && \ + pgrep -u "${TARGET_USER}" -x gnome-shell >/dev/null 2>&1; then + echo UTAH_INSTALLED_GRAPHICAL_OK + exit 0 + fi + sleep 2 +done +echo UTAH_INSTALLED_GRAPHICAL_FAIL +exit 1 +EOF +chmod 0755 "${root_mount}/etc/utah-e2e-graphical.sh" +cat >"${root_mount}/etc/systemd/system/utah-e2e-graphical.service" </dev/null +) +for entry in "${entries[@]}"; do + grep -q '^options ' "${entry}" || continue + sed -i 's/ rhgb//g; s/ quiet//g' "${entry}" + grep -q 'console=ttyS0' "${entry}" || \ + sed -i 's|^options .*|& console=ttyS0,115200n8|' "${entry}" + grep -q 'forward_to_console' "${entry}" || \ + sed -i 's|^options .*|& systemd.journald.forward_to_console=yes|' "${entry}" +done + +if [[ -f "${esp_mount}/EFI/fedora/shimx64.efi" ]]; then + mkdir -p "${esp_mount}/EFI/BOOT" + cp -f "${esp_mount}/EFI/fedora/shimx64.efi" "${esp_mount}/EFI/BOOT/BOOTX64.EFI" +fi +if [[ -f "${esp_mount}/EFI/fedora/grubx64.efi" ]]; then + cp -f "${esp_mount}/EFI/fedora/grubx64.efi" "${esp_mount}/EFI/BOOT/grubx64.efi" +fi + +sync +blockdev --flushbufs "${root_part}" 2>/dev/null || true +blockdev --flushbufs "${esp_part}" 2>/dev/null || true +echo "Utah identity and installed graphical assertion staged." +TARGET_SETUP + +echo "Powering off the live VM before booting its installed disk..." +ssh_live 'sudo systemctl poweroff --no-block' >/dev/null 2>&1 || true +live_pid="$(cat "${LIVE_PID_FILE}")" +for _ in $(seq 1 180); do + kill -0 "${live_pid}" 2>/dev/null || break + sleep 1 +done +if kill -0 "${live_pid}" 2>/dev/null; then + echo "Live VM did not power off cleanly; stopping it after disk flush." >&2 + kill_qemu "${live_pid}" +fi +cp -f "${LIVE_VARS}" "${INSTALLED_VARS}" + +echo "=== Phase 4/4: boot the installed Utah system without ISO or network ===" +"${QEMU}" \ + -name utah-installed-e2e \ + -machine q35 \ + "${ACCEL[@]}" \ + -cpu max \ + -m "${QEMU_MEMORY}" \ + -smp "${QEMU_CPUS}" \ + -drive "if=pflash,format=raw,readonly=on,file=${OVMF_CODE}" \ + -drive "if=pflash,format=raw,file=${INSTALLED_VARS}" \ + -drive "if=none,id=disk,file=${INSTALL_DISK},format=qcow2" \ + -device virtio-blk-pci,drive=disk \ + -net none \ + -display none \ + -vnc 127.0.0.1:3 \ + -monitor "unix:${INSTALLED_MONITOR},server,nowait" \ + -serial "file:${INSTALLED_SERIAL}" \ + -no-reboot \ + -daemonize \ + -pidfile "${INSTALLED_PID_FILE}" + +installed_ready=0 +for _ in $(seq 1 "${BOOT_TIMEOUT}"); do + if grep -aFq UTAH_INSTALLED_GRAPHICAL_OK "${INSTALLED_SERIAL}" 2>/dev/null; then + installed_ready=1 + break + fi + if grep -aEiq 'kernel panic|UTAH_INSTALLED_GRAPHICAL_FAIL|emergency mode' \ + "${INSTALLED_SERIAL}" 2>/dev/null; then + break + fi + sleep 1 +done +(( installed_ready == 1 )) || { + echo "--- installed serial log ---" >&2 + tail -200 "${INSTALLED_SERIAL}" >&2 2>/dev/null || true + fail "installed disk did not reach Utah's graphical-session marker" +} +capture_screen installed-desktop "${INSTALLED_MONITOR}" + +cat >"${WORK}/e2e-summary.txt" <