From 4ca27c65b9174693a93b8efaa177f8b27b303833 Mon Sep 17 00:00:00 2001 From: Douglas Baggett Date: Sat, 19 Sep 2026 15:14:24 -0400 Subject: [PATCH 1/4] ci: retain gated production ISO artifacts Compose DEBUG=0 ISO artifacts only after every flavor passes the offline LUKS matrix, retain checksums, and block testing promotion on production ISO composition. Assisted-by: GPT-5.6 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Douglas Baggett --- .github/workflows/post-testing-e2e.yml | 59 +++++++++++++++++++++++++- docs/skills/ci-workflows.md | 16 +++++-- tests/test_iso_ci.py | 19 ++++++++- 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/.github/workflows/post-testing-e2e.yml b/.github/workflows/post-testing-e2e.yml index 206e1a9e5..0a0adb92b 100644 --- a/.github/workflows/post-testing-e2e.yml +++ b/.github/workflows/post-testing-e2e.yml @@ -141,9 +141,64 @@ jobs: path: docs/verification/ # Never upload debug ISOs or test disks: they contain test passwords. - promote-to-testing: + # Test credentials are only present in the DEBUG=1 ISO above. Build this + # artifact after the whole matrix passes, so a retained ISO is production + # media and every configured image has completed its offline-install gate. + production-iso: needs: [resolve, luks] runs-on: ubuntu-26.04 + timeout-minutes: 90 + permissions: + contents: read + packages: read + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.resolve.outputs.matrix) }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + ref: ${{ needs.resolve.outputs.sha }} + persist-credentials: false + - name: Prepare production ISO host + run: | + set -euo pipefail + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc + sudo apt-get update + sudo apt-get install -y podman skopeo uidmap slirp4netns fuse-overlayfs \ + squashfs-tools xorriso dosfstools mtools + mkdir -p output + df -h . + - name: Pull exact published payload + env: + GH_TOKEN: ${{ github.token }} + IMAGE_REF: ${{ matrix.ref }} + run: | + set -euo pipefail + echo "$GH_TOKEN" | podman login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + podman pull "$IMAGE_REF" + podman logout ghcr.io + - name: Compose production ISO and checksum + env: + IMAGE_REF: ${{ matrix.ref }} + UTAH_ISO_WORKDIR: ${{ runner.temp }} + run: | + set -euo pipefail + bash iso/scripts/build-iso.sh "$IMAGE_REF" "$PWD/output/utah-live.iso" \ + "Utah Live" 0 "$IMAGE_REF" + sha256sum output/utah-live.iso > output/utah-live.iso.sha256 + - name: Retain production ISO + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: production-iso-${{ matrix.image }} + retention-days: 30 + if-no-files-found: error + path: | + output/utah-live.iso + output/utah-live.iso.sha256 + + promote-to-testing: + needs: [resolve, luks, production-iso] + runs-on: ubuntu-26.04 permissions: contents: read packages: write @@ -168,7 +223,7 @@ jobs: done < <(jq -r '.include[].ref' <<< "$MATRIX") documentation: - needs: [resolve, luks] + needs: [resolve, luks, production-iso] runs-on: ubuntu-26.04 permissions: contents: write diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index 017b713d9..abbe270a4 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -166,12 +166,22 @@ restricted networking. CI requires KVM, PNG screenshots, and OCR evidence that fastfetch ran in the graphical terminal. Test credentials are confined to the disposable ISO/disk; neither is uploaded or released. +Only after the entire LUKS matrix succeeds, `production-iso` composes a fresh +`DEBUG=0` x86_64 UEFI ISO from each same digest, writes its SHA-256 checksum, +and retains both as a 30-day Actions artifact. It is deliberately an artifact, +not a release: destination, flavor policy, naming, signing, and Secure Boot +are product decisions tracked by #186. Production-ISO composition is a +promotion prerequisite, so a failed production build cannot advance testing +tags. Debug ISOs and guest disks are never uploaded because they contain test +credentials. + Every matrix job preserves build/test logs, serial logs, and screenshots, including on failure. Only passing jobs upload `docs/verification` with the source commit, original build run, E2E run, image digest and ISO checksum. -The promotion job depends on the entire matrix; a superseded testing commit -cannot move tags. Registry tag copies are sequential, not an atomic multi-tag -transaction: a registry failure can interrupt promotion after a partial copy. +The promotion job depends on the LUKS and production-ISO matrices; a +superseded testing commit cannot move tags. Registry tag copies are sequential, +not an atomic multi-tag transaction: a registry failure can interrupt promotion +after a partial copy. A separate least-privilege job proposes the main desktop's screenshots in `automation/iso-verification`, a documentation PR. It updates only the README diff --git a/tests/test_iso_ci.py b/tests/test_iso_ci.py index b0f35a9aa..d84643abe 100644 --- a/tests/test_iso_ci.py +++ b/tests/test_iso_ci.py @@ -161,12 +161,27 @@ def test_readme_update_is_idempotent_and_preserves_other_text(self): self.assertIn("Keep this paragraph.", result) self.assertIn("actions/runs/123", result) - def test_publication_needs_all_luks_jobs_and_debug_images_are_not_uploaded(self): + def test_production_iso_artifacts_follow_the_full_luks_matrix(self): import yaml jobs = yaml.safe_load((ROOT / ".github/workflows/post-testing-e2e.yml").read_text())["jobs"] + production = jobs["production-iso"] + self.assertIn("luks", production["needs"]) + self.assertFalse(production["strategy"]["fail-fast"]) + compose = next(step for step in production["steps"] + if step.get("name") == "Compose production ISO and checksum") + self.assertIn('"Utah Live" 0 "$IMAGE_REF"', compose["run"]) + self.assertIn("sha256sum output/utah-live.iso", compose["run"]) + artifact = next(step for step in production["steps"] + if step.get("name") == "Retain production ISO") + self.assertIn("output/utah-live.iso", artifact["with"]["path"]) + self.assertIn("output/utah-live.iso.sha256", artifact["with"]["path"]) for name in ["promote-to-testing", "documentation"]: - self.assertIn("luks", jobs[name]["needs"]) + self.assertIn("production-iso", jobs[name]["needs"]) self.assertNotIn("if", jobs[name]) + + def test_debug_images_and_test_disks_are_not_uploaded(self): + import yaml + jobs = yaml.safe_load((ROOT / ".github/workflows/post-testing-e2e.yml").read_text())["jobs"] steps = jobs["luks"]["steps"] self.assertFalse(jobs["luks"]["strategy"]["fail-fast"]) test = next(step for step in steps if "Run existing LUKS" in step.get("name", "")) From 659207f1b7a68c1f2e1cc2f55e2b7070e369e8d6 Mon Sep 17 00:00:00 2001 From: Danathar Date: Sun, 20 Sep 2026 22:52:48 +0000 Subject: [PATCH 2/4] ci: generate the retained ISO checksum with a bare filename sha256sum output/utah-live.iso > output/utah-live.iso.sha256 embedded the output/ path prefix inside the checksum file. upload-artifact roots the artifact at output/, so the downloaded artifact holds utah-live.iso and utah-live.iso.sha256 side by side and sha256sum -c utah-live.iso.sha256 fails with 'No such file or directory: output/utah-live.iso'. Generate it with cd output so the recorded filename is bare and verifiable as shipped. Updated the matching assertion in tests/test_iso_ci.py. This environment has no PyYAML (documented pre-existing gap in the PR body), so full-suite execution is deferred to CI; verified by grep that the workflow literal and the test's assertIn string match byte-for-byte. Signed-off-by: Danathar --- .github/workflows/post-testing-e2e.yml | 2 +- tests/test_iso_ci.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/post-testing-e2e.yml b/.github/workflows/post-testing-e2e.yml index 0a0adb92b..0a5a020ad 100644 --- a/.github/workflows/post-testing-e2e.yml +++ b/.github/workflows/post-testing-e2e.yml @@ -185,7 +185,7 @@ jobs: set -euo pipefail bash iso/scripts/build-iso.sh "$IMAGE_REF" "$PWD/output/utah-live.iso" \ "Utah Live" 0 "$IMAGE_REF" - sha256sum output/utah-live.iso > output/utah-live.iso.sha256 + (cd output && sha256sum utah-live.iso > utah-live.iso.sha256) - name: Retain production ISO uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: diff --git a/tests/test_iso_ci.py b/tests/test_iso_ci.py index d84643abe..11a3be0bb 100644 --- a/tests/test_iso_ci.py +++ b/tests/test_iso_ci.py @@ -170,7 +170,7 @@ def test_production_iso_artifacts_follow_the_full_luks_matrix(self): compose = next(step for step in production["steps"] if step.get("name") == "Compose production ISO and checksum") self.assertIn('"Utah Live" 0 "$IMAGE_REF"', compose["run"]) - self.assertIn("sha256sum output/utah-live.iso", compose["run"]) + self.assertIn("cd output && sha256sum utah-live.iso", compose["run"]) artifact = next(step for step in production["steps"] if step.get("name") == "Retain production ISO") self.assertIn("output/utah-live.iso", artifact["with"]["path"]) From 152424389f30e0bf26760a6bc060bef31f05667e Mon Sep 17 00:00:00 2001 From: Danathar Date: Mon, 21 Sep 2026 01:28:29 +0000 Subject: [PATCH 3/4] docs: document that documentation job also gates on production-iso The intent-alignment review noted the PR body scopes the gate to testing-tag promotion, but post-testing-e2e.yml also adds production-iso to the documentation job's needs (asserted intentionally by tests/test_iso_ci.py:167-169). Neither the body nor the doc mentioned this, so a maintainer reading only the doc would be surprised the screenshot-refresh PR now waits on the whole production ISO matrix. --- docs/skills/ci-workflows.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index abbe270a4..8d76672f7 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -184,11 +184,15 @@ not an atomic multi-tag transaction: a registry failure can interrupt promotion after a partial copy. A separate least-privilege job proposes the main desktop's screenshots in -`automation/iso-verification`, a documentation PR. It updates only the README -evidence block and `docs/verification/`, preserving the current README's other -content. The repository must allow Actions to create pull requests; a denied -write fails this job visibly, without deleting test artifacts. It does not -auto-merge the evidence PR or imply a fresh pass for a different commit. +`automation/iso-verification`, a documentation PR. It also depends on +`production-iso`, so a failed production-ISO composition blocks this +screenshot-refresh PR too, not just testing-tag promotion — a LUKS-only +concern in `docs/verification` still has to wait on the whole matrix +composing cleanly. It updates only the README evidence block and +`docs/verification/`, preserving the current README's other content. The +repository must allow Actions to create pull requests; a denied write fails +this job visibly, without deleting test artifacts. It does not auto-merge +the evidence PR or imply a fresh pass for a different commit. For a deliberate rerun, dispatch Post-Testing E2E with `build_run_id` from a successful testing build containing the current harness. Do not pass a PR From 75f6fc266dd8f823b3925994797c09e63f5f6888 Mon Sep 17 00:00:00 2001 From: Danathar Date: Mon, 21 Sep 2026 12:08:39 +0000 Subject: [PATCH 4/4] docs(iso): say what the retained "production" ISO actually is The retained DEBUG=0 ISO boots `enforcing=0 console=ttyS0,115200n8` -- both boot paths in build-iso.sh apply that cmdline for every DEBUG value, because the E2E harness drives the guest over serial and the live root has no policy relabel pass. That was unremarkable while every ISO was disposable. This PR keeps them for 30 days under the name "production-iso-*", which makes the name a claim the artifact does not support. No behaviour change: the cmdline and the retention stay as they are. Whether the live environment ships SELinux-permissive is a product decision, and it belongs to #186 with signing and Secure Boot rather than to a CI change. What was missing was saying so where someone would look -- the cmdline itself, the retain step, and the CI skill doc -- so a green production-iso job is not mistaken for release-ready media. Also records the storage shape the same review raised: multi-GB per flavor per dispatch at 30-day retention, revisit the window before widening the matrix. DEBUG=0 does exclude the test credentials and sshd path (iso/live/src/configure-live.sh), so nothing secret is retained. 241 tests pass, YAML parses, build-iso.sh passes bash -n. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/post-testing-e2e.yml | 9 +++++++++ docs/skills/ci-workflows.md | 19 +++++++++++++++++++ iso/scripts/build-iso.sh | 9 +++++++++ 3 files changed, 37 insertions(+) diff --git a/.github/workflows/post-testing-e2e.yml b/.github/workflows/post-testing-e2e.yml index 0a5a020ad..12a632247 100644 --- a/.github/workflows/post-testing-e2e.yml +++ b/.github/workflows/post-testing-e2e.yml @@ -186,6 +186,15 @@ jobs: bash iso/scripts/build-iso.sh "$IMAGE_REF" "$PWD/output/utah-live.iso" \ "Utah Live" 0 "$IMAGE_REF" (cd output && sha256sum utah-live.iso > utah-live.iso.sha256) + # "production" is the DEBUG=0 build, not shippable media. DEBUG=0 keeps + # the test credentials and sshd path out (iso/live/src/configure-live.sh), + # so nothing secret is retained -- but build-iso.sh applies + # `enforcing=0 console=ttyS0,115200n8` for every DEBUG value, so this + # boots SELinux-permissive. Whether the live environment ships that way + # is a product decision tracked in #186 alongside signing and Secure + # Boot; a green job here is not evidence the media is release-ready. + # Multi-GB per flavor per dispatch at 30 days: revisit the window before + # widening the matrix. - name: Retain production ISO uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index 8d76672f7..f494c4d10 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -175,6 +175,25 @@ promotion prerequisite, so a failed production build cannot advance testing tags. Debug ISOs and guest disks are never uploaded because they contain test credentials. +**"Production" names the `DEBUG=0` build, not shippable media.** `DEBUG=0` +does exclude the test credentials and sshd path (`iso/live/src/configure-live.sh`), +so the retained artifact carries no secrets — but `iso/scripts/build-iso.sh` +hard-codes `enforcing=0 console=ttyS0,115200n8` on the boot entry for every +`DEBUG` value (the documented Issue #22 exception: rootless `podman unshare` +cannot write `security.selinux` xattrs into the squashfs root), so this ISO +boots SELinux-permissive with a serial console. +That was unremarkable while ISOs were disposable; retaining them for +30 days under the name "production" makes it worth stating plainly. Running +the live environment permissive is a product decision, and it belongs to #186 +along with signing and Secure Boot — it must be settled there before any of +these ISOs reach users. Do not treat a green `production-iso` job as evidence +that the media is release-ready. + +Each retained ISO is multi-GB (the size budget in `iso/scripts/build-iso.sh` +is the ceiling, not the measured size), per flavor, per dispatch, at 30-day +retention. That is real Actions storage; if the matrix widens, revisit the +retention window before the flavor count. + Every matrix job preserves build/test logs, serial logs, and screenshots, including on failure. Only passing jobs upload `docs/verification` with the source commit, original build run, E2E run, image digest and ISO checksum. diff --git a/iso/scripts/build-iso.sh b/iso/scripts/build-iso.sh index 06c19e5af..4068e4151 100644 --- a/iso/scripts/build-iso.sh +++ b/iso/scripts/build-iso.sh @@ -145,6 +145,15 @@ mcopy -i "${ESP}" "${INITRD}" ::/images/pxeboot/initrd.img # Documented exception (Issue #22): rootless podman unshare cannot write security.selinux # xattrs into the squashfs root, leaving it unlabeled. enforcing=0 is required for live boot # to avoid systemd/GDM denials until xattr-preserving rootfs assembly is implemented. +# +# This applies for every DEBUG value, which was unremarkable while every ISO +# was disposable. Since post-testing-e2e.yml began retaining the DEBUG=0 build +# for 30 days as "production-iso-*", it is no longer only a test concern: a +# permissive live boot is a product decision, tracked with signing and Secure +# Boot in #186, and must be settled there before any of these ISOs reach +# users. DEBUG=0 does still exclude the test credentials and sshd path +# (iso/live/src/configure-live.sh), so the retained artifact carries no +# secrets; it is simply not release media. cat > "${WORK}/utah-live.conf" <