From 8db77662c5afa506c942afadf966a70b798dc0c3 Mon Sep 17 00:00:00 2001 From: kylerankin Date: Fri, 18 Sep 2026 23:53:18 +0000 Subject: [PATCH 1/3] fix: stop shipping the 4 GB package repository in every image (#128) The live ISO grew from 7.7G (2026-09-06, last passing run) to 8.6G (2026-09-18) between builds. Root cause: the Containerfile copied the digest-pinned `packages` stage's RPM repository into a layer with `COPY --from=packages /repository /etc/utah-packages`, and nothing ever removed it. That ~4 GB is two thirds of every published image and of every live ISO, whose squashfs holds the whole mounted image. Bind the repository into the two RUN steps that install from it with a BuildKit `RUN --mount=type=bind,from=packages`, so it costs no layer and leaves nothing on disk. Reproducibility still comes from the digest-pinned `packages` stage, the only source the package transaction sees. The flavor step flips `utah-packages.repo` to enabled=0 last, so later dnf calls on the image (the live ISO build's included) do not fail on a file:// baseurl that no longer exists. Add a size budget to iso/scripts/build-iso.sh that fails closed above ISO_MAX_GB (override UTAH_ISO_MAX_GB) once the ISO is written, so drift stops landing silently. Raise N only after #105 (linux-firmware + parity packages) is evaluated. just check asserts the mount and refuses a `COPY --from=packages`. Signed-off-by: kylerankin --- Containerfile | 26 +++++++++++++++++++------- Justfile | 5 ++++- docs/skills/ci-workflows.md | 7 +++++++ docs/skills/containerfile.md | 17 ++++++++++++++--- docs/skills/index.json | 4 ++-- docs/skills/index.md | 2 +- iso/scripts/build-iso.sh | 20 ++++++++++++++++++++ tests/test_iso_ci.py | 25 +++++++++++++++++++++++++ 8 files changed, 92 insertions(+), 14 deletions(-) diff --git a/Containerfile b/Containerfile index a7c4b7f9e..80b0c3107 100644 --- a/Containerfile +++ b/Containerfile @@ -41,10 +41,15 @@ FROM ${BASE_IMAGE} # turn below them. COPY packages/bluefin.toml packages/utah.toml contracts/bluefin-desktop.toml /usr/share/utah/ COPY packages/hummingbird.repo packages/nvidia-container.repo packages/utah-packages.repo /etc/yum.repos.d/ -# The package image is an RPM repository, not a runtime dependency. Its -# contents are intentionally copied into the image so the package transaction -# is reproducible and does not depend on a mutable Pages mirror. -COPY --from=packages /repository /etc/utah-packages +# The package image is an RPM repository, not a runtime dependency. It is +# bind mounted into the two RUN steps that install from it and never copied +# into a layer: a COPY used to put the whole ~4 GB repository at +# /etc/utah-packages, nothing ever removed it, and it was two thirds of every +# published image and of every live ISO, whose squashfs holds the image (#128). +# Reproducibility still comes from the digest-pinned `packages` stage, which is +# the only source the package transaction can see -- that is what the old +# comment meant by "does not depend on a mutable Pages mirror." The mount is a +# BuildKit RUN --mount, so it costs no layer and leaves nothing on disk. # One layer for all of Utah's scripts. They are staged under /tmp and installed # by name in the RUN below, because a multi-source COPY cannot rename and # every downstream path expects the utah- prefix. @@ -112,7 +117,8 @@ RUN for pair in install-packages.py:utah-install-packages \ # The package lists live in the manifests, not here. When they were spelled # out in this RUN as well, the two copies drifted and the contract check was # asserting a different set than the install had asked for. -RUN /usr/local/libexec/utah-install-packages \ +RUN --mount=type=bind,from=packages,source=/repository,target=/etc/utah-packages,ro \ + /usr/local/libexec/utah-install-packages \ /usr/share/utah/bluefin.toml /usr/share/utah/utah.toml && \ IMAGE_FLAVOR=main /usr/local/libexec/utah-verify-rpm-contract \ /usr/share/utah/bluefin.toml /usr/share/utah/utah.toml && \ @@ -180,7 +186,8 @@ RUN mkdir -p /tmp/uupd && \ # Dakota-compatible flavors: OGC is built and asserted before NVIDIA so the # NVIDIA path can bind its module to the exact kernel tree it will boot. -RUN case "${IMAGE_FLAVOR}" in \ +RUN --mount=type=bind,from=packages,source=/repository,target=/etc/utah-packages,ro \ + case "${IMAGE_FLAVOR}" in \ gaming|nvidia-gaming) /usr/local/libexec/utah-install-ogc-kernel ;; \ main|nvidia) ;; \ *) echo "Unknown Utah image flavor: ${IMAGE_FLAVOR}" >&2; exit 2 ;; \ @@ -190,7 +197,12 @@ RUN case "${IMAGE_FLAVOR}" in \ main|gaming) ;; \ esac && \ IMAGE_FLAVOR="${IMAGE_FLAVOR}" /usr/local/libexec/utah-verify-rpm-contract \ - /usr/share/utah/bluefin.toml /usr/share/utah/utah.toml + /usr/share/utah/bluefin.toml /usr/share/utah/utah.toml && \ + # The package repository is now only ever bind mounted, so it is absent from + # the committed image. Flip it disabled here -- the last step that installs + # anything -- so later dnf calls on the image (the live ISO build's included) + # do not fail on a file:// baseurl that no longer exists. + sed -i 's/^enabled=1$/enabled=0/' /etc/yum.repos.d/utah-packages.repo # Everything above writes build-time residue that bootc lint rejects: dnf logs # under /var/log, cockpit and dnf state under /run, and ~45 /var directories diff --git a/Justfile b/Justfile index 28eda9363..bafcb9ceb 100644 --- a/Justfile +++ b/Justfile @@ -64,7 +64,10 @@ check: grep -q 'ENABLE_SSHD="${ENABLE_SSHD:-0}"' Justfile grep -q 'ARG PACKAGE_IMAGE_SHA=' Containerfile grep -q 'ARG PACKAGE_IMAGE_REF=' Containerfile - grep -q 'COPY --from=packages /repository /etc/utah-packages' Containerfile + grep -q -- '--mount=type=bind,from=packages,source=/repository,target=/etc/utah-packages,ro' Containerfile + # The package repository is bind mounted, never committed; a COPY would ship + # the whole ~4 GB RPM repository in every image and of every ISO (#128). + ! grep -q 'COPY --from=packages' Containerfile # Every executable release asset fetched during composition must be pinned # and verified; no build may resolve a mutable latest release. python3 scripts/check-download-integrity.py diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index 000c5add3..764d610e8 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -165,6 +165,13 @@ successful testing build containing the current harness. Do not pass a PR build: PRs do not publish immutable images. This matrix validates emulated UEFI desktop installation, not Secure Boot, TPM unlock, or physical GPUs. +The ISO size is bounded in `iso/scripts/build-iso.sh`: it fails closed above +`ISO_MAX_GB` (override per-run with `UTAH_ISO_MAX_GB`) once the ISO is written, +so a drift like the 7.7G -> 8.6G jump in #128 fails the job instead of landing +silently. The guard lives in the build script, so it holds for every caller +(local `just iso`, the CI LUKS job, and any deliberate rerun), not just one +workflow. + ## Verification ```bash diff --git a/docs/skills/containerfile.md b/docs/skills/containerfile.md index 696ce1d97..bd81f3211 100644 --- a/docs/skills/containerfile.md +++ b/docs/skills/containerfile.md @@ -1,7 +1,7 @@ --- name: containerfile version: "1.0" -last_updated: "2026-09-05" +last_updated: "2026-09-18" id: containerfile one_line_purpose: Edit the Containerfile without regressing layer count or cache hits. entry_point: docs/skills/containerfile.md @@ -46,6 +46,16 @@ In summary: - External executable release assets (such as `uupd`) are pinned by version and verified with explicit sha256 checksums (`UUPD_SHA256`) before extraction. +- The digest-pinned `packages` stage is the package factory's RPM repository, + and it is **bind mounted, never copied**: `RUN --mount=type=bind, + from=packages,source=/repository,target=/etc/utah-packages,ro` feeds it to the + two RUN steps that install from it. A `COPY --from=packages /repository + /etc/utah-packages` committed the whole ~4 GB repository to a layer that + nothing removed -- two thirds of every published image and of every live ISO + (#128). The mount costs no layer. The flavor step flips `utah-packages.repo` + to `enabled=0` last, so later dnf calls on the image (the live ISO build's + included) do not fail on a `file://` baseurl that is no longer present. `just + check` asserts the mount and refuses a `COPY --from=packages`. ## Where the build time goes @@ -138,5 +148,6 @@ just check grep -c '^COPY\|^RUN' Containerfile ``` -The count stays at its current value (13 as of 2026-09-05) unless the change -justifies a new layer against the timings table above. +The count stays at its current value (12 as of 2026-09-18) unless the change +justifies a new layer against the timings table above. Removing the package +repository COPY dropped it from 13 to 12; a later change must earn its layer. diff --git a/docs/skills/index.json b/docs/skills/index.json index 02838e2dc..19fa9876f 100644 --- a/docs/skills/index.json +++ b/docs/skills/index.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-16", + "generated_at": "2026-09-18", "schema_version": "1.0", "skills": [ { @@ -35,7 +35,7 @@ ], "description": "Layer discipline: COPY folding, late per-image ARGs, script staging, clean+lint in one layer. Use when modifying the Containerfile, adding a script, or investigating build time.", "version": "1.0", - "last_updated": "2026-09-05", + "last_updated": "2026-09-18", "doc_type": "policy" }, { diff --git a/docs/skills/index.md b/docs/skills/index.md index 688ac5214..c62b2f2eb 100644 --- a/docs/skills/index.md +++ b/docs/skills/index.md @@ -3,7 +3,7 @@ This file is a human-readable mirror of `index.json`. Both are generated by `scripts/generate_skill_index.py` — do not hand-edit either file. -Generated: 2026-09-16 · schema 1.0 · 8 skills +Generated: 2026-09-18 · schema 1.0 · 8 skills | id | category | status | one-line purpose | |---|---|---|---| diff --git a/iso/scripts/build-iso.sh b/iso/scripts/build-iso.sh index 10f49b3d1..d1a0461a2 100644 --- a/iso/scripts/build-iso.sh +++ b/iso/scripts/build-iso.sh @@ -10,6 +10,17 @@ DEBUG="${4:-0}" # SOURCE_IMAGE may be localhost for development, but the embedded store and # installer recipe use this stable, publishable reference. PUBLISHED_IMAGE="${5:-ghcr.io/projectbluefin/utah:testing}" +# Live-ISO size budget (#128). The ISO embeds the full container store for +# offline install, so image growth shows up doubled on the ISO. The last fully +# passing run (2026-09-06) was 7.7G; the 2026-09-18 run was 8.6G once the +# ~4 GB package repository stopped being removed from the image. The ceiling +# sits between those two: 8 GB is above the last passing run so a clean build +# passes, but below the grown size, so the exact #128 regression fails the job +# instead of landing silently. Raise N only after the real fix -- unmount, +# never COPY, the package repository (#128) -- lands; #105 adds linux-firmware +# and ~30 parity packages on top, so expect to revisit N. +# Override per-run with UTAH_ISO_MAX_GB (GB) without editing this script. +ISO_MAX_GB="${UTAH_ISO_MAX_GB:-8}" LABEL="UTAH_LIVE" ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" mkdir -p "$(dirname "${OUTPUT_ISO}")" @@ -159,4 +170,13 @@ xorriso -as mkisofs -iso-level 3 -r -J --joliet-long -V "${LABEL}" \ --efi-boot EFI/efi.img -efi-boot-part --efi-boot-image \ -o "${OUTPUT_ISO}" "${ISO_ROOT}" echo "ISO ready: ${OUTPUT_ISO} ($(du -sh "${OUTPUT_ISO}" | cut -f1))" +# Budget guard: a silent drift of nearly a gigabyte in twelve days (#128) is +# exactly what this fails closed against. Compare byte counts so the GB ceiling +# is exact regardless of how `du -h` rounds. +iso_max_bytes=$(( ISO_MAX_GB * 1024 * 1024 * 1024 )) +iso_size_bytes=$(du -b "${OUTPUT_ISO}" | cut -f1) +if [ "${iso_size_bytes}" -gt "${iso_max_bytes}" ]; then + echo "ERROR: live ISO ($(du -sh "${OUTPUT_ISO}" | cut -f1)) exceeds ${ISO_MAX_GB} GB budget (#128)" >&2 + exit 1 +fi ASSEMBLY diff --git a/tests/test_iso_ci.py b/tests/test_iso_ci.py index b2fcd0385..fb0b3a8f9 100644 --- a/tests/test_iso_ci.py +++ b/tests/test_iso_ci.py @@ -95,6 +95,31 @@ def test_offline_payload_preserves_manifest_digest(self): self.assertIn('"dir:${PAYLOAD_EXPORT}"', script) self.assertIn('dir:/payload "containers-storage:$1"', script) + def test_iso_budget_guard_fails_closed_above_ceiling(self): + # The budget guard (#128) is the whole point of the size drift this PR + # closes. Extract the real block and run it with du stubbed so we can + # drive both the byte count (-b) and the human size (-sh) without a + # real ISO on disk -- the test exercises the logic, not a copy of it. + script = (ROOT / "iso/scripts/build-iso.sh").read_text() + self.assertIn("ISO_MAX_GB", script) + self.assertIn("UTAH_ISO_MAX_GB", script) + start = script.index("iso_max_bytes=$(( ISO_MAX_GB") + end = script.index("\nfi\n", start) + len("\nfi\n") + guard = script[start:end] + run = ( + "du() { if [ \"$1\" = \"-b\" ]; then echo \"$DU_BYTES\"; " + "else echo \"$DU_HUMAN\"; fi; };\n" + "ISO_MAX_GB=${ISO_MAX_GB:-8}; OUTPUT_ISO=/tmp/utah-fakeiso\n" + + guard + ) + under = {"DU_BYTES": str(7 * 1024 ** 3), "DU_HUMAN": "7.0G"} + result = subprocess.run(["bash", "-eu", "-c", run], capture_output=True, text=True, env=under) + self.assertEqual(result.returncode, 0, result.stderr) + over = {"DU_BYTES": str(8 * 1024 ** 3 + 512 * 1024 ** 2), "DU_HUMAN": "8.5G"} + result = subprocess.run(["bash", "-eu", "-c", run], capture_output=True, text=True, env=over) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertIn("exceeds 8 GB budget", result.stderr) + def test_build_explicitly_dispatches_iso_after_both_image_jobs(self): import yaml build = yaml.safe_load((ROOT / ".github/workflows/build.yml").read_text()) From d43e4e1ac16c750ab228d31187b7f9b4bb9d7c88 Mon Sep 17 00:00:00 2001 From: kylerankin Date: Sat, 19 Sep 2026 01:22:36 +0000 Subject: [PATCH 2/3] fix(iso): pass ISO_MAX_GB into the assembly guard heredoc The budget guard (#128) lived inside the <<'ASSEMBLY' quoted heredoc, which runs under 'podman unshare bash -s' in a separate set -u shell that only received the seven positional args. ISO_MAX_GB was a plain, unexported assignment in the outer script, so the guard died with 'ISO_MAX_GB: unbound variable' on every build -- after the ISO was already written -- and the UTAH_ISO_MAX_GB override never reached it either. Pass ISO_MAX_GB as an eighth positional arg alongside the others. Update the budget guard test, which injected ISO_MAX_GB itself and thus masked exactly this scoping failure, to stop injecting it and to assert the variable is actually delivered into the heredoc arg list. Fixes review feedback (#156). --- iso/scripts/build-iso.sh | 3 ++- tests/test_iso_ci.py | 15 ++++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/iso/scripts/build-iso.sh b/iso/scripts/build-iso.sh index d1a0461a2..b1da18c93 100644 --- a/iso/scripts/build-iso.sh +++ b/iso/scripts/build-iso.sh @@ -57,7 +57,7 @@ podman build --layers \ # Image mounts live in rootless Podman's user namespace. Keep the complete # mount/copy/assembly operation inside podman unshare rather than leaking a # namespace-private mount path back to the host shell. -podman unshare bash -s -- "${LIVE_IMAGE}" "${IMAGE}" "${PUBLISHED_IMAGE}" "${OUTPUT_ISO}" "${TITLE}" "${LABEL}" "${WORK}" <<'ASSEMBLY' +podman unshare bash -s -- "${LIVE_IMAGE}" "${IMAGE}" "${PUBLISHED_IMAGE}" "${OUTPUT_ISO}" "${TITLE}" "${LABEL}" "${WORK}" "${ISO_MAX_GB}" <<'ASSEMBLY' set -euo pipefail LIVE_IMAGE="$1" PAYLOAD_IMAGE="$2" @@ -66,6 +66,7 @@ OUTPUT_ISO="$4" TITLE="$5" LABEL="$6" WORK="$7" +ISO_MAX_GB="$8" MOUNT="$(podman image mount "${LIVE_IMAGE}")" cleanup() { set +e diff --git a/tests/test_iso_ci.py b/tests/test_iso_ci.py index fb0b3a8f9..b5e64608a 100644 --- a/tests/test_iso_ci.py +++ b/tests/test_iso_ci.py @@ -101,21 +101,26 @@ def test_iso_budget_guard_fails_closed_above_ceiling(self): # drive both the byte count (-b) and the human size (-sh) without a # real ISO on disk -- the test exercises the logic, not a copy of it. script = (ROOT / "iso/scripts/build-iso.sh").read_text() - self.assertIn("ISO_MAX_GB", script) - self.assertIn("UTAH_ISO_MAX_GB", script) + # The guard must receive ISO_MAX_GB the way the real script delivers it: + # as a positional arg into the <<'ASSEMBLY' heredoc, not from the outer + # shell's environment. Assert that plumbing exists so a regression back + # to an unexported, unpassed variable (which dies under set -u inside + # the assembly) is caught here before it breaks every ISO build. + self.assertRegex(script, r"podman unshare bash -s -- .*\$\{ISO_MAX_GB\}") + self.assertIn('ISO_MAX_GB="$8"', script) start = script.index("iso_max_bytes=$(( ISO_MAX_GB") end = script.index("\nfi\n", start) + len("\nfi\n") guard = script[start:end] run = ( "du() { if [ \"$1\" = \"-b\" ]; then echo \"$DU_BYTES\"; " "else echo \"$DU_HUMAN\"; fi; };\n" - "ISO_MAX_GB=${ISO_MAX_GB:-8}; OUTPUT_ISO=/tmp/utah-fakeiso\n" + "OUTPUT_ISO=/tmp/utah-fakeiso\n" + guard ) - under = {"DU_BYTES": str(7 * 1024 ** 3), "DU_HUMAN": "7.0G"} + under = {"ISO_MAX_GB": "8", "DU_BYTES": str(7 * 1024 ** 3), "DU_HUMAN": "7.0G"} result = subprocess.run(["bash", "-eu", "-c", run], capture_output=True, text=True, env=under) self.assertEqual(result.returncode, 0, result.stderr) - over = {"DU_BYTES": str(8 * 1024 ** 3 + 512 * 1024 ** 2), "DU_HUMAN": "8.5G"} + over = {"ISO_MAX_GB": "8", "DU_BYTES": str(8 * 1024 ** 3 + 512 * 1024 ** 2), "DU_HUMAN": "8.5G"} result = subprocess.run(["bash", "-eu", "-c", run], capture_output=True, text=True, env=over) self.assertNotEqual(result.returncode, 0, result.stdout) self.assertIn("exceeds 8 GB budget", result.stderr) From 37e03e29f43197c83cca7cf4684934d195642520 Mon Sep 17 00:00:00 2001 From: kylerankin Date: Sat, 19 Sep 2026 03:22:41 +0000 Subject: [PATCH 3/3] test(Containerfile): fail fast on silent sed no-op; give budget test a PATH - Containerfile: assert enabled=0 actually took after the sed so a formatting change can't silently leave the repo enabled and break a later dnf call on a missing file:// baseurl. - test_iso_ci: pass PATH into the budget-guard harness so it no longer relies on bash's unset-PATH fallback for the cut pipe. --- Containerfile | 3 ++- tests/test_iso_ci.py | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Containerfile b/Containerfile index 80b0c3107..52edb70a5 100644 --- a/Containerfile +++ b/Containerfile @@ -202,7 +202,8 @@ RUN --mount=type=bind,from=packages,source=/repository,target=/etc/utah-packages # the committed image. Flip it disabled here -- the last step that installs # anything -- so later dnf calls on the image (the live ISO build's included) # do not fail on a file:// baseurl that no longer exists. - sed -i 's/^enabled=1$/enabled=0/' /etc/yum.repos.d/utah-packages.repo + sed -i 's/^enabled=1$/enabled=0/' /etc/yum.repos.d/utah-packages.repo \ + && grep -q '^enabled=0$' /etc/yum.repos.d/utah-packages.repo # Everything above writes build-time residue that bootc lint rejects: dnf logs # under /var/log, cockpit and dnf state under /run, and ~45 /var directories diff --git a/tests/test_iso_ci.py b/tests/test_iso_ci.py index b5e64608a..4dcc04612 100644 --- a/tests/test_iso_ci.py +++ b/tests/test_iso_ci.py @@ -1,6 +1,7 @@ """CI must test the complete exact-digest set before publication.""" import importlib.util import json +import os import subprocess from pathlib import Path import tempfile @@ -117,10 +118,12 @@ def test_iso_budget_guard_fails_closed_above_ceiling(self): "OUTPUT_ISO=/tmp/utah-fakeiso\n" + guard ) - under = {"ISO_MAX_GB": "8", "DU_BYTES": str(7 * 1024 ** 3), "DU_HUMAN": "7.0G"} + under = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "ISO_MAX_GB": "8", "DU_BYTES": str(7 * 1024 ** 3), "DU_HUMAN": "7.0G"} result = subprocess.run(["bash", "-eu", "-c", run], capture_output=True, text=True, env=under) self.assertEqual(result.returncode, 0, result.stderr) - over = {"ISO_MAX_GB": "8", "DU_BYTES": str(8 * 1024 ** 3 + 512 * 1024 ** 2), "DU_HUMAN": "8.5G"} + over = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "ISO_MAX_GB": "8", "DU_BYTES": str(8 * 1024 ** 3 + 512 * 1024 ** 2), "DU_HUMAN": "8.5G"} result = subprocess.run(["bash", "-eu", "-c", run], capture_output=True, text=True, env=over) self.assertNotEqual(result.returncode, 0, result.stdout) self.assertIn("exceeds 8 GB budget", result.stderr)