Skip to content
39 changes: 36 additions & 3 deletions iso/live/src/install-flatpaks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,34 @@
# Flatpak repository is part of the ISO and is available offline to fisherman.
set -euo pipefail

# Flathub pulls are the largest network operation in the whole ISO build --
# Firefox alone is ~200 MB -- and one timed-out object fails the compose, which
# fails the flavor's entire end-to-end run. It is not hypothetical: run
# 35432516418 lost utah-gaming to
#
# Failed to install org.mozilla.firefox: While pulling
# app/org.mozilla.firefox/x86_64/stable from remote flathub: While fetching
# .../88680ed7...commitmeta: [28] Timeout was reached
#
# after 3m44s, with nothing wrong in the image. The curl above already retries
# for the same reason. flatpak resumes a partial pull from the local repository,
# so a retry re-fetches only what is still missing, and every install below
# passes --or-update, which makes a retry a no-op for refs already complete.
retry_flatpak() {
local attempt
for attempt in 1 2 3; do
if flatpak "$@"; then
return 0
fi
echo "flatpak $1 attempt ${attempt} of 3 failed" >&2
if (( attempt < 3 )); then
sleep $(( attempt * 15 ))
fi
done
echo "ERROR: flatpak $1 failed after 3 attempts: $*" >&2
return 1
}

FLATPAK_CACHE=/var/cache/flatpak-dl
INSTALLER_APP_ID=org.bootcinstaller.Installer
INSTALLER_REPO=projectbluefin/bootc-installer
Expand Down Expand Up @@ -69,7 +97,12 @@ ostree init --repo="${local_repo}" --mode=archive-z2
flatpak build-import-bundle "${local_repo}" /tmp/bootc-installer.flatpak
rm -f /tmp/bootc-installer.flatpak
flatpak remote-add --system --no-gpg-verify installer-local "file://${local_repo}"
flatpak install --system --noninteractive installer-local "${INSTALLER_APP_ID}"
# --or-update for the same reason the Flathub installs below carry it: a retry
# must be a no-op for a ref that already completed. Without it, an attempt
# that installed the app but still exited nonzero would make attempts 2 and 3
# fail with "already installed", turning a flaky success into a hard failure.
retry_flatpak install --system --noninteractive --or-update installer-local \
"${INSTALLER_APP_ID}"
flatpak remote-delete --system --force installer-local || true
rm -rf "${local_repo}"

Expand All @@ -94,7 +127,7 @@ if (( ${#apps[@]} == 0 )); then
echo "No flatpaks listed in /tmp/flatpaks-list; the Brewfile conversion is broken" >&2
exit 1
fi
flatpak install --system --noninteractive --no-related --or-update flathub "${apps[@]}"
retry_flatpak install --system --noninteractive --no-related --or-update flathub "${apps[@]}"

# Ghostty, from the TunaOS OCI remote.
#
Expand All @@ -109,7 +142,7 @@ flatpak install --system --noninteractive --no-related --or-update flathub "${ap
# This is Utah's own addition and does not belong in it.
flatpak remote-add --system --if-not-exists tuna-os \
https://tunaos.org/flatpak/tuna-os.flatpakrepo
flatpak install --system --noninteractive --no-related --or-update \
retry_flatpak install --system --noninteractive --no-related --or-update \
tuna-os com.mitchellh.ghostty
flatpak uninstall --system --noninteractive --unused || true

Expand Down
16 changes: 16 additions & 0 deletions scripts/install-ogc-kernel.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,22 @@ DNF="$(command -v dnf5 || command -v dnf)"
# SYSFB_SIMPLEFB makes sysfb hand the EFI/VESA framebuffer to a
# simple-framebuffer device, and DRM_SIMPLEDRM drives it: an unaccelerated but
# real KMS device on any hardware, until a native driver takes over.
#
# The root filesystem, too. Utah installs to btrfs on LUKS, and x86_64
# defconfig has no BTRFS_FS, so the gaming flavors could format a root
# volume and then not mount it. The installer got all the way to
#
# mkfs.btrfs -f -L root /dev/mapper/fisherman-root (ok)
# mount -t btrfs /dev/mapper/fisherman-root /mnt/fisherman-target
# mount: unknown filesystem type 'btrfs'
# fisherman: fatal: mounting root: ... exit status 32
#
# and the ISO test failed at phase 3 on both gaming flavors. It had never
# reached that phase before: the missing KMS device below stopped it at
# phase 2, so this list was written for the live-boot and LUKS paths and
# nobody had yet needed the filesystem the disk is actually made of.
# BTRFS_FS selects its own crc, zlib, lzo, zstd, raid6 and xor helpers, so
# naming it is enough for olddefconfig to pull the rest in.
required_config=(SCHED_CLASS_EXT NTSYNC ANDROID_BINDERFS
OVERLAY_FS SQUASHFS SQUASHFS_ZSTD EROFS_FS BTRFS_FS
BLK_DEV_LOOP ISO9660_FS BLK_DEV_DM DM_SNAPSHOT DM_CRYPT
Expand Down
131 changes: 131 additions & 0 deletions tests/test_iso_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import importlib.util
import json
import os
import re
import subprocess
from pathlib import Path
import tempfile
Expand Down Expand Up @@ -229,6 +230,136 @@ def test_harness_delegates_the_decision_and_prints_the_transcript_on_failure(sel
self.assertIn("last OCR transcript", script)


class FlatpakRetryTests(unittest.TestCase):
"""A Flathub timeout must not fail a whole flavor's end-to-end run.

Run 35432516418 lost utah-gaming to a single timed-out object while pulling
Firefox, three minutes into composing the ISO, with nothing wrong in the
image. The installs are the largest network operation in the build and had
no retry, while the curl beside them has had one all along.
"""

SCRIPT = ROOT / "iso/live/src/install-flatpaks.sh"

def drive(self, stub: str) -> subprocess.CompletedProcess:
"""Run the real retry_flatpak against a stub flatpak, with sleep off."""
harness = f"""
set -uo pipefail
eval "$(sed -n '/^retry_flatpak() {{/,/^}}/p' {self.SCRIPT})"
sleep() {{ :; }}
attempts=0
{stub}
retry_flatpak install org.example.App >/dev/null 2>&1
echo "rc=$? attempts=$attempts"
"""
return subprocess.run(["bash", "-c", harness], capture_output=True, text=True,
cwd=ROOT)

def test_a_transient_failure_is_retried_and_succeeds(self):
result = self.drive('flatpak() { attempts=$((attempts+1)); [ "$attempts" -ge 3 ]; }')
self.assertEqual(result.stdout.strip(), "rc=0 attempts=3", result.stderr)

def test_a_persistent_failure_still_fails_after_three_attempts(self):
# The point is resilience, not swallowing errors: a repository that is
# genuinely gone must still fail the build.
result = self.drive("flatpak() { attempts=$((attempts+1)); return 1; }")
self.assertEqual(result.stdout.strip(), "rc=1 attempts=3", result.stderr)

def test_every_network_install_goes_through_the_retry(self):
script = self.SCRIPT.read_text()
installs = [line for line in script.splitlines()
if line.startswith("flatpak install")
or line.startswith("retry_flatpak install")]
self.assertTrue(installs)
for line in installs:
with self.subTest(line=line):
self.assertTrue(line.startswith("retry_flatpak install"),
f"unretried network install: {line}")

def test_every_retried_install_is_idempotent(self):
# The retry is only safe if re-running it is a no-op for a ref that
# already completed. Without --or-update, an attempt that installed the
# app but still exited nonzero makes the next attempt fail with
# "already installed" -- the retry would turn a flaky success into a
# hard failure, which is the opposite of why it was added.
script = self.SCRIPT.read_text()
calls = re.findall(r"^retry_flatpak install.*?(?=\n\S|\Z)", script,
re.MULTILINE | re.DOTALL)
self.assertTrue(calls)
for call in calls:
with self.subTest(call=call.splitlines()[0]):
self.assertIn("--or-update", call)


class OgcKernelConfigGateTests(unittest.TestCase):
"""The OGC kernel must be rejected if it cannot mount Utah's root filesystem.

Utah installs to btrfs on LUKS, and x86_64_defconfig has no BTRFS_FS -- I
checked upstream's own defconfig, where DM_CRYPT is likewise absent and
VFAT_FS is present. The gaming flavors therefore formatted a root volume and
then failed to mount it, in run 35432516418:

mkfs.btrfs -f -L root /dev/mapper/fisherman-root (ok)
mount -t btrfs /dev/mapper/fisherman-root /mnt/...
mount: unknown filesystem type 'btrfs'

A full kernel build is the only complete proof, and it takes 45 minutes. The
gate itself is a shell function, so its contract can be tested in
milliseconds: it must reject a config missing any required symbol, and name
the one it rejected.
"""

SCRIPT = ROOT / "scripts/install-ogc-kernel.sh"

def gate(self, config_body: str) -> subprocess.CompletedProcess:
"""Run the real required_config/verify_config against a fake .config."""
with tempfile.TemporaryDirectory() as tmp:
config = Path(tmp) / ".config"
config.write_text(config_body)
harness = f"""
set -uo pipefail
eval "$(sed -n '/^required_config=(/,/)$/p' {self.SCRIPT})"
eval "$(sed -n '/^verify_config() {{/,/^}}/p' {self.SCRIPT})"
verify_config "{config}"
"""
return subprocess.run(["bash", "-c", harness],
capture_output=True, text=True)

def required_symbols(self) -> list[str]:
block = self.SCRIPT.read_text().split("required_config=(", 1)[1]
block = block.split(")", 1)[0]
return block.split()

def test_btrfs_is_required(self):
self.assertIn("BTRFS_FS", self.required_symbols())

def test_a_config_missing_btrfs_is_rejected_by_name(self):
body = "".join(f"CONFIG_{s}=y\n" for s in self.required_symbols()
if s != "BTRFS_FS")
result = self.gate(body)
self.assertEqual(result.returncode, 1)
self.assertIn("missing CONFIG_BTRFS_FS", result.stderr)

def test_a_complete_config_passes(self):
body = "".join(f"CONFIG_{s}=y\n" for s in self.required_symbols())
self.assertEqual(self.gate(body).returncode, 0, self.gate(body).stderr)

def test_a_module_satisfies_the_gate_as_well_as_builtin(self):
# BTRFS_FS is enabled with --module, so =m has to count.
body = "".join(f"CONFIG_{s}=m\n" for s in self.required_symbols())
self.assertEqual(self.gate(body).returncode, 0, self.gate(body).stderr)

def test_every_required_symbol_is_actually_checked(self):
# A symbol in the list that verify_config never looks at would be
# documentation pretending to be a gate.
for symbol in self.required_symbols():
with self.subTest(symbol=symbol):
body = "".join(f"CONFIG_{s}=y\n" for s in self.required_symbols()
if s != symbol)
result = self.gate(body)
self.assertEqual(result.returncode, 1, f"{symbol} is not gated")
self.assertIn(f"missing CONFIG_{symbol}", result.stderr)

class ForkPullRequestKernelCacheTests(unittest.TestCase):
"""A fork PR cannot publish the kernel cache, and must not go red for it.

Expand Down