diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aafe8235..2fb67932 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,10 +15,8 @@ concurrency: cancel-in-progress: true jobs: - # Cheap, network-only gate. A contract package that none of Utah's - # repositories provide used to - # surface as an opaque `exit status 71` from the image build; catching it here - # costs seconds instead of the full matrix. + # Resolve the complete transaction against the same pinned inputs as the + # image before starting the flavor matrix, including extension build tools. contract: runs-on: ubuntu-24.04 permissions: @@ -34,8 +32,6 @@ jobs: submodules: recursive - name: Install just uses: extractions/setup-just@f8a3cce218d9f83db3a2ecd90e41ac3de6cdfd9b # v3 - - name: Install zstandard - run: pip install --quiet zstandard - name: Validate manifests run: just check - name: Check Bluefin parity diff --git a/Containerfile b/Containerfile index fa5246d3..64456f10 100644 --- a/Containerfile +++ b/Containerfile @@ -3,7 +3,7 @@ ARG BASE_IMAGE=quay.io/hummingbird-community/bootc-os:latest@sha256:c5539f9ed4d9 # Keep this pin in Utah so an image build is reproducible and can be reviewed # against the exact package set it consumes. ARG PACKAGE_IMAGE=ghcr.io/projectbluefin/utah-packages -ARG PACKAGE_IMAGE_SHA=sha256:2848c60d51fc6d75c3c89b246aad0e5ebf1fe84c5b7696203f02f14727bd158b +ARG PACKAGE_IMAGE_SHA=sha256:ca320b39b5f40bea9516f6f1c11e70d352c35f1c3d109b3aaf0816be007468f7 # CI keeps PACKAGE_IMAGE_SHA pinned. PACKAGE_IMAGE_REF supports a local image # in containers-storage, where no registry digest is available. ARG PACKAGE_IMAGE_REF=${PACKAGE_IMAGE}@${PACKAGE_IMAGE_SHA} @@ -78,7 +78,7 @@ RUN for pair in install-packages.py:utah-install-packages \ configure-branding.sh:utah-configure-branding \ verify-desktop-contract.py:utah-verify-desktop-contract \ verify-gnome-extensions.py:utah-verify-gnome-extensions; do \ - install -m 0755 "/tmp/utah-scripts/${pair%%:*}" "/usr/local/libexec/${pair##*:}" || exit 1; \ + install -Dm 0755 "/tmp/utah-scripts/${pair%%:*}" "/usr/local/libexec/${pair##*:}" || exit 1; \ done && \ cp -a /tmp/utah-common/. / && \ cp -a /tmp/utah-bluefin/. / && \ diff --git a/Justfile b/Justfile index 6706129f..3faf8ad0 100644 --- a/Justfile +++ b/Justfile @@ -66,6 +66,7 @@ check: python3 -m py_compile scripts/check-repo-availability.py python3 scripts/install-packages.py --check packages/bluefin.toml python3 scripts/verify-rpm-contract.py --check packages/bluefin.toml + python3 -m unittest discover -s tests -p 'test_package_resolution.py' grep -qE 'reusable-build\.yml@(v1|[0-9a-f]{40} # v1)$' .github/workflows/build.yml test -f Containerfile.kernel bash -n scripts/install-ogc-kernel.sh @@ -105,7 +106,7 @@ check-desktop-contract image_ref="localhost/utah:testing": # Fail fast when a contract package is in none of the repositories the image # actually enables, instead of discovering it twenty minutes into a build. -# Checks names only -- it does not assert versions. Needs network access. +# Resolves dependencies on the pinned base and package image. Needs podman and network. check-repos: python3 scripts/check-repo-availability.py packages/bluefin.toml packages/utah.toml diff --git a/docs/skills/ci-workflows.md b/docs/skills/ci-workflows.md index 24c8c643..7e0ea07e 100644 --- a/docs/skills/ci-workflows.md +++ b/docs/skills/ci-workflows.md @@ -37,7 +37,7 @@ as a `docs/building.md` design bullet; now lives in this skill). ## contract: the cheap gate -`build.yml` opens with a network-only gate so a contract package that none of +`build.yml` opens with a container-based gate so a contract package that none of Utah's repositories provide fails in seconds instead of surfacing as an opaque `exit status 71` from the image build (comment, `.github/workflows/build.yml`). It runs three checks: @@ -46,8 +46,8 @@ opaque `exit status 71` from the image build (comment, (`scripts/check_workflow_outputs.py`) and the ban on flavor literals in workflows. - `just check-parity` -- `packages/bluefin.toml` against Bluefin's upstream. -- `just check-repos` -- contract package names against the enabled - repositories. +- `just check-repos` -- the complete installation transaction against the + digest-pinned base and package repository, including extension build tools. The same job resolves the flavor set and splits it in two by what each flavor builds on -- `main` on the pristine Hummingbird base, the rest on the diff --git a/docs/skills/containerfile.md b/docs/skills/containerfile.md index 7d1bbe3c..696ce1d9 100644 --- a/docs/skills/containerfile.md +++ b/docs/skills/containerfile.md @@ -116,6 +116,11 @@ because a multi-source COPY cannot rename, and installed by name into `utah-` prefix. 3. Run `just check`. +The destination directory may be absent in the Hummingbird base. Use +`install -Dm 0755` in the loop, retaining `${pair%%:*}` for the source and +`${pair##*:}` for the destination. `${pair##:*}` does not strip the source +name: it installs a filename containing the entire colon-separated pair. + ## Clean and lint share a layer Everything above writes build-time residue that bootc lint rejects: dnf logs diff --git a/docs/skills/package-contract.md b/docs/skills/package-contract.md index d19d53f3..f9aefcda 100644 --- a/docs/skills/package-contract.md +++ b/docs/skills/package-contract.md @@ -103,12 +103,18 @@ about installation. ## Failure semantics -- A contract package missing from every repository the image enables is a - **build failure**: the dnf transaction in `install-packages.py` exits - non-zero and the Containerfile RUN step fails. `just check-repos` exists to - fail fast on that case — names only, no versions — instead of discovering - it twenty minutes into a build (recipe comment, `Justfile`, `check-repos`; - needs network access). +- A contract package or dependency missing from the installation transaction + is a **build failure**. `just check-repos` reads the base and package-image + digests from `Containerfile`, copies the same repository configuration, and + runs `install-packages.py --resolve` inside that base. This includes the + release-specific Bluefin section, GNOME, services, and extension build tools. + It needs Podman and network access. A name lookup on GitHub Pages is not + evidence that the pinned OCI repository is complete or ABI-compatible. + The factory's leading metadata layer is checked against the pinned manifest + and blob hashes, then mounted for resolution without downloading RPMs. The + factory must copy the same repodata into the leading layer and payload layer. + DNF's `--assumeno` may return 1 for a valid declined transaction; the checker + requires a transaction summary and rejects dependency and repository errors. - `[unavailable]` entries still present in the install set are a validation error (`install-packages.py --check`). - Drift in `packages/bluefin.toml` from upstream is a CI failure diff --git a/packages/utah.toml b/packages/utah.toml index dbc79677..b868da4a 100644 --- a/packages/utah.toml +++ b/packages/utah.toml @@ -110,26 +110,25 @@ packages = [ # either a ruby exclusion in the buildroot or dropping the html subpackage. # # firefox -# Bluefin's desktop contract names Firefox, but this factory currently only -# carries mozjs140 (the embeddable JavaScript runtime), not the Firefox -# browser build. Keep this explicit until a Hummingbird-compatible Firefox -# recipe is added. Tracked by #35. -# -# grub2-efi-x64-cdboot -# This boot-media subpackage is not present in either Hummingbird's overlay -# or the Utah package repository. The image uses Hummingbird's bootable base -# and does not build installer media here. Tracked by #35. -# -# pipewire-libs-extra -# Not published by Hummingbird or utah-packages. Bluefin obtains it from -# negativo17's fedora-multimedia repository, which Utah does not enable. -# Tracked by #35. +# Utah installs org.mozilla.firefox as a Flatpak through the desktop +# contract. The RPM browser is deliberately excluded, while mozjs140 is +# retained for GNOME Shell. Tracked by #35 and utah-packages#112. +# +# anaconda-live, slitherer +# Utah's ISO uses bootc-installer. Both alternative installer packages need +# anaconda-webui, which the factory excludes because Cockpit authentication +# needs a working GHmac backend in Hummingbird's GLib. Slitherer also lacks +# its Qt WebEngine dependency closure. Match the factory's installer policy +# until that path is supported; tracked by #83. +# +# grub2-efi-x64-cdboot and pipewire-libs-extra are supplied by the pinned +# September 10 factory candidate and return to the install contract (#35). packages = [ + "anaconda-live", "evolution-ews-core", "firefox", "fish", - "grub2-efi-x64-cdboot", - "pipewire-libs-extra", + "slitherer", "zsh", ] diff --git a/scripts/check-repo-availability.py b/scripts/check-repo-availability.py index fb1b05e0..c3a94ea7 100644 --- a/scripts/check-repo-availability.py +++ b/scripts/check-repo-availability.py @@ -1,124 +1,108 @@ #!/usr/bin/env python3 -"""Check the package contract against the repositories Utah actually installs from. +"""Resolve Utah's real install transaction against its pinned OCI inputs. -Every package Utah claims parity on is looked up in the repodata of the same -repositories the image build enables: Hummingbird's own overlay and the -utah-packages factory. A package that is neither available nor listed under -[unavailable] in packages/utah.toml fails here, in the seconds-long preflight -job, instead of twenty minutes into a container build. - -Checking Rawhide instead would be actively misleading: Rawhide has moved to -OpenSSL 4 while the Hummingbird base pins 3.5.6, so a package being present -in Rawhide says nothing about whether Utah can install it. +A name lookup against Pages cannot verify the digest in Containerfile, library +dependencies, or packages supplied only by the base image's RPM database. +Use the same installer and repository files as the image, with --assumeno. """ from __future__ import annotations import argparse +import hashlib import io +import json import re -import sys -import tomllib +import subprocess +import tarfile +import tempfile +import urllib.parse import urllib.request -import xml.etree.ElementTree as ET from pathlib import Path -# The same repositories the image installs from, in the same precedence order: -# Hummingbird's own overlay plus the utah-packages factory (its Pages mirror, -# since check-repos runs in CI before the OCI repo is copied into the image). -REPOS = { - "public-hummingbird": "https://packages.redhat.com/api/pulp-content/public-hummingbird/x86_64/", - "utah-packages": "https://projectbluefin.github.io/utah-packages/", -} - - -def section(path: Path, name: str) -> list[str]: - data = tomllib.loads(path.read_text()) - return list(data.get(name, {}).get("packages", [])) - - -def fetch(url: str) -> bytes: - with urllib.request.urlopen(url, timeout=120) as response: - return response.read() - - -def repo_package_names(base: str) -> set[str]: - repomd = ET.fromstring(fetch(base + "repodata/repomd.xml")) - ns = {"repo": "http://linux.duke.edu/metadata/repo"} - href = next( - location.get("href") - for data in repomd.findall("repo:data", ns) - if data.get("type") == "primary" - for location in data.findall("repo:location", ns) - ) - raw = fetch(base + href) - if href.endswith(".zst"): - try: - import zstandard - except ModuleNotFoundError: - print( - "ERROR: python3-zstandard is required to read Rawhide repodata", - file=sys.stderr, - ) - raise SystemExit(2) - stream = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(raw)) - else: - import gzip - - stream = gzip.GzipFile(fileobj=io.BytesIO(raw)) - - names = set() - # Stream the ~16 MiB of metadata rather than holding a parse tree for - # 66k packages; only at package scope matters here. - for line in io.TextIOWrapper(stream, encoding="utf-8", errors="replace"): - match = re.match(r"\s*([^<]+)", line) - if match: - names.add(match.group(1)) - return names + +def pinned_inputs(containerfile: Path) -> tuple[str, str]: + args = dict(re.findall(r"^ARG ([A-Z_]+)=(\S+)$", containerfile.read_text(), re.M)) + base = args["BASE_IMAGE"] + packages = f"{args['PACKAGE_IMAGE']}@{args['PACKAGE_IMAGE_SHA']}" + for image in (base, packages): + if not re.fullmatch(r"[a-zA-Z0-9./:_-]+@sha256:[0-9a-f]{64}", image): + raise ValueError(f"Expected a digest-pinned image, got {image!r}") + return base, packages + + +def verified_bytes(raw: bytes, digest: str) -> bytes: + if digest != "sha256:" + hashlib.sha256(raw).hexdigest(): + raise ValueError(f"Registry content does not match {digest}") + return raw + + +def repository_metadata(image: str, destination: Path) -> None: + registry, reference = image.split("/", 1) + if registry != "ghcr.io": + raise ValueError("The factory metadata reader currently supports ghcr.io images") + repository, digest = reference.split("@", 1) + query = urllib.parse.urlencode({"service": registry, "scope": f"repository:{repository}:pull"}) + with urllib.request.urlopen(f"https://{registry}/token?{query}", timeout=120) as response: + token = json.load(response)["token"] + + def fetch(kind: str, digest: str) -> bytes: + request = urllib.request.Request( + f"https://{registry}/v2/{repository}/{kind}/{digest}", + headers={"Authorization": f"Bearer {token}", + "Accept": "application/vnd.oci.image.manifest.v1+json"}, + ) + with urllib.request.urlopen(request, timeout=120) as response: + return verified_bytes(response.read(), digest) + + manifest = json.loads(fetch("manifests", digest)) + # The factory publishes repodata first, separately from the large RPM + # payload. Do not silently fall back to Pages or another image tag. + layer = manifest["layers"][0] + if layer["size"] > 64 * 1024 * 1024: + raise ValueError("Package image lacks a small leading metadata layer; republish with repodata first") + unpack_metadata(fetch("blobs", layer["digest"]), destination) + + +def unpack_metadata(raw: bytes, destination: Path) -> None: + with tarfile.open(fileobj=io.BytesIO(raw), mode="r:gz") as archive: + for entry in archive: + path = Path(entry.name) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"Unsafe metadata path: {entry.name}") + if entry.isdir(): + continue + if not entry.isfile() or path.parts[:2] != ("repository", "repodata"): + raise ValueError(f"Unexpected entry in metadata layer: {entry.name}") + target = destination / Path(*path.parts[1:]) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(archive.extractfile(entry).read()) + if not (destination / "repodata/repomd.xml").is_file(): + raise ValueError("Pinned metadata layer contains no repomd.xml") def main() -> int: - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("manifest", type=Path) parser.add_argument("overlay", type=Path, nargs="?", default=None) + parser.add_argument("--engine", default="podman") args = parser.parse_args() + root = Path(__file__).resolve().parent.parent overlay = args.overlay or args.manifest.with_name("utah.toml") - - unavailable = set(section(overlay, "unavailable")) - wanted = sorted( - set(section(args.manifest, "fedora")) - | set(section(overlay, "gnome")) - | set(section(overlay, "build")) - ) - names: set[str] = set() - for label, base in REPOS.items(): - found = repo_package_names(base) - print(f"{label}: {len(found)} binary packages") - names |= found - - missing = [pkg for pkg in wanted if pkg not in names and pkg not in unavailable] - resolved = sorted(pkg for pkg in unavailable if pkg in names) - - for pkg in resolved: - print(f"NOTE: {pkg} is now available and can be removed from [unavailable]") - if missing: - print( - f"ERROR: {len(missing)} contract packages are in none of the configured repositories.", - file=sys.stderr, - ) - print( - "Add each to [unavailable] in packages/utah.toml with a tracking " - "issue, or fix the name:", - file=sys.stderr, - ) - for pkg in missing: - print(f" - {pkg}", file=sys.stderr) - return 1 - print( - f"All {len(wanted) - len(unavailable)} contract packages are available " - f"({len(unavailable)} documented as unavailable)." - ) - return 0 + base, packages = pinned_inputs(root / "Containerfile") + print(f"Checking package repository {packages} on {base}", flush=True) + with tempfile.TemporaryDirectory(prefix="utah-repodata-") as tmp: + repository_metadata(packages, Path(tmp)) + return subprocess.run([ + args.engine, "run", "--rm", "--platform", "linux/amd64", + "-v", f"{tmp}:/etc/utah-packages:ro,Z", + "-v", f"{root / 'packages'}:/etc/yum.repos.d:ro,Z", + "-v", f"{args.manifest.resolve()}:/tmp/bluefin.toml:ro,Z", + "-v", f"{overlay.resolve()}:/tmp/utah.toml:ro,Z", + "-v", f"{root / 'scripts/install-packages.py'}:/tmp/install-packages.py:ro,Z", + base, "python3", "/tmp/install-packages.py", "--resolve", + "/tmp/bluefin.toml", "/tmp/utah.toml", + ], check=False).returncode if __name__ == "__main__": diff --git a/scripts/install-packages.py b/scripts/install-packages.py index dc536b06..09d715ba 100755 --- a/scripts/install-packages.py +++ b/scripts/install-packages.py @@ -9,6 +9,8 @@ from __future__ import annotations import argparse +import os +import re import shutil import subprocess import sys @@ -91,6 +93,8 @@ def installed(packages: list[str]) -> list[str]: def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--check", action="store_true") + parser.add_argument("--resolve", action="store_true", + help="resolve the full transaction without installing packages") parser.add_argument("manifest", type=Path) parser.add_argument( "overlay", type=Path, nargs="?", default=None, @@ -118,6 +122,26 @@ def main() -> int: build_deps = section(overlay, "build") excluded = section(args.manifest, "excluded") + if args.resolve: + result = subprocess.run( + [dnf, "--assumeno", "--disablerepo=*", + *(f"--enablerepo={r}" for r in REPOS), + "-x", "PackageKit*", "install", *packages, *build_deps], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + env={**os.environ, "LC_ALL": "C"}, check=False, + ) + print(result.stdout, end="", flush=True) + # DNF exits nonzero when --assumeno declines a valid transaction. + # A missing package or dependency must never be accepted as that case. + errors = r"No match for argument|nothing provides|conflicting requests|cannot install both|Error:|Failed to" + summary = r"(?m)^Transaction Summary:?\s*$|^Nothing to do\.?\s*$" + if (result.returncode not in (0, 1) + or re.search(errors, result.stdout, re.IGNORECASE) + or not re.search(summary, result.stdout)): + return 1 + print(f"Resolved {len(set(packages + build_deps))} runtime and build packages on the pinned base") + return 0 + # Record exactly what this run resolved, so the contract check asserts the # set that was actually asked for rather than recomputing it and drifting. # It drifted once: install added [fedora_v] and the verifier never diff --git a/tests/test_package_resolution.py b/tests/test_package_resolution.py new file mode 100644 index 00000000..14350d01 --- /dev/null +++ b/tests/test_package_resolution.py @@ -0,0 +1,129 @@ +"""The preflight must exercise the install contract and fail closed.""" + +import importlib.util +import hashlib +import io +import re +from pathlib import Path +import subprocess +import tarfile +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] + + +def load(name): + spec = importlib.util.spec_from_file_location(name, ROOT / "scripts" / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +installer = load("install-packages") +checker = load("check-repo-availability") + + +class PackageResolutionTests(unittest.TestCase): + def test_metadata_digest_is_verified(self): + raw = b"metadata" + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + self.assertEqual(checker.verified_bytes(raw, digest), raw) + with self.assertRaises(ValueError): + checker.verified_bytes(b"changed", digest) + + def metadata_archive(self, name): + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w:gz") as archive: + entry = tarfile.TarInfo(name) + entry.size = len(b"") + archive.addfile(entry, io.BytesIO(b"")) + return stream.getvalue() + + def test_metadata_layer_extracts_only_repository_metadata(self): + with tempfile.TemporaryDirectory() as tmp: + checker.unpack_metadata(self.metadata_archive("repository/repodata/repomd.xml"), Path(tmp)) + self.assertEqual((Path(tmp) / "repodata/repomd.xml").read_bytes(), b"") + + def test_metadata_layer_rejects_paths_outside_repodata(self): + for name in ("../outside", "/absolute", "repository/payload.rpm"): + with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(ValueError): + checker.unpack_metadata(self.metadata_archive(name), Path(tmp)) + + def test_containerfile_installs_scripts_into_absent_destination(self): + text = (ROOT / "Containerfile").read_text() + loop = re.search(r"RUN (for pair in .*?\bdone) &&", text, re.S).group(1) + pairs = re.findall(r"([\w.-]+):(utah-[\w.-]+)", loop) + self.assertTrue(pairs) + with tempfile.TemporaryDirectory() as tmp: + source = Path(tmp) / "sources" + destination = Path(tmp) / "missing" / "libexec" + source.mkdir() + for name, _ in pairs: + (source / name).write_bytes((ROOT / "scripts" / name).read_bytes()) + script = loop.replace("/tmp/utah-scripts", str(source)).replace( + "/usr/local/libexec", str(destination)) + subprocess.run(["bash", "-eu", "-c", script], check=True) + self.assertEqual({p.name for p in destination.iterdir()}, {p[1] for p in pairs}) + for name, installed in pairs: + self.assertEqual((source / name).read_bytes(), (destination / installed).read_bytes()) + + def resolve(self, output, code=1): + with tempfile.TemporaryDirectory() as tmp: + base = Path(tmp) / "bluefin.toml" + overlay = Path(tmp) / "utah.toml" + base.write_text('[fedora]\npackages=["base", "unavailable"]\n' + '[fedora_v44]\npackages=["release-specific"]\n') + overlay.write_text('[gnome]\npackages=["shell"]\n' + '[services]\npackages=["resolver"]\n' + '[build]\npackages=["compiler"]\n' + '[unavailable]\npackages=["unavailable"]\n') + with patch("sys.argv", ["install", "--resolve", str(base), str(overlay)]), \ + patch.object(installer, "fedora_major", return_value="44"), \ + patch.object(installer, "dnf_path", return_value="dnf5"), \ + patch.object(installer.subprocess, "run", return_value= + subprocess.CompletedProcess([], code, stdout=output)) as run: + rc = installer.main() + return rc, run.call_args.args[0] + + def test_valid_declined_transaction_includes_every_install_section(self): + rc, command = self.resolve("Transaction Summary:\nInstall 12 Packages\nOperation aborted.\n") + self.assertEqual(rc, 0) + self.assertEqual(command[command.index("install") + 1:], + ["base", "release-specific", "shell", "resolver", "compiler"]) + self.assertIn("--assumeno", command) + self.assertIn("--disablerepo=*", command) + for repo in installer.REPOS: + self.assertIn(f"--enablerepo={repo}", command) + + def test_resolution_failures_do_not_pass(self): + for output in ("nothing provides libmissing.so.1\n", + "No match for argument: compiler\nTransaction Summary\n", + "Error: Failed to download metadata\n", + "Operation aborted.\n", ""): + with self.subTest(output=output): + self.assertEqual(self.resolve(output)[0], 1) + + def test_unexpected_exit_code_fails_even_with_summary(self): + self.assertEqual(self.resolve("Transaction Summary\n", code=125)[0], 1) + + def test_already_installed_contract_passes(self): + self.assertEqual(self.resolve("Nothing to do.\n", code=0)[0], 0) + + def test_pins_come_from_containerfile(self): + base, packages = checker.pinned_inputs(ROOT / "Containerfile") + self.assertIn("@sha256:", base) + self.assertIn("@sha256:", packages) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "Containerfile" + path.write_text("ARG BASE_IMAGE=example/base:latest\n" + "ARG PACKAGE_IMAGE=example/packages\n" + f"ARG PACKAGE_IMAGE_SHA=sha256:{'1' * 64}\n") + with self.assertRaises(ValueError): + checker.pinned_inputs(path) + + +if __name__ == "__main__": + unittest.main()