diff --git a/AGENTS.md b/AGENTS.md index 92965df25..2a22420fd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,10 +38,9 @@ checks (`scripts/check-skill-frontmatter.sh`, `scripts/check-skill-index.sh`, ## Invariants — do not break -- **`packages/bluefin.toml` is a byte-for-byte copy** of - `projectbluefin/bluefin` `build_files/packages/base.toml`. Never hand-edit - it; sync it verbatim from upstream. CI diffs it on every run - (`just check-parity`). +- **`packages/bluefin.toml` is derived from Bluefin's effective package payload** + (base, version-specific, external, and multimedia transactions). + CI verifies it against upstream on every run (`just check-parity`). - **Utah's own package changes live in `packages/utah.toml`** (`[gnome]`, `[build]`, `[unavailable]`). Every `[unavailable]` entry MUST carry a tracking issue. A missing contract package is a build failure, never a diff --git a/Justfile b/Justfile index 6706129fc..ba195b2be 100644 --- a/Justfile +++ b/Justfile @@ -64,6 +64,7 @@ check: python3 -m py_compile scripts/install-packages.py python3 -m py_compile scripts/verify-rpm-contract.py python3 -m py_compile scripts/check-repo-availability.py + python3 -m py_compile scripts/check-parity.py python3 scripts/install-packages.py --check packages/bluefin.toml python3 scripts/verify-rpm-contract.py --check packages/bluefin.toml grep -qE 'reusable-build\.yml@(v1|[0-9a-f]{40} # v1)$' .github/workflows/build.yml @@ -109,21 +110,11 @@ check-desktop-contract image_ref="localhost/utah:testing": check-repos: python3 scripts/check-repo-availability.py packages/bluefin.toml packages/utah.toml -# packages/bluefin.toml is a verbatim copy of Bluefin's base.toml. Drift here -# is a parity bug, so make it loud rather than letting it accumulate quietly. +# packages/bluefin.toml is derived from Bluefin's effective package payload +# (base, version-specific, external, and multimedia transactions). +# Drift here is a parity bug, so make it loud rather than letting it accumulate quietly. check-parity: - #!/usr/bin/env bash - set -euo pipefail - upstream=$(mktemp) - trap 'rm -f "$upstream"' EXIT - curl -fsSL -o "$upstream" \ - https://raw.githubusercontent.com/projectbluefin/bluefin/main/build_files/packages/base.toml - if diff -u "$upstream" packages/bluefin.toml; then - echo "packages/bluefin.toml matches projectbluefin/bluefin" - else - echo "packages/bluefin.toml has drifted from projectbluefin/bluefin" >&2 - exit 1 - fi + python3 scripts/check-parity.py image_name base_name stream flavor: #!/usr/bin/env bash diff --git a/README.md b/README.md index 1ffae678c..0d68c6128 100644 --- a/README.md +++ b/README.md @@ -47,15 +47,17 @@ is built to produce, not something you can pull today. ## Package parity with Bluefin -`packages/bluefin.toml` is a byte-for-byte copy of Bluefin's `base.toml`, and CI -diffs it against upstream on every run, so drift fails the build rather than -being noticed later. +`packages/bluefin.toml` is a machine-readable contract derived from Bluefin's +effective package payload (covering base Fedora, version-specific, external, +and multimedia transactions). CI derives and verifies it against upstream on +every run (`just check-parity`), so additions or removals fail the build rather +than accumulating quietly. | | count | | --- | --- | -| Bluefin contract installed | **61** | +| Bluefin contract installed | **64** | | Utah additions (GNOME 51, desktop services) | 12 | -| Genuinely unavailable | **4** | +| Documented unavailable exceptions | **28** | The install writes its resolved list to `/usr/share/utah/contract.txt` and the verify step asserts *that file*, so the two cannot disagree. diff --git a/docs/skills/index.json b/docs/skills/index.json index 9a380aa0a..3f383ba4f 100644 --- a/docs/skills/index.json +++ b/docs/skills/index.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-09-05", + "generated_at": "2026-09-11", "schema_version": "1.0", "skills": [ { @@ -123,9 +123,9 @@ "bluefin", "contracts" ], - "description": "Bluefin parity contract: verbatim bluefin.toml, utah.toml overlay, [unavailable] rules, repository policy. Use when adding, removing, or debugging packages or parity/check-repos failures.", - "version": "1.0", - "last_updated": "2026-09-05", + "description": "Bluefin parity contract: machine-readable effective payload bluefin.toml, utah.toml overlay, [unavailable] rules, repository policy. Use when adding, removing, or debugging packages or parity/check-repos failures.", + "version": "1.1", + "last_updated": "2026-09-11", "doc_type": "policy" }, { diff --git a/docs/skills/index.md b/docs/skills/index.md index 2864a0b10..32e7fa052 100644 --- a/docs/skills/index.md +++ b/docs/skills/index.md @@ -3,7 +3,7 @@ This file is a human-readable mirror of `index.json`. Both are generated by `scripts/generate_skill_index.py` — do not hand-edit either file. -Generated: 2026-09-05 · schema 1.0 · 8 skills +Generated: 2026-09-11 · schema 1.0 · 8 skills | id | category | status | one-line purpose | |---|---|---|---| diff --git a/docs/skills/package-contract.md b/docs/skills/package-contract.md index d19d53f3e..22f9c2634 100644 --- a/docs/skills/package-contract.md +++ b/docs/skills/package-contract.md @@ -1,7 +1,7 @@ --- name: package-contract -version: "1.0" -last_updated: "2026-09-05" +version: "1.1" +last_updated: "2026-09-11" id: package-contract one_line_purpose: Maintain Bluefin package parity and Utah's overlay manifest. entry_point: docs/skills/package-contract.md @@ -12,9 +12,9 @@ status: active dependencies: [] tags: [packages, parity, bluefin, contracts] description: >- - Bluefin parity contract: verbatim bluefin.toml, utah.toml overlay, - [unavailable] rules, repository policy. Use when adding, removing, or - debugging packages or parity/check-repos failures. + Bluefin parity contract: machine-readable effective payload bluefin.toml, + utah.toml overlay, [unavailable] rules, repository policy. Use when adding, + removing, or debugging packages or parity/check-repos failures. metadata: type: policy --- @@ -27,12 +27,14 @@ policy for changing them. ## The two manifests -- **`packages/bluefin.toml`** — the parity contract. It is a byte-for-byte - copy of projectbluefin/bluefin's `build_files/packages/base.toml` and must - stay that way. **Never hand-edit it.** Sync it verbatim from upstream; any - drift is a parity bug. `just check-parity` diffs it against upstream on - every CI run so drift fails the build rather than accumulating quietly - (recipe comment: `Justfile`, `check-parity`). +- **`packages/bluefin.toml`** — the parity contract. It is a machine-readable + contract derived from the package transactions that Bluefin actually executes: + base Fedora packages, version-specific additions, external/COPR transactions + (such as `tailscale` and `uupd`), and multimedia transactions (including + overrides and codecs). **Never hand-edit it without validation.** + `just check-parity` runs `scripts/check-parity.py` on every CI run, deriving + the effective payload from upstream Bluefin's `build_files/packages/base.toml` + and `build_files/base/03-packages.sh` to detect additions and removals. - **`packages/utah.toml`** — Utah's overlay. Everything Utah needs *in addition to* or *instead of* the contract lives here. The full rules are in the header comment of that file (cite it; do not move or copy it): @@ -40,7 +42,8 @@ policy for changing them. - `[gnome]` — GNOME 51 desktop contract Hummingbird does not ship. - `[build]` — toolchain needed to build the pinned GNOME extensions (`scripts/build-gnome-extensions.sh`). - - `[services]` — desktop services Bluefin adds on top of the server base. + - `[services]` — desktop services Hummingbird's server base does not install + (`fwupd`, `systemd-resolved`). - `[unavailable]` — Bluefin contract packages none of Utah's repositories provide. @@ -51,23 +54,19 @@ policy for changing them. a dumping ground for packages that are merely inconvenient (header comment, `packages/utah.toml`). -## multimedia_overrides are not missing packages +## Multimedia transactions and overrides -Bluefin's `[multimedia_overrides]` (twelve names: mesa-libGL, -mesa-vulkan-drivers, libva, intel-mediasdk, libheif and friends) are **not** -extra packages. They are the same names Fedora already ships, which Bluefin -*replaces* with negativo17 builds by enabling `fedora-multimedia`. Utah does -not enable that repository, so it installs Fedora's builds instead. Nothing -is absent from the image; what differs is which build it carries, and the -practical consequence is hardware-accelerated codec support. +Bluefin executes an extensive multimedia transaction combining negativo17's +`fedora-multimedia` repository overrides (`mesa`, `libva`, etc.) and codecs +(`ffmpeg`, `gstreamer1-plugins-*`, `lame`, `libfdk-aac`, `libjxl`). Utah does +not enable negativo17 or Fedora runtime repositories. -That is why they are absent from the contract rather than listed under -`[unavailable]`: recording them as missing would be wrong (a source does -provide the name), and recording them as satisfied would hide a real -functional difference. The factory already builds several of them in -projectbluefin/hummingbird-github; when that overlay is published and enabled -here, these can move into the contract as a version assertion rather than a -name one (header comment, `packages/utah.toml`). +Where Hummingbird or `utah-packages` already provides the required build +(`intel-gmmlib`, `intel-mediasdk`, `intel-vpl-gpu-rt`, `libheif`, `libva`), +Utah installs it directly as part of the contract. Packages not yet available +in Utah's enabled repositories are recorded in `[unavailable]` with tracking +issue #12, until the complete multimedia closure from `utah-packages#24` is +published. ## Repository policy @@ -114,9 +113,9 @@ about installation. - Drift in `packages/bluefin.toml` from upstream is a CI failure (`just check-parity`). -Current counts, per the README "Package parity" section: 61 Bluefin contract -packages installed, 12 Utah additions (GNOME 51, desktop services), 4 -genuinely unavailable. +Current counts: 64 Bluefin contract packages installed (58 fedora base + +1 fedora_v44 + 5 multimedia overrides), 12 Utah additions (10 GNOME 51, +2 desktop services), 28 documented as unavailable. ## Verification diff --git a/packages/bluefin.toml b/packages/bluefin.toml index 8d5ef38f5..e5dcb501a 100644 --- a/packages/bluefin.toml +++ b/packages/bluefin.toml @@ -1,34 +1,21 @@ -# Package manifest for the bluefin base image. -# Read by build_files/base/03-packages.sh via build_files/shared/read-packages. +# Machine-readable contract derived from Bluefin's effective package payload. +# +# Sources in projectbluefin/bluefin: +# - build_files/packages/base.toml (fedora, version-specific, overrides, excluded) +# - build_files/base/03-packages.sh (external and multimedia transactions) # # Sections: -# [multimedia_overrides] packages synced from fedora-multimedia repo -# [fedora] base Fedora packages, all versions -# [fedora_v42] additions for Fedora 42 only -# [fedora_v43] additions for Fedora 43 only -# [fedora_v44] additions for Fedora 44 only +# [fedora] base Fedora packages common to all supported versions +# [fedora_v*] version-specific Fedora additions +# [external] packages from external repositories/COPRs +# [multimedia] codecs and multimedia packages from 03-packages.sh +# [multimedia_overrides] packages synced from negativo17/fedora-multimedia # [excluded] packages removed from the base image +# +# Validated by scripts/check-parity.py on every CI run (`just check-parity`). -[multimedia_overrides] -# Replace mesa and friends with less-crippled versions from negativo17/fedora-multimedia. -# Versions are pinned with `dnf5 versionlock` after install. -packages = [ - "intel-gmmlib", - "intel-mediasdk", - "intel-vpl-gpu-rt", - "libheif", - "libva", - "libva-intel-media-driver", - "mesa-dri-drivers", - "mesa-filesystem", - "mesa-libEGL", - "mesa-libGL", - "mesa-libgbm", - "mesa-vulkan-drivers", -] - -[fedora] # Base packages from Fedora repos — common to all supported Fedora versions. +[fedora] packages = [ "adw-gtk3-theme", "adwaita-fonts-all", @@ -43,10 +30,10 @@ packages = [ "dracut-live", "evtest", "fastfetch", + "firefox", "firewall-config", "fish", "flatpak-spawn", - "firefox", "fzf", "gcc", "gcc-c++", @@ -95,25 +82,69 @@ packages = [ "zsh", ] +# Fedora 42 additions [fedora_v42] -# TODO: remove when F42 is dropped packages = [ "evolution-ews-core", ] +# Fedora 43 additions [fedora_v43] packages = [ "evolution-ews-core", "gnupg2-scdaemon", ] +# Fedora 44 additions [fedora_v44] packages = [ "gnupg2-scdaemon", ] -[excluded] +# External and COPR packages installed in Bluefin's package transaction. +[external] +packages = [ + "tailscale", + "uupd", +] + +# Codecs and extra multimedia packages installed in Bluefin's transaction. +[multimedia] +packages = [ + "@multimedia", + "ffmpeg", + "ffmpeg-libs", + "ffmpegthumbnailer", + "gstreamer1-plugins-bad-free", + "gstreamer1-plugins-bad-free-libs", + "gstreamer1-plugins-base", + "gstreamer1-plugins-good", + "lame", + "lame-libs", + "libavcodec", + "libfdk-aac", + "libjxl", +] + +# Packages synced from negativo17/fedora-multimedia repo. +[multimedia_overrides] +packages = [ + "intel-gmmlib", + "intel-mediasdk", + "intel-vpl-gpu-rt", + "libheif", + "libva", + "libva-intel-media-driver", + "mesa-dri-drivers", + "mesa-filesystem", + "mesa-libEGL", + "mesa-libGL", + "mesa-libgbm", + "mesa-vulkan-drivers", +] + # Packages removed from the base image — conflicts with or replaced by image content. +[excluded] packages = [ "default-fonts-cjk-sans", "fedora-bookmarks", diff --git a/packages/utah.toml b/packages/utah.toml index dbc796776..6fc73e468 100644 --- a/packages/utah.toml +++ b/packages/utah.toml @@ -1,36 +1,29 @@ # Utah's package overlay on top of the Bluefin package contract. # -# packages/bluefin.toml is a byte-for-byte copy of projectbluefin/bluefin's -# build_files/packages/base.toml and must stay that way -- it is the parity -# contract, and any drift there is a parity bug. Everything Utah needs *in -# addition to* (or *instead of*) that contract lives here. +# packages/bluefin.toml is the machine-readable contract derived from Bluefin's +# effective package payload (base Fedora, version-specific, external, and +# multimedia transactions). Everything Utah needs *in addition to* (or *instead +# of*) that contract lives here. # # Sections: # [gnome] GNOME 51 desktop contract Hummingbird does not ship # [build] toolchain needed to build the pinned GNOME extensions +# [services] desktop services Hummingbird server base does not install # [unavailable] Bluefin contract packages none of Utah's repositories provide # -# On [multimedia_overrides], which Utah deliberately does not install: -# -# Those twelve entries -- mesa-libGL, mesa-vulkan-drivers, libva, -# intel-mediasdk, libheif and friends -- are not extra packages. They are the -# same package names Fedora already ships, which Bluefin *replaces* with -# negativo17 builds by enabling fedora-multimedia. Utah does not enable that -# repository, so it installs Fedora's builds of those names instead. Nothing -# is missing from the image; what differs is which build of them it carries, -# and the practical consequence is hardware-accelerated codec support. -# -# This is why they are absent from the contract rather than listed under -# [unavailable]: [unavailable] means "no source provides this name at all", -# which is false of these. Recording them as missing would be wrong, and -# recording them as satisfied would hide a real functional difference from -# Bluefin. -# -# The factory is already building several of them for Hummingbird -- -# intel-gmmlib, intel-mediasdk, intel-vpl-gpu-rt, libheif and libva all built -# in projectbluefin/hummingbird-github. When that overlay is published and enabled -# here, it becomes the source that negativo17 is for Bluefin, and these can -# move into the contract as a version assertion rather than a name one. +# On multimedia transactions: +# +# Bluefin executes an extensive multimedia transaction combining negativo17's +# fedora-multimedia repository overrides (mesa, libva, etc.) and codecs +# (ffmpeg, gstreamer plugins, lame, fdk-aac, libjxl). Utah does not enable +# negativo17 or Fedora runtime repositories. +# +# Where Hummingbird or utah-packages already provides the required build +# (intel-gmmlib, intel-mediasdk, intel-vpl-gpu-rt, libheif, libva), Utah +# installs it directly as part of the contract. Packages not yet available +# in Utah's enabled repositories are recorded in [unavailable] with tracking +# issue #12, until the complete multimedia closure from utah-packages#24 is +# published. [gnome] # Hummingbird supplies the bootable base but not a desktop. The pinned @@ -69,8 +62,11 @@ packages = [ [services] # Bluefin's desktop image adds these to the server base before 40-services.sh -# enables their units. uupd is downloaded separately because it is published -# as a release binary rather than an RPM in the Hummingbird repositories. +# enables their units. +# +# tailscale is part of Bluefin's external contract (packages/bluefin.toml [external]). +# uupd is downloaded separately because it is published as a release binary +# rather than an RPM in the Hummingbird repositories. # # systemd-resolved is not in the Bluefin contract because Fedora's base image # already carries it. Hummingbird's bootable base does not: the package exists @@ -79,7 +75,6 @@ packages = [ packages = [ "fwupd", "systemd-resolved", - "tailscale", ] [unavailable] @@ -93,7 +88,7 @@ packages = [ # evolution-data-server stack -- camel, evolution-mail, evolution-shell, # libebook, libecal, libedata-book -- which the factory does not yet build. # Disproportionate cost for one Exchange plugin binary; the EDS closure can -# be added to the factory if Exchange support is ever required. +# be added to the factory if Exchange support is ever required. Tracked by #10. # # fish # Fedora's fish spec uses %cargo_generate_buildrequires, which requires @@ -101,13 +96,14 @@ packages = [ # 44 does not ship Rust crate binary packages -- they exist only in # Rawhide -- so the buildroot cannot satisfy them. Building fish needs a # vendored/cargo-fetch rebuild of the spec rather than the system-crate -# model. +# model. Tracked by #35. # # zsh # The HTML docs build target pulls texi2html, which drags Fedora's ruby # (ruby-default-gems) into a buildroot that already carries Hummingbird's # ruby4.0. The two conflict on file paths, so the transaction fails. Needs # either a ruby exclusion in the buildroot or dropping the html subpackage. +# Tracked by #35. # # firefox # Bluefin's desktop contract names Firefox, but this factory currently only @@ -123,13 +119,59 @@ packages = [ # pipewire-libs-extra # Not published by Hummingbird or utah-packages. Bluefin obtains it from # negativo17's fedora-multimedia repository, which Utah does not enable. -# Tracked by #35. +# Tracked by #35 and #12. +# +# uupd +# Bluefin installs uupd via the ublue-os/packages COPR. Utah does not enable +# COPR repositories at runtime; instead, uupd is downloaded as a pinned, +# digest-verified release binary in the Containerfile via +# scripts/check-download-integrity.py. Tracked by #20 and #11. +# +# tailscale +# Bluefin installs Tailscale from pkgs.tailscale.com. In Utah, Tailscale is +# packaged by utah-packages, but the binary RPM is not yet published in the +# repository build due to utah-packages#75. Tracked by #54. +# +# Multimedia packages and overrides: +# @multimedia, ffmpeg, ffmpeg-libs, ffmpegthumbnailer, +# gstreamer1-plugins-bad-free, gstreamer1-plugins-bad-free-libs, +# gstreamer1-plugins-base, gstreamer1-plugins-good, lame, lame-libs, +# libavcodec, libfdk-aac, libjxl, libva-intel-media-driver, mesa-dri-drivers, +# mesa-filesystem, mesa-libEGL, mesa-libGL, mesa-libgbm, mesa-vulkan-drivers +# +# Bluefin installs these from negativo17's fedora-multimedia repository. +# Utah does not enable third-party or negativo17 repositories at runtime. +# Instead, the utah-packages factory is building the complete multimedia +# and codec closure for Hummingbird (utah-packages#24). Until that closure +# is built and published, these packages cannot be installed. Tracked by #12. packages = [ + "@multimedia", "evolution-ews-core", + "ffmpeg", + "ffmpeg-libs", + "ffmpegthumbnailer", "firefox", "fish", "grub2-efi-x64-cdboot", + "gstreamer1-plugins-bad-free", + "gstreamer1-plugins-bad-free-libs", + "gstreamer1-plugins-base", + "gstreamer1-plugins-good", + "lame", + "lame-libs", + "libavcodec", + "libfdk-aac", + "libjxl", + "libva-intel-media-driver", + "mesa-dri-drivers", + "mesa-filesystem", + "mesa-libEGL", + "mesa-libGL", + "mesa-libgbm", + "mesa-vulkan-drivers", "pipewire-libs-extra", + "tailscale", + "uupd", "zsh", ] diff --git a/scripts/check-parity.py b/scripts/check-parity.py new file mode 100644 index 000000000..97b03ed1e --- /dev/null +++ b/scripts/check-parity.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +"""Check that Utah's Bluefin package contract matches Bluefin's effective payload. + +Bluefin executes package installations across multiple sources: +1. Base Fedora packages (base.toml [fedora]) +2. Version-specific Fedora packages (base.toml [fedora_v*]) +3. External transactions in 03-packages.sh (e.g. tailscale from tailscale-stable, + uupd from ublue-os/packages COPR) +4. Multimedia transactions: + - Override packages synced from fedora-multimedia (base.toml [multimedia_overrides]) + - Codecs and packages installed via --enablerepo='fedora-multimedia' in 03-packages.sh +5. Excluded packages (base.toml [excluded]) + +This verifier derives the effective package payload from upstream Bluefin +and asserts that packages/bluefin.toml tracks all effective requirements. +""" + +from __future__ import annotations + +import argparse +import re +import sys +import tomllib +import urllib.request +from pathlib import Path +from typing import Any + +BASE_TOML_URL = ( + "https://raw.githubusercontent.com/projectbluefin/bluefin/main/build_files/packages/base.toml" +) +PACKAGES_SH_URL = ( + "https://raw.githubusercontent.com/projectbluefin/bluefin/main/build_files/base/03-packages.sh" +) + + +def read_source(location: str) -> str: + if location.startswith("http://") or location.startswith("https://"): + req = urllib.request.Request( + location, headers={"User-Agent": "utah-parity-checker"} + ) + with urllib.request.urlopen(req, timeout=60) as resp: + return resp.read().decode("utf-8") + return Path(location).read_text(encoding="utf-8") + + +def expand_braces(text: str) -> list[str]: + match = re.search(r"([^{}\s]*)\{([^{}]+)\}([^{}\s]*)", text) + if not match: + return [text] + prefix, options, suffix = match.groups() + results = [] + for opt in options.split(","): + results.extend(expand_braces(f"{prefix}{opt}{suffix}")) + return results + + +def derive_effective_payload(base_toml_text: str, packages_sh_text: str) -> dict[str, list[str]]: + base_data = tomllib.loads(base_toml_text) + + # 1. Copr packages from 03-packages.sh (e.g. copr_install_isolated "ublue-os/packages" "uupd") + copr_packages = [] + for m in re.finditer( + r'copr_install_isolated\s+["\']([^"\']+)["\'](?:\s*\\\s*|\s+)["\']([^"\']+)["\']', + packages_sh_text, + ): + copr_packages.extend(m.group(2).split()) + + # 2. Extract dnf install arguments from 03-packages.sh + lines = packages_sh_text.splitlines() + in_install = False + install_tokens = [] + for line in lines: + stripped = line.strip() + if not in_install: + if re.search(r"\bdnf[0-9]?\s+.*install\b", stripped): + in_install = True + parts = re.split(r"\bdnf[0-9]?\s+.*install\s+", stripped, maxsplit=1) + token_part = parts[1] if len(parts) > 1 else "" + ends_cont = token_part.endswith("\\") + token_part = token_part.rstrip("\\").strip() + install_tokens.extend(token_part.split()) + if not ends_cont: + in_install = False + else: + ends_cont = stripped.endswith("\\") + token_part = stripped.rstrip("\\").strip() + install_tokens.extend(token_part.split()) + if not ends_cont: + in_install = False + + external_pkgs = list(copr_packages) + multimedia_pkgs = [] + skip_next = False + current_repo = None + + for token in install_tokens: + token = token.strip(" \"'") + if skip_next: + skip_next = False + continue + if token in ("-x", "--exclude"): + skip_next = True + continue + if token.startswith("-x=") or token.startswith("--exclude="): + continue + if token.startswith("--enablerepo="): + current_repo = token.split("=", 1)[1].strip(" \"'") + continue + if token.startswith("-"): + continue + if token.startswith('"${') or token.startswith("${"): + continue + + expanded = expand_braces(token) + for pkg in expanded: + if current_repo == "tailscale-stable" or pkg == "tailscale": + external_pkgs.append(pkg) + elif current_repo == "fedora-multimedia" or any( + m in pkg + for m in ( + "ffmpeg", + "gstreamer", + "lame", + "multimedia", + "libjxl", + "libfdk", + "libavcodec", + ) + ): + multimedia_pkgs.append(pkg) + else: + external_pkgs.append(pkg) + + payload: dict[str, list[str]] = { + "fedora": sorted(set(base_data.get("fedora", {}).get("packages", []))), + } + + # Version-specific additions (e.g. fedora_v42, fedora_v43, fedora_v44) + for k in sorted(base_data.keys()): + if k.startswith("fedora_v"): + payload[k] = sorted(set(base_data[k].get("packages", []))) + + payload["external"] = sorted(set(external_pkgs)) + payload["multimedia"] = sorted(set(multimedia_pkgs)) + payload["multimedia_overrides"] = sorted( + set(base_data.get("multimedia_overrides", {}).get("packages", [])) + ) + payload["excluded"] = sorted(set(base_data.get("excluded", {}).get("packages", []))) + + # Floor assertions: guard against silent under-detection if upstream shell structure shifts + min_fedora = 20 + min_external = 2 + if len(payload["fedora"]) < min_fedora: + raise ValueError( + f"Parsing error: parsed {len(payload['fedora'])} base Fedora packages, " + f"expected at least {min_fedora}. Upstream base.toml structure may have changed." + ) + if len(payload["external"]) < min_external: + raise ValueError( + f"Parsing error: parsed {len(payload['external'])} external packages, " + f"expected at least {min_external}. Upstream 03-packages.sh structure may have changed." + ) + + return payload + + +def format_toml(payload: dict[str, list[str]]) -> str: + lines = [ + "# Machine-readable contract derived from Bluefin's effective package payload.", + "#", + "# Sources in projectbluefin/bluefin:", + "# - build_files/packages/base.toml (fedora, version-specific, overrides, excluded)", + "# - build_files/base/03-packages.sh (external and multimedia transactions)", + "#", + "# Sections:", + "# [fedora] base Fedora packages common to all supported versions", + "# [fedora_v*] version-specific Fedora additions", + "# [external] packages from external repositories/COPRs", + "# [multimedia] codecs and multimedia packages from 03-packages.sh", + "# [multimedia_overrides] packages synced from negativo17/fedora-multimedia", + "# [excluded] packages removed from the base image", + "#", + "# Validated by scripts/check-parity.py on every CI run (`just check-parity`).", + "", + ] + + section_comments = { + "fedora": "# Base packages from Fedora repos — common to all supported Fedora versions.", + "fedora_v42": "# Fedora 42 additions", + "fedora_v43": "# Fedora 43 additions", + "fedora_v44": "# Fedora 44 additions", + "external": "# External and COPR packages installed in Bluefin's package transaction.", + "multimedia": "# Codecs and extra multimedia packages installed in Bluefin's transaction.", + "multimedia_overrides": "# Packages synced from negativo17/fedora-multimedia repo.", + "excluded": "# Packages removed from the base image — conflicts with or replaced by image content.", + } + + for section_name, packages in payload.items(): + comment = section_comments.get(section_name) + if comment: + lines.append(comment) + lines.append(f"[{section_name}]") + lines.append("packages = [") + for pkg in packages: + lines.append(f' "{pkg}",') + lines.append("]") + lines.append("") + + return "\n".join(lines).rstrip() + "\n" + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Verify parity between Utah's Bluefin contract and upstream Bluefin." + ) + parser.add_argument( + "--manifest", + type=Path, + default=Path("packages/bluefin.toml"), + help="Path to local Bluefin package contract (default: packages/bluefin.toml)", + ) + parser.add_argument( + "--upstream-manifest", + default=BASE_TOML_URL, + help="URL or path to Bluefin base.toml", + ) + parser.add_argument( + "--upstream-script", + default=PACKAGES_SH_URL, + help="URL or path to Bluefin 03-packages.sh", + ) + parser.add_argument( + "--write", + "--update", + action="store_true", + help="Write derived effective payload to the local contract file", + ) + args = parser.parse_args() + + base_toml_text = read_source(args.upstream_manifest) + packages_sh_text = read_source(args.upstream_script) + effective_payload = derive_effective_payload(base_toml_text, packages_sh_text) + + if args.write: + formatted = format_toml(effective_payload) + args.manifest.write_text(formatted, encoding="utf-8") + print(f"Updated {args.manifest} from upstream Bluefin effective payload.") + return 0 + + if not args.manifest.exists(): + print(f"ERROR: {args.manifest} does not exist.", file=sys.stderr) + return 1 + + local_data = tomllib.loads(args.manifest.read_text(encoding="utf-8")) + diff_found = False + + all_sections = sorted(set(effective_payload.keys()) | set(local_data.keys())) + for section_name in all_sections: + upstream_pkgs = set(effective_payload.get(section_name, [])) + local_pkgs = set(local_data.get(section_name, {}).get("packages", [])) + + missing = sorted(upstream_pkgs - local_pkgs) + unexpected = sorted(local_pkgs - upstream_pkgs) + + if missing or unexpected: + diff_found = True + print(f"Drift detected in section [{section_name}]:", file=sys.stderr) + if missing: + print(f" Missing additions from upstream ({len(missing)}):", file=sys.stderr) + for pkg in missing: + print(f" + {pkg}", file=sys.stderr) + if unexpected: + print(f" Unexpected packages in contract ({len(unexpected)}):", file=sys.stderr) + for pkg in unexpected: + print(f" - {pkg}", file=sys.stderr) + + if diff_found: + print( + "\nERROR: packages/bluefin.toml has drifted from Bluefin's effective payload.", + file=sys.stderr, + ) + print("Run `python3 scripts/check-parity.py --write` to update.", file=sys.stderr) + return 1 + + print("packages/bluefin.toml matches Bluefin's effective package payload.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check-repo-availability.py b/scripts/check-repo-availability.py index fb1b05e0a..317f18bba 100644 --- a/scripts/check-repo-availability.py +++ b/scripts/check-repo-availability.py @@ -85,11 +85,17 @@ def main() -> int: overlay = args.overlay or args.manifest.with_name("utah.toml") unavailable = set(section(overlay, "unavailable")) - wanted = sorted( - set(section(args.manifest, "fedora")) - | set(section(overlay, "gnome")) - | set(section(overlay, "build")) - ) + manifest_data = tomllib.loads(args.manifest.read_text()) + manifest_pkgs: set[str] = set() + for sec_name, sec_val in manifest_data.items(): + if sec_name == "excluded": + continue + if sec_name.startswith("fedora_v") and sec_name != "fedora_v44": + continue + manifest_pkgs.update(sec_val.get("packages", [])) + + overlay_pkgs = set(section(overlay, "gnome")) | set(section(overlay, "build")) + wanted = sorted(manifest_pkgs | overlay_pkgs) names: set[str] = set() for label, base in REPOS.items(): found = repo_package_names(base) diff --git a/scripts/install-packages.py b/scripts/install-packages.py index dc536b060..f4020dbe9 100755 --- a/scripts/install-packages.py +++ b/scripts/install-packages.py @@ -42,15 +42,17 @@ def fedora_major() -> str: def contract(base: Path, overlay: Path, major: str | None) -> list[str]: """The exact set of packages the built image must contain. - Bluefin installs [fedora] plus the [fedora_v] section for the - Fedora release it targets, and simply skips that section when it does not - exist. The lookup is dynamic: whatever release the base image reports, - Utah installs the matching section when upstream defines one and skips it - otherwise, so a new upstream section is picked up for free. + Bluefin executes base [fedora], version-specific [fedora_v], + [external] (e.g. tailscale, uupd), [multimedia], and [multimedia_overrides] + transactions. Utah installs all of these plus Utah's desktop additions + ([gnome], [services]), minus any documented exceptions in [unavailable]. """ packages = section(base, "fedora") if major: packages += section(base, f"fedora_v{major}") + packages += section(base, "external") + packages += section(base, "multimedia") + packages += section(base, "multimedia_overrides") packages += section(overlay, "gnome") # Service packages are part of the desktop contract as well: 40-services.sh # cannot enable what the server base never installed. @@ -101,7 +103,7 @@ def main() -> int: if args.check: # No rpmdb to consult off-image, so validate the manifests only. - packages = contract(args.manifest, overlay, major=None) + packages = contract(args.manifest, overlay, major="44") if not packages: raise ValueError("Bluefin package manifest is empty") unavailable = section(overlay, "unavailable") diff --git a/scripts/verify-rpm-contract.py b/scripts/verify-rpm-contract.py index 08fb22ff3..0a25de141 100644 --- a/scripts/verify-rpm-contract.py +++ b/scripts/verify-rpm-contract.py @@ -66,7 +66,14 @@ def main() -> int: gnome = [p for p in contract if p in gnome_names] services = [p for p in contract if p in service_names] else: - bluefin = [p for p in section(args.manifest, "fedora") if p not in unavailable] + bluefin_pkgs = ( + section(args.manifest, "fedora") + + section(args.manifest, "fedora_v44") + + section(args.manifest, "external") + + section(args.manifest, "multimedia") + + section(args.manifest, "multimedia_overrides") + ) + bluefin = [p for p in bluefin_pkgs if p not in unavailable] gnome = section(overlay, "gnome") services = section(overlay, "services") nvidia = list(NVIDIA_PACKAGES) if "nvidia" in flavor else [] diff --git a/tests/unit/test_check_parity.py b/tests/unit/test_check_parity.py new file mode 100644 index 000000000..8ed503cb2 --- /dev/null +++ b/tests/unit/test_check_parity.py @@ -0,0 +1,119 @@ +"""Unit tests for scripts/check-parity.py derivation and parsing logic.""" + +from __future__ import annotations + +import importlib.util +import tomllib +import pytest +from pathlib import Path +import sys + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + +spec = importlib.util.spec_from_file_location( + "check_parity", REPO_ROOT / "scripts" / "check-parity.py" +) +check_parity = importlib.util.module_from_spec(spec) +spec.loader.exec_module(check_parity) + +expand_braces = check_parity.expand_braces +derive_effective_payload = check_parity.derive_effective_payload +format_toml = check_parity.format_toml + + +def test_expand_braces_single(): + assert expand_braces("package") == ["package"] + + +def test_expand_braces_variants(): + assert sorted(expand_braces("gstreamer1-{plugins-base,plugins-good}")) == [ + "gstreamer1-plugins-base", + "gstreamer1-plugins-good", + ] + assert sorted(expand_braces("pkg{,-devel}")) == ["pkg", "pkg-devel"] + + +def test_derive_effective_payload_basic(): + base_toml = """ +[fedora] +packages = [ + "pkg01", "pkg02", "pkg03", "pkg04", "pkg05", + "pkg06", "pkg07", "pkg08", "pkg09", "pkg10", + "pkg11", "pkg12", "pkg13", "pkg14", "pkg15", + "pkg16", "pkg17", "pkg18", "pkg19", "pkg20" +] + +[multimedia_overrides] +packages = ["ffmpeg"] + +[excluded] +packages = ["nano"] +""" + + packages_sh = """ +#!/usr/bin/env bash +# Some comments +copr_install_isolated "ublue-os/packages" "uupd" + +dnf5 -y install \\ + --enablerepo="tailscale-stable" \\ + tailscale + +dnf5 -y install \\ + --enablerepo="fedora-multimedia" \\ + --exclude=excluded-codec \\ + gstreamer1-plugins-{bad,ugly} +""" + + payload = derive_effective_payload(base_toml, packages_sh) + assert len(payload["fedora"]) == 20 + assert "uupd" in payload["external"] + assert "tailscale" in payload["external"] + assert "gstreamer1-plugins-bad" in payload["multimedia"] + assert "gstreamer1-plugins-ugly" in payload["multimedia"] + assert "excluded-codec" not in payload["multimedia"] + assert payload["multimedia_overrides"] == ["ffmpeg"] + assert payload["excluded"] == ["nano"] + + +def test_derive_effective_payload_floor_assertions(): + # Fewer than 20 Fedora packages should fail loudly + low_fedora = """ +[fedora] +packages = ["single-pkg"] +""" + packages_sh = """ +copr_install_isolated "ublue-os/packages" "uupd" +dnf install tailscale +""" + with pytest.raises(ValueError, match="parsed 1 base Fedora packages, expected at least 20"): + derive_effective_payload(low_fedora, packages_sh) + + # Fewer than 2 external packages should fail loudly + valid_fedora = """ +[fedora] +packages = [ + "p01", "p02", "p03", "p04", "p05", "p06", "p07", "p08", "p09", "p10", + "p11", "p12", "p13", "p14", "p15", "p16", "p17", "p18", "p19", "p20" +] +""" + empty_packages_sh = "# No packages" + with pytest.raises(ValueError, match="parsed 0 external packages, expected at least 2"): + derive_effective_payload(valid_fedora, empty_packages_sh) + + +def test_format_toml_roundtrip(): + payload = { + "fedora": ["curl", "git"], + "external": ["tailscale", "uupd"], + "multimedia": ["ffmpeg"], + "multimedia_overrides": ["pipewire"], + "excluded": ["nano"], + } + toml_str = format_toml(payload) + data = tomllib.loads(toml_str) + assert data["fedora"]["packages"] == ["curl", "git"] + assert data["external"]["packages"] == ["tailscale", "uupd"] + assert data["multimedia"]["packages"] == ["ffmpeg"] + assert data["multimedia_overrides"]["packages"] == ["pipewire"] + assert data["excluded"]["packages"] == ["nano"]