diff --git a/AGENTS.md b/AGENTS.md index 02eedd25c..ab849ec76 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -151,7 +151,11 @@ tooling arrives; do not copy them as-is. - Sources come from **upstream releases**, verified and SHA-512 locked. Fedora dist-git supplies the **recipe only**, pinned by commit in - `.hummingbird-upstream.json`. + `.hummingbird-upstream.json`. Fedora belongs in `fallback_urls`, never in the + primary `url`: `tools/source_pipeline.py` fetches the primary directly, so a + Fedora URL there builds from the lookaside. `tools/validate.py` enforces this + as a ratchet against `config/fedora-primary-sources.txt`, which records the + entries still waiting on the bulk re-pin; that list only shrinks. - Build order is solved from real BuildRequires (`tools/build_graph.py`): a package builds in a later wave than every factory package it BuildRequires. A `stage` in `config/upstream-sources.json` only orders the diff --git a/config/fedora-primary-sources.txt b/config/fedora-primary-sources.txt new file mode 100644 index 000000000..6f22cf51a --- /dev/null +++ b/config/fedora-primary-sources.txt @@ -0,0 +1,259 @@ +# Packages whose source lock still takes its payload from Fedora's lookaside +# instead of the project's own release (utah-packages#42). tools/validate.py +# reads this as a ratchet: a package outside this list may not use a Fedora +# primary `url`, and a package re-pinned upstream must be deleted from here. +# The list only shrinks. +ModemManager +SDL3 +abseil-cpp +adwaita-icon-theme-legacy +alsa-firmware +alsa-sof-firmware +alsa-tools +alsa-utils +appstream +aribb24 +at-spi2-core +bluez +cairo +cdparanoia +cjson +codec2 +colord +colord-gtk +containerd +cups-pk-helper +desktop-file-utils +device-mapper-persistent-data +double-conversion +enchant2 +evolution-data-server +fdk-aac-free +ffmpeg +flac +flatpak +flite +fribidi +fuse +fwupd +fzf +game-music-emu +gcr +gcr3 +gdk-pixbuf2 +geoclue2 +geocode-glib +gjs +glib-networking +glycin +gnome-autoar +gnome-bluetooth +gnome-keyring +gnome-online-accounts +gobject-introspection +graphene +gsl +gsm +gsound +gssdp +gstreamer1 +gstreamer1-plugins-bad-free +gstreamer1-plugins-base +gstreamer1-plugins-good +gtk2 +gtk3 +gtksourceview4 +gupnp +gupnp-igd +gweather-locations +hicolor-icon-theme +hidapi +highway +hunspell +hunspell-en +hwdata +hwinfo +hyphen +i2c-tools +ibus +icu +iio-sensor-proxy +ilbc +iputils +iso-codes +jpegxl +json-glib +kde-filesystem +kde-settings +kf5 +lame +lcms2 +libICE +libSM +libXcursor +libXdamage +libXdmcp +libXfixes +libXfont2 +libXinerama +libXmu +libXrandr +libXt +libXv +libaribcaption +libasyncns +libatasmart +libayatana-ido +libayatana-indicator +libburn +libbytesize +libcue +libdatrie +libdbusmenu +libdecor +libdisplay-info +libdrm +libebur128 +libei +libepoxy +libevdev +libfyaml +libglvnd +libgtop2 +libgudev +libgusb +libgweather +libical +libimobiledevice +libimobiledevice-glue +libinput +libisofs +liblc3 +libldac +libmanette +libnfs +libnice +libnma +libnotify +libnvme +libogg +liboping +libpciaccess +libplist +libproxy +librsvg2 +libsass +libsecret +libshout +libsndfile +libsoup3 +libtdb +libthai +libtheora +libusbmuxd +libvdpau +libvisual +libvorbis +libvpl +libvpx +libwacom +libx86emu +libxcvt +libxkbfile +libxmlb +libxshmfence +linux-firmware +livesys-scripts +lm_sensors +localsearch +lpcnetfreedv +lttng-ust +mdadm +mesa +mesa-demos +mobile-broadband-provider-info +mozc +mozjs140 +mpg123 +msgraph +mtdev +nautilus-python +noopenh264 +openapv +opencore-amr +openjpeg +opus +orc +parted +passim +pciutils +pipewire +pixman +protobuf +protobuf3 +pulseaudio +pycairo +python-annotated-types +python-argcomplete +python-dasbus +python-dbus-next +python-distro +python-evdev +python-pam +python-psutil +python-pydantic +python-typing-inspection +rest +rtkit +runc +samba +sbc +sdl2-compat +setools +setxkbmap +snowball +sound-theme-freedesktop +soxr +spandsp +speex +spirv-tools +srt +taglib +twolame +udisks2 +upower +v4l-utils +vo-amrwbenc +volume_key +vulkan-headers +vulkan-loader +wavpack +wayland +webkitgtk +wireguard-tools +wireplumber +wsdd +xcb-util +xcb-util-image +xcb-util-keysyms +xcb-util-renderutil +xcb-util-wm +xdg-dbus-proxy +xdg-desktop-portal-gtk +xdg-user-dirs +xdg-user-dirs-gtk +xevd +xeve +xhost +xkbcomp +xmlrpc-c +xmodmap +xorg-x11-server-Xwayland +xorg-x11-xauth +xorg-x11-xinit +xprop +xrdb +xrefresh +xvidcore +yyjson +zvbi diff --git a/docs/architecture.md b/docs/architecture.md index 346b17d12..0788e5ae3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -315,7 +315,7 @@ The repository gates are enforced across CI workflows and collected in `Justfile | Gate | Command | Enforced in | What it gates | | --- | --- | --- | --- | | Factory onboarding contract | `tools/factory_contract.py` | `.github/workflows/validate.yml` | Skill router coverage, skill front-matter, the `AGENTS.md` self-improvement mandate, the pinned `projectbluefin/common` sidecar, banned changelog and session-notes files, and relative documentation links | -| Package factory configuration | `tools/validate.py` | `.github/workflows/validate.yml` | Import provenance in `.hummingbird-upstream.json`, source-lock coverage, and Packit configuration for every recipe | +| Package factory configuration | `tools/validate.py` | `.github/workflows/validate.yml` | Import provenance in `.hummingbird-upstream.json`, source-lock coverage, Packit configuration for every recipe, and the Fedora-primary-URL ratchet in `config/fedora-primary-sources.txt` | | Workflow shell quoting | `tools/check_workflow_quoting.py` | `.github/workflows/validate.yml`, `.github/workflows/rebuild-rpms.yml` (`prepare`) | Shell-quoting safety of build scripts embedded in GitHub Actions workflows | | Runtime contract | `tools/runtime_contract.py config/bluefin-packages.toml config/runtime-contract.toml --check` | `.github/workflows/validate.yml`, `.github/workflows/rebuild-rpms.yml` (`prepare`) | Image manifest resolution against the pinned Hummingbird runtime contract | | Unit tests | `pytest tests` / `unittest discover` | `.github/workflows/validate.yml`, `.github/workflows/rebuild-rpms.yml` (`prepare`) | The tooling in `tools/`, including `tools/publish_gate.py`, whose regression test asserts the rebuild-rpms.yml publish job replaces only what a run built, keeps a failed or precedence-losing package at its previous build, and never publishes a candidate whose Hummingbird-only transaction does not resolve | diff --git a/tests/test_validate.py b/tests/test_validate.py index c7e9bed1e..103733f49 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -20,6 +20,11 @@ ROOT = Path(__file__).resolve().parent.parent VALIDATE = ROOT / "tools" / "validate.py" +LOOKASIDE_URL = ( + "https://src.fedoraproject.org/repo/pkgs/rpms/example/example-1.tar.xz/" + "sha512/" + "0" * 128 + "/example-1.tar.xz" +) + RAWHIDE_PROVENANCE = { "package": "example", "branch": "rawhide", @@ -64,6 +69,8 @@ def build( packit=None, buildroot_lock=DEFAULT_BUILDROOT_LOCK, buildroot_pin=DEFAULT_BUILDROOT_PIN, + source_urls=None, + fedora_baseline=None, ) -> None: """Write a factory tree validate.py accepts unless a case breaks one rule.""" locked = packages if locked is None else locked @@ -77,9 +84,17 @@ def build( json.dumps({**provenance, "package": name}) ) (root / "config").mkdir() + locks = [{"name": name} for name in locked] + for entry in locks: + if entry["name"] in (source_urls or {}): + entry["url"] = source_urls[entry["name"]] (root / "config" / "upstream-sources.json").write_text( - json.dumps({"packages": [{"name": name} for name in locked]}) + json.dumps({"packages": locks}) ) + if fedora_baseline is not None: + (root / "config" / "fedora-primary-sources.txt").write_text( + "# baseline\n" + "".join(f"{name}\n" for name in fedora_baseline) + ) if buildroot_lock is not None: (root / "config" / "buildroot-lock.json").write_text( json.dumps(buildroot_lock) @@ -271,6 +286,56 @@ def test_returns_zero_when_packages_directory_is_absent(self) -> None: result = self.run_validate(root) assert result.returncode == 0, result.stderr + def test_rejects_a_source_lock_that_fetches_from_fedora(self) -> None: + # source_pipeline.py fetches the primary `url` directly, so a Fedora URL + # there builds the package from the lookaside -- the dependency the + # direct-source pipeline exists to remove (utah-packages#42). + result = self.check( + source_urls={"example": LOOKASIDE_URL}, + ) + assert result.returncode != 0 + assert "takes its payload from Fedora, not upstream: example" in result.stderr + assert "fallback_urls" in result.stderr + + def test_accepts_a_fedora_url_that_the_baseline_already_records(self) -> None: + # The bulk re-pin is long; until it lands the recorded set must still + # pass, or the gate cannot be turned on at all. + result = self.check( + source_urls={"example": LOOKASIDE_URL}, + fedora_baseline=("example",), + ) + assert result.returncode == 0, result.stdout + result.stderr + + def test_rejects_a_new_fedora_url_alongside_a_recorded_one(self) -> None: + result = self.check( + packages=("example", "second"), + source_urls={"example": LOOKASIDE_URL, "second": LOOKASIDE_URL}, + fedora_baseline=("example",), + ) + assert result.returncode != 0 + offenders = result.stderr.splitlines()[0].rsplit(": ", 1)[-1] + assert offenders == "second" + + def test_rejects_a_baseline_entry_that_is_no_longer_an_offender(self) -> None: + # A package re-pinned upstream but left listed holds the slot open for + # the same name to regress unnoticed, so the list only shrinks. + result = self.check(fedora_baseline=("example",)) + assert result.returncode != 0 + assert "no longer use a Fedora primary URL: example" in result.stderr + + def test_accepts_an_upstream_url(self) -> None: + result = self.check( + source_urls={"example": "https://github.com/example/example/releases/example-1.tar.xz"}, + ) + assert result.returncode == 0, result.stdout + result.stderr + + def test_rejects_any_fedora_subdomain_in_the_primary_position(self) -> None: + result = self.check( + source_urls={"example": "https://kojipkgs.fedoraproject.org/packages/example.tar.xz"}, + ) + assert result.returncode != 0 + assert "takes its payload from Fedora" in result.stderr + def test_the_checked_in_factory_tree_passes_its_own_gate(self) -> None: result = self.run_validate(ROOT) assert result.returncode == 0, result.stdout + result.stderr diff --git a/tools/validate.py b/tools/validate.py index f301caee8..a0df0ea29 100755 --- a/tools/validate.py +++ b/tools/validate.py @@ -6,6 +6,7 @@ import json import re import sys +import urllib.parse from pathlib import Path FULL_SHA = re.compile(r"^[0-9a-f]{40}$") @@ -14,6 +15,7 @@ from tools.buildroot_pin import parse as parse_buildroot_pin from tools.check_suppressed_tests import main as check_suppressed_tests +from tools.bootstrap_upstream_sources import FEDORA_HOSTS from tools.package_inventory import inventory @@ -99,6 +101,78 @@ def check_provenance(path: Path, data: dict) -> None: raise SystemExit(f"rawhide import must carry a full {key} SHA: {path}") +def fedora_primary_sources(path: Path) -> list[str]: + """Names in the source lock whose primary `url` is Fedora infrastructure. + + `tools/source_pipeline.py` fetches the primary `url` directly, so an entry + pointing at Fedora takes its payload from the lookaside rather than from + the project's own release -- which is the one thing AGENTS.md and + docs/targeting-hummingbird.md say the factory does not do. The SHA-512 lock + still holds, so this is provenance, not integrity: Fedora belongs in + `fallback_urls`, which source_pipeline already consults after an upstream + transport failure, not in the primary position. + """ + if not path.is_file(): + return [] + try: + data = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + raise SystemExit(f"invalid source lock: {path}") + offenders = [] + for entry in data.get("packages", []): + if not isinstance(entry, dict): + continue + host = urllib.parse.urlparse(entry.get("url") or "").hostname or "" + if host in FEDORA_HOSTS or host.endswith(".fedoraproject.org"): + offenders.append(entry.get("name") or entry.get("filename") or "") + return sorted(offenders) + + +def read_fedora_primary_baseline(path: Path) -> list[str]: + if not path.is_file(): + return [] + names = [] + for line in path.read_text().splitlines(): + stripped = line.split("#", 1)[0].strip() + if stripped: + names.append(stripped) + return sorted(names) + + +def check_fedora_primary_sources(root: Path) -> None: + """A ratchet: the Fedora-primary set may shrink, never grow. + + Re-pointing every entry at its upstream release is a long bulk edit + (utah-packages#42); until it finishes, the gap has to stop widening, so the + packages still on a Fedora primary URL are listed in the baseline and + anything outside it fails. The baseline is also checked for staleness -- + a package re-pinned upstream but left in the file would hold a slot open + for the gap to reopen under the same name. + """ + offenders = fedora_primary_sources(root / "config" / "upstream-sources.json") + baseline_path = root / "config" / "fedora-primary-sources.txt" + baseline = read_fedora_primary_baseline(baseline_path) + added = sorted(set(offenders) - set(baseline)) + removed = sorted(set(baseline) - set(offenders)) + if added: + raise SystemExit( + "source lock takes its payload from Fedora, not upstream: " + f"{', '.join(added)}\n" + "Point the entry's `url` at the project's own release and keep the " + "Fedora URL in `fallback_urls`, which source_pipeline.py already " + "uses after an upstream transport failure. Where no upstream " + "artifact exists verbatim, add a generator in " + "tools/generated_sources.py instead." + ) + if removed: + raise SystemExit( + f"{baseline_path.name} lists packages that no longer use a Fedora " + f"primary URL: {', '.join(removed)}\n" + "Delete those lines. The list only shrinks; a stale entry holds the " + "slot open for the same package to regress unnoticed." + ) + + def main(root: Path = Path(".")) -> int: packages_dir = root / "packages" if not packages_dir.is_dir(): @@ -117,6 +191,9 @@ def main(root: Path = Path(".")) -> int: # Before the recipe tally, so a package that is merely missing a Packit # entry cannot hide a buildroot that drifted from its lock. validate_buildroots(root / "config" / "buildroot-lock.json") + # Before the tally too: a lock that builds from the lookaside is a + # provenance failure whether or not every recipe is otherwise accounted for. + check_fedora_primary_sources(root) records = inventory(root) missing_locks = sorted(record.name for record in records if not record.source_locked) missing_packit = sorted(record.name for record in records if not record.packit_configured)