From d7edb254fbac2d9aac94983f8ca9f5a6e90ff509 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 00:23:16 -0500 Subject: [PATCH 1/7] fix(keys): gen-dev-keys refuses partial key sets without --force A complete set was already kept, but the check looked only at DB.key and sysupdate-signing.asc: with DB.key gone the script regenerated PK, KEK and the module certificate over the existing files, and with the signing key gone it replaced import-pubring.pgp. Both sets are now checked file by file (an empty file counts as missing) and a partial set exits 1 before anything is written. tests/unit/test_gen_dev_keys.py runs the script against a scratch tree with openssl and gpg stubbed: a complete set is left untouched, every partial set is refused without calling either tool, and --force still regenerates. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/gen-dev-keys.sh | 40 +++++++++-- tests/unit/test_gen_dev_keys.py | 119 ++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 tests/unit/test_gen_dev_keys.py diff --git a/scripts/gen-dev-keys.sh b/scripts/gen-dev-keys.sh index 0b45b9c3..ad83208d 100644 --- a/scripts/gen-dev-keys.sh +++ b/scripts/gen-dev-keys.sh @@ -12,8 +12,13 @@ # files/boot-keys/import-pubring.pgp its public keyring, installed as # /etc/systemd/import-pubring.pgp (importd, sysupdate) # -# Changing modules/linux-module-cert.crt changes the kernel's cache key and -# forces a kernel rebuild, so existing keys are kept unless --force is given. +# Existing keys are never overwritten without --force: every key is baked +# into or signs the image, so replacing one needs a new image-version (see +# "Keys" in docs/skills/ddi-installer-build.md), and changing +# modules/linux-module-cert.crt also forces a kernel rebuild. A partial set +# (some files of a pair or of the boot set missing) is an error rather than +# something to fill in, since filling it in would replace the files that do +# exist. set -euo pipefail dir="$(cd "$(dirname "$0")/.." && pwd)/files/boot-keys" @@ -37,11 +42,38 @@ gen_signing_key() { echo "Generated image signing key in ${dir}" } -if [ ! -s "${dir}/sysupdate-signing.asc" ] || [ "${force}" = 1 ]; then +signing_keys=(sysupdate-signing.asc import-pubring.pgp) +boot_keys=(PK.key PK.crt KEK.key KEK.crt DB.key DB.crt linux-module-cert.key modules/linux-module-cert.crt) + +# Prints "none", "all" or "partial" for the named files under ${dir}. +key_set_state() { + local present=0 f + for f in "$@"; do + [ -s "${dir}/${f}" ] && present=$((present + 1)) + done + if [ "${present}" = 0 ]; then echo none + elif [ "${present}" = "$#" ]; then echo all + else echo partial + fi +} + +signing_state="$(key_set_state "${signing_keys[@]}")" +boot_state="$(key_set_state "${boot_keys[@]}")" +if [ "${force}" = 0 ]; then + for state in "signing:${signing_state}" "boot:${boot_state}"; do + if [ "${state#*:}" = partial ]; then + echo "ERROR: ${dir} holds a partial ${state%%:*} key set; refusing to overwrite the files that exist." >&2 + echo " Restore the missing files, or pass --force to regenerate every key (then bump image-version)." >&2 + exit 1 + fi + done +fi + +if [ "${signing_state}" != all ] || [ "${force}" = 1 ]; then gen_signing_key fi -if [ -e "${dir}/DB.key" ] && [ "${force}" = 0 ]; then +if [ "${boot_state}" = all ] && [ "${force}" = 0 ]; then echo "Keys already exist in ${dir}; pass --force to regenerate (rebuilds the kernel)." exit 0 fi diff --git a/tests/unit/test_gen_dev_keys.py b/tests/unit/test_gen_dev_keys.py new file mode 100644 index 00000000..d341022a --- /dev/null +++ b/tests/unit/test_gen_dev_keys.py @@ -0,0 +1,119 @@ +"""scripts/gen-dev-keys.sh never replaces existing keys without --force. + +Each case runs a copy of the script against a scratch files/boot-keys/ with +openssl and gpg replaced by stubs that log their calls, so no key material is +generated and a refusal can be told apart from a regeneration. +""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import subprocess +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "scripts" / "gen-dev-keys.sh" + +SIGNING = ["sysupdate-signing.asc", "import-pubring.pgp"] +BOOT = [ + "PK.key", + "PK.crt", + "KEK.key", + "KEK.crt", + "DB.key", + "DB.crt", + "linux-module-cert.key", + "modules/linux-module-cert.crt", +] + + +@pytest.fixture +def tree(tmp_path: Path) -> Path: + (tmp_path / "scripts").mkdir() + shutil.copy(SCRIPT, tmp_path / "scripts" / SCRIPT.name) + stubs = tmp_path / "stubs" + stubs.mkdir() + for tool in ("openssl", "gpg"): + stub = stubs / tool + stub.write_text(f'#!/bin/sh\necho {tool} >> "$STUB_LOG"\nexit 1\n') + stub.chmod(0o755) + return tmp_path + + +def seed(tree: Path, names: list[str]) -> None: + keys = tree / "files" / "boot-keys" + for name in names: + path = keys / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(f"existing {name}\n") + + +def snapshot(tree: Path) -> dict[str, str]: + keys = tree / "files" / "boot-keys" + return { + str(p.relative_to(keys)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(keys.rglob("*")) + if p.is_file() + } + + +def run(tree: Path, *args: str) -> subprocess.CompletedProcess[str]: + env = dict(os.environ) + env["PATH"] = f"{tree / 'stubs'}:{env['PATH']}" + env["STUB_LOG"] = str(tree / "stub.log") + return subprocess.run( + ["bash", str(tree / "scripts" / SCRIPT.name), *args], + capture_output=True, + text=True, + env=env, + ) + + +def stub_calls(tree: Path) -> str: + log = tree / "stub.log" + return log.read_text() if log.exists() else "" + + +def test_complete_key_set_is_kept_without_force(tree: Path) -> None: + seed(tree, SIGNING + BOOT) + before = snapshot(tree) + result = run(tree) + assert result.returncode == 0, result.stderr + assert "pass --force to regenerate" in result.stdout + assert snapshot(tree) == before + assert stub_calls(tree) == "" + + +@pytest.mark.parametrize( + "missing", + ["DB.key", "PK.crt", "modules/linux-module-cert.crt", "sysupdate-signing.asc", "import-pubring.pgp"], +) +def test_partial_key_set_is_refused_without_force(tree: Path, missing: str) -> None: + seed(tree, [name for name in SIGNING + BOOT if name != missing]) + before = snapshot(tree) + result = run(tree) + assert result.returncode == 1 + assert "partial" in result.stderr + assert "--force" in result.stderr + assert snapshot(tree) == before + assert stub_calls(tree) == "" + + +def test_empty_key_file_counts_as_missing(tree: Path) -> None: + seed(tree, SIGNING + BOOT) + (tree / "files" / "boot-keys" / "KEK.key").write_text("") + result = run(tree) + assert result.returncode == 1 + assert "partial boot key set" in result.stderr + assert stub_calls(tree) == "" + + +def test_force_regenerates_an_existing_set(tree: Path) -> None: + seed(tree, SIGNING + BOOT) + result = run(tree, "--force") + assert result.returncode != 0, "the failing gpg stub must have been reached" + assert stub_calls(tree).splitlines()[0] == "gpg" From 2211a7fd396a56a4e391752abf3b550fb6eb9c7b Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 00:23:16 -0500 Subject: [PATCH 2/7] docs(keys): rotating a key needs a new image-version Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/ddi-installer-build.md | 17 ++++++++++++++++- docs/skills/systemd-sysupdate-verification.md | 7 +++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 0d8e1067..f65f7214 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -57,13 +57,28 @@ with the module signing certificate, and the release `SHA256SUMS` with the image signing key. `build-image` (and `validate`) depends on `gen-dev-keys`, which generates throwaway keys in `files/boot-keys/` (gitignored) on first run: PK/KEK/DB, the module certificate, and the `sysupdate-signing.asc` / -`import-pubring.pgp` pair. Keys are kept unless `--force` is given. CI builds +`import-pubring.pgp` pair. Keys are kept unless `--force` is given, and a +partial set (a file of a pair or of the boot set missing or empty) is an error +rather than something to fill in. CI builds on `main` unpack the `BOOT_KEYS_TARBALL` secret, write `SYSUPDATE_SIGNING_KEY` to `files/boot-keys/sysupdate-signing.asc`, and copy the committed release keyring `files/os/sysupdate-keys/import-pubring.gpg` to `files/boot-keys/import-pubring.pgp`; pull requests get throwaway keys and their images are never published. +**Rotating any key needs a new `image-version`.** Every key ends up in the +image bits: DB signs the UKIs and systemd-boot, the module certificate is +built into the kernel, and `import-pubring.pgp` ships in `/usr`. An image +version names one immutable set of bits, and `systemd-sysupdate` only +installs a version newer than the one it runs, so rebuilding the same version +with new keys publishes different bits under a released name and never +reaches nodes already on it. Rotate keys (`just gen-dev-keys --force`, or new +CI secrets), then `just set-version` to a version that sorts higher before +building. Nodes also need the new Secure Boot keys enrolled and, for the +image signing key, the new keyring; see +"Rotating the Signing Key" in +[systemd-sysupdate-verification.md](systemd-sysupdate-verification.md). + ## Dogfood: boot it in QEMU All dogfood paths boot with Secure Boot firmware (OVMF secboot). The firmware diff --git a/docs/skills/systemd-sysupdate-verification.md b/docs/skills/systemd-sysupdate-verification.md index db982f08..b84012ce 100644 --- a/docs/skills/systemd-sysupdate-verification.md +++ b/docs/skills/systemd-sysupdate-verification.md @@ -132,8 +132,11 @@ proves both). ``` 2. Update the GitHub Actions repository secret `SYSUPDATE_SIGNING_KEY` with the new ASCII-armored private key. -3. Rebuild and publish a release. Existing hosts will only trust updates signed - by the new key, so plan the rotation around a release boundary. +3. Rebuild and publish a release under a new `image-version` (a key rotation + is never a rebuild of an existing version; see + "Keys" in [ddi-installer-build.md](ddi-installer-build.md)). Existing hosts only + trust updates signed by the key in their keyring, so plan the rotation + around a release boundary. For a throwaway local signing key, `just gen-dev-keys` writes `files/boot-keys/sysupdate-signing.asc` and `files/boot-keys/import-pubring.pgp` From 5e3454e324dcc5d08665106fb42b2c00cc0d7c50 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 00:25:16 -0500 Subject: [PATCH 3/7] docs(patches): document every FSDK patch and its drop condition patches/README.md records why each patch exists, whether it went upstream and when to drop it, and that 0003-0005 name full library versions (libgirepository 1.0.0, gdk-pixbuf 0.4400.7, appstream 1.1.6) that a component bump leaves dangling while the patch still applies. bump-fsdk-version.md pointed at a single remaining CAS patch; it now links the README and adds the version comparison to the bump procedure. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/bump-fsdk-version.md | 10 +++++---- patches/README.md | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 patches/README.md diff --git a/docs/skills/bump-fsdk-version.md b/docs/skills/bump-fsdk-version.md index 541f2df1..2e867e11 100644 --- a/docs/skills/bump-fsdk-version.md +++ b/docs/skills/bump-fsdk-version.md @@ -37,9 +37,11 @@ There is no application version for these images. Two version axes exist: 2. Update the `ref:` in `elements/freedesktop-sdk.bst` to the new tag/commit. -3. Re-check patches still apply — FSDK ships local patches under - `patches/freedesktop-sdk/`. If a release changed the patched files, refresh or - drop them. `just validate` surfaces patch failures. +3. Re-check the local patches in `patches/freedesktop-sdk/` + ([`patches/README.md`](../../patches/README.md) says why each exists and + when to drop it). `just validate` surfaces a patch that no longer applies; + it does not catch 0003-0005 pointing at a library version the new ref no + longer builds, so compare those by hand (see the README). 4. Rebuild and verify: @@ -71,7 +73,7 @@ Before merging a bump: - `components/systemd-base.bst` was dropped in FSDK 26.08, and FSDK now ships its own systemd directly, so the previous `gnome-build-meta` systemd overrides were removed from `elements/freedesktop-sdk.bst`. - A point-release tag is immutable: once a GitHub Release for a given `image-version` is published, never republish different bits under it. -- **Only one CAS-config patch remains.** FSDK 26.08 absorbed the old `0001` CAS-limits patch (upstream `project.conf` sets `retry-limit`/`retry-delay`/`request-timeout` itself), so `patches/freedesktop-sdk/` carries only `0001-project.conf-Add-GNOME-CAS-servers.patch`. +- FSDK 26.08 absorbed the old CAS-limits patch (upstream `project.conf` sets `retry-limit`/`retry-delay`/`request-timeout` itself); the remaining patches are listed in [`patches/README.md`](../../patches/README.md). - Junction overrides are only meaningful for components your local elements reference directly. The 25 GNOME sdk/* overrides (cairo, gtk3, pango, glib, gdk-pixbuf…) were dead weight — none of our `base-stack`, `brew-deps` etc. ever diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 00000000..0cae08c9 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,37 @@ +# patches/ + +`patch_queue` sources apply every file in a directory, in file name order, +on top of the junction they belong to. `just validate` fails if one no longer +applies. + +## freedesktop-sdk/ + +Applied to the freedesktop-sdk junction (`elements/freedesktop-sdk.bst`). +Every patch that edits an FSDK element (0002-0006) changes that element's +cache key and the keys of everything that depends on it, so those artifacts +come from the Bluefin cache or a local build, never from FSDK's cache. Keep +the queue short. + +| Patch | Why | Upstream status | Drop when | +|---|---|---|---| +| `0001-project.conf-Add-GNOME-CAS-servers.patch` | Adds the GNOME artifact and source cache to FSDK's own `project.conf`, so the junction pulls from it without per-user BuildStream config. Taken from gnome-build-meta. | Downstream by design (GNOME carries it too). | GNOME's cache stops serving FSDK 26.08 artifacts. gnome-50 builds on FSDK 25.08, so check the hit rate before the next FSDK minor. | +| `0002-glib-stage1-disable-tests.patch` | `-Dtests=false` for the bootstrap glib: building its tests ran the uninstalled `glib-compile-resources` before libgio existed and failed with exit 127 in from-source bootstrap builds (projectbluefin/server#50). | Not reported upstream. | A from-source build of `components/_private/glib-stage1.bst` passes without it. | +| `0003-gobject-introspection-base-library-symlinks.patch` | Remote-execution (BuildBarn) builds: creates the uninstalled SONAME links for `libgirepository-1.0.so.1` and sets `LD_LIBRARY_PATH` before ninja runs `g-ir-compiler` (projectbluefin/server#53). | Not reported upstream. | Builds no longer run under remote execution, or they pass without it. | +| `0004-gdk-pixbuf-library-symlinks.patch` | Same failure for `gdk-pixbuf-print-mime-types` and `libgdk_pixbuf-2.0.so.0` (projectbluefin/server#55). | Not reported upstream. | As 0003. | +| `0005-appstream-library-symlinks.patch` | Same failure for `appstreamcli` and `libappstream.so.5` / `libappstream-compose.so.0` in `appstream` and `appstream-minimal` (projectbluefin/server#57). | Not reported upstream. | As 0003. | +| `0006-linux-kubernetes-cilium-networking.patch` | Builds VXLAN, GENEVE, the tc BPF classifier/action, ingress qdisc, INET_DIAG (+TCP/UDP, DIAG_DESTROY) and `NOTRACK` for Cilium and the kubeadm/k0s sysexts. Written into `fdsdk-config.sh`, so FSDK's expected-config check fails the build if Kconfig drops one. | Not proposed upstream. | FSDK's kernel config enables them. | + +### Hardcoded library versions in 0003-0005 + +These three patches name the full library file each link points at: + +| Patch | Link target | Pinned component (FSDK 26.08.0) | +|---|---|---| +| 0003 | `libgirepository-1.0.so.1.0.0` | gobject-introspection 1.86.0 | +| 0004 | `libgdk_pixbuf-2.0.so.0.4400.7` | gdk-pixbuf 2.44.7 | +| 0005 | `libappstream.so.1.1.6`, `libappstream-compose.so.1.1.6` | appstream 1.1.6 | + +An FSDK point release that bumps one of these components leaves the patch +applying cleanly but its links dangling, and remote builds fail again with +exit 127. On every FSDK bump, compare these names with the component +versions in the new FSDK ref and update the patch in the same change. From 0a11bc9a12b9e0be2fc4c2bd853346e4755da2f5 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 01:11:46 -0500 Subject: [PATCH 4/7] fix(boot): clamp initrd file times to SOURCE_DATE_EPOCH cpio --reproducible fixes inode and device numbers but keeps mtimes, and everything the boot element creates in the initrd tree (symlinks, sysusers output, depmod indexes) carried the build time, so every rebuild gave a different initrd and UKIs. Rebuilds now differ in no initrd byte. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/oci/bluefin-server-boot.bst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elements/oci/bluefin-server-boot.bst b/elements/oci/bluefin-server-boot.bst index 9437a2c3..2650d0a1 100644 --- a/elements/oci/bluefin-server-boot.bst +++ b/elements/oci/bluefin-server-boot.bst @@ -165,6 +165,8 @@ config: rm -rf usr/share/man usr/share/doc usr/share/info usr/include usr/lib/*/pkgconfig find . -name '*.a' -delete du -sh /initrd + # cpio records mtimes; everything created above carries the build time. + find . -mindepth 1 -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + find . -mindepth 1 -printf '%P\n' | LC_ALL=C sort | cpio -o -H newc --reproducible --quiet | zstd -19 -T0 -q > /tmp/initrd.cpio.zst ls -lh /tmp/initrd.cpio.zst From 0ff7c7edff6a19b5873fcaccf0dd9ee3265a7c71 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 01:11:46 -0500 Subject: [PATCH 5/7] fix(boot): sign systemd-boot and the UKIs with systemd-sbsign sbsign puts the current time in each Authenticode signature's signingTime attribute; systemd-sbsign (FSDK systemd, /usr/lib/systemd) takes it from SOURCE_DATE_EPOCH, so a rebuild with the same keys signs identically. sbverify still checks every output. ukify needs --no-sign-kernel with this signtool, which keeps today's behaviour: the embedded kernel was never signed on its own, only the UKI. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/bluefin-server/os-sd-boot-signed.bst | 7 +++++-- elements/oci/bluefin-server-boot.bst | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/elements/bluefin-server/os-sd-boot-signed.bst b/elements/bluefin-server/os-sd-boot-signed.bst index 55e79f25..d81a785b 100644 --- a/elements/bluefin-server/os-sd-boot-signed.bst +++ b/elements/bluefin-server/os-sd-boot-signed.bst @@ -24,6 +24,9 @@ config: set -eu src=%{indep-libdir}/systemd/boot/efi/systemd-boot%{efi-arch}.efi install -d "%{install-root}%{indep-libdir}/systemd/boot/efi" - sbsign --key /boot-keys/DB.key --cert /boot-keys/DB.crt \ - --output "%{install-root}${src}.signed" "${src}" + # systemd-sbsign, unlike sbsign, takes the signing time from + # SOURCE_DATE_EPOCH, so the signature is reproducible. + %{indep-libdir}/systemd/systemd-sbsign sign \ + --private-key=/boot-keys/DB.key --certificate=/boot-keys/DB.crt \ + --output="%{install-root}${src}.signed" "${src}" sbverify --cert /boot-keys/DB.crt "%{install-root}${src}.signed" diff --git a/elements/oci/bluefin-server-boot.bst b/elements/oci/bluefin-server-boot.bst index 2650d0a1..b36e860e 100644 --- a/elements/oci/bluefin-server-boot.bst +++ b/elements/oci/bluefin-server-boot.bst @@ -183,7 +183,8 @@ config: --cmdline=@"/tmp/$1.cmdline" \ --os-release=@/osrel/usr/lib/os-release \ --uname="$(ls /kmods/usr/lib/modules)" \ - --signtool=sbsign \ + --signtool=systemd-sbsign \ + --no-sign-kernel \ --secureboot-private-key=/boot-keys/DB.key \ --secureboot-certificate=/boot-keys/DB.crt \ --output="%{install-root}/$1.efi" From 42e1d75072137ff670b8860b5b632f3b6ceb0134 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 01:11:46 -0500 Subject: [PATCH 6/7] fix(image): build the ESP images reproducibly The staged ESP trees carried the build time into the FAT entries (mcopy keeps mtimes) and mkfs.fat from dosfstools 4.2, which ignores SOURCE_DATE_EPOCH, stamped each volume label with the current time. Clamp the trees before systemd-repart and pass --invariant to mkfs.fat through SYSTEMD_REPART_MKFS_OPTIONS_VFAT (which also fixes the ESP volume ID). Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/oci/bluefin-server-image.bst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/elements/oci/bluefin-server-image.bst b/elements/oci/bluefin-server-image.bst index a39e4275..5344d25a 100644 --- a/elements/oci/bluefin-server-image.bst +++ b/elements/oci/bluefin-server-image.bst @@ -37,6 +37,7 @@ build-depends: - base/base-stack.bst - freedesktop-sdk.bst:public-stacks/runtime-gnu.bst - freedesktop-sdk.bst:components/dosfstools.bst +- freedesktop-sdk.bst:components/findutils.bst - freedesktop-sdk.bst:components/mtools.bst - freedesktop-sdk.bst:components/systemd.bst - filename: oci/bluefin-server-usr.bst @@ -73,6 +74,12 @@ variables: - arch == "aarch64": efi-arch: aa64 +environment: + # dosfstools 4.2 ignores SOURCE_DATE_EPOCH and stamps each ESP's volume + # label with the current time; --invariant fixes the label time and the + # volume ID. + SYSTEMD_REPART_MKFS_OPTIONS_VFAT: --invariant + config: commands: - mkdir -p /tmp /var/tmp @@ -99,6 +106,8 @@ config: printf '[Partition]\nType=usr\nLabel=bluefin_usr_%{image-version}\nUUID=%s\nCopyBlocks=/usr-image/%{image-name}.usr.raw\n' "$(cat /tmp/usr.uuid)" > "${d}/10-usr.conf" printf '[Partition]\nType=usr-verity\nLabel=bluefin_usr_verity_%{image-version}\nUUID=%s\nCopyBlocks=/usr-image/%{image-name}.usr-verity.raw\n' "$(cat /tmp/usr-verity.uuid)" > "${d}/20-usr-verity.conf" printf '[Partition]\nType=esp\nFormat=vfat\nCopyFiles=%s:/\nSizeMinBytes=128M\nSizeMaxBytes=128M\n' "${esp}" > "${d}/30-esp.conf" + # mcopy keeps mtimes; the staged ESP tree carries the build time. + find "${esp}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + systemd-repart --empty=create --size=auto --offline=yes --dry-run=no \ --seed=6f1b0fe2-3c2a-4f2e-9f0a-b1e5e7a5e003 \ --definitions="${d}" "%{install-root}/%{image-name}.raw" @@ -117,6 +126,7 @@ config: d=/tmp/netboot.repart.d mkdir -p "${d}" printf '[Partition]\nType=esp\nFormat=vfat\nCopyFiles=%s:/\nSizeMinBytes=160M\n' "${esp}" > "${d}/10-esp.conf" + find "${esp}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + systemd-repart --empty=create --size=auto --offline=yes --dry-run=no \ --seed=6f1b0fe2-3c2a-4f2e-9f0a-b1e5e7a5e002 \ --definitions="${d}" "%{install-root}/%{netboot-name}.esp.raw" @@ -140,6 +150,7 @@ config: printf '[Partition]\nType=usr\nLabel=bluefin-installer-usr\nCopyBlocks=/usr-image/%{image-name}.usr.raw\n' > "${d}/10-usr.conf" printf '[Partition]\nType=usr-verity\nLabel=bluefin-installer-usr-verity\nCopyBlocks=/usr-image/%{image-name}.usr-verity.raw\n' > "${d}/20-usr-verity.conf" printf '[Partition]\nType=esp\nLabel=bluefin-installer\nFormat=vfat\nCopyFiles=%s:/\nSizeMinBytes=256M\n' "${esp}" > "${d}/30-esp.conf" + find "${esp}" -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + systemd-repart --empty=create --size=auto --offline=yes --dry-run=no \ --seed=6f1b0fe2-3c2a-4f2e-9f0a-b1e5e7a5e004 \ --definitions="${d}" "%{install-root}/bluefin-server-installer_%{image-version}.raw" From f171e8e0743e912cfd7943f3dc833e5e44a2ec24 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 01:11:46 -0500 Subject: [PATCH 7/7] ci: weekly reproducibility check of the image set reproducibility.yml builds the image set, deletes the final-assembly artifacts, rebuilds them without remote caches and diffs every output except *.gpg. efi-keys.bst stays cached: sbvarsign stamps the current time into each signed variable update and has no option to fix it. ddi-installer-build.md records what makes the assembly reproducible. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/reproducibility.yml | 95 +++++++++++++++++++++++++++ docs/skills/ci-tooling.md | 1 + docs/skills/ddi-installer-build.md | 17 +++++ 3 files changed, 113 insertions(+) create mode 100644 .github/workflows/reproducibility.yml diff --git a/.github/workflows/reproducibility.yml b/.github/workflows/reproducibility.yml new file mode 100644 index 00000000..b088fd36 --- /dev/null +++ b/.github/workflows/reproducibility.yml @@ -0,0 +1,95 @@ +name: Reproducibility + +# Builds the image set, deletes the artifacts of the final assembly steps +# (images, UKIs, initrd, sysexts, signed systemd-boot) from the local +# BuildStream cache, rebuilds them without pulling from any remote, and +# compares every output file except the GPG signatures, which embed their +# signing time. Both builds share one checkout, one set of throwaway keys and +# one image-version, so any difference comes from the assembly itself. +# +# bluefin-server/keys/efi-keys.bst is kept: sbvarsign stamps the current time +# into each Secure Boot variable update it signs (PK/KEK/db.auth), the same +# way gpg timestamps SHA256SUMS.gpg, and has no option to fix it. +on: + schedule: + - cron: '0 9 * * 1' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: reproducibility + cancel-in-progress: false + +jobs: + reproducibility: + runs-on: ubuntu-26.04 + timeout-minutes: 300 + env: + FINAL_ASSEMBLY: >- + oci/bluefin-server-image.bst + oci/bluefin-server-boot.bst + oci/bluefin-server-usr.bst + oci/k0s-sysext.bst + oci/kubestellar-sysext.bst + oci/kubeadm-sysext.bst + oci/zfs-sysext.bst + bluefin-server/os-sd-boot-signed.bst + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up just + uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2 + with: + tool: just@1.58.0 + + - name: Set up runner disk space + run: | + set -euo pipefail + sudo mkdir -p /mnt/podman /mnt/buildstream + sudo chown -R "$USER:$USER" /mnt/podman /mnt/buildstream + mkdir -p ~/.config/containers + cat > ~/.config/containers/storage.conf < first.sha256 + rm -rf dist/first + + - name: Rebuild the final assembly + run: | + set -euo pipefail + # shellcheck disable=SC2086 + just bst artifact delete ${FINAL_ASSEMBLY} + just bst build --ignore-project-artifact-remotes oci/bluefin-server-image.bst + just bst artifact checkout oci/bluefin-server-image.bst --directory /src/dist/second + (cd dist/second && find . -type f ! -name '*.gpg' -exec sha256sum {} + | sort -k2) > second.sha256 + + - name: Compare + run: | + set -euo pipefail + cat first.sha256 + if ! diff -u first.sha256 second.sha256; then + echo "::error::The rebuilt image set differs from the first build (files listed above)." + exit 1 + fi + echo "Both builds produced identical files." diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index 4eb28c2e..d88ce746 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -120,6 +120,7 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e | `boot-test` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Downloads the build job's exported image set and runs the Secure Boot QEMU checks: `scripts/dogfood-diskless.sh --check` (diskless boot) and `scripts/dogfood-install.sh` (install to disk and boot it). Read-only token. | | `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`) (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write` + `packages: write`. | | `docs` | `docs-checks.yml` | `pull_request`, `push/main` | Runs markdown and skill metadata checks via `docs-checks.py`. Read-only token. | +| `reproducibility` | `reproducibility.yml` | `schedule` (Mondays 09:00 UTC), `workflow_dispatch` | Builds the image set, deletes the final-assembly artifacts, rebuilds them without remote caches and diffs every output except `*.gpg` (see "Reproducible builds" in `ddi-installer-build.md`). Throwaway keys, nothing published. Read-only token. | | `unit` | `unit-tests.yml` | `pull_request`, `push/main` | Runs pytest and BATS unit test suites. Read-only token. | GitHub Actions runs the **complete BuildStream compilation pipeline** using `/mnt` diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index f65f7214..dcd8f27a 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -79,6 +79,23 @@ image signing key, the new keyring; see "Rotating the Signing Key" in [systemd-sysupdate-verification.md](systemd-sysupdate-verification.md). +## Reproducible builds + +With the same checkout, keys and `image-version`, rebuilding the final +assembly gives the same bytes. BuildStream exports `SOURCE_DATE_EPOCH` into +every sandbox and the assembly steps honor it: `mkfs.erofs` and +`systemd-repart` clamp file times to it, the initrd and ESP trees are clamped +before `cpio` and `mcopy` copy them, and `systemd-sbsign` (not `sbsign`) +uses it as the signing time of systemd-boot and the UKIs. `systemd-repart +--seed` fixes partition UUIDs. + +Two outputs carry a signing time of their own: `SHA256SUMS.gpg` and the +`efi-keys/*.auth` updates, which `sbvarsign` stamps with the current time +(they stay fixed as long as `bluefin-server/keys/efi-keys.bst` stays +cached). `.github/workflows/reproducibility.yml` checks the rest weekly: +it builds, deletes the final-assembly artifacts, rebuilds them without remote +caches, and compares every file except `*.gpg`. + ## Dogfood: boot it in QEMU All dogfood paths boot with Secure Boot firmware (OVMF secboot). The firmware