From 1efc2bc716867d66f81149701816cc11ab1322fd Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 21:54:14 -0500 Subject: [PATCH 1/7] feat(sbom): ship an SPDX 2.3 SBOM in the signed release set oci/bluefin-server-sbom.bst runs FSDK's collect_manifest plugin (output-type: spdx) over the runtime closure of the release payload: the /usr and initrd stacks, the kernel, and the k0s, KubeStellar, kubeadm and OpenZFS sysext payloads. The image element stamps creationInfo.created, drops the identical duplicate packages FSDK bootstrap elements produce, and lists bluefin-server_.spdx.json in SHA256SUMS so the GPG signature covers it. It also exports the keyring the build trusts as sysupdate-keys/import-pubring.pgp (not published, like efi-keys/) so a pull request's release dry run can check SHA256SUMS.gpg against the throwaway key. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/oci/bluefin-server-image.bst | 23 ++++++++++++++- elements/oci/bluefin-server-sbom.bst | 41 +++++++++++++++++++++++++++ project.conf | 2 ++ tests/unit/test_kubeadm_sysext.py | 2 +- 4 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 elements/oci/bluefin-server-sbom.bst diff --git a/elements/oci/bluefin-server-image.bst b/elements/oci/bluefin-server-image.bst index a39e4275..ca45390a 100644 --- a/elements/oci/bluefin-server-image.bst +++ b/elements/oci/bluefin-server-image.bst @@ -26,9 +26,14 @@ description: | zfs_.raw.zst, kubestellar_.raw.zst, kubeadm_.raw.zst opt-in sysexts locked to this image version k0s-.raw.zst opt-in k0s sysext (own version axis) + bluefin-server_.spdx.json SPDX 2.3 SBOM (oci/bluefin-server-sbom.bst) SHA256SUMS, SHA256SUMS.gpg manifest of all of the above, signed with files/boot-keys/sysupdate-signing.asc; the image trusts the matching import-pubring.pgp + sysupdate-keys/import-pubring.pgp that keyring, for the release dry-run to + check SHA256SUMS.gpg against a pull request's + throwaway key; like efi-keys/ it is not + published (releases carry top-level files only) (@): - include/image.yml @@ -49,6 +54,7 @@ build-depends: config: location: /sd-boot - freedesktop-sdk.bst:components/gnupg.bst +- freedesktop-sdk.bst:components/jq.bst - bluefin-server/keys/boot-keys.bst - filename: oci/k0s-sysext.bst config: @@ -62,6 +68,9 @@ build-depends: - filename: oci/kubeadm-sysext.bst config: location: /sysext/kubeadm +- filename: oci/bluefin-server-sbom.bst + config: + location: /sbom variables: image-name: bluefin-server_%{image-version} @@ -150,10 +159,21 @@ config: /sysext/zfs/zfs_%{image-version}.raw.zst \ /sysext/kubeadm/kubeadm_%{image-version}.raw.zst "%{install-root}/" + - | + set -eu + # SPDX requires a creation time, which the plugin leaves to the build, + # and unique SPDXIDs: FSDK bootstrap elements reached through two + # junction paths are collected twice, as identical packages. + jq --arg created "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '.creationInfo.created = $created + | .packages |= unique_by(.SPDXID) + | .relationships |= unique_by(.relatedSpdxElement)' \ + /sbom/bluefin-server.spdx.json > "%{install-root}/%{image-name}.spdx.json" + - | set -eu cd "%{install-root}" - sha256sum --binary *.raw *.efi *.raw.zst > SHA256SUMS + sha256sum --binary *.raw *.efi *.raw.zst *.spdx.json > SHA256SUMS cat SHA256SUMS export GNUPGHOME=/tmp/gnupg mkdir -m 0700 -p "${GNUPGHOME}" @@ -162,3 +182,4 @@ config: --detach-sign --output SHA256SUMS.gpg SHA256SUMS # The keyring the image ships must accept this signature. gpgv --keyring /boot-keys/import-pubring.pgp SHA256SUMS.gpg SHA256SUMS + install -Dm644 /boot-keys/import-pubring.pgp sysupdate-keys/import-pubring.pgp diff --git a/elements/oci/bluefin-server-sbom.bst b/elements/oci/bluefin-server-sbom.bst new file mode 100644 index 00000000..cf7a6c20 --- /dev/null +++ b/elements/oci/bluefin-server-sbom.bst @@ -0,0 +1,41 @@ +kind: collect_manifest +description: | + SPDX 2.3 SBOM of one release set, written by the collect_manifest plugin + that freedesktop-sdk uses for its own CVE manifests + (buildstream-plugins-community, source provenance API). + + One SPDX package per source of every element in the runtime closure of + the dependencies below: name, version (from the git-describe ref or the + tarball name), download location and source kind. Local sources are + listed by path. + + The dependencies are the payload of the release set, not the tools that + assemble it: the /usr and initrd stacks, the kernel, and the sysext + payloads. Build-only elements (signing keys, image tooling) are not + runtime dependencies of these and never appear. + + oci/bluefin-server-image.bst stamps creationInfo.created and ships the + result as bluefin-server_.spdx.json inside the signed SHA256SUMS. + +(@): +- include/image.yml + +build-depends: +- bluefin-server/os-stack.bst +- bluefin-server/initrd/initrd-stack.bst +- freedesktop-sdk.bst:components/linux.bst +- k0s/k0s-bin.bst +- oci/kubestellar-sysext.bst +- kubeadm/kubeadm-bin.bst +- zfs/openzfs.bst + +config: + path: /bluefin-server.spdx.json + output-type: spdx + spdx-info: + name: bluefin-server-%{image-version} + data-license: CC0-1.0 + doc-namespace: https://github.com/projectbluefin/server/spdx/bluefin-server-%{image-version} + creators: + - "Organization: Project Bluefin" + - "Tool: buildstream-plugins-community-collect_manifest" diff --git a/project.conf b/project.conf index 82f7015b..02bf09cf 100644 --- a/project.conf +++ b/project.conf @@ -60,6 +60,8 @@ source-caches: plugins: - origin: junction junction: plugins/buildstream-plugins-community.bst + elements: + - collect_manifest sources: - git_repo - patch_queue diff --git a/tests/unit/test_kubeadm_sysext.py b/tests/unit/test_kubeadm_sysext.py index c92f393d..08b5351a 100644 --- a/tests/unit/test_kubeadm_sysext.py +++ b/tests/unit/test_kubeadm_sysext.py @@ -79,7 +79,7 @@ def test_sysext_is_version_locked_and_in_the_signed_release_set() -> None: image = IMAGE.read_text(encoding="utf-8") assert "filename: oci/kubeadm-sysext.bst" in image assert "/sysext/kubeadm/kubeadm_%{image-version}.raw.zst" in image - assert "sha256sum --binary *.raw *.efi *.raw.zst > SHA256SUMS" in image + assert "sha256sum --binary *.raw *.efi *.raw.zst *.spdx.json > SHA256SUMS" in image assert "oci/kubeadm-sysext.bst" in (ROOT / "Justfile").read_text(encoding="utf-8") From be5d61670a449ba676b54c8397a9e4cf66335035 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 21:54:14 -0500 Subject: [PATCH 2/7] feat(release): add scripts/publish-release.sh One publish path for main and for the pull-request rehearsal: verify (gpgv against a keyring, every SHA256SUMS entry present and matching, every published file listed, file names match the version, SBOM is valid SPDX 2.3), release (gh release create at the built commit, or print it with --dry-run), and oci (oras push, then compare the registry manifest layer by layer with the local files; optionally pull it back). Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/publish-release.sh | 223 +++++++++++++++++++++++++++++ tests/unit/test_publish_release.py | 208 +++++++++++++++++++++++++++ 2 files changed, 431 insertions(+) create mode 100755 scripts/publish-release.sh create mode 100644 tests/unit/test_publish_release.py diff --git a/scripts/publish-release.sh b/scripts/publish-release.sh new file mode 100755 index 00000000..26d199ae --- /dev/null +++ b/scripts/publish-release.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# Publish one Bluefin Server release set (dist/diskless/), or rehearse it. +# +# publish-release.sh verify DIR VERSION KEYRING +# publish-release.sh release DIR VERSION [--dry-run] +# publish-release.sh oci DIR VERSION REF [--plain-http] [--pull-back] +# +# .github/workflows/build.yml runs these same commands in the `release` job +# on main and in the `release-dry-run` job on every pull request. The dry +# run only prints the `gh release create` command, and pushes the OCI +# artifact to a throwaway local registry and pulls it back. +# +# A release publishes the top-level files of DIR: the ones SHA256SUMS lists, +# plus SHA256SUMS and SHA256SUMS.gpg. Subdirectories (efi-keys/, +# sysupdate-keys/) stay local. +set -euo pipefail + +ARTIFACT_TYPE=application/vnd.projectbluefin.server.release.v1 +TITLE_KEY=org.opencontainers.image.title + +die() { echo "publish-release: $*" >&2; exit 1; } + +# Top-level regular files of DIR, sorted: exactly what a release publishes. +published_files() { + find "$1" -mindepth 1 -maxdepth 1 -type f -printf '%f\n' | LC_ALL=C sort +} + +# " " for every published file, from the verified SHA256SUMS +# plus the two files it cannot list itself. +published_digests() { + { + sed -E 's/^([0-9a-f]{64}) [ *]/\1 /' "$1/SHA256SUMS" + (cd "$1" && sha256sum SHA256SUMS SHA256SUMS.gpg) | sed -E 's/ / /' + } | LC_ALL=C sort -k2 +} + +output() { + echo "$1=$2" + if [ -n "${GITHUB_OUTPUT:-}" ]; then + echo "$1=$2" >> "${GITHUB_OUTPUT}" + fi +} + +check_version() { + [[ "$1" =~ ^[0-9][0-9A-Za-z.]{0,16}$ ]] || die "invalid version: $1" +} + +# The file names one image version ships (see the description of +# elements/oci/bluefin-server-image.bst). Each must appear exactly once; +# k0s carries its own version axis. +expected_patterns() { + local v="${1//./\\.}" + local uuid='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' + printf '%s\n' \ + "bluefin-server_${v}\\.raw" \ + "bluefin-server_${v}_${uuid}\\.usr\\.raw" \ + "bluefin-server_${v}_${uuid}\\.usr-verity\\.raw" \ + "bluefin-server-${v}\\.efi" \ + "bluefin-server-netboot_${v}\\.efi" \ + "bluefin-server-netboot_${v}\\.esp\\.raw" \ + "bluefin-server-installer_${v}\\.raw" \ + "bluefin-server_${v}\\.spdx\\.json" \ + "zfs_${v}\\.raw\\.zst" \ + "kubestellar_${v}\\.raw\\.zst" \ + "kubeadm_${v}\\.raw\\.zst" \ + "k0s-[0-9][0-9A-Za-z.+-]*\\.raw\\.zst" +} + +cmd_verify() { + [ $# -eq 3 ] || die "usage: verify DIR VERSION KEYRING" + local dir="$1" ver="$2" keyring + check_version "${ver}" + [ -d "${dir}" ] || die "no such directory: ${dir}" + keyring="$(realpath -e "$3")" || die "no such keyring: $3" + [ -f "${dir}/SHA256SUMS" ] || die "missing SHA256SUMS" + [ -f "${dir}/SHA256SUMS.gpg" ] || die "missing SHA256SUMS.gpg" + + # The same check systemd-sysupdate and systemd-importd make on nodes. + gpgv --keyring "${keyring}" "${dir}/SHA256SUMS.gpg" "${dir}/SHA256SUMS" \ + || die "SHA256SUMS.gpg does not verify against ${keyring}" + + local line name listed=() + while IFS= read -r line; do + [[ "${line}" =~ ^[0-9a-f]{64}\ [\ *]([^/]+)$ ]] \ + || die "malformed SHA256SUMS line: ${line}" + name="${BASH_REMATCH[1]}" + [ -f "${dir}/${name}" ] || die "listed in SHA256SUMS but missing: ${name}" + listed+=("${name}") + done < "${dir}/SHA256SUMS" + [ "${#listed[@]}" -gt 0 ] || die "SHA256SUMS is empty" + + local dup + dup="$(printf '%s\n' "${listed[@]}" | LC_ALL=C sort | uniq -d)" + [ -z "${dup}" ] || die "listed more than once in SHA256SUMS: ${dup}" + + (cd "${dir}" && sha256sum --check --strict --quiet SHA256SUMS) \ + || die "checksum mismatch" + + # Everything a release would publish is covered by the signature. + local unsigned + unsigned="$(LC_ALL=C comm -23 \ + <(published_files "${dir}" | grep -vxE 'SHA256SUMS(\.gpg)?') \ + <(printf '%s\n' "${listed[@]}" | LC_ALL=C sort))" + [ -z "${unsigned}" ] || die "not covered by SHA256SUMS: ${unsigned//$'\n'/ }" + + local pattern count + for name in "${listed[@]}"; do + grep -qxEf <(expected_patterns "${ver}") <<<"${name}" \ + || die "${name} is not a file of release ${ver}" + done + while IFS= read -r pattern; do + count="$(printf '%s\n' "${listed[@]}" | grep -cxE "${pattern}" || true)" + [ "${count}" -eq 1 ] \ + || die "release ${ver} needs exactly one file matching ${pattern}, found ${count}" + done < <(expected_patterns "${ver}") + + # What actions/attest needs to accept it as an SPDX SBOM. + local sbom="${dir}/bluefin-server_${ver}.spdx.json" + jq -e '.spdxVersion == "SPDX-2.3" and (.packages | length > 0) + and ([.packages[].SPDXID] | length == (unique | length)) + and (.creationInfo.created | type == "string")' "${sbom}" >/dev/null \ + || die "$(basename "${sbom}") is not an SPDX 2.3 document with unique packages and a creation time" + + # Attestation subjects: every published file (sha256sum format). + local subjects="${RUNNER_TEMP:-${TMPDIR:-/tmp}}/release-subjects-${ver}.sha256" + published_digests "${dir}" | sed 's/ / /' > "${subjects}" + echo "release set ${ver}: ${#listed[@]} files match SHA256SUMS, signature verified" + output subjects "${subjects}" +} + +cmd_release() { + [ $# -ge 2 ] || die "usage: release DIR VERSION [--dry-run]" + local dir="$1" ver="$2" dry=0 + shift 2 + case "${1:-}" in + --dry-run) dry=1 ;; + "") ;; + *) die "unknown option: $1" ;; + esac + check_version "${ver}" + local sha="${GITHUB_SHA:-$(git rev-parse HEAD)}" + local files=() + mapfile -t files < <(published_files "${dir}" | sed "s|^|${dir%/}/|") + [ "${#files[@]}" -gt 0 ] || die "nothing to publish in ${dir}" + + # A version is published exactly once: gh refuses an existing tag, so + # assets nodes may already trust are never overwritten. + local cmd=(gh release create "v${ver}" --target "${sha}" + --title "Bluefin Server ${ver}" + --notes "Image ${ver} built from ${sha}." + "${files[@]}") + if [ "${dry}" = 1 ]; then + echo "dry run, would run:" + printf '%q ' "${cmd[@]}" + echo + else + "${cmd[@]}" + fi +} + +cmd_oci() { + [ $# -ge 3 ] || die "usage: oci DIR VERSION REF [--plain-http] [--pull-back]" + local dir="$1" ver="$2" ref="$3" pull_back=0 flags=() + shift 3 + while [ $# -gt 0 ]; do + case "$1" in + --plain-http) flags+=(--plain-http) ;; + --pull-back) pull_back=1 ;; + *) die "unknown option: $1" ;; + esac + shift + done + check_version "${ver}" + local source="${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-projectbluefin/server}" + local files=() + mapfile -t files < <(published_files "${dir}") + + (cd "${dir}" && oras push "${flags[@]}" \ + --artifact-type "${ARTIFACT_TYPE}" \ + --annotation "org.opencontainers.image.version=${ver}" \ + --annotation "org.opencontainers.image.source=${source}" \ + --annotation "org.opencontainers.image.revision=${GITHUB_SHA:-$(git rev-parse HEAD)}" \ + "${ref}:${ver},latest" "${files[@]}") + + local digest latest + digest="$(oras resolve "${flags[@]}" "${ref}:${ver}")" + latest="$(oras resolve "${flags[@]}" "${ref}:latest")" + [ "${digest}" = "${latest}" ] || die "${ref}:latest is ${latest}, not ${digest}" + + # The registry holds exactly the published files, byte for byte. + diff -u <(published_digests "${dir}") \ + <(oras manifest fetch "${flags[@]}" "${ref}@${digest}" \ + | jq -r --arg k "${TITLE_KEY}" '.layers[] | "\(.digest | ltrimstr("sha256:")) \(.annotations[$k])"' \ + | LC_ALL=C sort -k2) \ + || die "${ref}@${digest} does not match ${dir}" + + if [ "${pull_back}" = 1 ]; then + local tmp + tmp="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '${tmp}'" EXIT + oras pull "${flags[@]}" -o "${tmp}" "${ref}@${digest}" + diff -u <(published_files "${dir}") <(published_files "${tmp}") \ + || die "pulled file list differs" + (cd "${tmp}" && sha256sum --check --strict --quiet SHA256SUMS) \ + || die "pulled files do not match SHA256SUMS" + cmp "${dir}/SHA256SUMS.gpg" "${tmp}/SHA256SUMS.gpg" \ + || die "pulled SHA256SUMS.gpg differs" + echo "pulled ${ref}@${digest} back: ${#files[@]} files match" + fi + + output name "${ref}" + output digest "${digest}" +} + +sub="${1:-}" +[ $# -gt 0 ] && shift +case "${sub}" in + verify) cmd_verify "$@" ;; + release) cmd_release "$@" ;; + oci) cmd_oci "$@" ;; + *) die "usage: $0 verify|release|oci ..." ;; +esac diff --git a/tests/unit/test_publish_release.py b/tests/unit/test_publish_release.py new file mode 100644 index 00000000..7995f10c --- /dev/null +++ b/tests/unit/test_publish_release.py @@ -0,0 +1,208 @@ +"""Behavior of scripts/publish-release.sh, the one publish path build.yml runs +on main (release) and on pull requests (release-dry-run).""" + +from __future__ import annotations + +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "publish-release.sh" +VERSION = "26.09.7" +UUID_A = "d3107d37-a9da-32cf-3c19-49fa3b0cb1df" +UUID_B = "7c6a26c3-2990-8ab6-966f-1e38b85bcd4e" +SPDX = { + "spdxVersion": "SPDX-2.3", + "creationInfo": {"created": "2026-09-28T00:00:00Z"}, + "packages": [{"SPDXID": "SPDXRef-a-0", "name": "systemd", "versionInfo": "260"}], +} + +pytestmark = pytest.mark.skipif(shutil.which("gpg") is None, reason="needs gpg") + + +def release_files(version: str) -> list[str]: + return [ + f"bluefin-server_{version}.raw", + f"bluefin-server_{version}_{UUID_A}.usr.raw", + f"bluefin-server_{version}_{UUID_B}.usr-verity.raw", + f"bluefin-server-{version}.efi", + f"bluefin-server-netboot_{version}.efi", + f"bluefin-server-netboot_{version}.esp.raw", + f"bluefin-server-installer_{version}.raw", + f"bluefin-server_{version}.spdx.json", + f"zfs_{version}.raw.zst", + f"kubestellar_{version}.raw.zst", + f"kubeadm_{version}.raw.zst", + "k0s-1.36.4-k0s.0.raw.zst", + ] + + +class Signer: + def __init__(self, home: Path) -> None: + self.env = {**os.environ, "GNUPGHOME": str(home)} + home.chmod(0o700) + self.gpg("--quick-gen-key", "Test Release ", "ed25519", "sign", "never") + self.keyring = home / "import-pubring.pgp" + self.gpg("--output", str(self.keyring), "--export", "release@example.invalid") + + def gpg(self, *args: str) -> None: + subprocess.run( + ["gpg", "--batch", "--quiet", "--yes", "--pinentry-mode", "loopback", "--passphrase", "", *args], + env=self.env, + check=True, + capture_output=True, + ) + + def sign(self, sums: Path) -> None: + self.gpg("--detach-sign", "--output", f"{sums}.gpg", str(sums)) + + def close(self) -> None: + subprocess.run(["gpgconf", "--kill", "all"], env=self.env, check=False, capture_output=True) + + +@pytest.fixture(scope="module") +def signers(): + homes = [Path(tempfile.mkdtemp(prefix="gpg", dir="/tmp")) for _ in range(2)] + made = [Signer(home) for home in homes] + yield made + for signer, home in zip(made, homes): + signer.close() + shutil.rmtree(home, ignore_errors=True) + + +def write_sums(release: Path, names: list[str]) -> None: + lines = [f"{hashlib.sha256((release / n).read_bytes()).hexdigest()} *{n}\n" for n in names] + (release / "SHA256SUMS").write_text("".join(lines)) + + +@pytest.fixture +def release(tmp_path: Path, signers) -> Path: + out = tmp_path / "diskless" + out.mkdir() + names = release_files(VERSION) + for name in names: + (out / name).write_text(f"content of {name}\n") + (out / f"bluefin-server_{VERSION}.spdx.json").write_text(json.dumps(SPDX)) + (out / "efi-keys").mkdir() + (out / "efi-keys" / "db.auth").write_text("not published\n") + write_sums(out, names) + signers[0].sign(out / "SHA256SUMS") + return out + + +def run(*args: str, cwd: Path | None = None, tmp: Path | None = None) -> subprocess.CompletedProcess[str]: + env = {**os.environ, "GITHUB_SHA": "0" * 40} + env.pop("GITHUB_OUTPUT", None) + if tmp is not None: + env["RUNNER_TEMP"] = str(tmp) + return subprocess.run(["bash", str(SCRIPT), *args], cwd=cwd, env=env, capture_output=True, text=True) + + +def verify(release: Path, keyring: Path, version: str = VERSION) -> subprocess.CompletedProcess[str]: + return run("verify", str(release), version, str(keyring), tmp=release.parent) + + +def test_complete_release_set_verifies(release: Path, signers) -> None: + result = verify(release, signers[0].keyring) + assert result.returncode == 0, result.stderr + assert f"release set {VERSION}: 12 files match SHA256SUMS, signature verified" in result.stdout + subjects = release.parent / f"release-subjects-{VERSION}.sha256" + assert f"subjects={subjects}" in result.stdout + names = sorted(line.split(" ", 1)[1] for line in subjects.read_text().splitlines()) + assert names == sorted([*release_files(VERSION), "SHA256SUMS", "SHA256SUMS.gpg"]) + + +def test_missing_file_is_refused(release: Path, signers) -> None: + (release / f"zfs_{VERSION}.raw.zst").unlink() + result = verify(release, signers[0].keyring) + assert result.returncode != 0 + assert f"listed in SHA256SUMS but missing: zfs_{VERSION}.raw.zst" in result.stderr + + +def test_bad_checksum_is_refused(release: Path, signers) -> None: + (release / f"bluefin-server_{VERSION}.raw").write_text("tampered\n") + result = verify(release, signers[0].keyring) + assert result.returncode != 0 + assert "checksum mismatch" in result.stderr + + +def test_rehashed_unsigned_manifest_is_refused(release: Path, signers) -> None: + (release / f"bluefin-server_{VERSION}.raw").write_text("tampered\n") + write_sums(release, release_files(VERSION)) + result = verify(release, signers[0].keyring) + assert result.returncode != 0 + assert "SHA256SUMS.gpg does not verify" in result.stderr + + +def test_signature_from_another_key_is_refused(release: Path, signers) -> None: + signers[1].sign(release / "SHA256SUMS") + result = verify(release, signers[0].keyring) + assert result.returncode != 0 + assert "SHA256SUMS.gpg does not verify" in result.stderr + + +def test_version_mismatch_is_refused(release: Path, signers) -> None: + result = verify(release, signers[0].keyring, version="26.09.8") + assert result.returncode != 0 + assert "is not a file of release 26.09.8" in result.stderr + + +def test_file_of_another_version_is_refused(release: Path, signers) -> None: + names = release_files(VERSION) + old = f"kubeadm_{VERSION}.raw.zst" + new = "kubeadm_26.09.6.raw.zst" + (release / old).rename(release / new) + write_sums(release, [new if n == old else n for n in names]) + signers[0].sign(release / "SHA256SUMS") + result = verify(release, signers[0].keyring) + assert result.returncode != 0 + assert f"{new} is not a file of release {VERSION}" in result.stderr + + +def test_missing_artifact_kind_is_refused(release: Path, signers) -> None: + names = [n for n in release_files(VERSION) if not n.endswith(".spdx.json")] + (release / f"bluefin-server_{VERSION}.spdx.json").unlink() + write_sums(release, names) + signers[0].sign(release / "SHA256SUMS") + result = verify(release, signers[0].keyring) + assert result.returncode != 0 + assert "needs exactly one file matching bluefin-server_26\\.09\\.7\\.spdx\\.json, found 0" in result.stderr + + +@pytest.mark.parametrize( + "broken", + [{"creationInfo": {}}, {"packages": [*SPDX["packages"], *SPDX["packages"]]}, {"spdxVersion": "SPDX-2.2"}], + ids=["no-created", "duplicate-spdxid", "wrong-version"], +) +def test_invalid_sbom_is_refused(release: Path, signers, broken: dict) -> None: + sbom = release / f"bluefin-server_{VERSION}.spdx.json" + sbom.write_text(json.dumps({**SPDX, **broken})) + write_sums(release, release_files(VERSION)) + signers[0].sign(release / "SHA256SUMS") + result = verify(release, signers[0].keyring) + assert result.returncode != 0 + assert "is not an SPDX 2.3 document" in result.stderr + + +def test_unsigned_top_level_file_is_refused(release: Path, signers) -> None: + (release / "dogfood-serial.log").write_text("local leftovers\n") + result = verify(release, signers[0].keyring) + assert result.returncode != 0 + assert "not covered by SHA256SUMS: dogfood-serial.log" in result.stderr + + +def test_dry_run_renders_gh_release_for_top_level_files_only(release: Path) -> None: + result = run("release", str(release), VERSION, "--dry-run") + assert result.returncode == 0, result.stderr + command = result.stdout.splitlines()[-1] + assert command.startswith(f"gh release create v{VERSION} --target {'0' * 40} ") + for name in [*release_files(VERSION), "SHA256SUMS", "SHA256SUMS.gpg"]: + assert f"{release}/{name}" in command + assert "efi-keys" not in command From 5a40cb24b2788f96a93fb328a4a82d2675535b4f Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 21:54:14 -0500 Subject: [PATCH 3/7] ci(release): dry-run the publish path on pull requests; attest provenance and SBOM The release job now calls scripts/publish-release.sh for every publish step and verifies the set against the committed release keyring first. It attests SLSA build provenance and the SPDX SBOM for every published file and for the OCI artifact digest (push-to-registry, so the Sigstore bundles live next to the artifact as referrers). release-dry-run runs the same verify, release --dry-run and oci commands on every pull request, pushing to a digest-pinned registry service container and pulling the artifact back. Read-only token, no secrets. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 123 +++++++++++++++++++++---- tests/unit/test_release_workflow.py | 66 +++++++++++++ tests/unit/test_sysupdate_transfers.py | 4 +- 3 files changed, 172 insertions(+), 21 deletions(-) create mode 100644 tests/unit/test_release_workflow.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5bcf73f0..ffdf9212 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -148,9 +148,12 @@ jobs: run: bash scripts/dogfood-install.sh dist/diskless # Publishes the signed image set as a new, immutable GitHub Release and as - # an ORAS OCI artifact (one layer per file, tags and latest). - # Only runs on pushes/dispatches to main, where the code is already - # reviewed, and is the only job in this workflow that holds write tokens. + # an ORAS OCI artifact (one layer per file, tags and latest), each + # with SLSA build provenance and SBOM attestations. Only runs on + # pushes/dispatches to main, where the code is already reviewed, and is the + # only job in this workflow that holds write tokens. Every publish command + # is scripts/publish-release.sh, which release-dry-run rehearses on each + # pull request. release: if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }} needs: [build, boot-test] @@ -158,26 +161,45 @@ jobs: timeout-minutes: 45 permissions: contents: write # creates the GitHub Release and uploads its assets - packages: write # pushes the OCI artifact to ghcr.io + packages: write # pushes the OCI artifact and its attestations to ghcr.io + id-token: write # Sigstore signing certificate for the attestations + attestations: write # stores the attestations with the repository + artifact-metadata: write # storage record for the attested ghcr.io artifact + env: + VERSION: ${{ needs.build.outputs.version }} steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Download image artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: image path: dist/diskless - # A version is published exactly once: creating an existing tag fails - # the job instead of overwriting assets that nodes may already trust. + # Against the committed release keyring, not the build's own copy. + - name: Verify the release set + id: verify + run: bash scripts/publish-release.sh verify dist/diskless "${VERSION}" files/os/sysupdate-keys/import-pubring.gpg + + - name: Attest build provenance of the release files + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-checksums: ${{ steps.verify.outputs.subjects }} + + - name: Attest the SBOM of the release files + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-checksums: ${{ steps.verify.outputs.subjects }} + sbom-path: dist/diskless/bluefin-server_${{ env.VERSION }}.spdx.json + - name: Create GitHub Release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} - run: | - set -euo pipefail - V="${{ needs.build.outputs.version }}" - find dist/diskless -maxdepth 1 -type f -print0 | xargs -0 gh release create "v${V}" \ - --title "Bluefin Server ${V}" \ - --notes "Image ${V} built from ${GITHUB_SHA}." + run: bash scripts/publish-release.sh release dist/diskless "${VERSION}" # setup-oras v2.0.1's built-in release table stops at ORAS 1.3.3, so pin # 1.3.4 (matching the Justfile's oras_image) by URL and checksum. @@ -188,16 +210,77 @@ jobs: checksum: f27adb935022d94df8dc77719c322dda592c78a0d57a6f7dcdd8d900b248c454 - name: Push OCI artifact + id: oci env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail - V="${{ needs.build.outputs.version }}" - REF="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/bluefin-server" echo "${GITHUB_TOKEN}" | oras login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin - cd dist/diskless - mapfile -t files < <(find . -maxdepth 1 -type f -printf '%f\n' | sort) - oras push --artifact-type application/vnd.projectbluefin.server.release.v1 \ - --annotation "org.opencontainers.image.version=${V}" \ - --annotation "org.opencontainers.image.source=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ - "${REF}:${V},latest" "${files[@]}" + bash scripts/publish-release.sh oci dist/diskless "${VERSION}" "ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/bluefin-server" + + # push-to-registry stores the signed attestation next to the artifact + # as an OCI referrer, verifiable with gh attestation or cosign. + - name: Attest build provenance of the OCI artifact + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.oci.outputs.name }} + subject-digest: ${{ steps.oci.outputs.digest }} + push-to-registry: true + + - name: Attest the SBOM of the OCI artifact + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ steps.oci.outputs.name }} + subject-digest: ${{ steps.oci.outputs.digest }} + sbom-path: dist/diskless/bluefin-server_${{ env.VERSION }}.spdx.json + push-to-registry: true + + # Rehearses `release` on every pull request with the same + # scripts/publish-release.sh commands: verifies the release set against the + # keyring the build trusted (a throwaway key on pull requests), renders the + # gh release command without running it, and pushes the OCI artifact to a + # local registry and pulls it back. No secrets, read-only token. + release-dry-run: + if: ${{ github.event_name == 'pull_request' }} + needs: build + runs-on: ubuntu-26.04 + timeout-minutes: 45 + permissions: + contents: read + services: + registry: + image: docker.io/library/registry:3.1.2@sha256:c87f33837722a100572e95d7dc4bf539fc42cf68202b13c3bc03c0ff54c3a649 + ports: + - 5000:5000 + env: + VERSION: ${{ needs.build.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.ref || github.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + persist-credentials: false + + - name: Download image artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: image + path: dist/diskless + + - name: Verify the release set + run: bash scripts/publish-release.sh verify dist/diskless "${VERSION}" dist/diskless/sysupdate-keys/import-pubring.pgp + + - name: Render the GitHub Release + run: bash scripts/publish-release.sh release dist/diskless "${VERSION}" --dry-run + + # setup-oras v2.0.1's built-in release table stops at ORAS 1.3.3, so pin + # 1.3.4 (matching the Justfile's oras_image) by URL and checksum. + - name: Set up oras + uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + with: + url: https://github.com/oras-project/oras/releases/download/v1.3.4/oras_1.3.4_linux_amd64.tar.gz + checksum: f27adb935022d94df8dc77719c322dda592c78a0d57a6f7dcdd8d900b248c454 + + - name: Push the OCI artifact to a local registry and pull it back + run: bash scripts/publish-release.sh oci dist/diskless "${VERSION}" localhost:5000/bluefin-server --plain-http --pull-back diff --git a/tests/unit/test_release_workflow.py b/tests/unit/test_release_workflow.py new file mode 100644 index 00000000..32472e73 --- /dev/null +++ b/tests/unit/test_release_workflow.py @@ -0,0 +1,66 @@ +"""The release job and its pull-request rehearsal run the same publish path.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +JOBS = yaml.safe_load((ROOT / ".github" / "workflows" / "build.yml").read_text(encoding="utf-8"))["jobs"] +RELEASE = JOBS["release"] +DRY_RUN = JOBS["release-dry-run"] + + +def steps(job: dict, uses_prefix: str) -> list[dict]: + return [s for s in job["steps"] if s.get("uses", "").startswith(uses_prefix)] + + +def publish_commands(job: dict) -> list[str]: + commands = [] + for step in job["steps"]: + for line in step.get("run", "").splitlines(): + if "scripts/publish-release.sh" in line: + commands.append(line.split("scripts/publish-release.sh", 1)[1].split()[0]) + return commands + + +def test_both_jobs_publish_through_the_script_only() -> None: + assert publish_commands(RELEASE) == ["verify", "release", "oci"] + assert publish_commands(DRY_RUN) == ["verify", "release", "oci"] + for job in (RELEASE, DRY_RUN): + runs = "\n".join(s.get("run", "") for s in job["steps"]) + assert "oras push" not in runs + assert "gh release create" not in runs + + +def test_oras_is_pinned_identically_in_both_jobs() -> None: + pins = [json.dumps({"uses": s["uses"], "with": s["with"]}, sort_keys=True) for job in (RELEASE, DRY_RUN) for s in steps(job, "oras-project/setup-oras@")] + assert len(pins) == 2 + assert pins[0] == pins[1] + assert "checksum" in json.loads(pins[0])["with"] + + +def test_dry_run_is_read_only_and_secret_free() -> None: + assert DRY_RUN["if"] == "${{ github.event_name == 'pull_request' }}" + assert DRY_RUN["permissions"] == {"contents": "read"} + assert "secrets." not in json.dumps(DRY_RUN) + assert "@sha256:" in DRY_RUN["services"]["registry"]["image"] + oci = [s["run"] for s in DRY_RUN["steps"] if "publish-release.sh oci" in s.get("run", "")] + assert oci and "--plain-http" in oci[0] and "--pull-back" in oci[0] + render = [s["run"] for s in DRY_RUN["steps"] if "publish-release.sh release" in s.get("run", "")] + assert render and render[0].rstrip().endswith("--dry-run") + + +def test_release_attests_files_and_oci_artifact() -> None: + attest = steps(RELEASE, "actions/attest@") + assert len({s["uses"] for s in attest}) == 1 + subjects = [s["with"] for s in attest] + assert sum("subject-checksums" in w and "sbom-path" not in w for w in subjects) == 1 + assert sum("subject-checksums" in w and "sbom-path" in w for w in subjects) == 1 + oci = [w for w in subjects if "subject-digest" in w] + assert len(oci) == 2 and all(w["push-to-registry"] is True for w in oci) + assert RELEASE["permissions"]["id-token"] == "write" + assert RELEASE["permissions"]["attestations"] == "write" + assert not any("id-token" in (job.get("permissions") or {}) for name, job in JOBS.items() if name != "release") diff --git a/tests/unit/test_sysupdate_transfers.py b/tests/unit/test_sysupdate_transfers.py index 57fe9866..a83ee08c 100644 --- a/tests/unit/test_sysupdate_transfers.py +++ b/tests/unit/test_sysupdate_transfers.py @@ -261,7 +261,9 @@ def test_every_source_artifact_is_in_the_signed_image_set(path: Path): image = IMAGE_ELEMENT.read_text() assert prefix in image, f"{path.name}: {prefix!r} assets are not in {IMAGE_ELEMENT.name}" build_yml = (REPO_ROOT / ".github" / "workflows" / "build.yml").read_text() - assert "find dist/diskless -maxdepth 1 -type f" in build_yml + assert "scripts/publish-release.sh release dist/diskless" in build_yml + publish = (REPO_ROOT / "scripts" / "publish-release.sh").read_text() + assert "-maxdepth 1 -type f" in publish assert "gpg --batch --yes --pinentry-mode loopback" in image assert "gpgv --keyring /boot-keys/import-pubring.pgp SHA256SUMS.gpg SHA256SUMS" in image From e31529ba559e151d425e7d4c6019fc5a15534662 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 21:54:14 -0500 Subject: [PATCH 4/7] docs: document release provenance, SBOM and verification systemd-sysupdate-verification.md owns the trust story and the gh attestation / cosign verify commands, replacing the dangling signing-and-sbom reference; ci-tooling.md lists the release-dry-run job and the release job's new permissions. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 3 +- README.md | 1 + docs/skills/ci-tooling.md | 23 +++++-- docs/skills/systemd-sysupdate-verification.md | 66 ++++++++++++++++++- 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4d24050c..951d5dad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,9 +7,10 @@ Bluefin Server is an image-based Linux server OS composed from freedesktop-sdk ( - a netboot ESP image with signed systemd-boot and Secure Boot key enrollment payloads - an offline USB installer `bluefin-server-installer_.raw` (usr + usr-verity + ESP with systemd-boot, the installer UKI, the disk UKI and `repart.d`) that boots into `systemd-sysinstall` - optional opt-in `systemd-sysext` images: `oci/k0s-sysext.bst` (controller, or worker when `/etc/k0s/token` exists), `oci/kubestellar-sysext.bst` (Argo CD, KubeStellar, kiosk; needs k0s), `oci/kubeadm-sysext.bst` (kubeadm worker: kubelet, containerd) and `oci/zfs-sysext.bst` +- an SPDX 2.3 SBOM `bluefin-server_.spdx.json` (`oci/bluefin-server-sbom.bst`) - a `SHA256SUMS` over the whole set, signed in-element (`SHA256SUMS.gpg`); nodes verify it against `/etc/systemd/import-pubring.pgp` -A release publishes `dist/diskless/` as-is: a GitHub Release `v` and an ORAS OCI artifact `ghcr.io//bluefin-server:,latest`. +A release publishes `dist/diskless/` as-is: a GitHub Release `v` and an ORAS OCI artifact `ghcr.io//bluefin-server:,latest`, both with provenance and SBOM attestations; pull requests rehearse it in `release-dry-run`. ## What agents should know first diff --git a/README.md b/README.md index 6d593ab5..3cd9d4ad 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ See [`CONTRIBUTING.md`](CONTRIBUTING.md) for the contributor checklist, Conventi - **Signed boot chain**: Secure Boot keys enroll from the ESP on first boot (`secure-boot-enroll if-safe` in VMs, or manually via systemd-boot menu in firmware Setup Mode on bare metal); local builds use throwaway keys from `just gen-dev-keys`. - **Signed manifests**: the build signs one combined `SHA256SUMS` over the whole image set (OS images, UKIs, sysexts) inside `oci/bluefin-server-image.bst`; a release publishes `dist/diskless/` as-is to GitHub Releases and as an OCI artifact. - **Sysupdate verification**: installed nodes verify updates against the signed manifest (`Verify=yes`), and the diskless pull checks the same signature in the initrd; see [`docs/skills/systemd-sysupdate-verification.md`](docs/skills/systemd-sysupdate-verification.md) for details. +- **Provenance and SBOM**: releases carry SLSA provenance and SPDX SBOM attestations; see [`docs/skills/systemd-sysupdate-verification.md`](docs/skills/systemd-sysupdate-verification.md) for verification. - **Vulnerability disclosure**: See [`SECURITY.md`](SECURITY.md) for policy details and how to report security issues. ## License diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index 4eb28c2e..a4775331 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -4,7 +4,7 @@ description: CI workflow conventions for Bluefin Server. Use when writing or edi metadata: type: reference status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" context7-sources: - /websites/github_en_actions - /websites/cli_github_manual @@ -72,8 +72,13 @@ may hold a write token. In `build.yml`, write tokens are granted to exactly one job: - `release` — creates the GitHub Release and pushes the OCI artifact; gated to - `refs/heads/main`. It holds `contents: write` (release) and - `packages: write` (ghcr.io push). + `refs/heads/main`. It holds `contents: write` (release), + `packages: write` (ghcr.io push and attestation referrers), and + `id-token: write`, `attestations: write` and `artifact-metadata: write` + for the provenance and SBOM attestations. + +Its pull-request rehearsal, `release-dry-run`, keeps the read-only default +and uses no secrets. Junction ref tracking must never run on `pull_request`. It used to, as a `track-refs` job gated on `startsWith(github.head_ref, 'renovate/')`, and a @@ -118,7 +123,8 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e | `track-junctions` | `track-junctions.yml` | `schedule` (08:00 UTC), `workflow_dispatch` | Resolves the `freedesktop-sdk.bst` + `gnome-build-meta.bst` junction refs, syncs `project.conf`'s `installer-version`, and opens/updates its own PR on `auto/track-junctions`. `contents: write` + `pull-requests: write`, never on `pull_request`. | | `build` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Resolves the element graph, sets `image-version`, and runs the full BuildStream compile of the image set (OS DDI, signed UKIs, netboot ESP, k0s/KubeStellar/OpenZFS sysext assets), which also writes and signs the combined `SHA256SUMS` inside `oci/bluefin-server-image.bst`. On `main` it installs the `BOOT_KEYS_TARBALL` and `SYSUPDATE_SIGNING_KEY` secrets; both are required there. Read-only token. | | `boot-test` | `build.yml` | `pull_request`, `push/main`, `workflow_dispatch` | Downloads the build job's exported image set and runs the Secure Boot QEMU checks: `scripts/dogfood-diskless.sh --check` (diskless boot) and `scripts/dogfood-install.sh` (install to disk and boot it). Read-only token. | -| `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`) (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write` + `packages: write`. | +| `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is through `scripts/publish-release.sh`: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`), with provenance and SBOM attestations for both (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). Write permissions listed above. | +| `release-dry-run` | `build.yml` | `pull_request` | Runs the same `scripts/publish-release.sh` commands against the PR's image set: verify, render `gh release create`, and a real `oras push` to a `registry` service container (pinned by digest) that it pulls back. Read-only token, no secrets. | | `docs` | `docs-checks.yml` | `pull_request`, `push/main` | Runs markdown and skill metadata checks via `docs-checks.py`. Read-only token. | | `unit` | `unit-tests.yml` | `pull_request`, `push/main` | Runs pytest and BATS unit test suites. Read-only token. | @@ -151,7 +157,11 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). merges), GitHub Actions publishes `dist/diskless/` as-is: an immutable GitHub Release `v` and an ORAS OCI artifact `ghcr.io//bluefin-server:,latest`. Nodes verify updates - against the `SHA256SUMS` / `SHA256SUMS.gpg` already in that set. + against the `SHA256SUMS` / `SHA256SUMS.gpg` already in that set. Every + pull request rehearses this path in `release-dry-run`, so the publish + code is exercised before it first runs on main. Attestations, the SBOM + and the verify commands are in + [`systemd-sysupdate-verification.md`](systemd-sysupdate-verification.md). ## Common Rationalizations @@ -179,6 +189,9 @@ uploaded to a GitHub Release tagged `v` (`YY.MM.` on main). - [ ] Release signing happens in `oci/bluefin-server-image.bst`; there is no separate CI signing step, and the release job publishes `dist/diskless/` as-is (GitHub Release + OCI artifact). +- [ ] Publish logic lives in `scripts/publish-release.sh`, not inline in the + workflow; `tests/unit/test_release_workflow.py` keeps `release` and + `release-dry-run` on the same commands and the same `setup-oras` pin. - [ ] The signing secret names (`BOOT_KEYS_TARBALL`, `SYSUPDATE_SIGNING_KEY`) match the ones documented in `docs/skills/systemd-sysupdate-verification.md`. diff --git a/docs/skills/systemd-sysupdate-verification.md b/docs/skills/systemd-sysupdate-verification.md index db982f08..7b1ecde0 100644 --- a/docs/skills/systemd-sysupdate-verification.md +++ b/docs/skills/systemd-sysupdate-verification.md @@ -4,7 +4,7 @@ description: Configure and operate GPG signature verification for Bluefin Server metadata: type: reference status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" context7-sources: - /systemd/systemd --- @@ -22,11 +22,12 @@ the OS image. - Rotating or replacing the image signing key. - Debugging `systemd-sysupdate` or diskless `rd.systemd.pull` failures related to `SHA256SUMS.gpg` verification. +- Verifying a published release: provenance attestations and the SBOM. ## When NOT to Use - General BuildStream element or dependency questions (use `avoid-over-engineering` or `ddi-installer`). -- SBOM or container-image signing questions (use `signing-and-sbom`). +- CI job wiring, permissions and action pins (use `ci-tooling`). ## How It Works @@ -91,6 +92,63 @@ keyring (`bluefin-server/initrd/initrd-stack.bst` depends on DDI and a re-hashed, unsigned manifest alike (`DOGFOOD_TAMPER=raw|sums` proves both). +## Provenance, SBOM and publishing + +The GPG signature is what nodes check. On top of it, every release carries +GitHub artifact attestations (SLSA build provenance, signed keylessly through +Sigstore with the `build.yml` workflow identity on `refs/heads/main`) that +tell a human or a policy engine which commit and workflow run produced it. + +- **SBOM.** `oci/bluefin-server-sbom.bst` runs FSDK's `collect_manifest` + plugin (`output-type: spdx`) over the payload of the release set: the /usr + and initrd stacks, the kernel, and the k0s, KubeStellar, kubeadm and + OpenZFS sysext payloads. It lists one SPDX package per source (name, + version, download URL, source kind). Build-only elements such as the + signing keys are not runtime dependencies of that payload and never + appear. The image element stamps `creationInfo.created` and ships it as + `bluefin-server_.spdx.json`, listed in `SHA256SUMS`, so the GPG + signature covers it too. No transfer matches it, so nodes never download it. +- **Publishing.** `scripts/publish-release.sh` is the only publish path. + `verify` checks what nodes will check (`gpgv` against the keyring, every + `SHA256SUMS` entry present and matching), plus: every published file is + listed, every file name carries the release version, and the SBOM is valid + SPDX 2.3. `release` creates the GitHub Release at the built commit. `oci` + pushes the ORAS artifact, then checks the registry manifest layer by layer + against the local files. On main the `release` job verifies against the + committed `files/os/sysupdate-keys/import-pubring.gpg`. On pull requests + `release-dry-run` verifies against the throwaway keyring the build + exported in `sysupdate-keys/`, prints the `gh release create` command, and + pushes to a local registry and pulls the artifact back. +- **Attestations.** The `release` job runs `actions/attest` four times: + provenance and SBOM for every published file (subjects are the verified + checksums), and provenance and SBOM for the OCI artifact digest with + `push-to-registry: true`, which stores the Sigstore bundles next to the + artifact as OCI referrers. That bundle is the registry signature. A + separate `cosign sign` would only repeat the same workflow identity, so + there is none. + +Verify a downloaded file, or the OCI artifact, with the GitHub CLI: + +```bash +gh attestation verify bluefin-server_.raw --repo projectbluefin/server \ + --signer-workflow projectbluefin/server/.github/workflows/build.yml --source-ref refs/heads/main +gh attestation verify bluefin-server_.raw --repo projectbluefin/server \ + --predicate-type https://spdx.dev/Document/v2.3 +gh attestation verify oci://ghcr.io/projectbluefin/bluefin-server: \ + --repo projectbluefin/server --bundle-from-oci +``` + +Or with cosign, against the referrers stored in the registry: + +```bash +cosign verify-attestation --new-bundle-format --type slsaprovenance1 \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com \ + --certificate-identity 'https://github.com/projectbluefin/server/.github/workflows/build.yml@refs/heads/main' \ + ghcr.io/projectbluefin/bluefin-server@sha256: +``` + +Use `--type spdxjson` to get the SBOM attestation instead. + ## Repository Layout - `files/os/sysupdate-keys/import-pubring.gpg` — the release public keyring, @@ -174,6 +232,10 @@ on its own; no manual gpg step is needed. proves it with `gpgv` against the keyring the image ships. - [ ] CI publishes `dist/diskless/` as-is to the GitHub Release and to the OCI artifact; there is no separate signing step. +- [ ] Every publish command in `build.yml` goes through + `scripts/publish-release.sh`, and `release-dry-run` runs the same + `verify`, `release --dry-run` and `oci` commands on pull requests. +- [ ] `bluefin-server_.spdx.json` is listed in `SHA256SUMS`. - [ ] Every transfer in `files/os/sysupdate.d/*.transfer` and the k0s component directory uses a static `Path=` with no `@v` placeholder. - [ ] Every transfer uses `@v` only inside `MatchPattern=`. From ffa818c40416c341b858b5f2137020b90c8da9ba Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 01:21:13 -0500 Subject: [PATCH 5/7] fix(sbom): stamp the SBOM with SOURCE_DATE_EPOCH, not the build time A wall-clock creation time made every rebuild of the release set differ and defeats the reproducibility check proposed in #281. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/oci/bluefin-server-image.bst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elements/oci/bluefin-server-image.bst b/elements/oci/bluefin-server-image.bst index ca45390a..aa3eada4 100644 --- a/elements/oci/bluefin-server-image.bst +++ b/elements/oci/bluefin-server-image.bst @@ -164,7 +164,7 @@ config: # SPDX requires a creation time, which the plugin leaves to the build, # and unique SPDXIDs: FSDK bootstrap elements reached through two # junction paths are collected twice, as identical packages. - jq --arg created "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + jq --arg created "$(date -u -d "@${SOURCE_DATE_EPOCH}" +%Y-%m-%dT%H:%M:%SZ)" \ '.creationInfo.created = $created | .packages |= unique_by(.SPDXID) | .relationships |= unique_by(.relatedSpdxElement)' \ From 8c2ff71cf17a631ccc0492d514e63b53b52e075d Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 02:01:19 -0500 Subject: [PATCH 6/7] fix(sbom): list the k0s, kubeadm and ZFS sysext elements The SBOM listed k0s-bin, kubeadm-bin and openzfs but not the sysexts that ship them, so their local sources (files/{k0s,kubeadm,zfs}/sysext) were missing, while the KubeStellar sysext was listed directly. A sysext only build-depends on its payload, which collect_manifest does not follow, so both stay listed: 183 -> 186 SPDX packages. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- elements/oci/bluefin-server-sbom.bst | 8 ++++++++ tests/unit/test_build_depends.py | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/elements/oci/bluefin-server-sbom.bst b/elements/oci/bluefin-server-sbom.bst index cf7a6c20..e94f06da 100644 --- a/elements/oci/bluefin-server-sbom.bst +++ b/elements/oci/bluefin-server-sbom.bst @@ -14,6 +14,11 @@ description: | payloads. Build-only elements (signing keys, image tooling) are not runtime dependencies of these and never appear. + A sysext only build-depends on the upstream binaries or source it + stages, which the plugin does not follow, so each is listed twice: the + oci/*-sysext.bst element for its own local sources (units, + extension-release, config), and the element it stages them from. + oci/bluefin-server-image.bst stamps creationInfo.created and ships the result as bluefin-server_.spdx.json inside the signed SHA256SUMS. @@ -24,9 +29,12 @@ build-depends: - bluefin-server/os-stack.bst - bluefin-server/initrd/initrd-stack.bst - freedesktop-sdk.bst:components/linux.bst +- oci/k0s-sysext.bst - k0s/k0s-bin.bst - oci/kubestellar-sysext.bst +- oci/kubeadm-sysext.bst - kubeadm/kubeadm-bin.bst +- oci/zfs-sysext.bst - zfs/openzfs.bst config: diff --git a/tests/unit/test_build_depends.py b/tests/unit/test_build_depends.py index 63136085..0d5ffda4 100644 --- a/tests/unit/test_build_depends.py +++ b/tests/unit/test_build_depends.py @@ -113,3 +113,22 @@ def test_os_countme_depends_on_curl_and_jq(): "(projectbluefin/server#96)" ) + +def _build_depends(element): + data = yaml.safe_load((ELEMENTS_DIR / element).read_text(encoding="utf-8")) + return {d if isinstance(d, str) else d["filename"] for d in data.get("build-depends", [])} + + +def test_sbom_lists_every_published_sysext_and_its_payload(): + """collect_manifest follows only runtime dependencies of what it lists. + + A sysext only build-depends on the upstream payload it stages, so the + SBOM must list the sysext (its own local sources) and the payload. + """ + sysexts = {d for d in _build_depends("oci/bluefin-server-image.bst") if d.endswith("-sysext.bst")} + sbom = _build_depends("oci/bluefin-server-sbom.bst") + + assert sysexts + assert sysexts <= sbom, f"SBOM misses {sorted(sysexts - sbom)}" + assert {"k0s/k0s-bin.bst", "kubeadm/kubeadm-bin.bst", "zfs/openzfs.bst"} <= sbom + From 941edb7ed5c475b08ccb919f440136d460eecd74 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Tue, 29 Sep 2026 02:01:27 -0500 Subject: [PATCH 7/7] ci(build): check out the triggering commit in build and release-dry-run Both checked out pull_request.head.ref, the branch, so a push landing mid-run could pair release-dry-run's scripts with artifacts built from another commit. Check out head.sha on pull requests and github.sha otherwise, the commit release passes to gh release --target. boot-test keeps head.ref: #270 rewrites that job. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 7 +++++-- tests/unit/test_release_workflow.py | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ffdf9212..1538cd15 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,9 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.pull_request.head.ref || github.ref }} + # The triggering commit, not its branch tip: release and + # release-dry-run pair these artifacts with that commit. + ref: ${{ github.event.pull_request.head.sha || github.sha }} repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} - name: Set up just @@ -258,7 +260,8 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.event.pull_request.head.ref || github.ref }} + # Same commit as `build`: the branch tip may have moved mid-run. + ref: ${{ github.event.pull_request.head.sha || github.sha }} repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} persist-credentials: false diff --git a/tests/unit/test_release_workflow.py b/tests/unit/test_release_workflow.py index 32472e73..a258626e 100644 --- a/tests/unit/test_release_workflow.py +++ b/tests/unit/test_release_workflow.py @@ -42,6 +42,12 @@ def test_oras_is_pinned_identically_in_both_jobs() -> None: assert "checksum" in json.loads(pins[0])["with"] +def test_build_and_dry_run_check_out_the_triggering_commit() -> None: + for job in (JOBS["build"], DRY_RUN): + checkout = steps(job, "actions/checkout@")[0]["with"] + assert checkout["ref"] == "${{ github.event.pull_request.head.sha || github.sha }}" + + def test_dry_run_is_read_only_and_secret_free() -> None: assert DRY_RUN["if"] == "${{ github.event_name == 'pull_request' }}" assert DRY_RUN["permissions"] == {"contents": "read"}