From dfb33372f0672fd502b9ab0da12bd855943812e6 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 14 Nov 2026 11:58:58 -0500 Subject: [PATCH 1/3] fix(cache): validate seed payloads instead of trusting remote metadata Remote availability can advertise an artifact whose CAS payload cannot be pulled. Let BuildStream perform native pull/build/push checks instead of reporting seed success from metadata alone. Make the observed missing GCC stage2 compiler an explicit seed target, rather than relying on a build dependency being needed. Assisted-by: GPT-6.1 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- docs/skills/ci-tooling.md | 15 +++++++-------- scripts/kernel-cache.sh | 21 ++++++++++----------- tests/unit/test_kernel_cache.py | 26 +++++++++++++++++--------- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index b520732f..1024783a 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -348,7 +348,6 @@ those of the org config `projectbluefin/.project/prow.yaml`; the hive ## Build time and caches The Brew machine shares the server's SDK junction, not a second bootstrap. -Land cache-key fixes before queueing dependent updates; hold pre-fix entries. The canonical build-time figures; other files link here instead of restating them: @@ -398,7 +397,7 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about `tests/fixtures/`, `project.conf`, `Justfile`, `build.yml`) would skip. A skipped job reports as passing, so its required check does not hold it. - **Kernel cache in the project CAS, key-free by construction.** Two jobs, - on releases only, run `scripts/kernel-cache.sh seed`, one per module + on every push to `main` (`release=true`), run `scripts/kernel-cache.sh seed`, one per module certificate the kernel can trust (it is part of the kernel's cache key): - `kernel-cache` runs `seed release` with the committed release certificate (`files/release-keys/linux-module-cert.crt`); `build` waits for it. @@ -409,11 +408,11 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about With no signing secrets and that one certificate staged as the only file of `files/boot-keys/modules/` (`seed` refuses anything else there: that - directory's import is pushed with the kernel), each builds FSDK's kernel - and Go plus `fsdk-containers.bst:oci/brew-nspawn.bst`. The BuildStream config - pushes artifacts and sources to `cache.projectbluefin.io:11002`, unless - `bst artifact show` reports every target `available` on a remote. The push - endpoint takes mTLS: the client certificate is the repository variable + directory's import is pushed with the kernel), each builds FSDK's kernel, + Go and `bootstrap/build/gcc-stage2.bst`, plus the Brew machine. BuildStream + pulls usable payloads or rebuilds missing ones, then pushes artifacts and + sources to `cache.projectbluefin.io:11002`; metadata alone never skips seeding. + The push endpoint takes mTLS: its client certificate is the repository variable `CASD_CLIENT_CERT` and its key the `bst-cache` environment secret `CASD_CLIENT_KEY`, given to that step only and written to a gitignored `.casd.*/` directory that the script removes on exit. The CAS trusts that certificate in its list of client @@ -433,7 +432,7 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about seed step is `continue-on-error`, so a failed seed only costs time; the next step then posts a warning and a step summary line. The former kernel/Go-only seed took 44 min and 15 GB on 16+ CPUs. - Kernel, Go, Brew or their dependency changes reseed the affected targets. + Kernel, Go, GCC, Brew or their dependency changes reseed the affected targets. - **No other cache push.** `bluefin-server/keys/boot-keys.bst` imports `files/boot-keys/`, which on `main` holds the Secure Boot, module-signing and sysupdate private keys, and the image, UKIs, `kernel-modules.bst`, diff --git a/scripts/kernel-cache.sh b/scripts/kernel-cache.sh index 0a968c06..e6794158 100755 --- a/scripts/kernel-cache.sh +++ b/scripts/kernel-cache.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Key-free BuildStream cache of FSDK's kernel, Go and the Brew machine. +# Key-free BuildStream cache of FSDK's kernel, Go, GCC and the Brew machine. # # CASD_CLIENT_CERT= CASD_CLIENT_KEY= kernel-cache.sh seed release|dev # @@ -24,9 +24,8 @@ # sources to the CAS's mTLS endpoint; every build reads them back through the # anonymous remote that patches/freedesktop-sdk/0001 adds to the junction's # project.conf (the top-level project.conf remote does not reach junction -# elements such as the kernel). It builds nothing when a remote already -# holds every element (`available`; an element only cached locally still goes -# through `bst build`, which pushes it). +# elements such as the kernel). `available` only proves remote metadata exists; +# `bst build` must check the payload, pull or rebuild it, and push the result. # # The CAS is publicly readable. What `seed` pushes is key-free by # construction: the seed job holds no signing secrets, `seed` refuses to @@ -36,7 +35,12 @@ # directory's import is pushed with the kernel). set -euo pipefail -ELEMENTS=(freedesktop-sdk.bst:components/linux.bst freedesktop-sdk.bst:components/go.bst fsdk-containers.bst:oci/brew-nspawn.bst) +ELEMENTS=( + freedesktop-sdk.bst:components/linux.bst + freedesktop-sdk.bst:components/go.bst + freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst + fsdk-containers.bst:oci/brew-nspawn.bst +) KEYS_ELEMENT=bluefin-server/keys/boot-keys.bst PUSH_URL=https://cache.projectbluefin.io:11002 @@ -95,7 +99,7 @@ stage_cert() { auth="" seed() { - local variant="$1" graph states available + local variant="$1" graph states : "${CASD_CLIENT_CERT:?kernel-cache: CASD_CLIENT_CERT is not set}" : "${CASD_CLIENT_KEY:?kernel-cache: CASD_CLIENT_KEY is not set}" stage_cert "${variant}" @@ -125,11 +129,6 @@ seed() { echo "kernel-cache: cannot reach ${PUSH_URL}; refusing to build without pushing" >&2 exit 1 fi - available="$(grep -cE '^ *available ' <<< "${states}" || true)" - if [ "${available}" -eq "${#ELEMENTS[@]}" ]; then - echo "kernel-cache: ${variant}: ${ELEMENTS[*]} already on a remote" - return 0 - fi just bst build "${ELEMENTS[@]}" echo "kernel-cache: ${variant}: pushed ${ELEMENTS[*]} to ${PUSH_URL}" } diff --git a/tests/unit/test_kernel_cache.py b/tests/unit/test_kernel_cache.py index cff65362..c97767fa 100644 --- a/tests/unit/test_kernel_cache.py +++ b/tests/unit/test_kernel_cache.py @@ -2,9 +2,9 @@ The CAS is publicly readable, so `seed` must refuse whenever the graph it builds reaches the element that stages private keys, before it writes the -client credentials anywhere; it must build nothing when a remote already holds -every element; and the push config it hands BuildStream must authenticate -with the client certificate and leave no credential behind. `just` is stubbed. +client credentials anywhere; remote metadata alone must not count as a successful +seed; and the push config it hands BuildStream must authenticate with the +client certificate and leave no credential behind. `just` is stubbed. Each variant stages exactly one committed certificate as the only file of files/boot-keys/modules/ (whose import is pushed with the kernel): release @@ -36,7 +36,8 @@ conf="${BST_FLAGS##*--config /src/}" cp "$conf" "$LOG/buildstream.conf" cp "$(dirname "$conf")/client.key" "$LOG/client.key" - ls -ld "$(dirname "$conf")" > "$LOG/auth-dir" ;; + ls -ld "$(dirname "$conf")" > "$LOG/auth-dir" + exit "${BUILD_STATUS:-0}" ;; esac """ @@ -65,7 +66,12 @@ def env(tmp_path: Path) -> dict[str, str]: "PATH": f"{bin_dir}:{os.environ['PATH']}", "LOG": str(tmp_path / "log"), "GRAPH": "freedesktop-sdk.bst:components/linux.bst bluefin-server/keys/linux-module-cert.bst", - "STATES": " not cached freedesktop-sdk.bst:components/linux.bst\n available freedesktop-sdk.bst:components/go.bst\n available fsdk-containers.bst:oci/brew-nspawn.bst", + "STATES": ( + " not cached freedesktop-sdk.bst:components/linux.bst\n" + " available freedesktop-sdk.bst:components/go.bst\n" + " available freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst\n" + " available fsdk-containers.bst:oci/brew-nspawn.bst" + ), "CASD_CLIENT_CERT": "-----BEGIN CERTIFICATE-----\ncert\n-----END CERTIFICATE-----", "CASD_CLIENT_KEY": "-----BEGIN PRIVATE KEY-----\nkey\n-----END PRIVATE KEY-----", "BST_FLAGS": "", @@ -141,12 +147,12 @@ def test_seed_refuses_to_build_when_the_push_remote_is_unreachable(checkout: Pat assert not leftovers(checkout) -def test_seed_skips_elements_a_remote_already_holds(checkout: Path, env: dict[str, str]) -> None: +def test_seed_does_not_accept_metadata_when_payload_build_fails(checkout: Path, env: dict[str, str]) -> None: env["STATES"] = env["STATES"].replace("not cached", " available") + env["BUILD_STATUS"] = "19" result = run(checkout, env) - assert result.returncode == 0 - assert "already on a remote" in result.stdout - assert not (Path(env["LOG"]) / "built").exists() + assert result.returncode == 19 + assert "pushed" not in result.stdout assert not leftovers(checkout) @@ -161,6 +167,7 @@ def test_seed_builds_when_only_brew_is_missing(checkout: Path, env: dict[str, st env["STATES"] = ( " available freedesktop-sdk.bst:components/linux.bst\n" " available freedesktop-sdk.bst:components/go.bst\n" + " available freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst\n" " not cached fsdk-containers.bst:oci/brew-nspawn.bst" ) assert run(checkout, env).returncode == 0 @@ -176,6 +183,7 @@ def test_seed_pushes_with_the_client_certificate(checkout: Path, env: dict[str, "build", "freedesktop-sdk.bst:components/linux.bst", "freedesktop-sdk.bst:components/go.bst", + "freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst", "fsdk-containers.bst:oci/brew-nspawn.bst", ] conf = (log / "buildstream.conf").read_text() From fbca5f6dc845509e21e01f0e6be4dd24f303231e Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 14 Nov 2026 12:05:56 -0500 Subject: [PATCH 2/3] fix(cache): replicate upstream build dependencies into project CAS A native two-CAS test confirms that building a cached target leaves build-only dependencies outside the destination cache. Explicit artifact push --deps all pulls and replicates those dependencies. Only report successful seeding after replication succeeds. Assisted-by: GPT-6.1 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- docs/skills/ci-tooling.md | 4 ++-- scripts/kernel-cache.sh | 2 ++ tests/unit/test_kernel_cache.py | 11 ++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index 1024783a..14dc1e5f 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -410,8 +410,8 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about `files/boot-keys/modules/` (`seed` refuses anything else there: that directory's import is pushed with the kernel), each builds FSDK's kernel, Go and `bootstrap/build/gcc-stage2.bst`, plus the Brew machine. BuildStream - pulls usable payloads or rebuilds missing ones, then pushes artifacts and - sources to `cache.projectbluefin.io:11002`; metadata alone never skips seeding. + pulls or rebuilds payloads, then `bst artifact push --deps all` replicates + upstream artifacts into `cache.projectbluefin.io:11002`; metadata never skips seeding. The push endpoint takes mTLS: its client certificate is the repository variable `CASD_CLIENT_CERT` and its key the `bst-cache` environment secret `CASD_CLIENT_KEY`, given to that step only and written to a gitignored `.casd.*/` directory that the script diff --git a/scripts/kernel-cache.sh b/scripts/kernel-cache.sh index e6794158..371628aa 100755 --- a/scripts/kernel-cache.sh +++ b/scripts/kernel-cache.sh @@ -130,6 +130,8 @@ seed() { exit 1 fi just bst build "${ELEMENTS[@]}" + # Builds need not push dependencies pulled from upstream caches. + just bst artifact push --deps all "${ELEMENTS[@]}" echo "kernel-cache: ${variant}: pushed ${ELEMENTS[*]} to ${PUSH_URL}" } diff --git a/tests/unit/test_kernel_cache.py b/tests/unit/test_kernel_cache.py index c97767fa..95d7d3f1 100644 --- a/tests/unit/test_kernel_cache.py +++ b/tests/unit/test_kernel_cache.py @@ -26,11 +26,12 @@ # would read it from /src (the checkout). JUST = """#!/bin/sh case "$*" in - *"--deps all"*) + "bst show --deps all"*) printf '%s\\n' $GRAPH i=0; while [ "$i" -lt "${GRAPH_TAIL:-0}" ]; do echo "freedesktop-sdk.bst:components/filler-$i.bst"; i=$((i + 1)); done exit "${GRAPH_STATUS:-0}" ;; "bst artifact show"*) printf '%s\\n' "$STATES"; [ -z "$UNREACHABLE" ] || echo "$UNREACHABLE" >&2 ;; + "bst artifact push"*) exit "${PUSH_STATUS:-0}" ;; "bst build"*) echo "$*" >> "$LOG/built" conf="${BST_FLAGS##*--config /src/}" @@ -156,6 +157,14 @@ def test_seed_does_not_accept_metadata_when_payload_build_fails(checkout: Path, assert not leftovers(checkout) +def test_seed_does_not_report_success_when_replication_fails(checkout: Path, env: dict[str, str]) -> None: + env["PUSH_STATUS"] = "23" + result = run(checkout, env) + assert result.returncode == 23 + assert "pushed" not in result.stdout + assert not leftovers(checkout) + + def test_seed_builds_elements_only_cached_locally(checkout: Path, env: dict[str, str]) -> None: # A local artifact may never have been pushed; `bst build` pushes it. env["STATES"] = env["STATES"].replace("not cached", " cached") From 47ce75e01338338a3003ed0397d13de4252361a2 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 14 Nov 2026 13:36:28 -0500 Subject: [PATCH 3/3] fix(boot): preserve a reproducible native systemd clock epoch PID 1 resets a correct 2026 RTC to the SDK build epoch in 2011 once its 15-year window expires, causing GPG to reject matching image keys as future-dated. Produce /usr/lib/clock-epoch from the pinned SDK year/month and restore its mtime after initrd clamping and before repart. Keep signature verification and SDK cache keys unchanged. Assisted-by: GPT-6.1 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- docs/skills/ddi-installer-build.md | 26 +++++++++++---------- elements/bluefin-server/os-release.bst | 8 +++++++ elements/oci/bluefin-server-boot.bst | 2 ++ elements/oci/bluefin-server-usr.bst | 3 +++ tests/unit/test_clock_epoch.py | 31 ++++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 tests/unit/test_clock_epoch.py diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 8852ca56..e29fd7b2 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -48,11 +48,10 @@ payloads, and a `SHA256SUMS` over all of it with its detached signature [ddi-installer.md](ddi-installer.md) for what each artifact is. Home Server builds `homebrew/image.bst` through `fsdk-containers.bst`; the -producer exports its independently pinned Brew machine, not host binaries. -Keep the host and producer FSDK junctions explicitly duplicated in `project.conf`: -their filesystems are packaged separately. Share the identical community-plugin -junction through the consumer override; recheck its pins before changing that -mapping. Text-only manual elements use `strip-binaries: ""`, matching +producer exports a separate Brew machine, not host binaries. Its SDK and +community-plugin junctions are overridden with the server's junctions, reusing +artifact keys without sharing a root filesystem. Keep that mapping when +updating the producer pin. Text-only manual elements use `strip-binaries: ""`, matching `os-creds-prov.bst`, rather than adding a binary-stripping toolchain. The separate `oci/homebrew-integration-sysext.bst` supplies the optional host bridge and Python closure, not base `/usr`. Its version-locked asset and the @@ -121,13 +120,16 @@ version that sorts higher before building. ## Reproducible builds -With the same checkout, keys and `image-version`, rebuilding the final -assembly gives the same bytes. BuildStream exports `SOURCE_DATE_EPOCH` into -every sandbox and the assembly steps honor it: `mkfs.erofs` and -`systemd-repart` clamp file times to it, the initrd and ESP trees are clamped -before `cpio` and `mcopy` copy them, and `systemd-sbsign` (not `sbsign`) -uses it as the signing time of systemd-boot and the UKIs. `systemd-repart ---seed` fixes partition UUIDs. +With the same checkout, keys and `image-version`, rebuilding final assembly +gives the same bytes. BuildStream exports a fixed `SOURCE_DATE_EPOCH`; cpio, +ESP assembly and signatures honor it, and `systemd-repart --seed` fixes UUIDs. +PID 1's `/usr/lib/clock-epoch` is a deliberate exception: its **mtime** is the +first day of the pinned SDK's year/month (`installer-version`), preventing the +15-year clock sanity window from resetting a valid RTC to 2011 and rejecting +GPG keys as future-dated. `os-release.bst` records that reproducible epoch; +initrd assembly restores its mtime after clamping, and repart uses the same +epoch for the erofs image. Never substitute wall-clock build time or disable +signature-time checks. Two outputs carry a signing time of their own: `SHA256SUMS.gpg` and the `efi-keys/*.auth` updates, which `sbvarsign` stamps with the current time diff --git a/elements/bluefin-server/os-release.bst b/elements/bluefin-server/os-release.bst index a36ad138..364f6e5c 100644 --- a/elements/bluefin-server/os-release.bst +++ b/elements/bluefin-server/os-release.bst @@ -34,3 +34,11 @@ config: HOME_URL="https://github.com/projectbluefin/server" BUG_REPORT_URL="https://github.com/projectbluefin/server/issues" EOF + - | + # PID 1 uses this file's mtime as its clock floor, not the SDK's 2011 epoch. + v='%{installer-version}' + month="${v#*.}" + epoch="%{install-root}/usr/lib/clock-epoch" + date -u -d "20${v%%.*}-${month%%.*}-01" +%s > "${epoch}" + touch -d "@$(cat "${epoch}")" "${epoch}" + chmod 0644 "${epoch}" diff --git a/elements/oci/bluefin-server-boot.bst b/elements/oci/bluefin-server-boot.bst index ed68bd35..15de4135 100644 --- a/elements/oci/bluefin-server-boot.bst +++ b/elements/oci/bluefin-server-boot.bst @@ -223,6 +223,8 @@ config: du -sh /initrd # cpio records mtimes; everything created above carries the build time. find . -mindepth 1 -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + # Staging and the clamp reset mtimes; PID 1 must not reset a valid RTC to 2011. + touch -d "@$(cat usr/lib/clock-epoch)" usr/lib/clock-epoch find . -mindepth 1 -printf '%P\n' | LC_ALL=C sort | cpio -o -H newc --reproducible --quiet | zstd -19 -T0 -q > /tmp/initrd.cpio.zst ls -lh /tmp/initrd.cpio.zst diff --git a/elements/oci/bluefin-server-usr.bst b/elements/oci/bluefin-server-usr.bst index 56f29e6f..d580c7ef 100644 --- a/elements/oci/bluefin-server-usr.bst +++ b/elements/oci/bluefin-server-usr.bst @@ -211,6 +211,9 @@ config: - | set -eu cd "%{install-root}" + # Preserve PID 1's native clock floor through repart's reproducible clamp. + export SOURCE_DATE_EPOCH="$(cat /sysroot/usr/lib/clock-epoch)" + touch -d "@${SOURCE_DATE_EPOCH}" /sysroot/usr/lib/clock-epoch systemd-repart \ --empty=create --size=auto --offline=yes --dry-run=no \ --seed=6f1b0fe2-3c2a-4f2e-9f0a-b1e5e7a5e001 \ diff --git a/tests/unit/test_clock_epoch.py b/tests/unit/test_clock_epoch.py new file mode 100644 index 00000000..db379171 --- /dev/null +++ b/tests/unit/test_clock_epoch.py @@ -0,0 +1,31 @@ +"""The image clock floor follows the SDK release, not BuildStream's 2011 epoch.""" + +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ELEMENT = ROOT / "elements" / "bluefin-server" / "os-release.bst" + + +@pytest.mark.parametrize( + ("sdk", "expected"), + [("26.08.3", datetime(2026, 8, 1, tzinfo=UTC)), ("27.02.1", datetime(2027, 2, 1, tzinfo=UTC))], +) +def test_image_clock_epoch_tracks_sdk_release(tmp_path: Path, sdk: str, expected: datetime) -> None: + commands = yaml.safe_load(ELEMENT.read_text())["config"]["install-commands"] + script = "\n".join(commands) + for key, value in { + "install-root": str(tmp_path), + "image-version": "0.1", + "systemd-arch": "x86-64", + "installer-version": sdk, + }.items(): + script = script.replace(f"%{{{key}}}", value) + subprocess.run(["bash", "-eu", "-c", script], check=True, capture_output=True, text=True) + epoch = int((tmp_path / "usr/lib/clock-epoch").read_text()) + assert epoch == int(expected.timestamp()) + assert int((tmp_path / "usr/lib/clock-epoch").stat().st_mtime) == int(expected.timestamp())