From d6e01452748a3bbd5111d529bc7708bbe6e49c77 Mon Sep 17 00:00:00 2001 From: castrojo Date: Fri, 13 Nov 2026 20:00:52 -0500 Subject: [PATCH 1/6] fix(build): collapse fsdk-containers FSDK junction onto host FSDK Override freedesktop-sdk.bst onto host freedesktop-sdk.bst in elements/fsdk-containers.bst, bump junction ref to b79382bd, and remove dead duplicate junction declaration from project.conf. This eliminates 311 duplicate elements (including the 64-element FSDK bootstrap), collapses the homebrew build closure from 713 to 361 elements, and aligns glibc/toolchain cache keys so builds pull directly from cache.projectbluefin.io rather than compiling from source for 5 hours. Assisted-by: GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- elements/fsdk-containers.bst | 3 ++- project.conf | 6 ------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/elements/fsdk-containers.bst b/elements/fsdk-containers.bst index a14b9a95..3fba2c51 100644 --- a/elements/fsdk-containers.bst +++ b/elements/fsdk-containers.bst @@ -4,10 +4,11 @@ sources: - kind: git_repo url: https://github.com/projectbluefin/fsdk-containers.git track: main - ref: 5b2df6e22b2dd69ec4e86d9283eaac6def355915 + ref: b79382bd18aebbe805f018cf9d777a4492f92ce8 config: options: arch: '%{arch}' overrides: + freedesktop-sdk.bst: freedesktop-sdk.bst plugins/buildstream-plugins-community.bst: plugins/buildstream-plugins-community.bst diff --git a/project.conf b/project.conf index 17207e8b..72fee663 100644 --- a/project.conf +++ b/project.conf @@ -79,12 +79,6 @@ source-caches: retry-delay: 500 request-timeout: 180 -# Host /usr and the independently packaged Brew machine never share a rootfs. -junctions: - duplicates: - freedesktop-sdk: - - freedesktop-sdk.bst - - fsdk-containers.bst:freedesktop-sdk.bst plugins: - origin: junction From faf8e309f7cbf0e0176faa8b029b0167aa81ebb5 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 14 Nov 2026 09:53:32 -0500 Subject: [PATCH 2/6] fix(ci): delete stale tpm2_id waiver and ignore test fixture ELFs in check-image Assisted-by: Gemini-3.8-flash via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- .github/scripts/check-image.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/scripts/check-image.py b/.github/scripts/check-image.py index 4585a39b..826b7994 100644 --- a/.github/scripts/check-image.py +++ b/.github/scripts/check-image.py @@ -53,8 +53,6 @@ "/usr/lib/systemd/system/quotaon-root.service runs missing /usr/bin/quotaon", # Only started when swtpm is installed (systemd-tpm2-swtpm finds none). "/usr/lib/systemd/system/systemd-tpm2-swtpm.service runs missing swtpm_ioctl", - # tpm2-tools' tpm2_id is not in FSDK; the rule only adds udev properties. - "/usr/lib/udev/rules.d/60-tpm2-id.rules runs missing tpm2_id", # Only for TI am65-cpsw-nuss switchdev NICs (iproute2 has no devlink here). "/usr/lib/udev/rules.d/81-net-bridge.rules runs missing /usr/sbin/devlink", # NVIDIA's GLX and EGL-GBM display libraries; the headless image has no @@ -187,7 +185,7 @@ def check_libraries(name: str, layers: list[Path], own: Path) -> list[str]: elves = [ p for p in iter_files(own) - if "/lib/modules/" not in str(p) and is_elf(p) + if "/lib/modules/" not in str(p) and "/test/support/fixtures/" not in str(p) and is_elf(p) ] needed, runpaths = elf_needed(elves) for path, libs in sorted(needed.items()): From f96222b500d6c4a0b68bd6912b5177d4d32fe209 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 14 Nov 2026 11:28:12 -0500 Subject: [PATCH 3/6] fix(cache): land shared SDK caching before dependency builds Seed the Brew machine with kernel and Go artifacts so cold runners do not repeatedly rebuild the uncached Brew closure. Prefer the project CAS for Bluefin artifacts and sources. Fold in the exact install-action version comments so this fix does not depend on the dependency-update queue. Assisted-by: GPT-6.1 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- .github/workflows/build.yml | 8 +++---- .github/workflows/reproducibility.yml | 2 +- .github/workflows/track-junctions.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- docs/skills/ci-tooling.md | 32 ++++++++++++--------------- project.conf | 9 ++++---- scripts/kernel-cache.sh | 4 ++-- tests/unit/test_kernel_cache.py | 13 ++++++++++- 8 files changed, 39 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e59ab4c9..f57be0e1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -139,7 +139,7 @@ jobs: persist-credentials: false - name: Set up just - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2 + uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 with: tool: just@1.59.0 @@ -179,7 +179,7 @@ jobs: persist-credentials: false - name: Set up just - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2 + uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 with: tool: just@1.59.0 @@ -219,7 +219,7 @@ jobs: persist-credentials: false - name: Set up just - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2 + uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 with: tool: just@1.59.0 @@ -294,7 +294,7 @@ jobs: persist-credentials: false - name: Set up just - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2 + uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 with: tool: just@1.59.0 diff --git a/.github/workflows/reproducibility.yml b/.github/workflows/reproducibility.yml index 044f7c8c..c646fa3d 100644 --- a/.github/workflows/reproducibility.yml +++ b/.github/workflows/reproducibility.yml @@ -61,7 +61,7 @@ jobs: persist-credentials: false - name: Set up just - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2 + uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 with: tool: just@1.59.0 diff --git a/.github/workflows/track-junctions.yml b/.github/workflows/track-junctions.yml index 33e844a3..21311ffb 100644 --- a/.github/workflows/track-junctions.yml +++ b/.github/workflows/track-junctions.yml @@ -42,7 +42,7 @@ jobs: persist-credentials: false - name: Set up just - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2 + uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 with: tool: just@1.59.0 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 7d86f81c..1212ef9c 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -42,7 +42,7 @@ jobs: fi - name: Set up just - uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2 + uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2 with: tool: just@1.59.0 diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index 8637dc2b..b520732f 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -347,6 +347,9 @@ those of the org config `projectbluefin/.project/prow.yaml`; the hive ## Build time and caches +The Brew machine shares the server's SDK junction, not a second bootstrap. +Land cache-key fixes before queueing dependent updates; hold pre-fix entries. + The canonical build-time figures; other files link here instead of restating them: @@ -406,10 +409,10 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about With no signing secrets and that one certificate staged as the only file of `files/boot-keys/modules/` (`seed` refuses anything else there: that - directory's import is pushed with the kernel), each builds FSDK's - `components/linux.bst` and `components/go.bst` with a BuildStream config - that pushes artifacts and sources to `cache.projectbluefin.io:11002`, - unless `bst artifact show` reports both `available` on a remote. The push + directory's import is pushed with the kernel), each builds FSDK's kernel + and Go plus `fsdk-containers.bst:oci/brew-nspawn.bst`. The BuildStream config + pushes artifacts and sources to `cache.projectbluefin.io:11002`, unless + `bst artifact show` reports every target `available` on a remote. The push endpoint takes mTLS: the client certificate is the repository variable `CASD_CLIENT_CERT` and its key the `bst-cache` environment secret `CASD_CLIENT_KEY`, given to that step only and written to a gitignored `.casd.*/` directory that the script @@ -429,9 +432,8 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about sources (Go's TLS test data and others) carry 307 PEM private keys. The seed step is `continue-on-error`, so a failed seed only costs time; the next step then posts a warning and a step summary line. - Measured in the lab: the seed build takes 44 min on 16+ CPUs and fills - 15 GB. Only a kernel or Go change (FSDK bump, a kernel config patch such as - `0006` or `0007`, module certificate) reseeds. + The former kernel/Go-only seed took 44 min and 15 GB on 16+ CPUs. + Kernel, Go, Brew or their dependency changes reseed the affected targets. - **No other cache push.** `bluefin-server/keys/boot-keys.bst` imports `files/boot-keys/`, which on `main` holds the Secure Boot, module-signing and sysupdate private keys, and the image, UKIs, `kernel-modules.bst`, @@ -439,17 +441,11 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about and the `nvidia-open-*-signed.bst` elements build-depend on it, so a release build's own cache must never be saved or pushed anywhere a pull request can read. -- **Most pull requests build the image only in the merge queue.** The full - build costs up to 2 h (figures above), most of it FSDK's kernel, so a pull - request builds before the queue only with the `full-build` label or when it - changes `elements/freedesktop-sdk.bst` or `patches/`; the rest run - `validate`, and the merge queue builds and boot-tests every merge group - that can change the image set before it reaches `main`. When a pull - request, a merge group or the nightly build does build, it pulls the kernel that `kernel-cache-dev` seeded: `just - gen-dev-keys` always stages the same public dev module certificate, so - the dev kernel's cache key is the same on every run. It compiles the - kernel (1 h 43 min) only until the next release after a kernel change - seeds it. +- **Most pull requests build the image only in the merge queue.** Before + the queue, only `full-build`, SDK junction and patch changes build; the + rest run `validate`. Every image-changing merge group builds and boots. + Dev builds reuse the public module certificate, so their kernel keys + match `kernel-cache-dev`; after a kernel change it stays cold until seeded. - **No `actions/cache`.** A full build's cache holds `boot-keys.bst`, and pull requests can restore caches saved on `main`; the key-free kernel cache lives in the project CAS instead. diff --git a/project.conf b/project.conf index 72fee663..6bf50894 100644 --- a/project.conf +++ b/project.conf @@ -52,13 +52,13 @@ variables: # Pull-only: read from the shared GNOME + Bluefin BuildStream CAS caches. artifacts: - - url: https://gbm.gnome.org:11003 + - url: https://cache.projectbluefin.io:11001 connection-config: keepalive-time: 180 retry-limit: 5 retry-delay: 500 request-timeout: 180 - - url: https://cache.projectbluefin.io:11001 + - url: https://gbm.gnome.org:11003 connection-config: keepalive-time: 180 retry-limit: 5 @@ -66,20 +66,19 @@ artifacts: request-timeout: 180 source-caches: - - url: https://gbm.gnome.org:11003 + - url: https://cache.projectbluefin.io:11001 connection-config: keepalive-time: 180 retry-limit: 5 retry-delay: 500 request-timeout: 180 - - url: https://cache.projectbluefin.io:11001 + - url: https://gbm.gnome.org:11003 connection-config: keepalive-time: 180 retry-limit: 5 retry-delay: 500 request-timeout: 180 - plugins: - origin: junction junction: plugins/buildstream-plugins-community.bst diff --git a/scripts/kernel-cache.sh b/scripts/kernel-cache.sh index 70a2a210..0a968c06 100755 --- a/scripts/kernel-cache.sh +++ b/scripts/kernel-cache.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Key-free BuildStream cache of FSDK's kernel (and Go), in the project CAS. +# Key-free BuildStream cache of FSDK's kernel, Go and the Brew machine. # # CASD_CLIENT_CERT= CASD_CLIENT_KEY= kernel-cache.sh seed release|dev # @@ -36,7 +36,7 @@ # directory's import is pushed with the kernel). set -euo pipefail -ELEMENTS=(freedesktop-sdk.bst:components/linux.bst freedesktop-sdk.bst:components/go.bst) +ELEMENTS=(freedesktop-sdk.bst:components/linux.bst freedesktop-sdk.bst:components/go.bst fsdk-containers.bst:oci/brew-nspawn.bst) KEYS_ELEMENT=bluefin-server/keys/boot-keys.bst PUSH_URL=https://cache.projectbluefin.io:11002 diff --git a/tests/unit/test_kernel_cache.py b/tests/unit/test_kernel_cache.py index 8a44cf03..cff65362 100644 --- a/tests/unit/test_kernel_cache.py +++ b/tests/unit/test_kernel_cache.py @@ -65,7 +65,7 @@ def env(tmp_path: Path) -> dict[str, str]: "PATH": f"{bin_dir}:{os.environ['PATH']}", "LOG": str(tmp_path / "log"), "GRAPH": "freedesktop-sdk.bst:components/linux.bst bluefin-server/keys/linux-module-cert.bst", - "STATES": " not cached freedesktop-sdk.bst:components/linux.bst\n available freedesktop-sdk.bst:components/go.bst", + "STATES": " not cached freedesktop-sdk.bst:components/linux.bst\n available freedesktop-sdk.bst:components/go.bst\n available fsdk-containers.bst:oci/brew-nspawn.bst", "CASD_CLIENT_CERT": "-----BEGIN CERTIFICATE-----\ncert\n-----END CERTIFICATE-----", "CASD_CLIENT_KEY": "-----BEGIN PRIVATE KEY-----\nkey\n-----END PRIVATE KEY-----", "BST_FLAGS": "", @@ -157,6 +157,16 @@ def test_seed_builds_elements_only_cached_locally(checkout: Path, env: dict[str, assert (Path(env["LOG"]) / "built").exists() +def test_seed_builds_when_only_brew_is_missing(checkout: Path, env: dict[str, str]) -> None: + env["STATES"] = ( + " available freedesktop-sdk.bst:components/linux.bst\n" + " available freedesktop-sdk.bst:components/go.bst\n" + " not cached fsdk-containers.bst:oci/brew-nspawn.bst" + ) + assert run(checkout, env).returncode == 0 + assert (Path(env["LOG"]) / "built").exists() + + def test_seed_pushes_with_the_client_certificate(checkout: Path, env: dict[str, str]) -> None: result = run(checkout, env) assert result.returncode == 0, result.stderr @@ -166,6 +176,7 @@ def test_seed_pushes_with_the_client_certificate(checkout: Path, env: dict[str, "build", "freedesktop-sdk.bst:components/linux.bst", "freedesktop-sdk.bst:components/go.bst", + "fsdk-containers.bst:oci/brew-nspawn.bst", ] conf = (log / "buildstream.conf").read_text() for kind in ("artifacts:", "source-caches:"): From dfb33372f0672fd502b9ab0da12bd855943812e6 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 14 Nov 2026 11:58:58 -0500 Subject: [PATCH 4/6] fix(cache): validate seed payloads instead of trusting remote metadata Remote availability can advertise an artifact whose CAS payload cannot be pulled. Let BuildStream perform native pull/build/push checks instead of reporting seed success from metadata alone. Make the observed missing GCC stage2 compiler an explicit seed target, rather than relying on a build dependency being needed. Assisted-by: GPT-6.1 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- docs/skills/ci-tooling.md | 15 +++++++-------- scripts/kernel-cache.sh | 21 ++++++++++----------- tests/unit/test_kernel_cache.py | 26 +++++++++++++++++--------- 3 files changed, 34 insertions(+), 28 deletions(-) diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index b520732f..1024783a 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -348,7 +348,6 @@ those of the org config `projectbluefin/.project/prow.yaml`; the hive ## Build time and caches The Brew machine shares the server's SDK junction, not a second bootstrap. -Land cache-key fixes before queueing dependent updates; hold pre-fix entries. The canonical build-time figures; other files link here instead of restating them: @@ -398,7 +397,7 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about `tests/fixtures/`, `project.conf`, `Justfile`, `build.yml`) would skip. A skipped job reports as passing, so its required check does not hold it. - **Kernel cache in the project CAS, key-free by construction.** Two jobs, - on releases only, run `scripts/kernel-cache.sh seed`, one per module + on every push to `main` (`release=true`), run `scripts/kernel-cache.sh seed`, one per module certificate the kernel can trust (it is part of the kernel's cache key): - `kernel-cache` runs `seed release` with the committed release certificate (`files/release-keys/linux-module-cert.crt`); `build` waits for it. @@ -409,11 +408,11 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about With no signing secrets and that one certificate staged as the only file of `files/boot-keys/modules/` (`seed` refuses anything else there: that - directory's import is pushed with the kernel), each builds FSDK's kernel - and Go plus `fsdk-containers.bst:oci/brew-nspawn.bst`. The BuildStream config - pushes artifacts and sources to `cache.projectbluefin.io:11002`, unless - `bst artifact show` reports every target `available` on a remote. The push - endpoint takes mTLS: the client certificate is the repository variable + directory's import is pushed with the kernel), each builds FSDK's kernel, + Go and `bootstrap/build/gcc-stage2.bst`, plus the Brew machine. BuildStream + pulls usable payloads or rebuilds missing ones, then pushes artifacts and + sources to `cache.projectbluefin.io:11002`; metadata alone never skips seeding. + The push endpoint takes mTLS: its client certificate is the repository variable `CASD_CLIENT_CERT` and its key the `bst-cache` environment secret `CASD_CLIENT_KEY`, given to that step only and written to a gitignored `.casd.*/` directory that the script removes on exit. The CAS trusts that certificate in its list of client @@ -433,7 +432,7 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about seed step is `continue-on-error`, so a failed seed only costs time; the next step then posts a warning and a step summary line. The former kernel/Go-only seed took 44 min and 15 GB on 16+ CPUs. - Kernel, Go, Brew or their dependency changes reseed the affected targets. + Kernel, Go, GCC, Brew or their dependency changes reseed the affected targets. - **No other cache push.** `bluefin-server/keys/boot-keys.bst` imports `files/boot-keys/`, which on `main` holds the Secure Boot, module-signing and sysupdate private keys, and the image, UKIs, `kernel-modules.bst`, diff --git a/scripts/kernel-cache.sh b/scripts/kernel-cache.sh index 0a968c06..e6794158 100755 --- a/scripts/kernel-cache.sh +++ b/scripts/kernel-cache.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Key-free BuildStream cache of FSDK's kernel, Go and the Brew machine. +# Key-free BuildStream cache of FSDK's kernel, Go, GCC and the Brew machine. # # CASD_CLIENT_CERT= CASD_CLIENT_KEY= kernel-cache.sh seed release|dev # @@ -24,9 +24,8 @@ # sources to the CAS's mTLS endpoint; every build reads them back through the # anonymous remote that patches/freedesktop-sdk/0001 adds to the junction's # project.conf (the top-level project.conf remote does not reach junction -# elements such as the kernel). It builds nothing when a remote already -# holds every element (`available`; an element only cached locally still goes -# through `bst build`, which pushes it). +# elements such as the kernel). `available` only proves remote metadata exists; +# `bst build` must check the payload, pull or rebuild it, and push the result. # # The CAS is publicly readable. What `seed` pushes is key-free by # construction: the seed job holds no signing secrets, `seed` refuses to @@ -36,7 +35,12 @@ # directory's import is pushed with the kernel). set -euo pipefail -ELEMENTS=(freedesktop-sdk.bst:components/linux.bst freedesktop-sdk.bst:components/go.bst fsdk-containers.bst:oci/brew-nspawn.bst) +ELEMENTS=( + freedesktop-sdk.bst:components/linux.bst + freedesktop-sdk.bst:components/go.bst + freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst + fsdk-containers.bst:oci/brew-nspawn.bst +) KEYS_ELEMENT=bluefin-server/keys/boot-keys.bst PUSH_URL=https://cache.projectbluefin.io:11002 @@ -95,7 +99,7 @@ stage_cert() { auth="" seed() { - local variant="$1" graph states available + local variant="$1" graph states : "${CASD_CLIENT_CERT:?kernel-cache: CASD_CLIENT_CERT is not set}" : "${CASD_CLIENT_KEY:?kernel-cache: CASD_CLIENT_KEY is not set}" stage_cert "${variant}" @@ -125,11 +129,6 @@ seed() { echo "kernel-cache: cannot reach ${PUSH_URL}; refusing to build without pushing" >&2 exit 1 fi - available="$(grep -cE '^ *available ' <<< "${states}" || true)" - if [ "${available}" -eq "${#ELEMENTS[@]}" ]; then - echo "kernel-cache: ${variant}: ${ELEMENTS[*]} already on a remote" - return 0 - fi just bst build "${ELEMENTS[@]}" echo "kernel-cache: ${variant}: pushed ${ELEMENTS[*]} to ${PUSH_URL}" } diff --git a/tests/unit/test_kernel_cache.py b/tests/unit/test_kernel_cache.py index cff65362..c97767fa 100644 --- a/tests/unit/test_kernel_cache.py +++ b/tests/unit/test_kernel_cache.py @@ -2,9 +2,9 @@ The CAS is publicly readable, so `seed` must refuse whenever the graph it builds reaches the element that stages private keys, before it writes the -client credentials anywhere; it must build nothing when a remote already holds -every element; and the push config it hands BuildStream must authenticate -with the client certificate and leave no credential behind. `just` is stubbed. +client credentials anywhere; remote metadata alone must not count as a successful +seed; and the push config it hands BuildStream must authenticate with the +client certificate and leave no credential behind. `just` is stubbed. Each variant stages exactly one committed certificate as the only file of files/boot-keys/modules/ (whose import is pushed with the kernel): release @@ -36,7 +36,8 @@ conf="${BST_FLAGS##*--config /src/}" cp "$conf" "$LOG/buildstream.conf" cp "$(dirname "$conf")/client.key" "$LOG/client.key" - ls -ld "$(dirname "$conf")" > "$LOG/auth-dir" ;; + ls -ld "$(dirname "$conf")" > "$LOG/auth-dir" + exit "${BUILD_STATUS:-0}" ;; esac """ @@ -65,7 +66,12 @@ def env(tmp_path: Path) -> dict[str, str]: "PATH": f"{bin_dir}:{os.environ['PATH']}", "LOG": str(tmp_path / "log"), "GRAPH": "freedesktop-sdk.bst:components/linux.bst bluefin-server/keys/linux-module-cert.bst", - "STATES": " not cached freedesktop-sdk.bst:components/linux.bst\n available freedesktop-sdk.bst:components/go.bst\n available fsdk-containers.bst:oci/brew-nspawn.bst", + "STATES": ( + " not cached freedesktop-sdk.bst:components/linux.bst\n" + " available freedesktop-sdk.bst:components/go.bst\n" + " available freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst\n" + " available fsdk-containers.bst:oci/brew-nspawn.bst" + ), "CASD_CLIENT_CERT": "-----BEGIN CERTIFICATE-----\ncert\n-----END CERTIFICATE-----", "CASD_CLIENT_KEY": "-----BEGIN PRIVATE KEY-----\nkey\n-----END PRIVATE KEY-----", "BST_FLAGS": "", @@ -141,12 +147,12 @@ def test_seed_refuses_to_build_when_the_push_remote_is_unreachable(checkout: Pat assert not leftovers(checkout) -def test_seed_skips_elements_a_remote_already_holds(checkout: Path, env: dict[str, str]) -> None: +def test_seed_does_not_accept_metadata_when_payload_build_fails(checkout: Path, env: dict[str, str]) -> None: env["STATES"] = env["STATES"].replace("not cached", " available") + env["BUILD_STATUS"] = "19" result = run(checkout, env) - assert result.returncode == 0 - assert "already on a remote" in result.stdout - assert not (Path(env["LOG"]) / "built").exists() + assert result.returncode == 19 + assert "pushed" not in result.stdout assert not leftovers(checkout) @@ -161,6 +167,7 @@ def test_seed_builds_when_only_brew_is_missing(checkout: Path, env: dict[str, st env["STATES"] = ( " available freedesktop-sdk.bst:components/linux.bst\n" " available freedesktop-sdk.bst:components/go.bst\n" + " available freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst\n" " not cached fsdk-containers.bst:oci/brew-nspawn.bst" ) assert run(checkout, env).returncode == 0 @@ -176,6 +183,7 @@ def test_seed_pushes_with_the_client_certificate(checkout: Path, env: dict[str, "build", "freedesktop-sdk.bst:components/linux.bst", "freedesktop-sdk.bst:components/go.bst", + "freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst", "fsdk-containers.bst:oci/brew-nspawn.bst", ] conf = (log / "buildstream.conf").read_text() From fbca5f6dc845509e21e01f0e6be4dd24f303231e Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 14 Nov 2026 12:05:56 -0500 Subject: [PATCH 5/6] fix(cache): replicate upstream build dependencies into project CAS A native two-CAS test confirms that building a cached target leaves build-only dependencies outside the destination cache. Explicit artifact push --deps all pulls and replicates those dependencies. Only report successful seeding after replication succeeds. Assisted-by: GPT-6.1 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- docs/skills/ci-tooling.md | 4 ++-- scripts/kernel-cache.sh | 2 ++ tests/unit/test_kernel_cache.py | 11 ++++++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index 1024783a..14dc1e5f 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -410,8 +410,8 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about `files/boot-keys/modules/` (`seed` refuses anything else there: that directory's import is pushed with the kernel), each builds FSDK's kernel, Go and `bootstrap/build/gcc-stage2.bst`, plus the Brew machine. BuildStream - pulls usable payloads or rebuilds missing ones, then pushes artifacts and - sources to `cache.projectbluefin.io:11002`; metadata alone never skips seeding. + pulls or rebuilds payloads, then `bst artifact push --deps all` replicates + upstream artifacts into `cache.projectbluefin.io:11002`; metadata never skips seeding. The push endpoint takes mTLS: its client certificate is the repository variable `CASD_CLIENT_CERT` and its key the `bst-cache` environment secret `CASD_CLIENT_KEY`, given to that step only and written to a gitignored `.casd.*/` directory that the script diff --git a/scripts/kernel-cache.sh b/scripts/kernel-cache.sh index e6794158..371628aa 100755 --- a/scripts/kernel-cache.sh +++ b/scripts/kernel-cache.sh @@ -130,6 +130,8 @@ seed() { exit 1 fi just bst build "${ELEMENTS[@]}" + # Builds need not push dependencies pulled from upstream caches. + just bst artifact push --deps all "${ELEMENTS[@]}" echo "kernel-cache: ${variant}: pushed ${ELEMENTS[*]} to ${PUSH_URL}" } diff --git a/tests/unit/test_kernel_cache.py b/tests/unit/test_kernel_cache.py index c97767fa..95d7d3f1 100644 --- a/tests/unit/test_kernel_cache.py +++ b/tests/unit/test_kernel_cache.py @@ -26,11 +26,12 @@ # would read it from /src (the checkout). JUST = """#!/bin/sh case "$*" in - *"--deps all"*) + "bst show --deps all"*) printf '%s\\n' $GRAPH i=0; while [ "$i" -lt "${GRAPH_TAIL:-0}" ]; do echo "freedesktop-sdk.bst:components/filler-$i.bst"; i=$((i + 1)); done exit "${GRAPH_STATUS:-0}" ;; "bst artifact show"*) printf '%s\\n' "$STATES"; [ -z "$UNREACHABLE" ] || echo "$UNREACHABLE" >&2 ;; + "bst artifact push"*) exit "${PUSH_STATUS:-0}" ;; "bst build"*) echo "$*" >> "$LOG/built" conf="${BST_FLAGS##*--config /src/}" @@ -156,6 +157,14 @@ def test_seed_does_not_accept_metadata_when_payload_build_fails(checkout: Path, assert not leftovers(checkout) +def test_seed_does_not_report_success_when_replication_fails(checkout: Path, env: dict[str, str]) -> None: + env["PUSH_STATUS"] = "23" + result = run(checkout, env) + assert result.returncode == 23 + assert "pushed" not in result.stdout + assert not leftovers(checkout) + + def test_seed_builds_elements_only_cached_locally(checkout: Path, env: dict[str, str]) -> None: # A local artifact may never have been pushed; `bst build` pushes it. env["STATES"] = env["STATES"].replace("not cached", " cached") From 47ce75e01338338a3003ed0397d13de4252361a2 Mon Sep 17 00:00:00 2001 From: castrojo Date: Sat, 14 Nov 2026 13:36:28 -0500 Subject: [PATCH 6/6] fix(boot): preserve a reproducible native systemd clock epoch PID 1 resets a correct 2026 RTC to the SDK build epoch in 2011 once its 15-year window expires, causing GPG to reject matching image keys as future-dated. Produce /usr/lib/clock-epoch from the pinned SDK year/month and restore its mtime after initrd clamping and before repart. Keep signature verification and SDK cache keys unchanged. Assisted-by: GPT-6.1 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: castrojo --- docs/skills/ddi-installer-build.md | 26 +++++++++++---------- elements/bluefin-server/os-release.bst | 8 +++++++ elements/oci/bluefin-server-boot.bst | 2 ++ elements/oci/bluefin-server-usr.bst | 3 +++ tests/unit/test_clock_epoch.py | 31 ++++++++++++++++++++++++++ 5 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 tests/unit/test_clock_epoch.py diff --git a/docs/skills/ddi-installer-build.md b/docs/skills/ddi-installer-build.md index 8852ca56..e29fd7b2 100644 --- a/docs/skills/ddi-installer-build.md +++ b/docs/skills/ddi-installer-build.md @@ -48,11 +48,10 @@ payloads, and a `SHA256SUMS` over all of it with its detached signature [ddi-installer.md](ddi-installer.md) for what each artifact is. Home Server builds `homebrew/image.bst` through `fsdk-containers.bst`; the -producer exports its independently pinned Brew machine, not host binaries. -Keep the host and producer FSDK junctions explicitly duplicated in `project.conf`: -their filesystems are packaged separately. Share the identical community-plugin -junction through the consumer override; recheck its pins before changing that -mapping. Text-only manual elements use `strip-binaries: ""`, matching +producer exports a separate Brew machine, not host binaries. Its SDK and +community-plugin junctions are overridden with the server's junctions, reusing +artifact keys without sharing a root filesystem. Keep that mapping when +updating the producer pin. Text-only manual elements use `strip-binaries: ""`, matching `os-creds-prov.bst`, rather than adding a binary-stripping toolchain. The separate `oci/homebrew-integration-sysext.bst` supplies the optional host bridge and Python closure, not base `/usr`. Its version-locked asset and the @@ -121,13 +120,16 @@ version that sorts higher before building. ## Reproducible builds -With the same checkout, keys and `image-version`, rebuilding the final -assembly gives the same bytes. BuildStream exports `SOURCE_DATE_EPOCH` into -every sandbox and the assembly steps honor it: `mkfs.erofs` and -`systemd-repart` clamp file times to it, the initrd and ESP trees are clamped -before `cpio` and `mcopy` copy them, and `systemd-sbsign` (not `sbsign`) -uses it as the signing time of systemd-boot and the UKIs. `systemd-repart ---seed` fixes partition UUIDs. +With the same checkout, keys and `image-version`, rebuilding final assembly +gives the same bytes. BuildStream exports a fixed `SOURCE_DATE_EPOCH`; cpio, +ESP assembly and signatures honor it, and `systemd-repart --seed` fixes UUIDs. +PID 1's `/usr/lib/clock-epoch` is a deliberate exception: its **mtime** is the +first day of the pinned SDK's year/month (`installer-version`), preventing the +15-year clock sanity window from resetting a valid RTC to 2011 and rejecting +GPG keys as future-dated. `os-release.bst` records that reproducible epoch; +initrd assembly restores its mtime after clamping, and repart uses the same +epoch for the erofs image. Never substitute wall-clock build time or disable +signature-time checks. Two outputs carry a signing time of their own: `SHA256SUMS.gpg` and the `efi-keys/*.auth` updates, which `sbvarsign` stamps with the current time diff --git a/elements/bluefin-server/os-release.bst b/elements/bluefin-server/os-release.bst index a36ad138..364f6e5c 100644 --- a/elements/bluefin-server/os-release.bst +++ b/elements/bluefin-server/os-release.bst @@ -34,3 +34,11 @@ config: HOME_URL="https://github.com/projectbluefin/server" BUG_REPORT_URL="https://github.com/projectbluefin/server/issues" EOF + - | + # PID 1 uses this file's mtime as its clock floor, not the SDK's 2011 epoch. + v='%{installer-version}' + month="${v#*.}" + epoch="%{install-root}/usr/lib/clock-epoch" + date -u -d "20${v%%.*}-${month%%.*}-01" +%s > "${epoch}" + touch -d "@$(cat "${epoch}")" "${epoch}" + chmod 0644 "${epoch}" diff --git a/elements/oci/bluefin-server-boot.bst b/elements/oci/bluefin-server-boot.bst index ed68bd35..15de4135 100644 --- a/elements/oci/bluefin-server-boot.bst +++ b/elements/oci/bluefin-server-boot.bst @@ -223,6 +223,8 @@ config: du -sh /initrd # cpio records mtimes; everything created above carries the build time. find . -mindepth 1 -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} + + # Staging and the clamp reset mtimes; PID 1 must not reset a valid RTC to 2011. + touch -d "@$(cat usr/lib/clock-epoch)" usr/lib/clock-epoch find . -mindepth 1 -printf '%P\n' | LC_ALL=C sort | cpio -o -H newc --reproducible --quiet | zstd -19 -T0 -q > /tmp/initrd.cpio.zst ls -lh /tmp/initrd.cpio.zst diff --git a/elements/oci/bluefin-server-usr.bst b/elements/oci/bluefin-server-usr.bst index 56f29e6f..d580c7ef 100644 --- a/elements/oci/bluefin-server-usr.bst +++ b/elements/oci/bluefin-server-usr.bst @@ -211,6 +211,9 @@ config: - | set -eu cd "%{install-root}" + # Preserve PID 1's native clock floor through repart's reproducible clamp. + export SOURCE_DATE_EPOCH="$(cat /sysroot/usr/lib/clock-epoch)" + touch -d "@${SOURCE_DATE_EPOCH}" /sysroot/usr/lib/clock-epoch systemd-repart \ --empty=create --size=auto --offline=yes --dry-run=no \ --seed=6f1b0fe2-3c2a-4f2e-9f0a-b1e5e7a5e001 \ diff --git a/tests/unit/test_clock_epoch.py b/tests/unit/test_clock_epoch.py new file mode 100644 index 00000000..db379171 --- /dev/null +++ b/tests/unit/test_clock_epoch.py @@ -0,0 +1,31 @@ +"""The image clock floor follows the SDK release, not BuildStream's 2011 epoch.""" + +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ELEMENT = ROOT / "elements" / "bluefin-server" / "os-release.bst" + + +@pytest.mark.parametrize( + ("sdk", "expected"), + [("26.08.3", datetime(2026, 8, 1, tzinfo=UTC)), ("27.02.1", datetime(2027, 2, 1, tzinfo=UTC))], +) +def test_image_clock_epoch_tracks_sdk_release(tmp_path: Path, sdk: str, expected: datetime) -> None: + commands = yaml.safe_load(ELEMENT.read_text())["config"]["install-commands"] + script = "\n".join(commands) + for key, value in { + "install-root": str(tmp_path), + "image-version": "0.1", + "systemd-arch": "x86-64", + "installer-version": sdk, + }.items(): + script = script.replace(f"%{{{key}}}", value) + subprocess.run(["bash", "-eu", "-c", script], check=True, capture_output=True, text=True) + epoch = int((tmp_path / "usr/lib/clock-epoch").read_text()) + assert epoch == int(expected.timestamp()) + assert int((tmp_path / "usr/lib/clock-epoch").stat().st_mtime) == int(expected.timestamp())