Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions .github/scripts/check-image.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,6 @@
"/usr/lib/systemd/system/quotaon-root.service runs missing /usr/bin/quotaon",
# Only started when swtpm is installed (systemd-tpm2-swtpm finds none).
"/usr/lib/systemd/system/systemd-tpm2-swtpm.service runs missing swtpm_ioctl",
# tpm2-tools' tpm2_id is not in FSDK; the rule only adds udev properties.
"/usr/lib/udev/rules.d/60-tpm2-id.rules runs missing tpm2_id",
# Only for TI am65-cpsw-nuss switchdev NICs (iproute2 has no devlink here).
"/usr/lib/udev/rules.d/81-net-bridge.rules runs missing /usr/sbin/devlink",
# NVIDIA's GLX and EGL-GBM display libraries; the headless image has no
Expand Down Expand Up @@ -187,7 +185,7 @@ def check_libraries(name: str, layers: list[Path], own: Path) -> list[str]:
elves = [
p
for p in iter_files(own)
if "/lib/modules/" not in str(p) and is_elf(p)
if "/lib/modules/" not in str(p) and "/test/support/fixtures/" not in str(p) and is_elf(p)
]
needed, runpaths = elf_needed(elves)
for path, libs in sorted(needed.items()):
Expand Down
8 changes: 4 additions & 4 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ jobs:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
with:
tool: just@1.59.0

Expand Down Expand Up @@ -179,7 +179,7 @@ jobs:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
with:
tool: just@1.59.0

Expand Down Expand Up @@ -219,7 +219,7 @@ jobs:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
with:
tool: just@1.59.0

Expand Down Expand Up @@ -294,7 +294,7 @@ jobs:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
with:
tool: just@1.59.0

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/reproducibility.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ jobs:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
with:
tool: just@1.59.0

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/track-junctions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
persist-credentials: false

- name: Set up just
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
with:
tool: just@1.59.0

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ jobs:
fi

- name: Set up just
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2
uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
with:
tool: just@1.59.0

Expand Down
35 changes: 15 additions & 20 deletions docs/skills/ci-tooling.md
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,8 @@ those of the org config `projectbluefin/.project/prow.yaml`; the hive

## Build time and caches

The Brew machine shares the server's SDK junction, not a second bootstrap.

The canonical build-time figures; other files link here instead of
restating them:

Expand Down Expand Up @@ -395,7 +397,7 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about
`tests/fixtures/`, `project.conf`, `Justfile`, `build.yml`) would skip.
A skipped job reports as passing, so its required check does not hold it.
- **Kernel cache in the project CAS, key-free by construction.** Two jobs,
on releases only, run `scripts/kernel-cache.sh seed`, one per module
on every push to `main` (`release=true`), run `scripts/kernel-cache.sh seed`, one per module
certificate the kernel can trust (it is part of the kernel's cache key):
- `kernel-cache` runs `seed release` with the committed release certificate
(`files/release-keys/linux-module-cert.crt`); `build` waits for it.
Expand All @@ -406,11 +408,11 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about

With no signing secrets and that one certificate staged as the only file of
`files/boot-keys/modules/` (`seed` refuses anything else there: that
directory's import is pushed with the kernel), each builds FSDK's
`components/linux.bst` and `components/go.bst` with a BuildStream config
that pushes artifacts and sources to `cache.projectbluefin.io:11002`,
unless `bst artifact show` reports both `available` on a remote. The push
endpoint takes mTLS: the client certificate is the repository variable
directory's import is pushed with the kernel), each builds FSDK's kernel,
Go and `bootstrap/build/gcc-stage2.bst`, plus the Brew machine. BuildStream
pulls or rebuilds payloads, then `bst artifact push --deps all` replicates
upstream artifacts into `cache.projectbluefin.io:11002`; metadata never skips seeding.
The push endpoint takes mTLS: its client certificate is the repository variable
`CASD_CLIENT_CERT` and its key the `bst-cache` environment secret
`CASD_CLIENT_KEY`, given to that step only and written to a gitignored `.casd.*/` directory that the script
removes on exit. The CAS trusts that certificate in its list of client
Expand All @@ -429,27 +431,20 @@ version) and the NVIDIA Container Toolkit (Go); the driver sysext adds about
sources (Go's TLS test data and others) carry 307 PEM private keys. The
seed step is `continue-on-error`, so a failed seed only costs time; the
next step then posts a warning and a step summary line.
Measured in the lab: the seed build takes 44 min on 16+ CPUs and fills
15 GB. Only a kernel or Go change (FSDK bump, a kernel config patch such as
`0006` or `0007`, module certificate) reseeds.
The former kernel/Go-only seed took 44 min and 15 GB on 16+ CPUs.
Kernel, Go, GCC, Brew or their dependency changes reseed the affected targets.
- **No other cache push.** `bluefin-server/keys/boot-keys.bst` imports
`files/boot-keys/`, which on `main` holds the Secure Boot, module-signing
and sysupdate private keys, and the image, UKIs, `kernel-modules.bst`,
`efi-keys.bst`, `os-sd-boot-signed.bst`, `shim.bst`, `openzfs-signed.bst`
and the `nvidia-open-*-signed.bst` elements
build-depend on it, so a release build's own cache must never be saved or
pushed anywhere a pull request can read.
- **Most pull requests build the image only in the merge queue.** The full
build costs up to 2 h (figures above), most of it FSDK's kernel, so a pull
request builds before the queue only with the `full-build` label or when it
changes `elements/freedesktop-sdk.bst` or `patches/`; the rest run
`validate`, and the merge queue builds and boot-tests every merge group
that can change the image set before it reaches `main`. When a pull
request, a merge group or the nightly build does build, it pulls the kernel that `kernel-cache-dev` seeded: `just
gen-dev-keys` always stages the same public dev module certificate, so
the dev kernel's cache key is the same on every run. It compiles the
kernel (1 h 43 min) only until the next release after a kernel change
seeds it.
- **Most pull requests build the image only in the merge queue.** Before
the queue, only `full-build`, SDK junction and patch changes build; the
rest run `validate`. Every image-changing merge group builds and boots.
Dev builds reuse the public module certificate, so their kernel keys
match `kernel-cache-dev`; after a kernel change it stays cold until seeded.
- **No `actions/cache`.** A full build's cache holds `boot-keys.bst`, and
pull requests can restore caches saved on `main`; the key-free kernel
cache lives in the project CAS instead.
Expand Down
26 changes: 14 additions & 12 deletions docs/skills/ddi-installer-build.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,10 @@ payloads, and a `SHA256SUMS` over all of it with its detached signature
[ddi-installer.md](ddi-installer.md) for what each artifact is.

Home Server builds `homebrew/image.bst` through `fsdk-containers.bst`; the
producer exports its independently pinned Brew machine, not host binaries.
Keep the host and producer FSDK junctions explicitly duplicated in `project.conf`:
their filesystems are packaged separately. Share the identical community-plugin
junction through the consumer override; recheck its pins before changing that
mapping. Text-only manual elements use `strip-binaries: ""`, matching
producer exports a separate Brew machine, not host binaries. Its SDK and
community-plugin junctions are overridden with the server's junctions, reusing
artifact keys without sharing a root filesystem. Keep that mapping when
updating the producer pin. Text-only manual elements use `strip-binaries: ""`, matching
`os-creds-prov.bst`, rather than adding a binary-stripping toolchain.
The separate `oci/homebrew-integration-sysext.bst` supplies the optional host
bridge and Python closure, not base `/usr`. Its version-locked asset and the
Expand Down Expand Up @@ -121,13 +120,16 @@ version that sorts higher before building.

## Reproducible builds

With the same checkout, keys and `image-version`, rebuilding the final
assembly gives the same bytes. BuildStream exports `SOURCE_DATE_EPOCH` into
every sandbox and the assembly steps honor it: `mkfs.erofs` and
`systemd-repart` clamp file times to it, the initrd and ESP trees are clamped
before `cpio` and `mcopy` copy them, and `systemd-sbsign` (not `sbsign`)
uses it as the signing time of systemd-boot and the UKIs. `systemd-repart
--seed` fixes partition UUIDs.
With the same checkout, keys and `image-version`, rebuilding final assembly
gives the same bytes. BuildStream exports a fixed `SOURCE_DATE_EPOCH`; cpio,
ESP assembly and signatures honor it, and `systemd-repart --seed` fixes UUIDs.
PID 1's `/usr/lib/clock-epoch` is a deliberate exception: its **mtime** is the
first day of the pinned SDK's year/month (`installer-version`), preventing the
15-year clock sanity window from resetting a valid RTC to 2011 and rejecting
GPG keys as future-dated. `os-release.bst` records that reproducible epoch;
initrd assembly restores its mtime after clamping, and repart uses the same
epoch for the erofs image. Never substitute wall-clock build time or disable
signature-time checks.

Two outputs carry a signing time of their own: `SHA256SUMS.gpg` and the
`efi-keys/*.auth` updates, which `sbvarsign` stamps with the current time
Expand Down
8 changes: 8 additions & 0 deletions elements/bluefin-server/os-release.bst
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,11 @@ config:
HOME_URL="https://github.com/projectbluefin/server"
BUG_REPORT_URL="https://github.com/projectbluefin/server/issues"
EOF
- |
# PID 1 uses this file's mtime as its clock floor, not the SDK's 2011 epoch.
v='%{installer-version}'
month="${v#*.}"
epoch="%{install-root}/usr/lib/clock-epoch"
date -u -d "20${v%%.*}-${month%%.*}-01" +%s > "${epoch}"
touch -d "@$(cat "${epoch}")" "${epoch}"
chmod 0644 "${epoch}"
3 changes: 2 additions & 1 deletion elements/fsdk-containers.bst
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@ sources:
- kind: git_repo
url: https://github.com/projectbluefin/fsdk-containers.git
track: main
ref: 5b2df6e22b2dd69ec4e86d9283eaac6def355915
ref: b79382bd18aebbe805f018cf9d777a4492f92ce8

config:
options:
arch: '%{arch}'
overrides:
freedesktop-sdk.bst: freedesktop-sdk.bst
plugins/buildstream-plugins-community.bst: plugins/buildstream-plugins-community.bst
2 changes: 2 additions & 0 deletions elements/oci/bluefin-server-boot.bst
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,8 @@ config:
du -sh /initrd
# cpio records mtimes; everything created above carries the build time.
find . -mindepth 1 -exec touch -h -d "@${SOURCE_DATE_EPOCH}" {} +
# Staging and the clamp reset mtimes; PID 1 must not reset a valid RTC to 2011.
touch -d "@$(cat usr/lib/clock-epoch)" usr/lib/clock-epoch
find . -mindepth 1 -printf '%P\n' | LC_ALL=C sort | cpio -o -H newc --reproducible --quiet | zstd -19 -T0 -q > /tmp/initrd.cpio.zst
ls -lh /tmp/initrd.cpio.zst

Expand Down
3 changes: 3 additions & 0 deletions elements/oci/bluefin-server-usr.bst
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,9 @@ config:
- |
set -eu
cd "%{install-root}"
# Preserve PID 1's native clock floor through repart's reproducible clamp.
export SOURCE_DATE_EPOCH="$(cat /sysroot/usr/lib/clock-epoch)"
touch -d "@${SOURCE_DATE_EPOCH}" /sysroot/usr/lib/clock-epoch
systemd-repart \
--empty=create --size=auto --offline=yes --dry-run=no \
--seed=6f1b0fe2-3c2a-4f2e-9f0a-b1e5e7a5e001 \
Expand Down
15 changes: 4 additions & 11 deletions project.conf
Original file line number Diff line number Diff line change
Expand Up @@ -52,40 +52,33 @@ variables:

# Pull-only: read from the shared GNOME + Bluefin BuildStream CAS caches.
artifacts:
- url: https://gbm.gnome.org:11003
- url: https://cache.projectbluefin.io:11001
connection-config:
keepalive-time: 180
retry-limit: 5
retry-delay: 500
request-timeout: 180
- url: https://cache.projectbluefin.io:11001
- url: https://gbm.gnome.org:11003
connection-config:
keepalive-time: 180
retry-limit: 5
retry-delay: 500
request-timeout: 180

source-caches:
- url: https://gbm.gnome.org:11003
- url: https://cache.projectbluefin.io:11001
connection-config:
keepalive-time: 180
retry-limit: 5
retry-delay: 500
request-timeout: 180
- url: https://cache.projectbluefin.io:11001
- url: https://gbm.gnome.org:11003
connection-config:
keepalive-time: 180
retry-limit: 5
retry-delay: 500
request-timeout: 180

# Host /usr and the independently packaged Brew machine never share a rootfs.
junctions:
duplicates:
freedesktop-sdk:
- freedesktop-sdk.bst
- fsdk-containers.bst:freedesktop-sdk.bst

plugins:
- origin: junction
junction: plugins/buildstream-plugins-community.bst
Expand Down
23 changes: 12 additions & 11 deletions scripts/kernel-cache.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Key-free BuildStream cache of FSDK's kernel (and Go), in the project CAS.
# Key-free BuildStream cache of FSDK's kernel, Go, GCC and the Brew machine.
#
# CASD_CLIENT_CERT=<pem> CASD_CLIENT_KEY=<pem> kernel-cache.sh seed release|dev
#
Expand All @@ -24,9 +24,8 @@
# sources to the CAS's mTLS endpoint; every build reads them back through the
# anonymous remote that patches/freedesktop-sdk/0001 adds to the junction's
# project.conf (the top-level project.conf remote does not reach junction
# elements such as the kernel). It builds nothing when a remote already
# holds every element (`available`; an element only cached locally still goes
# through `bst build`, which pushes it).
# elements such as the kernel). `available` only proves remote metadata exists;
# `bst build` must check the payload, pull or rebuild it, and push the result.
#
# The CAS is publicly readable. What `seed` pushes is key-free by
# construction: the seed job holds no signing secrets, `seed` refuses to
Expand All @@ -36,7 +35,12 @@
# directory's import is pushed with the kernel).
set -euo pipefail

ELEMENTS=(freedesktop-sdk.bst:components/linux.bst freedesktop-sdk.bst:components/go.bst)
ELEMENTS=(
freedesktop-sdk.bst:components/linux.bst
freedesktop-sdk.bst:components/go.bst
freedesktop-sdk.bst:bootstrap/build/gcc-stage2.bst
fsdk-containers.bst:oci/brew-nspawn.bst
)
KEYS_ELEMENT=bluefin-server/keys/boot-keys.bst
PUSH_URL=https://cache.projectbluefin.io:11002

Expand Down Expand Up @@ -95,7 +99,7 @@ stage_cert() {
auth=""

seed() {
local variant="$1" graph states available
local variant="$1" graph states
: "${CASD_CLIENT_CERT:?kernel-cache: CASD_CLIENT_CERT is not set}"
: "${CASD_CLIENT_KEY:?kernel-cache: CASD_CLIENT_KEY is not set}"
stage_cert "${variant}"
Expand Down Expand Up @@ -125,12 +129,9 @@ seed() {
echo "kernel-cache: cannot reach ${PUSH_URL}; refusing to build without pushing" >&2
exit 1
fi
available="$(grep -cE '^ *available ' <<< "${states}" || true)"
if [ "${available}" -eq "${#ELEMENTS[@]}" ]; then
echo "kernel-cache: ${variant}: ${ELEMENTS[*]} already on a remote"
return 0
fi
just bst build "${ELEMENTS[@]}"
# Builds need not push dependencies pulled from upstream caches.
just bst artifact push --deps all "${ELEMENTS[@]}"
echo "kernel-cache: ${variant}: pushed ${ELEMENTS[*]} to ${PUSH_URL}"
}

Expand Down
31 changes: 31 additions & 0 deletions tests/unit/test_clock_epoch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
"""The image clock floor follows the SDK release, not BuildStream's 2011 epoch."""

import subprocess
from datetime import UTC, datetime
from pathlib import Path

import pytest
import yaml

ROOT = Path(__file__).resolve().parents[2]
ELEMENT = ROOT / "elements" / "bluefin-server" / "os-release.bst"


@pytest.mark.parametrize(
("sdk", "expected"),
[("26.08.3", datetime(2026, 8, 1, tzinfo=UTC)), ("27.02.1", datetime(2027, 2, 1, tzinfo=UTC))],
)
def test_image_clock_epoch_tracks_sdk_release(tmp_path: Path, sdk: str, expected: datetime) -> None:
commands = yaml.safe_load(ELEMENT.read_text())["config"]["install-commands"]
script = "\n".join(commands)
for key, value in {
"install-root": str(tmp_path),
"image-version": "0.1",
"systemd-arch": "x86-64",
"installer-version": sdk,
}.items():
script = script.replace(f"%{{{key}}}", value)
subprocess.run(["bash", "-eu", "-c", script], check=True, capture_output=True, text=True)
epoch = int((tmp_path / "usr/lib/clock-epoch").read_text())
assert epoch == int(expected.timestamp())
assert int((tmp_path / "usr/lib/clock-epoch").stat().st_mtime) == int(expected.timestamp())
Loading
Loading