Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 39 additions & 12 deletions .github/scripts/check-release-version.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,14 @@
release asset names via `@v`. The k0s and OpenZFS sysexts pin their own
upstream versions in include/k0s.yml and include/zfs.yml.

This script fails closed on drift.
This script fails closed on drift. It is also the one parser of the pinned
FSDK point release:

--print-fsdk print it (the Justfile's fsdk_version)
--fix set installer-version to it, then check (track-junctions.yml)
"""

import argparse
import re
import sys
from pathlib import Path
Expand All @@ -25,6 +30,8 @@
r"^\s*installer-version:\s*[\"']?([0-9]+\.[0-9]+\.[0-9]+)[\"']?\s*$", re.MULTILINE
)
FSDK_REF_RE = re.compile(r"freedesktop-sdk-([0-9]+\.[0-9]+\.[0-9]+)")
# Any installer-version value, valid or not, for --fix to replace.
INSTALLER_LINE_RE = re.compile(r"^([ \t]*installer-version:[ \t]*).*$", re.MULTILINE)


def read(path):
Expand All @@ -33,10 +40,38 @@ def read(path):
return path.read_text(encoding="utf-8")


def main():
def pinned_fsdk_version():
fsdk_match = FSDK_REF_RE.search(read(FSDK_JUNCTION))
if not fsdk_match:
sys.exit(
"ERROR: elements/freedesktop-sdk.bst has no "
"'freedesktop-sdk-X.Y.Z' point release in its ref."
)
return fsdk_match.group(1)


def fix_installer_version(fsdk_pinned):
conf = read(PROJECT_CONF)
junction = read(FSDK_JUNCTION)
fixed = INSTALLER_LINE_RE.sub(lambda m: f'{m.group(1)}"{fsdk_pinned}"', conf, count=1)
if fixed != conf:
PROJECT_CONF.write_text(fixed, encoding="utf-8")


def main(argv=()):
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
action = parser.add_mutually_exclusive_group()
action.add_argument("--print-fsdk", action="store_true", help="print the pinned FSDK point release")
action.add_argument("--fix", action="store_true", help="sync installer-version to the pinned FSDK point release")
args = parser.parse_args(argv)

fsdk_pinned = pinned_fsdk_version()
if args.print_fsdk:
print(fsdk_pinned)
return
if args.fix:
fix_installer_version(fsdk_pinned)

conf = read(PROJECT_CONF)
installer_match = INSTALLER_VERSION_RE.search(conf)
if not installer_match:
sys.exit(
Expand All @@ -45,14 +80,6 @@ def main():
)
installer_declared = installer_match.group(1)

fsdk_match = FSDK_REF_RE.search(junction)
if not fsdk_match:
sys.exit(
"ERROR: elements/freedesktop-sdk.bst has no "
"'freedesktop-sdk-X.Y.Z' point release in its ref."
)
fsdk_pinned = fsdk_match.group(1)

if installer_declared != fsdk_pinned:
sys.exit(
"ERROR: installer-version drift.\n"
Expand All @@ -71,4 +98,4 @@ def main():


if __name__ == "__main__":
main()
main(sys.argv[1:])
3 changes: 2 additions & 1 deletion .github/scripts/image-build-needed.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
Reads the PR's changed paths (one per line, renames listed under both names)
on stdin and prints ``true`` or ``false`` for the build workflow's `changes`
job, which runs this script from the PR's base revision. Only changes that
cannot reach the image set or the boot test skip the ~2 h build: docs,
cannot reach the image set or the boot test skip the full build (figures in
docs/skills/ci-tooling.md, "Build time and caches"): docs,
Markdown outside the build inputs, unit/e2e tests and the workflows that run
them. Anything else, and anything under a build input root, builds; so does
any change to this classifier or to the build workflow, and an empty or
Expand Down
94 changes: 94 additions & 0 deletions .github/scripts/propose-pr.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
#!/usr/bin/env bash
# Propose the working-tree changes to PATHs as one commit on BRANCH and open
# or update its pull request against BASE. Shared by the trackers
# (track-junctions.yml, track-binaries.yml).
#
# propose-pr.sh --branch B --title T --body-file F [--base main]
# [--remote origin] [--skip-closed-title] -- PATH...
#
# Does nothing when:
# - PATHs do not differ from the checked-out commit (nothing to propose);
# - an open pull request's branch already holds exactly these PATHs, so a
# daily run neither force-pushes an identical commit nor restarts its CI;
# - --skip-closed-title and a closed pull request from BRANCH had this
# title: it was turned down, and the next upstream version brings a new
# title.
# Otherwise it commits onto BRANCH (from the checked-out commit), force-pushes
# with a lease on the branch state it saw, and creates or edits the pull
# request. GH_TOKEN authenticates `gh` and the push: checkouts do not persist
# credentials, so the push passes it as a one-off header that never reaches
# .git/config.
#
# No `|| true` on the queries: a failed `gh` or `git ls-remote` must stop the
# run, not pass for "no pull request" or "no branch".
set -euo pipefail

die() { echo "propose-pr: $*" >&2; exit 1; }

branch="" title="" body="" base=main remote=origin skip_closed=0
while [ $# -gt 0 ]; do
case "$1" in
--branch) branch="$2"; shift 2 ;;
--title) title="$2"; shift 2 ;;
--body-file) body="$2"; shift 2 ;;
--base) base="$2"; shift 2 ;;
--remote) remote="$2"; shift 2 ;;
--skip-closed-title) skip_closed=1; shift ;;
--) shift; break ;;
*) die "unknown argument: $1" ;;
esac
done
[ -n "${branch}" ] && [ -n "${title}" ] || die "--branch and --title are required"
[ -f "${body}" ] || die "--body-file must name a file"
[ $# -gt 0 ] || die "no paths to propose"
paths=("$@")

git add -- "${paths[@]}"
if git diff --cached --quiet -- "${paths[@]}"; then
echo "Nothing to propose: ${paths[*]} already match $(git rev-parse --short HEAD)."
exit 0
fi

remote_head=""
if git ls-remote --exit-code --heads "${remote}" "refs/heads/${branch}" >/dev/null; then
git fetch --quiet "${remote}" "+refs/heads/${branch}:refs/remotes/${remote}/${branch}"
remote_head="$(git rev-parse "refs/remotes/${remote}/${branch}")"
else
status=$?
[ "${status}" -eq 2 ] || die "cannot list ${branch} on ${remote} (git ls-remote exit ${status})"
fi

open="$(gh pr list --head "${branch}" --base "${base}" --state open --json number --jq '.[0].number // empty')"
if [ -n "${open}" ] && [ -n "${remote_head}" ] && git diff --cached --quiet "${remote_head}" -- "${paths[@]}"; then
echo "#${open} already proposes this content on ${branch}; leaving it and its CI alone."
exit 0
fi
if [ -z "${open}" ] && [ "${skip_closed}" = 1 ]; then
closed="$(gh pr list --head "${branch}" --base "${base}" --state closed --json title --jq '.[].title')"
if grep -Fxq -- "${title}" <<< "${closed}"; then
echo "A closed pull request already proposed '${title}'; not reopening it."
exit 0
fi
fi

# Deliberately no attribution trailers: the trackers' commits are the
# deterministic output of their tools, with no AI authorship to attribute
# (AGENTS.md's trailers are for AI-authored commits). Do not re-add the
# Copilot trailer the old track-refs job carried.
git checkout --quiet -B "${branch}"
git -c user.name="github-actions[bot]" \
-c user.email="41898282+github-actions[bot]@users.noreply.github.com" \
commit --quiet -m "${title}"
push_auth=()
if [ -n "${GH_TOKEN:-}" ]; then
push_auth=(-c "http.${GITHUB_SERVER_URL:-https://github.com}/.extraheader=AUTHORIZATION: basic $(printf 'x-access-token:%s' "${GH_TOKEN}" | base64 -w0)")
fi
git "${push_auth[@]}" push --quiet --force-with-lease="refs/heads/${branch}:${remote_head}" \
"${remote}" "HEAD:refs/heads/${branch}"

if [ -n "${open}" ]; then
gh pr edit "${open}" --title "${title}" --body-file "${body}"
echo "Updated PR: $(gh pr view "${open}" --json url --jq '.url')"
else
echo "Created PR: $(gh pr create --base "${base}" --head "${branch}" --title "${title}" --body-file "${body}")"
fi
40 changes: 8 additions & 32 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@ jobs:
# full-build label or when it changes the FSDK junction or its patches,
# and never when it cannot change the image set (docs, unit tests; see
# .github/scripts/image-build-needed.py). Other pull requests run
# `validate` in minutes instead of the ~1.5 h build.
# `validate` in minutes instead of the full build (build-time figures:
# docs/skills/ci-tooling.md#build-time-and-caches).
# A skipped job reports as passing, so `build` and `boot-test` can be
# required checks without blocking those PRs. The classifier script is
# checked out from the base revision, so editing it does not skip the PR's
Expand Down Expand Up @@ -131,9 +132,10 @@ jobs:
- name: Validate Element Graph
run: just validate

# FSDK's kernel is ~70 of the ~85 build minutes, and no upstream cache holds
# it (our module certificate and kernel config patches change its key). This
# job builds it (and Go) with the committed release module certificate and no
# FSDK's kernel is most of a cold build
# (docs/skills/ci-tooling.md#build-time-and-caches), and no upstream cache
# holds it (our module certificate and kernel config patches change its
# key). This job builds it (and Go) with the committed release module certificate and no
# signing secrets, and pushes the key-free artifacts to the project CAS unless
# a remote already holds them; every build pulls them through project.conf
# (scripts/kernel-cache.sh).
Expand All @@ -157,22 +159,8 @@ jobs:
with:
tool: just@1.58.0

# BuildStream's cache on the runner's /mnt disk; the kernel build alone
# fills 15 GB.
- name: Set up runner disk space
run: |
set -euo pipefail
sudo mkdir -p /mnt/podman /mnt/buildstream
sudo chown -R "$USER:$USER" /mnt/podman /mnt/buildstream
mkdir -p ~/.config/containers
cat > ~/.config/containers/storage.conf <<EOF
[storage]
driver = "overlay"
runroot = "/run/user/1001"
graphroot = "/mnt/podman"
EOF
mkdir -p ~/.cache
ln -sfn /mnt/buildstream ~/.cache/buildstream
run: bash scripts/ci-runner-disk.sh

- name: Stage the release module certificate
run: |
Expand Down Expand Up @@ -290,19 +278,7 @@ jobs:
echo "version=${V}" >> "$GITHUB_OUTPUT"

- name: Set up runner disk space
run: |
set -euo pipefail
sudo mkdir -p /mnt/podman /mnt/buildstream
sudo chown -R "$USER:$USER" /mnt/podman /mnt/buildstream
mkdir -p ~/.config/containers
cat > ~/.config/containers/storage.conf <<EOF
[storage]
driver = "overlay"
runroot = "/run/user/1001"
graphroot = "/mnt/podman"
EOF
mkdir -p ~/.cache
ln -sf /mnt/buildstream ~/.cache/buildstream
run: bash scripts/ci-runner-disk.sh

- name: Validate Element Graph
run: just validate
Expand Down
21 changes: 8 additions & 13 deletions .github/workflows/reproducibility.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,25 @@ jobs:
runs-on: ubuntu-26.04
timeout-minutes: 300
env:
# Everything oci/bluefin-server-image.bst stages into the release set,
# and every oci/* target `just validate` resolves;
# tests/unit/test_reproducibility_workflow.py fails when a shipped element is missing.
FINAL_ASSEMBLY: >-
oci/bluefin-server-image.bst
oci/bluefin-server-boot.bst
oci/bluefin-server-usr.bst
oci/bluefin-server-sbom.bst
oci/k0s-sysext.bst
oci/kubestellar-sysext.bst
oci/kubeadm-sysext.bst
oci/homelab-sysext.bst
oci/argo-workflows-sysext.bst
oci/mcp-sysext.bst
oci/zfs-sysext.bst
oci/nvidia-open-595-sysext.bst
oci/nvidia-container-toolkit-sysext.bst
bluefin-server/os-sd-boot-signed.bst
homelab/homelab-templates.bst
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -51,19 +58,7 @@ jobs:
tool: just@1.58.0

- name: Set up runner disk space
run: |
set -euo pipefail
sudo mkdir -p /mnt/podman /mnt/buildstream
sudo chown -R "$USER:$USER" /mnt/podman /mnt/buildstream
mkdir -p ~/.config/containers
cat > ~/.config/containers/storage.conf <<EOF
[storage]
driver = "overlay"
runroot = "/run/user/1001"
graphroot = "/mnt/podman"
EOF
mkdir -p ~/.cache
ln -sf /mnt/buildstream ~/.cache/buildstream
run: bash scripts/ci-runner-disk.sh

# Throwaway keys and a 0.* version: nothing from this job is published.
- name: Generate keys and set the image version
Expand Down
45 changes: 7 additions & 38 deletions .github/workflows/track-binaries.yml
Original file line number Diff line number Diff line change
Expand Up @@ -107,46 +107,15 @@ jobs:
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
branch="auto/track-binaries/${COMPONENT}"
title="chore(deps): update ${COMPONENT} ${FROM} -> ${TO}"
if [ ! -s "$RUNNER_TEMP/files.txt" ]; then
echo "${COMPONENT} needs no change."
exit 0
fi

# One pull request per component. An open one with this exact title
# already proposes this release, so leave it and its CI run alone. A
# closed one with this title was turned down (or reverted); do not
# reopen it every day, the next release brings a new title. No
# pipes into grep and no `|| true`: a failed query must stop here,
# not pass for "no pull request".
open=$(gh pr list --head "$branch" --base main --state open --json number,title --jq '.[0] | select(.) | "\(.number)\t\(.title)"')
number="${open%%$'\t'*}"
if [ -n "$open" ] && [ "${open#*$'\t'}" = "$title" ]; then
echo "#${number} already proposes ${COMPONENT} ${TO}."
exit 0
fi
closed=$(gh pr list --head "$branch" --base main --state closed --json title --jq '.[].title')
if [ -z "$open" ] && grep -Fxq -- "$title" <<< "$closed"; then
echo "A closed pull request already proposed '${title}'; not reopening it."
exit 0
fi

# No attribution trailers, as in track-junctions.yml: this commit is
# the deterministic output of track-binaries.py.
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -B "$branch"
mapfile -t files < "$RUNNER_TEMP/files.txt"
git add -- "${files[@]}"
git commit -m "$title"
git -c "http.${GITHUB_SERVER_URL}/.extraheader=AUTHORIZATION: basic $(printf 'x-access-token:%s' "$GH_TOKEN" | base64 -w0)" \
push --force-with-lease origin "$branch"

printf '\n---\n*Generated by [Track upstream binaries](%s)*\n' "$RUN_URL" >> "$RUNNER_TEMP/pr-body.md"
if [ -n "$open" ]; then
gh pr edit "$number" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md"
echo "Updated PR: $(gh pr view "$number" --json url --jq '.url')"
else
echo "Created PR: $(gh pr create --base main --head "$branch" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md")"
fi
mapfile -t files < "$RUNNER_TEMP/files.txt"
# One pull request per component. A closed one with this exact title
# was turned down (or reverted); the next release brings a new title.
bash .github/scripts/propose-pr.sh --branch "auto/track-binaries/${COMPONENT}" \
--title "chore(deps): update ${COMPONENT} ${FROM} -> ${TO}" \
--body-file "$RUNNER_TEMP/pr-body.md" --skip-closed-title \
-- "${files[@]}"
Loading