From e515fddcc1c69666d85b4b5c210fe6164f82588b Mon Sep 17 00:00:00 2001 From: Tomas Srnka Date: Fri, 31 Jul 2026 18:24:02 +0200 Subject: [PATCH 1/2] ci: propose the NixOS base image nixpkgs pin bump on a schedule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The base image pins an exact nixpkgs release, which is what makes it reproducible but also what makes it go stale silently: nothing moves the pin, so nothing picks up upstream's security fixes. nixos-26.05 has taken 100+ commits in the last month alone. Add a monthly job that resolves the current release of whatever series the pin is already on and, if it moved, opens a draft PR updating build.sh and the README pin table. It deliberately does not build and does not publish. The value is turning "the userland is drifting" into a reviewable diff that CI can run the distro build test against; publishing stays manual, because the publish job's only gate inspects the rootfs tar, and a tar can pass every assertion while the image cannot boot — which is exactly what 25.05 replacing $toplevel/init with the systemd binary produced. The bot refuses to cross a series: 24.05 -> 26.05 needed an activation shim, a stateVersion decision and real KVM verification, none of which a cron should attempt. It uses the GitHub App token rather than GITHUB_TOKEN so the bump PR actually triggers CI, and it verifies the release's nixexprs tarball is served before proposing a pin the build could not fetch. Also drops the README's duplicate dated release row, so the release id lives in exactly two places the job keeps in sync rather than three it could skew. --- .github/workflows/nixos-pin-bump.yml | 176 ++++++++++++++++++ .../base/distro/nixos-base-image/README.md | 1 - 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/nixos-pin-bump.yml diff --git a/.github/workflows/nixos-pin-bump.yml b/.github/workflows/nixos-pin-bump.yml new file mode 100644 index 0000000000..9b3c38ca64 --- /dev/null +++ b/.github/workflows/nixos-pin-bump.yml @@ -0,0 +1,176 @@ +name: Bump the NixOS base image nixpkgs pin + +# Opens a PR moving the base image's nixpkgs pin to the current release of the +# series it is already on. It never builds and never publishes: the point is to +# turn "the pinned userland is getting stale" into a reviewable diff that CI can +# run the distro boot test against, not to ship an unattended image. +# +# Publishing stays manual (nixos-base-image.yml) and follows the merge, because +# the publish job's own gate only inspects the rootfs tar. A tar can pass every +# assertion and still not boot — observed when NixOS 25.05 turned +# $toplevel/init into the systemd binary. + +on: + schedule: + # 06:00 UTC on the 1st. Monthly: nixpkgs release branches move constantly + # (100+ commits/month on nixos-26.05), but the channel — the revision that + # has been through upstream's own tests, and the only thing worth pinning — + # is re-cut far less often, so a tighter cadence mostly opens empty PRs. + - cron: "0 6 1 * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +env: + BUILD_SH: packages/orchestrator/pkg/template/build/phases/base/distro/nixos-base-image/build.sh + +jobs: + bump: + name: Propose a nixpkgs pin bump + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + # An App token, not GITHUB_TOKEN: PRs opened with GITHUB_TOKEN do not + # trigger workflows, and a bump PR nobody can see CI for is worthless. + - name: Generate GitHub App installation token + id: app-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ secrets.app_id }} + private-key: ${{ secrets.app_secret }} + + - name: Checkout + uses: actions/checkout@v6 + with: + token: ${{ steps.app-token.outputs.token }} + + - name: Resolve the current channel release + id: resolve + run: | + set -euo pipefail + read_default() { + # NIXOS_SERIES=${NIXOS_SERIES:-26.05} -> 26.05 + sed -n "s/^$1=\${$1:-\(.*\)}$/\1/p" "${BUILD_SH}" + } + series=$(read_default NIXOS_SERIES) + current=$(read_default NIXPKGS_RELEASE) + if [[ -z "${series}" || -z "${current}" ]]; then + echo "::error::Could not read NIXOS_SERIES/NIXPKGS_RELEASE from ${BUILD_SH}." \ + "The pin format changed — this job needs updating alongside it." + exit 1 + fi + + # channels.nixos.org redirects to the release directory, whose name is + # the release id we pin. + location=$(curl -sI --max-time 30 "https://channels.nixos.org/nixos-${series}" | + tr -d '\r' | sed -n 's/^[Ll]ocation: //p') + latest=$(basename "${location:-}") + if [[ -z "${latest}" ]]; then + echo "::error::No redirect from channels.nixos.org for nixos-${series};" \ + "cannot tell what the current release is. Failing rather than guessing." + exit 1 + fi + + # Never let the bot cross a series. A series bump is a different kind + # of change: 24.05 -> 26.05 replaced the stage-2 init outright and + # broke the boot until an activation shim was added. That needs a + # human, a `stateVersion` decision and a real boot verification. + if [[ "${latest}" != nixos-${series}.* ]]; then + echo "::error::Channel nixos-${series} now resolves to '${latest}', which is not" \ + "in series ${series}. A series bump must be done by hand — see the" \ + "nixpkgs pin section of the base image README." + exit 1 + fi + + if [[ "${latest}" == "${current}" ]]; then + echo "Pin is already current (${current}); nothing to do." + echo "changed=false" >> "${GITHUB_OUTPUT}" + exit 0 + fi + + # A release directory can exist before its nixexprs tarball is served. + # Proposing a pin the build cannot fetch would just burn a CI run. + url="https://releases.nixos.org/nixos/${series}/${latest}/nixexprs.tar.xz" + code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 60 -I "${url}") + if [[ "${code}" != "200" ]]; then + echo "::error::${url} returned HTTP ${code}; the release is not fully published yet." + exit 1 + fi + + { + echo "changed=true" + echo "series=${series}" + echo "current=${current}" + echo "latest=${latest}" + echo "branch=chore/nixos-pin-${latest}" + } >> "${GITHUB_OUTPUT}" + echo "Pin bump available: ${current} -> ${latest}" + + - name: Open the bump PR + if: ${{ steps.resolve.outputs.changed == 'true' }} + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + BRANCH: ${{ steps.resolve.outputs.branch }} + CURRENT: ${{ steps.resolve.outputs.current }} + LATEST: ${{ steps.resolve.outputs.latest }} + SERIES: ${{ steps.resolve.outputs.series }} + run: | + set -euo pipefail + if gh pr list --head "${BRANCH}" --state open --json number --jq 'length' | grep -qv '^0$'; then + echo "A PR for ${BRANCH} is already open; leaving it alone." + exit 0 + fi + + # The release id appears in build.sh (the functional pin) and in the + # README's pin table. Replace it in both, or the docs silently drift — + # the table is the only place a reader looks up what is pinned. + before=$(sha256sum "${BUILD_SH}") + sed -i "s|^NIXPKGS_RELEASE=\${NIXPKGS_RELEASE:-${CURRENT}}$|NIXPKGS_RELEASE=\${NIXPKGS_RELEASE:-${LATEST}}|" "${BUILD_SH}" + if [[ "$(sha256sum "${BUILD_SH}")" == "${before}" ]]; then + echo "::error::Pin line did not change; the assignment format in ${BUILD_SH} moved." + exit 1 + fi + readme="$(dirname "${BUILD_SH}")/README.md" + sed -i "s|${CURRENT}|${LATEST}|g" "${readme}" + if grep -q "${CURRENT}" "${BUILD_SH}" "${readme}"; then + echo "::error::'${CURRENT}' still appears after the bump:" + grep -n "${CURRENT}" "${BUILD_SH}" "${readme}" + exit 1 + fi + + git config user.name "e2b-bot" + git config user.email "bot@e2b.dev" + git checkout -b "${BRANCH}" + + cat > /tmp/commit-msg.txt < /tmp/pr-body.md < Date: Fri, 31 Jul 2026 18:35:57 +0200 Subject: [PATCH 2/2] ci: use the version-bumper App and handle an orphaned bump branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review findings, both real. secrets.app_id/app_secret are workflow_call secret inputs declared by pr-no-generated-changes.yml, not repository secrets, so minting the token would have failed on every scheduled run; use the version-bumper App (vars.VERSION_BUMPER_APPID / secrets.VERSION_BUMPER_SECRET), which is org-level and exactly this job's purpose. The retry guard also only skipped on an open PR, so a branch left behind by a failed run made the next push non-fast-forward and the bump could never reopen. Skip on a PR in any state — a closed one means the release was rejected and re-proposing it monthly would be nagging — and delete a branch that no PR references at all. --- .github/workflows/nixos-pin-bump.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nixos-pin-bump.yml b/.github/workflows/nixos-pin-bump.yml index 9b3c38ca64..0bad826cfa 100644 --- a/.github/workflows/nixos-pin-bump.yml +++ b/.github/workflows/nixos-pin-bump.yml @@ -41,8 +41,8 @@ jobs: id: app-token uses: actions/create-github-app-token@v2 with: - app-id: ${{ secrets.app_id }} - private-key: ${{ secrets.app_secret }} + app-id: ${{ vars.VERSION_BUMPER_APPID }} + private-key: ${{ secrets.VERSION_BUMPER_SECRET }} - name: Checkout uses: actions/checkout@v6 @@ -121,11 +121,23 @@ jobs: SERIES: ${{ steps.resolve.outputs.series }} run: | set -euo pipefail - if gh pr list --head "${BRANCH}" --state open --json number --jq 'length' | grep -qv '^0$'; then - echo "A PR for ${BRANCH} is already open; leaving it alone." + # Any state, not just open: a closed PR for this exact release means a + # human rejected it, and re-proposing it every month would be nagging. + # The branch name carries the release id, so once upstream cuts a newer + # one this stops mattering by itself. + if [[ "$(gh pr list --head "${BRANCH}" --state all --json number --jq 'length')" != "0" ]]; then + echo "A PR for ${BRANCH} already exists; not re-proposing." exit 0 fi + # No PR ever existed, but the branch might still be there from a run + # that pushed and then failed. Left in place it makes the push below + # non-fast-forward and the bump would never reopen. + if git ls-remote --exit-code --heads origin "${BRANCH}" >/dev/null 2>&1; then + echo "Deleting orphaned remote branch ${BRANCH} (no PR references it)." + git push origin --delete "${BRANCH}" + fi + # The release id appears in build.sh (the functional pin) and in the # README's pin table. Replace it in both, or the docs silently drift — # the table is the only place a reader looks up what is pinned.