diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 4fb94b25..e6b7b903 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -21,7 +21,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4.2.2 - name: Run actionlint - uses: reviewdog/action-actionlint@068bf0ce873c499b29c1c799fa813417db01f578 # v1.9.0 + uses: reviewdog/action-actionlint@6fb7acc99f4a1008869fa8a0f09cfca740837d9d # v1.72.0 with: reporter: github-pr-check fail_on_error: true diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index 589243ca..4aa8a13a 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -322,7 +322,6 @@ jobs: if: github.event_name != 'pull_request' shell: bash run: | - PUSH_END=$(date +%s) # push-image action handles its own timing internally; we measure wall-clock from here echo "push_seconds=0" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/skill-drift.yml b/.github/workflows/skill-drift.yml index 789acd2b..d5383eb7 100644 --- a/.github/workflows/skill-drift.yml +++ b/.github/workflows/skill-drift.yml @@ -8,5 +8,5 @@ jobs: skill-drift: uses: ./.github/workflows/skill-drift-check.yml with: - code-paths: '["bootc-build/**/action.yml", ".github/workflows/reusable-*.yml"]' + code-paths: '["bootc-build/**/action.yml", ".github/workflows/reusable-*.yml", ".github/workflows/upgrade-*.yml"]' skill-paths: '["docs/skills/**"]' diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml new file mode 100644 index 00000000..55e3066c --- /dev/null +++ b/.github/workflows/upgrade-test.yml @@ -0,0 +1,86 @@ +name: bootc Upgrade and Rollback Test +# Reusable workflow: post-build gate that boots the newly pushed image in QEMU +# and runs the projectbluefin/testsuite lifecycle suite (upgrade + rollback). +# +# Wire it after your build job and gate promotion on the result: +# +# upgrade-test: +# needs: build +# uses: projectbluefin/actions/.github/workflows/upgrade-test.yml@v1 +# with: +# image: ghcr.io/your-org/your-image@${{ needs.build.outputs.digest }} +# +# promote: +# needs: upgrade-test +# if: needs.upgrade-test.outputs.result == 'success' +# +# The lifecycle suite validates: bootc status, upgrade, reboot into new +# deployment, rollback to previous deployment, and /etc persistence. +# See: https://github.com/projectbluefin/testsuite/blob/main/tests/lifecycle/features/bootc.feature +# +# NOTE: explicit from→to cross-version upgrade path testing requires testsuite +# changes tracked in: +# https://github.com/projectbluefin/testsuite/issues/223 (target-image input) +# https://github.com/projectbluefin/testsuite/issues/222 (no-op upgrade guard) + +on: + workflow_call: + inputs: + image: + description: > + Full OCI image reference to boot and gate on — accepts a tag or pinned digest. + Pass the freshly built digest from your build job to gate promotion on it. + Example (digest gate): ghcr.io/ublue-os/bluefin@sha256:abc123 + Example (tag gate): ghcr.io/ublue-os/bluefin:stable + Example (LTS): ghcr.io/ublue-os/bluefin-lts:lts + required: true + type: string + suites: + description: > + Comma-separated test suites to run. + Default is 'lifecycle' which covers: bootc status, upgrade, rollback, + /etc merge, and idempotency. + Any suite supported by projectbluefin/testsuite may be added. + type: string + default: "lifecycle" + skip_native_apps: + description: "Skip @native_app scenarios (non-Flatpak app tests)" + type: boolean + default: false + screenshot_flatpaks: + description: > + Comma-separated Flatpak app IDs to launch-and-screenshot after the + test run. Leave empty to skip. + type: string + default: "" + + outputs: + result: + description: "Outcome of the upgrade test job: success, failure, cancelled, or skipped" + value: ${{ jobs.report.outputs.result }} + +permissions: + contents: read + packages: write # required by testsuite e2e.yml for screenshot OCI push + +jobs: + upgrade-test: + name: Upgrade & rollback — ${{ inputs.image }} + uses: projectbluefin/testsuite/.github/workflows/e2e.yml@4ca86c64f3ed92dabe24d31f5a08b82fe3a3df8a # main + permissions: + contents: read + packages: write + with: + image: ${{ inputs.image }} + suites: ${{ inputs.suites }} + skip_native_apps: ${{ inputs.skip_native_apps }} + screenshot_flatpaks: ${{ inputs.screenshot_flatpaks }} + + report: + needs: upgrade-test + if: always() + runs-on: ubuntu-latest + outputs: + result: ${{ needs.upgrade-test.result }} + steps: + - run: echo "upgrade-test result=${{ needs.upgrade-test.result }}" diff --git a/AGENTS.md b/AGENTS.md index a41b3ede..27c2842c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,14 +11,14 @@ projectbluefin/actions ←── shared CI building blocks │ ├── projectbluefin/bluefin (bootc desktop image, Path 1 — reusable-build.yml) ├── projectbluefin/bluefin-lts (LTS on CentOS Stream 10, Path 2 — à la carte) - ├── projectbluefin/dakota (BST/BuildStream image, Path 2 — à la carte, deferred) + ├── projectbluefin/dakota (BST/BuildStream image, Path 2 — à la carte; ghcr-cleanup deployed) ├── ublue-os/aurora (Aurora desktop image) └── ublue-os/bazzite (Bazzite gaming image) ``` **Path 1** (full reusable workflow): consumer calls `reusable-build.yml@v1` and satisfies the Justfile contract. Used by bluefin and aurora. -**Path 2** (à la carte composite actions): consumer calls individual actions. Used by bluefin-lts (CentOS Stream 10 base, multi-arch, `chunka` not `rechunk`) and dakota (BST build engine — partial adoption, see issue #16). +**Path 2** (à la carte composite actions): consumer calls individual actions. Used by bluefin-lts (CentOS Stream 10 base, multi-arch, `chunka` not `rechunk`) and dakota (BST build engine; `ghcr-cleanup` deployed, full adoption tracked in issue #16). Actions are referenced as `projectbluefin/actions/bootc-build/@v1`. Breaking changes to an action require a version bump and coordinated update across all consuming repos. @@ -34,7 +34,7 @@ Actions are referenced as `projectbluefin/actions/bootc-build/@v1`. Breaki **No breaking changes without a version signal:** Removing or renaming an input, or changing default behavior, requires coordinating with consuming repos. Document the blast radius in the PR description. -**Consumer validation (required before merging):** For any action change, open a draft PR in at least one consuming repo (`projectbluefin/bluefin` is the primary) pinned to your feature branch SHA. CI must pass there before merging to `main` and moving the `@v1` tag. See the Rollout strategy section in `docs/skills/composite-actions.md` for the full protocol. +**Consumer validation (required before merging):** For any action change, open a draft PR in at least one consuming repo pinned to your feature branch SHA. CI must pass there before merging to `main` and moving the `@v1` tag. Use `projectbluefin/bluefin` as the primary validator for build-pipeline actions. For maintenance-only actions (e.g. `ghcr-cleanup`) that aren't used by bluefin, validate in the affected consumer repo instead (e.g. dakota). See the Rollout strategy section in `docs/skills/composite-actions.md` for the full protocol. **Verification:** Every PR must confirm that the action change was exercised in a real workflow (link to a CI run or test job). No untested changes. diff --git a/bootc-build/ghcr-cleanup/action.yml b/bootc-build/ghcr-cleanup/action.yml index da2cf18c..b4fb391f 100644 --- a/bootc-build/ghcr-cleanup/action.yml +++ b/bootc-build/ghcr-cleanup/action.yml @@ -15,6 +15,9 @@ inputs: keep-n-untagged: description: "Minimum number of untagged images to keep" default: "7" + delete-orphaned-images: + description: "Delete orphaned images (untagged images not referenced by a manifest list)" + default: "true" github-token: description: "Token with packages:write permission" required: true @@ -23,10 +26,11 @@ runs: using: "composite" steps: - name: Clean up old GHCR images - uses: dataaxiom/ghcr-cleanup-action@cd0cdb900b85ee49ba34d2048b2ccb7c55efb9e1 # v1 + uses: dataaxiom/ghcr-cleanup-action@f092b48ba3b604b2a83690dc4b2bbb3392e1045f # v1.2.1 with: packages: ${{ inputs.packages }} older-than: ${{ inputs.older-than }} keep-n-tagged: ${{ inputs.keep-n-tagged }} keep-n-untagged: ${{ inputs.keep-n-untagged }} + delete-orphaned-images: ${{ inputs.delete-orphaned-images }} token: ${{ inputs.github-token }} diff --git a/docs/SKILL.md b/docs/SKILL.md index 4e28efa8..8afeb0ba 100644 --- a/docs/SKILL.md +++ b/docs/SKILL.md @@ -6,6 +6,7 @@ Agent entry point for `projectbluefin/actions`. Load only the skill relevant to | I need to... | Load | |---|---| +| Add a weekly GHCR image cleanup schedule to a consumer repo | `docs/skills/composite-actions.md` → "ghcr-cleanup" | | Detect changed paths for PR build matrix | `docs/skills/composite-actions.md` → "detect-changes" | | Validate a PR (just check, shellcheck, hadolint, pre-commit) | `docs/skills/composite-actions.md` → "validate-pr" | | Author or modify a composite action | `docs/skills/composite-actions.md` | @@ -20,9 +21,12 @@ Agent entry point for `projectbluefin/actions`. Load only the skill relevant to | Author or modify the reusable build workflow | `docs/skills/composite-actions.md` → "Reusable workflow" | | Understand when to use `generate-tags` vs `just generate-build-tags` | `docs/skills/composite-actions.md` → "generate-tags" + "Reusable workflow" | | Wire a consuming repo to the shared reusable workflow | `docs/skills/composite-actions.md` → "Reusable workflow" | +| Add upgrade/rollback testing to a bootc image repo | `docs/skills/consumer-guide.md` → "Upgrade test" | | Use these actions in a new or external bootc image repo | `docs/skills/consumer-guide.md` | | Understand the Justfile contract for the reusable workflow | `docs/skills/consumer-guide.md` → "Justfile contract" | | Wire a skill-drift check to a consumer repo | `docs/skills/consumer-guide.md` → "Checklist" + `docs/skills/factory-operations.md` | +| Understand `force-compression` for CentOS Stream consumers | `docs/skills/composite-actions.md` → "Force-compression input rationale" | +| Understand zstd:chunked compression invariants or run regression tests | `docs/skills/composite-actions.md` → "Compression regression tests" | | Audit non-deterministic surfaces in the factory | `docs/skills/determinism.md` | | Find or report floating action tags | `docs/skills/determinism.md` → "Already pinned" or file a GitHub issue | | Understand the gold standards for determinism | `docs/skills/determinism.md` → "Gold standards" | diff --git a/docs/skills/composite-actions.md b/docs/skills/composite-actions.md index 6a1cfdf6..35e92e86 100644 --- a/docs/skills/composite-actions.md +++ b/docs/skills/composite-actions.md @@ -206,7 +206,7 @@ Key design decisions: - `CHUNKAH_CONFIG_STR=$(sudo podman inspect "${SOURCE}")` passes existing OCI labels through so `containers.bootc=1` and other metadata are preserved. - Mandatory cleanup flags (`--prune /sysroot/ --label ostree.commit- --label ostree.final-diffid-`) strip stale OSTree annotations and are hardcoded — they are correctness requirements, not tuning knobs. - `output-image` defaults to `source-image` (in-place rechunk). -- `force-compression` input is optional and defaults to `false` (preserves existing compression). Use `true` for images that must migrate from existing registry compression (e.g. CentOS Stream bases transitioning from gzip to zstd:chunked). +- `force-compression` input is optional and defaults to `false` (preserves existing compression). When `true`, passes `--compression-format zstd:chunked --force-compression` to `buildah build` — use for images that must migrate existing gzip layers (e.g. CentOS Stream bases transitioning from gzip to zstd:chunked). Fedora consumers must leave this at `false` because Fedora images are already zstd:chunked and forcing recompression strips `ostree.components` layer annotations. **Workarounds carried from consuming repos:** @@ -221,7 +221,28 @@ Key design decisions: ### `ghcr-cleanup` -Thin wrapper around `dataaxiom/ghcr-cleanup-action`. Deletes untagged/old images older than `older-than` (default: 90 days), keeping at least `keep-n-tagged` and `keep-n-untagged` (both default: 7). +Thin wrapper around `dataaxiom/ghcr-cleanup-action`. Deletes untagged/old images older than `older-than` (default: 90 days), keeping at least `keep-n-tagged` and `keep-n-untagged` (both default: 7). Also prunes orphaned images (untagged layers not referenced by any manifest list) when `delete-orphaned-images` is `true` (default). + +**Standard usage** (weekly Sunday schedule, mirrors bluefin `clean.yml`): + +```yaml +jobs: + delete-older-than-90: + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - uses: projectbluefin/actions/bootc-build/ghcr-cleanup@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + packages: myimage,myimage-nvidia # comma-separated GHCR package names + older-than: 90 days + delete-orphaned-images: true + keep-n-tagged: 7 + keep-n-untagged: 7 +``` + +Deploy as a scheduled workflow (`cron: "15 0 * * 0"`) with `workflow_dispatch` for manual runs. Deployed in: bluefin (direct upstream call), dakota (`clean.yml` via this action). ### `detect-changes` @@ -406,6 +427,18 @@ When refactoring a consumer workflow to use shared actions, the real metric is " The `chunka` and `push-image` actions expose an optional `force-compression` input (default: `false`). This input exists for CentOS Stream 10 and other non-Fedora consumers that need to migrate existing registry layers from `gzip` to `zstd:chunked`. Fedora consumers should leave it at the default because Fedora images are already `zstd:chunked` and forcing recompression strips `ostree.components` layer annotations. +In `chunka`, `force-compression: true` passes `--compression-format zstd:chunked --force-compression` to `buildah build`, which forces zstd:chunked on all intermediate layers built by buildah. In `push-image`, it passes `--force-compression` to `podman push`, which forces re-encoding of any remaining gzip layers on the registry side. For CentOS Stream bases, both flags are needed together. + +### Compression regression tests + +`tests/test-zstd-chunked.sh` contains static and functional tests verifying all compression invariants. Run locally with: + +```bash +bash tests/test-zstd-chunked.sh +``` + +The `.github/workflows/test-zstd-chunked.yml` workflow runs these tests automatically on PRs and pushes touching `bootc-build/**` or `reusable-build.yml`. + ### Consumer validation flow 1. Land change on a **feature branch** in this repo @@ -481,3 +514,5 @@ Never add a new inline `uses:` for a third-party action in a consumer workflow i | `-v $(pwd):/run/src` + `--security-opt=label=disable` | `chunka` | buildah < v1.44 drops bind-mounts without these; needed for `out.ociarchive` to survive to final stage | | `sudo rm -f out.ociarchive` | `chunka` | Containerfile.splitter leaves artifact in CWD; stale file breaks re-runs | | `sudo podman save \| podman load` | `chunka` | buildah (root) and podman (user) use separate container stores | +| `reviewdog/action-actionlint` must be ≥ v1.72.0 | `actionlint.yml` | v1.9.0 bundles an actionlint binary that predates `workflow_call` support — all reusable workflows get false-positive errors; `filter_mode: added` hides it on existing files, surfacing only on new files | +| Expose reusable-workflow job result via a `report` job | any `workflow_call` workflow | actionlint cannot model `jobs..result` in `workflow_call` outputs when the job calls a reusable workflow; add an `if: always()` `report` job that outputs `needs..result`, then reference `jobs.report.outputs.result` in `workflow_call` outputs | diff --git a/docs/skills/consumer-guide.md b/docs/skills/consumer-guide.md index 4f0f0a18..31b6acd2 100644 --- a/docs/skills/consumer-guide.md +++ b/docs/skills/consumer-guide.md @@ -263,7 +263,7 @@ When your repo has multiple git remotes (e.g., forked from upstream), take care: | Pushing to the wrong remote | Run `git remote -v` before pushing. If your feature branch is tracking `origin`, but you meant to push to `projectbluefin`, specify `git push projectbluefin ` explicitly. In high-activity repos, always double-check the remote name. | | Feature branches go stale quickly | High-velocity repos like `bluefin-lts` (1400+ merged PRs, constant digest bumps via Renovate) can make a feature branch outdated within hours. Rebase onto the target remote's `main` before opening or re-opening a PR: `git rebase projectbluefin/main` | | Multi-arch matrix stays in the consumer workflow | Do not move matrix orchestration (`generate_matrix`, per-arch build matrix, conditional arm64 logic) into shared actions. Shared actions are per-arch steps; the matrix stays in the consumer workflow. This keeps the action catalog focused and lets each consumer tune its own build strategy. | -| CentOS Stream requires explicit compression | If your consumer is CentOS Stream 10 or another non-Fedora OS, set `force-compression: true` on both `chunka` and `push-image` actions due to zstd layer migration requirements. Fedora consumers can leave this at the default `false`. | +| CentOS Stream requires explicit compression | If your consumer is CentOS Stream 10 or another non-Fedora OS, set `force-compression: true` on both `chunka` and `push-image` actions. In `chunka` this passes `--compression-format zstd:chunked --force-compression` to `buildah build`. In `push-image` it adds `--force-compression` to `podman push`. Fedora consumers must leave both at the default `false` — Fedora images are already zstd:chunked and forcing recompression strips `ostree.components` layer annotations. | --- @@ -294,6 +294,132 @@ When your repo has multiple git remotes (e.g., forked from upstream), take care: --- +## Upgrade test + +A post-build gate: after your image is built and pushed, call this workflow with +the new image ref. It boots it in QEMU via `projectbluefin/testsuite` and runs the +`lifecycle` suite — upgrade, reboot, rollback, `/etc` persistence, and idempotency. +Only if the gate passes should the image be promoted or tagged stable. + +### Minimal wiring — build → gate → promote + +```yaml +jobs: + build: + # ... your build steps ... + outputs: + image: ${{ steps.push.outputs.registry-path }}/your-image + digest: ${{ steps.push.outputs.digest }} + + upgrade-test: + needs: build + uses: projectbluefin/actions/.github/workflows/upgrade-test.yml@v1 + permissions: + contents: read + packages: write + with: + image: ${{ needs.build.outputs.image }}@${{ needs.build.outputs.digest }} + + promote: + needs: upgrade-test + if: needs.upgrade-test.outputs.result == 'success' + # ... move tag, cut release, etc. ... +``` + +### Hello-world: bluefin + +```yaml +# .github/workflows/upgrade-test.yml (in projectbluefin/bluefin) +name: Upgrade Test + +on: + workflow_dispatch: + inputs: + image: + description: "Full OCI ref of the build to gate on" + required: false + default: "ghcr.io/ublue-os/bluefin:stable" + schedule: + - cron: "0 6 * * 1" # weekly on Monday + +jobs: + upgrade-test: + uses: projectbluefin/actions/.github/workflows/upgrade-test.yml@v1 + permissions: + contents: read + packages: write + with: + image: ${{ inputs.image || 'ghcr.io/ublue-os/bluefin:stable' }} +``` + +### Hello-world: bluefin-lts + +```yaml +# .github/workflows/upgrade-test.yml (in projectbluefin/bluefin-lts) +name: Upgrade Test + +on: + workflow_dispatch: + inputs: + image: + description: "Full OCI ref of the build to gate on" + required: false + default: "ghcr.io/ublue-os/bluefin-lts:lts" + schedule: + - cron: "0 6 * * 1" # weekly on Monday + +jobs: + upgrade-test: + uses: projectbluefin/actions/.github/workflows/upgrade-test.yml@v1 + permissions: + contents: read + packages: write + with: + image: ${{ inputs.image || 'ghcr.io/ublue-os/bluefin-lts:lts' }} +``` + +### Inputs reference + +| Input | Required | Default | Description | +|---|---|---|---| +| `image` | **yes** | — | Full OCI ref of the image to gate on — tag or digest (e.g. `ghcr.io/ublue-os/bluefin:stable` or `…@sha256:abc`) | +| `suites` | no | `lifecycle` | Comma-separated testsuite suites to run | +| `skip_native_apps` | no | `false` | Skip `@native_app` scenarios | +| `screenshot_flatpaks` | no | `""` | Comma-separated Flatpak IDs to launch-and-screenshot | + +### Outputs reference + +| Output | Description | +|---|---| +| `result` | Job outcome: `success`, `failure`, `cancelled`, or `skipped` — use in `if:` conditions for downstream promote jobs | + +### What the `lifecycle` suite tests + +| Scenario | Tags | +|---|---| +| `bootc status` reports expected image and is not dirty | `@lifecycle @status` | +| Pin and unpin the current deployment | `@lifecycle @pin` | +| `bootc upgrade` stages a new deployment | `@lifecycle @upgrade` | +| VM boots into upgraded deployment after reboot | `@lifecycle @upgrade @reboot` | +| `bootc rollback` reverts to previous deployment | `@lifecycle @rollback` | +| `/etc` customizations survive upgrade | `@lifecycle @etc_merge` | +| `ostree admin status` shows two deployments | `@lifecycle @ostree` | +| `os-release` version changes tracked after upgrade | `@lifecycle @upgrade @version` | +| `bootc upgrade` is idempotent when already at latest | `@lifecycle @upgrade @idempotent` | +| Auto-update timer is present and not masked | `@lifecycle @autoupdate` | + +Source: [`tests/lifecycle/features/bootc.feature`](https://github.com/projectbluefin/testsuite/blob/main/tests/lifecycle/features/bootc.feature) + +### Permissions required + +```yaml +permissions: + contents: read + packages: write # testsuite pushes desktop screenshots as OCI artifacts +``` + +--- + ## Getting help - File issues at [projectbluefin/actions](https://github.com/projectbluefin/actions/issues) @@ -309,6 +435,6 @@ bluefin-lts uses CentOS Stream 10 (non-Fedora base) and cannot use the full reus |---|---|---| | `validate-pr` | `shellcheck-glob: "build_scripts/**/*.sh"` | lts uses `build_scripts/`, not `build_files/` | | `detect-changes` | `filters:` with `build_scripts/**`, `image-versions.yaml` | default paths are bluefin-specific | -| `chunka` | `force-compression: true` | CentOS base must migrate gzip layers to zstd:chunked | +| `chunka` | `force-compression: true` | CentOS base must migrate gzip layers to zstd:chunked — passes `--compression-format zstd:chunked --force-compression` to `buildah build` | This is the reference implementation for any bootc image repo that diverges from the bluefin path convention. diff --git a/docs/skills/factory-operations.md b/docs/skills/factory-operations.md index 9fd5446e..41fd92b0 100644 --- a/docs/skills/factory-operations.md +++ b/docs/skills/factory-operations.md @@ -108,7 +108,7 @@ These are the committed reference configs currently deployed in each repo's `.gi | Repo | `code-paths` | `skill-paths` | |---|---|---| -| `projectbluefin/actions` | `'["bootc-build/**/action.yml", ".github/workflows/reusable-*.yml"]'` | `'["docs/skills/**"]'` | +| `projectbluefin/actions` | `'["bootc-build/**/action.yml", ".github/workflows/reusable-*.yml", ".github/workflows/upgrade-*.yml"]'` | `'["docs/skills/**"]'` | | `projectbluefin/bluefin` | `'[".github/workflows/**", "build_files/**", "Justfile", "recipes/**"]'` | `'["docs/skills/**", "docs/*.md", "AGENTS.md"]'` | | `projectbluefin/bluefin-lts` | `'[".github/workflows/**", "build_files/**", "Justfile"]'` | `'["docs/skills/**", "docs/*.md", "AGENTS.md"]'` | | `projectbluefin/dakota` | `'[".github/workflows/**", "build_files/**", "Justfile", "elements/**"]'` | `'["docs/skills/**", "docs/*.md", "AGENTS.md"]'` |