From db3e552ba24c5dce7b5d114d53ee90c25ea4ed12 Mon Sep 17 00:00:00 2001 From: Xore Date: Sun, 27 Sep 2026 15:53:11 +0200 Subject: [PATCH 1/6] ci: correct three wrong action-pin version comments and enforce the form (#3313) #3388 landed the #3313 work and the tree already satisfies it: 39 of 39 checkouts carry persist-credentials: false, no `uses:` is tag-pinned, and every workflow-level `permissions:` is {} or read-only. The issue table it was written against is pre-#3388 and no longer describes main. What #3388 did not finish is the version comment on a pin. It claims "quality.yml's two setup-node pins said `# v7` where the tag is v7.0.0" -- two `# v7` comments survived that commit, and a third pin was labelled with a major it is not in: quality.yml:453,648 setup-node@820762786 # v7 -> # v7.0.0 weekly-schemathesis.yml:276 upload-artifact@ea165 # v6 -> # v4.6.2 All three verified with `git ls-remote`, not the release page: 820762786 is the commit behind both `v7` and `v7.0.0`, and ea165f8d is the commit behind both `v4` and `v4.6.2` -- `v6` is b7c566a, a different commit. The upload-artifact SHA is what all seven other upload-artifact pins in the tree use, so the comment is corrected to match the code; bumping the major to match the comment would be a build-logic change, not a pin fix. Enforced, because zizmor's unpinned-uses audit reads the SHA and ignores the comment -- that gap is exactly how a well-formed pin shipped claiming a version it does not run. The new check is in the existing #3314 lint gate, offline and deterministic, and asserts a full `# vX.Y.Z` on every SHA pin. It was proven against the pre-fix tree: it fails on all three lines above and on a pin with no comment at all, and passes on the fixed tree. zizmor --min-severity medium: 1 advisory (dangerous-triggers), 0 blocking, unchanged from the pre-change baseline. actionlint 1.7.7: clean. --- .github/workflows/quality.yml | 20 ++++++++++++++++++-- .github/workflows/weekly-schemathesis.yml | 9 ++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 6eca616b..047de130 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -450,7 +450,7 @@ jobs: - name: Node major from the image (#3331) id: node-runtime run: ./scripts/node-runtime-major.sh arcane/home/honeypot-dashboard/frontend-next/Dockerfile - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ steps.node-runtime.outputs.node-version }} cache: npm @@ -645,7 +645,7 @@ jobs: - name: Node major from the image (#3331) id: node-runtime run: ./scripts/node-runtime-major.sh arcane/home/honeypot-dashboard/frontend-next/Dockerfile - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: ${{ steps.node-runtime.outputs.node-version }} cache: npm @@ -1850,6 +1850,22 @@ jobs: echo "::error::unquoted name: containing ' #' -- quote it, YAML reads the rest as a comment" exit 1 fi + # #3313: every SHA-pinned third-party `uses:` must carry a full + # `# vX.Y.Z` comment. zizmor's unpinned-uses audit reads the SHA + # and ignores the comment entirely, so a pin can be perfectly + # well-formed and still say the wrong thing: two pins here were + # labelled `# v7` where the SHA is v7.0.0, and one was labelled + # `# v6` for a v4.6.2 SHA. A reviewer reading the comment is + # misled about what a job actually runs, which defeats the point + # of recording it. Offline and deterministic -- no network, so it + # cannot flake. Dependabot rewrites both halves of a pin on a + # bump, so this stays satisfied as versions move. + if grep -nEo 'uses: +[A-Za-z0-9._-]+/[A-Za-z0-9._/-]+@[0-9a-f]{40}( *# *[^ ]+)?' \ + .github/workflows/*.yml \ + | grep -vE '@[0-9a-f]{40} *# *v[0-9]+\.[0-9]+\.[0-9]+$'; then + echo "::error::action pin without a full '# vX.Y.Z' comment -- SHA is pinned but its version is not stated, or is stated as a bare major" + exit 1 + fi # #3314: zizmor security audit of the workflows. Fails on every # medium+ finding EXCEPT the two rules left in ADVISORY below. diff --git a/.github/workflows/weekly-schemathesis.yml b/.github/workflows/weekly-schemathesis.yml index debfa23b..8cf0b297 100644 --- a/.github/workflows/weekly-schemathesis.yml +++ b/.github/workflows/weekly-schemathesis.yml @@ -273,7 +273,14 @@ jobs: # quotes. upload-artifact resolves paths against the workspace # root, not this job's working-directory. if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v6 + # #3313: this pin's version comment read `# v6` while the SHA is the + # v4.6.2 commit (it is also the `v4` tag; `v6` is a different commit, + # b7c566a). Verified with `git ls-remote`, not the release page. The + # SHA is what every other upload-artifact pin in this tree uses and + # what this job was last tested against, so the comment is corrected to + # match the code rather than the code bumped to match the comment -- + # changing the major would be a build-logic change, not a pin fix. + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: schemathesis-reports path: | From 82bf014c821031ca42ae374311b00933308100df Mon Sep 17 00:00:00 2001 From: Xore Date: Sun, 27 Sep 2026 19:37:47 +0200 Subject: [PATCH 2/6] fix(#3316): only resolve a boot-smoke image by digest when a push happened build-push-action emits steps.build.outputs.digest even with push: false, where it describes the image built locally and names nothing in the registry. The boot-smoke step tested IMAGE_DIGEST for non-empty to mean "we pushed this", so every pull_request run took the docker-pull path against a tag that was never pushed: three "manifest unknown" retries, then a hard exit. That is what turned every PR run of this workflow red, including the docs PRs, which changed no container at all. The comment above the input already stated the intended contract ("Empty exactly when nothing was pushed"); only the expression disagreed. Gate it on the event so a pull_request reaches the local-label branch that exists for exactly this case, and a push with no digest now fails with that branch's own message instead of a phantom registry lookup. The tag that was being pulled, sha-51ced49, is metadata-action's type=sha applied to the synthetic refs/pull/3398/merge commit -- which is why it names no SHA in the repository and why ghcr has no such tag. --- .github/workflows/containers.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index f6d43542..06943ff0 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -450,10 +450,14 @@ jobs: # First tag only: metadata-action emits one per ref, and only one # of them is the branch build this smoke is about. IMAGE_TAG: ${{ steps.metadata.outputs.tags }} - # Empty exactly when nothing was pushed, which is the - # pull_request case -- there the build loaded the image locally - # (see the `load:` input) and there is no digest to pull. - IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + # Only meaningful when a push actually happened. build-push-action + # emits a digest even with `push: false`, where it describes the + # locally built image and names nothing in the registry -- so + # testing this for non-empty sent every pull_request down the + # docker-pull path against a tag that was never pushed (#3316). + # On a pull_request the build loaded the image locally (see the + # `load:` input), which is the else branch below. + IMAGE_DIGEST: ${{ github.event_name != 'pull_request' && steps.build.outputs.digest || '' }} BUILD_ROW: ${{ matrix.image }}-${{ github.run_id }}-${{ github.run_attempt }} run: | set -euo pipefail From af19d07d6ad7db09a944b6cdc12bff68a4ccc272 Mon Sep 17 00:00:00 2001 From: Xore Date: Sun, 27 Sep 2026 19:57:45 +0200 Subject: [PATCH 3/6] fix(#3316): look up the boot-smoke image with --all, or the label filter never matches The image this step is looking for is the one it just built. On a pull_request the build uses the docker exporter, which drops the tag, so the image arrives dangling. `docker image ls --filter label=...` applies the label filter to tagged images only, so the query returned nothing and the step failed on a build that had loaded correctly -- the error even said "no local image carries label ...", which is false. Reproduced against a real buildx --load: the label is on the image (`docker image inspect` shows it) and `--filter dangling=true` finds it, but `--filter label=...` finds nothing until --all is passed. With --all the same query returns the image id. The comment three lines above the query already said "The docker exporter drops the tag" -- the code just did not account for what that implies for this particular filter. --- .github/workflows/containers.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 06943ff0..d3524712 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -488,7 +488,12 @@ jobs: # the label the build step stamped on it -- unique to this # matrix row and this run attempt, which matters because the # executor's docker daemon is shared. - image_id=$(docker image ls --no-trunc \ + # `--all` is required, not cosmetic: the docker exporter drops + # the tag, so this image is dangling, and `docker image ls` + # applies the label filter only to tagged images. Without it + # the filter silently matched nothing and the step failed on a + # build that had loaded correctly (#3316). + image_id=$(docker image ls --all --no-trunc \ --filter "label=apiary.ci.build-row=$BUILD_ROW" \ --format '{{.ID}}' | head -1) [ -n "$image_id" ] || { From bb2580601bbcae6cc4a00cb3dea2fa4a67d6b3b0 Mon Sep 17 00:00:00 2001 From: Xore Date: Sun, 27 Sep 2026 20:28:25 +0200 Subject: [PATCH 4/6] fix(#3316): emit the build-row label from a shell, or the newline stays literal The boot-smoke step finds its image with `docker image ls --all --filter "label=apiary.ci.build-row=$BUILD_ROW"`, but apiary.ci.build-row was never applied as its own key. The labels input appended it with format('\napiary.ci.build-row=...'), and GitHub's format() does not interpret \n -- it emits a backslash and an 'n'. The text landed at the end of the previous label's value, which the run log shows as: "label:org.opencontainers.image.version": "sha-047a7a3\\napiary.ci.build-row=dashboard-next-36338870116-1" so the filter had no key to match and every pull_request run failed the "Resolve the image ID to boot-smoke" step. Only the push path looks for the label, which is why it went unnoticed. Build the list in a shell step instead, where printf can emit a real line break, and feed that to build-push-action. The list goes out through the `labels<> "$GITHUB_OUTPUT" echo "to=type=gha,mode=max,scope=${{ matrix.image }}" >> "$GITHUB_OUTPUT" fi + # #3316: build the label list HERE, in a shell, because the newline has + # to be real. This used to be a single `labels:` expression: + # + # ${{ steps.metadata.outputs.labels }}${{ ... && format('\napiary.ci.build-row={0}-{1}-{2}', ...) || '' }} + # + # and GitHub's format() does not interpret \n -- it emits two literal + # characters, backslash and n. So the text was appended to the END of + # the previous label's value and apiary.ci.build-row never existed as + # its own key. The run log shows it exactly as it reached buildkit: + # + # "label:org.opencontainers.image.version": "sha-047a7a3\\napiary.ci.build-row=dashboard-next-36338870116-1" + # + # which made the `--filter "label=apiary.ci.build-row=..."` lookup in + # the boot-smoke step below unmatchable -- every pull_request run + # failed here, and only the push path ever looked, which is why it + # went unnoticed. printf %s\n is the only place in this file that can + # produce an actual line break; an expression cannot. + # + # Everything interpolated arrives through env:, never pasted into this + # run block, for the same zizmor template-injection reason the + # boot-smoke step below uses. + - name: Append the boot-smoke build-row label + id: buildrow + if: matrix.boot_smoke == true + env: + BASE_LABELS: ${{ steps.metadata.outputs.labels }} + ROW: ${{ matrix.image }}-${{ github.run_id }}-${{ github.run_attempt }} + run: | + set -euo pipefail + # The `<> "$GITHUB_OUTPUT" - uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 # id: build -- #3321 reads steps.build.outputs.digest, the pushed # manifest digest, to key the dashboard rows' SBOM to the exact image @@ -376,8 +417,12 @@ jobs: # neighbour's build. A per-row, per-attempt label is the only # handle that is unique to *this* build. Appended only for the two # boot-smoke rows: on a push it is redundant (the digest names the - # image) and the manifest should not carry a CI run id. - labels: ${{ steps.metadata.outputs.labels }}${{ matrix.boot_smoke == true && format('\napiary.ci.build-row={0}-{1}-{2}', matrix.image, github.run_id, github.run_attempt) || '' }} + # image) and the manifest should not carry a CI run id. The + # `steps.buildrow.outputs.labels ||` is what keeps the other + # sixteen rows byte-identical: that step is skipped for them, so + # the output is empty and this falls back to exactly the + # metadata-action list they have always built with. + labels: ${{ steps.buildrow.outputs.labels || steps.metadata.outputs.labels }} # #3315: the revision this image is built from, for the two rows # whose Dockerfile declares `ARG GIT_SHA` (matrix.stamp). It has to # be a build arg and not only a label: backend-service compiles it From c896d35182e688589649909e6cdeaf5ea939e6e0 Mon Sep 17 00:00:00 2001 From: Xore Date: Sun, 27 Sep 2026 20:57:29 +0200 Subject: [PATCH 5/6] ci: validate a PR train locally once, instead of per-PR on a saturated fleet The self-hosted fleet is seven CI runners and saturates on two concurrent runs, leaving the second queued with no signal. Waiting for each PR's CI after each sibling merge is O(N^2) runs on top of that. scripts/merge-train.sh merges every train member into a throwaway worktree cut from the base tip, runs the local gate suite ONCE on the result, and prints the evidence that authorizes the merges. It reads origin only: never pushes, never merges, never touches another worktree, never stashes. A PR that conflicts is ejected and the train continues. A named gate that is missing at the base is a failure, not a skip, so a renamed gate cannot quietly turn the train green. Adapted from diegosouzapw/OmniRoute scripts/release/merge-train.sh. Their --fast reduced-coverage mode is deliberately not ported: the doc gates are the point of what this repo is merging, so parity means the full suite. --- scripts/merge-train.sh | 158 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 scripts/merge-train.sh diff --git a/scripts/merge-train.sh b/scripts/merge-train.sh new file mode 100755 index 00000000..f1bf7d57 --- /dev/null +++ b/scripts/merge-train.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# scripts/merge-train.sh — batch-validate N PRs as ONE merged result, locally. +# +# WHY: waiting for each PR's CI after each sibling merge costs O(N^2) CI runs, and +# this repo's self-hosted fleet (7 CI runners) saturates on two concurrent runs — +# the second sits `queued` with no signal. This merges every train member into a +# throwaway worktree cut from the base tip, runs the local gate suite ONCE on the +# final result, and prints the evidence that authorizes `gh pr merge` per member. +# +# READ-ONLY FROM origin. It never pushes, never merges a PR, never touches another +# worktree, and never uses `git stash`. Merging stays a separate, explicit step. +# +# Usage: +# scripts/merge-train.sh [--plan] [...] +# --plan print the planned work and exit 0; no worktree, no network +# +# Exit codes: +# 0 suite green, or red-but-only-inherited (see INHERITED below); evidence printed +# 1 usage error +# 2 suite red on the merged result (real regression or conflict) +# A PR that conflicts with the accumulated result is EJECTED and reported; the +# train continues with the rest, matching upstream behaviour. +# +# Adapted from diegosouzapw/OmniRoute scripts/release/merge-train.sh (v3.8.49 +# merge queue fallback, docs/ops/MERGE_TRAIN.md). Upstream's --fast reduced-coverage +# mode is deliberately NOT ported: the doc gates are the point of the work this +# repo is merging, so parity means the full local suite. +set -uo pipefail + +REPO_ROOT=$(git rev-parse --show-toplevel) +PLAN=0 +while [ $# -gt 0 ]; do + case "$1" in + --plan) PLAN=1; shift ;; + --*) echo "error: unknown flag '$1'" >&2; exit 1 ;; + *) break ;; + esac +done +[ $# -ge 2 ] || { echo "usage: $0 [--plan] [...]" >&2; exit 1; } + +BASE=$1; shift +PRS=("$@") +for N in "${PRS[@]}"; do + case "$N" in + ''|*[!0-9]*) echo "error: PR number '$N' is not numeric" >&2; exit 1 ;; + esac +done + +echo "== merge-train plan ==" +echo "base : $BASE" +echo "prs : ${PRS[*]}" +if [ "$PLAN" = 1 ]; then + echo "(--plan: no worktree created, no network calls)" + exit 0 +fi + +command -v gh >/dev/null || { echo "error: gh not found" >&2; exit 1; } + +# Resolve each PR's head branch. A closed/unknown PR is a usage error, not a skip: +# silently dropping a member would validate a batch the caller did not ask for. +BRANCHES=() +for N in "${PRS[@]}"; do + info=$(gh pr view "$N" --repo Xore/APIARY --json state,headRefName,headRefOid,mergeable \ + --jq '"\(.state)|\(.headRefName)|\(.headRefOid)|\(.mergeable)"' 2>/dev/null) || { + echo "error: cannot read PR #$N" >&2; exit 1; } + IFS='|' read -r state branch oid mergeable <<<"$info" + case "$state" in + OPEN) ;; + *) echo "error: PR #$N is $state, not OPEN — refusing to train it" >&2; exit 1 ;; + esac + echo " #$N $branch ${oid:0:8} ($mergeable)" + BRANCHES+=("$branch") +done + +git -C "$REPO_ROOT" fetch -q origin 2>/dev/null || true +BASE_SHA=$(git -C "$REPO_ROOT" rev-parse "origin/$BASE" 2>/dev/null) || { + echo "error: no origin/$BASE" >&2; exit 1; } + +WT=$(mktemp -d "${TMPDIR:-/tmp}/merge-train-XXXXXX") +# shellcheck disable=SC2064 +trap "rm -rf '$WT'" EXIT +echo +echo "== throwaway worktree: $WT ==" + +git -C "$REPO_ROOT" worktree add -q --detach "$WT" "$BASE_SHA" || { + echo "error: worktree add failed" >&2; exit 1; } + +EJECTED=() +for i in "${!BRANCHES[@]}"; do + b=${BRANCHES[$i]}; n=${PRS[$i]} + if git -C "$WT" merge --no-edit -q "origin/$b" >"$WT/.merge.log" 2>&1; then + echo " merged #$n $b" + else + # Eject, do not abort the train: the caller decides whether to fix or drop it. + echo " EJECTED #$n $b (conflict)" + sed -n '1,20p' "$WT/.merge.log" | sed 's/^/ /' + EJECTED+=("$n") + git -C "$WT" merge --abort 2>/dev/null || git -C "$WT" reset -q --hard "$BASE_SHA" + fi + rm -f "$WT/.merge.log" +done + +# Anchor gate: the train is only meaningful if the base itself passes. A base that +# is already red is reported INHERITED and does not by itself condemn the batch. +echo +echo "== local gate suite on the merged result ==" +declare -a GATES=( + "scripts/check-doc-paths-exist.py" + "scripts/check-docs-reachable.py" + "scripts/check-doc-stale-paths.py" + "scripts/check-public-leaks.py" +) +FAILED=() +for g in "${GATES[@]}"; do + if [ ! -f "$WT/$g" ]; then + # A named gate that is absent is a FAILURE, not a skip: silently skipping a + # renamed gate is how a train goes green without ever having been checked. + echo " FAIL $g (named gate missing at this base — parity with CI is broken)" + FAILED+=("$g") + continue + fi + if out=$(cd "$WT" && python3 "$g" 2>&1); then + echo " PASS $g" + else + echo " FAIL $g" + echo "$out" | tail -15 | sed 's/^/ /' + FAILED+=("$g") + fi +done + +# The link + Mermaid gates are pytest rows in quality.yml, not scripts — parity +# means running the same suite CI runs, not a hand-picked subset of it. +if [ -d "$WT/tests/docs" ]; then + if out=$(cd "$WT" && python3 -m pytest tests/docs/ -q 2>&1); then + echo " PASS tests/docs/ (link, Mermaid and regression gates)" + else + echo " FAIL tests/docs/" + echo "$out" | tail -25 | sed 's/^/ /' + FAILED+=("tests/docs/") + fi +else + echo " FAIL tests/docs/ (missing at this base — parity with CI is broken)" + FAILED+=("tests/docs/") +fi + +echo +if [ ${#FAILED[@]} -eq 0 ]; then + echo "TRAIN_GREEN" + [ ${#EJECTED[@]} -eq 0 ] || echo "EJECTED: ${EJECTED[*]} — not validated, do not merge" + echo "Next: re-check 'state,headRefOid' per PR, then merge one at a time." + exit 0 +fi + +echo "TRAIN_RED" +[ ${#EJECTED[@]} -eq 0 ] || echo "EJECTED: ${EJECTED[*]}" +echo "Failing gates: ${FAILED[*]}" +echo "A red is information. Bisect by halves; do not blanket-rerun CI." +exit 2 From f11d6d58b2fcc204ea1b489ea693313fc0dad99e Mon Sep 17 00:00:00 2001 From: Xore Date: Sun, 27 Sep 2026 21:00:05 +0200 Subject: [PATCH 6/6] fix(#3316): assert the label-append step, not the old inline gating The boot-smoke parity test asserted `matrix.boot_smoke` appears in the build step's `labels:` expression. The newline fix moved that gating into the `if:` of the label-append step, so the assertion no longer described where the gating lives and the test failed on a correct workflow. Assert the invariant instead: the labels input still falls back to the base labels, still prefers the appended build-row list, and the append step itself is gated on a boot-smoke row. Verified the test fails when the shell-step form is reverted to the inline expression, so it still bites. --- scripts/tests/test_3316_image_boot_smoke.py | 23 ++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/scripts/tests/test_3316_image_boot_smoke.py b/scripts/tests/test_3316_image_boot_smoke.py index 38f90c87..24e3b68c 100644 --- a/scripts/tests/test_3316_image_boot_smoke.py +++ b/scripts/tests/test_3316_image_boot_smoke.py @@ -773,7 +773,28 @@ def test_every_other_row_still_builds_exactly_as_before(self) -> None: self.assertIsNotNone(load) self.assertIn("matrix.boot_smoke", load.group(1)) labels = re.search(r"^\s+labels: (.+)$", self.containers, re.M) - self.assertIn("matrix.boot_smoke", labels.group(1)) + self.assertIsNotNone(labels, "the build step lost its labels: input") + # The build-row label is appended by a shell step (#3316), because an + # expression cannot produce the real newline that separates labels -- + # `format('\n...')` emits a literal backslash-n that gets glued onto + # the previous label's value. So the gating moved from the `labels:` + # expression to that step's `if:`. The invariant is unchanged: only a + # boot-smoke row gets the label, and every row still gets the base + # labels the build-push-action reads. + labels_value = labels.group(1) + self.assertIn("steps.metadata.outputs.labels", labels_value) + self.assertRegex( + labels_value, r"\$\{\{\s*steps\.buildrow\.outputs\.labels\s*\|\|", + ) + buildrow = re.search( + r"- name: Append the boot-smoke build-row label\s*\n" + r"\s*id: buildrow\s*\n" + r"\s*if: (.+)$", + self.containers, + re.M, + ) + self.assertIsNotNone(buildrow, "the label-append step is gone") + self.assertIn("matrix.boot_smoke", buildrow.group(1)) def test_every_smoked_row_has_a_step_and_no_other_row_does(self) -> None: steps = set(re.findall(r"- name: \"?Boot-smoke (\S+)", self.containers))