lczyk · lczyk · Sep 11, 2026 · Sep 11, 2026 · Sep 11, 2026 · Sep 11, 2026
diff --git a/.github/workflows/binaries.yaml b/.github/workflows/binaries.yaml
@@ -1,9 +1,10 @@
 name: Binaries
 
-# Reusable. Cross-compiles chisel + spread + the docker CLI (via
-# Canonical's ubuntu/go:1.25-26.04_edge builder) for all target
-# arches. Uploads cache/binaries/ + the matching
-# .stamp/binaries so downstream jobs can short-circuit any rebuild.
+# Reusable. One job per target arch cross-compiles chisel + spread + the
+# docker CLI (via Canonical's ubuntu/go:1.25-26.04_edge builder; pure go,
+# so every lane runs on amd64). Each uploads its cache/binaries/ + the
+# matching .stamp/binaries-<arch> as binaries-<arch> so downstream jobs
+# can short-circuit any rebuild.
 
 on:
   workflow_call:
@@ -13,6 +14,10 @@ permissions:
 
 jobs:
   build:
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [amd64, arm64, s390x, ppc64le]
     runs-on: ubuntu-24.04
     timeout-minutes: 30
 
@@ -32,17 +37,17 @@ jobs:
         with:
           path: |
             cache/binaries/
-            .stamp/binaries
-          key: binaries-${{ hashFiles('hack/build_binaries.sh', 'patches/chisel/*.patch', 'makefile') }}
+            .stamp/binaries-${{ matrix.arch }}
+          key: binaries-${{ matrix.arch }}-${{ hashFiles('hack/build_binaries.sh', 'patches/chisel/*.patch', 'makefile') }}
 
-      - name: Build go binaries (chisel + spread + docker static)
+      - name: Build go binaries for ${{ matrix.arch }} (chisel + spread + docker static)
         if: steps.binaries-cache.outputs.cache-hit != 'true'
-        run: make .stamp/binaries
+        run: make binaries ARCH=${{ matrix.arch }}
 
-      - name: Upload cache/binaries + .stamp/binaries
+      - name: Upload cache/binaries + .stamp/binaries-${{ matrix.arch }}
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
-          name: binaries-cache
+          name: binaries-${{ matrix.arch }}
           path: |
             cache/binaries/
-            .stamp/binaries
+            .stamp/binaries-${{ matrix.arch }}
diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml
@@ -32,7 +32,7 @@ jobs:
       - name: Restore cached binaries + stamp
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: binaries-cache
+          name: binaries-${{ matrix.arch }}
           path: .
 
       - name: Build images for ${{ matrix.arch }}

diff --git a/.github/workflows/build-oci.yaml b/.github/workflows/build-oci.yaml
@@ -47,20 +47,20 @@ jobs:
       - name: Restore cached binaries + stamp
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: binaries-cache
+          name: binaries-${{ matrix.arch }}
           path: .
 
       # Cache the saved tarballs keyed on everything that affects image
       # content (same input set hack/hash_inputs.sh stamps on, incl.
-      # .stamp/binaries restored above). Hit -> skip the build entirely;
-      # matters most for the slow qemu lanes. Miss -> build and the cache
-      # auto-saves at job end.
+      # .stamp/binaries-<arch> restored above). Hit -> skip the build
+      # entirely; matters most for the slow qemu lanes. Miss -> build and
+      # the cache auto-saves at job end.
       - name: Restore image tarball cache
         id: images-cache
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: images-out/
-          key: images-${{ matrix.arch }}-${{ hashFiles('images/Dockerfile.*', 'hack/bread-warning.sh', 'hack/banner.txt', 'hack/lazy-apt.sh', 'hack/apt-mirror.sh', 'hack/build_image.sh', '.stamp/binaries') }}
+          key: images-${{ matrix.arch }}-${{ hashFiles('images/Dockerfile.*', 'hack/bread-warning.sh', 'hack/banner.txt', 'hack/lazy-apt.sh', 'hack/apt-mirror.sh', 'hack/build_image.sh', format('.stamp/binaries-{0}', matrix.arch)) }}
 
       - name: Set up qemu (emulated arches only)
         if: matrix.qemu && steps.images-cache.outputs.cache-hit != 'true'

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -51,10 +51,11 @@ jobs:
       - name: Regenerate inlined yamls
         run: make inline
 
-      - name: Restore cached binaries
+      - name: Restore cached binaries (all arches)
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
-          name: binaries-cache
+          pattern: binaries-*
+          merge-multiple: true
           path: .
 
       - name: Assemble release assets

diff --git a/AGENTS.md b/AGENTS.md
@@ -0,0 +1,8 @@
+# agents.md
+
+This file gets edited ONLY by humans, not by agents. If there are thigns you think ought to be saved to long-term project memory, and if you are working in an interactive session, you MAY prompt the user with that suggestion. NEVER make edits to agents.md yourself.
+
+---
+
+## general
+
diff --git a/README.md b/README.md
@@ -94,7 +94,7 @@ the cert-identity regex above is approximate; consult the actual issued cert on
 spread-bread/
   makefile                       # build images + generate inlined yamls + run contract tests
   hack/
-    build_binaries.sh            # cross-compile chisel + spread + docker cli in one ubuntu/go builder
+    build_binaries.sh            # cross-compile chisel + spread + docker cli in an ubuntu/go builder (per arch in ci)
     build_image.sh               # per-image build w/ hash-stamp short-circuit
     hash_inputs.sh               # per-image input hash (drives stamp invalidation)
     check_base.sh                # detect upstream ubuntu base digest drift; rewrite @sha256 pins

diff --git a/hack/build_binaries.sh b/hack/build_binaries.sh
@@ -1,12 +1,15 @@
 #!/usr/bin/env bash
-# Cross-compile chisel + spread + docker cli for all target arches inside a
+# Cross-compile chisel + spread + docker cli for the target arches inside a
 # single Canonical ubuntu/go:1.25-26.04_edge container. Output binaries land
 # in ./cache/binaries/{chisel,chisel-hacked,spread,docker}-<arch>.
 #
 # Required env vars (set by makefile):
 #   CHISEL_REF        git ref (tag, branch, or SHA) for canonical/chisel
 #   SPREAD_REF        git ref for canonical/spread
 #   GO_BUILDER_IMAGE  builder image tag
+#   DOCKER_VERSION    docker/cli tag, without the leading v
+# Optional:
+#   TARGET_ARCHES     space-separated GOARCH list (default: all four)
 set -euo pipefail
 
 cd "$(dirname "$0")/.."
@@ -15,6 +18,7 @@ cd "$(dirname "$0")/.."
 : "${SPREAD_REF:?required}"
 : "${GO_BUILDER_IMAGE:?required}"
 : "${DOCKER_VERSION:?required}"
+TARGET_ARCHES="${TARGET_ARCHES:-amd64 arm64 s390x ppc64le}"
 
 mkdir -p cache/binaries
 
@@ -28,13 +32,13 @@ docker run --rm \
     -e CHISEL_REF="$CHISEL_REF" \
     -e SPREAD_REF="$SPREAD_REF" \
     -e DOCKER_VERSION="$DOCKER_VERSION" \
+    -e TARGET_ARCHES="$TARGET_ARCHES" \
     -e HUID="$HUID" \
     -e HGID="$HGID" \
     "$GO_BUILDER_IMAGE" -ceuo pipefail '
 # Builder runs natively on host arch and cross-compiles via GOARCH for
-# the other arches. Both binaries are pure Go (no CGO), so cross-compile
+# each target arch. All binaries are pure Go (no CGO), so cross-compile
 # is clean.
-TARGET_ARCHES="amd64 arm64 s390x ppc64le"
 
 # Canonical ubuntu/go image has /usr/bin/go as a broken symlink in some
 # revisions; pick the actual go binary out of /usr/lib/go-*/bin.

diff --git a/hack/hash_inputs.sh b/hack/hash_inputs.sh
@@ -4,7 +4,7 @@
 #
 # Usage:
 #   hash_inputs.sh <flavour-ver-arch>     # for image stamps
-#   hash_inputs.sh binaries               # for cache/binaries stamp
+#   hash_inputs.sh binaries-<arch>        # for the per-arch cache/binaries stamp
 #
 # Stdout: hex digest only.
 set -euo pipefail
@@ -13,20 +13,23 @@ cd "$(dirname "$0")/.."
 
 name="$1"
 
-if [ "$name" = "binaries" ]; then
-    # Binary cache hash combines:
-    #   - hack/build_binaries.sh content (drives how things build)
-    #   - CHISEL_REF + SPREAD_REF + GO_BUILDER_IMAGE env vars (drive what is built)
-    : "${CHISEL_REF:?required}"
-    : "${SPREAD_REF:?required}"
-    : "${GO_BUILDER_IMAGE:?required}"
-    : "${DOCKER_VERSION:?required}"
-    { sha256sum hack/build_binaries.sh patches/chisel/*.patch; \
-      printf 'CHISEL_REF=%s\nSPREAD_REF=%s\nGO_BUILDER_IMAGE=%s\nDOCKER_VERSION=%s\n' \
-          "$CHISEL_REF" "$SPREAD_REF" "$GO_BUILDER_IMAGE" "$DOCKER_VERSION"; \
-    } | sha256sum | cut -d' ' -f1
-    exit 0
-fi
+case "$name" in
+    binaries-*)
+        # Binary cache hash combines:
+        #   - hack/build_binaries.sh + the chisel patches (drive how things build)
+        #   - CHISEL_REF + SPREAD_REF + GO_BUILDER_IMAGE + DOCKER_VERSION (drive what is built)
+        #   - the target arch
+        : "${CHISEL_REF:?required}"
+        : "${SPREAD_REF:?required}"
+        : "${GO_BUILDER_IMAGE:?required}"
+        : "${DOCKER_VERSION:?required}"
+        { sha256sum hack/build_binaries.sh patches/chisel/*.patch; \
+          printf 'ARCH=%s\nCHISEL_REF=%s\nSPREAD_REF=%s\nGO_BUILDER_IMAGE=%s\nDOCKER_VERSION=%s\n' \
+              "${name#binaries-}" "$CHISEL_REF" "$SPREAD_REF" "$GO_BUILDER_IMAGE" "$DOCKER_VERSION"; \
+        } | sha256sum | cut -d' ' -f1
+        exit 0
+        ;;
+esac
 
 arch="${name##*-}"
 rest="${name%-*}"
@@ -49,14 +52,14 @@ case "$flavour" in
             "hack/lazy-apt.sh"
             "hack/apt-mirror.sh"
             ".stamp/bread-$ver-$arch"
-            ".stamp/binaries"
+            ".stamp/binaries-$arch"
         )
         ;;
     bread-test)
         inputs=(
             "tests/Dockerfile.bread-test-$ver"
             ".stamp/bread-$ver-$arch"
-            ".stamp/binaries"
+            ".stamp/binaries-$arch"
         )
         ;;
     *)

diff --git a/makefile b/makefile
@@ -70,19 +70,18 @@ build-bread-chisel-releases: $(CHISEL_STAMPS)  ## Build bread-chisel-releases im
 .PHONY: build-bread-test
 build-bread-test: $(BREAD_TEST_STAMPS)  ## Build the bread-test (26.04 only, native arches) test-host image
 
+.PHONY: binaries
+binaries: $(addprefix .stamp/binaries-,$(SELECTED_ARCHES))  ## Cross-compile chisel + spread + docker cli (narrow via ARCH=...)
+
 # Run the contract/integration spread suite (tests/spread.yaml). Builds the
 # test-host image + inlined yamls first. Pass extra spread args via SPREAD_ARGS,
 # e.g. make test SPREAD_ARGS='-debug'.
 SPREAD_ARGS ?=
 # The contract-bread-chisel-releases run task allocates the per-version
 # bread-chisel-releases:<ver>-<arch> images via the host docker socket, so all
-# four versions (host arch) must exist before the suite runs.
-# List the bread base stamps explicitly (not just via the chisel-releases
-# prereq) so make builds them as direct goals -- the .stamp/bread-% pattern
-# also matches bread-chisel-releases-%, so chained-implicit base builds are
-# unreliable.
+# versions (host arch) must exist before the suite runs.
 TEST_STAMPS := .stamp/bread-test-26.04-$(SELECTED_ARCH) \
-	$(foreach v,$(VERSIONS),.stamp/bread-$(v)-$(SELECTED_ARCH) .stamp/bread-chisel-releases-$(v)-$(SELECTED_ARCH))
+	$(foreach v,$(VERSIONS),.stamp/bread-chisel-releases-$(v)-$(SELECTED_ARCH))
 .PHONY: test
 test: $(TEST_STAMPS) inline  ## Run the spread test suite (host arch; ARCH=... to override, SPREAD_ARGS=... for flags)
 	cd tests && spread $(SPREAD_ARGS) outer:ubuntu-26.04-$(SELECTED_ARCH)
@@ -112,35 +111,47 @@ inline: $(INLINED)  ## Generate inlined/*.yaml from templates/*.yaml.in
 .PHONY: FORCE
 FORCE:
 
-.PRECIOUS: .stamp/bread-% .stamp/bread-chisel-releases-% .stamp/bread-test-% .stamp/binaries
+.PRECIOUS: .stamp/bread-% .stamp/bread-chisel-releases-% .stamp/bread-test-% .stamp/binaries-%
 
 .stamp:
 	@mkdir -p $@
 
-# Cross-compile chisel + spread for both arches via a single
-# Canonical ubuntu/go:1.25-26.04_edge builder container. Stamp content =
-# hash of inputs (CHISEL_REF + SPREAD_REF + builder image + script).
+# Static pattern rules over the full matrix: an implicit .stamp/bread-% would
+# also match bread-chisel-releases-* and win whenever a prerequisite stamp is
+# not on disk yet.
+FULL_BINARIES_STAMPS := $(addprefix .stamp/binaries-,$(ARCHES))
+FULL_BREAD_STAMPS    := $(addprefix .stamp/bread-,$(FULL_VER_ARCH))
+FULL_CHISEL_STAMPS   := $(addprefix .stamp/bread-chisel-releases-,$(FULL_VER_ARCH))
+FULL_TEST_STAMPS     := $(addprefix .stamp/bread-test-26.04-,$(ARCHES))
+
+# Cross-compile chisel + spread + docker cli for one arch in a Canonical
+# ubuntu/go:1.25-26.04_edge builder container. Stamp content = hash of inputs
+# (version pins + builder image + script + patches + arch).
 BINARIES_ENV := CHISEL_REF="$(CHISEL_REF)" SPREAD_REF="$(SPREAD_REF)" GO_BUILDER_IMAGE="$(GO_BUILDER_IMAGE)" DOCKER_VERSION="$(DOCKER_VERSION)"
 
-.stamp/binaries: FORCE | .stamp
+$(FULL_BINARIES_STAMPS): .stamp/binaries-%: FORCE | .stamp
 	@set -e ; \
-		new=$$($(BINARIES_ENV) hack/hash_inputs.sh binaries) ; \
+		new=$$($(BINARIES_ENV) hack/hash_inputs.sh binaries-$*) ; \
 		cur=$$(cat $@ 2>/dev/null || true) ; \
 		if [ "$$new" != "$$cur" ]; then \
-			echo "==> building go binaries (chisel + spread + docker, both arches)" ; \
-			$(BINARIES_ENV) hack/build_binaries.sh ; \
+			echo "==> building go binaries for $* (chisel + spread + docker)" ; \
+			$(BINARIES_ENV) TARGET_ARCHES="$*" hack/build_binaries.sh ; \
 			echo "$$new" > $@ ; \
 		else \
-			echo "==> go binaries up-to-date (stamp matches)" ; \
+			echo "==> go binaries for $* up-to-date (stamp matches)" ; \
 		fi
 
-.stamp/bread-%: FORCE | .stamp
+$(FULL_BREAD_STAMPS): .stamp/bread-%: FORCE | .stamp
 	@hack/build_image.sh bread-$*
 
-.stamp/bread-chisel-releases-%: .stamp/bread-% .stamp/binaries FORCE | .stamp
+# The arch is the last dash-separated field of a <ver>-<arch> stem.
+arch_of = $(lastword $(subst -, ,$(1)))
+
+.SECONDEXPANSION:
+$(FULL_CHISEL_STAMPS): .stamp/bread-chisel-releases-%: .stamp/bread-% .stamp/binaries-$$(call arch_of,$$*) FORCE | .stamp
 	@hack/build_image.sh bread-chisel-releases-$*
 
-.stamp/bread-test-%: .stamp/bread-% .stamp/binaries FORCE | .stamp
+$(FULL_TEST_STAMPS): .stamp/bread-test-%: .stamp/bread-% .stamp/binaries-$$(call arch_of,$$*) FORCE | .stamp
 	@hack/build_image.sh bread-test-$*
 
 inlined/%.yaml: templates/%.yaml.in hack/inline_scripts.rb $(SCRIPTS)