fix(26.10): sshd login

.github/workflows/build-oci.yaml

@@ -7,7 +7,8 @@ name: Build OCI image tarballs
 # manifest or attach to releases without rebuilding.
 #
 # amd64 / arm64 build on native runners; s390x / ppc64le have no hosted
-# runners and build under qemu on amd64 (slow, publish-untested).
+# runners and build under qemu on amd64 (slow; published after only a login
+# check).
 #
 # bread-test is intentionally not built here (test-infra-only).
 
@@ -60,10 +61,11 @@ jobs:
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:
           path: images-out/
-          key: images-${{ matrix.arch }}-${{ hashFiles('images/Dockerfile.*', 'hack/bread-warning.sh', 'hack/banner.txt', 'hack/tar-shim.sh', 'hack/lazy-apt.sh', 'hack/apt-mirror.sh', 'hack/build_image.sh', format('.stamp/binaries-{0}', matrix.arch)) }}
+          key: images-${{ matrix.arch }}-${{ hashFiles('images/Dockerfile.*', 'hack/bread-warning.sh', 'hack/banner.txt', 'hack/tar-shim.sh', 'hack/seccomp-shim.c', 'hack/lazy-apt.sh', 'hack/apt-mirror.sh', 'hack/build_image.sh', format('.stamp/binaries-{0}', matrix.arch)) }}
 
+      # Needed on a cache hit too: the login check below runs the images.
       - name: Set up qemu (emulated arches only)
-        if: matrix.qemu && steps.images-cache.outputs.cache-hit != 'true'
+        if: matrix.qemu
         uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4.3.0
 
       # Emulated builds are ~single-core and the per-version chains are
@@ -86,6 +88,43 @@ jobs:
             done
           done
 
+      # A cache hit skips the bake, so the daemon has nothing to run yet.
+      - name: Load cached ${{ matrix.arch }} images
+        if: matrix.qemu && steps.images-cache.outputs.cache-hit == 'true'
+        run: for f in images-out/*.tar; do docker load -i "$f"; done
+
+      # Emulated images get no test suite, so at least log into each one the
+      # way spread does. A running sshd is not enough; see hack/seccomp-shim.c.
+      - name: Log into every ${{ matrix.arch }} image
+        if: matrix.qemu
+        run: |
+          set -e
+          fail() {
+            echo "::error::$1"
+            docker logs "$cid" 2>&1 || true
+            exit 1
+          }
+          for flavour in bread bread-chisel-releases; do
+            for ver in 22.04 24.04 25.10 26.04 26.10; do
+              tag="$flavour:$ver-${{ matrix.arch }}"
+              echo "==> logging into $tag"
+              cid=$(docker run -d --platform "linux/${{ matrix.arch }}" -p 127.0.0.1::22 "$tag")
+              port=$(docker port "$cid" 22 | head -n1 | cut -d: -f2)
+              [ -n "$port" ] || fail "$tag has no published sshd port"
+              ok=""
+              for _ in $(seq 30); do
+                if sshpass -p bread ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+                    -o LogLevel=ERROR -o ConnectTimeout=10 -p "$port" root@127.0.0.1 true; then
+                  ok=1
+                  break
+                fi
+                sleep 1
+              done
+              [ -n "$ok" ] || fail "cannot log into $tag"
+              docker rm -f "$cid" >/dev/null
+            done
+          done
+
       - name: Upload ${{ matrix.arch }} image tarballs
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:

README.md

@@ -20,7 +20,7 @@ prebuilt docker images + ready-to-use spread yamls so you can drop Canonical [`s
 
 ## flavours
 
-two image flavours, each for ubuntu 22.04, 24.04, 25.10, 26.04, 26.10 x amd64 / arm64 / s390x / ppc64le, published as multiarch tags at `ghcr.io/lczyk/spread-bread/<flavour>:<ver>`. heads-up: s390x + ppc64le images build under qemu and ship untested (no native runners); amd64 + arm64 are tested on every release:
+two image flavours, each for ubuntu 22.04, 24.04, 25.10, 26.04, 26.10 x amd64 / arm64 / s390x / ppc64le, published as multiarch tags at `ghcr.io/lczyk/spread-bread/<flavour>:<ver>`. heads-up: s390x + ppc64le images build under qemu and ship with only a login check (no native runners); amd64 + arm64 are tested on every release:
 
 - **`bread`** -- base: ubuntu + sshd. general-purpose spread system; the test suite installs whatever else it needs.
 - **`bread-chisel-releases`** -- bread + `chisel` + the shell + container tooling typically needed by [chisel-releases](https://github.com/canonical/chisel-releases) spread tests (curl, wget, git, jq, file, sudo, tree, docker, skopeo). `chisel` and `docker` are built from source (canonical/chisel + docker/cli pinned by version tag) so the bundled binaries are go 1.25+ and survive qemu emulation.
@@ -61,6 +61,14 @@ ubuntu 26.04's patched GNU tar (`1.35+dfsg-4ubuntu0.x`) resolves extraction path
 
 the images work around it: `/bin/tar` is a shim that probes GNU tar once per container and, if it is broken, routes *extraction* to `bsdtar` (see `hack/tar-shim.sh`). creation stays on GNU tar, which spread needs for `--sort=name` when it packs artifacts. on unaffected hosts the shim is inert.
 
+### sshd under emulation (26.10)
+
+since openssh 10.4, sshd drops a connection before auth when it cannot install its pre-auth seccomp filter, and qemu-user (s390x, ppc64le) and Docker Desktop's Rosetta (amd64 on apple silicon) always refuse that filter. 26.10 ships openssh `10.5p1`, so under emulation every connection ends with `ssh_sandbox_child: prctl(PR_SET_SECCOMP): Invalid argument [preauth]` in sshd's log, and spread gives up with `ssh: handshake failed: EOF`.
+
+the 26.10 images work around it: sshd runs with a preloaded shim that reports success for that one refused call (see `hack/seccomp-shim.c`), so sshd carries on without the sandbox, as openssh did before 10.4. on real hardware the call succeeds and the shim is inert. it only reaches the image's own sshd: one a test starts itself, e.g. from a chisel rootfs under `chroot`, still fails under emulation.
+
+the allocate scripts also wait for a completed ssh handshake (when the host has `ssh-keyscan`) and fail with sshd's log if none comes, so a failure of this kind shows up at allocation rather than as a bare `EOF` from spread.
+
 ## install spread
 
 prefer a precompiled spread CLI over `go install`? same release ships statically-linked binaries for linux amd64 / arm64 / s390x / ppc64le:
@@ -100,6 +108,7 @@ spread-bread/
     check_base.sh                # detect upstream ubuntu base digest drift; rewrite @sha256 pins
     inline_scripts.rb            # splice scripts/*.sh into yaml templates
     tar-shim.sh                  # image /bin/tar; routes extraction to bsdtar where gnu tar is broken
+    seccomp-shim.c               # preloaded into 26.10's sshd; lets it log in where emulation refuses seccomp
     apt-mirror.sh                # build-time apt mirror override, bind-mounted into image builds by ci
   scripts/                       # allocate / discard scripts, one pair per flavour
   images/                        # one Dockerfile per (flavour, ubuntu version)

demo/spread.yaml

@@ -125,12 +125,39 @@ backends:
       fi
 
       if [ "$mode" = publish ]; then
+          host=127.0.0.1
           port=$(docker port "$container_name" 22 | head -n1 | cut -d: -f2)
           [ -n "$port" ] || { echo "could not find published sshd port for $container_name" >&2; exit 1; }
-          ADDRESS "127.0.0.1:$port"
+          address="$host:$port"
       else
-          ADDRESS "$(docker inspect "$container_name" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}')"
+          host=$(docker inspect "$container_name" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}')
+          port=22
+          address="$host"
       fi
+
+      # A running sshd can still fail every handshake (openssh's seccomp
+      # sandbox under emulation does; see hack/seccomp-shim.c). Say so here,
+      # with the container's logs, rather than leave spread to report a bare
+      # EOF.
+      if command -v ssh-keyscan >/dev/null 2>&1; then
+          handshake=""
+          for ((i = 0; i < 10; i++)); do
+              if ssh-keyscan -T 5 -t ed25519 -p "$port" "$host" 2>/dev/null | grep -q .; then
+                  handshake=1
+                  break
+              fi
+              sleep 1
+          done
+          if [ -z "$handshake" ]; then
+              docker logs "$container_name" >&2 2>&1 || true
+              # --rm only fires once the container stops, and a plain exit 1
+              # has spread retry the allocation for 5 minutes; FATAL does not.
+              docker rm -f "$container_name" >/dev/null 2>&1 || true
+              FATAL "sshd in $container_name is running but no ssh handshake completes"
+          fi
+      fi
+
+      ADDRESS "$address"
     discard: |
       #!/bin/bash
 

hack/check_base.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Detect upstream ubuntu base-image drift for the bread base Dockerfiles.
 #
-# Each images/Dockerfile.bread-<ver> pins its base by digest:
+# Each images/Dockerfile.bread-<ver> pins its base by digest in its first FROM:
 #   FROM <registry/repo>:<tag>@sha256:<manifest-list-digest>
 # This script resolves the *live* manifest-list digest for each base and
 # compares it to the pinned one. The pin is part of hack/hash_inputs.sh's
@@ -58,10 +58,9 @@ for df in "${dockerfiles[@]}"; do
     echo "$ver: drift ${pinned:-<unpinned>} -> $live"
 
     if [ "$mode" = "write" ]; then
-        # Exactly one base FROM per file; rewrite that line, keep the rest.
-        awk -v repl="FROM ${base}@${live}" \
-            '/^FROM / && !seen {print repl; seen=1; next} {print}' \
-            "$df" > "$df.tmp"
+        # Swap the ref inside every FROM that uses it: keeps stage names, and
+        # moves a build-platform stage pinned to the same base along with it.
+        sed "/^FROM /s|${from}|${base}@${live}|" "$df" > "$df.tmp"
         mv "$df.tmp" "$df"
     fi
 done

hack/hash_inputs.sh

@@ -43,6 +43,7 @@ case "$flavour" in
             "hack/bread-warning.sh"
             "hack/banner.txt"
             "hack/tar-shim.sh"
+            "hack/seccomp-shim.c"
             "hack/apt-mirror.sh"
         )
         ;;

hack/seccomp-shim.c

@@ -0,0 +1,48 @@
+/*
+ * seccomp-shim: lets sshd take logins where seccomp filters cannot be
+ * installed.
+ *
+ * Preloaded into the image's sshd via LD_PRELOAD in the Dockerfile CMD;
+ * sshd-session and sshd-auth inherit it. openssh 10.4+ aborts the pre-auth
+ * child when prctl(PR_SET_SECCOMP) fails, and qemu-user (s390x, ppc64le) and
+ * Docker Desktop's Rosetta (amd64 on Apple Silicon) refuse that call with
+ * EINVAL. Reporting success for that one failure lets sshd carry on
+ * unsandboxed, as openssh did before 10.4. Natively the call succeeds and the
+ * shim changes nothing.
+ *
+ * Written without libc headers so a bare clang can cross-compile it; the
+ * constants are the same on every Linux arch. prctl stays variadic because
+ * calling glibc's variadic prctl through a fixed-argument pointer is unsafe
+ * on ppc64le.
+ */
+
+#include <stdarg.h>
+
+#define PR_SET_SECCOMP 22
+#define EINVAL 22
+#define RTLD_NEXT ((void *) -1l)
+
+extern void *dlsym(void *, const char *);
+extern int *__errno_location(void);
+
+int prctl(int option, ...)
+{
+    static int (*real)(int, ...);
+    unsigned long a2, a3, a4, a5;
+    va_list ap;
+    int r;
+
+    va_start(ap, option);
+    a2 = va_arg(ap, unsigned long);
+    a3 = va_arg(ap, unsigned long);
+    a4 = va_arg(ap, unsigned long);
+    a5 = va_arg(ap, unsigned long);
+    va_end(ap);
+
+    if (!real)
+        real = (int (*)(int, ...))dlsym(RTLD_NEXT, "prctl");
+    r = real(option, a2, a3, a4, a5);
+    if (r == -1 && option == PR_SET_SECCOMP && *__errno_location() == EINVAL)
+        return 0;
+    return r;
+}

images/Dockerfile.bread-26.10

@@ -1,6 +1,6 @@
 # Bread image: Ubuntu 26.10 + sshd only.
 
-FROM docker.io/library/ubuntu:26.10@sha256:49077a16b772f8bc6e6f160ad2bfc218919f3455037387ed19a8309174328603
+FROM docker.io/library/ubuntu:26.10@sha256:49077a16b772f8bc6e6f160ad2bfc218919f3455037387ed19a8309174328603 AS base
 
 ARG APT_MIRROR=
 
@@ -45,5 +45,31 @@ RUN chmod 0755 /usr/local/bin/bread-tar-shim && \
     dpkg-divert --local --divert /usr/bin/tar.distrib --rename /usr/bin/tar && \
     ln -s /usr/local/bin/bread-tar-shim /usr/bin/tar
 
+# seccomp shim for sshd, inert unless the host's emulation refuses seccomp
+# filters. See hack/seccomp-shim.c. Cross-compiled on the build platform, from
+# the same pin as the base above (hack/check_base.sh moves both).
+FROM --platform=$BUILDPLATFORM docker.io/library/ubuntu:26.10@sha256:49077a16b772f8bc6e6f160ad2bfc218919f3455037387ed19a8309174328603 AS seccomp-shim
+ARG APT_MIRROR=
+RUN --mount=type=bind,source=hack/apt-mirror.sh,target=/usr/local/bin/bread-apt-mirror \
+    --mount=type=tmpfs,target=/run/apt-mirror \
+    export APT_CONFIG=/run/apt-mirror/apt.conf && bread-apt-mirror /run/apt-mirror && \
+    apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends clang lld
+ARG TARGETARCH
+RUN --mount=type=bind,source=hack/seccomp-shim.c,target=/src/seccomp-shim.c \
+    case "$TARGETARCH" in \
+        amd64)   triple=x86_64-linux-gnu ;; \
+        arm64)   triple=aarch64-linux-gnu ;; \
+        s390x)   triple=s390x-linux-gnu ;; \
+        ppc64le) triple=powerpc64le-linux-gnu ;; \
+        *) echo "no seccomp-shim target for $TARGETARCH" >&2; exit 1 ;; \
+    esac && \
+    clang --target="$triple" -O2 -Wall -fPIC -shared -nostdlib -fuse-ld=lld \
+        -o /seccomp-shim.so /src/seccomp-shim.c
+
+FROM base
+COPY --from=seccomp-shim /seccomp-shim.so /usr/local/lib/bread-seccomp-shim.so
+
 EXPOSE 22
-CMD ["/usr/sbin/sshd", "-D"]
+# -e: log to stderr, i.e. to `docker logs`; the container has no syslog.
+CMD ["env", "LD_PRELOAD=/usr/local/lib/bread-seccomp-shim.so", "/usr/sbin/sshd", "-D", "-e"]

inlined/bread-22.04.yaml

@@ -124,13 +124,39 @@ backends:
       fi
 
       if [ "$mode" = publish ]; then
+          host=127.0.0.1
           # The ephemeral host port docker mapped to the container's sshd.
           port=$(docker port "$container_name" 22 | head -n1 | cut -d: -f2)
           [ -n "$port" ] || { echo "could not find published sshd port for $container_name" >&2; exit 1; }
-          ADDRESS "127.0.0.1:$port"
+          address="$host:$port"
       else
-          ADDRESS "$(docker inspect "$container_name" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}')"
+          host=$(docker inspect "$container_name" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}')
+          port=22
+          address="$host"
       fi
+
+      # A running sshd can still fail every handshake (openssh's seccomp sandbox
+      # under emulation does; see hack/seccomp-shim.c). Say so here, with the
+      # container's logs, rather than leave spread to report a bare EOF.
+      if command -v ssh-keyscan >/dev/null 2>&1; then
+          handshake=""
+          for ((i = 0; i < 10; i++)); do
+              if ssh-keyscan -T 5 -t ed25519 -p "$port" "$host" 2>/dev/null | grep -q .; then
+                  handshake=1
+                  break
+              fi
+              sleep 1
+          done
+          if [ -z "$handshake" ]; then
+              docker logs "$container_name" >&2 2>&1 || true
+              # --rm only fires once the container stops, and a plain exit 1 has
+              # spread retry the allocation for 5 minutes; FATAL does not.
+              docker rm -f "$container_name" >/dev/null 2>&1 || true
+              FATAL "sshd in $container_name is running but no ssh handshake completes"
+          fi
+      fi
+
+      ADDRESS "$address"
     discard: |
       #!/bin/bash
 

inlined/bread-24.04.yaml

@@ -124,13 +124,39 @@ backends:
       fi
 
       if [ "$mode" = publish ]; then
+          host=127.0.0.1
           # The ephemeral host port docker mapped to the container's sshd.
           port=$(docker port "$container_name" 22 | head -n1 | cut -d: -f2)
           [ -n "$port" ] || { echo "could not find published sshd port for $container_name" >&2; exit 1; }
-          ADDRESS "127.0.0.1:$port"
+          address="$host:$port"
       else
-          ADDRESS "$(docker inspect "$container_name" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}')"
+          host=$(docker inspect "$container_name" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}')
+          port=22
+          address="$host"
       fi
+
+      # A running sshd can still fail every handshake (openssh's seccomp sandbox
+      # under emulation does; see hack/seccomp-shim.c). Say so here, with the
+      # container's logs, rather than leave spread to report a bare EOF.
+      if command -v ssh-keyscan >/dev/null 2>&1; then
+          handshake=""
+          for ((i = 0; i < 10; i++)); do
+              if ssh-keyscan -T 5 -t ed25519 -p "$port" "$host" 2>/dev/null | grep -q .; then
+                  handshake=1
+                  break
+              fi
+              sleep 1
+          done
+          if [ -z "$handshake" ]; then
+              docker logs "$container_name" >&2 2>&1 || true
+              # --rm only fires once the container stops, and a plain exit 1 has
+              # spread retry the allocation for 5 minutes; FATAL does not.
+              docker rm -f "$container_name" >/dev/null 2>&1 || true
+              FATAL "sshd in $container_name is running but no ssh handshake completes"
+          fi
+      fi
+
+      ADDRESS "$address"
     discard: |
       #!/bin/bash
 

inlined/bread-25.10.yaml

@@ -124,13 +124,39 @@ backends:
       fi
 
       if [ "$mode" = publish ]; then
+          host=127.0.0.1
           # The ephemeral host port docker mapped to the container's sshd.
           port=$(docker port "$container_name" 22 | head -n1 | cut -d: -f2)
           [ -n "$port" ] || { echo "could not find published sshd port for $container_name" >&2; exit 1; }
-          ADDRESS "127.0.0.1:$port"
+          address="$host:$port"
       else
-          ADDRESS "$(docker inspect "$container_name" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}')"
+          host=$(docker inspect "$container_name" --format '{{.NetworkSettings.Networks.bridge.IPAddress}}')
+          port=22
+          address="$host"
       fi
+
+      # A running sshd can still fail every handshake (openssh's seccomp sandbox
+      # under emulation does; see hack/seccomp-shim.c). Say so here, with the
+      # container's logs, rather than leave spread to report a bare EOF.
+      if command -v ssh-keyscan >/dev/null 2>&1; then
+          handshake=""
+          for ((i = 0; i < 10; i++)); do
+              if ssh-keyscan -T 5 -t ed25519 -p "$port" "$host" 2>/dev/null | grep -q .; then
+                  handshake=1
+                  break
+              fi
+              sleep 1
+          done
+          if [ -z "$handshake" ]; then
+              docker logs "$container_name" >&2 2>&1 || true
+              # --rm only fires once the container stops, and a plain exit 1 has
+              # spread retry the allocation for 5 minutes; FATAL does not.
+              docker rm -f "$container_name" >/dev/null 2>&1 || true
+              FATAL "sshd in $container_name is running but no ssh handshake completes"
+          fi
+      fi
+
+      ADDRESS "$address"
     discard: |
       #!/bin/bash