builder: fail-closed sccache gate with mutation tests

.github/scripts/assert-image.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+# usage: assert-image.sh <image> <cyclonedx-sbom.json>
+set -u
+HERE=$(cd "$(dirname "$0")" && pwd)
+IMG="$1"; SBOM="$2"; fail=0
+check() { d=$1; shift; if "$@"; then echo "PASS  $d"; else echo "FAIL  $d"; fail=1; fi; }
+count_purl() { jq --arg p "$1" '[.components[]|select((.purl//"")|startswith($p))]|length' "$SBOM"; }
+
+have_sccache()  { podman run --rm --entrypoint sh "$IMG" -c 'test -x /opt/cargo/bin/sccache'; }
+no_dist_bin()   { ! podman run --rm --entrypoint sh "$IMG" -c 'test -e /opt/cargo/bin/sccache-dist'; }
+sbom_has_cargo(){ [ "$(count_purl pkg:cargo/)" -ge 100 ]; }
+no_netstack()   { for c in opendal reqwest rustls openssl hyper-rustls; do [ "$(count_purl "pkg:cargo/$c")" = 0 ] || return 1; done; }
+pinned()        { [ "$(count_purl "pkg:cargo/$1")" -ge 1 ]; }
+frame_env()     { podman image inspect "$IMG" --format '{{range .Config.Env}}{{println .}}{{end}}' | grep -q '^SCCACHE_MAX_FRAME_LENGTH='; }
+listen_ok()     { podman run --rm -v "$HERE/listen-check.sh:/listen-check.sh:ro" --entrypoint sh "$IMG" /listen-check.sh; }
+
+check "control: sccache binary exists in image"           have_sccache
+check "no sccache-dist binary"                            no_dist_bin
+check "control: SBOM has >=100 cargo components"          sbom_has_cargo
+check "no network-stack crates in SBOM"                   no_netstack
+check "pinned tar@0.4.40 present"                         pinned tar@0.4.40
+check "pinned bytes@1.10.1 present"                       pinned bytes@1.10.1
+check "pinned rand@0.8.5 present"                         pinned rand@0.8.5
+check "SCCACHE_MAX_FRAME_LENGTH set in image"             frame_env
+check "server seen on loopback; no other listeners"       listen_ok
+exit $fail

.github/scripts/listen-check.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+# runs inside the image: start sccache on a known port, list TCP listeners
+export SCCACHE_SERVER_PORT=4777 SCCACHE_DIR=/tmp/c
+sccache --start-server >/dev/null 2>&1
+sleep 1
+awk '$4=="0A" {print $2}' /proc/net/tcp /proc/net/tcp6 2>/dev/null > /tmp/listeners
+sccache --stop-server >/dev/null 2>&1
+grep -q '^0100007F:12A9$' /tmp/listeners || { echo "control failed: no loopback listener on 4777 seen"; exit 2; }
+if grep -v -E '^(0100007F:|00000000000000000000000001000000:)' /tmp/listeners | grep -q .; then
+  echo "non-loopback listener(s):"; grep -v -E '^(0100007F:|00000000000000000000000001000000:)' /tmp/listeners; exit 1
+fi

.github/workflows/build-environment-image.yml

@@ -6,6 +6,7 @@ on:
     paths:
       - "environments/alpine-3.24-musl/Containerfile"
       - "environments/alpine-3.24-musl/requirements-builder.txt"
+      - "environments/alpine-3.24-musl/assert-sccache-build.sh"
       - "environments/alpine-3.24-musl/apk.lock.*"
       - ".github/workflows/build-environment-image.yml"
   workflow_dispatch: {}
@@ -42,7 +43,7 @@ jobs:
         id: tag
         env:
           ARCH: ${{ matrix.arch }}
-        run: echo "tag=${{ hashFiles('environments/alpine-3.24-musl/Containerfile', 'environments/alpine-3.24-musl/requirements-builder.txt') }}-${ARCH}" >> "$GITHUB_OUTPUT"
+        run: echo "tag=${{ hashFiles('environments/alpine-3.24-musl/Containerfile', 'environments/alpine-3.24-musl/requirements-builder.txt', 'environments/alpine-3.24-musl/assert-sccache-build.sh') }}-${ARCH}" >> "$GITHUB_OUTPUT"
 
       - name: Log in to GHCR
         env:
@@ -148,13 +149,16 @@ jobs:
             --arg rustup_sha256 "$(arg "$rustup_sha_var")" \
             --arg rust_version "$(arg RUST_VERSION)" \
             --arg sccache_version "$(arg SCCACHE_VERSION)" \
+            --arg cargo_auditable_version "$(arg CARGO_AUDITABLE_VERSION)" \
+            --arg assert_script_sha256 "$(sha256sum "$CONTEXT/assert-sccache-build.sh" | cut -d' ' -f1)" \
             --arg requirements_sha256 "$(sha256sum "$CONTEXT/requirements-builder.txt" | cut -d' ' -f1)" \
             --arg apk_lock_sha256 "$(sha256sum "$CONTEXT/apk.lock.$ARCH" | cut -d' ' -f1)" \
             --arg builder_image "$IMAGE@$DIGEST" \
             --arg commit "$COMMIT" \
             '{base_image_ref:$base_ref, base_image_index_digest:$base_index, base_platform:$base_platform,
               base_platform_manifest_digest:$base_platform_digest, rustup_version:$rustup_version,
               rustup_sha256:$rustup_sha256, rust_version:$rust_version, sccache_version:$sccache_version,
+              cargo_auditable_version:$cargo_auditable_version, assert_script_sha256:$assert_script_sha256,
               requirements_builder_sha256:$requirements_sha256, apk_lock_sha256:$apk_lock_sha256,
               builder_image:$builder_image, source_commit:$commit}' > "builder-inputs.$ARCH.json"
           cat "builder-inputs.$ARCH.json"

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+.pytest_cache/

environments/alpine-3.24-musl/Containerfile

@@ -84,16 +84,19 @@ ARG CARGO_AUDITABLE_VERSION=0.7.7
 # in the binary so Syft/Trivy scan what was actually compiled. The tool, the crate
 # sources and the registry are removed in the same layer. The guard fails the build if
 # the network stack is back.
+COPY assert-sccache-build.sh /tmp/assert-sccache-build.sh
 RUN set -eux; \
     cargo install cargo-auditable --version "${CARGO_AUDITABLE_VERSION}" --locked --root /tmp/auditable; \
     PATH="/tmp/auditable/bin:${PATH}" \
         cargo auditable install sccache --version "${SCCACHE_VERSION}" --locked --no-default-features; \
-    rm -rf /tmp/auditable /opt/cargo/registry /opt/cargo/git; \
+    sh /tmp/assert-sccache-build.sh "${SCCACHE_VERSION}"; \
+    rm -rf /tmp/auditable /tmp/assert-sccache-build.sh /opt/cargo/registry /opt/cargo/git; \
     sccache --version | grep -F "${SCCACHE_VERSION}"; \
     if grep -q -a -e opendal -e reqwest /opt/cargo/bin/sccache; then \
         echo "sccache contains network backend code" >&2; exit 1; fi
 ENV RUSTC_WRAPPER=/opt/cargo/bin/sccache \
     SCCACHE_DIR=/sccache \
+    SCCACHE_MAX_FRAME_LENGTH=8388608 \
     CARGO_TARGET_DIR=/cargo-target
 
 # maturin: exact version with hashes for both architectures.

environments/alpine-3.24-musl/assert-sccache-build.sh

@@ -0,0 +1,157 @@
+#!/bin/sh
+# Build-time gate for the sccache install. Runs in the same layer as the
+# install, before the cargo registry is deleted. Exit non-zero => no image.
+#
+# The VEX statements in vex/builder.openvex.json depend on these facts:
+# change a pin, review the VEX (tests/ checks the two stay in sync).
+#
+# Each check is its own assert_* function so tests/ can prove every one of
+# them rejects a deliberate violation. The Containerfile passes only the
+# version; the --*-dir/--tree-file flags exist for the tests.
+set -eu
+
+usage() {
+  echo "usage: $0 <sccache-version> [--tree-file F] [--src-dir D] [--bin-dir D] [--out-dir D] [--cargo-home D]" >&2
+  exit 2
+}
+[ $# -ge 1 ] || usage
+V="$1"; shift
+TREE_FILE=""; SRC_DIR=""; BIN_DIR=/opt/cargo/bin; OUT_DIR=/usr/share/builder; CARGO_HOME_DIR=/opt/cargo
+while [ $# -gt 0 ]; do
+  [ $# -ge 2 ] || usage
+  case "$1" in
+    --tree-file) TREE_FILE="$2" ;;
+    --src-dir)   SRC_DIR="$2" ;;
+    --bin-dir)   BIN_DIR="$2" ;;
+    --out-dir)   OUT_DIR="$2" ;;
+    --cargo-home) CARGO_HOME_DIR="$2" ;;
+    *) usage ;;
+  esac
+  shift 2
+done
+
+# crate version: the VEX file must mention each as <crate>@<version>.
+PINNED_CRATES='tar 0.4.40
+bytes 1.10.1
+rand 0.8.5'
+FORBIDDEN_CRATES='opendal reqwest rustls rustls-webpki openssl hyper-rustls'
+
+fail() { echo "ASSERT FAIL: $*" >&2; exit 1; }
+
+load_inputs() {
+  if [ -n "$TREE_FILE" ]; then
+    TREE=$(sort -u "$TREE_FILE")
+  fi
+  if [ -z "$SRC_DIR" ]; then
+    SRC_DIR=$(ls -d /opt/cargo/registry/src/*/sccache-"$V" 2>/dev/null) \
+      || fail "sccache $V sources not found in the cargo registry"
+  fi
+  [ -d "$SRC_DIR" ] || fail "source dir missing: $SRC_DIR"
+  cd "$SRC_DIR"
+  if [ -z "$TREE_FILE" ]; then
+    TREE=$(cargo tree --locked --no-default-features -e normal --prefix none | sort -u)
+  fi
+  [ -n "$TREE" ] || fail "empty dependency tree"
+  SRC=$(find src -name '*.rs')
+  [ -n "$SRC" ] || fail "no .rs files found under src/"
+}
+
+# The dependency tree below is computed with --no-default-features. That only
+# describes the real binary if the install used the same flags, so verify what
+# cargo recorded for the actual install (not what the Containerfile says).
+assert_install_flags() {
+  msg=$(python3 - "$CARGO_HOME_DIR/.crates2.json" "$V" <<'PYEOF'
+import json, sys
+path, v = sys.argv[1:3]
+try:
+    installs = json.load(open(path))["installs"]
+except Exception as e:
+    print(f"cannot read cargo install record {path}: {e}")
+    sys.exit(1)
+keys = [k for k in installs if k.startswith(f"sccache {v} ")]
+if len(keys) != 1:
+    print(f"expected exactly one install record for sccache {v}, found {len(keys)}")
+    sys.exit(1)
+r = installs[keys[0]]
+if r.get("no_default_features") is not True:
+    print("sccache was installed WITHOUT --no-default-features")
+    sys.exit(1)
+if r.get("features") != [] or r.get("all_features") is not False:
+    print(f"sccache was installed with extra features: {r.get('features')} all={r.get('all_features')}")
+    sys.exit(1)
+if r.get("bins") != ["sccache"]:
+    print(f"unexpected installed binaries: {r.get('bins')}")
+    sys.exit(1)
+PYEOF
+  ) || fail "install flags: $msg"
+}
+
+assert_version_matches() {
+  found=$(sed -n 's/^version = "\(.*\)"$/\1/p' Cargo.toml | head -n 1)
+  [ "$found" = "$V" ] || fail "Cargo.toml version is '$found', expected $V"
+}
+
+assert_no_network_crates() {
+  for c in $FORBIDDEN_CRATES; do
+    if echo "$TREE" | grep -q "^$c "; then fail "network-stack crate in dependency tree: $c"; fi
+  done
+}
+
+assert_pinned_crates() {
+  while read -r name ver; do
+    [ -n "$name" ] || continue
+    ver_re=$(printf '%s' "$ver" | sed 's/\./\\./g')
+    echo "$TREE" | grep -Eq "^$name v$ver_re( |\$)" \
+      || fail "expected $name v$ver in tree; review the VEX before changing pins"
+  done <<EOF2
+$PINNED_CRATES
+EOF2
+}
+
+assert_tar_confined_to_dist() {
+  grep -l 'tar::Archive' $SRC | grep -q '^src/bin/sccache-dist/' \
+    || fail "control: expected tar::Archive in src/bin/sccache-dist"
+  other=$(grep -l 'tar::Archive' $SRC | grep -v '^src/bin/sccache-dist/' || true)
+  [ -z "$other" ] || fail "tar::Archive used outside sccache-dist: $other"
+}
+
+assert_dist_gated() {
+  grep -q 'required-features = \["dist-server"\]' Cargo.toml \
+    || fail "sccache-dist no longer gated by dist-server"
+}
+
+assert_no_thread_rng() {
+  hits=$(grep -n -E 'thread_rng|rand::rng\(|ThreadRng' $SRC || true)
+  [ -z "$hits" ] || fail "thread-local rand pattern found: $hits"
+}
+
+assert_no_custom_logger() {
+  hits=$(grep -n -E 'set_logger|set_boxed_logger|impl[^{]*Log for' $SRC || true)
+  [ -z "$hits" ] || fail "custom-logger pattern found: $hits"
+}
+
+assert_no_dist_binary() {
+  [ ! -e "$BIN_DIR/sccache-dist" ] || fail "sccache-dist binary present"
+}
+
+write_deps_record() {
+  mkdir -p "$OUT_DIR"
+  echo "$TREE" > "$OUT_DIR/sccache-$V.deps.txt"
+}
+
+main() {
+  load_inputs
+  assert_install_flags
+  assert_version_matches
+  assert_no_network_crates
+  assert_pinned_crates
+  assert_tar_confined_to_dist
+  assert_dist_gated
+  assert_no_thread_rng
+  assert_no_custom_logger
+  assert_no_dist_binary
+  write_deps_record
+  echo "sccache build assertions passed"
+}
+
+main

environments/alpine-3.24-musl/vex/builder.openvex.json

@@ -0,0 +1,43 @@
+{
+  "@context": "https://openvex.dev/ns/v0.2.0",
+  "@id": "https://github.com/patrickryankenneth/python-wheels-builder/vex/2026-10-03",
+  "author": "patrickryankenneth@gmail.com",
+  "timestamp": "2026-10-03T18:49:01Z",
+  "version": 1,
+  "statements": [
+    {
+      "vulnerability": {"name": "CVE-2026-33055"},
+      "products": [{"@id": "pkg:cargo/tar@0.4.40"}],
+      "status": "not_affected",
+      "justification": "vulnerable_code_not_in_execute_path",
+      "impact_statement": "tar 0.4.40 is linked into sccache 0.18.0 only for creating archives (tar::Builder). The vulnerable behavior is in reading archives. tar::Archive is used only under src/bin/sccache-dist/, which requires the dist-server feature. The builder installs with --no-default-features and the image has no sccache-dist binary (asserted by assert-image.sh). Review by 2027-01-03, or on any sccache or tar version change."
+    },
+    {
+      "vulnerability": {"name": "CVE-2026-33056"},
+      "products": [{"@id": "pkg:cargo/tar@0.4.40"}],
+      "status": "not_affected",
+      "justification": "vulnerable_code_not_in_execute_path",
+      "impact_statement": "The flaw is in unpacking crafted archives. sccache calls no unpack path outside sccache-dist, which is not built or shipped (see CVE-2026-33055). Review by 2027-01-03, or on any sccache or tar version change."
+    },
+    {
+      "vulnerability": {"name": "GHSA-3pv8-6f4r-ffg2"},
+      "products": [{"@id": "pkg:cargo/tar@0.4.40"}],
+      "status": "not_affected",
+      "justification": "vulnerable_code_not_in_execute_path",
+      "impact_statement": "The flaw is PAX header desynchronization when reading a tar stream. sccache reads tar streams only in sccache-dist, which is not built or shipped (see CVE-2026-33055). Review by 2027-01-03, or on any sccache or tar version change."
+    },
+    {
+      "vulnerability": {"name": "GHSA-cq8v-f236-94qc"},
+      "products": [{"@id": "pkg:cargo/rand@0.8.5"}],
+      "status": "not_affected",
+      "justification": "vulnerable_code_not_in_execute_path",
+      "impact_statement": "The advisory requires a custom logger that calls into rand. sccache 0.18.0 source has no set_logger, set_boxed_logger or impl Log, and its only rand usage is OsRng and RngCore in four files. Review by 2027-01-03, or on any sccache or rand version change."
+    },
+    {
+      "vulnerability": {"name": "CVE-2026-25541", "aliases": ["GHSA-434x-w66g-qw3r"]},
+      "products": [{"@id": "pkg:cargo/bytes@1.10.1"}],
+      "status": "affected",
+      "action_statement": "Accepted, not suppressed. bytes 1.10.1 is in sccache's server protocol path (BytesMut in src/server.rs). Supporting controls, not a proof of non-exploitability: the server binds 127.0.0.1 or a Unix socket, and the frame length is capped at 8 MiB. Upstream main still resolves bytes 1.10.1 and has no release after 0.18.0. Severity MEDIUM, below the HIGH gate. Review by 2027-01-03, or when sccache updates bytes."
+    }
+  ]
+}

tests/conftest.py

@@ -0,0 +1,24 @@
+import shutil
+
+import pytest
+
+
+def pytest_addoption(parser):
+    parser.addoption("--integration", action="store_true",
+                     help="also run the slow podman image-build tests")
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "integration: builds the real image with podman")
+
+
+def pytest_collection_modifyitems(config, items):
+    skip = None
+    if not config.getoption("--integration"):
+        skip = pytest.mark.skip(reason="needs --integration")
+    elif shutil.which("podman") is None:
+        skip = pytest.mark.skip(reason="podman not installed")
+    if skip:
+        for item in items:
+            if "integration" in item.keywords:
+                item.add_marker(skip)