ci: apt mirror failover, retried LLVM install, no fail-fast between build variants

.github/workflows/build-reusable.yml

@@ -32,6 +32,8 @@ jobs:
     name: Linux
     runs-on: ${{matrix.os}}
     strategy:
+      # One variant's failure shouldn't cancel the others.
+      fail-fast: false
       matrix:
         include:
           - lto_flag: ""
@@ -305,6 +307,8 @@ jobs:
     name: Linux musl
     runs-on: ${{matrix.os}}
     strategy:
+      # One variant's failure shouldn't cancel the others.
+      fail-fast: false
       matrix:
         include:
           - lto_flag: ""
@@ -390,6 +394,8 @@ jobs:
     # mac-release.bash remains for building on a real Mac locally.
     runs-on: linux-x64-gh
     strategy:
+      # One variant's failure shouldn't cancel the others.
+      fail-fast: false
       matrix:
         include:
           - label: bun-webkit-macos-arm64
@@ -484,6 +490,8 @@ jobs:
     # so all FreeBSD targets build on x64 regardless of target arch.
     runs-on: linux-x64-gh
     strategy:
+      # One variant's failure shouldn't cancel the others.
+      fail-fast: false
       matrix:
         include:
           - label: bun-webkit-freebsd-amd64
@@ -555,6 +563,8 @@ jobs:
     # cross-compile from x64 regardless of target arch.
     runs-on: linux-x64-gh
     strategy:
+      # One variant's failure shouldn't cancel the others.
+      fail-fast: false
       matrix:
         include:
           - label: bun-webkit-linux-arm64-android

Dockerfile

@@ -32,15 +32,18 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 # Both archive.ubuntu.com and azure.archive.ubuntu.com have intermittently
 # timed out from inside the GitHub-hosted docker-buildx network at different
-# times. Prefer Azure (faster on Azure-hosted runners) but fall back to the
-# canonical mirror if `apt-get update` can't reach it. arm64 uses
-# ports.ubuntu.com which has been reachable, so leave it alone.
-RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list
+# times, sometimes halfway through a step. Give apt both through its mirror
+# method -- Azure first (faster on Azure-hosted runners), the canonical mirror
+# as fallback -- so every individual fetch that fails on one, in any later
+# `apt-get` too (llvm.sh, cmake, gcc), is retried on the other; and retry
+# transient errors. arm64 uses ports.ubuntu.com which has been reachable, so
+# its sources.list is left alone (the sed matches nothing there).
+RUN printf 'http://azure.archive.ubuntu.com/ubuntu/\tpriority:1\nhttp://archive.ubuntu.com/ubuntu/\tpriority:2\n' > /etc/apt/mirrors.txt \
+    && sed -i 's|http://archive.ubuntu.com/ubuntu/\?|mirror+file:/etc/apt/mirrors.txt|g' /etc/apt/sources.list \
+    && printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' > /etc/apt/apt.conf.d/80-retries
 
 # Install basic build dependencies
-RUN ( apt-get update || \
-      ( sed -i 's|http://azure.archive.ubuntu.com/ubuntu|http://archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && apt-get update ) \
-    ) && apt-get install -y \
+RUN apt-get update && apt-get install -y \
     wget \
     curl \
     git \
@@ -103,11 +106,9 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 \
     --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-13 \
     --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-13
 
-# Install LLVM 21
-RUN wget https://apt.llvm.org/llvm.sh \
-    && chmod +x llvm.sh \
-    && ./llvm.sh 21 all \
-    && rm llvm.sh \
+# Install LLVM: apt.llvm.org's llvm.sh with retries, see scripts/install-llvm.sh.
+RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \
+    bash /install-llvm.sh ${LLVM_VERSION} all \
     && rm -rf /var/lib/apt/lists/*
 
 # Configure library paths

Dockerfile.android

@@ -20,16 +20,18 @@ ARG LLVM_VERSION
 ARG NDK_VERSION
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    wget unzip xz-utils ca-certificates \
+    wget curl unzip xz-utils ca-certificates \
     cmake ninja-build make git \
     ruby ruby-getoptlong perl python3 rsync file cpio \
     lsb-release software-properties-common gnupg \
     && rm -rf /var/lib/apt/lists/*
 
 # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot,
-# not via the NDK's bundled clang. apt.llvm.org installs version-suffixed names
-# only (ld.lld-21, not ld.lld), so add unversioned links for -fuse-ld=lld.
-RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \
+# not via the NDK's bundled clang. Installed through scripts/install-llvm.sh:
+# apt.llvm.org's llvm.sh with retries. apt.llvm.org installs version-suffixed
+# names only (ld.lld-21, not ld.lld), so add unversioned links for -fuse-ld=lld.
+RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \
+    bash /install-llvm.sh ${LLVM_VERSION} && \
     for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \
       ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \
     done

Dockerfile.freebsd

@@ -23,16 +23,18 @@ ARG FREEBSD_VERSION
 ARG FREEBSD_ARCH
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    wget unzip xz-utils ca-certificates \
+    wget curl unzip xz-utils ca-certificates \
     cmake ninja-build make git \
     ruby ruby-getoptlong perl python3 rsync file cpio \
     lsb-release software-properties-common gnupg \
     && rm -rf /var/lib/apt/lists/*
 
 # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot.
-# apt.llvm.org installs version-suffixed names only, so add unversioned links
-# for -fuse-ld=lld.
-RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \
+# Installed through scripts/install-llvm.sh: apt.llvm.org's llvm.sh with
+# retries. apt.llvm.org installs version-suffixed names only, so add
+# unversioned links for -fuse-ld=lld.
+RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \
+    bash /install-llvm.sh ${LLVM_VERSION} && \
     for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \
       ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \
     done

Dockerfile.macos

@@ -63,16 +63,18 @@ ARG COMPILER_RT_DARWIN_TAG
 ARG COMPILER_RT_DARWIN_SHA256
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    wget unzip xz-utils ca-certificates \
+    wget curl unzip xz-utils ca-certificates \
     cmake ninja-build make git \
     ruby ruby-getoptlong perl python3 rsync file cpio \
     flex bison \
     lsb-release software-properties-common gnupg \
     && rm -rf /var/lib/apt/lists/*
 
 # Host clang (same version Bun uses) — we cross-compile via --target/-isysroot
-# and link with ld64.lld (lld's Mach-O port).
-RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \
+# and link with ld64.lld (lld's Mach-O port). Installed through
+# scripts/install-llvm.sh: apt.llvm.org's llvm.sh with retries.
+RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \
+    bash /install-llvm.sh ${LLVM_VERSION} && \
     for t in clang clang++ ld64.lld lld llvm-ar llvm-ranlib llvm-nm llvm-cxxfilt; do \
       ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \
     done

Dockerfile.windows

@@ -66,7 +66,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     lsb-release software-properties-common gnupg \
     && rm -rf /var/lib/apt/lists/*
 
-RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \
+# Host clang: scripts/install-llvm.sh runs apt.llvm.org's llvm.sh with retries.
+RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \
+    bash /install-llvm.sh ${LLVM_VERSION} && \
     for t in clang clang++ clang-cl lld-link ld.lld llvm-ar llvm-lib llvm-mt llvm-rc llvm-ranlib llvm-objcopy; do \
       ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \
     done

scripts/install-llvm.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Install LLVM from apt.llvm.org inside the Docker builds. Usage, from a
+# Dockerfile RUN step: install-llvm.sh <major version> [all]
+#
+# This runs apt.llvm.org's own llvm.sh, it only adds the retries llvm.sh does
+# not have. Every docker build (about 25 per CI run) fetches llvm.sh, probes the
+# repo and the signing key with HEAD requests, downloads the key, then pulls
+# about 30 debs, all from apt.llvm.org and none of it retried. One failed
+# request fails the build, and during a burst of builds that happens to a few
+# variants per run: the connection to apt.llvm.org fails, or llvm.sh exits with
+# "GPG key not reachable", or a repo probe fails and llvm.sh misreports it as
+# "Distribution 'ubuntu' ... is not supported by this script".
+#
+# So the whole install is one attempt, retried with a growing pause: fetch the
+# key, fetch the script, run it. llvm.sh only runs add-apt-repository, apt-get
+# update and apt-get install, which are idempotent, and it skips its own
+# unretried key download because the key file is already there. The retry is
+# a shell loop rather than curl --retry: curl --retry covers timeouts, a few
+# HTTP codes and (with --retry-connrefused) ECONNREFUSED, but not the other
+# connect failures seen here, and focal's curl has no --retry-all-errors.
+set -euo pipefail
+
+if [ $# -lt 1 ]; then
+  echo "usage: $0 <llvm major version> [all]" >&2
+  exit 2
+fi
+
+# Both files are a few KB. The time limits turn a stalled server into a failed
+# attempt instead of a job that sits there until its 90 minute timeout.
+fetch() {
+  curl -fsSL --connect-timeout 15 --max-time 60 "$1" -o "$2"
+}
+
+attempt() {
+  fetch https://apt.llvm.org/llvm-snapshot.gpg.key /etc/apt/trusted.gpg.d/apt.llvm.org.asc &&
+    fetch https://apt.llvm.org/llvm.sh /tmp/llvm.sh &&
+    bash /tmp/llvm.sh "$@"
+}
+
+for n in 1 2 3 4 5; do
+  if attempt "$@"; then
+    rm -f /tmp/llvm.sh
+    exit 0
+  fi
+  echo "installing LLVM from apt.llvm.org failed (attempt $n of 5)" >&2
+  if [ "$n" -lt 5 ]; then
+    sleep $((n * 5))
+  fi
+done
+exit 1