From 85d8f38b9c16ff67f8334dead453dffb89a1ca9b Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Wed, 19 Aug 2026 09:50:59 +0000 Subject: [PATCH] ci: apt mirror failover, retried LLVM install, no fail-fast On 2026-08-19 about a dozen preview builds failed a few minutes into their docker builds. The job logs show two causes. azure.archive.ubuntu.com was unreachable from the amd64 runners for over an hour (22 of the 30 failed jobs). Dockerfile's fallback for this, `apt-get update || `, never runs: apt-get update exits 0 when a mirror is unreachable and only warns, so the apt-get install after it fails with "has no installation candidate". It also only covered the first of the eight apt-get runs in the image. The Dockerfile hunk here is b840022e0f from #471 (also in #302), unchanged: an apt mirror list with the azure mirror first and archive.ubuntu.com second, pointed at from sources.list with mirror+file:, so apt tries the other mirror for every index and .deb that fails, in every apt-get run. Verified with focal's apt 2.0.2 against a local repo whose first mirror refuses, answers 503, is unreachable, or hangs. apt.llvm.org requests failed (8 jobs, across Dockerfile on arm64 and the freebsd, macos and windows cross images): the download of llvm.sh itself, llvm.sh's HEAD probe of the signing key ("GPG key not reachable"), its HEAD probe of the repo (reported as "Distribution 'ubuntu' ... is not supported"), and the key download. None of it is retried. scripts/install-llvm.sh, now used by all five apt based Dockerfiles, retries the whole install (key, script, llvm.sh) up to five times with a growing pause. llvm.sh is idempotent and skips its own key download when the key file exists. The first version of this script fetched the key and the script with curl --retry outside the loop, and the preview build of that version failed on exactly that: curl retries timeouts, a few HTTP codes and ECONNREFUSED, not the connect failure it got from apt.llvm.org. The linux, linux-musl, macos-cross, freebsd and linux-android matrices had the default fail-fast, so one variant that failed in its first minutes cancelled its healthy siblings, and they all had to be rebuilt. windows-cross already had fail-fast: false. Set it on every matrix, the same hunk as 418804c0a0 in #296. Co-authored-by: Dylan Conway --- .github/workflows/build-reusable.yml | 10 ++++++ Dockerfile | 25 +++++++------- Dockerfile.android | 10 +++--- Dockerfile.freebsd | 10 +++--- Dockerfile.macos | 8 +++-- Dockerfile.windows | 4 ++- scripts/install-llvm.sh | 50 ++++++++++++++++++++++++++++ 7 files changed, 93 insertions(+), 24 deletions(-) create mode 100755 scripts/install-llvm.sh diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index 9cfcdaa7f515..6b0d78c9edb0 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -32,6 +32,8 @@ jobs: name: Linux runs-on: ${{matrix.os}} strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - lto_flag: "" @@ -305,6 +307,8 @@ jobs: name: Linux musl runs-on: ${{matrix.os}} strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - lto_flag: "" @@ -390,6 +394,8 @@ jobs: # mac-release.bash remains for building on a real Mac locally. runs-on: linux-x64-gh strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - label: bun-webkit-macos-arm64 @@ -484,6 +490,8 @@ jobs: # so all FreeBSD targets build on x64 regardless of target arch. runs-on: linux-x64-gh strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - label: bun-webkit-freebsd-amd64 @@ -555,6 +563,8 @@ jobs: # cross-compile from x64 regardless of target arch. runs-on: linux-x64-gh strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - label: bun-webkit-linux-arm64-android diff --git a/Dockerfile b/Dockerfile index afe5c1f43bf0..9a40477104bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,15 +32,18 @@ ENV DEBIAN_FRONTEND=noninteractive # Both archive.ubuntu.com and azure.archive.ubuntu.com have intermittently # timed out from inside the GitHub-hosted docker-buildx network at different -# times. Prefer Azure (faster on Azure-hosted runners) but fall back to the -# canonical mirror if `apt-get update` can't reach it. arm64 uses -# ports.ubuntu.com which has been reachable, so leave it alone. -RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list +# times, sometimes halfway through a step. Give apt both through its mirror +# method -- Azure first (faster on Azure-hosted runners), the canonical mirror +# as fallback -- so every individual fetch that fails on one, in any later +# `apt-get` too (llvm.sh, cmake, gcc), is retried on the other; and retry +# transient errors. arm64 uses ports.ubuntu.com which has been reachable, so +# its sources.list is left alone (the sed matches nothing there). +RUN printf 'http://azure.archive.ubuntu.com/ubuntu/\tpriority:1\nhttp://archive.ubuntu.com/ubuntu/\tpriority:2\n' > /etc/apt/mirrors.txt \ + && sed -i 's|http://archive.ubuntu.com/ubuntu/\?|mirror+file:/etc/apt/mirrors.txt|g' /etc/apt/sources.list \ + && printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' > /etc/apt/apt.conf.d/80-retries # Install basic build dependencies -RUN ( apt-get update || \ - ( sed -i 's|http://azure.archive.ubuntu.com/ubuntu|http://archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && apt-get update ) \ - ) && apt-get install -y \ +RUN apt-get update && apt-get install -y \ wget \ curl \ git \ @@ -103,11 +106,9 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 \ --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-13 \ --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-13 -# Install LLVM 21 -RUN wget https://apt.llvm.org/llvm.sh \ - && chmod +x llvm.sh \ - && ./llvm.sh 21 all \ - && rm llvm.sh \ +# Install LLVM: apt.llvm.org's llvm.sh with retries, see scripts/install-llvm.sh. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} all \ && rm -rf /var/lib/apt/lists/* # Configure library paths diff --git a/Dockerfile.android b/Dockerfile.android index b07ead881617..16eb0ad9eaa5 100644 --- a/Dockerfile.android +++ b/Dockerfile.android @@ -20,16 +20,18 @@ ARG LLVM_VERSION ARG NDK_VERSION RUN apt-get update && apt-get install -y --no-install-recommends \ - wget unzip xz-utils ca-certificates \ + wget curl unzip xz-utils ca-certificates \ cmake ninja-build make git \ ruby ruby-getoptlong perl python3 rsync file cpio \ lsb-release software-properties-common gnupg \ && rm -rf /var/lib/apt/lists/* # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot, -# not via the NDK's bundled clang. apt.llvm.org installs version-suffixed names -# only (ld.lld-21, not ld.lld), so add unversioned links for -fuse-ld=lld. -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# not via the NDK's bundled clang. Installed through scripts/install-llvm.sh: +# apt.llvm.org's llvm.sh with retries. apt.llvm.org installs version-suffixed +# names only (ld.lld-21, not ld.lld), so add unversioned links for -fuse-ld=lld. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} && \ for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.freebsd b/Dockerfile.freebsd index 7dc042f49e13..d4cb3f141f76 100644 --- a/Dockerfile.freebsd +++ b/Dockerfile.freebsd @@ -23,16 +23,18 @@ ARG FREEBSD_VERSION ARG FREEBSD_ARCH RUN apt-get update && apt-get install -y --no-install-recommends \ - wget unzip xz-utils ca-certificates \ + wget curl unzip xz-utils ca-certificates \ cmake ninja-build make git \ ruby ruby-getoptlong perl python3 rsync file cpio \ lsb-release software-properties-common gnupg \ && rm -rf /var/lib/apt/lists/* # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot. -# apt.llvm.org installs version-suffixed names only, so add unversioned links -# for -fuse-ld=lld. -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# Installed through scripts/install-llvm.sh: apt.llvm.org's llvm.sh with +# retries. apt.llvm.org installs version-suffixed names only, so add +# unversioned links for -fuse-ld=lld. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} && \ for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.macos b/Dockerfile.macos index e8d4761f438c..d54feba8d32c 100644 --- a/Dockerfile.macos +++ b/Dockerfile.macos @@ -63,7 +63,7 @@ ARG COMPILER_RT_DARWIN_TAG ARG COMPILER_RT_DARWIN_SHA256 RUN apt-get update && apt-get install -y --no-install-recommends \ - wget unzip xz-utils ca-certificates \ + wget curl unzip xz-utils ca-certificates \ cmake ninja-build make git \ ruby ruby-getoptlong perl python3 rsync file cpio \ flex bison \ @@ -71,8 +71,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Host clang (same version Bun uses) — we cross-compile via --target/-isysroot -# and link with ld64.lld (lld's Mach-O port). -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# and link with ld64.lld (lld's Mach-O port). Installed through +# scripts/install-llvm.sh: apt.llvm.org's llvm.sh with retries. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} && \ for t in clang clang++ ld64.lld lld llvm-ar llvm-ranlib llvm-nm llvm-cxxfilt; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.windows b/Dockerfile.windows index 5edb4fb6bef4..f50c9b57a869 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -66,7 +66,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ lsb-release software-properties-common gnupg \ && rm -rf /var/lib/apt/lists/* -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# Host clang: scripts/install-llvm.sh runs apt.llvm.org's llvm.sh with retries. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} && \ for t in clang clang++ clang-cl lld-link ld.lld llvm-ar llvm-lib llvm-mt llvm-rc llvm-ranlib llvm-objcopy; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/scripts/install-llvm.sh b/scripts/install-llvm.sh new file mode 100755 index 000000000000..2a408da603f1 --- /dev/null +++ b/scripts/install-llvm.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Install LLVM from apt.llvm.org inside the Docker builds. Usage, from a +# Dockerfile RUN step: install-llvm.sh [all] +# +# This runs apt.llvm.org's own llvm.sh, it only adds the retries llvm.sh does +# not have. Every docker build (about 25 per CI run) fetches llvm.sh, probes the +# repo and the signing key with HEAD requests, downloads the key, then pulls +# about 30 debs, all from apt.llvm.org and none of it retried. One failed +# request fails the build, and during a burst of builds that happens to a few +# variants per run: the connection to apt.llvm.org fails, or llvm.sh exits with +# "GPG key not reachable", or a repo probe fails and llvm.sh misreports it as +# "Distribution 'ubuntu' ... is not supported by this script". +# +# So the whole install is one attempt, retried with a growing pause: fetch the +# key, fetch the script, run it. llvm.sh only runs add-apt-repository, apt-get +# update and apt-get install, which are idempotent, and it skips its own +# unretried key download because the key file is already there. The retry is +# a shell loop rather than curl --retry: curl --retry covers timeouts, a few +# HTTP codes and (with --retry-connrefused) ECONNREFUSED, but not the other +# connect failures seen here, and focal's curl has no --retry-all-errors. +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "usage: $0 [all]" >&2 + exit 2 +fi + +# Both files are a few KB. The time limits turn a stalled server into a failed +# attempt instead of a job that sits there until its 90 minute timeout. +fetch() { + curl -fsSL --connect-timeout 15 --max-time 60 "$1" -o "$2" +} + +attempt() { + fetch https://apt.llvm.org/llvm-snapshot.gpg.key /etc/apt/trusted.gpg.d/apt.llvm.org.asc && + fetch https://apt.llvm.org/llvm.sh /tmp/llvm.sh && + bash /tmp/llvm.sh "$@" +} + +for n in 1 2 3 4 5; do + if attempt "$@"; then + rm -f /tmp/llvm.sh + exit 0 + fi + echo "installing LLVM from apt.llvm.org failed (attempt $n of 5)" >&2 + if [ "$n" -lt 5 ]; then + sleep $((n * 5)) + fi +done +exit 1