diff --git a/.github/workflows/build-reusable.yml b/.github/workflows/build-reusable.yml index 9cfcdaa7f515..6b0d78c9edb0 100644 --- a/.github/workflows/build-reusable.yml +++ b/.github/workflows/build-reusable.yml @@ -32,6 +32,8 @@ jobs: name: Linux runs-on: ${{matrix.os}} strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - lto_flag: "" @@ -305,6 +307,8 @@ jobs: name: Linux musl runs-on: ${{matrix.os}} strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - lto_flag: "" @@ -390,6 +394,8 @@ jobs: # mac-release.bash remains for building on a real Mac locally. runs-on: linux-x64-gh strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - label: bun-webkit-macos-arm64 @@ -484,6 +490,8 @@ jobs: # so all FreeBSD targets build on x64 regardless of target arch. runs-on: linux-x64-gh strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - label: bun-webkit-freebsd-amd64 @@ -555,6 +563,8 @@ jobs: # cross-compile from x64 regardless of target arch. runs-on: linux-x64-gh strategy: + # One variant's failure shouldn't cancel the others. + fail-fast: false matrix: include: - label: bun-webkit-linux-arm64-android diff --git a/Dockerfile b/Dockerfile index afe5c1f43bf0..9a40477104bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,15 +32,18 @@ ENV DEBIAN_FRONTEND=noninteractive # Both archive.ubuntu.com and azure.archive.ubuntu.com have intermittently # timed out from inside the GitHub-hosted docker-buildx network at different -# times. Prefer Azure (faster on Azure-hosted runners) but fall back to the -# canonical mirror if `apt-get update` can't reach it. arm64 uses -# ports.ubuntu.com which has been reachable, so leave it alone. -RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list +# times, sometimes halfway through a step. Give apt both through its mirror +# method -- Azure first (faster on Azure-hosted runners), the canonical mirror +# as fallback -- so every individual fetch that fails on one, in any later +# `apt-get` too (llvm.sh, cmake, gcc), is retried on the other; and retry +# transient errors. arm64 uses ports.ubuntu.com which has been reachable, so +# its sources.list is left alone (the sed matches nothing there). +RUN printf 'http://azure.archive.ubuntu.com/ubuntu/\tpriority:1\nhttp://archive.ubuntu.com/ubuntu/\tpriority:2\n' > /etc/apt/mirrors.txt \ + && sed -i 's|http://archive.ubuntu.com/ubuntu/\?|mirror+file:/etc/apt/mirrors.txt|g' /etc/apt/sources.list \ + && printf 'Acquire::Retries "5";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\n' > /etc/apt/apt.conf.d/80-retries # Install basic build dependencies -RUN ( apt-get update || \ - ( sed -i 's|http://azure.archive.ubuntu.com/ubuntu|http://archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && apt-get update ) \ - ) && apt-get install -y \ +RUN apt-get update && apt-get install -y \ wget \ curl \ git \ @@ -103,11 +106,9 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 \ --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-13 \ --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-13 -# Install LLVM 21 -RUN wget https://apt.llvm.org/llvm.sh \ - && chmod +x llvm.sh \ - && ./llvm.sh 21 all \ - && rm llvm.sh \ +# Install LLVM: apt.llvm.org's llvm.sh with retries, see scripts/install-llvm.sh. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} all \ && rm -rf /var/lib/apt/lists/* # Configure library paths diff --git a/Dockerfile.android b/Dockerfile.android index b07ead881617..16eb0ad9eaa5 100644 --- a/Dockerfile.android +++ b/Dockerfile.android @@ -20,16 +20,18 @@ ARG LLVM_VERSION ARG NDK_VERSION RUN apt-get update && apt-get install -y --no-install-recommends \ - wget unzip xz-utils ca-certificates \ + wget curl unzip xz-utils ca-certificates \ cmake ninja-build make git \ ruby ruby-getoptlong perl python3 rsync file cpio \ lsb-release software-properties-common gnupg \ && rm -rf /var/lib/apt/lists/* # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot, -# not via the NDK's bundled clang. apt.llvm.org installs version-suffixed names -# only (ld.lld-21, not ld.lld), so add unversioned links for -fuse-ld=lld. -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# not via the NDK's bundled clang. Installed through scripts/install-llvm.sh: +# apt.llvm.org's llvm.sh with retries. apt.llvm.org installs version-suffixed +# names only (ld.lld-21, not ld.lld), so add unversioned links for -fuse-ld=lld. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} && \ for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.freebsd b/Dockerfile.freebsd index 7dc042f49e13..d4cb3f141f76 100644 --- a/Dockerfile.freebsd +++ b/Dockerfile.freebsd @@ -23,16 +23,18 @@ ARG FREEBSD_VERSION ARG FREEBSD_ARCH RUN apt-get update && apt-get install -y --no-install-recommends \ - wget unzip xz-utils ca-certificates \ + wget curl unzip xz-utils ca-certificates \ cmake ninja-build make git \ ruby ruby-getoptlong perl python3 rsync file cpio \ lsb-release software-properties-common gnupg \ && rm -rf /var/lib/apt/lists/* # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot. -# apt.llvm.org installs version-suffixed names only, so add unversioned links -# for -fuse-ld=lld. -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# Installed through scripts/install-llvm.sh: apt.llvm.org's llvm.sh with +# retries. apt.llvm.org installs version-suffixed names only, so add +# unversioned links for -fuse-ld=lld. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} && \ for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.macos b/Dockerfile.macos index e8d4761f438c..d54feba8d32c 100644 --- a/Dockerfile.macos +++ b/Dockerfile.macos @@ -63,7 +63,7 @@ ARG COMPILER_RT_DARWIN_TAG ARG COMPILER_RT_DARWIN_SHA256 RUN apt-get update && apt-get install -y --no-install-recommends \ - wget unzip xz-utils ca-certificates \ + wget curl unzip xz-utils ca-certificates \ cmake ninja-build make git \ ruby ruby-getoptlong perl python3 rsync file cpio \ flex bison \ @@ -71,8 +71,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Host clang (same version Bun uses) — we cross-compile via --target/-isysroot -# and link with ld64.lld (lld's Mach-O port). -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# and link with ld64.lld (lld's Mach-O port). Installed through +# scripts/install-llvm.sh: apt.llvm.org's llvm.sh with retries. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} && \ for t in clang clang++ ld64.lld lld llvm-ar llvm-ranlib llvm-nm llvm-cxxfilt; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.windows b/Dockerfile.windows index 5edb4fb6bef4..f50c9b57a869 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -66,7 +66,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ lsb-release software-properties-common gnupg \ && rm -rf /var/lib/apt/lists/* -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# Host clang: scripts/install-llvm.sh runs apt.llvm.org's llvm.sh with retries. +RUN --mount=type=bind,source=scripts/install-llvm.sh,target=/install-llvm.sh \ + bash /install-llvm.sh ${LLVM_VERSION} && \ for t in clang clang++ clang-cl lld-link ld.lld llvm-ar llvm-lib llvm-mt llvm-rc llvm-ranlib llvm-objcopy; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/scripts/install-llvm.sh b/scripts/install-llvm.sh new file mode 100755 index 000000000000..2a408da603f1 --- /dev/null +++ b/scripts/install-llvm.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Install LLVM from apt.llvm.org inside the Docker builds. Usage, from a +# Dockerfile RUN step: install-llvm.sh [all] +# +# This runs apt.llvm.org's own llvm.sh, it only adds the retries llvm.sh does +# not have. Every docker build (about 25 per CI run) fetches llvm.sh, probes the +# repo and the signing key with HEAD requests, downloads the key, then pulls +# about 30 debs, all from apt.llvm.org and none of it retried. One failed +# request fails the build, and during a burst of builds that happens to a few +# variants per run: the connection to apt.llvm.org fails, or llvm.sh exits with +# "GPG key not reachable", or a repo probe fails and llvm.sh misreports it as +# "Distribution 'ubuntu' ... is not supported by this script". +# +# So the whole install is one attempt, retried with a growing pause: fetch the +# key, fetch the script, run it. llvm.sh only runs add-apt-repository, apt-get +# update and apt-get install, which are idempotent, and it skips its own +# unretried key download because the key file is already there. The retry is +# a shell loop rather than curl --retry: curl --retry covers timeouts, a few +# HTTP codes and (with --retry-connrefused) ECONNREFUSED, but not the other +# connect failures seen here, and focal's curl has no --retry-all-errors. +set -euo pipefail + +if [ $# -lt 1 ]; then + echo "usage: $0 [all]" >&2 + exit 2 +fi + +# Both files are a few KB. The time limits turn a stalled server into a failed +# attempt instead of a job that sits there until its 90 minute timeout. +fetch() { + curl -fsSL --connect-timeout 15 --max-time 60 "$1" -o "$2" +} + +attempt() { + fetch https://apt.llvm.org/llvm-snapshot.gpg.key /etc/apt/trusted.gpg.d/apt.llvm.org.asc && + fetch https://apt.llvm.org/llvm.sh /tmp/llvm.sh && + bash /tmp/llvm.sh "$@" +} + +for n in 1 2 3 4 5; do + if attempt "$@"; then + rm -f /tmp/llvm.sh + exit 0 + fi + echo "installing LLVM from apt.llvm.org failed (attempt $n of 5)" >&2 + if [ "$n" -lt 5 ]; then + sleep $((n * 5)) + fi +done +exit 1