From dbbd5e7c0436c7191aafa271449e0fbaa497a368 Mon Sep 17 00:00:00 2001 From: Sosuke Suzuki Date: Fri, 21 Aug 2026 11:14:37 +0900 Subject: [PATCH] ci: install LLVM from a mirrored GitHub release instead of apt.llvm.org Every docker build fetched llvm.sh and ~30 debs from apt.llvm.org with no cache and no retry, so one failed request there failed the image build. Mirror the debs the Dockerfiles install (the llvm.sh package set, closure resolved by apt) to the llvm-21-debs release, SHA-256 pinned in each Dockerfile, and install them with apt-get install ./*.deb so the Ubuntu-archive dependencies resolve as before. scripts/mirror-llvm-debs.sh regenerates the release; the mirror-llvm-debs workflow runs it on a runner. --- .github/workflows/mirror-llvm-debs.yml | 31 ++++++++ Dockerfile | 22 +++-- Dockerfile.android | 10 ++- Dockerfile.freebsd | 11 ++- Dockerfile.macos | 12 ++- Dockerfile.windows | 10 ++- scripts/mirror-llvm-debs.sh | 106 +++++++++++++++++++++++++ 7 files changed, 190 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/mirror-llvm-debs.yml create mode 100755 scripts/mirror-llvm-debs.sh diff --git a/.github/workflows/mirror-llvm-debs.yml b/.github/workflows/mirror-llvm-debs.yml new file mode 100644 index 000000000000..15f236a7eee9 --- /dev/null +++ b/.github/workflows/mirror-llvm-debs.yml @@ -0,0 +1,31 @@ +name: Mirror LLVM debs + +# Rebuilds the llvm--debs release that the Dockerfiles install LLVM +# from. Run it when the LLVM version changes or apt.llvm.org ships a build +# worth picking up, then pin the printed SHA-256s in the Dockerfiles. +on: + workflow_dispatch: + inputs: + llvm_version: + description: 'LLVM major version to mirror' + type: string + default: '21' + +permissions: + contents: write + +jobs: + mirror: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + sparse-checkout: scripts + - name: Mirror + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + LLVM_VERSION: ${{ inputs.llvm_version }} + OUT: ${{ runner.temp }}/llvm-debs + run: bash scripts/mirror-llvm-debs.sh diff --git a/Dockerfile b/Dockerfile index afe5c1f43bf0..8dff2f78e788 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,12 +103,22 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 \ --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-13 \ --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-13 -# Install LLVM 21 -RUN wget https://apt.llvm.org/llvm.sh \ - && chmod +x llvm.sh \ - && ./llvm.sh 21 all \ - && rm llvm.sh \ - && rm -rf /var/lib/apt/lists/* +# Install LLVM +# The `llvm.sh all` package set, mirrored from apt.llvm.org to a +# GitHub release so the image doesn't depend on apt.llvm.org at build time. +# Ubuntu-archive dependencies still come from apt. Regenerate via +# scripts/mirror-llvm-debs.sh (or the mirror-llvm-debs workflow). +ARG LLVM_DEBS_SHA256_amd64=759ea9d6d50de9b6062cf40161a24a3a9d70aaf11aa1a544074d126590eb55f7 +ARG LLVM_DEBS_SHA256_arm64=4d4923baa663cb2e1be67e8e7097220604489b0da8b7a4ab5911ac2baf1e0ba6 +RUN curl -fsSL --retry 5 --retry-connrefused \ + "https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-focal-${TARGETARCH}.tar.gz" \ + -o /tmp/llvm.tar.gz \ + && eval "expected=\$LLVM_DEBS_SHA256_${TARGETARCH}" \ + && echo "${expected} /tmp/llvm.tar.gz" | sha256sum -c - \ + && mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm \ + && apt-get update \ + && apt-get install -y /tmp/llvm/*.deb \ + && rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* # Configure library paths RUN if [ "$TARGETARCH" = "arm64" ]; then \ diff --git a/Dockerfile.android b/Dockerfile.android index b07ead881617..88b403c094e4 100644 --- a/Dockerfile.android +++ b/Dockerfile.android @@ -2,6 +2,7 @@ ARG MARCH_FLAG="-march=armv8-a+crc -mtune=cortex-a78" ARG WEBKIT_RELEASE_TYPE=Release ARG LTO_FLAG="" ARG LLVM_VERSION="21" +ARG LLVM_DEBS_SHA256="4a79b0eae89af72b082997d361198f16aaefce3fa8ad3c56c8e97311ff31dd5f" ARG NDK_VERSION="r27c" ARG NDK_SHA256="59c2f6dc96743b5daf5d1626684640b20a6bd2b1d85b13156b90333741bad5cc" ARG ANDROID_API="28" @@ -17,6 +18,7 @@ FROM ubuntu:24.04 AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG LLVM_VERSION +ARG LLVM_DEBS_SHA256 ARG NDK_VERSION RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -29,7 +31,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot, # not via the NDK's bundled clang. apt.llvm.org installs version-suffixed names # only (ld.lld-21, not ld.lld), so add unversioned links for -fuse-ld=lld. -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# The debs are mirrored from apt.llvm.org to a GitHub release (see +# scripts/mirror-llvm-debs.sh) so the build doesn't depend on apt.llvm.org. +ADD --checksum=sha256:${LLVM_DEBS_SHA256} \ + https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-noble-amd64.tar.gz /tmp/llvm.tar.gz +RUN mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm && \ + apt-get update && apt-get install -y /tmp/llvm/*.deb && \ + rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* && \ for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.freebsd b/Dockerfile.freebsd index 7dc042f49e13..05bf38e59e8d 100644 --- a/Dockerfile.freebsd +++ b/Dockerfile.freebsd @@ -2,6 +2,7 @@ ARG MARCH_FLAG="-march=nehalem" ARG WEBKIT_RELEASE_TYPE=Release ARG LTO_FLAG="" ARG LLVM_VERSION="21" +ARG LLVM_DEBS_SHA256="4a79b0eae89af72b082997d361198f16aaefce3fa8ad3c56c8e97311ff31dd5f" ARG FREEBSD_VERSION="14.3" ARG FREEBSD_ARCH="x86_64" ARG DEFAULT_CFLAGS="-mno-omit-leaf-frame-pointer -fno-omit-frame-pointer -ffunction-sections -fdata-sections -faddrsig -fno-unwind-tables -fno-asynchronous-unwind-tables -DU_STATIC_IMPLEMENTATION=1 " @@ -19,6 +20,7 @@ FROM ubuntu:24.04 AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG LLVM_VERSION +ARG LLVM_DEBS_SHA256 ARG FREEBSD_VERSION ARG FREEBSD_ARCH @@ -31,8 +33,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot. # apt.llvm.org installs version-suffixed names only, so add unversioned links -# for -fuse-ld=lld. -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# for -fuse-ld=lld. The debs are mirrored from apt.llvm.org to a GitHub +# release (see scripts/mirror-llvm-debs.sh) so the build doesn't depend on it. +ADD --checksum=sha256:${LLVM_DEBS_SHA256} \ + https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-noble-amd64.tar.gz /tmp/llvm.tar.gz +RUN mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm && \ + apt-get update && apt-get install -y /tmp/llvm/*.deb && \ + rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* && \ for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.macos b/Dockerfile.macos index e8d4761f438c..a613843c98d6 100644 --- a/Dockerfile.macos +++ b/Dockerfile.macos @@ -30,6 +30,7 @@ ARG WEBKIT_RELEASE_TYPE=Release ARG LTO_FLAG="" ARG MARCH_FLAG="-mcpu=apple-m1" ARG LLVM_VERSION="21" +ARG LLVM_DEBS_SHA256="4a79b0eae89af72b082997d361198f16aaefce3fa8ad3c56c8e97311ff31dd5f" ARG BOOTSTRAP_CMDS_TAG="bootstrap_cmds-138" # Matches the native macOS lane's CMAKE_C_FLAGS (build-reusable.yml) plus -g # from mac-release.bash. Cross-only additions live in the build stage. @@ -54,6 +55,7 @@ FROM ubuntu:24.04 AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG LLVM_VERSION +ARG LLVM_DEBS_SHA256 ARG MACOS_ARCH ARG MACOS_SDK_VERSION ARG MACOS_SDK_CLT_RELEASE @@ -71,8 +73,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Host clang (same version Bun uses) — we cross-compile via --target/-isysroot -# and link with ld64.lld (lld's Mach-O port). -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# and link with ld64.lld (lld's Mach-O port). The debs are mirrored from +# apt.llvm.org to a GitHub release (see scripts/mirror-llvm-debs.sh) so the +# build doesn't depend on it. +ADD --checksum=sha256:${LLVM_DEBS_SHA256} \ + https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-noble-amd64.tar.gz /tmp/llvm.tar.gz +RUN mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm && \ + apt-get update && apt-get install -y /tmp/llvm/*.deb && \ + rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* && \ for t in clang clang++ ld64.lld lld llvm-ar llvm-ranlib llvm-nm llvm-cxxfilt; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.windows b/Dockerfile.windows index 5edb4fb6bef4..c344daedbefd 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -35,6 +35,7 @@ ARG ENABLE_SANITIZERS="" ARG USE_MIMALLOC="OFF" ARG USE_EXTERNAL_MIMALLOC="OFF" ARG LLVM_VERSION="21" +ARG LLVM_DEBS_SHA256="4a79b0eae89af72b082997d361198f16aaefce3fa8ad3c56c8e97311ff31dd5f" ARG XWIN_VERSION="0.9.0" ARG XWIN_SHA256="31e1033f30608ba6b821d17f1461042bd54c23424813c9b4e9ae15b6d32fa4cd" # Pinned MSVC CRT + Windows SDK versions. Bump deliberately; the manifest on @@ -53,6 +54,7 @@ FROM ubuntu:24.04 AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG LLVM_VERSION +ARG LLVM_DEBS_SHA256 ARG XWIN_VERSION ARG XWIN_SHA256 ARG MSVC_CRT_VERSION @@ -66,7 +68,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ lsb-release software-properties-common gnupg \ && rm -rf /var/lib/apt/lists/* -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# Host clang. The debs are mirrored from apt.llvm.org to a GitHub release +# (see scripts/mirror-llvm-debs.sh) so the build doesn't depend on it. +ADD --checksum=sha256:${LLVM_DEBS_SHA256} \ + https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-noble-amd64.tar.gz /tmp/llvm.tar.gz +RUN mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm && \ + apt-get update && apt-get install -y /tmp/llvm/*.deb && \ + rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* && \ for t in clang clang++ clang-cl lld-link ld.lld llvm-ar llvm-lib llvm-mt llvm-rc llvm-ranlib llvm-objcopy; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/scripts/mirror-llvm-debs.sh b/scripts/mirror-llvm-debs.sh new file mode 100755 index 000000000000..6be3e766f5ec --- /dev/null +++ b/scripts/mirror-llvm-debs.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Mirror the LLVM .debs that the Dockerfiles install from apt.llvm.org to a +# GitHub release on oven-sh/WebKit, so the docker builds stop depending on +# apt.llvm.org being reachable. Same idea as scripts/mirror-gcc13-debs.sh. +# +# One tarball per (Ubuntu release, arch) the Dockerfiles build on: +# focal amd64, arm64 -> Dockerfile (llvm.sh $LLVM_VERSION all) +# noble amd64 -> Dockerfile.{android,freebsd,macos,windows} +# (llvm.sh $LLVM_VERSION) +# Only packages served from apt.llvm.org go in; their Ubuntu-archive +# dependencies are resolved by `apt-get install ./*.deb` at image build time +# as before. apt verifies every download against the signed repo index. +# +# Needs docker and gh. Run .github/workflows/mirror-llvm-debs.yml to do this +# on a runner. Afterwards paste the printed SHA-256s into the Dockerfiles. +set -euo pipefail + +REPO="${REPO:-oven-sh/WebKit}" +LLVM_VERSION="${LLVM_VERSION:-21}" +TAG="${TAG:-llvm-${LLVM_VERSION}-debs}" +OUT="${OUT:-/tmp/llvm-debs}" +V="$LLVM_VERSION" + +# What `llvm.sh $V` and `llvm.sh $V all` install (see PKG= in llvm.sh). +BASE_PKGS="clang-$V lldb-$V lld-$V clangd-$V" +ALL_PKGS="$BASE_PKGS clang-tidy-$V clang-format-$V clang-tools-$V llvm-$V-dev \ + llvm-$V-tools libomp-$V-dev libc++-$V-dev libc++abi-$V-dev \ + libclang-common-$V-dev libclang-$V-dev libclang-cpp$V-dev liblldb-$V-dev \ + libunwind-$V-dev libclang-rt-$V-dev libpolly-$V-dev" + +rm -rf "$OUT" && mkdir -p "$OUT" + +# mirror ... +mirror() { + local distro="$1" image="$2" pkgs="$3"; shift 3 + local arches="$*" + mkdir -p "$OUT/$distro" + docker run --rm -v "$OUT/$distro":/out -e V="$V" -e DISTRO="$distro" \ + -e PKGS="$pkgs" -e ARCHES="$arches" "$image" bash -euo pipefail -c ' + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq ca-certificates curl gnupg >/dev/null + mkdir -p /etc/apt/keyrings + curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /etc/apt/keyrings/llvm.gpg + native=$(dpkg --print-architecture) + for a in $ARCHES; do + [ "$a" = "$native" ] || dpkg --add-architecture "$a" + done + # Keep the Ubuntu archive native-only: the foreign arch is served from + # ports.ubuntu.com and we only need its apt.llvm.org debs. + if [ -f /etc/apt/sources.list ]; then + sed -i "s|^deb |deb [arch=$native] |" /etc/apt/sources.list + fi + if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then + sed -i "/^Types:/a Architectures: $native" /etc/apt/sources.list.d/ubuntu.sources + fi + echo "deb [arch=$(echo $ARCHES | tr " " ,) signed-by=/etc/apt/keyrings/llvm.gpg] https://apt.llvm.org/$DISTRO/ llvm-toolchain-$DISTRO-$V main" \ + > /etc/apt/sources.list.d/llvm.list + apt-get update -qq + # Resolve the install set on the native arch and keep the names that come + # from apt.llvm.org; the same names exist for every arch in that repo. + names=$(apt-get install -y -qq --print-uris $PKGS \ + | grep -o "https://apt.llvm.org/[^ '"'"']*\.deb" | xargs -n1 basename | cut -d_ -f1 | sort -u) + echo "$(echo "$names" | wc -l) packages from apt.llvm.org" + for a in $ARCHES; do + mkdir -p /out/$a && cd /out/$a + for n in $names; do + apt-get download -qq "$n:$a" + done + ls *.deb | wc -l + done + ' +} + +mirror focal ubuntu:20.04 "$ALL_PKGS" amd64 arm64 +mirror noble ubuntu:24.04 "$BASE_PKGS" amd64 + +NOTES="$OUT/notes.md" +{ + echo "LLVM $V .debs from apt.llvm.org, mirrored so the Dockerfiles do not depend on apt.llvm.org at build time. Regenerate with scripts/mirror-llvm-debs.sh (or the mirror-llvm-debs workflow)." + echo + echo "| tarball | clang-$V version | files | sha256 |" + echo "|---|---|---|---|" +} > "$NOTES" +ASSETS=() +for dir in "$OUT"/*/*/; do + distro=$(basename "$(dirname "$dir")"); arch=$(basename "$dir") + ls "$dir"/clang-${V}_*.deb >/dev/null || { echo "!! $distro/$arch missing clang-$V"; exit 1; } + tarball="$OUT/llvm-$V-$distro-$arch.tar.gz" + # Sorted names + fixed mtime/owner so the SHA-256 is reproducible. + tar --sort=name --mtime='UTC 2020-01-01' --owner=0 --group=0 --numeric-owner \ + -czf "$tarball" -C "$dir" . + sha=$(sha256sum "$tarball" | cut -d' ' -f1) + ver=$(dpkg-deb -f "$(ls "$dir"/clang-${V}_*.deb)" Version) + echo "| llvm-$V-$distro-$arch.tar.gz | $ver | $(ls "$dir" | wc -l) | \`$sha\` |" >> "$NOTES" + echo "LLVM_DEBS_SHA256_${distro}_${arch}=$sha ($(du -h "$tarball" | cut -f1))" + ASSETS+=("$tarball") +done + +if gh release view "$TAG" -R "$REPO" >/dev/null 2>&1; then + gh release upload "$TAG" -R "$REPO" --clobber "${ASSETS[@]}" + gh release edit "$TAG" -R "$REPO" --notes-file "$NOTES" +else + gh release create "$TAG" -R "$REPO" --title "LLVM $V .debs (mirror of apt.llvm.org)" \ + --notes-file "$NOTES" "${ASSETS[@]}" +fi