diff --git a/.github/workflows/mirror-llvm-debs.yml b/.github/workflows/mirror-llvm-debs.yml new file mode 100644 index 000000000000..15f236a7eee9 --- /dev/null +++ b/.github/workflows/mirror-llvm-debs.yml @@ -0,0 +1,31 @@ +name: Mirror LLVM debs + +# Rebuilds the llvm--debs release that the Dockerfiles install LLVM +# from. Run it when the LLVM version changes or apt.llvm.org ships a build +# worth picking up, then pin the printed SHA-256s in the Dockerfiles. +on: + workflow_dispatch: + inputs: + llvm_version: + description: 'LLVM major version to mirror' + type: string + default: '21' + +permissions: + contents: write + +jobs: + mirror: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + sparse-checkout: scripts + - name: Mirror + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + LLVM_VERSION: ${{ inputs.llvm_version }} + OUT: ${{ runner.temp }}/llvm-debs + run: bash scripts/mirror-llvm-debs.sh diff --git a/Dockerfile b/Dockerfile index afe5c1f43bf0..8dff2f78e788 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,12 +103,22 @@ RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 \ --slave /usr/bin/gcc-nm gcc-nm /usr/bin/gcc-nm-13 \ --slave /usr/bin/gcc-ranlib gcc-ranlib /usr/bin/gcc-ranlib-13 -# Install LLVM 21 -RUN wget https://apt.llvm.org/llvm.sh \ - && chmod +x llvm.sh \ - && ./llvm.sh 21 all \ - && rm llvm.sh \ - && rm -rf /var/lib/apt/lists/* +# Install LLVM +# The `llvm.sh all` package set, mirrored from apt.llvm.org to a +# GitHub release so the image doesn't depend on apt.llvm.org at build time. +# Ubuntu-archive dependencies still come from apt. Regenerate via +# scripts/mirror-llvm-debs.sh (or the mirror-llvm-debs workflow). +ARG LLVM_DEBS_SHA256_amd64=759ea9d6d50de9b6062cf40161a24a3a9d70aaf11aa1a544074d126590eb55f7 +ARG LLVM_DEBS_SHA256_arm64=4d4923baa663cb2e1be67e8e7097220604489b0da8b7a4ab5911ac2baf1e0ba6 +RUN curl -fsSL --retry 5 --retry-connrefused \ + "https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-focal-${TARGETARCH}.tar.gz" \ + -o /tmp/llvm.tar.gz \ + && eval "expected=\$LLVM_DEBS_SHA256_${TARGETARCH}" \ + && echo "${expected} /tmp/llvm.tar.gz" | sha256sum -c - \ + && mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm \ + && apt-get update \ + && apt-get install -y /tmp/llvm/*.deb \ + && rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* # Configure library paths RUN if [ "$TARGETARCH" = "arm64" ]; then \ diff --git a/Dockerfile.android b/Dockerfile.android index b07ead881617..88b403c094e4 100644 --- a/Dockerfile.android +++ b/Dockerfile.android @@ -2,6 +2,7 @@ ARG MARCH_FLAG="-march=armv8-a+crc -mtune=cortex-a78" ARG WEBKIT_RELEASE_TYPE=Release ARG LTO_FLAG="" ARG LLVM_VERSION="21" +ARG LLVM_DEBS_SHA256="4a79b0eae89af72b082997d361198f16aaefce3fa8ad3c56c8e97311ff31dd5f" ARG NDK_VERSION="r27c" ARG NDK_SHA256="59c2f6dc96743b5daf5d1626684640b20a6bd2b1d85b13156b90333741bad5cc" ARG ANDROID_API="28" @@ -17,6 +18,7 @@ FROM ubuntu:24.04 AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG LLVM_VERSION +ARG LLVM_DEBS_SHA256 ARG NDK_VERSION RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -29,7 +31,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot, # not via the NDK's bundled clang. apt.llvm.org installs version-suffixed names # only (ld.lld-21, not ld.lld), so add unversioned links for -fuse-ld=lld. -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# The debs are mirrored from apt.llvm.org to a GitHub release (see +# scripts/mirror-llvm-debs.sh) so the build doesn't depend on apt.llvm.org. +ADD --checksum=sha256:${LLVM_DEBS_SHA256} \ + https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-noble-amd64.tar.gz /tmp/llvm.tar.gz +RUN mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm && \ + apt-get update && apt-get install -y /tmp/llvm/*.deb && \ + rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* && \ for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.freebsd b/Dockerfile.freebsd index 7dc042f49e13..05bf38e59e8d 100644 --- a/Dockerfile.freebsd +++ b/Dockerfile.freebsd @@ -2,6 +2,7 @@ ARG MARCH_FLAG="-march=nehalem" ARG WEBKIT_RELEASE_TYPE=Release ARG LTO_FLAG="" ARG LLVM_VERSION="21" +ARG LLVM_DEBS_SHA256="4a79b0eae89af72b082997d361198f16aaefce3fa8ad3c56c8e97311ff31dd5f" ARG FREEBSD_VERSION="14.3" ARG FREEBSD_ARCH="x86_64" ARG DEFAULT_CFLAGS="-mno-omit-leaf-frame-pointer -fno-omit-frame-pointer -ffunction-sections -fdata-sections -faddrsig -fno-unwind-tables -fno-asynchronous-unwind-tables -DU_STATIC_IMPLEMENTATION=1 " @@ -19,6 +20,7 @@ FROM ubuntu:24.04 AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG LLVM_VERSION +ARG LLVM_DEBS_SHA256 ARG FREEBSD_VERSION ARG FREEBSD_ARCH @@ -31,8 +33,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Host clang (same version Bun uses) — we cross-compile via --target/--sysroot. # apt.llvm.org installs version-suffixed names only, so add unversioned links -# for -fuse-ld=lld. -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# for -fuse-ld=lld. The debs are mirrored from apt.llvm.org to a GitHub +# release (see scripts/mirror-llvm-debs.sh) so the build doesn't depend on it. +ADD --checksum=sha256:${LLVM_DEBS_SHA256} \ + https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-noble-amd64.tar.gz /tmp/llvm.tar.gz +RUN mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm && \ + apt-get update && apt-get install -y /tmp/llvm/*.deb && \ + rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* && \ for t in clang clang++ ld.lld lld llvm-ar llvm-ranlib; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.macos b/Dockerfile.macos index e8d4761f438c..a613843c98d6 100644 --- a/Dockerfile.macos +++ b/Dockerfile.macos @@ -30,6 +30,7 @@ ARG WEBKIT_RELEASE_TYPE=Release ARG LTO_FLAG="" ARG MARCH_FLAG="-mcpu=apple-m1" ARG LLVM_VERSION="21" +ARG LLVM_DEBS_SHA256="4a79b0eae89af72b082997d361198f16aaefce3fa8ad3c56c8e97311ff31dd5f" ARG BOOTSTRAP_CMDS_TAG="bootstrap_cmds-138" # Matches the native macOS lane's CMAKE_C_FLAGS (build-reusable.yml) plus -g # from mac-release.bash. Cross-only additions live in the build stage. @@ -54,6 +55,7 @@ FROM ubuntu:24.04 AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG LLVM_VERSION +ARG LLVM_DEBS_SHA256 ARG MACOS_ARCH ARG MACOS_SDK_VERSION ARG MACOS_SDK_CLT_RELEASE @@ -71,8 +73,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* # Host clang (same version Bun uses) — we cross-compile via --target/-isysroot -# and link with ld64.lld (lld's Mach-O port). -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# and link with ld64.lld (lld's Mach-O port). The debs are mirrored from +# apt.llvm.org to a GitHub release (see scripts/mirror-llvm-debs.sh) so the +# build doesn't depend on it. +ADD --checksum=sha256:${LLVM_DEBS_SHA256} \ + https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-noble-amd64.tar.gz /tmp/llvm.tar.gz +RUN mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm && \ + apt-get update && apt-get install -y /tmp/llvm/*.deb && \ + rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* && \ for t in clang clang++ ld64.lld lld llvm-ar llvm-ranlib llvm-nm llvm-cxxfilt; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/Dockerfile.windows b/Dockerfile.windows index 5edb4fb6bef4..c344daedbefd 100644 --- a/Dockerfile.windows +++ b/Dockerfile.windows @@ -35,6 +35,7 @@ ARG ENABLE_SANITIZERS="" ARG USE_MIMALLOC="OFF" ARG USE_EXTERNAL_MIMALLOC="OFF" ARG LLVM_VERSION="21" +ARG LLVM_DEBS_SHA256="4a79b0eae89af72b082997d361198f16aaefce3fa8ad3c56c8e97311ff31dd5f" ARG XWIN_VERSION="0.9.0" ARG XWIN_SHA256="31e1033f30608ba6b821d17f1461042bd54c23424813c9b4e9ae15b6d32fa4cd" # Pinned MSVC CRT + Windows SDK versions. Bump deliberately; the manifest on @@ -53,6 +54,7 @@ FROM ubuntu:24.04 AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG LLVM_VERSION +ARG LLVM_DEBS_SHA256 ARG XWIN_VERSION ARG XWIN_SHA256 ARG MSVC_CRT_VERSION @@ -66,7 +68,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ lsb-release software-properties-common gnupg \ && rm -rf /var/lib/apt/lists/* -RUN wget -qO- https://apt.llvm.org/llvm.sh | bash -s -- ${LLVM_VERSION} && \ +# Host clang. The debs are mirrored from apt.llvm.org to a GitHub release +# (see scripts/mirror-llvm-debs.sh) so the build doesn't depend on it. +ADD --checksum=sha256:${LLVM_DEBS_SHA256} \ + https://github.com/oven-sh/WebKit/releases/download/llvm-${LLVM_VERSION}-debs/llvm-${LLVM_VERSION}-noble-amd64.tar.gz /tmp/llvm.tar.gz +RUN mkdir -p /tmp/llvm && tar xzf /tmp/llvm.tar.gz -C /tmp/llvm && \ + apt-get update && apt-get install -y /tmp/llvm/*.deb && \ + rm -rf /tmp/llvm /tmp/llvm.tar.gz /var/lib/apt/lists/* && \ for t in clang clang++ clang-cl lld-link ld.lld llvm-ar llvm-lib llvm-mt llvm-rc llvm-ranlib llvm-objcopy; do \ ln -sf /usr/bin/${t}-${LLVM_VERSION} /usr/local/bin/${t}; \ done diff --git a/scripts/mirror-llvm-debs.sh b/scripts/mirror-llvm-debs.sh new file mode 100755 index 000000000000..6be3e766f5ec --- /dev/null +++ b/scripts/mirror-llvm-debs.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Mirror the LLVM .debs that the Dockerfiles install from apt.llvm.org to a +# GitHub release on oven-sh/WebKit, so the docker builds stop depending on +# apt.llvm.org being reachable. Same idea as scripts/mirror-gcc13-debs.sh. +# +# One tarball per (Ubuntu release, arch) the Dockerfiles build on: +# focal amd64, arm64 -> Dockerfile (llvm.sh $LLVM_VERSION all) +# noble amd64 -> Dockerfile.{android,freebsd,macos,windows} +# (llvm.sh $LLVM_VERSION) +# Only packages served from apt.llvm.org go in; their Ubuntu-archive +# dependencies are resolved by `apt-get install ./*.deb` at image build time +# as before. apt verifies every download against the signed repo index. +# +# Needs docker and gh. Run .github/workflows/mirror-llvm-debs.yml to do this +# on a runner. Afterwards paste the printed SHA-256s into the Dockerfiles. +set -euo pipefail + +REPO="${REPO:-oven-sh/WebKit}" +LLVM_VERSION="${LLVM_VERSION:-21}" +TAG="${TAG:-llvm-${LLVM_VERSION}-debs}" +OUT="${OUT:-/tmp/llvm-debs}" +V="$LLVM_VERSION" + +# What `llvm.sh $V` and `llvm.sh $V all` install (see PKG= in llvm.sh). +BASE_PKGS="clang-$V lldb-$V lld-$V clangd-$V" +ALL_PKGS="$BASE_PKGS clang-tidy-$V clang-format-$V clang-tools-$V llvm-$V-dev \ + llvm-$V-tools libomp-$V-dev libc++-$V-dev libc++abi-$V-dev \ + libclang-common-$V-dev libclang-$V-dev libclang-cpp$V-dev liblldb-$V-dev \ + libunwind-$V-dev libclang-rt-$V-dev libpolly-$V-dev" + +rm -rf "$OUT" && mkdir -p "$OUT" + +# mirror ... +mirror() { + local distro="$1" image="$2" pkgs="$3"; shift 3 + local arches="$*" + mkdir -p "$OUT/$distro" + docker run --rm -v "$OUT/$distro":/out -e V="$V" -e DISTRO="$distro" \ + -e PKGS="$pkgs" -e ARCHES="$arches" "$image" bash -euo pipefail -c ' + export DEBIAN_FRONTEND=noninteractive + apt-get update -qq + apt-get install -y -qq ca-certificates curl gnupg >/dev/null + mkdir -p /etc/apt/keyrings + curl -fsSL https://apt.llvm.org/llvm-snapshot.gpg.key | gpg --dearmor -o /etc/apt/keyrings/llvm.gpg + native=$(dpkg --print-architecture) + for a in $ARCHES; do + [ "$a" = "$native" ] || dpkg --add-architecture "$a" + done + # Keep the Ubuntu archive native-only: the foreign arch is served from + # ports.ubuntu.com and we only need its apt.llvm.org debs. + if [ -f /etc/apt/sources.list ]; then + sed -i "s|^deb |deb [arch=$native] |" /etc/apt/sources.list + fi + if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then + sed -i "/^Types:/a Architectures: $native" /etc/apt/sources.list.d/ubuntu.sources + fi + echo "deb [arch=$(echo $ARCHES | tr " " ,) signed-by=/etc/apt/keyrings/llvm.gpg] https://apt.llvm.org/$DISTRO/ llvm-toolchain-$DISTRO-$V main" \ + > /etc/apt/sources.list.d/llvm.list + apt-get update -qq + # Resolve the install set on the native arch and keep the names that come + # from apt.llvm.org; the same names exist for every arch in that repo. + names=$(apt-get install -y -qq --print-uris $PKGS \ + | grep -o "https://apt.llvm.org/[^ '"'"']*\.deb" | xargs -n1 basename | cut -d_ -f1 | sort -u) + echo "$(echo "$names" | wc -l) packages from apt.llvm.org" + for a in $ARCHES; do + mkdir -p /out/$a && cd /out/$a + for n in $names; do + apt-get download -qq "$n:$a" + done + ls *.deb | wc -l + done + ' +} + +mirror focal ubuntu:20.04 "$ALL_PKGS" amd64 arm64 +mirror noble ubuntu:24.04 "$BASE_PKGS" amd64 + +NOTES="$OUT/notes.md" +{ + echo "LLVM $V .debs from apt.llvm.org, mirrored so the Dockerfiles do not depend on apt.llvm.org at build time. Regenerate with scripts/mirror-llvm-debs.sh (or the mirror-llvm-debs workflow)." + echo + echo "| tarball | clang-$V version | files | sha256 |" + echo "|---|---|---|---|" +} > "$NOTES" +ASSETS=() +for dir in "$OUT"/*/*/; do + distro=$(basename "$(dirname "$dir")"); arch=$(basename "$dir") + ls "$dir"/clang-${V}_*.deb >/dev/null || { echo "!! $distro/$arch missing clang-$V"; exit 1; } + tarball="$OUT/llvm-$V-$distro-$arch.tar.gz" + # Sorted names + fixed mtime/owner so the SHA-256 is reproducible. + tar --sort=name --mtime='UTC 2020-01-01' --owner=0 --group=0 --numeric-owner \ + -czf "$tarball" -C "$dir" . + sha=$(sha256sum "$tarball" | cut -d' ' -f1) + ver=$(dpkg-deb -f "$(ls "$dir"/clang-${V}_*.deb)" Version) + echo "| llvm-$V-$distro-$arch.tar.gz | $ver | $(ls "$dir" | wc -l) | \`$sha\` |" >> "$NOTES" + echo "LLVM_DEBS_SHA256_${distro}_${arch}=$sha ($(du -h "$tarball" | cut -f1))" + ASSETS+=("$tarball") +done + +if gh release view "$TAG" -R "$REPO" >/dev/null 2>&1; then + gh release upload "$TAG" -R "$REPO" --clobber "${ASSETS[@]}" + gh release edit "$TAG" -R "$REPO" --notes-file "$NOTES" +else + gh release create "$TAG" -R "$REPO" --title "LLVM $V .debs (mirror of apt.llvm.org)" \ + --notes-file "$NOTES" "${ASSETS[@]}" +fi