fix(ci): retry refused and unreachable connections on CI downloads

.github/workflows/build.yml

@@ -333,8 +333,10 @@ jobs:
           bash scripts/ci/install-intel-ocloc.sh
           sudo apt-get update
           sudo -E apt-get -yq install gpg-agent cmake
-          wget -qO- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
-            | sudo gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg
+          wget --tries=3 --retry-connrefused --retry-on-host-error -qO "$RUNNER_TEMP/oneapi-key.asc" \
+            https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          sudo gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg "$RUNNER_TEMP/oneapi-key.asc"
+          rm -f "$RUNNER_TEMP/oneapi-key.asc"
           echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \
             | sudo tee /etc/apt/sources.list.d/oneAPI.list
           sudo apt-get update

.github/workflows/ffmpeg-integration.yml

@@ -195,8 +195,10 @@ jobs:
           # ocloc compiles the SYCL ahead-of-time images; the Linux compiler
           # does not ship it and meson setup refuses without it (ADR-1360).
           bash scripts/ci/install-intel-ocloc.sh
-          wget -qO- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
-            | sudo gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg
+          wget --tries=3 --retry-connrefused --retry-on-host-error -qO "$RUNNER_TEMP/oneapi-key.asc" \
+            https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          sudo gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg "$RUNNER_TEMP/oneapi-key.asc"
+          rm -f "$RUNNER_TEMP/oneapi-key.asc"
           echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \
             | sudo tee /etc/apt/sources.list.d/oneAPI.list
           sudo apt-get update

.github/workflows/libvmaf-build-matrix.yml

@@ -414,8 +414,10 @@ jobs:
           sudo -E apt-get -yq install ccache ninja-build nasm gcc g++ gpg-agent pkg-config libzimg-dev
           # ADR-1360 (design notes)
           bash scripts/ci/install-intel-ocloc.sh
-          wget -qO- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
-            | sudo gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg
+          wget --tries=3 --retry-connrefused --retry-on-host-error -qO "$RUNNER_TEMP/oneapi-key.asc" \
+            https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          sudo gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg "$RUNNER_TEMP/oneapi-key.asc"
+          rm -f "$RUNNER_TEMP/oneapi-key.asc"
           echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \
             | sudo tee /etc/apt/sources.list.d/oneAPI.list
           sudo apt-get update
@@ -948,7 +950,7 @@ jobs:
           WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/1f18901e-877d-469d-a41a-a10f11b39336/intel-oneapi-base-toolkit-2025.3.0.372_offline.exe
           WINDOWS_DPCPP_COMPONENTS: intel.oneapi.win.cpp-dpcpp-common
         run: |
-          curl.exe --output %TEMP%\webimage.exe --url %WINDOWS_BASEKIT_URL% --retry 5 --retry-delay 5
+          curl.exe --output %TEMP%\webimage.exe --url %WINDOWS_BASEKIT_URL% --retry 5 --retry-all-errors --retry-delay 5
           start /b /wait %TEMP%\webimage.exe -s -x -f webimage_extracted --log extract.log
           del %TEMP%\webimage.exe
           webimage_extracted\bootstrapper.exe -s --action install --components=%WINDOWS_DPCPP_COMPONENTS% --eula=accept -p=NEED_VS2017_INTEGRATION=0 -p=NEED_VS2019_INTEGRATION=0 -p=NEED_VS2022_INTEGRATION=0 --log-dir=.

.github/workflows/lint-and-format.yml

@@ -662,10 +662,10 @@ jobs:
         run: |
           printf 'Acquire::Retries "3";\nAcquire::http::Timeout "30";\nAcquire::https::Timeout "30";\nAPT::Keep-Downloaded-Packages "true";\n' \
             | sudo tee /etc/apt/apt.conf.d/80ci-retries >/dev/null
-          wget -qO- \
-            https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
-            | sudo gpg --dearmor \
-                -o /usr/share/keyrings/oneapi-archive-keyring.gpg
+          wget --tries=3 --retry-connrefused --retry-on-host-error -qO "$RUNNER_TEMP/oneapi-key.asc" \
+            https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
+          sudo gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg "$RUNNER_TEMP/oneapi-key.asc"
+          rm -f "$RUNNER_TEMP/oneapi-key.asc"
           echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] \
             https://apt.repos.intel.com/oneapi all main" \
             | sudo tee /etc/apt/sources.list.d/oneAPI.list
@@ -1132,7 +1132,7 @@ jobs:
           fi
           url="https://github.com/stoplightio/spectral/releases/download/v${version}/spectral-linux-x64"
           mkdir -p "${RUNNER_TEMP}/spectral"
-          curl -fsSL --retry 3 --max-time 300 -o "${RUNNER_TEMP}/spectral/spectral" "${url}"
+          curl -fsSL --retry 3 --retry-all-errors --max-time 300 -o "${RUNNER_TEMP}/spectral/spectral" "${url}"
           echo "${sha}  ${RUNNER_TEMP}/spectral/spectral" | sha256sum -c -
           chmod +x "${RUNNER_TEMP}/spectral/spectral"
           echo "${RUNNER_TEMP}/spectral" >> "${GITHUB_PATH}"
@@ -1271,7 +1271,7 @@ jobs:
           mkdir -p "${RUNNER_TEMP}/trivy-bin"
           tarball="trivy_${version}_Linux-64bit.tar.gz"
           url="https://github.com/aquasecurity/trivy/releases/download/v${version}/${tarball}"
-          curl -fsSL --retry 3 --max-time 300 -o "${RUNNER_TEMP}/${tarball}" "${url}"
+          curl -fsSL --retry 3 --retry-all-errors --max-time 300 -o "${RUNNER_TEMP}/${tarball}" "${url}"
           echo "${expected_sha}  ${RUNNER_TEMP}/${tarball}" | sha256sum -c -
           tar -xzf "${RUNNER_TEMP}/${tarball}" -C "${RUNNER_TEMP}/trivy-bin" trivy
           echo "${RUNNER_TEMP}/trivy-bin" >> "${GITHUB_PATH}"

.github/workflows/security-scans.yml

@@ -308,9 +308,10 @@ jobs:
       - name: Install gitleaks
         run: |
           GITLEAKS_VERSION=8.24.3
-          curl -sSfL --max-time 300 \
-            "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" \
-            | tar -xz -C /usr/local/bin gitleaks
+          curl --retry 3 --retry-all-errors -sSfL --max-time 300 -o "$RUNNER_TEMP/gitleaks.tar.gz" \
+            "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
+          tar -xzf "$RUNNER_TEMP/gitleaks.tar.gz" -C /usr/local/bin gitleaks
+          rm -f "$RUNNER_TEMP/gitleaks.tar.gz"
           gitleaks version
       # Scan the history of the checked-out commit only. Without --log-opts
       # gitleaks runs `git log --all`, and fetch-depth: 0 fetches every

.github/workflows/supply-chain.yml

@@ -976,7 +976,7 @@ jobs:
           for crate in vmafx-sys vmafx; do
             version="$(python3 -c 'import sys, tomllib; print(tomllib.load(open(sys.argv[1], "rb"))["package"]["version"])' \
               "bindings/rust/$crate/Cargo.toml")"
-            http_code="$(curl --silent --show-error --max-time 60 \
+            http_code="$(curl --retry 3 --retry-all-errors --silent --show-error --max-time 60 \
               --user-agent 'VMAFx/vmafx release publisher (https://github.com/VMAFx/vmafx)' \
               --output "$RUNNER_TEMP/$crate-version.json" --write-out '%{http_code}' \
               "https://crates.io/api/v1/crates/$crate/$version")"
@@ -1071,7 +1071,7 @@ jobs:
           fi
           local_files=("${wheels[0]}" "${sdists[0]}")
           response="$RUNNER_TEMP/pypi-release.json"
-          http_code="$(curl --retry 3 --max-time 60 --silent --show-error \
+          http_code="$(curl --retry 3 --retry-all-errors --max-time 60 --silent --show-error \
             --output "$response" --write-out '%{http_code}' \
             "https://pypi.org/pypi/vmaf-mcp/$VMAFX_PEP440_VERSION/json")"
           publish=false
@@ -1165,7 +1165,7 @@ jobs:
           response="$RUNNER_TEMP/pypi-release-after.json"
           http_code=000
           for attempt in {1..12}; do
-            http_code="$(curl --retry 3 --max-time 60 --silent --show-error --output "$response" \
+            http_code="$(curl --retry 3 --retry-all-errors --max-time 60 --silent --show-error --output "$response" \
               --write-out '%{http_code}' \
               "https://pypi.org/pypi/vmaf-mcp/$VMAFX_PEP440_VERSION/json")"
             if [[ "$http_code" == "200" &&
@@ -1373,7 +1373,7 @@ jobs:
         run: |
           set -euo pipefail
           url="$(python3 scripts/release/distribution.py source-url --version "$VMAFX_VERSION")"
-          curl --fail --location --silent --show-error --retry 3 --max-time 120 \
+          curl --fail --location --silent --show-error --retry 3 --retry-all-errors --max-time 120 \
             --output "$RUNNER_TEMP/source.tar.gz" "$url"
           digest="$(sha256sum "$RUNNER_TEMP/source.tar.gz" | cut -d' ' -f1)"
           python3 scripts/release/distribution.py homebrew --version "$VMAFX_VERSION" \

.github/workflows/tests-and-quality-gates.yml

@@ -1047,7 +1047,7 @@ jobs:
           mkdir -p "${RUNNER_TEMP}/ort-cache"
           tarball="${RUNNER_TEMP}/ort-cache/onnxruntime-linux-x64-gpu-${ORT_VERSION}.tgz"
           if [ ! -f "$tarball" ]; then
-            curl -fsSL --retry 3 --retry-delay 5 --max-time 600 -o "$tarball" \
+            curl -fsSL --retry 3 --retry-all-errors --retry-delay 5 --max-time 600 -o "$tarball" \
               "https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/onnxruntime-linux-x64-gpu-${ORT_VERSION}.tgz"
           fi
           if ! printf '%s  %s\n' "${ORT_SHA256}" "$tarball" | sha256sum --check --strict; then

.github/workflows/windows-tester-bundle.yml

@@ -312,7 +312,7 @@ jobs:
             $pins[$name] = $line.Matches[0].Groups[1].Value
           }
           $exe = Join-Path $env:RUNNER_TEMP 'oneapi-basekit.exe'
-          curl.exe --fail --location --retry 5 --retry-delay 5 --output $exe --url $pins['INTEL_BASEKIT_WINDOWS_URL']
+          curl.exe --fail --location --retry 5 --retry-all-errors --retry-delay 5 --output $exe --url $pins['INTEL_BASEKIT_WINDOWS_URL']
           if ($LASTEXITCODE -ne 0) { throw "download failed ($LASTEXITCODE)" }
           $size = (Get-Item $exe).Length
           if ($size -ne [int64]$pins['INTEL_BASEKIT_WINDOWS_SIZE']) { throw "installer is $size bytes, not $($pins['INTEL_BASEKIT_WINDOWS_SIZE'])" }

.pre-commit-config.yaml

@@ -852,6 +852,15 @@ repos:
         files: '^(\.github/workflows/|scripts/ci/(pr_workflow_budget\.py|tests/test_pr_workflow_budget\.py)|\.pre-commit-config\.yaml$)'
         pass_filenames: false
 
+      - id: test-fetch-connect-retry
+        name: HTTP fetches retry connection and host errors
+        entry: python3 -B scripts/ci/tests/test_fetch_connect_retry.py
+        language: system
+        files: '\.(sh|bash|ksh|zsh|fish|ps1|cmd|bat)$|(^|/)([^/]*\.)?(Dockerfile|Containerfile)(\.[^/]+)?$|^\.github/workflows/[^/]+\.ya?ml$|^scripts/ci/tests/test_fetch_connect_retry\.py$|^\.pre-commit-config\.yaml$'
+        # Shebangs can also identify shell scripts with arbitrary filenames.
+        always_run: true
+        pass_filenames: false
+
       - id: check-api-doc-coverage
         name: Every public vmafx symbol is named in docs/api (#2436)
         entry: python3 scripts/ci/check_api_doc_coverage.py

Containerfile.vmafx

@@ -128,9 +128,10 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
 # umf/tcm/tbb component dirs together without missing-dependency surprises.
 # The ~10 GB it installs never leaves this stage — stage 3 copies out only
 # the specific .so's `ldd` says vmaf/ffmpeg/libvmaf actually link (below).
-RUN curl -fsSL https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
-        | gpg --dearmor \
-        | tee /usr/share/keyrings/intel-sw-products.gpg > /dev/null \
+RUN key="$(mktemp)" \
+    && curl --retry 3 --retry-all-errors -fsSL -o "$key" https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+    && gpg --dearmor < "$key" | tee /usr/share/keyrings/intel-sw-products.gpg > /dev/null \
+    && rm -f "$key" \
     && printf 'deb [signed-by=/usr/share/keyrings/intel-sw-products.gpg] https://apt.repos.intel.com/oneapi all main\n' \
         > /etc/apt/sources.list.d/intel-oneapi.list
 

Dockerfile

@@ -87,8 +87,10 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     --mount=type=cache,target=/var/lib/apt,sharing=locked \
     if [ "$ENABLE_SYCL" = "true" ]; then \
         apt-get update && apt-get install -y --no-install-recommends gnupg2 ca-certificates && \
-        wget -qO- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
-            | gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg && \
+        key="$(mktemp)" && \
+        wget --tries=3 --retry-connrefused --retry-on-host-error -qO "$key" https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB && \
+        gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg "$key" && \
+        rm -f "$key" && \
         echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" \
             > /etc/apt/sources.list.d/oneapi.list && \
         apt-get update && apt-get install -y --no-install-recommends \
@@ -101,7 +103,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
 
 # ---------- nv-codec-headers ----------
 WORKDIR /tmp/nv-codec
-RUN wget -q "https://github.com/FFmpeg/nv-codec-headers/archive/${NV_CODEC_TAG}.zip" && \
+RUN wget --tries=3 --retry-connrefused --retry-on-host-error -q "https://github.com/FFmpeg/nv-codec-headers/archive/${NV_CODEC_TAG}.zip" && \
     unzip -q "${NV_CODEC_TAG}.zip"
 WORKDIR /tmp/nv-codec/nv-codec-headers-${NV_CODEC_TAG}
 RUN make && make install
@@ -129,7 +131,7 @@ RUN --mount=type=cache,target=/root/.cache/ccache,sharing=locked \
 RUN echo /usr/local/lib/x86_64-linux-gnu > /etc/ld.so.conf.d/vmaf-local.conf && ldconfig
 
 # ---------- build FFmpeg ----------
-RUN wget -q "${FFMPEG_REMOTE%.git}/archive/${FFMPEG_TAG}.zip" && \
+RUN wget --tries=3 --retry-connrefused --retry-on-host-error -q "${FFMPEG_REMOTE%.git}/archive/${FFMPEG_TAG}.zip" && \
     unzip -q "${FFMPEG_TAG}.zip" && rm "${FFMPEG_TAG}.zip"
 
 COPY ffmpeg-patches /tmp/ffmpeg-patches

Dockerfile.ffmpeg

@@ -17,7 +17,7 @@ ARG FFMPEG_REMOTE=https://github.com/FFmpeg/FFmpeg.git
 ARG NVCC_FLAGS="-gencode arch=compute_75,code=sm_75 -gencode arch=compute_80,code=sm_80 -gencode arch=compute_90,code=sm_90 -gencode arch=compute_120,code=sm_120 -gencode arch=compute_120,code=compute_120 -O2"
 
 # get ffmpeg
-RUN wget -q "${FFMPEG_REMOTE%.git}/archive/${FFMPEG_TAG}.zip" && \
+RUN wget --tries=3 --retry-connrefused --retry-on-host-error -q "${FFMPEG_REMOTE%.git}/archive/${FFMPEG_TAG}.zip" && \
     unzip -q "${FFMPEG_TAG}.zip"
 
 # Apply the same cumulative patch series as every other FFmpeg image: the

changelog.d/fixed/fetch-connect-retry.md

@@ -0,0 +1 @@
+- **CI:** a refused or unreachable connection no longer ends a CI download on the first attempt: curl and wget fetches in shell scripts, workflows and container recipes retry connection and host-resolution failures, the tidy lane's APT downloads retry, and a contract test checks curl and wget fetches for the retry flags and requires retried downloads to write to a file before use.