Skip to content

fix(ci): a retry budget sized on the outage that was measured, not on a guess #794

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephrobert merged 1 commit into from
Sep 25, 2026
Merged

fix(ci): a retry budget sized on the outage that was measured, not on a guess #794

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 14 additions & 15 deletions .github/workflows/conformance.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,8 +122,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://github.com/scaleway/scaleway-cli/releases/download/v${SCW_VERSION}"
asset="scaleway-cli_${SCW_VERSION}_linux_amd64"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/SHA256SUMS"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/SHA256SUMS" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/scw
scw version
Expand All @@ -148,8 +148,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}"
asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin
terraform version
Expand Down Expand Up @@ -259,8 +259,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://github.com/scaleway/scaleway-cli/releases/download/v${SCW_VERSION}"
asset="scaleway-cli_${SCW_VERSION}_linux_amd64"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/SHA256SUMS"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/SHA256SUMS" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/scw
scw version
Expand All @@ -287,9 +287,8 @@ jobs:
version="${OCTL_VERSION#v}"
base="https://github.com/outscale/octl/releases/download/${OCTL_VERSION}"
asset="octl_Linux_x86_64"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" \
"${base}/octl_${version}_checksums.txt"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/octl_${version}_checksums.txt" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/octl
octl --version
Expand All @@ -312,8 +311,8 @@ jobs:
version="${EXO_VERSION#v}"
base="https://github.com/exoscale/cli/releases/download/${EXO_VERSION}"
asset="exoscale-cli_${version}_linux_amd64.tar.gz"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/exoscale-cli_${version}_checksums.txt"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/exoscale-cli_${version}_checksums.txt" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
tar -xzf "${workdir}/${asset}" -C "${workdir}" exo
sudo install -m 0755 "${workdir}/exo" /usr/local/bin/exo
Expand Down Expand Up @@ -347,8 +346,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}"
asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin
terraform version
Expand All @@ -366,8 +365,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://github.com/opentofu/opentofu/releases/download/v${TOFU_VERSION}"
asset="tofu_${TOFU_VERSION}_linux_amd64.zip"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/tofu_${TOFU_VERSION}_SHA256SUMS"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/tofu_${TOFU_VERSION}_SHA256SUMS" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo unzip -o "${workdir}/${asset}" tofu -d /usr/local/bin
tofu version
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/corpus-cloud.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ jobs:
set -euo pipefail
version="2.56.3"
url="https://github.com/scaleway/scaleway-cli/releases/download/v${version}/scaleway-cli_${version}_linux_amd64"
curl --fail --silent --show-error --location "$url" -o "$RUNNER_TEMP/scw"
tools/ci/fetch.sh "$url" "$RUNNER_TEMP/scw"
chmod +x "$RUNNER_TEMP/scw"
echo "$RUNNER_TEMP" >> "$GITHUB_PATH"

Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/go.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,8 @@ jobs:
set -euo pipefail
workdir="$(mktemp -d)"
asset="shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz"
curl -fsSL -o "${workdir}/${asset}" \
"https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${asset}"
tools/ci/fetch.sh "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${asset}" \
"${workdir}/${asset}"
tar -xJf "${workdir}/${asset}" -C "${workdir}"
sudo install -m 0755 "${workdir}/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" /usr/local/bin/shellcheck
shellcheck --version
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/plumber.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://github.com/getplumber/plumber/releases/download/v${PLUMBER_VERSION}"
asset="plumber-linux-amd64"
curl -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl -fsSL -o "${workdir}/checksums.txt" "${base}/checksums.txt"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/checksums.txt" "${workdir}/checksums.txt"
(cd "${workdir}" && grep " ${asset}$" checksums.txt | sha256sum -c -)
sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/plumber
plumber version
Expand Down
25 changes: 12 additions & 13 deletions .github/workflows/runtime-proof.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ jobs:
run: |
set -euo pipefail
workdir="$(mktemp -d)"
curl --retry 3 --retry-connrefused -fsSL https://pkgs.zabbly.com/key.asc -o "${workdir}/zabbly.asc"
tools/ci/fetch.sh https://pkgs.zabbly.com/key.asc "${workdir}/zabbly.asc"
fpr="$(gpg --show-keys --with-colons "${workdir}/zabbly.asc" | awk -F: '/^fpr/ {print $10; exit}')"
if [ "${fpr}" != "${ZABBLY_FPR}" ]; then
echo "unexpected Zabbly key fingerprint: ${fpr}" >&2
Expand Down Expand Up @@ -309,8 +309,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://github.com/scaleway/scaleway-cli/releases/download/v${SCW_VERSION}"
asset="scaleway-cli_${SCW_VERSION}_linux_amd64"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/SHA256SUMS"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/SHA256SUMS" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/scw
scw version
Expand All @@ -327,9 +327,8 @@ jobs:
version="${OCTL_VERSION#v}"
base="https://github.com/outscale/octl/releases/download/${OCTL_VERSION}"
asset="octl_Linux_x86_64"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" \
"${base}/octl_${version}_checksums.txt"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/octl_${version}_checksums.txt" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/octl
octl --version
Expand All @@ -343,8 +342,8 @@ jobs:
version="${EXO_VERSION#v}"
base="https://github.com/exoscale/cli/releases/download/${EXO_VERSION}"
asset="exoscale-cli_${version}_linux_amd64.tar.gz"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/exoscale-cli_${version}_checksums.txt"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/exoscale-cli_${version}_checksums.txt" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
tar -xzf "${workdir}/${asset}" -C "${workdir}" exo
sudo install -m 0755 "${workdir}/exo" /usr/local/bin/exo
Expand Down Expand Up @@ -490,8 +489,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}"
asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin
terraform version
Expand Down Expand Up @@ -650,7 +649,7 @@ jobs:
run: |
set -euo pipefail
workdir="$(mktemp -d)"
curl --retry 3 --retry-connrefused -fsSL https://pkgs.zabbly.com/key.asc -o "${workdir}/zabbly.asc"
tools/ci/fetch.sh https://pkgs.zabbly.com/key.asc "${workdir}/zabbly.asc"
fpr="$(gpg --show-keys --with-colons "${workdir}/zabbly.asc" | awk -F: '/^fpr/ {print $10; exit}')"
if [ "${fpr}" != "${ZABBLY_FPR}" ]; then
echo "unexpected Zabbly key fingerprint: ${fpr}" >&2
Expand Down Expand Up @@ -804,8 +803,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}"
asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin
terraform version
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/secrets.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://github.com/trufflesecurity/trufflehog/releases/download/v${TRUFFLEHOG_VERSION}"
asset="trufflehog_${TRUFFLEHOG_VERSION}_linux_amd64.tar.gz"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/trufflehog_${TRUFFLEHOG_VERSION}_checksums.txt"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/trufflehog_${TRUFFLEHOG_VERSION}_checksums.txt" "${workdir}/sums"
(cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -)
tar -xzf "${workdir}/${asset}" -C "${workdir}" trufflehog
sudo install -m 0755 "${workdir}/trufflehog" /usr/local/bin/trufflehog
Expand Down
11 changes: 5 additions & 6 deletions .github/workflows/workflow-security.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,7 @@ jobs:
set -euo pipefail
workdir="$(mktemp -d)"
asset="zizmor-x86_64-unknown-linux-gnu.tar.gz"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" \
"https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/${asset}"
tools/ci/fetch.sh "https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/${asset}" "${workdir}/${asset}"
gh attestation verify "${workdir}/${asset}" --repo zizmorcore/zizmor
tar -xzf "${workdir}/${asset}" -C "${workdir}" zizmor
sudo install -m 0755 "${workdir}/zizmor" /usr/local/bin/zizmor
Expand Down Expand Up @@ -129,8 +128,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}"
asset="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/checksums.txt" "${base}/actionlint_${ACTIONLINT_VERSION}_checksums.txt"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/actionlint_${ACTIONLINT_VERSION}_checksums.txt" "${workdir}/checksums.txt"
(cd "${workdir}" && grep " ${asset}$" checksums.txt | sha256sum -c -)
tar -xzf "${workdir}/${asset}" -C "${workdir}" actionlint
sudo install -m 0755 "${workdir}/actionlint" /usr/local/bin/actionlint
Expand Down Expand Up @@ -167,8 +166,8 @@ jobs:
workdir="$(mktemp -d)"
base="https://github.com/boostsecurityio/poutine/releases/download/v${POUTINE_VERSION}"
asset="poutine_Linux_x86_64.tar.gz"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}"
curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/checksums.txt" "${base}/poutine_${POUTINE_VERSION}_checksums.txt"
tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}"
tools/ci/fetch.sh "${base}/poutine_${POUTINE_VERSION}_checksums.txt" "${workdir}/checksums.txt"
(cd "${workdir}" && grep " ${asset}$" checksums.txt | sha256sum -c -)
tar -xzf "${workdir}/${asset}" -C "${workdir}" poutine
sudo install -m 0755 "${workdir}/poutine" /usr/local/bin/poutine
Expand Down
66 changes: 66 additions & 0 deletions tools/ci/fetch.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/usr/bin/env bash
# Fetch a release asset, with a retry budget sized on a measured outage.
#
# Usage: tools/ci/fetch.sh <url> <destination>
#
# WHY THIS EXISTS
#
# Every install step in this repository already wrote `curl --retry 3
# --retry-connrefused`, and the nightly conformance run of 2026-09-25 still died
# on `curl: (22) The requested URL returned error: 504` installing the Scaleway
# CLI. The reflex reading is "curl does not retry on 504". Measured, it does:
# against a local server answering 504 forever, `--retry 3` sends four requests,
# one second apart then two then four.
#
# So retrying was never the gap. The BUDGET was: seven seconds.
#
# HOW LONG AN OUTAGE ACTUALLY LASTS HERE
#
# The one real measurement this repository has, taken 2026-09-14 while the same
# family of failure was killing three jobs: the asset
# `terraform-provider-outscale v1.8.0` answered 200 to five requests out of
# fifteen for roughly twenty minutes, then fifteen out of fifteen. A 67% failure
# rate, not a total one.
#
# Against that rate, and taking attempts as independent — which they are not
# when a nearby cache is what broke, so this is a floor rather than a promise:
#
# 4 attempts 19.8% chance of failing anyway <- what the workflows had
# 8 attempts 3.9%
# 10 attempts 1.7%
#
# 19.8% is exactly what was observed: passing most nights, failing some.
#
# WHY A FIXED INTERVAL RATHER THAN A BACKOFF
#
# Exponential backoff exists to spare a service that is struggling under load.
# What breaks here is a CDN answering 504 to everyone, which more waiting does
# not help. At an equal time budget a fixed interval buys far more attempts:
# ten of them cost 135s spaced 15s apart, against 511s doubling.
#
# WHAT IS NOT RETRIED
#
# A 404 is not an outage, it is a pin naming an asset that does not exist, and
# retrying it for two minutes turns a clear error into a slow one. curl's own
# notion of a transient error is the right one and it excludes 404: timeouts,
# and HTTP 408, 429, 500, 502, 503, 504.
set -euo pipefail

url="${1:?usage: fetch.sh <url> <destination>}"
dest="${2:?usage: fetch.sh <url> <destination>}"

# Overridable so a test can drive this in seconds rather than minutes. The
# defaults are the measured ones, and a caller that lowers them is choosing a
# higher chance of a red night.
attempts="${FEINT_FETCH_ATTEMPTS:-10}"
delay="${FEINT_FETCH_DELAY:-15}"
max_time="${FEINT_FETCH_MAX_TIME:-200}"

# --retry counts retries, not attempts: `--retry 9` sends ten requests.
curl --fail --silent --show-error --location \
--retry "$((attempts - 1))" \
--retry-delay "$delay" \
--retry-max-time "$max_time" \
--retry-connrefused \
--output "$dest" \
"$url"
Loading
Loading