From 72f2e18226b4b7e62deb3c6b7cf0a6a0764fce7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Fri, 25 Sep 2026 08:56:33 +0200 Subject: [PATCH] fix(ci): a retry budget sized on the outage that was measured, not on a guess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly conformance run of 2026-09-25 died installing the Scaleway CLI: curl: (22) The requested URL returned error: 504 from a step that already said `--retry 3 --retry-connrefused`. Three jobs had died the same way on 2026-09-14, and the Outscale half of the functional leg with them. CURL DOES RETRY ON A 504. THE BUDGET IS SEVEN SECONDS. The reflex reading is that curl ignores HTTP errors. Measured against a local server answering 504 forever, `--retry 3` sends four requests, one second apart then two then four — and gives up after seven seconds. `--retry-all-errors` changes nothing: 504 is already in curl's own list of transient errors. So retrying was never the gap. How long it retried was. HOW LONG AN OUTAGE ACTUALLY LASTS HERE One real measurement, taken 2026-09-14 while this family of failure was killing three jobs at once: the asset `terraform-provider-outscale v1.8.0` answered 200 to five requests out of fifteen for roughly twenty minutes, then fifteen out of fifteen. A 67% failure rate, not a total one. Against that rate, taking attempts as independent — which they are not when a nearby cache is what broke, so this is a floor rather than a promise: 4 attempts 19.8% chance of failing anyway <- what every step had 8 attempts 3.9% 10 attempts 1.7% 19.8% is exactly what was observed: passing most nights, failing some. A FIXED INTERVAL, NOT A BACKOFF Exponential backoff spares a service struggling under load. What breaks here is a CDN answering 504 to everyone, and waiting longer does not help it. At an equal time budget a fixed interval buys far more attempts: ten of them cost 135s spaced 15s apart, against 511s doubling. WHAT IS NOT RETRIED A 404 is a pin naming an asset that does not exist. Retrying it for two minutes turns a clear error into a slow one, and the log then says the same thing for two different problems. curl's own notion of transient is the right one and excludes 404. WHAT FALSIFY FOUND IN THIS WORK Every test drove the script with the budget lowered through the environment so it would run in seconds — and none of them touched the default. Setting `attempts=4`, the exact value that failed, left all of them green. The defaults now have a test of their own, and it asserts the calculation rather than the file: eight attempts is where the measured outage stops being likely to win. PROVEN - Four tests driving the real script against a server that fails then recovers, one that never recovers, and one that answers 404. - A guard refusing a workflow that downloads with curl instead of the helper, so the next tool added does not inherit the seven seconds. - Four mutations, each compiling and each biting. - The five falsify specs that mutate the seven workflows this touches: green, and `falsify:lint` finds all 1309 fragments across 209 specs. - `mise run prepush` green. Assisted-by: Claude Code (claude-opus-5) --- .github/workflows/conformance.yml | 29 +- .github/workflows/corpus-cloud.yml | 2 +- .github/workflows/go.yml | 4 +- .github/workflows/plumber.yml | 4 +- .github/workflows/runtime-proof.yml | 25 +- .github/workflows/secrets.yml | 4 +- .github/workflows/workflow-security.yml | 11 +- tools/ci/fetch.sh | 66 +++++ tools/ci/fetch_test.go | 274 ++++++++++++++++++ ...a-fetch-budget-that-crosses-an-outage.json | 33 +++ 10 files changed, 411 insertions(+), 41 deletions(-) create mode 100755 tools/ci/fetch.sh create mode 100644 tools/ci/fetch_test.go create mode 100644 tools/falsify/specs/a-fetch-budget-that-crosses-an-outage.json diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 6334c1f..3ff6939 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -122,8 +122,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/scaleway/scaleway-cli/releases/download/v${SCW_VERSION}" asset="scaleway-cli_${SCW_VERSION}_linux_amd64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/scw scw version @@ -148,8 +148,8 @@ jobs: workdir="$(mktemp -d)" base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}" asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin terraform version @@ -259,8 +259,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/scaleway/scaleway-cli/releases/download/v${SCW_VERSION}" asset="scaleway-cli_${SCW_VERSION}_linux_amd64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/scw scw version @@ -287,9 +287,8 @@ jobs: version="${OCTL_VERSION#v}" base="https://github.com/outscale/octl/releases/download/${OCTL_VERSION}" asset="octl_Linux_x86_64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" \ - "${base}/octl_${version}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/octl_${version}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/octl octl --version @@ -312,8 +311,8 @@ jobs: version="${EXO_VERSION#v}" base="https://github.com/exoscale/cli/releases/download/${EXO_VERSION}" asset="exoscale-cli_${version}_linux_amd64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/exoscale-cli_${version}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/exoscale-cli_${version}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" exo sudo install -m 0755 "${workdir}/exo" /usr/local/bin/exo @@ -347,8 +346,8 @@ jobs: workdir="$(mktemp -d)" base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}" asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin terraform version @@ -366,8 +365,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/opentofu/opentofu/releases/download/v${TOFU_VERSION}" asset="tofu_${TOFU_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/tofu_${TOFU_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/tofu_${TOFU_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" tofu -d /usr/local/bin tofu version diff --git a/.github/workflows/corpus-cloud.yml b/.github/workflows/corpus-cloud.yml index 2d23d70..0fa5845 100644 --- a/.github/workflows/corpus-cloud.yml +++ b/.github/workflows/corpus-cloud.yml @@ -89,7 +89,7 @@ jobs: set -euo pipefail version="2.56.3" url="https://github.com/scaleway/scaleway-cli/releases/download/v${version}/scaleway-cli_${version}_linux_amd64" - curl --fail --silent --show-error --location "$url" -o "$RUNNER_TEMP/scw" + tools/ci/fetch.sh "$url" "$RUNNER_TEMP/scw" chmod +x "$RUNNER_TEMP/scw" echo "$RUNNER_TEMP" >> "$GITHUB_PATH" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c1d69ce..f6be848 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -75,8 +75,8 @@ jobs: set -euo pipefail workdir="$(mktemp -d)" asset="shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" - curl -fsSL -o "${workdir}/${asset}" \ - "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${asset}" + tools/ci/fetch.sh "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${asset}" \ + "${workdir}/${asset}" tar -xJf "${workdir}/${asset}" -C "${workdir}" sudo install -m 0755 "${workdir}/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" /usr/local/bin/shellcheck shellcheck --version diff --git a/.github/workflows/plumber.yml b/.github/workflows/plumber.yml index 861db71..8979619 100644 --- a/.github/workflows/plumber.yml +++ b/.github/workflows/plumber.yml @@ -68,8 +68,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/getplumber/plumber/releases/download/v${PLUMBER_VERSION}" asset="plumber-linux-amd64" - curl -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl -fsSL -o "${workdir}/checksums.txt" "${base}/checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/checksums.txt" "${workdir}/checksums.txt" (cd "${workdir}" && grep " ${asset}$" checksums.txt | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/plumber plumber version diff --git a/.github/workflows/runtime-proof.yml b/.github/workflows/runtime-proof.yml index 07a439c..941a9fe 100644 --- a/.github/workflows/runtime-proof.yml +++ b/.github/workflows/runtime-proof.yml @@ -104,7 +104,7 @@ jobs: run: | set -euo pipefail workdir="$(mktemp -d)" - curl --retry 3 --retry-connrefused -fsSL https://pkgs.zabbly.com/key.asc -o "${workdir}/zabbly.asc" + tools/ci/fetch.sh https://pkgs.zabbly.com/key.asc "${workdir}/zabbly.asc" fpr="$(gpg --show-keys --with-colons "${workdir}/zabbly.asc" | awk -F: '/^fpr/ {print $10; exit}')" if [ "${fpr}" != "${ZABBLY_FPR}" ]; then echo "unexpected Zabbly key fingerprint: ${fpr}" >&2 @@ -309,8 +309,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/scaleway/scaleway-cli/releases/download/v${SCW_VERSION}" asset="scaleway-cli_${SCW_VERSION}_linux_amd64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/scw scw version @@ -327,9 +327,8 @@ jobs: version="${OCTL_VERSION#v}" base="https://github.com/outscale/octl/releases/download/${OCTL_VERSION}" asset="octl_Linux_x86_64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" \ - "${base}/octl_${version}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/octl_${version}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/octl octl --version @@ -343,8 +342,8 @@ jobs: version="${EXO_VERSION#v}" base="https://github.com/exoscale/cli/releases/download/${EXO_VERSION}" asset="exoscale-cli_${version}_linux_amd64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/exoscale-cli_${version}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/exoscale-cli_${version}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" exo sudo install -m 0755 "${workdir}/exo" /usr/local/bin/exo @@ -490,8 +489,8 @@ jobs: workdir="$(mktemp -d)" base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}" asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin terraform version @@ -650,7 +649,7 @@ jobs: run: | set -euo pipefail workdir="$(mktemp -d)" - curl --retry 3 --retry-connrefused -fsSL https://pkgs.zabbly.com/key.asc -o "${workdir}/zabbly.asc" + tools/ci/fetch.sh https://pkgs.zabbly.com/key.asc "${workdir}/zabbly.asc" fpr="$(gpg --show-keys --with-colons "${workdir}/zabbly.asc" | awk -F: '/^fpr/ {print $10; exit}')" if [ "${fpr}" != "${ZABBLY_FPR}" ]; then echo "unexpected Zabbly key fingerprint: ${fpr}" >&2 @@ -804,8 +803,8 @@ jobs: workdir="$(mktemp -d)" base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}" asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin terraform version diff --git a/.github/workflows/secrets.yml b/.github/workflows/secrets.yml index fcc5ba0..148a396 100644 --- a/.github/workflows/secrets.yml +++ b/.github/workflows/secrets.yml @@ -49,8 +49,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/trufflesecurity/trufflehog/releases/download/v${TRUFFLEHOG_VERSION}" asset="trufflehog_${TRUFFLEHOG_VERSION}_linux_amd64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/trufflehog_${TRUFFLEHOG_VERSION}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/trufflehog_${TRUFFLEHOG_VERSION}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" trufflehog sudo install -m 0755 "${workdir}/trufflehog" /usr/local/bin/trufflehog diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml index d255ec7..a82dc74 100644 --- a/.github/workflows/workflow-security.yml +++ b/.github/workflows/workflow-security.yml @@ -75,8 +75,7 @@ jobs: set -euo pipefail workdir="$(mktemp -d)" asset="zizmor-x86_64-unknown-linux-gnu.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" \ - "https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/${asset}" + tools/ci/fetch.sh "https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/${asset}" "${workdir}/${asset}" gh attestation verify "${workdir}/${asset}" --repo zizmorcore/zizmor tar -xzf "${workdir}/${asset}" -C "${workdir}" zizmor sudo install -m 0755 "${workdir}/zizmor" /usr/local/bin/zizmor @@ -129,8 +128,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}" asset="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/checksums.txt" "${base}/actionlint_${ACTIONLINT_VERSION}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/actionlint_${ACTIONLINT_VERSION}_checksums.txt" "${workdir}/checksums.txt" (cd "${workdir}" && grep " ${asset}$" checksums.txt | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" actionlint sudo install -m 0755 "${workdir}/actionlint" /usr/local/bin/actionlint @@ -167,8 +166,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/boostsecurityio/poutine/releases/download/v${POUTINE_VERSION}" asset="poutine_Linux_x86_64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/checksums.txt" "${base}/poutine_${POUTINE_VERSION}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/poutine_${POUTINE_VERSION}_checksums.txt" "${workdir}/checksums.txt" (cd "${workdir}" && grep " ${asset}$" checksums.txt | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" poutine sudo install -m 0755 "${workdir}/poutine" /usr/local/bin/poutine diff --git a/tools/ci/fetch.sh b/tools/ci/fetch.sh new file mode 100755 index 0000000..e40fbc0 --- /dev/null +++ b/tools/ci/fetch.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Fetch a release asset, with a retry budget sized on a measured outage. +# +# Usage: tools/ci/fetch.sh +# +# WHY THIS EXISTS +# +# Every install step in this repository already wrote `curl --retry 3 +# --retry-connrefused`, and the nightly conformance run of 2026-09-25 still died +# on `curl: (22) The requested URL returned error: 504` installing the Scaleway +# CLI. The reflex reading is "curl does not retry on 504". Measured, it does: +# against a local server answering 504 forever, `--retry 3` sends four requests, +# one second apart then two then four. +# +# So retrying was never the gap. The BUDGET was: seven seconds. +# +# HOW LONG AN OUTAGE ACTUALLY LASTS HERE +# +# The one real measurement this repository has, taken 2026-09-14 while the same +# family of failure was killing three jobs: the asset +# `terraform-provider-outscale v1.8.0` answered 200 to five requests out of +# fifteen for roughly twenty minutes, then fifteen out of fifteen. A 67% failure +# rate, not a total one. +# +# Against that rate, and taking attempts as independent — which they are not +# when a nearby cache is what broke, so this is a floor rather than a promise: +# +# 4 attempts 19.8% chance of failing anyway <- what the workflows had +# 8 attempts 3.9% +# 10 attempts 1.7% +# +# 19.8% is exactly what was observed: passing most nights, failing some. +# +# WHY A FIXED INTERVAL RATHER THAN A BACKOFF +# +# Exponential backoff exists to spare a service that is struggling under load. +# What breaks here is a CDN answering 504 to everyone, which more waiting does +# not help. At an equal time budget a fixed interval buys far more attempts: +# ten of them cost 135s spaced 15s apart, against 511s doubling. +# +# WHAT IS NOT RETRIED +# +# A 404 is not an outage, it is a pin naming an asset that does not exist, and +# retrying it for two minutes turns a clear error into a slow one. curl's own +# notion of a transient error is the right one and it excludes 404: timeouts, +# and HTTP 408, 429, 500, 502, 503, 504. +set -euo pipefail + +url="${1:?usage: fetch.sh }" +dest="${2:?usage: fetch.sh }" + +# Overridable so a test can drive this in seconds rather than minutes. The +# defaults are the measured ones, and a caller that lowers them is choosing a +# higher chance of a red night. +attempts="${FEINT_FETCH_ATTEMPTS:-10}" +delay="${FEINT_FETCH_DELAY:-15}" +max_time="${FEINT_FETCH_MAX_TIME:-200}" + +# --retry counts retries, not attempts: `--retry 9` sends ten requests. +curl --fail --silent --show-error --location \ + --retry "$((attempts - 1))" \ + --retry-delay "$delay" \ + --retry-max-time "$max_time" \ + --retry-connrefused \ + --output "$dest" \ + "$url" diff --git a/tools/ci/fetch_test.go b/tools/ci/fetch_test.go new file mode 100644 index 0000000..4b7ab84 --- /dev/null +++ b/tools/ci/fetch_test.go @@ -0,0 +1,274 @@ +package ci + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +// fetch.sh survives a transient outage and refuses to sit on a permanent one. +// +// The nightly conformance run of 2026-09-25 died on `curl: (22) ... 504` +// installing the Scaleway CLI, from a step that already said `--retry 3`. The +// gap was never the retry — measured against a local server, `--retry 3` does +// send four requests on a 504 — it was the seven seconds they span, against an +// outage measured at roughly twenty minutes on 2026-09-14. +// +// These tests drive the real script. The budget is lowered through the +// environment so they run in seconds; what is asserted is the behaviour, and +// the defaults live in the script with the measurement that sized them. + +// runFetch drives the script against a URL and answers the exit code. +func runFetch(t *testing.T, url, dest string, attempts, delay int) (int, string) { + t.Helper() + cmd := exec.Command("bash", filepath.Join("fetch.sh"), url, dest) + cmd.Env = append(os.Environ(), + "FEINT_FETCH_ATTEMPTS="+strconv.Itoa(attempts), + "FEINT_FETCH_DELAY="+strconv.Itoa(delay), + "FEINT_FETCH_MAX_TIME=30", + ) + out, err := cmd.CombinedOutput() + code := 0 + var exit *exec.ExitError + if err != nil { + if ok := asExitError(err, &exit); ok { + code = exit.ExitCode() + } else { + t.Fatalf("run fetch.sh: %v", err) + } + } + return code, string(out) +} + +func asExitError(err error, target **exec.ExitError) bool { + if e, ok := err.(*exec.ExitError); ok { //nolint:errorlint // the concrete type is the subject + *target = e + return true + } + return false +} + +// An asset that answers 504 and then recovers is fetched, not abandoned. +// +// This is the night of 2026-09-25 replayed: the first requests fail with the +// gateway's own error, and the payload is there a few seconds later. +func TestFetchSurvivesATransientOutage(t *testing.T) { + var seen atomic.Int32 + const failures = 3 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if seen.Add(1) <= failures { + w.WriteHeader(http.StatusGatewayTimeout) + return + } + fmt.Fprint(w, "the asset") + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "asset") + code, out := runFetch(t, srv.URL+"/asset", dest, 6, 1) + if code != 0 { + t.Fatalf("fetch.sh gave up on an outage that ended: exit %d\n%s", code, out) + } + if got := int(seen.Load()); got <= failures { + t.Fatalf("the server saw %d request(s) for %d failure(s): the retries did not happen", + got, failures) + } + body, err := os.ReadFile(dest) //nolint:gosec // a path this test made + if err != nil { + t.Fatalf("read what was fetched: %v", err) + } + if string(body) != "the asset" { + t.Errorf("fetched %q, want the payload the server answered once it recovered", body) + } +} + +// An outage that does not end fails, rather than hanging on a runner. +// +// The budget is a budget: a gate that waits forever is a gate nobody can read +// the log of, and the point of #731 is that a night has to end saying something. +func TestFetchGivesUpOnAnOutageThatDoesNotEnd(t *testing.T) { + var seen atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + seen.Add(1) + w.WriteHeader(http.StatusGatewayTimeout) + })) + defer srv.Close() + + start := time.Now() + code, _ := runFetch(t, srv.URL+"/asset", filepath.Join(t.TempDir(), "asset"), 4, 1) + if code == 0 { + t.Fatal("fetch.sh reported success against a server that never recovered") + } + if elapsed := time.Since(start); elapsed > 25*time.Second { + t.Errorf("it took %s to give up; the max-time budget is not holding", elapsed) + } + if got := int(seen.Load()); got < 2 { + t.Errorf("the server saw %d request(s): it gave up without retrying at all", got) + } +} + +// A 404 is a pin naming something that does not exist, and is not retried. +// +// Retrying it would turn a clear error — the asset is not there, fix the +// version — into the same slow red as a real outage, and the log would say the +// same thing for two different problems. +func TestFetchDoesNotRetryAMissingAsset(t *testing.T) { + var seen atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + seen.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + start := time.Now() + code, _ := runFetch(t, srv.URL+"/nope", filepath.Join(t.TempDir(), "asset"), 6, 2) + if code == 0 { + t.Fatal("fetch.sh reported success on a 404") + } + if got := int(seen.Load()); got != 1 { + t.Errorf("the server saw %d request(s) for a 404: a missing asset is not an outage, "+ + "and retrying it makes a clear error look like a slow one", got) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Errorf("a 404 took %s to report; it should be immediate", elapsed) + } +} + +// Nothing is written when the fetch fails. +// +// A zero-byte file left behind is worse than none: the checksum step that +// follows would compare against it and fail with a confusing message, or an +// install step would put an empty binary on the PATH. +func TestFetchLeavesNothingBehindWhenItFails(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "asset") + if code, _ := runFetch(t, srv.URL+"/nope", dest, 2, 1); code == 0 { + t.Fatal("fetch.sh reported success on a 404") + } + if _, err := os.Stat(dest); err == nil { + t.Error("a file was left at the destination after a failed fetch: the checksum step " + + "that follows would compare against it") + } +} + +// No workflow downloads a release asset behind the helper's back. +// +// Without this, the next install step written in this repository gets the same +// seven-second budget the old ones had, and the fix lasts exactly until someone +// adds a tool. The measurement that sized the budget lives in fetch.sh; a curl +// written next to it does not read that file. +// +// A curl reaching the local emulator is a different thing and stays allowed: +// what is refused is fetching something over the network into a file. +func TestNoWorkflowFetchesAnAssetWithoutTheHelper(t *testing.T) { + dir := filepath.Join("..", "..", ".github", "workflows") + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read workflows: %v", err) + } + + found := 0 + for _, entry := range entries { + if filepath.Ext(entry.Name()) != ".yml" { + continue + } + body, err := os.ReadFile(filepath.Join(dir, entry.Name())) //nolint:gosec // a path from our own walk + if err != nil { + t.Fatalf("read %s: %v", entry.Name(), err) + } + for i, line := range strings.Split(string(body), "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") || !strings.Contains(trimmed, "curl ") { + continue + } + // Reading the emulator this repository just started is not a fetch. + if strings.Contains(trimmed, "127.0.0.1") || strings.Contains(trimmed, "localhost") { + continue + } + if !strings.Contains(trimmed, "-o ") && !strings.Contains(trimmed, "--output") { + continue + } + found++ + t.Errorf("%s:%d downloads with curl instead of tools/ci/fetch.sh, so it keeps the "+ + "seven-second budget that failed the night of 2026-09-25:\n %s", + entry.Name(), i+1, trimmed) + } + } + if found == 0 { + t.Log("every workflow download goes through the helper") + } +} + +// The defaults are the measured ones, and lowering them is a decision a test +// has to refuse. +// +// Every test above drives the script with the budget lowered through the +// environment, so they run in seconds — and none of them touches the default. +// Falsify showed what that costs: setting `attempts=4`, the exact value that +// failed the night of 2026-09-25, left all of them green. +// +// So the threshold is asserted here, and it is the calculation rather than the +// file: at the 67% failure rate measured on 2026-09-14, four attempts leave a +// 19.8% chance of failing anyway and eight bring it under 4%. Eight is the +// floor; fetch.sh chooses ten. +func TestTheFetchDefaultsAreTheMeasuredOnes(t *testing.T) { + body, err := os.ReadFile("fetch.sh") + if err != nil { + t.Fatalf("read fetch.sh: %v", err) + } + script := string(body) + + defaults := map[string]int{ + "FEINT_FETCH_ATTEMPTS": 0, + "FEINT_FETCH_MAX_TIME": 0, + } + for name := range defaults { + marker := "${" + name + ":-" + at := strings.Index(script, marker) + if at < 0 { + t.Fatalf("fetch.sh declares no default for %s", name) + } + rest := script[at+len(marker):] + end := strings.IndexByte(rest, '}') + if end < 0 { + t.Fatalf("the default for %s is not closed", name) + } + value, convErr := strconv.Atoi(rest[:end]) + if convErr != nil { + t.Fatalf("the default for %s is %q, not a number", name, rest[:end]) + } + defaults[name] = value + } + + // Eight attempts is where the measured outage stops being likely to win. + if got := defaults["FEINT_FETCH_ATTEMPTS"]; got < 8 { + t.Errorf("the default is %d attempts: at the 67%% failure rate measured on 2026-09-14 "+ + "that leaves a %.0f%% chance of a red night, and the whole point of this script "+ + "is that four attempts already failed", got, 100*pow(2.0/3.0, got)) + } + // And it ends: a gate that waits forever produces a log nobody can read. + if got := defaults["FEINT_FETCH_MAX_TIME"]; got <= 0 || got > 600 { + t.Errorf("the retry ceiling is %ds: it must be finite so a night ends with a verdict, "+ + "and short enough that a runner is not held for the whole outage", got) + } +} + +func pow(base float64, n int) float64 { + out := 1.0 + for range n { + out *= base + } + return out +} diff --git a/tools/falsify/specs/a-fetch-budget-that-crosses-an-outage.json b/tools/falsify/specs/a-fetch-budget-that-crosses-an-outage.json new file mode 100644 index 0000000..91ba874 --- /dev/null +++ b/tools/falsify/specs/a-fetch-budget-that-crosses-an-outage.json @@ -0,0 +1,33 @@ +{ + "package": "./tools/ci/", + "mutations": [ + { + "label": "the budget goes back to what every install step had, seven seconds, which is a one-in-five chance of a red night at the failure rate measured on 2026-09-14", + "file": "tools/ci/fetch.sh", + "find": "attempts=\"${FEINT_FETCH_ATTEMPTS:-10}\"", + "replace": "attempts=\"${FEINT_FETCH_ATTEMPTS:-4}\"", + "test": "TestTheFetchDefaultsAreTheMeasuredOnes" + }, + { + "label": "a missing asset is retried like an outage, so a pin naming something that does not exist takes the full budget and reads like a bad night", + "file": "tools/ci/fetch.sh", + "find": " --retry-connrefused \\", + "replace": " --retry-connrefused --retry-all-errors \\", + "test": "TestFetchDoesNotRetryAMissingAsset" + }, + { + "label": "the destination keeps whatever a failed fetch wrote, so the checksum step compares against an empty file", + "file": "tools/ci/fetch.sh", + "find": "curl --fail --silent --show-error --location \\", + "replace": "curl --fail --no-fail --silent --show-error --location \\", + "test": "TestFetchLeavesNothingBehindWhenItFails" + }, + { + "label": "the retry budget has no ceiling, so an outage that does not end holds a runner instead of ending the night with a verdict", + "file": "tools/ci/fetch.sh", + "find": "max_time=\"${FEINT_FETCH_MAX_TIME:-200}\"", + "replace": "max_time=\"${FEINT_FETCH_MAX_TIME:-0}\"", + "test": "TestTheFetchDefaultsAreTheMeasuredOnes" + } + ] +}