diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 6334c1f..3ff6939 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -122,8 +122,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/scaleway/scaleway-cli/releases/download/v${SCW_VERSION}" asset="scaleway-cli_${SCW_VERSION}_linux_amd64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/scw scw version @@ -148,8 +148,8 @@ jobs: workdir="$(mktemp -d)" base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}" asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin terraform version @@ -259,8 +259,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/scaleway/scaleway-cli/releases/download/v${SCW_VERSION}" asset="scaleway-cli_${SCW_VERSION}_linux_amd64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/scw scw version @@ -287,9 +287,8 @@ jobs: version="${OCTL_VERSION#v}" base="https://github.com/outscale/octl/releases/download/${OCTL_VERSION}" asset="octl_Linux_x86_64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" \ - "${base}/octl_${version}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/octl_${version}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/octl octl --version @@ -312,8 +311,8 @@ jobs: version="${EXO_VERSION#v}" base="https://github.com/exoscale/cli/releases/download/${EXO_VERSION}" asset="exoscale-cli_${version}_linux_amd64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/exoscale-cli_${version}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/exoscale-cli_${version}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" exo sudo install -m 0755 "${workdir}/exo" /usr/local/bin/exo @@ -347,8 +346,8 @@ jobs: workdir="$(mktemp -d)" base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}" asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin terraform version @@ -366,8 +365,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/opentofu/opentofu/releases/download/v${TOFU_VERSION}" asset="tofu_${TOFU_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/tofu_${TOFU_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/tofu_${TOFU_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" tofu -d /usr/local/bin tofu version diff --git a/.github/workflows/corpus-cloud.yml b/.github/workflows/corpus-cloud.yml index 2d23d70..0fa5845 100644 --- a/.github/workflows/corpus-cloud.yml +++ b/.github/workflows/corpus-cloud.yml @@ -89,7 +89,7 @@ jobs: set -euo pipefail version="2.56.3" url="https://github.com/scaleway/scaleway-cli/releases/download/v${version}/scaleway-cli_${version}_linux_amd64" - curl --fail --silent --show-error --location "$url" -o "$RUNNER_TEMP/scw" + tools/ci/fetch.sh "$url" "$RUNNER_TEMP/scw" chmod +x "$RUNNER_TEMP/scw" echo "$RUNNER_TEMP" >> "$GITHUB_PATH" diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c1d69ce..f6be848 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -75,8 +75,8 @@ jobs: set -euo pipefail workdir="$(mktemp -d)" asset="shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" - curl -fsSL -o "${workdir}/${asset}" \ - "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${asset}" + tools/ci/fetch.sh "https://github.com/koalaman/shellcheck/releases/download/v${SHELLCHECK_VERSION}/${asset}" \ + "${workdir}/${asset}" tar -xJf "${workdir}/${asset}" -C "${workdir}" sudo install -m 0755 "${workdir}/shellcheck-v${SHELLCHECK_VERSION}/shellcheck" /usr/local/bin/shellcheck shellcheck --version diff --git a/.github/workflows/plumber.yml b/.github/workflows/plumber.yml index 861db71..8979619 100644 --- a/.github/workflows/plumber.yml +++ b/.github/workflows/plumber.yml @@ -68,8 +68,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/getplumber/plumber/releases/download/v${PLUMBER_VERSION}" asset="plumber-linux-amd64" - curl -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl -fsSL -o "${workdir}/checksums.txt" "${base}/checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/checksums.txt" "${workdir}/checksums.txt" (cd "${workdir}" && grep " ${asset}$" checksums.txt | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/plumber plumber version diff --git a/.github/workflows/runtime-proof.yml b/.github/workflows/runtime-proof.yml index 07a439c..941a9fe 100644 --- a/.github/workflows/runtime-proof.yml +++ b/.github/workflows/runtime-proof.yml @@ -104,7 +104,7 @@ jobs: run: | set -euo pipefail workdir="$(mktemp -d)" - curl --retry 3 --retry-connrefused -fsSL https://pkgs.zabbly.com/key.asc -o "${workdir}/zabbly.asc" + tools/ci/fetch.sh https://pkgs.zabbly.com/key.asc "${workdir}/zabbly.asc" fpr="$(gpg --show-keys --with-colons "${workdir}/zabbly.asc" | awk -F: '/^fpr/ {print $10; exit}')" if [ "${fpr}" != "${ZABBLY_FPR}" ]; then echo "unexpected Zabbly key fingerprint: ${fpr}" >&2 @@ -309,8 +309,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/scaleway/scaleway-cli/releases/download/v${SCW_VERSION}" asset="scaleway-cli_${SCW_VERSION}_linux_amd64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/scw scw version @@ -327,9 +327,8 @@ jobs: version="${OCTL_VERSION#v}" base="https://github.com/outscale/octl/releases/download/${OCTL_VERSION}" asset="octl_Linux_x86_64" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" \ - "${base}/octl_${version}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/octl_${version}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo install -m 0755 "${workdir}/${asset}" /usr/local/bin/octl octl --version @@ -343,8 +342,8 @@ jobs: version="${EXO_VERSION#v}" base="https://github.com/exoscale/cli/releases/download/${EXO_VERSION}" asset="exoscale-cli_${version}_linux_amd64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/exoscale-cli_${version}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/exoscale-cli_${version}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" exo sudo install -m 0755 "${workdir}/exo" /usr/local/bin/exo @@ -490,8 +489,8 @@ jobs: workdir="$(mktemp -d)" base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}" asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin terraform version @@ -650,7 +649,7 @@ jobs: run: | set -euo pipefail workdir="$(mktemp -d)" - curl --retry 3 --retry-connrefused -fsSL https://pkgs.zabbly.com/key.asc -o "${workdir}/zabbly.asc" + tools/ci/fetch.sh https://pkgs.zabbly.com/key.asc "${workdir}/zabbly.asc" fpr="$(gpg --show-keys --with-colons "${workdir}/zabbly.asc" | awk -F: '/^fpr/ {print $10; exit}')" if [ "${fpr}" != "${ZABBLY_FPR}" ]; then echo "unexpected Zabbly key fingerprint: ${fpr}" >&2 @@ -804,8 +803,8 @@ jobs: workdir="$(mktemp -d)" base="https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}" asset="terraform_${TERRAFORM_VERSION}_linux_amd64.zip" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/terraform_${TERRAFORM_VERSION}_SHA256SUMS" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) sudo unzip -o "${workdir}/${asset}" terraform -d /usr/local/bin terraform version diff --git a/.github/workflows/secrets.yml b/.github/workflows/secrets.yml index fcc5ba0..148a396 100644 --- a/.github/workflows/secrets.yml +++ b/.github/workflows/secrets.yml @@ -49,8 +49,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/trufflesecurity/trufflehog/releases/download/v${TRUFFLEHOG_VERSION}" asset="trufflehog_${TRUFFLEHOG_VERSION}_linux_amd64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/sums" "${base}/trufflehog_${TRUFFLEHOG_VERSION}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/trufflehog_${TRUFFLEHOG_VERSION}_checksums.txt" "${workdir}/sums" (cd "${workdir}" && grep " ${asset}$" sums | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" trufflehog sudo install -m 0755 "${workdir}/trufflehog" /usr/local/bin/trufflehog diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml index d255ec7..a82dc74 100644 --- a/.github/workflows/workflow-security.yml +++ b/.github/workflows/workflow-security.yml @@ -75,8 +75,7 @@ jobs: set -euo pipefail workdir="$(mktemp -d)" asset="zizmor-x86_64-unknown-linux-gnu.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" \ - "https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/${asset}" + tools/ci/fetch.sh "https://github.com/zizmorcore/zizmor/releases/download/v${ZIZMOR_VERSION}/${asset}" "${workdir}/${asset}" gh attestation verify "${workdir}/${asset}" --repo zizmorcore/zizmor tar -xzf "${workdir}/${asset}" -C "${workdir}" zizmor sudo install -m 0755 "${workdir}/zizmor" /usr/local/bin/zizmor @@ -129,8 +128,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}" asset="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/checksums.txt" "${base}/actionlint_${ACTIONLINT_VERSION}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/actionlint_${ACTIONLINT_VERSION}_checksums.txt" "${workdir}/checksums.txt" (cd "${workdir}" && grep " ${asset}$" checksums.txt | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" actionlint sudo install -m 0755 "${workdir}/actionlint" /usr/local/bin/actionlint @@ -167,8 +166,8 @@ jobs: workdir="$(mktemp -d)" base="https://github.com/boostsecurityio/poutine/releases/download/v${POUTINE_VERSION}" asset="poutine_Linux_x86_64.tar.gz" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/${asset}" "${base}/${asset}" - curl --retry 3 --retry-connrefused -fsSL -o "${workdir}/checksums.txt" "${base}/poutine_${POUTINE_VERSION}_checksums.txt" + tools/ci/fetch.sh "${base}/${asset}" "${workdir}/${asset}" + tools/ci/fetch.sh "${base}/poutine_${POUTINE_VERSION}_checksums.txt" "${workdir}/checksums.txt" (cd "${workdir}" && grep " ${asset}$" checksums.txt | sha256sum -c -) tar -xzf "${workdir}/${asset}" -C "${workdir}" poutine sudo install -m 0755 "${workdir}/poutine" /usr/local/bin/poutine diff --git a/tools/ci/fetch.sh b/tools/ci/fetch.sh new file mode 100755 index 0000000..e40fbc0 --- /dev/null +++ b/tools/ci/fetch.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Fetch a release asset, with a retry budget sized on a measured outage. +# +# Usage: tools/ci/fetch.sh +# +# WHY THIS EXISTS +# +# Every install step in this repository already wrote `curl --retry 3 +# --retry-connrefused`, and the nightly conformance run of 2026-09-25 still died +# on `curl: (22) The requested URL returned error: 504` installing the Scaleway +# CLI. The reflex reading is "curl does not retry on 504". Measured, it does: +# against a local server answering 504 forever, `--retry 3` sends four requests, +# one second apart then two then four. +# +# So retrying was never the gap. The BUDGET was: seven seconds. +# +# HOW LONG AN OUTAGE ACTUALLY LASTS HERE +# +# The one real measurement this repository has, taken 2026-09-14 while the same +# family of failure was killing three jobs: the asset +# `terraform-provider-outscale v1.8.0` answered 200 to five requests out of +# fifteen for roughly twenty minutes, then fifteen out of fifteen. A 67% failure +# rate, not a total one. +# +# Against that rate, and taking attempts as independent — which they are not +# when a nearby cache is what broke, so this is a floor rather than a promise: +# +# 4 attempts 19.8% chance of failing anyway <- what the workflows had +# 8 attempts 3.9% +# 10 attempts 1.7% +# +# 19.8% is exactly what was observed: passing most nights, failing some. +# +# WHY A FIXED INTERVAL RATHER THAN A BACKOFF +# +# Exponential backoff exists to spare a service that is struggling under load. +# What breaks here is a CDN answering 504 to everyone, which more waiting does +# not help. At an equal time budget a fixed interval buys far more attempts: +# ten of them cost 135s spaced 15s apart, against 511s doubling. +# +# WHAT IS NOT RETRIED +# +# A 404 is not an outage, it is a pin naming an asset that does not exist, and +# retrying it for two minutes turns a clear error into a slow one. curl's own +# notion of a transient error is the right one and it excludes 404: timeouts, +# and HTTP 408, 429, 500, 502, 503, 504. +set -euo pipefail + +url="${1:?usage: fetch.sh }" +dest="${2:?usage: fetch.sh }" + +# Overridable so a test can drive this in seconds rather than minutes. The +# defaults are the measured ones, and a caller that lowers them is choosing a +# higher chance of a red night. +attempts="${FEINT_FETCH_ATTEMPTS:-10}" +delay="${FEINT_FETCH_DELAY:-15}" +max_time="${FEINT_FETCH_MAX_TIME:-200}" + +# --retry counts retries, not attempts: `--retry 9` sends ten requests. +curl --fail --silent --show-error --location \ + --retry "$((attempts - 1))" \ + --retry-delay "$delay" \ + --retry-max-time "$max_time" \ + --retry-connrefused \ + --output "$dest" \ + "$url" diff --git a/tools/ci/fetch_test.go b/tools/ci/fetch_test.go new file mode 100644 index 0000000..4b7ab84 --- /dev/null +++ b/tools/ci/fetch_test.go @@ -0,0 +1,274 @@ +package ci + +import ( + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +// fetch.sh survives a transient outage and refuses to sit on a permanent one. +// +// The nightly conformance run of 2026-09-25 died on `curl: (22) ... 504` +// installing the Scaleway CLI, from a step that already said `--retry 3`. The +// gap was never the retry — measured against a local server, `--retry 3` does +// send four requests on a 504 — it was the seven seconds they span, against an +// outage measured at roughly twenty minutes on 2026-09-14. +// +// These tests drive the real script. The budget is lowered through the +// environment so they run in seconds; what is asserted is the behaviour, and +// the defaults live in the script with the measurement that sized them. + +// runFetch drives the script against a URL and answers the exit code. +func runFetch(t *testing.T, url, dest string, attempts, delay int) (int, string) { + t.Helper() + cmd := exec.Command("bash", filepath.Join("fetch.sh"), url, dest) + cmd.Env = append(os.Environ(), + "FEINT_FETCH_ATTEMPTS="+strconv.Itoa(attempts), + "FEINT_FETCH_DELAY="+strconv.Itoa(delay), + "FEINT_FETCH_MAX_TIME=30", + ) + out, err := cmd.CombinedOutput() + code := 0 + var exit *exec.ExitError + if err != nil { + if ok := asExitError(err, &exit); ok { + code = exit.ExitCode() + } else { + t.Fatalf("run fetch.sh: %v", err) + } + } + return code, string(out) +} + +func asExitError(err error, target **exec.ExitError) bool { + if e, ok := err.(*exec.ExitError); ok { //nolint:errorlint // the concrete type is the subject + *target = e + return true + } + return false +} + +// An asset that answers 504 and then recovers is fetched, not abandoned. +// +// This is the night of 2026-09-25 replayed: the first requests fail with the +// gateway's own error, and the payload is there a few seconds later. +func TestFetchSurvivesATransientOutage(t *testing.T) { + var seen atomic.Int32 + const failures = 3 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if seen.Add(1) <= failures { + w.WriteHeader(http.StatusGatewayTimeout) + return + } + fmt.Fprint(w, "the asset") + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "asset") + code, out := runFetch(t, srv.URL+"/asset", dest, 6, 1) + if code != 0 { + t.Fatalf("fetch.sh gave up on an outage that ended: exit %d\n%s", code, out) + } + if got := int(seen.Load()); got <= failures { + t.Fatalf("the server saw %d request(s) for %d failure(s): the retries did not happen", + got, failures) + } + body, err := os.ReadFile(dest) //nolint:gosec // a path this test made + if err != nil { + t.Fatalf("read what was fetched: %v", err) + } + if string(body) != "the asset" { + t.Errorf("fetched %q, want the payload the server answered once it recovered", body) + } +} + +// An outage that does not end fails, rather than hanging on a runner. +// +// The budget is a budget: a gate that waits forever is a gate nobody can read +// the log of, and the point of #731 is that a night has to end saying something. +func TestFetchGivesUpOnAnOutageThatDoesNotEnd(t *testing.T) { + var seen atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + seen.Add(1) + w.WriteHeader(http.StatusGatewayTimeout) + })) + defer srv.Close() + + start := time.Now() + code, _ := runFetch(t, srv.URL+"/asset", filepath.Join(t.TempDir(), "asset"), 4, 1) + if code == 0 { + t.Fatal("fetch.sh reported success against a server that never recovered") + } + if elapsed := time.Since(start); elapsed > 25*time.Second { + t.Errorf("it took %s to give up; the max-time budget is not holding", elapsed) + } + if got := int(seen.Load()); got < 2 { + t.Errorf("the server saw %d request(s): it gave up without retrying at all", got) + } +} + +// A 404 is a pin naming something that does not exist, and is not retried. +// +// Retrying it would turn a clear error — the asset is not there, fix the +// version — into the same slow red as a real outage, and the log would say the +// same thing for two different problems. +func TestFetchDoesNotRetryAMissingAsset(t *testing.T) { + var seen atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + seen.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + start := time.Now() + code, _ := runFetch(t, srv.URL+"/nope", filepath.Join(t.TempDir(), "asset"), 6, 2) + if code == 0 { + t.Fatal("fetch.sh reported success on a 404") + } + if got := int(seen.Load()); got != 1 { + t.Errorf("the server saw %d request(s) for a 404: a missing asset is not an outage, "+ + "and retrying it makes a clear error look like a slow one", got) + } + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Errorf("a 404 took %s to report; it should be immediate", elapsed) + } +} + +// Nothing is written when the fetch fails. +// +// A zero-byte file left behind is worse than none: the checksum step that +// follows would compare against it and fail with a confusing message, or an +// install step would put an empty binary on the PATH. +func TestFetchLeavesNothingBehindWhenItFails(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "asset") + if code, _ := runFetch(t, srv.URL+"/nope", dest, 2, 1); code == 0 { + t.Fatal("fetch.sh reported success on a 404") + } + if _, err := os.Stat(dest); err == nil { + t.Error("a file was left at the destination after a failed fetch: the checksum step " + + "that follows would compare against it") + } +} + +// No workflow downloads a release asset behind the helper's back. +// +// Without this, the next install step written in this repository gets the same +// seven-second budget the old ones had, and the fix lasts exactly until someone +// adds a tool. The measurement that sized the budget lives in fetch.sh; a curl +// written next to it does not read that file. +// +// A curl reaching the local emulator is a different thing and stays allowed: +// what is refused is fetching something over the network into a file. +func TestNoWorkflowFetchesAnAssetWithoutTheHelper(t *testing.T) { + dir := filepath.Join("..", "..", ".github", "workflows") + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read workflows: %v", err) + } + + found := 0 + for _, entry := range entries { + if filepath.Ext(entry.Name()) != ".yml" { + continue + } + body, err := os.ReadFile(filepath.Join(dir, entry.Name())) //nolint:gosec // a path from our own walk + if err != nil { + t.Fatalf("read %s: %v", entry.Name(), err) + } + for i, line := range strings.Split(string(body), "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") || !strings.Contains(trimmed, "curl ") { + continue + } + // Reading the emulator this repository just started is not a fetch. + if strings.Contains(trimmed, "127.0.0.1") || strings.Contains(trimmed, "localhost") { + continue + } + if !strings.Contains(trimmed, "-o ") && !strings.Contains(trimmed, "--output") { + continue + } + found++ + t.Errorf("%s:%d downloads with curl instead of tools/ci/fetch.sh, so it keeps the "+ + "seven-second budget that failed the night of 2026-09-25:\n %s", + entry.Name(), i+1, trimmed) + } + } + if found == 0 { + t.Log("every workflow download goes through the helper") + } +} + +// The defaults are the measured ones, and lowering them is a decision a test +// has to refuse. +// +// Every test above drives the script with the budget lowered through the +// environment, so they run in seconds — and none of them touches the default. +// Falsify showed what that costs: setting `attempts=4`, the exact value that +// failed the night of 2026-09-25, left all of them green. +// +// So the threshold is asserted here, and it is the calculation rather than the +// file: at the 67% failure rate measured on 2026-09-14, four attempts leave a +// 19.8% chance of failing anyway and eight bring it under 4%. Eight is the +// floor; fetch.sh chooses ten. +func TestTheFetchDefaultsAreTheMeasuredOnes(t *testing.T) { + body, err := os.ReadFile("fetch.sh") + if err != nil { + t.Fatalf("read fetch.sh: %v", err) + } + script := string(body) + + defaults := map[string]int{ + "FEINT_FETCH_ATTEMPTS": 0, + "FEINT_FETCH_MAX_TIME": 0, + } + for name := range defaults { + marker := "${" + name + ":-" + at := strings.Index(script, marker) + if at < 0 { + t.Fatalf("fetch.sh declares no default for %s", name) + } + rest := script[at+len(marker):] + end := strings.IndexByte(rest, '}') + if end < 0 { + t.Fatalf("the default for %s is not closed", name) + } + value, convErr := strconv.Atoi(rest[:end]) + if convErr != nil { + t.Fatalf("the default for %s is %q, not a number", name, rest[:end]) + } + defaults[name] = value + } + + // Eight attempts is where the measured outage stops being likely to win. + if got := defaults["FEINT_FETCH_ATTEMPTS"]; got < 8 { + t.Errorf("the default is %d attempts: at the 67%% failure rate measured on 2026-09-14 "+ + "that leaves a %.0f%% chance of a red night, and the whole point of this script "+ + "is that four attempts already failed", got, 100*pow(2.0/3.0, got)) + } + // And it ends: a gate that waits forever produces a log nobody can read. + if got := defaults["FEINT_FETCH_MAX_TIME"]; got <= 0 || got > 600 { + t.Errorf("the retry ceiling is %ds: it must be finite so a night ends with a verdict, "+ + "and short enough that a runner is not held for the whole outage", got) + } +} + +func pow(base float64, n int) float64 { + out := 1.0 + for range n { + out *= base + } + return out +} diff --git a/tools/falsify/specs/a-fetch-budget-that-crosses-an-outage.json b/tools/falsify/specs/a-fetch-budget-that-crosses-an-outage.json new file mode 100644 index 0000000..91ba874 --- /dev/null +++ b/tools/falsify/specs/a-fetch-budget-that-crosses-an-outage.json @@ -0,0 +1,33 @@ +{ + "package": "./tools/ci/", + "mutations": [ + { + "label": "the budget goes back to what every install step had, seven seconds, which is a one-in-five chance of a red night at the failure rate measured on 2026-09-14", + "file": "tools/ci/fetch.sh", + "find": "attempts=\"${FEINT_FETCH_ATTEMPTS:-10}\"", + "replace": "attempts=\"${FEINT_FETCH_ATTEMPTS:-4}\"", + "test": "TestTheFetchDefaultsAreTheMeasuredOnes" + }, + { + "label": "a missing asset is retried like an outage, so a pin naming something that does not exist takes the full budget and reads like a bad night", + "file": "tools/ci/fetch.sh", + "find": " --retry-connrefused \\", + "replace": " --retry-connrefused --retry-all-errors \\", + "test": "TestFetchDoesNotRetryAMissingAsset" + }, + { + "label": "the destination keeps whatever a failed fetch wrote, so the checksum step compares against an empty file", + "file": "tools/ci/fetch.sh", + "find": "curl --fail --silent --show-error --location \\", + "replace": "curl --fail --no-fail --silent --show-error --location \\", + "test": "TestFetchLeavesNothingBehindWhenItFails" + }, + { + "label": "the retry budget has no ceiling, so an outage that does not end holds a runner instead of ending the night with a verdict", + "file": "tools/ci/fetch.sh", + "find": "max_time=\"${FEINT_FETCH_MAX_TIME:-200}\"", + "replace": "max_time=\"${FEINT_FETCH_MAX_TIME:-0}\"", + "test": "TestTheFetchDefaultsAreTheMeasuredOnes" + } + ] +}