fix: retry a failing download server and say it was an outage

CHANGELOG.md

@@ -7,6 +7,22 @@ Versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 today keeps working across minor and patch releases; a change that would break
 one waits for a major.
 
+## [Unreleased]
+
+### Fixed
+
+- **A download server that fails for a moment no longer ends the command.** The
+  release listings, the VM image and its `sha256sums`, and out-of-feed packages
+  are asked again on a 5xx, 429 or dropped connection: 4 attempts, 2, 4 and 8 s
+  apart. A 404 is still asked once. When a failure outlasts the retries, the
+  message says it is an outage to wait out, not a problem with `owlab.yaml`.
+
+- **`setup` tells a GitHub outage from a bad release.** `gh release download`
+  and `gh release view` are retried while GitHub answers 5xx or cannot be
+  reached, and exit 8 when that outlasts 4 attempts. When `gh attestation verify`
+  cannot reach GitHub or Sigstore, the step retries and reports an outage
+  (exit 8) instead of "does not verify".
+
 ## [0.6.4] - 2026-09-14
 
 ### Fixed

docs/troubleshooting.md

@@ -562,6 +562,33 @@ looking at is worth doing before debugging anything else.
 
 ---
 
+## "A download failed"
+
+### `503 Service Unavailable (the server is failing, not refusing …)`
+
+The download server or GitHub was failing, and kept failing through owlab's
+retries (4 attempts over about 14 seconds). Nothing is wrong with `owlab.yaml`.
+Run the command again later. For GitHub, check
+[githubstatus.com](https://www.githubstatus.com).
+
+The same applies to `unreachable after 4 attempts (a network or server outage …)`,
+which covers timeouts, DNS failures and refused connections.
+
+### `404 Not Found` on an image, a rootfs or a package
+
+The server answered, and the file is not there. owlab asks for a 404 once. Check
+the release number and target in `owlab.yaml`: the release may not publish that
+image yet, or the out-of-feed package URL is wrong.
+
+### `setup` step: `Upstream outage (safe to rerun)`, exit code 8
+
+`gh release download`, `gh release view` or `gh attestation verify` could not reach
+GitHub or Sigstore after 4 attempts. The binary was not checked, which is different
+from failing verification. Rerun the job. `does not verify as built by
+owfeed/owlab's release workflow`, exit code 1, is the real refusal.
+
+---
+
 ## Three upstream behaviours worth knowing
 
 ### `kmods` must stay in the feed list

docs/troubleshooting_ru.md

@@ -561,6 +561,32 @@ VM не грузится или виснет на середине, тогда 
 
 ---
 
+## «Загрузка упала»
+
+### `503 Service Unavailable (the server is failing, not refusing …)`
+
+Сервер загрузок или GitHub сбоил и продолжал сбоить на протяжении всех повторов owlab
+(4 попытки примерно за 14 секунд). С `owlab.yaml` всё в порядке. Запустите команду
+позже. Для GitHub загляните на [githubstatus.com](https://www.githubstatus.com).
+
+То же относится к `unreachable after 4 attempts (a network or server outage …)`: так
+выглядят таймауты, сбои DNS и отказы в соединении.
+
+### `404 Not Found` на образ, rootfs или пакет
+
+Сервер ответил, и файла там нет. На 404 owlab спрашивает один раз. Проверьте номер
+релиза и target в `owlab.yaml`: релиз может ещё не публиковать такой образ, или URL
+внефидового пакета неверен.
+
+### Шаг `setup`: `Upstream outage (safe to rerun)`, код выхода 8
+
+`gh release download`, `gh release view` или `gh attestation verify` не достучались до
+GitHub или Sigstore за 4 попытки. Бинарник не проверялся — это не то же самое, что
+провал проверки. Перезапустите job. Настоящий отказ — `does not verify as built by
+owfeed/owlab's release workflow` с кодом выхода 1.
+
+---
+
 ## Три поведения апстрима, которые стоит знать
 
 ### `kmods` должен остаться в списке фидов

internal/compose/extra.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"owfeed.org/owlab/internal/config"
+	"owfeed.org/owlab/internal/netx"
 )
 
 // fetchExtraPackages downloads every router's out-of-feed packages into the
@@ -90,14 +91,15 @@ func Fetch(url, cacheDir string) (string, error) {
 	}
 
 	fmt.Fprintf(os.Stderr, "owlab: fetching %s\n", url)
-	client := &http.Client{Timeout: 5 * time.Minute}
+	// Retries a 5xx or a dropped connection; see internal/netx.
+	client := netx.Client(&http.Client{Timeout: 5 * time.Minute})
 	resp, err := client.Get(url)
 	if err != nil {
 		return "", err
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("GET %s: %s", url, resp.Status)
+		return "", netx.Status(url, resp)
 	}
 
 	// Written to a temporary name and renamed, so an interrupted download

internal/netx/netx.go

@@ -0,0 +1,179 @@
+// Package netx tells a download server that is failing from one that answered, and
+// retries the first.
+//
+// owlab fetches a rootfs, a VM image and its sha256sums, out-of-feed packages and the
+// release listings from servers it does not run. Before this package one 502 from a
+// mirror ended the command with the same kind of message as a release that does not
+// exist, and nothing was asked twice. Now a 5xx, a 429 or a dropped connection is
+// retried a few times, and what still fails says whether it was an outage (run again
+// later) or an answer (fix the release or URL).
+//
+// owfeed has a package of the same name and shape. It is a copy on purpose: the two
+// tools share no Go module (docs/ECOSYSTEM.md in owfeed).
+package netx
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"syscall"
+	"time"
+)
+
+// StatusError is a response that was not 200.
+type StatusError struct {
+	URL    string
+	Code   int
+	Status string
+}
+
+func (e *StatusError) Error() string {
+	msg := fmt.Sprintf("GET %s: %s", e.URL, e.Status)
+	if TransientStatus(e.Code) {
+		msg += " (the server is failing, not refusing: owlab retried; run the command again later)"
+	}
+	return msg
+}
+
+// Status builds the error for a response that was not 200.
+func Status(url string, resp *http.Response) error {
+	return &StatusError{URL: url, Code: resp.StatusCode, Status: resp.Status}
+}
+
+// TransientStatus reports whether an HTTP status is one a server returns while it is
+// failing rather than while it is answering: 408, 425, 429 and every 5xx.
+func TransientStatus(code int) bool {
+	return code == http.StatusRequestTimeout || code == http.StatusTooEarly ||
+		code == http.StatusTooManyRequests || code >= 500
+}
+
+// Transient reports whether err is an outage rather than an answer. Unknown errors
+// are not transient.
+func Transient(err error) bool {
+	if err == nil || errors.Is(err, context.Canceled) {
+		return false
+	}
+	var oe *OutageError
+	if errors.As(err, &oe) {
+		return true
+	}
+	var se *StatusError
+	if errors.As(err, &se) {
+		return TransientStatus(se.Code)
+	}
+	// A certificate that does not verify does not heal by asking again, and it
+	// arrives wrapped in the same *url.Error as the network cases below.
+	var certErr *tls.CertificateVerificationError
+	var unknownAuth x509.UnknownAuthorityError
+	var hostErr x509.HostnameError
+	var invalid x509.CertificateInvalidError
+	if errors.As(err, &certErr) || errors.As(err, &unknownAuth) ||
+		errors.As(err, &hostErr) || errors.As(err, &invalid) {
+		return false
+	}
+	if errors.Is(err, context.DeadlineExceeded) ||
+		errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) ||
+		errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNREFUSED) {
+		return true
+	}
+	var dnsErr *net.DNSError
+	var opErr *net.OpError
+	if errors.As(err, &dnsErr) || errors.As(err, &opErr) {
+		return true
+	}
+	var ne net.Error
+	return errors.As(err, &ne) && ne.Timeout()
+}
+
+// Four attempts, 2 s, 4 s and 8 s apart. Variables so tests keep the attempts and
+// drop the waiting.
+var (
+	DefaultAttempts = 4
+	DefaultDelay    = 2 * time.Second
+)
+
+// OutageError is what Transport returns when a transient transport failure outlasted
+// every attempt.
+type OutageError struct {
+	Attempts int
+	Err      error
+}
+
+func (e *OutageError) Error() string {
+	return fmt.Sprintf("unreachable after %d attempts (a network or server outage, not a problem with the config; run the command again later): %v", e.Attempts, e.Err)
+}
+func (e *OutageError) Unwrap() error { return e.Err }
+
+// Transport retries GET and HEAD requests whose failure is Transient, with a delay
+// that doubles. Requests with a body pass through untouched. Only the response
+// headers are covered: a body that breaks off halfway is the caller's read error.
+type Transport struct {
+	Base     http.RoundTripper
+	Attempts int
+	Delay    time.Duration
+}
+
+func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	base := t.Base
+	if base == nil {
+		base = http.DefaultTransport
+	}
+	if (req.Method != http.MethodGet && req.Method != http.MethodHead) ||
+		(req.Body != nil && req.Body != http.NoBody) {
+		return base.RoundTrip(req)
+	}
+	attempts, delay := t.Attempts, t.Delay
+	if attempts <= 0 {
+		attempts = DefaultAttempts
+	}
+	if delay <= 0 {
+		delay = DefaultDelay
+	}
+
+	for attempt := 1; ; attempt++ {
+		resp, err := base.RoundTrip(req)
+		retry := false
+		switch {
+		case err != nil:
+			retry = Transient(err)
+		case TransientStatus(resp.StatusCode):
+			retry = true
+		}
+		if !retry {
+			return resp, err
+		}
+		if attempt >= attempts {
+			if err != nil {
+				return nil, &OutageError{Attempts: attempt, Err: err}
+			}
+			return resp, nil
+		}
+		if resp != nil {
+			_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
+			resp.Body.Close()
+		}
+		timer := time.NewTimer(delay)
+		select {
+		case <-req.Context().Done():
+			timer.Stop()
+			return nil, req.Context().Err()
+		case <-timer.C:
+		}
+		delay *= 2
+	}
+}
+
+// Client returns a copy of hc whose transport retries outages. hc is not modified.
+func Client(hc *http.Client) *http.Client {
+	if hc == nil {
+		hc = http.DefaultClient
+	}
+	c := *hc
+	c.Transport = &Transport{Base: hc.Transport}
+	return &c
+}