From 5a3573a074ebb83eb76d2bdd45ef4ac7e91ca3f8 Mon Sep 17 00:00:00 2001 From: VizzleTF Date: Mon, 14 Sep 2026 15:03:53 +0300 Subject: [PATCH] fix: retry a failing download server and say it was an outage Release listings, the VM image and its sha256sums, and out-of-feed packages are retried on a 5xx, 429 or dropped connection; a 404 is asked once. A failure that outlasts the retries says it is an outage to wait out, not a problem with owlab.yaml. setup retries gh release reads while GitHub is failing and exits 8 when that outlasts 4 attempts; an attestation API or Sigstore that cannot be reached is reported as an outage, not as a binary that does not verify. --- CHANGELOG.md | 16 +++ docs/troubleshooting.md | 27 +++++ docs/troubleshooting_ru.md | 26 +++++ internal/compose/extra.go | 6 +- internal/netx/netx.go | 179 +++++++++++++++++++++++++++++ internal/netx/netx_test.go | 118 +++++++++++++++++++ internal/qemu/fetch_outage_test.go | 54 +++++++++ internal/qemu/image.go | 11 +- internal/upstream/upstream.go | 7 +- setup/install.sh | 83 +++++++++++-- 10 files changed, 509 insertions(+), 18 deletions(-) create mode 100644 internal/netx/netx.go create mode 100644 internal/netx/netx_test.go create mode 100644 internal/qemu/fetch_outage_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index caf3eb3..df10a79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ Versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html). today keeps working across minor and patch releases; a change that would break one waits for a major. +## [Unreleased] + +### Fixed + +- **A download server that fails for a moment no longer ends the command.** The + release listings, the VM image and its `sha256sums`, and out-of-feed packages + are asked again on a 5xx, 429 or dropped connection: 4 attempts, 2, 4 and 8 s + apart. A 404 is still asked once. When a failure outlasts the retries, the + message says it is an outage to wait out, not a problem with `owlab.yaml`. + +- **`setup` tells a GitHub outage from a bad release.** `gh release download` + and `gh release view` are retried while GitHub answers 5xx or cannot be + reached, and exit 8 when that outlasts 4 attempts. When `gh attestation verify` + cannot reach GitHub or Sigstore, the step retries and reports an outage + (exit 8) instead of "does not verify". + ## [0.6.4] - 2026-09-14 ### Fixed diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index cb44946..7614fc4 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -562,6 +562,33 @@ looking at is worth doing before debugging anything else. --- +## "A download failed" + +### `503 Service Unavailable (the server is failing, not refusing …)` + +The download server or GitHub was failing, and kept failing through owlab's +retries (4 attempts over about 14 seconds). Nothing is wrong with `owlab.yaml`. +Run the command again later. For GitHub, check +[githubstatus.com](https://www.githubstatus.com). + +The same applies to `unreachable after 4 attempts (a network or server outage …)`, +which covers timeouts, DNS failures and refused connections. + +### `404 Not Found` on an image, a rootfs or a package + +The server answered, and the file is not there. owlab asks for a 404 once. Check +the release number and target in `owlab.yaml`: the release may not publish that +image yet, or the out-of-feed package URL is wrong. + +### `setup` step: `Upstream outage (safe to rerun)`, exit code 8 + +`gh release download`, `gh release view` or `gh attestation verify` could not reach +GitHub or Sigstore after 4 attempts. The binary was not checked, which is different +from failing verification. Rerun the job. `does not verify as built by +owfeed/owlab's release workflow`, exit code 1, is the real refusal. + +--- + ## Three upstream behaviours worth knowing ### `kmods` must stay in the feed list diff --git a/docs/troubleshooting_ru.md b/docs/troubleshooting_ru.md index 21f528e..5ff26ac 100644 --- a/docs/troubleshooting_ru.md +++ b/docs/troubleshooting_ru.md @@ -561,6 +561,32 @@ VM не грузится или виснет на середине, тогда --- +## «Загрузка упала» + +### `503 Service Unavailable (the server is failing, not refusing …)` + +Сервер загрузок или GitHub сбоил и продолжал сбоить на протяжении всех повторов owlab +(4 попытки примерно за 14 секунд). С `owlab.yaml` всё в порядке. Запустите команду +позже. Для GitHub загляните на [githubstatus.com](https://www.githubstatus.com). + +То же относится к `unreachable after 4 attempts (a network or server outage …)`: так +выглядят таймауты, сбои DNS и отказы в соединении. + +### `404 Not Found` на образ, rootfs или пакет + +Сервер ответил, и файла там нет. На 404 owlab спрашивает один раз. Проверьте номер +релиза и target в `owlab.yaml`: релиз может ещё не публиковать такой образ, или URL +внефидового пакета неверен. + +### Шаг `setup`: `Upstream outage (safe to rerun)`, код выхода 8 + +`gh release download`, `gh release view` или `gh attestation verify` не достучались до +GitHub или Sigstore за 4 попытки. Бинарник не проверялся — это не то же самое, что +провал проверки. Перезапустите job. Настоящий отказ — `does not verify as built by +owfeed/owlab's release workflow` с кодом выхода 1. + +--- + ## Три поведения апстрима, которые стоит знать ### `kmods` должен остаться в списке фидов diff --git a/internal/compose/extra.go b/internal/compose/extra.go index 09518f6..61420e8 100644 --- a/internal/compose/extra.go +++ b/internal/compose/extra.go @@ -12,6 +12,7 @@ import ( "time" "owfeed.org/owlab/internal/config" + "owfeed.org/owlab/internal/netx" ) // fetchExtraPackages downloads every router's out-of-feed packages into the @@ -90,14 +91,15 @@ func Fetch(url, cacheDir string) (string, error) { } fmt.Fprintf(os.Stderr, "owlab: fetching %s\n", url) - client := &http.Client{Timeout: 5 * time.Minute} + // Retries a 5xx or a dropped connection; see internal/netx. + client := netx.Client(&http.Client{Timeout: 5 * time.Minute}) resp, err := client.Get(url) if err != nil { return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("GET %s: %s", url, resp.Status) + return "", netx.Status(url, resp) } // Written to a temporary name and renamed, so an interrupted download diff --git a/internal/netx/netx.go b/internal/netx/netx.go new file mode 100644 index 0000000..ac9a94b --- /dev/null +++ b/internal/netx/netx.go @@ -0,0 +1,179 @@ +// Package netx tells a download server that is failing from one that answered, and +// retries the first. +// +// owlab fetches a rootfs, a VM image and its sha256sums, out-of-feed packages and the +// release listings from servers it does not run. Before this package one 502 from a +// mirror ended the command with the same kind of message as a release that does not +// exist, and nothing was asked twice. Now a 5xx, a 429 or a dropped connection is +// retried a few times, and what still fails says whether it was an outage (run again +// later) or an answer (fix the release or URL). +// +// owfeed has a package of the same name and shape. It is a copy on purpose: the two +// tools share no Go module (docs/ECOSYSTEM.md in owfeed). +package netx + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "syscall" + "time" +) + +// StatusError is a response that was not 200. +type StatusError struct { + URL string + Code int + Status string +} + +func (e *StatusError) Error() string { + msg := fmt.Sprintf("GET %s: %s", e.URL, e.Status) + if TransientStatus(e.Code) { + msg += " (the server is failing, not refusing: owlab retried; run the command again later)" + } + return msg +} + +// Status builds the error for a response that was not 200. +func Status(url string, resp *http.Response) error { + return &StatusError{URL: url, Code: resp.StatusCode, Status: resp.Status} +} + +// TransientStatus reports whether an HTTP status is one a server returns while it is +// failing rather than while it is answering: 408, 425, 429 and every 5xx. +func TransientStatus(code int) bool { + return code == http.StatusRequestTimeout || code == http.StatusTooEarly || + code == http.StatusTooManyRequests || code >= 500 +} + +// Transient reports whether err is an outage rather than an answer. Unknown errors +// are not transient. +func Transient(err error) bool { + if err == nil || errors.Is(err, context.Canceled) { + return false + } + var oe *OutageError + if errors.As(err, &oe) { + return true + } + var se *StatusError + if errors.As(err, &se) { + return TransientStatus(se.Code) + } + // A certificate that does not verify does not heal by asking again, and it + // arrives wrapped in the same *url.Error as the network cases below. + var certErr *tls.CertificateVerificationError + var unknownAuth x509.UnknownAuthorityError + var hostErr x509.HostnameError + var invalid x509.CertificateInvalidError + if errors.As(err, &certErr) || errors.As(err, &unknownAuth) || + errors.As(err, &hostErr) || errors.As(err, &invalid) { + return false + } + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) || + errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNREFUSED) { + return true + } + var dnsErr *net.DNSError + var opErr *net.OpError + if errors.As(err, &dnsErr) || errors.As(err, &opErr) { + return true + } + var ne net.Error + return errors.As(err, &ne) && ne.Timeout() +} + +// Four attempts, 2 s, 4 s and 8 s apart. Variables so tests keep the attempts and +// drop the waiting. +var ( + DefaultAttempts = 4 + DefaultDelay = 2 * time.Second +) + +// OutageError is what Transport returns when a transient transport failure outlasted +// every attempt. +type OutageError struct { + Attempts int + Err error +} + +func (e *OutageError) Error() string { + return fmt.Sprintf("unreachable after %d attempts (a network or server outage, not a problem with the config; run the command again later): %v", e.Attempts, e.Err) +} +func (e *OutageError) Unwrap() error { return e.Err } + +// Transport retries GET and HEAD requests whose failure is Transient, with a delay +// that doubles. Requests with a body pass through untouched. Only the response +// headers are covered: a body that breaks off halfway is the caller's read error. +type Transport struct { + Base http.RoundTripper + Attempts int + Delay time.Duration +} + +func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) { + base := t.Base + if base == nil { + base = http.DefaultTransport + } + if (req.Method != http.MethodGet && req.Method != http.MethodHead) || + (req.Body != nil && req.Body != http.NoBody) { + return base.RoundTrip(req) + } + attempts, delay := t.Attempts, t.Delay + if attempts <= 0 { + attempts = DefaultAttempts + } + if delay <= 0 { + delay = DefaultDelay + } + + for attempt := 1; ; attempt++ { + resp, err := base.RoundTrip(req) + retry := false + switch { + case err != nil: + retry = Transient(err) + case TransientStatus(resp.StatusCode): + retry = true + } + if !retry { + return resp, err + } + if attempt >= attempts { + if err != nil { + return nil, &OutageError{Attempts: attempt, Err: err} + } + return resp, nil + } + if resp != nil { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + resp.Body.Close() + } + timer := time.NewTimer(delay) + select { + case <-req.Context().Done(): + timer.Stop() + return nil, req.Context().Err() + case <-timer.C: + } + delay *= 2 + } +} + +// Client returns a copy of hc whose transport retries outages. hc is not modified. +func Client(hc *http.Client) *http.Client { + if hc == nil { + hc = http.DefaultClient + } + c := *hc + c.Transport = &Transport{Base: hc.Transport} + return &c +} diff --git a/internal/netx/netx_test.go b/internal/netx/netx_test.go new file mode 100644 index 0000000..8eab977 --- /dev/null +++ b/internal/netx/netx_test.go @@ -0,0 +1,118 @@ +package netx + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" +) + +func TestTransient(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"404", &StatusError{Code: 404}, false}, + {"502", fmt.Errorf("sums: %w", &StatusError{Code: 502}), true}, + {"429", &StatusError{Code: 429}, true}, + {"dns", &url.Error{Op: "Get", URL: "https://x", Err: &net.DNSError{Err: "no such host", Name: "x"}}, true}, + {"unexpected EOF", fmt.Errorf("copy: %w", io.ErrUnexpectedEOF), true}, + {"canceled", context.Canceled, false}, + {"unknown authority", &url.Error{Op: "Get", URL: "https://x", Err: x509.UnknownAuthorityError{}}, false}, + {"checksum mismatch", errors.New("checksum mismatch for x"), false}, + {"transport outage", &OutageError{Attempts: 4, Err: errors.New("refused")}, true}, + } + for _, c := range cases { + if got := Transient(c.err); got != c.want { + t.Errorf("%s: Transient(%v) = %v, want %v", c.name, c.err, got, c.want) + } + } +} + +func server(t *testing.T, codes ...int) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := int(hits.Add(1)) + code := codes[len(codes)-1] + if n <= len(codes) { + code = codes[n-1] + } + w.WriteHeader(code) + })) + t.Cleanup(s.Close) + return s, &hits +} + +func client() *http.Client { return &http.Client{Transport: &Transport{Delay: time.Millisecond}} } + +func TestTransportRetriesUntilTheServerAnswers(t *testing.T) { + s, hits := server(t, 502, 503, 200) + resp, err := client().Get(s.URL) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != 200 || hits.Load() != 3 { + t.Fatalf("status %d after %d requests, want 200 after 3", resp.StatusCode, hits.Load()) + } +} + +func TestTransportAsksForA404Once(t *testing.T) { + s, hits := server(t, 404) + resp, err := client().Get(s.URL) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != 404 || hits.Load() != 1 { + t.Fatalf("status %d after %d requests, want 404 after 1", resp.StatusCode, hits.Load()) + } +} + +func TestTransportSaysAPersistentFailureIsAnOutage(t *testing.T) { + s, hits := server(t, 503) + resp, err := client().Get(s.URL) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if int(hits.Load()) != DefaultAttempts { + t.Fatalf("%d requests, want %d", hits.Load(), DefaultAttempts) + } + if msg := Status(s.URL, resp).Error(); !strings.Contains(msg, "run the command again later") { + t.Fatalf("a persistent 503 should say it is an outage: %s", msg) + } + + closed := httptest.NewServer(http.NotFoundHandler()) + addr := closed.URL + closed.Close() + _, err = client().Get(addr) + var oe *OutageError + if !errors.As(err, &oe) || !strings.Contains(err.Error(), "not a problem with the config") { + t.Fatalf("an unreachable host should be an OutageError that says so, got %v", err) + } +} + +func TestTransportDoesNotReplayAWrite(t *testing.T) { + s, hits := server(t, 503) + resp, err := client().Post(s.URL, "text/plain", strings.NewReader("x")) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if hits.Load() != 1 { + t.Fatalf("POST sent %d times, want 1", hits.Load()) + } +} diff --git a/internal/qemu/fetch_outage_test.go b/internal/qemu/fetch_outage_test.go new file mode 100644 index 0000000..0b48fd7 --- /dev/null +++ b/internal/qemu/fetch_outage_test.go @@ -0,0 +1,54 @@ +package qemu + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "owfeed.org/owlab/internal/netx" +) + +// A mirror that answers 502 once and then serves the image. Before netx the first 502 +// ended the download; now it is asked again, and a 404 still is not. +func TestFetchToRetriesAFailingMirrorButNotAMissingImage(t *testing.T) { + old := netx.DefaultDelay + netx.DefaultDelay = time.Millisecond + defer func() { netx.DefaultDelay = old }() + + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := hits.Add(1) + switch { + case strings.HasSuffix(r.URL.Path, "/missing.img.gz"): + http.NotFound(w, r) + case n == 1: + w.WriteHeader(http.StatusBadGateway) + default: + _, _ = w.Write([]byte("image")) + } + })) + defer srv.Close() + + dest := filepath.Join(t.TempDir(), "img.gz") + if _, err := fetchTo(context.Background(), srv.URL+"/openwrt.img.gz", dest); err != nil { + t.Fatalf("a single 502 should be retried: %v", err) + } + if hits.Load() != 2 { + t.Fatalf("%d requests, want 2", hits.Load()) + } + + hits.Store(10) // past the 502 + before := hits.Load() + _, err := fetchTo(context.Background(), srv.URL+"/missing.img.gz", dest) + if err == nil || !strings.Contains(err.Error(), "not found") { + t.Fatalf("want the not-found message, got %v", err) + } + if hits.Load()-before != 1 { + t.Fatalf("a 404 was asked %d times, want 1", hits.Load()-before) + } +} diff --git a/internal/qemu/image.go b/internal/qemu/image.go index 05a40ac..1c1502e 100644 --- a/internal/qemu/image.go +++ b/internal/qemu/image.go @@ -14,6 +14,7 @@ import ( "time" "owfeed.org/owlab/internal/config" + "owfeed.org/owlab/internal/netx" ) // CacheDir is where downloaded disk images live. @@ -97,13 +98,13 @@ func expectedSum(ctx context.Context, r *config.Router) (string, error) { if err != nil { return "", err } - resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + resp, err := netx.Client(&http.Client{Timeout: 30 * time.Second}).Do(req) if err != nil { return "", err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return "", fmt.Errorf("%s: %s", r.SumsURL(), resp.Status) + return "", netx.Status(r.SumsURL(), resp) } body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) if err != nil { @@ -129,7 +130,9 @@ func fetchTo(ctx context.Context, url, dest string) (string, error) { if err != nil { return "", err } - resp, err := (&http.Client{Timeout: 30 * time.Minute}).Do(req) + // Retries a 5xx or a dropped connection before the download starts; a mirror's + // 502 used to end `owlab up` on the first try. + resp, err := netx.Client(&http.Client{Timeout: 30 * time.Minute}).Do(req) if err != nil { return "", err } @@ -138,7 +141,7 @@ func fetchTo(ctx context.Context, url, dest string) (string, error) { if resp.StatusCode == http.StatusNotFound { return "", fmt.Errorf("%s: not found.\n\nThis usually means the release or the target does not publish this image — check the release number in %s", url, config.FileName) } - return "", fmt.Errorf("%s: %s", url, resp.Status) + return "", netx.Status(url, resp) } f, err := os.Create(dest) diff --git a/internal/upstream/upstream.go b/internal/upstream/upstream.go index e6f45ea..caaf1c4 100644 --- a/internal/upstream/upstream.go +++ b/internal/upstream/upstream.go @@ -21,6 +21,7 @@ import ( "time" "owfeed.org/owlab/internal/config" + "owfeed.org/owlab/internal/netx" ) // pointRelease matches the href of a release directory: `href="25.12.4/"`. @@ -208,13 +209,13 @@ func get(ctx context.Context, url string) ([]byte, error) { if err != nil { return nil, err } - resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + resp, err := netx.Client(&http.Client{Timeout: 30 * time.Second}).Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("GET %s: %s", url, resp.Status) + return nil, netx.Status(url, resp) } return io.ReadAll(io.LimitReader(resp.Body, 8<<20)) } @@ -224,7 +225,7 @@ func head(ctx context.Context, url string) bool { if err != nil { return false } - resp, err := (&http.Client{Timeout: 30 * time.Second}).Do(req) + resp, err := netx.Client(&http.Client{Timeout: 30 * time.Second}).Do(req) if err != nil { return false } diff --git a/setup/install.sh b/setup/install.sh index cf7b2af..a7c7864 100755 --- a/setup/install.sh +++ b/setup/install.sh @@ -69,11 +69,56 @@ if [ -z "${OWLAB_VERSION:-}" ]; then esac fi +# gh_outage +# +# gh exits 1 for every failure, so the text decides whether GitHub was failing or +# answering. Measured with gh 2.99.0: a 5xx prints `HTTP 503: ()` or +# `(HTTP 503)`, an unreachable host `error connecting to check your internet +# connection`, an unreachable proxy `proxyconnect tcp: ... connection refused`, and +# `gh attestation verify` with no Sigstore trust root reachable `error creating +# Sigstore verifier: no valid Sigstore verifiers could be initialized`. An answer +# prints `release not found`, `no assets match the file pattern` or `HTTP 404`. +gh_outage() { + grep -Eqi 'HTTP (408|429|5[0-9][0-9])|status code: (408|429|5[0-9][0-9])|error connecting to|connection (refused|reset)|i/o timeout|TLS handshake timeout|timeout awaiting|no such host|unexpected EOF|server misbehaving|proxyconnect|rate limit|error creating Sigstore verifier' <<<"$1" +} + +# gh_read +# +# A read from GitHub, asked again while GitHub is failing: 4 attempts, 5, 10 and 20 s +# apart. Exit 8 when the outage outlasts them -- owfeed's code for "upstream outage, +# safe to rerun" -- and 7 at once for an answer. Before this a single 502 from the +# release API failed the step with gh's exit 1, the same as a tag that does not exist. +gh_read() { + local attempt=1 delay=5 err + err="$(mktemp)" + while :; do + if gh "$@" 2>"$err"; then + rm -f "$err" + return 0 + fi + if ! gh_outage "$(cat "$err")"; then + cat "$err" >&2 + rm -f "$err" + return 7 + fi + if [ "$attempt" -ge 4 ]; then + cat "$err" >&2 + rm -f "$err" + echo "::error title=Upstream outage (safe to rerun)::gh $1 $2 could not reach GitHub after $attempt attempts. Nothing was installed and nothing failed verification; rerun the job." + return 8 + fi + echo "gh $1 $2: GitHub did not answer (attempt $attempt of 4), retrying in ${delay}s: $(tr '\n' ' ' <"$err")" >&2 + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + done +} + # The version is needed before the download, not after: the asset name carries # it, so "latest" has to be resolved to a tag rather than guessed at. if [ "$OWLAB_VERSION" = "latest" ]; then echo "::warning::owlab pinned to \"latest\"; pin a tag so a CI result cannot change without a commit" - tag="$(gh release view --repo owfeed/owlab --json tagName --jq .tagName)" + tag="$(gh_read release view --repo owfeed/owlab --json tagName --jq .tagName)" else tag="$OWLAB_VERSION" fi @@ -99,7 +144,7 @@ fi # --clobber, because a job may have downloaded a DIFFERENT version into the same # directory: the short-circuit above returns only on an exact match, so reaching # here with the file present means it is the wrong one and has to be replaced. -gh release download "$tag" --repo owfeed/owlab --pattern "$asset" --dir "$dir" --clobber +gh_read release download "$tag" --repo owfeed/owlab --pattern "$asset" --dir "$dir" --clobber # Verify BEFORE the archive is unpacked or anything in it is executed. A check # that runs after the thing it checks is not a check. @@ -112,18 +157,38 @@ if [ "${OWLAB_VERIFY:-true}" = "true" ]; then # gh writes its result to stderr, which a composite step swallows, so it is # captured and echoed either way: a check nobody can see ran is one nobody # believes ran. - if out=$(gh attestation verify "$dir/$asset" \ - --repo owfeed/owlab \ - --signer-workflow owfeed/owlab/.github/workflows/release.yml 2>&1); then - echo "$out" - echo "verified $asset as built by owfeed/owlab .github/workflows/release.yml" - else + # + # Asked again when GitHub's attestation API or Sigstore is what failed: reporting + # that as "does not verify" told people the binary was suspect when the check had + # never run, and pushed them towards verify: false. + attempt=1 + delay=5 + while :; do + if out=$(gh attestation verify "$dir/$asset" \ + --repo owfeed/owlab \ + --signer-workflow owfeed/owlab/.github/workflows/release.yml 2>&1); then + echo "$out" + echo "verified $asset as built by owfeed/owlab .github/workflows/release.yml" + break + fi echo "$out" + if gh_outage "$out"; then + if [ "$attempt" -lt 4 ]; then + echo "the attestation check could not reach GitHub or Sigstore (attempt $attempt of 4); retrying in ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + continue + fi + echo "::error title=Upstream outage (safe to rerun)::the attestation check could not reach GitHub or Sigstore after $attempt attempts. The binary was not found to be wrong; it was not checked. Rerun the job." + rm -f "$dir/$asset" + exit 8 + fi echo "::error::$asset does not verify as built by owfeed/owlab's release workflow — refusing to install it" echo "::error::releases before v0.2.0 carry no attestation; pin a later tag, or set verify: false to accept an unverified download" rm -f "$dir/$asset" exit 1 - fi + done else echo "::warning::owlab installed without verifying its build attestation" fi