fix(tools): tell an upstream outage from a failed check

.github/workflows/intake.yml

@@ -40,13 +40,23 @@ jobs:
         run: |
           set -eu
           printf '%s' "$BODY" > "$RUNNER_TEMP/body.md"
-          ./tools/intake-check.sh "$RUNNER_TEMP/body.md" > "$RUNNER_TEMP/verdict.md" || {
+          rc=0
+          ./tools/intake-check.sh "$RUNNER_TEMP/body.md" > "$RUNNER_TEMP/verdict.md" || rc=$?
+          # 8 is an upstream outage (tools/net.sh): nothing was decided, and the
+          # requester has nothing to fix. Anything else non-zero is a bug here.
+          if [ "$rc" -eq 8 ]; then
+            {
+              echo "## Automated intake check"
+              echo
+              echo "GitHub or the manifest's host did not answer while this ran, so nothing was checked yet. You do not need to change anything; editing the issue or a maintainer's rerun checks it again."
+            } > "$RUNNER_TEMP/verdict.md"
+          elif [ "$rc" -ne 0 ]; then
             {
               echo "## Automated intake check"
               echo
               echo "The check itself failed to run. That is a bug here, not in your request."
             } > "$RUNNER_TEMP/verdict.md"
-          }
+          fi
           cat "$RUNNER_TEMP/verdict.md"
           gh issue comment "${{ github.event.issue.number }}" \
             --repo "${{ github.repository }}" \

.github/workflows/pr.yml

@@ -96,7 +96,7 @@ jobs:
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Self-test the scripts under tools/
-        run: sh tools/test-sources.sh && sh tools/test-land-updates.sh && sh tools/test-check-updates.sh
+        run: sh tools/test-sources.sh && sh tools/test-land-updates.sh && sh tools/test-check-updates.sh && sh tools/test-net.sh
 
   check:
     needs: tools

RUNBOOK.md

@@ -134,8 +134,19 @@ nothing reported.
 container output: usually a dependency that does not resolve on a stock image, or a file installed
 somewhere nothing looks.
 
-**`sha256 … pinned …`** in the fetch step. The upstream replaced a release in place. Do not update
-the pin to make it pass — find out why the bytes changed first.
+**`!! upstream outage: …` and exit code 8** in the fetch step. GitHub or another upstream answered
+5xx, 429, timed out, or could not be reached, and kept doing so for five attempts over about 75
+seconds. Nothing about the package was found wrong. Rerun the job once
+[githubstatus.com](https://www.githubstatus.com) is green. The same message comes from
+`check-updates.sh`, `land-updates.sh` and the intake job, all through `tools/net.sh`. Before this
+change the symptom was `curl: (22) The requested URL returned error: 500` repeated six times and
+`exit code 22` — also an outage, also a rerun.
+
+**`curl: (22) … 404` and exit code 7** in the fetch step. The asset is not in the release: the tag
+or the file name in `upstream.sh` is wrong, or upstream deleted it. A rerun gets the same answer.
+
+**`sha256 … pinned …`** in the fetch step, exit code 7. The upstream replaced a release in place. Do
+not update the pin to make it pass — find out why the bytes changed first.
 
 **`NO ORIGIN …`** from `tools/check-origin.sh`, after the index is built. A package reached the tree
 without saying where it comes from, and this feed does not publish it. Nothing here fixes that: the
@@ -145,8 +156,8 @@ somewhere a user can go.
 
 **`tools` failed.** A self-test of a script under `tools/` is red, and `check` did not start. The log
 prints `FAIL` beside the broken case. Run the tests locally before you push a change to `tools/`:
-`sh tools/test-sources.sh`, `sh tools/test-land-updates.sh`, `sh tools/test-check-updates.sh`. They
-need `git` and `jq` and reach no network.
+`sh tools/test-sources.sh`, `sh tools/test-land-updates.sh`, `sh tools/test-check-updates.sh`,
+`sh tools/test-net.sh`. They need `git` and `jq` and reach no network.
 
 ---
 

RUNBOOK_ru.md

@@ -134,8 +134,19 @@ MISSING luci-theme-footstrap luci-theme-footstrap_0.11.6-r1_all.ipk on 24.10: ab
 контейнера: обычно это зависимость, которая не резолвится на стоковом образе, или файл, положенный
 туда, куда никто не смотрит.
 
-**`sha256 … pinned …`** на шаге fetch. Апстрим подменил релиз на месте. Не правьте пин, чтобы
-прошло, — сначала выясните, почему изменились байты.
+**`!! upstream outage: …` и код выхода 8** на шаге fetch. GitHub или другой апстрим отвечал 5xx,
+429, не отвечал вовсе или не резолвился — и так пять попыток подряд, около 75 секунд. С пакетом
+ничего не нашлось. Перезапустите job, когда [githubstatus.com](https://www.githubstatus.com)
+позеленеет. То же сообщение печатают `check-updates.sh`, `land-updates.sh` и intake — все ходят
+через `tools/net.sh`. До этого изменения симптом выглядел как шесть строк
+`curl: (22) The requested URL returned error: 500` и `exit code 22` — тоже сбой апстрима, тоже
+перезапуск.
+
+**`curl: (22) … 404` и код выхода 7** на шаге fetch. Файла нет в релизе: в `upstream.sh` неверный
+тег или имя файла, или апстрим его удалил. Перезапуск даст тот же ответ.
+
+**`sha256 … pinned …`** на шаге fetch, код выхода 7. Апстрим подменил релиз на месте. Не правьте
+пин, чтобы прошло, — сначала выясните, почему изменились байты.
 
 **`NO ORIGIN …`** от `tools/check-origin.sh`, после сборки индекса. Пакет дошёл до дерева, не сказав,
 откуда он, и этот фид его не публикует. Здесь это не чинится: поле ставится там, где пакет
@@ -144,8 +155,8 @@ MISSING luci-theme-footstrap luci-theme-footstrap_0.11.6-r1_all.ipk on 24.10: ab
 
 **Упал `tools`.** Покраснел самотест одного из скриптов в `tools/`, и `check` не запустился. В логе
 рядом со сломанным случаем стоит `FAIL`. Прогоните тесты у себя, прежде чем пушить правку в `tools/`:
-`sh tools/test-sources.sh`, `sh tools/test-land-updates.sh`, `sh tools/test-check-updates.sh`. Им
-нужны `git` и `jq`, в сеть они не ходят.
+`sh tools/test-sources.sh`, `sh tools/test-land-updates.sh`, `sh tools/test-check-updates.sh`,
+`sh tools/test-net.sh`. Им нужны `git` и `jq`, в сеть они не ходят.
 
 ---
 

tools/check-updates.sh

@@ -48,6 +48,12 @@ set -eu
 # can guess a different repository than the one this checkout came from.
 SELF="${GITHUB_REPOSITORY:-$(gh repo view --json nameWithOwner -q .nameWithOwner)}"
 
+# Every read from an upstream release goes through tools/net.sh: it retries an outage
+# and exits 8 when one outlasts it, so "GitHub answered 502" never reads as "this
+# release is broken". Resolved beside this file, not from the working directory,
+# because the tests run this script from inside a repository with no tools/ in it.
+NET="$(cd "$(dirname "$0")" && pwd)/net.sh"
+
 # may_automerge <upstream.sh> <current version> <latest version> <package name>
 #
 # A signature says the author published these bytes. It does not say the release is
@@ -196,6 +202,7 @@ BASE="$(git rev-parse --abbrev-ref HEAD)"
 # Packages whose check stopped, reported together once the loop is done. See the end
 # of the loop for why one of them must not end it.
 failed=""
+outages=""
 
 for up in packages/*/upstream.sh; do
 	dir="$(dirname "$up")"
@@ -244,18 +251,31 @@ for up in packages/*/upstream.sh; do
 		# fails, the release list; a renamed or deleted upstream is a REPO to fix,
 		# not a quiet hour. Every other failure stops this package through the
 		# handler after the subshell.
-		if ! latest_tag="$(gh release view --repo "$REPO" --json tagName -q .tagName 2>"$tmp/view.err")"; then
-			if [ "$(cat "$tmp/view.err")" = "release not found" ] &&
-				gh api "repos/$REPO/releases?per_page=1" -q length >/dev/null 2>"$tmp/list.err"; then
-				echo "$name: upstream has no releases"
-				exit 0
+		#
+		# tools/net.sh passes gh's stderr through verbatim when the answer is definite,
+		# so the `release not found` comparison still sees exactly what gh printed.
+		rc=0
+		latest_tag="$("$NET" gh release view --repo "$REPO" --json tagName -q .tagName 2>"$tmp/view.err")" || rc=$?
+		if [ "$rc" -ne 0 ]; then
+			if [ "$rc" -ne 8 ] && [ "$(cat "$tmp/view.err")" = "release not found" ]; then
+				rc=0
+				"$NET" gh api "repos/$REPO/releases?per_page=1" -q length >/dev/null 2>"$tmp/list.err" || rc=$?
+				if [ "$rc" -eq 0 ]; then
+					echo "$name: upstream has no releases"
+					exit 0
+				fi
 			fi
 			{
 				echo "$name: could not read the latest release of $REPO"
 				cat "$tmp/view.err" "$tmp/list.err" 2>/dev/null | sed 's/^/  /'
 				echo "  failed: gh release view --repo $REPO --json tagName, then gh api repos/$REPO/releases"
-				echo "  a renamed or deleted repository needs REPO fixed in $up; an outage clears on a later run"
+				if [ "$rc" -eq 8 ]; then
+					echo "  GitHub did not answer after retries: an outage, not a finding; a later run checks it again"
+				else
+					echo "  a renamed or deleted repository needs REPO fixed in $up"
+				fi
 			} >&2
+			[ "$rc" -ne 8 ] || exit 8
 			exit 1
 		fi
 		[ -n "$latest_tag" ] || { echo "$name: gh release view answered an empty tag for $REPO" >&2; exit 1; }
@@ -326,7 +346,7 @@ for up in packages/*/upstream.sh; do
 		# none of them would be pure waste.
 		pattern='*'
 		[ "$KIND" = "manifest" ] && pattern='manifest.txt'
-		gh release download "$latest_tag" --repo "$REPO" --dir "$tmp" --pattern "$pattern" >/dev/null
+		"$NET" gh release download "$latest_tag" --repo "$REPO" --dir "$tmp" --pattern "$pattern" >/dev/null
 
 		# Recompute the pins from the bytes the release actually served, rewriting
 		# values in place. Nothing but data changes, so the diff is readable.
@@ -518,13 +538,28 @@ can be merged.")"; then
 		# both back: the branch and every tracked file.
 		echo "$name: stopped; the remaining packages are still checked" >&2
 		git checkout -q -f "$BASE"
-		failed="$failed $name"
+		# Exit 8 is tools/net.sh saying GitHub did not answer. Kept apart so the end
+		# of the run can say which of the two it was.
+		if [ "$stopped" -eq 8 ]; then
+			outages="$outages $name"
+		else
+			failed="$failed $name"
+		fi
 	fi
 done
 
 # Still red when anything stopped. Continuing past a failure is about checking the
 # rest, not about hiding this one: a check that could not run counts as failed.
+#
+# Exit 8 only when an outage is the whole story. One real failure beside it makes the
+# run a 1, because exit 8 tells whoever reads it that a rerun is all it needs.
 if [ -n "$failed" ]; then
 	echo "check stopped for:$failed" >&2
+	[ -z "$outages" ] || echo "and GitHub did not answer for:$outages" >&2
 	exit 1
 fi
+if [ -n "$outages" ]; then
+	echo "check stopped because GitHub did not answer for:$outages" >&2
+	echo "  an upstream outage, not a finding; the next scheduled run checks them again" >&2
+	exit 8
+fi

tools/fetch.sh

@@ -7,6 +7,11 @@
 # start with copying somebody else's shell.
 #
 # Usage: tools/fetch.sh packages/<name>
+#
+# Exit 8: upstream outage that outlasted the retries in tools/net.sh -- rerun the job.
+# Exit 7: a check failed -- a 404 of a pinned asset, a checksum, size or signature
+# that does not match. Never rerun that one to make it pass. Anything else: a bug here
+# or in upstream.sh.
 set -eu
 
 DIR="${1:?usage: tools/fetch.sh packages/<name>}"
@@ -31,21 +36,25 @@ base="https://github.com/${REPO}/releases/download/${TAG}"
 
 # get <url> <dest>
 #
-# --retry, because a release of ninety-odd assets meets a 502 from the CDN sooner or
-# later and a whole publish failing on one is noise, not a finding. Retrying is safe
-# here for the reason it usually is not: every byte fetched is checked against a hash
-# or a signature afterwards, so a retry cannot smuggle anything past.
+# Through tools/net.sh, which retries an outage in place and then exits 8, and exits 7
+# at once for a definite answer such as a 404. A release of ninety-odd assets meets a
+# 502 from the CDN sooner or later, and a whole publish failing on one is noise, not a
+# finding. Retrying is safe here for the reason it usually is not: every byte fetched
+# is checked against a hash or a signature afterwards, so a retry cannot smuggle
+# anything past.
 get() {
-	curl -fsSL --proto '=https' --tlsv1.2 \
-		--retry 5 --retry-delay 2 --retry-connrefused --retry-all-errors \
-		-o "$2" "$1"
+	"$ROOT/tools/net.sh" get "$1" "$2"
 }
 
 # download <url> <dest> <sha256>
+#
+# A mismatch is exit 7, a failed check: the bytes arrived and are not the pinned ones,
+# and no rerun changes that. `|| exit $?` rather than trusting errexit, so the 8 of an
+# outage reaches the job unchanged from inside the `while read` loops below too.
 download() {
-	get "$1" "$2"
+	get "$1" "$2" || exit $?
 	got="$(sha256sum "$2" | cut -d' ' -f1)"
-	[ "$got" = "$3" ] || { echo "$1: sha256 $got, pinned $3" >&2; rm -f "$2"; exit 1; }
+	[ "$got" = "$3" ] || { echo "$1: sha256 $got, pinned $3" >&2; rm -f "$2"; exit 7; }
 }
 
 # fetch_source
@@ -137,7 +146,14 @@ fetch_source() {
 		# known until the index is built -- so tools/sources.sh is what refuses,
 		# per package, once it can read one. Failing here instead would take down
 		# the publish of a permissive package whose upstream simply moved a tag.
-		if ! get "$url" "$dest"; then
+		#
+		# An outage is not that answer. Reading a 503 as "no source archive" would
+		# record the package as sourceless and, for a copyleft one, turn a GitHub
+		# incident into a refusal naming the package. So exit 8 goes straight out.
+		rc=0
+		get "$url" "$dest" || rc=$?
+		[ "$rc" -ne 8 ] || exit 8
+		if [ "$rc" -ne 0 ]; then
 			echo "!! $NAME: no source archive at $url" >&2
 			echo "   set SOURCE_URL in packages/$NAME/upstream.sh if the source lives elsewhere;" >&2
 			echo "   if this package is copyleft the publish will refuse it later, by name" >&2
@@ -352,7 +368,7 @@ manifest)
 		download "$base/$file" "$dest/$out" "$sum"
 
 		got_size="$(wc -c < "$dest/$out" | tr -d ' ')"
-		[ "$got_size" = "$size" ] || { echo "$file: $got_size bytes, manifest says $size" >&2; exit 1; }
+		[ "$got_size" = "$size" ] || { echo "$file: $got_size bytes, manifest says $size" >&2; exit 7; }
 
 		# The manifest's signature already covers this file's hash, so a detached
 		# signature beside it adds nothing here -- it exists for consumers that know

tools/intake-check.sh

@@ -15,7 +15,12 @@
 # Usage: tools/intake-check.sh <issue-body-file> > verdict.md
 set -eu
 
+#
+# Exit 8: GitHub or the manifest's host did not answer, so nothing was decided and
+# intake.yml says that instead of posting a verdict. A 503 read as "no such release"
+# would send a requester looking for a mistake they did not make.
 BODY="${1:?usage: tools/intake-check.sh <issue-body-file>}"
+NET="$(cd "$(dirname "$0")" && pwd)/net.sh"
 WORK="$(mktemp -d)"
 trap 'rm -rf "$WORK"' EXIT
 
@@ -56,7 +61,10 @@ say "\`$REPO\` @ \`$TAG\` — declared shape: \`$KIND\`"
 say ""
 
 # The release has to exist before anything else is worth checking.
-if ! gh release view "$TAG" --repo "$REPO" --json tagName >/dev/null 2>&1; then
+rc=0
+"$NET" gh release view "$TAG" --repo "$REPO" --json tagName >/dev/null 2>&1 || rc=$?
+[ "$rc" -ne 8 ] || exit 8
+if [ "$rc" -ne 0 ]; then
 	bad "No release \`$TAG\` in \`$REPO\`, or the repository is private."
 	say ""
 	say "Nothing else could be checked."
@@ -72,8 +80,13 @@ manifest)
 		bad "The manifest shape needs a manifest URL, and none was given."
 	else
 		printf '%s\n' "$PUBKEY" > "$WORK/claimed.pub"
-		if ! curl -fsSL --max-time 60 "$MF_URL" -o "$WORK/manifest.txt" ||
-		   ! curl -fsSL --max-time 60 "$MF_URL.sig" -o "$WORK/manifest.txt.sig"; then
+		# 60 s per attempt, as before: the URL is the requester's choice, and a
+		# server that trickles bytes must not hold the job for hours.
+		rc=0
+		NET_MAX_TIME=60 "$NET" get "$MF_URL" "$WORK/manifest.txt" || rc=$?
+		[ "$rc" -ne 0 ] || NET_MAX_TIME=60 "$NET" get "$MF_URL.sig" "$WORK/manifest.txt.sig" || rc=$?
+		[ "$rc" -ne 8 ] || exit 8
+		if [ "$rc" -ne 0 ]; then
 			bad "Could not fetch the manifest and its \`.sig\` from that URL."
 		else
 			# VERIFY BEFORE READING, exactly as ingest does: every value inside