From 6368535088bf1fb12ecb7056ad87a8d780cbce36 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Wed, 28 Oct 2026 20:00:12 -0400 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(ci):=20retry=20go=20mod=20do?= =?UTF-8?q?wnload=20in=20Go=20workflow=20jobs=20on=20transient=20proxy=20r?= =?UTF-8?q?esets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a reusable composite action (.github/actions/go-mod-download-retry) that runs `go mod download` with up to 5 attempts and exponential backoff, and call it right after the Go cache guard in every Go job of v2-ci, v2-tests, go-security-analysis and testutil-guard. The first module-fetching step (golangci-lint, govulncheck, gosec, go build, the sharded go test slices, the coverage scorer's go list) then reads a warm GOMODCACHE instead of racing proxy.golang.org, so a single "stream error: ... INTERNAL_ERROR; received from peer" no longer fails the job. Completes the Docker builder-stage retry from the earlier fix for the same tracker; job semantics are otherwise unchanged. Signed-off-by: Andrew Anderson --- .../actions/go-mod-download-retry/action.yml | 51 +++++++++++++++++++ .github/workflows/go-security-analysis.yml | 20 ++++++++ .github/workflows/testutil-guard.yml | 5 ++ .github/workflows/v2-ci.yml | 5 ++ .github/workflows/v2-tests.yml | 46 +++++++++++++++++ .../fixed-14976-ci-go-mod-download-retry.md | 1 + 6 files changed, 128 insertions(+) create mode 100644 .github/actions/go-mod-download-retry/action.yml create mode 100644 changelog.d/fixed-14976-ci-go-mod-download-retry.md diff --git a/.github/actions/go-mod-download-retry/action.yml b/.github/actions/go-mod-download-retry/action.yml new file mode 100644 index 0000000000..ded93fb7ea --- /dev/null +++ b/.github/actions/go-mod-download-retry/action.yml @@ -0,0 +1,51 @@ +name: Download Go modules with retries +description: >- + Runs `go mod download` with bounded retries and exponential backoff so a + transient proxy.golang.org stream reset ("stream error: stream ID N; + INTERNAL_ERROR; received from peer", #14976) fills the module cache instead + of failing the job. Every later `go build`/`go test`/golangci-lint/ + govulncheck/gosec step in the job then reads a warm GOMODCACHE and never + touches the proxy for the module graph. Mirrors the retry #14990 added to + the Docker builder stages (src/Dockerfile*). Go verifies every module + against go.sum regardless of how many attempts it took, so a retry can never + substitute a different artifact. A still-failing final attempt fails the + step exactly like the unguarded download did. +inputs: + working-directory: + description: Directory holding go.mod (and go.work, when present). + required: false + default: src + attempts: + description: Maximum number of `go mod download` attempts. + required: false + default: '5' +runs: + using: composite + steps: + - name: go mod download (retry transient proxy resets) + shell: bash + working-directory: ${{ inputs.working-directory }} + env: + ATTEMPTS: ${{ inputs.attempts }} + GOFLAGS: -mod=mod + run: | + set -uo pipefail + case "$ATTEMPTS" in + ''|*[!0-9]*|0) echo "::error::attempts must be a positive integer, got '$ATTEMPTS'"; exit 2 ;; + esac + delay=2 + for i in $(seq 1 "$ATTEMPTS"); do + rc=0 + go mod download || rc=$? + if [ "$rc" -eq 0 ]; then + echo "go mod download succeeded on attempt $i/$ATTEMPTS" + exit 0 + fi + if [ "$i" -eq "$ATTEMPTS" ]; then + echo "::error::go mod download failed after $ATTEMPTS attempts (last rc=$rc)" + exit "$rc" + fi + echo "::warning::go mod download failed (attempt $i/$ATTEMPTS, rc=$rc); retrying in ${delay}s" + sleep "$delay" + delay=$((delay * 2)) + done diff --git a/.github/workflows/go-security-analysis.yml b/.github/workflows/go-security-analysis.yml index 44f0913755..3bd404010a 100644 --- a/.github/workflows/go-security-analysis.yml +++ b/.github/workflows/go-security-analysis.yml @@ -163,6 +163,11 @@ jobs: go-version-file: src/go.mod - name: Guard shared Go cache (fall back when the runner volume is full) run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry # Installed by version rather than via a marketplace action so the tool # itself is pinned and the supply chain is one hop shorter. @@ -200,6 +205,11 @@ jobs: go-version-file: src/go.mod - name: Guard shared Go cache (fall back when the runner volume is full) run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry # Version floor is load-bearing: gosec v2.21.4 pins golang.org/x/tools # v0.25.0, which does not COMPILE under Go 1.25 ("invalid array length"). @@ -254,6 +264,11 @@ jobs: go-version-file: src/go.mod - name: Guard shared Go cache (fall back when the runner volume is full) run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry # skip-cache: the action's own cache save/restore is the same multi-GB # tar problem as setup-go's; the runners already keep GOCACHE warm. @@ -343,6 +358,11 @@ jobs: go-version-file: src/go.mod - name: Guard shared Go cache (fall back when the runner volume is full) run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry - name: Regenerate NOTICE from the current module graph run: src/scripts/generate-notice.sh /tmp/NOTICE.generated diff --git a/.github/workflows/testutil-guard.yml b/.github/workflows/testutil-guard.yml index 0c2cb50774..0439e68e77 100644 --- a/.github/workflows/testutil-guard.yml +++ b/.github/workflows/testutil-guard.yml @@ -139,6 +139,11 @@ jobs: go-version-file: src/go.mod - name: Guard shared Go cache (fall back when the runner volume is full) run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry - name: internal/testutil coverage floor working-directory: src diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index 458052a823..ed57da8313 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -143,6 +143,11 @@ jobs: go-version-file: src/go.mod - name: Guard shared Go cache (fall back when the runner volume is full) run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry # Warm compile cache (#7480 follow-up): the warm-build-cache job on every # v4 push compiles the whole tree (race + norace) and saves GOCACHE keyed diff --git a/.github/workflows/v2-tests.yml b/.github/workflows/v2-tests.yml index aacab0dc9f..d4990a0158 100644 --- a/.github/workflows/v2-tests.yml +++ b/.github/workflows/v2-tests.yml @@ -317,6 +317,12 @@ jobs: - name: Guard shared Go cache (fall back when the runner volume is full) if: steps.select.outputs.run == 'true' run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry - name: Install gotestsum if: steps.select.outputs.run == 'true' @@ -600,6 +606,12 @@ jobs: - name: Guard shared Go cache (fall back when the runner volume is full) if: steps.select.outputs.run == 'true' run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry - name: Install gotestsum if: steps.select.outputs.run == 'true' @@ -871,6 +883,12 @@ jobs: - name: Guard shared Go cache (fall back when the runner volume is full) if: steps.select.outputs.run == 'true' run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry - name: Install gotestsum if: steps.select.outputs.run == 'true' @@ -1107,6 +1125,11 @@ jobs: go-version-file: src/go.mod - name: Guard shared Go cache (fall back when the runner volume is full) run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry # Warm compile cache (#7480 follow-up): the warm-build-cache job on every # v4 push compiles the whole tree (race + norace) and saves GOCACHE keyed @@ -1444,6 +1467,12 @@ jobs: - name: Guard shared Go cache (fall back when the runner volume is full) if: steps.select.outputs.run == 'true' run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry - name: Install gotestsum if: steps.select.outputs.run == 'true' @@ -1659,6 +1688,12 @@ jobs: - name: Guard shared Go cache (fall back when the runner volume is full) if: steps.select.outputs.run == 'true' run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry - name: Install gotestsum if: steps.select.outputs.run == 'true' @@ -2028,6 +2063,12 @@ jobs: cache-dependency-path: | src/go.sum src/crux/go.sum + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + if: ${{ steps.cov-scope.outputs.score == 'true' }} + uses: ./.github/actions/go-mod-download-retry - name: List packages present in checkout if: ${{ steps.cov-scope.outputs.score == 'true' }} @@ -2664,6 +2705,11 @@ jobs: go-version-file: src/go.mod - name: Guard shared Go cache (fall back when the runner volume is full) run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + # Fill GOMODCACHE up front with bounded retries so a transient + # proxy.golang.org stream reset (INTERNAL_ERROR, #14976) cannot fail the + # job's first module-fetching step; see .github/actions/go-mod-download-retry. + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry - name: Restore warm Go build cache if: ${{ !contains(vars.HIVE_RUNNER_LABELS, 'hive-runners') }} diff --git a/changelog.d/fixed-14976-ci-go-mod-download-retry.md b/changelog.d/fixed-14976-ci-go-mod-download-retry.md new file mode 100644 index 0000000000..6e8e7aebdb --- /dev/null +++ b/changelog.d/fixed-14976-ci-go-mod-download-retry.md @@ -0,0 +1 @@ +- CI Go jobs (`v2-ci`, `v2-tests`, `go-security-analysis`, `testutil-guard`) now pre-fill the module cache through a shared `go-mod-download-retry` action that retries `go mod download` with exponential backoff, so a transient `proxy.golang.org` stream reset (`INTERNAL_ERROR`) no longer fails golangci-lint, govulncheck, gosec or the sharded tests; this completes the Docker builder-stage retry from #14990 ([#14976](https://github.com/hivecommons/hive/issues/14976)).