From ff4169ddba8a823ab11ae00dbd256a51aa6389a1 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 31 Aug 2026 11:36:10 +0200 Subject: [PATCH] docker: retry transient pull failures Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- __tests__/docker/docker.test.ts | 58 ++++++++++++++++++++++++++++++++ src/docker/docker.ts | 59 ++++++++++++++++++++++++++------- 2 files changed, 105 insertions(+), 12 deletions(-) diff --git a/__tests__/docker/docker.test.ts b/__tests__/docker/docker.test.ts index 9aef6e87..46a406ee 100644 --- a/__tests__/docker/docker.test.ts +++ b/__tests__/docker/docker.test.ts @@ -19,6 +19,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; import * as io from '@actions/io'; +import {ExecOutput} from '@actions/exec'; import * as rimraf from 'rimraf'; import {mockHomedir} from '../.helpers/os.js'; @@ -166,6 +167,55 @@ describe('getExecOutput', () => { }); }); +describe('pull', () => { + const originalDockerConfig = process.env.DOCKER_CONFIG; + + beforeEach(() => { + fs.mkdirSync(tmpDir, {recursive: true}); + process.env.DOCKER_CONFIG = path.join(tmpDir, 'docker-config'); + }); + + afterEach(() => { + process.env.DOCKER_CONFIG = originalDockerConfig; + vi.restoreAllMocks(); + vi.useRealTimers(); + }); + + it('retries transient registry errors', async () => { + vi.useFakeTimers(); + const execSpy = vi + .spyOn(Docker, 'getExecOutput') + .mockResolvedValueOnce( + execOutput( + 1, + '', + 'Error response from daemon: Head "https://registry-1.docker.io/v2/tonistiigi/binfmt/manifests/latest": Get "https://auth.docker.io/token": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' + ) + ) + .mockResolvedValueOnce(execOutput(1, '', 'Error response from daemon: Head "https://registry-1.docker.io/v2/tonistiigi/binfmt/manifests/latest": EOF')) + .mockResolvedValueOnce(execOutput(1, '', 'Error response from daemon: received unexpected HTTP status: 503 Service Unavailable')) + .mockResolvedValueOnce(execOutput(1, '', 'Error response from daemon: connection reset by peer')) + .mockResolvedValueOnce(execOutput(0, 'latest: Pulling from tonistiigi/binfmt', '')); + + const pull = Docker.pull('tonistiigi/binfmt'); + await vi.runAllTimersAsync(); + await pull; + expect(execSpy).toHaveBeenCalledTimes(5); + }); + + it('does not retry permanent pull errors', async () => { + const execSpy = vi.spyOn(Docker, 'getExecOutput').mockResolvedValue(execOutput(1, '', 'Error response from daemon: pull access denied for doesnotexist')); + await expect(Docker.pull('doesnotexist:foo')).rejects.toThrow('pull access denied for doesnotexist'); + expect(execSpy).toHaveBeenCalledTimes(1); + }); + + it('does not retry rate limit errors', async () => { + const execSpy = vi.spyOn(Docker, 'getExecOutput').mockResolvedValue(execOutput(1, '', 'Error response from daemon: toomanyrequests: You have reached your pull rate limit')); + await expect(Docker.pull('busybox')).rejects.toThrow('toomanyrequests'); + expect(execSpy).toHaveBeenCalledTimes(1); + }); +}); + describe('context', () => { it('call docker context show', async () => { const execSpy = vi.spyOn(Docker, 'getExecOutput'); @@ -241,3 +291,11 @@ describe('printInfo', () => { expect(callfunc).toEqual([['info']]); }); }); + +const execOutput = (exitCode: number, stdout: string, stderr: string): ExecOutput => { + return { + exitCode: exitCode, + stdout: stdout, + stderr: stderr + }; +}; diff --git a/src/docker/docker.ts b/src/docker/docker.ts index f7fac241..4ee6ca13 100644 --- a/src/docker/docker.ts +++ b/src/docker/docker.ts @@ -17,6 +17,7 @@ import fs from 'fs'; import os from 'os'; import path from 'path'; +import retry from 'async-retry'; import * as core from '@actions/core'; import {ExecOptions, ExecOutput} from '@actions/exec'; import * as io from '@actions/io'; @@ -185,19 +186,16 @@ export class Docker { } let pulled = true; - await Docker.getExecOutput(['pull', image], { - ignoreReturnCode: true - }).then(res => { - if (res.stderr.length > 0 && res.exitCode != 0) { - pulled = false; - const err = res.stderr.match(/(.*)\s*$/)?.[0]?.trim() ?? 'unknown error'; - if (cacheFoundPath) { - core.warning(`Failed to pull image, using one from cache: ${err}`); - } else { - throw new Error(err); - } + try { + await Docker.pullWithRetry(image); + } catch (e) { + pulled = false; + if (cacheFoundPath) { + core.warning(`Failed to pull image, using one from cache: ${(e as Error).message}`); + } else { + throw e; } - }); + } if (cache && pulled) { const imageTarPath = path.join(Context.tmpDir(), `${Util.hash(image)}.tar`); @@ -213,4 +211,41 @@ export class Docker { }); } } + + private static async pullWithRetry(image: string): Promise { + const retries = 5; + await retry( + async bail => { + const res = await Docker.getExecOutput(['pull', image], { + ignoreReturnCode: true + }); + if (res.stderr.length > 0 && res.exitCode != 0) { + const err = res.stderr.match(/(.*)\s*$/)?.[0]?.trim() ?? 'unknown error'; + if (!Docker.isPullTransientError(err)) { + bail(new Error(err)); + return; + } + throw new Error(err); + } + }, + { + retries: retries - 1, + minTimeout: 1000, + factor: 2, + onRetry: (err, i) => { + core.debug(`Docker pull failed, retrying (${i}/${retries})...\n${err}`); + } + } + ); + } + + private static isPullTransientError(err: string): boolean { + return ( + /Client\.Timeout exceeded|TLS handshake timeout|i\/o timeout|context deadline exceeded|request canceled|connection reset by peer|connection refused|connection timed out|temporary failure|unexpected EOF|\bEOF\b|server misbehaving/i.test( + err + ) || + /\b(500|502|503|504)\b/.test(err) || + /\b(service unavailable|bad gateway|gateway timeout|internal server error)\b/i.test(err) + ); + } }