docker: retry transient pull failures

__tests__/docker/docker.test.ts

@@ -19,6 +19,7 @@ import fs from 'fs';
 import os from 'os';
 import path from 'path';
 import * as io from '@actions/io';
+import {ExecOutput} from '@actions/exec';
 import * as rimraf from 'rimraf';
 
 import {mockHomedir} from '../.helpers/os.js';
@@ -166,6 +167,55 @@ describe('getExecOutput', () => {
   });
 });
 
+describe('pull', () => {
+  const originalDockerConfig = process.env.DOCKER_CONFIG;
+
+  beforeEach(() => {
+    fs.mkdirSync(tmpDir, {recursive: true});
+    process.env.DOCKER_CONFIG = path.join(tmpDir, 'docker-config');
+  });
+
+  afterEach(() => {
+    process.env.DOCKER_CONFIG = originalDockerConfig;
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+  });
+
+  it('retries transient registry errors', async () => {
+    vi.useFakeTimers();
+    const execSpy = vi
+      .spyOn(Docker, 'getExecOutput')
+      .mockResolvedValueOnce(
+        execOutput(
+          1,
+          '',
+          'Error response from daemon: Head "https://registry-1.docker.io/v2/tonistiigi/binfmt/manifests/latest": Get "https://auth.docker.io/token": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)'
+        )
+      )
+      .mockResolvedValueOnce(execOutput(1, '', 'Error response from daemon: Head "https://registry-1.docker.io/v2/tonistiigi/binfmt/manifests/latest": EOF'))
+      .mockResolvedValueOnce(execOutput(1, '', 'Error response from daemon: received unexpected HTTP status: 503 Service Unavailable'))
+      .mockResolvedValueOnce(execOutput(1, '', 'Error response from daemon: connection reset by peer'))
+      .mockResolvedValueOnce(execOutput(0, 'latest: Pulling from tonistiigi/binfmt', ''));
+
+    const pull = Docker.pull('tonistiigi/binfmt');
+    await vi.runAllTimersAsync();
+    await pull;
+    expect(execSpy).toHaveBeenCalledTimes(5);
+  });
+
+  it('does not retry permanent pull errors', async () => {
+    const execSpy = vi.spyOn(Docker, 'getExecOutput').mockResolvedValue(execOutput(1, '', 'Error response from daemon: pull access denied for doesnotexist'));
+    await expect(Docker.pull('doesnotexist:foo')).rejects.toThrow('pull access denied for doesnotexist');
+    expect(execSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not retry rate limit errors', async () => {
+    const execSpy = vi.spyOn(Docker, 'getExecOutput').mockResolvedValue(execOutput(1, '', 'Error response from daemon: toomanyrequests: You have reached your pull rate limit'));
+    await expect(Docker.pull('busybox')).rejects.toThrow('toomanyrequests');
+    expect(execSpy).toHaveBeenCalledTimes(1);
+  });
+});
+
 describe('context', () => {
   it('call docker context show', async () => {
     const execSpy = vi.spyOn(Docker, 'getExecOutput');
@@ -241,3 +291,11 @@ describe('printInfo', () => {
     expect(callfunc).toEqual([['info']]);
   });
 });
+
+const execOutput = (exitCode: number, stdout: string, stderr: string): ExecOutput => {
+  return {
+    exitCode: exitCode,
+    stdout: stdout,
+    stderr: stderr
+  };
+};

src/docker/docker.ts

@@ -17,6 +17,7 @@
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
+import retry from 'async-retry';
 import * as core from '@actions/core';
 import {ExecOptions, ExecOutput} from '@actions/exec';
 import * as io from '@actions/io';
@@ -185,19 +186,16 @@ export class Docker {
     }
 
     let pulled = true;
-    await Docker.getExecOutput(['pull', image], {
-      ignoreReturnCode: true
-    }).then(res => {
-      if (res.stderr.length > 0 && res.exitCode != 0) {
-        pulled = false;
-        const err = res.stderr.match(/(.*)\s*$/)?.[0]?.trim() ?? 'unknown error';
-        if (cacheFoundPath) {
-          core.warning(`Failed to pull image, using one from cache: ${err}`);
-        } else {
-          throw new Error(err);
-        }
+    try {
+      await Docker.pullWithRetry(image);
+    } catch (e) {
+      pulled = false;
+      if (cacheFoundPath) {
+        core.warning(`Failed to pull image, using one from cache: ${(e as Error).message}`);
+      } else {
+        throw e;
       }
-    });
+    }
 
     if (cache && pulled) {
       const imageTarPath = path.join(Context.tmpDir(), `${Util.hash(image)}.tar`);
@@ -213,4 +211,41 @@ export class Docker {
       });
     }
   }
+
+  private static async pullWithRetry(image: string): Promise<void> {
+    const retries = 5;
+    await retry(
+      async bail => {
+        const res = await Docker.getExecOutput(['pull', image], {
+          ignoreReturnCode: true
+        });
+        if (res.stderr.length > 0 && res.exitCode != 0) {
+          const err = res.stderr.match(/(.*)\s*$/)?.[0]?.trim() ?? 'unknown error';
+          if (!Docker.isPullTransientError(err)) {
+            bail(new Error(err));
+            return;
+          }
+          throw new Error(err);
+        }
+      },
+      {
+        retries: retries - 1,
+        minTimeout: 1000,
+        factor: 2,
+        onRetry: (err, i) => {
+          core.debug(`Docker pull failed, retrying (${i}/${retries})...\n${err}`);
+        }
+      }
+    );
+  }
+
+  private static isPullTransientError(err: string): boolean {
+    return (
+      /Client\.Timeout exceeded|TLS handshake timeout|i\/o timeout|context deadline exceeded|request canceled|connection reset by peer|connection refused|connection timed out|temporary failure|unexpected EOF|\bEOF\b|server misbehaving/i.test(
+        err
+      ) ||
+      /\b(500|502|503|504)\b/.test(err) ||
+      /\b(service unavailable|bad gateway|gateway timeout|internal server error)\b/i.test(err)
+    );
+  }
 }