diff --git a/src/cli/engine-install.ts b/src/cli/engine-install.ts index ed0ea6ff21..790258cf53 100644 --- a/src/cli/engine-install.ts +++ b/src/cli/engine-install.ts @@ -5,8 +5,12 @@ import { basename, join } from "node:path"; export const III_INSTALL_CONNECT_TIMEOUT_S = 15; export const III_INSTALL_MAX_TIME_S = 300; +export const III_INSTALL_RETRIES = 3; +export const III_INSTALL_RETRY_DELAY_S = 2; +export const III_INSTALL_RETRY_MAX_TIME_S = 120; export const III_INSTALL_SPAWN_TIMEOUT_MS = - (III_INSTALL_CONNECT_TIMEOUT_S + III_INSTALL_MAX_TIME_S + 30) * 1000; + (III_INSTALL_CONNECT_TIMEOUT_S + III_INSTALL_MAX_TIME_S + III_INSTALL_RETRY_MAX_TIME_S + 30) * 1000; +const RETRYABLE_HTTP_STATUS = new Set([408, 429, 500, 502, 503, 504]); type ChecksumTable = Readonly>>>; @@ -44,7 +48,11 @@ export function sha256File(path: string): string { } function curlTimeoutFlags(): string { - return `--connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} --max-time ${III_INSTALL_MAX_TIME_S}`; + return `--connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} --max-time ${III_INSTALL_MAX_TIME_S} ${curlRetryFlags()}`; +} + +function curlRetryFlags(): string { + return `--retry ${III_INSTALL_RETRIES} --retry-delay ${III_INSTALL_RETRY_DELAY_S} --retry-max-time ${III_INSTALL_RETRY_MAX_TIME_S}`; } export function iiiArchivePath(binDir: string): string { @@ -68,7 +76,7 @@ export function iiiManualInstallCommand( const verify = sha256 ? ` && printf '%s %s\\n' "${sha256}" "$archive" | { sha256sum -c - 2>/dev/null || shasum -a 256 -c -; }` : ""; - return `mkdir -p "${binDir}" && archive="$(mktemp "${binDir}/.iii-download.XXXXXX")" && { curl -fL --connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} -o "$archive" "${releaseUrl}"${verify} && tar -xzf "$archive" -C "${binDir}"; code=$?; rm -f "$archive"; (exit $code); } && chmod +x "${binPath}"`; + return `mkdir -p "${binDir}" && archive="$(mktemp "${binDir}/.iii-download.XXXXXX")" && { curl -fL --connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} ${curlRetryFlags()} -o "$archive" "${releaseUrl}"${verify} && tar -xzf "$archive" -C "${binDir}"; code=$?; rm -f "$archive"; (exit $code); } && chmod +x "${binPath}"`; } export function describeInstallFailure(result: { @@ -216,6 +224,12 @@ export function windowsCurlDownloadCommand( String(III_INSTALL_CONNECT_TIMEOUT_S), "--max-time", String(III_INSTALL_MAX_TIME_S), + "--retry", + String(III_INSTALL_RETRIES), + "--retry-delay", + String(III_INSTALL_RETRY_DELAY_S), + "--retry-max-time", + String(III_INSTALL_RETRY_MAX_TIME_S), "-o", dest, url, @@ -253,6 +267,8 @@ export async function downloadToFile( dest: string, env: NodeJS.ProcessEnv = process.env, run: ZipCommandRunner = runInstallCommand, + request: typeof fetch = fetch, + sleep: (ms: number) => Promise = (ms) => new Promise((done) => setTimeout(done, ms)), ): Promise { if (proxyConfigured(env)) { const step = windowsCurlDownloadCommand(url, dest, env); @@ -260,12 +276,39 @@ export async function downloadToFile( if (result.status !== 0) throw new Error(describeInstallFailure(result)); return; } - const res = await fetch(url, { - redirect: "follow", - signal: AbortSignal.timeout(III_INSTALL_MAX_TIME_S * 1000), - }); - if (!res.ok) throw new Error(`download failed with HTTP ${res.status} ${res.statusText}`.trim()); - writeFileSync(dest, Buffer.from(await res.arrayBuffer())); + const deadline = Date.now() + III_INSTALL_RETRY_MAX_TIME_S * 1000; + for (let attempt = 0; ; attempt++) { + let res: Response | undefined; + let body: Buffer | undefined; + let failure: unknown; + try { + res = await request(url, { redirect: "follow", signal: AbortSignal.timeout(III_INSTALL_MAX_TIME_S * 1000) }); + if (res.ok) body = Buffer.from(await res.arrayBuffer()); + else await res.body?.cancel().catch(() => {}); + } catch (error) { + failure = error; + } + if (body) { + writeFileSync(dest, body); + return; + } + if (res && !res.ok) { + failure = new Error(`download failed with HTTP ${res.status} ${res.statusText}`.trim()); + if (!RETRYABLE_HTTP_STATUS.has(res.status)) throw failure; + } + const delayMs = retryDelayMs(res); + if (attempt >= III_INSTALL_RETRIES || Date.now() + delayMs >= deadline) throw failure; + await sleep(delayMs); + } +} + +function retryDelayMs(res: Response | undefined): number { + const fallback = III_INSTALL_RETRY_DELAY_S * 1000; + const header = res?.ok ? null : res?.headers.get("retry-after"); + if (!header) return fallback; + const seconds = Number(header); + const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now(); + return Number.isFinite(ms) ? Math.max(ms, fallback) : fallback; } export function runInstallCommand(command: string, args: string[]): Promise { diff --git a/test/engine-install.test.ts b/test/engine-install.test.ts index b80114765f..bbee8c1c53 100644 --- a/test/engine-install.test.ts +++ b/test/engine-install.test.ts @@ -6,6 +6,9 @@ import { afterEach, describe, expect, it } from "vitest"; import { III_INSTALL_CONNECT_TIMEOUT_S, III_INSTALL_MAX_TIME_S, + III_INSTALL_RETRIES, + III_INSTALL_RETRY_DELAY_S, + III_INSTALL_RETRY_MAX_TIME_S, III_RELEASE_SHA256, describeInstallFailure, expectedIiiSha256, @@ -459,6 +462,61 @@ describe("iii engine Windows zip installer", () => { await expect(downloadToFile("https://example.invalid/iii.zip", "C:\\bin\\a.zip", { HTTPS_PROXY: "http://proxy:3128" }, () => ({ status: 22, stderr: "curl: (22) 407" }))).rejects.toThrow("407"); }); + it("retries transient download failures before giving up", async () => { + expect(iiiDownloadShellCommand()).toContain(`--retry ${III_INSTALL_RETRIES} --retry-delay ${III_INSTALL_RETRY_DELAY_S}`); + expect(iiiManualInstallCommand("https://x/iii.tar.gz", "/b", "/b/iii", null)).toContain(`--retry ${III_INSTALL_RETRIES}`); + const curlCalls: string[][] = []; + await downloadToFile("https://example.invalid/iii.zip", "C:\\bin\\a.zip", { HTTPS_PROXY: "http://proxy:3128" }, (_command, args) => { + curlCalls.push(args); + return { status: 0 }; + }); + expect(curlCalls[0]).toEqual(expect.arrayContaining(["--retry", String(III_INSTALL_RETRIES), "--retry-delay", String(III_INSTALL_RETRY_DELAY_S)])); + + const dir = mkdtempSync(join(tmpdir(), "am-retry-")); + const dest = join(dir, "iii.zip"); + const statuses = [504, 503, 200]; + const sleeps: number[] = []; + const request = (async () => { + const status = statuses.shift()!; + return new Response(status === 200 ? "zip-bytes" : "busy", { status }); + }) as typeof fetch; + await downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, request, async (ms) => { sleeps.push(ms); }); + expect(readFileSync(dest, "utf8")).toBe("zip-bytes"); + expect(sleeps).toEqual([III_INSTALL_RETRY_DELAY_S * 1000, III_INSTALL_RETRY_DELAY_S * 1000]); + + let attempts = 0; + const alwaysBusy = (async () => { attempts++; return new Response("busy", { status: 504 }); }) as typeof fetch; + await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, alwaysBusy, async () => {})).rejects.toThrow("HTTP 504"); + expect(attempts).toBe(III_INSTALL_RETRIES + 1); + + attempts = 0; + const missing = (async () => { attempts++; return new Response("missing", { status: 404 }); }) as typeof fetch; + await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, missing, async () => {})).rejects.toThrow("HTTP 404"); + expect(attempts).toBe(1); + + const limited = [new Response("slow down", { status: 429, headers: { "retry-after": "30" } }), new Response("zip-bytes", { status: 200 })]; + const waits: number[] = []; + await downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, (async () => limited.shift()!) as typeof fetch, async (ms) => { waits.push(ms); }); + expect(waits).toEqual([30_000]); + + attempts = 0; + const tooLong = (async () => { attempts++; return new Response("later", { status: 503, headers: { "retry-after": String(III_INSTALL_RETRY_MAX_TIME_S + 1) } }); }) as typeof fetch; + await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, tooLong, async () => {})).rejects.toThrow("HTTP 503"); + expect(attempts).toBe(1); + + attempts = 0; + const dropped = (async () => { + attempts++; + if (attempts > 1) return new Response("zip-bytes", { status: 200 }); + const stream = new ReadableStream({ start(controller) { controller.error(new Error("socket hang up")); } }); + return new Response(stream, { status: 200 }); + }) as typeof fetch; + writeFileSync(dest, ""); + await downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, dropped, async () => {}); + expect(attempts).toBe(2); + expect(readFileSync(dest, "utf8")).toBe("zip-bytes"); + }); + it("pins a SHA-256 for both Windows assets of the pinned release", () => { expect(expectedIiiSha256(III_PINNED_VERSION, "iii-x86_64-pc-windows-msvc.zip")).toMatch(/^[0-9a-f]{64}$/); expect(expectedIiiSha256(III_PINNED_VERSION, "iii-aarch64-pc-windows-msvc.zip")).toMatch(/^[0-9a-f]{64}$/);