From 2455a7a24b86e3276065a04896f1bf841e93e318 Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 9 Nov 2026 15:04:22 +0530 Subject: [PATCH 1/3] fix(install): retry transient iii engine download failures The iii engine download made a single attempt, so one GitHub 504, 503 or dropped connection failed a fresh install. On 2026-11-09 the release files for Linux and Windows returned 504 for hours. curl downloads (macOS, Linux, Windows behind a proxy, and the printed manual command) now pass --retry 3 --retry-delay 2 --retry-max-time 120; --retry covers timeouts and HTTP 408, 429, 500, 502, 503 and 504 and is accepted by every curl since 7.12. The Windows fetch download retries the same statuses and network errors up to three times inside the same 120-second window. A 404 or other client error still fails at once. The install spawn timeout grows by the retry window. --- src/cli/engine-install.ts | 48 ++++++++++++++++++++++++++++++------- test/engine-install.test.ts | 35 +++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/src/cli/engine-install.ts b/src/cli/engine-install.ts index ed0ea6ff21..29d4435c56 100644 --- a/src/cli/engine-install.ts +++ b/src/cli/engine-install.ts @@ -5,8 +5,12 @@ import { basename, join } from "node:path"; export const III_INSTALL_CONNECT_TIMEOUT_S = 15; export const III_INSTALL_MAX_TIME_S = 300; +export const III_INSTALL_RETRIES = 3; +export const III_INSTALL_RETRY_DELAY_S = 2; +export const III_INSTALL_RETRY_MAX_TIME_S = 120; export const III_INSTALL_SPAWN_TIMEOUT_MS = - (III_INSTALL_CONNECT_TIMEOUT_S + III_INSTALL_MAX_TIME_S + 30) * 1000; + (III_INSTALL_CONNECT_TIMEOUT_S + III_INSTALL_MAX_TIME_S + III_INSTALL_RETRY_MAX_TIME_S + 30) * 1000; +const RETRYABLE_HTTP_STATUS = new Set([408, 429, 500, 502, 503, 504]); type ChecksumTable = Readonly>>>; @@ -44,7 +48,11 @@ export function sha256File(path: string): string { } function curlTimeoutFlags(): string { - return `--connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} --max-time ${III_INSTALL_MAX_TIME_S}`; + return `--connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} --max-time ${III_INSTALL_MAX_TIME_S} ${curlRetryFlags()}`; +} + +function curlRetryFlags(): string { + return `--retry ${III_INSTALL_RETRIES} --retry-delay ${III_INSTALL_RETRY_DELAY_S} --retry-max-time ${III_INSTALL_RETRY_MAX_TIME_S}`; } export function iiiArchivePath(binDir: string): string { @@ -68,7 +76,7 @@ export function iiiManualInstallCommand( const verify = sha256 ? ` && printf '%s %s\\n' "${sha256}" "$archive" | { sha256sum -c - 2>/dev/null || shasum -a 256 -c -; }` : ""; - return `mkdir -p "${binDir}" && archive="$(mktemp "${binDir}/.iii-download.XXXXXX")" && { curl -fL --connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} -o "$archive" "${releaseUrl}"${verify} && tar -xzf "$archive" -C "${binDir}"; code=$?; rm -f "$archive"; (exit $code); } && chmod +x "${binPath}"`; + return `mkdir -p "${binDir}" && archive="$(mktemp "${binDir}/.iii-download.XXXXXX")" && { curl -fL --connect-timeout ${III_INSTALL_CONNECT_TIMEOUT_S} ${curlRetryFlags()} -o "$archive" "${releaseUrl}"${verify} && tar -xzf "$archive" -C "${binDir}"; code=$?; rm -f "$archive"; (exit $code); } && chmod +x "${binPath}"`; } export function describeInstallFailure(result: { @@ -216,6 +224,12 @@ export function windowsCurlDownloadCommand( String(III_INSTALL_CONNECT_TIMEOUT_S), "--max-time", String(III_INSTALL_MAX_TIME_S), + "--retry", + String(III_INSTALL_RETRIES), + "--retry-delay", + String(III_INSTALL_RETRY_DELAY_S), + "--retry-max-time", + String(III_INSTALL_RETRY_MAX_TIME_S), "-o", dest, url, @@ -253,6 +267,8 @@ export async function downloadToFile( dest: string, env: NodeJS.ProcessEnv = process.env, run: ZipCommandRunner = runInstallCommand, + request: typeof fetch = fetch, + sleep: (ms: number) => Promise = (ms) => new Promise((done) => setTimeout(done, ms)), ): Promise { if (proxyConfigured(env)) { const step = windowsCurlDownloadCommand(url, dest, env); @@ -260,12 +276,26 @@ export async function downloadToFile( if (result.status !== 0) throw new Error(describeInstallFailure(result)); return; } - const res = await fetch(url, { - redirect: "follow", - signal: AbortSignal.timeout(III_INSTALL_MAX_TIME_S * 1000), - }); - if (!res.ok) throw new Error(`download failed with HTTP ${res.status} ${res.statusText}`.trim()); - writeFileSync(dest, Buffer.from(await res.arrayBuffer())); + const started = Date.now(); + for (let attempt = 0; ; attempt++) { + const retryable = attempt < III_INSTALL_RETRIES && Date.now() - started < III_INSTALL_RETRY_MAX_TIME_S * 1000; + let res: Response; + try { + res = await request(url, { redirect: "follow", signal: AbortSignal.timeout(III_INSTALL_MAX_TIME_S * 1000) }); + } catch (error) { + if (!retryable) throw error; + await sleep(III_INSTALL_RETRY_DELAY_S * 1000); + continue; + } + if (res.ok) { + writeFileSync(dest, Buffer.from(await res.arrayBuffer())); + return; + } + if (!retryable || !RETRYABLE_HTTP_STATUS.has(res.status)) { + throw new Error(`download failed with HTTP ${res.status} ${res.statusText}`.trim()); + } + await sleep(III_INSTALL_RETRY_DELAY_S * 1000); + } } export function runInstallCommand(command: string, args: string[]): Promise { diff --git a/test/engine-install.test.ts b/test/engine-install.test.ts index b80114765f..fa61255c66 100644 --- a/test/engine-install.test.ts +++ b/test/engine-install.test.ts @@ -6,6 +6,8 @@ import { afterEach, describe, expect, it } from "vitest"; import { III_INSTALL_CONNECT_TIMEOUT_S, III_INSTALL_MAX_TIME_S, + III_INSTALL_RETRIES, + III_INSTALL_RETRY_DELAY_S, III_RELEASE_SHA256, describeInstallFailure, expectedIiiSha256, @@ -459,6 +461,39 @@ describe("iii engine Windows zip installer", () => { await expect(downloadToFile("https://example.invalid/iii.zip", "C:\\bin\\a.zip", { HTTPS_PROXY: "http://proxy:3128" }, () => ({ status: 22, stderr: "curl: (22) 407" }))).rejects.toThrow("407"); }); + it("retries transient download failures before giving up", async () => { + expect(iiiDownloadShellCommand()).toContain(`--retry ${III_INSTALL_RETRIES} --retry-delay ${III_INSTALL_RETRY_DELAY_S}`); + expect(iiiManualInstallCommand("https://x/iii.tar.gz", "/b", "/b/iii", null)).toContain(`--retry ${III_INSTALL_RETRIES}`); + const curlCalls: string[][] = []; + await downloadToFile("https://example.invalid/iii.zip", "C:\\bin\\a.zip", { HTTPS_PROXY: "http://proxy:3128" }, (_command, args) => { + curlCalls.push(args); + return { status: 0 }; + }); + expect(curlCalls[0]).toEqual(expect.arrayContaining(["--retry", String(III_INSTALL_RETRIES), "--retry-delay", String(III_INSTALL_RETRY_DELAY_S)])); + + const dir = mkdtempSync(join(tmpdir(), "am-retry-")); + const dest = join(dir, "iii.zip"); + const statuses = [504, 503, 200]; + const sleeps: number[] = []; + const request = (async () => { + const status = statuses.shift()!; + return new Response(status === 200 ? "zip-bytes" : "busy", { status }); + }) as typeof fetch; + await downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, request, async (ms) => { sleeps.push(ms); }); + expect(readFileSync(dest, "utf8")).toBe("zip-bytes"); + expect(sleeps).toEqual([III_INSTALL_RETRY_DELAY_S * 1000, III_INSTALL_RETRY_DELAY_S * 1000]); + + let attempts = 0; + const alwaysBusy = (async () => { attempts++; return new Response("busy", { status: 504 }); }) as typeof fetch; + await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, alwaysBusy, async () => {})).rejects.toThrow("HTTP 504"); + expect(attempts).toBe(III_INSTALL_RETRIES + 1); + + attempts = 0; + const missing = (async () => { attempts++; return new Response("missing", { status: 404 }); }) as typeof fetch; + await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, missing, async () => {})).rejects.toThrow("HTTP 404"); + expect(attempts).toBe(1); + }); + it("pins a SHA-256 for both Windows assets of the pinned release", () => { expect(expectedIiiSha256(III_PINNED_VERSION, "iii-x86_64-pc-windows-msvc.zip")).toMatch(/^[0-9a-f]{64}$/); expect(expectedIiiSha256(III_PINNED_VERSION, "iii-aarch64-pc-windows-msvc.zip")).toMatch(/^[0-9a-f]{64}$/); From 5f0367e0fbd7d37a3bae73a88e108ca7d9f754ca Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 9 Nov 2026 15:06:22 +0530 Subject: [PATCH 2/3] docs(changelog): add the engine download retry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab020d8c70..775403f7f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Consolidation keeps provenance and version heads across retries, and saved pattern memories reach the procedural tier (#1545, #1690). Reflection bounds its evidence and insight responses and keeps other projects' crystals out of a scoped reflection (#1544). - `health` reports CPU relative to the machine's cores, not one core (#1286 by @DanielCarmingham). - `agentmemory demo` stops the Compose workers it started on Windows (#1688 by @Mike-Animal-Counseling). Generated Compose values keep newlines (#1708 by @zhichao-poper). +- Fresh installs retry the iii engine download on timeouts and HTTP 408, 429 and 5xx responses, so a brief GitHub outage no longer fails setup (#1719). - `agentmemory connect` honors `CLAUDE_CONFIG_DIR` for Claude Code detection, MCP configuration and hooks (#1703 by @mothieras). ### Infrastructure From 37322a12a7f9f46370ec13d6bb22f14b24f47e3f Mon Sep 17 00:00:00 2001 From: Rohit Ghumare Date: Mon, 9 Nov 2026 17:26:14 +0530 Subject: [PATCH 3/3] fix(install): bound download retries by the retry window Recheck the two-minute window after each failed download and before the next one, retry a response body that drops mid-read, honor Retry-After on 408, 429 and 5xx, and release unread error bodies before retrying. Leave CHANGELOG to the release PR. Signed-off-by: Rohit Ghumare --- CHANGELOG.md | 1 - src/cli/engine-install.ts | 35 ++++++++++++++++++++++++----------- test/engine-install.test.ts | 23 +++++++++++++++++++++++ 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 775403f7f7..ab020d8c70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,7 +30,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Consolidation keeps provenance and version heads across retries, and saved pattern memories reach the procedural tier (#1545, #1690). Reflection bounds its evidence and insight responses and keeps other projects' crystals out of a scoped reflection (#1544). - `health` reports CPU relative to the machine's cores, not one core (#1286 by @DanielCarmingham). - `agentmemory demo` stops the Compose workers it started on Windows (#1688 by @Mike-Animal-Counseling). Generated Compose values keep newlines (#1708 by @zhichao-poper). -- Fresh installs retry the iii engine download on timeouts and HTTP 408, 429 and 5xx responses, so a brief GitHub outage no longer fails setup (#1719). - `agentmemory connect` honors `CLAUDE_CONFIG_DIR` for Claude Code detection, MCP configuration and hooks (#1703 by @mothieras). ### Infrastructure diff --git a/src/cli/engine-install.ts b/src/cli/engine-install.ts index 29d4435c56..790258cf53 100644 --- a/src/cli/engine-install.ts +++ b/src/cli/engine-install.ts @@ -276,28 +276,41 @@ export async function downloadToFile( if (result.status !== 0) throw new Error(describeInstallFailure(result)); return; } - const started = Date.now(); + const deadline = Date.now() + III_INSTALL_RETRY_MAX_TIME_S * 1000; for (let attempt = 0; ; attempt++) { - const retryable = attempt < III_INSTALL_RETRIES && Date.now() - started < III_INSTALL_RETRY_MAX_TIME_S * 1000; - let res: Response; + let res: Response | undefined; + let body: Buffer | undefined; + let failure: unknown; try { res = await request(url, { redirect: "follow", signal: AbortSignal.timeout(III_INSTALL_MAX_TIME_S * 1000) }); + if (res.ok) body = Buffer.from(await res.arrayBuffer()); + else await res.body?.cancel().catch(() => {}); } catch (error) { - if (!retryable) throw error; - await sleep(III_INSTALL_RETRY_DELAY_S * 1000); - continue; + failure = error; } - if (res.ok) { - writeFileSync(dest, Buffer.from(await res.arrayBuffer())); + if (body) { + writeFileSync(dest, body); return; } - if (!retryable || !RETRYABLE_HTTP_STATUS.has(res.status)) { - throw new Error(`download failed with HTTP ${res.status} ${res.statusText}`.trim()); + if (res && !res.ok) { + failure = new Error(`download failed with HTTP ${res.status} ${res.statusText}`.trim()); + if (!RETRYABLE_HTTP_STATUS.has(res.status)) throw failure; } - await sleep(III_INSTALL_RETRY_DELAY_S * 1000); + const delayMs = retryDelayMs(res); + if (attempt >= III_INSTALL_RETRIES || Date.now() + delayMs >= deadline) throw failure; + await sleep(delayMs); } } +function retryDelayMs(res: Response | undefined): number { + const fallback = III_INSTALL_RETRY_DELAY_S * 1000; + const header = res?.ok ? null : res?.headers.get("retry-after"); + if (!header) return fallback; + const seconds = Number(header); + const ms = Number.isFinite(seconds) ? seconds * 1000 : Date.parse(header) - Date.now(); + return Number.isFinite(ms) ? Math.max(ms, fallback) : fallback; +} + export function runInstallCommand(command: string, args: string[]): Promise { return new Promise((done) => { let stdout = ""; diff --git a/test/engine-install.test.ts b/test/engine-install.test.ts index fa61255c66..bbee8c1c53 100644 --- a/test/engine-install.test.ts +++ b/test/engine-install.test.ts @@ -8,6 +8,7 @@ import { III_INSTALL_MAX_TIME_S, III_INSTALL_RETRIES, III_INSTALL_RETRY_DELAY_S, + III_INSTALL_RETRY_MAX_TIME_S, III_RELEASE_SHA256, describeInstallFailure, expectedIiiSha256, @@ -492,6 +493,28 @@ describe("iii engine Windows zip installer", () => { const missing = (async () => { attempts++; return new Response("missing", { status: 404 }); }) as typeof fetch; await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, missing, async () => {})).rejects.toThrow("HTTP 404"); expect(attempts).toBe(1); + + const limited = [new Response("slow down", { status: 429, headers: { "retry-after": "30" } }), new Response("zip-bytes", { status: 200 })]; + const waits: number[] = []; + await downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, (async () => limited.shift()!) as typeof fetch, async (ms) => { waits.push(ms); }); + expect(waits).toEqual([30_000]); + + attempts = 0; + const tooLong = (async () => { attempts++; return new Response("later", { status: 503, headers: { "retry-after": String(III_INSTALL_RETRY_MAX_TIME_S + 1) } }); }) as typeof fetch; + await expect(downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, tooLong, async () => {})).rejects.toThrow("HTTP 503"); + expect(attempts).toBe(1); + + attempts = 0; + const dropped = (async () => { + attempts++; + if (attempts > 1) return new Response("zip-bytes", { status: 200 }); + const stream = new ReadableStream({ start(controller) { controller.error(new Error("socket hang up")); } }); + return new Response(stream, { status: 200 }); + }) as typeof fetch; + writeFileSync(dest, ""); + await downloadToFile("https://example.invalid/iii.zip", dest, {}, undefined, dropped, async () => {}); + expect(attempts).toBe(2); + expect(readFileSync(dest, "utf8")).toBe("zip-bytes"); }); it("pins a SHA-256 for both Windows assets of the pinned release", () => {