download.py

import shutil
import os
import contextlib
import http.client
import time
import urllib.error
import urllib.request
from typing import List

DOWNLOAD_ATTEMPTS_PER_URL = 3
DOWNLOAD_RETRY_DELAY_SECONDS = 5
DOWNLOAD_TIMEOUT_SECONDS = 30


def _download_once(url: str, target_file: str) -> None:
    part_file = target_file + ".part"

    try:
        with urllib.request.urlopen(
            url, timeout=DOWNLOAD_TIMEOUT_SECONDS
        ) as response, open(part_file, "wb") as out:
            shutil.copyfileobj(response, out)
            expected_size = response.headers.get("Content-Length")

        if expected_size is not None:
            actual_size = os.path.getsize(part_file)
            if actual_size != int(expected_size):
                raise OSError(f"got {actual_size} bytes out of "
                              f"{expected_size}")
    except BaseException:
        with contextlib.suppress(FileNotFoundError):
            os.remove(part_file)
        raise

    os.replace(part_file, target_file)


def download(urls: List[str], target_file: str) -> None:
    for url in urls:
        for attempt in range(1, DOWNLOAD_ATTEMPTS_PER_URL + 1):
            print(f"Downloading {url} (attempt {attempt})...")

            try:
                _download_once(url, target_file)
                return
            except urllib.error.HTTPError as err:
                print(f"Download failed: {err}")

                # The mirror answered and doesn't have the file, no point
                # asking it again
                if err.code < 500:
                    break
            except (OSError, http.client.HTTPException) as err:
                print(f"Download failed: {err}")

            if attempt != DOWNLOAD_ATTEMPTS_PER_URL:
                time.sleep(DOWNLOAD_RETRY_DELAY_SECONDS)

    raise RuntimeError(f"Failed to download {os.path.basename(target_file)} "
                       "from every mirror")