fix runner setup

packer/cirun-runner.pkr.hcl

@@ -100,28 +100,25 @@ build {
     "source.amazon-ebs.ubuntu"
   ]
 
+  # System preparation and shared prerequisites must run first so the later
+  # installers never hit a missing tool (unzip, gcc, gnupg, ...).
   provisioner "shell" {
-    script = "scripts/disable-upgrades.sh"
+    script = "scripts/01_setup-system.sh"
   }
 
   provisioner "shell" {
-    script = "scripts/install-docker.sh"
+    script = "scripts/02_install-docker.sh"
   }
 
   provisioner "shell" {
-    script = "scripts/install-nvidia-drivers.sh"
+    script = "scripts/03_install-nvidia.sh"
   }
 
   provisioner "shell" {
-    script = "scripts/preinstall-tools.sh"
+    script = "scripts/04_install-ci-tools.sh"
   }
 
   provisioner "shell" {
-    inline = [
-      "sudo apt-get update",
-      "sudo apt-get upgrade -y",
-      "sudo apt-get autoremove -y",
-      "sudo apt-get autoclean"
-    ]
+    script = "scripts/99_cleanup.sh"
   }
 }

packer/scripts/01_setup-system.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# Prepare the base system: disable unattended upgrades and install every
+# common package the later provisioning steps rely on. Installing all shared
+# prerequisites up front keeps the individual installer scripts from failing
+# on missing tools (e.g. unzip, gcc) and gives us a single apt update/upgrade.
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+echo "==> Disabling automatic upgrades and updates"
+# Stop, disable and mask the periodic apt units so they cannot race with our
+# provisioning (which would hold the dpkg lock) or mutate the image post-build.
+apt_units=(apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service)
+for unit in "${apt_units[@]}"; do
+    sudo systemctl stop "$unit" 2>/dev/null || true
+    sudo systemctl disable "$unit" 2>/dev/null || true
+    sudo systemctl mask "$unit" 2>/dev/null || true
+done
+
+sudo apt-get remove -y unattended-upgrades || true
+sudo tee /etc/apt/apt.conf.d/20auto-upgrades > /dev/null <<'EOF'
+APT::Periodic::Update-Package-Lists "0";
+APT::Periodic::Unattended-Upgrade "0";
+APT::Periodic::Download-Upgradeable-Packages "0";
+APT::Periodic::AutocleanInterval "0";
+EOF
+
+echo "==> Updating the package index and upgrading the base image"
+# The Ubuntu 24.04 base AMI ships with an older snapshot of packages (notably
+# the gnupg/libksba8 stack). Upgrading first avoids "unmet dependencies"
+# errors when later steps pull in newer packages.
+sudo apt-get update
+sudo apt-get upgrade -y
+
+echo "==> Installing common prerequisites"
+sudo apt-get install -y \
+    apt-transport-https \
+    build-essential \
+    ca-certificates \
+    curl \
+    git \
+    gnupg \
+    jq \
+    "linux-headers-$(uname -r)" \
+    lsb-release \
+    pipx \
+    python3-pip \
+    python3-venv \
+    software-properties-common \
+    unzip \
+    wget \
+    xvfb \
+    zip
+
+# Shared keyring directory for the third-party apt repositories added later.
+sudo install -m 0755 -d /etc/apt/keyrings
+
+echo "==> System preparation complete"

packer/scripts/02_install-docker.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Install Docker Engine plus the buildx and compose plugins from Docker's
+# official apt repository. Prerequisites (curl, gnupg, ca-certificates,
+# lsb-release and /etc/apt/keyrings) are provided by setup-system.sh.
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+echo "==> Adding Docker's official apt repository"
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | \
+    sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+sudo chmod a+r /etc/apt/keyrings/docker.gpg
+
+echo \
+  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+echo "==> Installing Docker Engine"
+sudo apt-get update
+sudo apt-get install -y \
+    containerd.io \
+    docker-buildx-plugin \
+    docker-ce \
+    docker-ce-cli \
+    docker-compose-plugin
+
+# Allow the default user to use Docker without sudo.
+sudo usermod -aG docker ubuntu
+
+sudo systemctl enable docker
+sudo systemctl start docker
+
+docker --version
+echo "==> Docker installation complete"

packer/scripts/03_install-nvidia.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Install the NVIDIA driver, the CUDA toolkit and the NVIDIA Container Toolkit.
+# build-essential and the matching linux-headers come from setup-system.sh;
+# Docker (configured here for the NVIDIA runtime) comes from install-docker.sh.
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+# This AMI targets Ubuntu 24.04 exclusively.
+DISTRO="ubuntu2404"
+
+echo "==> System information"
+uname -r
+lspci | grep -i nvidia || echo "Note: no NVIDIA GPU visible in the build environment"
+
+UBUNTU_VERSION="$(lsb_release -rs)"
+if [[ "$UBUNTU_VERSION" != "24.04" ]]; then
+    echo "WARNING: expected Ubuntu 24.04 but found ${UBUNTU_VERSION}; continuing with ${DISTRO} repositories"
+fi
+
+echo "==> Installing the recommended NVIDIA driver"
+sudo apt-get install -y ubuntu-drivers-common
+sudo ubuntu-drivers autoinstall
+
+# Fall back to a known-good driver version if autoinstall did not provide one.
+if ! command -v nvidia-smi >/dev/null 2>&1; then
+    echo "==> autoinstall provided no driver; falling back to nvidia-driver-535"
+    sudo apt-get install -y nvidia-driver-535 nvidia-utils-535
+fi
+
+echo "==> Installing the CUDA toolkit"
+# Best effort: the CUDA toolkit is large and occasionally unavailable. A
+# driver-only image is still usable, so do not fail the build if it errors.
+if ! command -v nvcc >/dev/null 2>&1; then
+    keyring="$(mktemp -d)/cuda-keyring.deb"
+    for attempt in 1 2 3; do
+        if wget -O "$keyring" \
+            "https://developer.download.nvidia.com/compute/cuda/repos/${DISTRO}/x86_64/cuda-keyring_1.1-1_all.deb"; then
+            break
+        fi
+        echo "CUDA keyring download attempt ${attempt} failed; retrying in 5s..."
+        sleep 5
+    done
+    sudo dpkg -i "$keyring"
+    rm -f "$keyring"
+
+    for attempt in 1 2 3; do
+        sudo apt-get update && break
+        echo "apt-get update attempt ${attempt} failed; retrying in 10s..."
+        sleep 10
+    done
+
+    sudo apt-get install -y cuda-toolkit || \
+        echo "WARNING: CUDA toolkit installation failed; continuing with driver only"
+fi
+
+echo "==> Configuring the CUDA environment"
+# Expose CUDA on PATH for all login shells (replaces the previous per-user
+# .bashrc / /etc/environment edits with a single system-wide profile drop-in).
+CUDA_DIR="$(ls -d /usr/local/cuda-* 2>/dev/null | head -1 || true)"
+CUDA_DIR="${CUDA_DIR:-/usr/local/cuda}"
+sudo tee /etc/profile.d/cuda.sh > /dev/null <<EOF
+export PATH="${CUDA_DIR}/bin:\$PATH"
+export LD_LIBRARY_PATH="${CUDA_DIR}/lib64:\${LD_LIBRARY_PATH:-}"
+EOF
+
+echo "==> Installing the NVIDIA Container Toolkit"
+# Best effort, but uses the official repository so it installs cleanly when the
+# network is healthy.
+curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | \
+    sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
+    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
+    sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list > /dev/null
+
+if sudo apt-get update && sudo apt-get install -y nvidia-container-toolkit; then
+    sudo nvidia-ctk runtime configure --runtime=docker
+    sudo systemctl restart docker || echo "Docker will pick up the NVIDIA runtime on next start"
+else
+    echo "WARNING: NVIDIA Container Toolkit installation failed; continuing without it"
+fi
+
+echo "==> Verifying installation"
+command -v nvcc || echo "nvcc not yet on PATH (available after re-login / reboot)"
+command -v nvidia-smi || echo "nvidia-smi not found (driver loads after reboot)"
+
+echo "==> NVIDIA installation complete (a reboot may be required for the driver)"

packer/scripts/04_install-ci-tools.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Install the tooling the CI runners expect that is not in the base repos:
+# the runnerx service account, AWS CLI v2, GitHub CLI, uv, Node.js/npm and
+# Miniconda. Common utilities (jq, git, unzip, python3-pip/venv, pipx, ...)
+# are installed by setup-system.sh.
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+echo "==> Creating the runnerx service account"
+if ! id runnerx &>/dev/null; then
+    sudo useradd -m -s /bin/bash runnerx
+fi
+sudo usermod -aG docker runnerx
+
+echo "==> Installing Node.js and npm"
+sudo apt-get install -y nodejs npm
+
+echo "==> Installing the GitHub CLI"
+# gh is not in the default Ubuntu repos; add its official apt repository.
+curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | \
+    sudo gpg --dearmor -o /etc/apt/keyrings/githubcli-archive-keyring.gpg
+sudo chmod a+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | \
+    sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+sudo apt-get update
+sudo apt-get install -y gh
+
+echo "==> Installing uv"
+# uv is not packaged for apt; install it system-wide so every user can run it.
+curl -LsSf https://astral.sh/uv/install.sh | sudo env UV_INSTALL_DIR=/usr/local/bin sh
+
+echo "==> Installing the AWS CLI v2"
+tmp_dir="$(mktemp -d)"
+curl -fsSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "${tmp_dir}/awscliv2.zip"
+unzip -q "${tmp_dir}/awscliv2.zip" -d "${tmp_dir}"
+sudo "${tmp_dir}/aws/install" --update
+rm -rf "${tmp_dir}"
+
+echo "==> Installing Miniconda for the runnerx user"
+sudo -u runnerx bash -c '
+    set -euo pipefail
+    wget -qO /home/runnerx/miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+    bash /home/runnerx/miniconda.sh -b -p /home/runnerx/miniconda3
+    rm -f /home/runnerx/miniconda.sh
+    echo "export PATH=\"/home/runnerx/miniconda3/bin:\$PATH\"" >> /home/runnerx/.bashrc
+'
+
+echo "==> Creating the /opt/runnerx working directory"
+sudo mkdir -p /opt/runnerx
+sudo chown ubuntu:ubuntu /opt/runnerx
+
+echo "==> CI tools installation complete"

packer/scripts/99_cleanup.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+# Final pass: apply any remaining upgrades and trim cached package data so the
+# resulting AMI is as small as possible.
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+
+sudo apt-get update
+sudo apt-get upgrade -y
+sudo apt-get autoremove -y
+sudo apt-get autoclean
+sudo rm -rf /var/lib/apt/lists/*
+
+echo "==> Cleanup complete"