diff --git a/.github/actions/ccache-stats/action.yml b/.github/actions/ccache-stats/action.yml new file mode 100644 index 0000000..7ca2b51 --- /dev/null +++ b/.github/actions/ccache-stats/action.yml @@ -0,0 +1,23 @@ +name: Report ccache stats +description: Print ccache statistics and append them to the job summary. +inputs: + label: + description: Lane name shown in the job summary heading + required: true +runs: + using: composite + steps: + - name: ccache stats + shell: bash + run: ccache --show-stats || true + - name: Append ccache stats to summary + shell: bash + env: + CCACHE_STATS_LABEL: ${{ inputs.label }} + run: | + { + echo "### ccache stats (${CCACHE_STATS_LABEL})" + echo '```' + ccache --show-stats || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/setup-ccache/action.yml b/.github/actions/setup-ccache/action.yml new file mode 100644 index 0000000..036a7c0 --- /dev/null +++ b/.github/actions/setup-ccache/action.yml @@ -0,0 +1,36 @@ +name: Setup ccache +description: Restore a build lane's ccache directory and prime it for a fresh run. +inputs: + cache-dir: + description: ccache directory to cache between runs + required: true + max-size: + description: Ceiling passed to ccache --max-size + required: true + key-prefix: + description: Cache key prefix for this lane; the run id is appended + required: true + extra-restore-keys: + description: Further restore-key prefixes, one per line, most specific first + required: false + default: '' +runs: + using: composite + steps: + - name: Restore ccache + uses: actions/cache@v6 + with: + path: ${{ inputs.cache-dir }} + key: ${{ inputs.key-prefix }}-${{ github.run_id }} + restore-keys: | + ${{ inputs.key-prefix }}- + ${{ inputs.extra-restore-keys }} + - name: Prime ccache + shell: bash + env: + CCACHE_DIR: ${{ inputs.cache-dir }} + CCACHE_MAXSIZE: ${{ inputs.max-size }} + run: | + mkdir -p "${CCACHE_DIR}" + ccache --max-size "${CCACHE_MAXSIZE}" + ccache --zero-stats diff --git a/.github/workflows/native_release.yml b/.github/workflows/native_release.yml index 409c404..b76e433 100644 --- a/.github/workflows/native_release.yml +++ b/.github/workflows/native_release.yml @@ -183,33 +183,20 @@ jobs: echo "Using pinned Android NDK: $NDK_DIR" - name: Install build deps run: sudo apt-get update && sudo apt-get install -y ninja-build ccache - - name: Restore ccache - uses: actions/cache@v6 + - name: Set up ccache + uses: ./.github/actions/setup-ccache with: - path: ${{ env.CCACHE_DIR }} - key: ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }}-${{ github.run_id }} - restore-keys: | - ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }}- - ccache-android-${{ matrix.android_abi }}- - - name: Prime ccache - run: | - mkdir -p "${CCACHE_DIR}" - ccache --max-size "${CCACHE_MAXSIZE}" - ccache --zero-stats + cache-dir: ${{ env.CCACHE_DIR }} + max-size: ${{ env.CCACHE_MAXSIZE }} + key-prefix: ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }} + extra-restore-keys: ccache-android-${{ matrix.android_abi }}- - name: Build run: python3 tools/build.py android --abi ${{ matrix.android_abi }} --backend ${{ matrix.backend }} - - name: ccache stats + - name: Report ccache stats if: always() - run: ccache --show-stats || true - - name: Append ccache stats to summary - if: always() - run: | - { - echo "### ccache stats (android/${{ matrix.android_abi }}/${{ matrix.backend }})" - echo '```' - ccache --show-stats || true - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + uses: ./.github/actions/ccache-stats + with: + label: android/${{ matrix.android_abi }}/${{ matrix.backend }} - name: Validate Android backend artifacts run: | set -euo pipefail @@ -337,18 +324,12 @@ jobs: run: | brew untap aws/tap 2>/dev/null || true brew install ccache - - name: Restore ccache - uses: actions/cache@v6 + - name: Set up ccache + uses: ./.github/actions/setup-ccache with: - path: ${{ env.CCACHE_DIR }} - key: ccache-apple-${{ matrix.target }}-${{ github.run_id }} - restore-keys: | - ccache-apple-${{ matrix.target }}- - - name: Prime ccache - run: | - mkdir -p "${CCACHE_DIR}" - ccache --max-size "${CCACHE_MAXSIZE}" - ccache --zero-stats + cache-dir: ${{ env.CCACHE_DIR }} + max-size: ${{ env.CCACHE_MAXSIZE }} + key-prefix: ccache-apple-${{ matrix.target }} - name: Build run: python3 tools/build.py apple --target ${{ matrix.target }} - name: Validate Apple wrapper exports @@ -356,18 +337,11 @@ jobs: OUT_LIB="bin/${{ matrix.out_dir }}/libllamadart.dylib" test -f "$OUT_LIB" || { echo "Missing $OUT_LIB"; exit 1; } python3 tools/validate_exports.py --format nm --tool nm "$OUT_LIB" - - name: ccache stats + - name: Report ccache stats if: always() - run: ccache --show-stats || true - - name: Append ccache stats to summary - if: always() - run: | - { - echo "### ccache stats (apple/${{ matrix.target }})" - echo '```' - ccache --show-stats || true - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + uses: ./.github/actions/ccache-stats + with: + label: apple/${{ matrix.target }} - name: Upload uses: actions/upload-artifact@v7 with: @@ -433,56 +407,13 @@ jobs: github_token: ${{ github.token }} - name: Install build deps run: | - apt_get_update() { - for attempt in 1 2 3; do - if sudo apt-get -o Acquire::Retries=3 update; then - return 0 - fi - if [ "$attempt" = "3" ]; then - return 1 - fi - sleep $((attempt * 15)) - done - } - apt_get_install() { - for attempt in 1 2 3; do - if sudo apt-get -o Acquire::Retries=3 install -y "$@"; then - return 0 - fi - if [ "$attempt" = "3" ]; then - return 1 - fi - sleep $((attempt * 15)) - done - } + . tools/apt_retry.sh if [ "${{ matrix.arch }}" = "arm64" ]; then sudo dpkg --add-architecture arm64 source /etc/os-release - CODENAME="${VERSION_CODENAME}" - { - printf '%s\n' \ - "Types: deb" \ - "URIs: http://archive.ubuntu.com/ubuntu" \ - "Suites: ${CODENAME} ${CODENAME}-updates ${CODENAME}-backports" \ - "Components: main universe restricted multiverse" \ - "Architectures: amd64" \ - "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" \ - "" \ - "Types: deb" \ - "URIs: http://security.ubuntu.com/ubuntu" \ - "Suites: ${CODENAME}-security" \ - "Components: main universe restricted multiverse" \ - "Architectures: amd64" \ - "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" \ - "" \ - "Types: deb" \ - "URIs: http://ports.ubuntu.com/ubuntu-ports" \ - "Suites: ${CODENAME} ${CODENAME}-updates ${CODENAME}-backports ${CODENAME}-security" \ - "Components: main universe restricted multiverse" \ - "Architectures: arm64" \ - "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" - } | sudo tee /etc/apt/sources.list.d/ubuntu.sources >/dev/null + sed "s/@VERSION_CODENAME@/${VERSION_CODENAME}/g" tools/docker/ubuntu.sources.template \ + | sudo tee /etc/apt/sources.list.d/ubuntu.sources >/dev/null fi apt_get_update apt_get_install ccache ninja-build pkg-config make @@ -526,33 +457,20 @@ jobs: exit 1 } fi - - name: Restore ccache - uses: actions/cache@v6 + - name: Set up ccache + uses: ./.github/actions/setup-ccache with: - path: ${{ env.CCACHE_DIR }} - key: ccache-linux-${{ matrix.arch }}-${{ matrix.backend }}-${{ github.run_id }} - restore-keys: | - ccache-linux-${{ matrix.arch }}-${{ matrix.backend }}- - ccache-linux-${{ matrix.arch }}- - - name: Prime ccache - run: | - mkdir -p "${CCACHE_DIR}" - ccache --max-size "${CCACHE_MAXSIZE}" - ccache --zero-stats + cache-dir: ${{ env.CCACHE_DIR }} + max-size: ${{ env.CCACHE_MAXSIZE }} + key-prefix: ccache-linux-${{ matrix.arch }}-${{ matrix.backend }} + extra-restore-keys: ccache-linux-${{ matrix.arch }}- - name: Build run: python3 tools/build.py linux --arch "${{ matrix.arch }}" --backend "${{ matrix.backend }}" - - name: ccache stats + - name: Report ccache stats if: always() - run: ccache --show-stats || true - - name: Append ccache stats to summary - if: always() - run: | - { - echo "### ccache stats (linux/${{ matrix.arch }}/${{ matrix.backend }})" - echo '```' - ccache --show-stats || true - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + uses: ./.github/actions/ccache-stats + with: + label: linux/${{ matrix.arch }}/${{ matrix.backend }} - name: Validate Linux backend artifacts run: | OUT_DIR="bin/linux/${{ matrix.arch }}" @@ -695,46 +613,23 @@ jobs: github_token: ${{ github.token }} - name: Install build deps run: | - apt_get_install() { - for attempt in 1 2 3; do - if DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 install -y "$@"; then - return 0 - fi - if [ "$attempt" = "3" ]; then - return 1 - fi - sleep $((attempt * 15)) - done - } - apt_get_install build-essential binutils ccache ninja-build pkg-config python3 python3-pip rocblas-dev hipblas-dev + . tools/apt_retry.sh + DEBIAN_FRONTEND=noninteractive apt_get_install build-essential binutils ccache ninja-build pkg-config python3 python3-pip rocblas-dev hipblas-dev python3 -m pip install --upgrade cmake cmake --version - - name: Restore ccache - uses: actions/cache@v6 + - name: Set up ccache + uses: ./.github/actions/setup-ccache with: - path: ${{ env.CCACHE_DIR }} - key: ccache-linux-x64-hip-${{ github.run_id }} - restore-keys: | - ccache-linux-x64-hip- - - name: Prime ccache - run: | - mkdir -p "${CCACHE_DIR}" - ccache --max-size "${CCACHE_MAXSIZE}" - ccache --zero-stats + cache-dir: ${{ env.CCACHE_DIR }} + max-size: ${{ env.CCACHE_MAXSIZE }} + key-prefix: ccache-linux-x64-hip - name: Build run: python3 tools/build.py linux --arch x64 --backend hip - - name: ccache stats + - name: Report ccache stats if: always() - run: ccache --show-stats || true - - name: Append ccache stats to summary - if: always() - run: | - { - echo "### ccache stats (linux/x64/hip)" - echo '```' - ccache --show-stats || true - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + uses: ./.github/actions/ccache-stats + with: + label: linux/x64/hip - name: Validate Linux HIP artifacts run: | OUT_DIR="bin/linux/x64" diff --git a/tests/test_native_release_dedup.py b/tests/test_native_release_dedup.py index 4463066..827e52b 100644 --- a/tests/test_native_release_dedup.py +++ b/tests/test_native_release_dedup.py @@ -17,6 +17,89 @@ WORKFLOW = ROOT / ".github/workflows/native_release.yml" BUILD_PY = ROOT / "tools/build.py" THIS_TEST = "tests/test_native_release_dedup.py" +SETUP_CCACHE = ROOT / ".github/actions/setup-ccache/action.yml" +CCACHE_STATS = ROOT / ".github/actions/ccache-stats/action.yml" +APT_RETRY = ROOT / "tools/apt_retry.sh" +SOURCES_TEMPLATE = ROOT / "tools/docker/ubuntu.sources.template" +DOCKERFILE = ROOT / "tools/docker/linux-builder.Dockerfile" + +CODENAME_PLACEHOLDER = "@VERSION_CODENAME@" +SED_RENDER = f's/{CODENAME_PLACEHOLDER}/${{VERSION_CODENAME}}/g' + +CCACHE_LANES = { + "build-android": { + "key-prefix": "ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }}", + "extra-restore-keys": "ccache-android-${{ matrix.android_abi }}-", + "label": "android/${{ matrix.android_abi }}/${{ matrix.backend }}", + "container": None, + "runner-default-shell": "bash", + }, + "build-apple": { + "key-prefix": "ccache-apple-${{ matrix.target }}", + "extra-restore-keys": None, + "label": "apple/${{ matrix.target }}", + "container": None, + "runner-default-shell": "bash", + }, + "build-linux": { + "key-prefix": "ccache-linux-${{ matrix.arch }}-${{ matrix.backend }}", + "extra-restore-keys": "ccache-linux-${{ matrix.arch }}-", + "label": "linux/${{ matrix.arch }}/${{ matrix.backend }}", + "container": None, + "runner-default-shell": "bash", + }, + "build-linux-hip": { + "key-prefix": "ccache-linux-x64-hip", + "extra-restore-keys": None, + "label": "linux/x64/hip", + "container": "rocm/dev-ubuntu-22.04:6.1.2", + "runner-default-shell": "sh", + }, +} + +CCACHE_ACTION_SHELL = "bash" + +LONE_PIPE = re.compile(r"(? list[str]: + found = [] + marker = f" - name: {name}\n" + start = text.find(marker) + while start != -1: + body = text[start + 1 :] + end = body.find("\n - ") + found.append(body if end == -1 else body[: end + 1]) + start = text.find(marker, start + 1) + return found + + +def workflow_jobs(text: str) -> dict[str, str]: + body = text[text.index("\njobs:\n") :] + starts = [(m.group(1), m.start()) for m in re.finditer(r"\n ([a-z][\w-]*):\n", body)] + return { + name: body[pos : starts[i + 1][1] if i + 1 < len(starts) else len(body)] + for i, (name, pos) in enumerate(starts) + } + + +def steps_by_job(text: str, name: str) -> dict[str, list[str]]: + found = {} + for job, block in workflow_jobs(text).items(): + steps = workflow_steps(block, name) + if steps: + found[job] = steps + return found + + +def step_input(step: str, key: str) -> str | None: + match = re.search(rf"^ *{re.escape(key)}: (.+)$", step, re.MULTILINE) + return match.group(1) if match else None LIST_COMMAND = "python3 tools/build.py list --android-arm64-cpu-variant-libs" @@ -134,5 +217,196 @@ def test_windows_cuda_root_derives_from_the_env_var(self) -> None: ) +class CcacheCompositeActionTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = WORKFLOW.read_text() + self.setup_action = SETUP_CCACHE.read_text() + self.stats_action = CCACHE_STATS.read_text() + + def test_cache_key_and_restore_keys_derive_from_the_key_prefix(self) -> None: + self.assertIn("path: ${{ inputs.cache-dir }}", self.setup_action) + self.assertIn("key: ${{ inputs.key-prefix }}-${{ github.run_id }}", self.setup_action) + self.assertIn( + "restore-keys: |\n" + " ${{ inputs.key-prefix }}-\n" + " ${{ inputs.extra-restore-keys }}\n", + self.setup_action, + ) + + def test_priming_still_sizes_and_zeroes_the_cache(self) -> None: + self.assertIn("CCACHE_DIR: ${{ inputs.cache-dir }}", self.setup_action) + self.assertIn("CCACHE_MAXSIZE: ${{ inputs.max-size }}", self.setup_action) + self.assertIn( + 'mkdir -p "${CCACHE_DIR}"\n' + ' ccache --max-size "${CCACHE_MAXSIZE}"\n' + " ccache --zero-stats\n", + self.setup_action, + ) + + def test_summary_heading_comes_from_the_label_input(self) -> None: + self.assertIn("CCACHE_STATS_LABEL: ${{ inputs.label }}", self.stats_action) + self.assertIn('echo "### ccache stats (${CCACHE_STATS_LABEL})"', self.stats_action) + self.assertEqual(2, self.stats_action.count("ccache --show-stats || true")) + + def test_every_lane_passes_the_cache_key_it_used_before(self) -> None: + setups = steps_by_job(self.workflow, "Set up ccache") + self.assertEqual(set(CCACHE_LANES), set(setups)) + for job, lane in CCACHE_LANES.items(): + self.assertEqual(1, len(setups[job]), job) + step = setups[job][0] + self.assertIn("uses: ./.github/actions/setup-ccache", step) + self.assertEqual("${{ env.CCACHE_DIR }}", step_input(step, "cache-dir"), job) + self.assertEqual("${{ env.CCACHE_MAXSIZE }}", step_input(step, "max-size"), job) + self.assertEqual(lane["key-prefix"], step_input(step, "key-prefix"), job) + self.assertEqual( + lane["extra-restore-keys"], step_input(step, "extra-restore-keys"), job + ) + + def test_every_lane_reports_stats_under_its_old_heading(self) -> None: + reports = steps_by_job(self.workflow, "Report ccache stats") + self.assertEqual(set(CCACHE_LANES), set(reports)) + for job, lane in CCACHE_LANES.items(): + self.assertEqual(1, len(reports[job]), job) + step = reports[job][0] + self.assertIn("if: always()", step) + self.assertIn("uses: ./.github/actions/ccache-stats", step) + self.assertEqual(lane["label"], step_input(step, "label"), job) + + def test_no_lane_keeps_its_own_copy_of_the_scaffolding(self) -> None: + self.assertNotIn("ccache --zero-stats", self.workflow) + self.assertNotIn("ccache --show-stats", self.workflow) + self.assertNotIn("actions/cache@v6", self.workflow) + self.assertNotIn("CCACHE_MAXSIZE}", self.workflow) + + def test_windows_keeps_its_own_sccache_setup(self) -> None: + self.assertIn("uses: mozilla-actions/sccache-action@v0.0.11", self.workflow) + self.assertIn("SCCACHE_GHA_ENABLED=true", self.workflow) + self.assertNotIn("sccache", self.setup_action) + + +class CcacheStepShellTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = WORKFLOW.read_text() + self.actions = { + "setup-ccache": SETUP_CCACHE.read_text(), + "ccache-stats": CCACHE_STATS.read_text(), + } + + def test_every_extracted_step_declares_one_shell(self) -> None: + for name, text in self.actions.items(): + shells = re.findall(r"^ *shell: (.+)$", text, re.MULTILINE) + runs = len(re.findall(r"^ *run: ", text, re.MULTILINE)) + self.assertEqual([CCACHE_ACTION_SHELL] * runs, shells, name) + + def test_each_lane_container_is_pinned(self) -> None: + jobs = workflow_jobs(self.workflow) + for job, lane in CCACHE_LANES.items(): + match = re.search(r"^ container: (.+)$", jobs[job], re.MULTILINE) + self.assertEqual(lane["container"], match.group(1) if match else None, job) + + def test_containerised_lanes_are_the_ones_that_defaulted_to_sh(self) -> None: + for job, lane in CCACHE_LANES.items(): + expected = "sh" if lane["container"] else "bash" + self.assertEqual(expected, lane["runner-default-shell"], job) + + def test_no_extracted_script_contains_a_pipeline(self) -> None: + for name, text in self.actions.items(): + for number, line in enumerate(text.splitlines(), 1): + self.assertIsNone( + LONE_PIPE.search(line.rstrip()), f"{name}:{number}: {line}" + ) + + +class AptRetryScriptTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = WORKFLOW.read_text() + self.script = APT_RETRY.read_text() + + def test_script_keeps_the_retry_ladder_and_apt_options(self) -> None: + self.assertIn("apt-get -o Acquire::Retries=3 \"$@\"", self.script) + self.assertIn("for attempt in 1 2 3; do", self.script) + self.assertIn("sleep $((attempt * 15))", self.script) + self.assertIn("_apt_get_retry update", self.script) + self.assertIn('_apt_get_retry install -y "$@"', self.script) + + def test_script_uses_sudo_only_when_not_root(self) -> None: + self.assertIn('if [ "$(id -u)" -eq 0 ]; then', self.script) + self.assertIn("sudo apt-get -o Acquire::Retries=3", self.script) + + def test_post_checkout_steps_source_the_script(self) -> None: + self.assertEqual(2, self.workflow.count(". tools/apt_retry.sh")) + + def test_only_the_pre_checkout_step_still_declares_the_helpers(self) -> None: + self.assertEqual(1, self.workflow.count("apt_get_install() {")) + self.assertEqual(1, self.workflow.count("apt_get_update() {")) + install_git = workflow_steps(self.workflow, "Install Git") + self.assertEqual(1, len(install_git)) + self.assertIn("apt_get_install() {", install_git[0]) + self.assertLess( + self.workflow.index("apt_get_install() {"), + self.workflow.index("uses: ./.github/actions/checkout-llama-ref\n", self.workflow.index("build-linux-hip:")), + ) + + def test_container_step_installs_noninteractively_without_exporting_it(self) -> None: + hip_deps = steps_by_job(self.workflow, "Install build deps")["build-linux-hip"][0] + self.assertEqual( + [HIP_APT_INSTALL], + [line.strip() for line in hip_deps.splitlines() if "DEBIAN_FRONTEND" in line], + ) + self.assertIn("python3 -m pip install --upgrade cmake", hip_deps) + + def test_the_hosted_linux_lane_sets_no_apt_frontend(self) -> None: + linux_deps = steps_by_job(self.workflow, "Install build deps")["build-linux"][0] + self.assertNotIn("DEBIAN_FRONTEND", linux_deps) + self.assertNotIn("DEBIAN_FRONTEND", APT_RETRY.read_text()) + + +class UbuntuSourcesTemplateTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = WORKFLOW.read_text() + self.template = SOURCES_TEMPLATE.read_text() + self.dockerfile = DOCKERFILE.read_text() + + def render(self, codename: str) -> str: + return self.template.replace(CODENAME_PLACEHOLDER, codename) + + def test_workflow_and_dockerfile_render_the_same_template(self) -> None: + self.assertFalse((ROOT / "tools/docker/ubuntu.sources").exists()) + self.assertIn(f'sed "{SED_RENDER}" tools/docker/ubuntu.sources.template', self.workflow) + self.assertIn(f'sed "{SED_RENDER}" /etc/apt/ubuntu.sources.template', self.dockerfile) + self.assertIn(". /etc/os-release", self.dockerfile) + self.assertIn("source /etc/os-release", self.workflow) + self.assertIn("COPY ubuntu.sources.template /etc/apt/ubuntu.sources.template", self.dockerfile) + + def test_no_codename_is_hardcoded(self) -> None: + self.assertNotIn("noble", self.template) + self.assertNotIn("noble", self.workflow) + self.assertNotIn("Signed-By", self.workflow) + + def test_rendered_list_keeps_all_three_suites(self) -> None: + rendered = self.render("plucky") + self.assertNotIn("@", rendered) + self.assertEqual( + [ + "Suites: plucky plucky-updates plucky-backports", + "Suites: plucky-security", + "Suites: plucky plucky-updates plucky-backports plucky-security", + ], + [line for line in rendered.splitlines() if line.startswith("Suites:")], + ) + self.assertEqual( + ["Architectures: amd64", "Architectures: amd64", "Architectures: arm64"], + [line for line in rendered.splitlines() if line.startswith("Architectures:")], + ) + self.assertEqual( + [ + "URIs: http://archive.ubuntu.com/ubuntu", + "URIs: http://security.ubuntu.com/ubuntu", + "URIs: http://ports.ubuntu.com/ubuntu-ports", + ], + [line for line in rendered.splitlines() if line.startswith("URIs:")], + ) + + if __name__ == "__main__": unittest.main() diff --git a/tools/apt_retry.sh b/tools/apt_retry.sh new file mode 100644 index 0000000..2eb34a5 --- /dev/null +++ b/tools/apt_retry.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +_apt_get() { + if [ "$(id -u)" -eq 0 ]; then + apt-get -o Acquire::Retries=3 "$@" + else + sudo apt-get -o Acquire::Retries=3 "$@" + fi +} + +_apt_get_retry() { + for attempt in 1 2 3; do + if _apt_get "$@"; then + return 0 + fi + if [ "$attempt" = "3" ]; then + return 1 + fi + sleep $((attempt * 15)) + done +} + +apt_get_update() { + _apt_get_retry update +} + +apt_get_install() { + _apt_get_retry install -y "$@" +} diff --git a/tools/docker/linux-builder.Dockerfile b/tools/docker/linux-builder.Dockerfile index 8fcae47..2e40a48 100644 --- a/tools/docker/linux-builder.Dockerfile +++ b/tools/docker/linux-builder.Dockerfile @@ -4,9 +4,13 @@ FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu24.04 ENV DEBIAN_FRONTEND=noninteractive -COPY ubuntu.sources /etc/apt/sources.list.d/ubuntu.sources +COPY ubuntu.sources.template /etc/apt/ubuntu.sources.template -RUN dpkg --add-architecture arm64 \ +RUN . /etc/os-release \ + && sed "s/@VERSION_CODENAME@/${VERSION_CODENAME}/g" /etc/apt/ubuntu.sources.template \ + > /etc/apt/sources.list.d/ubuntu.sources \ + && rm /etc/apt/ubuntu.sources.template \ + && dpkg --add-architecture arm64 \ && apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ diff --git a/tools/docker/ubuntu.sources b/tools/docker/ubuntu.sources.template similarity index 70% rename from tools/docker/ubuntu.sources rename to tools/docker/ubuntu.sources.template index 52bc124..26bc3b6 100644 --- a/tools/docker/ubuntu.sources +++ b/tools/docker/ubuntu.sources.template @@ -1,20 +1,20 @@ Types: deb URIs: http://archive.ubuntu.com/ubuntu -Suites: noble noble-updates noble-backports +Suites: @VERSION_CODENAME@ @VERSION_CODENAME@-updates @VERSION_CODENAME@-backports Components: main universe restricted multiverse Architectures: amd64 Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg Types: deb URIs: http://security.ubuntu.com/ubuntu -Suites: noble-security +Suites: @VERSION_CODENAME@-security Components: main universe restricted multiverse Architectures: amd64 Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg Types: deb URIs: http://ports.ubuntu.com/ubuntu-ports -Suites: noble noble-updates noble-backports noble-security +Suites: @VERSION_CODENAME@ @VERSION_CODENAME@-updates @VERSION_CODENAME@-backports @VERSION_CODENAME@-security Components: main universe restricted multiverse Architectures: arm64 Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg