From df4dd124cef0aa464466df0e16f384932b553c83 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Tue, 22 Sep 2026 08:44:58 -0400 Subject: [PATCH 1/3] ci: extract the repeated ccache and apt scaffolding from native_release The four ccache build lanes each carried the same restore/prime/stats/summary steps. They now call .github/actions/setup-ccache and .github/actions/ccache-stats with the cache-key prefix and summary label each lane already used; the cache key, restore keys, prime commands and summary heading are unchanged. The apt retry helpers move to tools/apt_retry.sh, sourced by the two install steps that run after checkout. The helper picks sudo by uid instead of hardcoding it, so the container lane and the hosted runner share one copy. The arm64 sources list is rendered from tools/docker/ubuntu.sources.template. The Docker builder image renders the same template from its own os-release instead of shipping a noble-pinned copy. --- .github/actions/ccache-stats/action.yml | 23 ++ .github/actions/setup-ccache/action.yml | 36 ++++ .github/workflows/native_release.yml | 191 ++++------------- tests/test_native_release_dedup.py | 199 ++++++++++++++++++ tools/apt_retry.sh | 29 +++ tools/docker/linux-builder.Dockerfile | 8 +- ...ubuntu.sources => ubuntu.sources.template} | 6 +- 7 files changed, 340 insertions(+), 152 deletions(-) create mode 100644 .github/actions/ccache-stats/action.yml create mode 100644 .github/actions/setup-ccache/action.yml create mode 100644 tools/apt_retry.sh rename tools/docker/{ubuntu.sources => ubuntu.sources.template} (70%) diff --git a/.github/actions/ccache-stats/action.yml b/.github/actions/ccache-stats/action.yml new file mode 100644 index 0000000..7ca2b51 --- /dev/null +++ b/.github/actions/ccache-stats/action.yml @@ -0,0 +1,23 @@ +name: Report ccache stats +description: Print ccache statistics and append them to the job summary. +inputs: + label: + description: Lane name shown in the job summary heading + required: true +runs: + using: composite + steps: + - name: ccache stats + shell: bash + run: ccache --show-stats || true + - name: Append ccache stats to summary + shell: bash + env: + CCACHE_STATS_LABEL: ${{ inputs.label }} + run: | + { + echo "### ccache stats (${CCACHE_STATS_LABEL})" + echo '```' + ccache --show-stats || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/actions/setup-ccache/action.yml b/.github/actions/setup-ccache/action.yml new file mode 100644 index 0000000..036a7c0 --- /dev/null +++ b/.github/actions/setup-ccache/action.yml @@ -0,0 +1,36 @@ +name: Setup ccache +description: Restore a build lane's ccache directory and prime it for a fresh run. +inputs: + cache-dir: + description: ccache directory to cache between runs + required: true + max-size: + description: Ceiling passed to ccache --max-size + required: true + key-prefix: + description: Cache key prefix for this lane; the run id is appended + required: true + extra-restore-keys: + description: Further restore-key prefixes, one per line, most specific first + required: false + default: '' +runs: + using: composite + steps: + - name: Restore ccache + uses: actions/cache@v6 + with: + path: ${{ inputs.cache-dir }} + key: ${{ inputs.key-prefix }}-${{ github.run_id }} + restore-keys: | + ${{ inputs.key-prefix }}- + ${{ inputs.extra-restore-keys }} + - name: Prime ccache + shell: bash + env: + CCACHE_DIR: ${{ inputs.cache-dir }} + CCACHE_MAXSIZE: ${{ inputs.max-size }} + run: | + mkdir -p "${CCACHE_DIR}" + ccache --max-size "${CCACHE_MAXSIZE}" + ccache --zero-stats diff --git a/.github/workflows/native_release.yml b/.github/workflows/native_release.yml index 409c404..d444c03 100644 --- a/.github/workflows/native_release.yml +++ b/.github/workflows/native_release.yml @@ -183,33 +183,20 @@ jobs: echo "Using pinned Android NDK: $NDK_DIR" - name: Install build deps run: sudo apt-get update && sudo apt-get install -y ninja-build ccache - - name: Restore ccache - uses: actions/cache@v6 + - name: Set up ccache + uses: ./.github/actions/setup-ccache with: - path: ${{ env.CCACHE_DIR }} - key: ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }}-${{ github.run_id }} - restore-keys: | - ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }}- - ccache-android-${{ matrix.android_abi }}- - - name: Prime ccache - run: | - mkdir -p "${CCACHE_DIR}" - ccache --max-size "${CCACHE_MAXSIZE}" - ccache --zero-stats + cache-dir: ${{ env.CCACHE_DIR }} + max-size: ${{ env.CCACHE_MAXSIZE }} + key-prefix: ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }} + extra-restore-keys: ccache-android-${{ matrix.android_abi }}- - name: Build run: python3 tools/build.py android --abi ${{ matrix.android_abi }} --backend ${{ matrix.backend }} - - name: ccache stats + - name: Report ccache stats if: always() - run: ccache --show-stats || true - - name: Append ccache stats to summary - if: always() - run: | - { - echo "### ccache stats (android/${{ matrix.android_abi }}/${{ matrix.backend }})" - echo '```' - ccache --show-stats || true - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + uses: ./.github/actions/ccache-stats + with: + label: android/${{ matrix.android_abi }}/${{ matrix.backend }} - name: Validate Android backend artifacts run: | set -euo pipefail @@ -337,18 +324,12 @@ jobs: run: | brew untap aws/tap 2>/dev/null || true brew install ccache - - name: Restore ccache - uses: actions/cache@v6 + - name: Set up ccache + uses: ./.github/actions/setup-ccache with: - path: ${{ env.CCACHE_DIR }} - key: ccache-apple-${{ matrix.target }}-${{ github.run_id }} - restore-keys: | - ccache-apple-${{ matrix.target }}- - - name: Prime ccache - run: | - mkdir -p "${CCACHE_DIR}" - ccache --max-size "${CCACHE_MAXSIZE}" - ccache --zero-stats + cache-dir: ${{ env.CCACHE_DIR }} + max-size: ${{ env.CCACHE_MAXSIZE }} + key-prefix: ccache-apple-${{ matrix.target }} - name: Build run: python3 tools/build.py apple --target ${{ matrix.target }} - name: Validate Apple wrapper exports @@ -356,18 +337,11 @@ jobs: OUT_LIB="bin/${{ matrix.out_dir }}/libllamadart.dylib" test -f "$OUT_LIB" || { echo "Missing $OUT_LIB"; exit 1; } python3 tools/validate_exports.py --format nm --tool nm "$OUT_LIB" - - name: ccache stats + - name: Report ccache stats if: always() - run: ccache --show-stats || true - - name: Append ccache stats to summary - if: always() - run: | - { - echo "### ccache stats (apple/${{ matrix.target }})" - echo '```' - ccache --show-stats || true - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + uses: ./.github/actions/ccache-stats + with: + label: apple/${{ matrix.target }} - name: Upload uses: actions/upload-artifact@v7 with: @@ -433,56 +407,13 @@ jobs: github_token: ${{ github.token }} - name: Install build deps run: | - apt_get_update() { - for attempt in 1 2 3; do - if sudo apt-get -o Acquire::Retries=3 update; then - return 0 - fi - if [ "$attempt" = "3" ]; then - return 1 - fi - sleep $((attempt * 15)) - done - } - apt_get_install() { - for attempt in 1 2 3; do - if sudo apt-get -o Acquire::Retries=3 install -y "$@"; then - return 0 - fi - if [ "$attempt" = "3" ]; then - return 1 - fi - sleep $((attempt * 15)) - done - } + . tools/apt_retry.sh if [ "${{ matrix.arch }}" = "arm64" ]; then sudo dpkg --add-architecture arm64 source /etc/os-release - CODENAME="${VERSION_CODENAME}" - { - printf '%s\n' \ - "Types: deb" \ - "URIs: http://archive.ubuntu.com/ubuntu" \ - "Suites: ${CODENAME} ${CODENAME}-updates ${CODENAME}-backports" \ - "Components: main universe restricted multiverse" \ - "Architectures: amd64" \ - "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" \ - "" \ - "Types: deb" \ - "URIs: http://security.ubuntu.com/ubuntu" \ - "Suites: ${CODENAME}-security" \ - "Components: main universe restricted multiverse" \ - "Architectures: amd64" \ - "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" \ - "" \ - "Types: deb" \ - "URIs: http://ports.ubuntu.com/ubuntu-ports" \ - "Suites: ${CODENAME} ${CODENAME}-updates ${CODENAME}-backports ${CODENAME}-security" \ - "Components: main universe restricted multiverse" \ - "Architectures: arm64" \ - "Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg" - } | sudo tee /etc/apt/sources.list.d/ubuntu.sources >/dev/null + sed "s/@VERSION_CODENAME@/${VERSION_CODENAME}/g" tools/docker/ubuntu.sources.template \ + | sudo tee /etc/apt/sources.list.d/ubuntu.sources >/dev/null fi apt_get_update apt_get_install ccache ninja-build pkg-config make @@ -526,33 +457,20 @@ jobs: exit 1 } fi - - name: Restore ccache - uses: actions/cache@v6 + - name: Set up ccache + uses: ./.github/actions/setup-ccache with: - path: ${{ env.CCACHE_DIR }} - key: ccache-linux-${{ matrix.arch }}-${{ matrix.backend }}-${{ github.run_id }} - restore-keys: | - ccache-linux-${{ matrix.arch }}-${{ matrix.backend }}- - ccache-linux-${{ matrix.arch }}- - - name: Prime ccache - run: | - mkdir -p "${CCACHE_DIR}" - ccache --max-size "${CCACHE_MAXSIZE}" - ccache --zero-stats + cache-dir: ${{ env.CCACHE_DIR }} + max-size: ${{ env.CCACHE_MAXSIZE }} + key-prefix: ccache-linux-${{ matrix.arch }}-${{ matrix.backend }} + extra-restore-keys: ccache-linux-${{ matrix.arch }}- - name: Build run: python3 tools/build.py linux --arch "${{ matrix.arch }}" --backend "${{ matrix.backend }}" - - name: ccache stats - if: always() - run: ccache --show-stats || true - - name: Append ccache stats to summary + - name: Report ccache stats if: always() - run: | - { - echo "### ccache stats (linux/${{ matrix.arch }}/${{ matrix.backend }})" - echo '```' - ccache --show-stats || true - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + uses: ./.github/actions/ccache-stats + with: + label: linux/${{ matrix.arch }}/${{ matrix.backend }} - name: Validate Linux backend artifacts run: | OUT_DIR="bin/linux/${{ matrix.arch }}" @@ -694,47 +612,26 @@ jobs: expected_commit: ${{ needs.resolve-tag.outputs.llama_cpp_commit }} github_token: ${{ github.token }} - name: Install build deps + env: + DEBIAN_FRONTEND: noninteractive run: | - apt_get_install() { - for attempt in 1 2 3; do - if DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::Retries=3 install -y "$@"; then - return 0 - fi - if [ "$attempt" = "3" ]; then - return 1 - fi - sleep $((attempt * 15)) - done - } + . tools/apt_retry.sh apt_get_install build-essential binutils ccache ninja-build pkg-config python3 python3-pip rocblas-dev hipblas-dev python3 -m pip install --upgrade cmake cmake --version - - name: Restore ccache - uses: actions/cache@v6 + - name: Set up ccache + uses: ./.github/actions/setup-ccache with: - path: ${{ env.CCACHE_DIR }} - key: ccache-linux-x64-hip-${{ github.run_id }} - restore-keys: | - ccache-linux-x64-hip- - - name: Prime ccache - run: | - mkdir -p "${CCACHE_DIR}" - ccache --max-size "${CCACHE_MAXSIZE}" - ccache --zero-stats + cache-dir: ${{ env.CCACHE_DIR }} + max-size: ${{ env.CCACHE_MAXSIZE }} + key-prefix: ccache-linux-x64-hip - name: Build run: python3 tools/build.py linux --arch x64 --backend hip - - name: ccache stats + - name: Report ccache stats if: always() - run: ccache --show-stats || true - - name: Append ccache stats to summary - if: always() - run: | - { - echo "### ccache stats (linux/x64/hip)" - echo '```' - ccache --show-stats || true - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + uses: ./.github/actions/ccache-stats + with: + label: linux/x64/hip - name: Validate Linux HIP artifacts run: | OUT_DIR="bin/linux/x64" diff --git a/tests/test_native_release_dedup.py b/tests/test_native_release_dedup.py index 4463066..9743b6c 100644 --- a/tests/test_native_release_dedup.py +++ b/tests/test_native_release_dedup.py @@ -17,6 +17,49 @@ WORKFLOW = ROOT / ".github/workflows/native_release.yml" BUILD_PY = ROOT / "tools/build.py" THIS_TEST = "tests/test_native_release_dedup.py" +SETUP_CCACHE = ROOT / ".github/actions/setup-ccache/action.yml" +CCACHE_STATS = ROOT / ".github/actions/ccache-stats/action.yml" +APT_RETRY = ROOT / "tools/apt_retry.sh" +SOURCES_TEMPLATE = ROOT / "tools/docker/ubuntu.sources.template" +DOCKERFILE = ROOT / "tools/docker/linux-builder.Dockerfile" + +CODENAME_PLACEHOLDER = "@VERSION_CODENAME@" +SED_RENDER = f's/{CODENAME_PLACEHOLDER}/${{VERSION_CODENAME}}/g' + +CCACHE_LANES = { + "android": { + "key-prefix": "ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }}", + "extra-restore-keys": "ccache-android-${{ matrix.android_abi }}-", + "label": "android/${{ matrix.android_abi }}/${{ matrix.backend }}", + }, + "apple": { + "key-prefix": "ccache-apple-${{ matrix.target }}", + "extra-restore-keys": None, + "label": "apple/${{ matrix.target }}", + }, + "linux": { + "key-prefix": "ccache-linux-${{ matrix.arch }}-${{ matrix.backend }}", + "extra-restore-keys": "ccache-linux-${{ matrix.arch }}-", + "label": "linux/${{ matrix.arch }}/${{ matrix.backend }}", + }, + "linux-hip": { + "key-prefix": "ccache-linux-x64-hip", + "extra-restore-keys": None, + "label": "linux/x64/hip", + }, +} + + +def workflow_steps(text: str, name: str) -> list[str]: + found = [] + marker = f" - name: {name}\n" + start = text.find(marker) + while start != -1: + body = text[start + 1 :] + end = body.find("\n - ") + found.append(body if end == -1 else body[: end + 1]) + start = text.find(marker, start + 1) + return found LIST_COMMAND = "python3 tools/build.py list --android-arm64-cpu-variant-libs" @@ -134,5 +177,161 @@ def test_windows_cuda_root_derives_from_the_env_var(self) -> None: ) +class CcacheCompositeActionTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = WORKFLOW.read_text() + self.setup_action = SETUP_CCACHE.read_text() + self.stats_action = CCACHE_STATS.read_text() + + def test_cache_key_and_restore_keys_derive_from_the_key_prefix(self) -> None: + self.assertIn("path: ${{ inputs.cache-dir }}", self.setup_action) + self.assertIn("key: ${{ inputs.key-prefix }}-${{ github.run_id }}", self.setup_action) + self.assertIn( + "restore-keys: |\n" + " ${{ inputs.key-prefix }}-\n" + " ${{ inputs.extra-restore-keys }}\n", + self.setup_action, + ) + + def test_priming_still_sizes_and_zeroes_the_cache(self) -> None: + self.assertIn("CCACHE_DIR: ${{ inputs.cache-dir }}", self.setup_action) + self.assertIn("CCACHE_MAXSIZE: ${{ inputs.max-size }}", self.setup_action) + self.assertIn( + 'mkdir -p "${CCACHE_DIR}"\n' + ' ccache --max-size "${CCACHE_MAXSIZE}"\n' + " ccache --zero-stats\n", + self.setup_action, + ) + + def test_summary_heading_comes_from_the_label_input(self) -> None: + self.assertIn("CCACHE_STATS_LABEL: ${{ inputs.label }}", self.stats_action) + self.assertIn('echo "### ccache stats (${CCACHE_STATS_LABEL})"', self.stats_action) + self.assertEqual(2, self.stats_action.count("ccache --show-stats || true")) + + def test_every_lane_passes_the_cache_key_it_used_before(self) -> None: + setups = workflow_steps(self.workflow, "Set up ccache") + self.assertEqual(len(CCACHE_LANES), len(setups)) + for step in setups: + self.assertIn("uses: ./.github/actions/setup-ccache", step) + self.assertIn("cache-dir: ${{ env.CCACHE_DIR }}", step) + self.assertIn("max-size: ${{ env.CCACHE_MAXSIZE }}", step) + prefixes = {lane["key-prefix"] for lane in CCACHE_LANES.values()} + self.assertEqual( + prefixes, + {re.search(r"key-prefix: (.+)\n", step).group(1) for step in setups}, + ) + extras = {lane["extra-restore-keys"] for lane in CCACHE_LANES.values()} + found_extras = set() + for step in setups: + match = re.search(r"extra-restore-keys: (.+)\n", step) + found_extras.add(match.group(1) if match else None) + self.assertEqual(extras, found_extras) + + def test_every_lane_reports_stats_under_its_old_heading(self) -> None: + reports = workflow_steps(self.workflow, "Report ccache stats") + self.assertEqual(len(CCACHE_LANES), len(reports)) + for step in reports: + self.assertIn("if: always()", step) + self.assertIn("uses: ./.github/actions/ccache-stats", step) + self.assertEqual( + {lane["label"] for lane in CCACHE_LANES.values()}, + {re.search(r"label: (.+)\n", step).group(1) for step in reports}, + ) + + def test_no_lane_keeps_its_own_copy_of_the_scaffolding(self) -> None: + self.assertNotIn("ccache --zero-stats", self.workflow) + self.assertNotIn("ccache --show-stats", self.workflow) + self.assertNotIn("actions/cache@v6", self.workflow) + self.assertNotIn("CCACHE_MAXSIZE}", self.workflow) + + def test_windows_keeps_its_own_sccache_setup(self) -> None: + self.assertIn("uses: mozilla-actions/sccache-action@v0.0.11", self.workflow) + self.assertIn("SCCACHE_GHA_ENABLED=true", self.workflow) + self.assertNotIn("sccache", self.setup_action) + + +class AptRetryScriptTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = WORKFLOW.read_text() + self.script = APT_RETRY.read_text() + + def test_script_keeps_the_retry_ladder_and_apt_options(self) -> None: + self.assertIn("apt-get -o Acquire::Retries=3 \"$@\"", self.script) + self.assertIn("for attempt in 1 2 3; do", self.script) + self.assertIn("sleep $((attempt * 15))", self.script) + self.assertIn("_apt_get_retry update", self.script) + self.assertIn('_apt_get_retry install -y "$@"', self.script) + + def test_script_uses_sudo_only_when_not_root(self) -> None: + self.assertIn('if [ "$(id -u)" -eq 0 ]; then', self.script) + self.assertIn("sudo apt-get -o Acquire::Retries=3", self.script) + + def test_post_checkout_steps_source_the_script(self) -> None: + self.assertEqual(2, self.workflow.count(". tools/apt_retry.sh")) + + def test_only_the_pre_checkout_step_still_declares_the_helpers(self) -> None: + self.assertEqual(1, self.workflow.count("apt_get_install() {")) + self.assertEqual(1, self.workflow.count("apt_get_update() {")) + install_git = workflow_steps(self.workflow, "Install Git") + self.assertEqual(1, len(install_git)) + self.assertIn("apt_get_install() {", install_git[0]) + self.assertLess( + self.workflow.index("apt_get_install() {"), + self.workflow.index("uses: ./.github/actions/checkout-llama-ref\n", self.workflow.index("build-linux-hip:")), + ) + + def test_container_step_still_installs_noninteractively(self) -> None: + hip_deps = workflow_steps(self.workflow, "Install build deps")[-1] + self.assertIn("DEBIAN_FRONTEND: noninteractive", hip_deps) + self.assertIn("apt_get_install build-essential binutils ccache", hip_deps) + + +class UbuntuSourcesTemplateTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = WORKFLOW.read_text() + self.template = SOURCES_TEMPLATE.read_text() + self.dockerfile = DOCKERFILE.read_text() + + def render(self, codename: str) -> str: + return self.template.replace(CODENAME_PLACEHOLDER, codename) + + def test_workflow_and_dockerfile_render_the_same_template(self) -> None: + self.assertFalse((ROOT / "tools/docker/ubuntu.sources").exists()) + self.assertIn(f'sed "{SED_RENDER}" tools/docker/ubuntu.sources.template', self.workflow) + self.assertIn(f'sed "{SED_RENDER}" /etc/apt/ubuntu.sources.template', self.dockerfile) + self.assertIn(". /etc/os-release", self.dockerfile) + self.assertIn("source /etc/os-release", self.workflow) + self.assertIn("COPY ubuntu.sources.template /etc/apt/ubuntu.sources.template", self.dockerfile) + + def test_no_codename_is_hardcoded(self) -> None: + self.assertNotIn("noble", self.template) + self.assertNotIn("noble", self.workflow) + self.assertNotIn("Signed-By", self.workflow) + + def test_rendered_list_keeps_all_three_suites(self) -> None: + rendered = self.render("plucky") + self.assertNotIn("@", rendered) + self.assertEqual( + [ + "Suites: plucky plucky-updates plucky-backports", + "Suites: plucky-security", + "Suites: plucky plucky-updates plucky-backports plucky-security", + ], + [line for line in rendered.splitlines() if line.startswith("Suites:")], + ) + self.assertEqual( + ["Architectures: amd64", "Architectures: amd64", "Architectures: arm64"], + [line for line in rendered.splitlines() if line.startswith("Architectures:")], + ) + self.assertEqual( + [ + "URIs: http://archive.ubuntu.com/ubuntu", + "URIs: http://security.ubuntu.com/ubuntu", + "URIs: http://ports.ubuntu.com/ubuntu-ports", + ], + [line for line in rendered.splitlines() if line.startswith("URIs:")], + ) + + if __name__ == "__main__": unittest.main() diff --git a/tools/apt_retry.sh b/tools/apt_retry.sh new file mode 100644 index 0000000..2eb34a5 --- /dev/null +++ b/tools/apt_retry.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +_apt_get() { + if [ "$(id -u)" -eq 0 ]; then + apt-get -o Acquire::Retries=3 "$@" + else + sudo apt-get -o Acquire::Retries=3 "$@" + fi +} + +_apt_get_retry() { + for attempt in 1 2 3; do + if _apt_get "$@"; then + return 0 + fi + if [ "$attempt" = "3" ]; then + return 1 + fi + sleep $((attempt * 15)) + done +} + +apt_get_update() { + _apt_get_retry update +} + +apt_get_install() { + _apt_get_retry install -y "$@" +} diff --git a/tools/docker/linux-builder.Dockerfile b/tools/docker/linux-builder.Dockerfile index 8fcae47..2e40a48 100644 --- a/tools/docker/linux-builder.Dockerfile +++ b/tools/docker/linux-builder.Dockerfile @@ -4,9 +4,13 @@ FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu24.04 ENV DEBIAN_FRONTEND=noninteractive -COPY ubuntu.sources /etc/apt/sources.list.d/ubuntu.sources +COPY ubuntu.sources.template /etc/apt/ubuntu.sources.template -RUN dpkg --add-architecture arm64 \ +RUN . /etc/os-release \ + && sed "s/@VERSION_CODENAME@/${VERSION_CODENAME}/g" /etc/apt/ubuntu.sources.template \ + > /etc/apt/sources.list.d/ubuntu.sources \ + && rm /etc/apt/ubuntu.sources.template \ + && dpkg --add-architecture arm64 \ && apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ diff --git a/tools/docker/ubuntu.sources b/tools/docker/ubuntu.sources.template similarity index 70% rename from tools/docker/ubuntu.sources rename to tools/docker/ubuntu.sources.template index 52bc124..26bc3b6 100644 --- a/tools/docker/ubuntu.sources +++ b/tools/docker/ubuntu.sources.template @@ -1,20 +1,20 @@ Types: deb URIs: http://archive.ubuntu.com/ubuntu -Suites: noble noble-updates noble-backports +Suites: @VERSION_CODENAME@ @VERSION_CODENAME@-updates @VERSION_CODENAME@-backports Components: main universe restricted multiverse Architectures: amd64 Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg Types: deb URIs: http://security.ubuntu.com/ubuntu -Suites: noble-security +Suites: @VERSION_CODENAME@-security Components: main universe restricted multiverse Architectures: amd64 Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg Types: deb URIs: http://ports.ubuntu.com/ubuntu-ports -Suites: noble noble-updates noble-backports noble-security +Suites: @VERSION_CODENAME@ @VERSION_CODENAME@-updates @VERSION_CODENAME@-backports @VERSION_CODENAME@-security Components: main universe restricted multiverse Architectures: arm64 Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg From a334fd94de5339c0fd0458e3d20dcade0e3bcae7 Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Tue, 22 Sep 2026 09:47:13 -0400 Subject: [PATCH 2/3] ci: scope the HIP apt frontend to the install call The extracted build-linux-hip step set DEBIAN_FRONTEND at step level, so it was also exported to python3 -m pip install --upgrade and cmake --version. Put it back on the apt_get_install call, where only apt-get sees it, as on main. tools/apt_retry.sh stays frontend-agnostic. The ccache lane tests compared the set of key prefixes and the set of labels across all four call sites, so swapping two lanes' values passed. They now resolve steps per job and assert each lane's own prefix, restore keys and label. --- .github/workflows/native_release.yml | 4 +- tests/test_native_release_dedup.py | 92 ++++++++++++++++++---------- 2 files changed, 62 insertions(+), 34 deletions(-) diff --git a/.github/workflows/native_release.yml b/.github/workflows/native_release.yml index d444c03..b76e433 100644 --- a/.github/workflows/native_release.yml +++ b/.github/workflows/native_release.yml @@ -612,11 +612,9 @@ jobs: expected_commit: ${{ needs.resolve-tag.outputs.llama_cpp_commit }} github_token: ${{ github.token }} - name: Install build deps - env: - DEBIAN_FRONTEND: noninteractive run: | . tools/apt_retry.sh - apt_get_install build-essential binutils ccache ninja-build pkg-config python3 python3-pip rocblas-dev hipblas-dev + DEBIAN_FRONTEND=noninteractive apt_get_install build-essential binutils ccache ninja-build pkg-config python3 python3-pip rocblas-dev hipblas-dev python3 -m pip install --upgrade cmake cmake --version - name: Set up ccache diff --git a/tests/test_native_release_dedup.py b/tests/test_native_release_dedup.py index 9743b6c..f97855e 100644 --- a/tests/test_native_release_dedup.py +++ b/tests/test_native_release_dedup.py @@ -27,28 +27,33 @@ SED_RENDER = f's/{CODENAME_PLACEHOLDER}/${{VERSION_CODENAME}}/g' CCACHE_LANES = { - "android": { + "build-android": { "key-prefix": "ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }}", "extra-restore-keys": "ccache-android-${{ matrix.android_abi }}-", "label": "android/${{ matrix.android_abi }}/${{ matrix.backend }}", }, - "apple": { + "build-apple": { "key-prefix": "ccache-apple-${{ matrix.target }}", "extra-restore-keys": None, "label": "apple/${{ matrix.target }}", }, - "linux": { + "build-linux": { "key-prefix": "ccache-linux-${{ matrix.arch }}-${{ matrix.backend }}", "extra-restore-keys": "ccache-linux-${{ matrix.arch }}-", "label": "linux/${{ matrix.arch }}/${{ matrix.backend }}", }, - "linux-hip": { + "build-linux-hip": { "key-prefix": "ccache-linux-x64-hip", "extra-restore-keys": None, "label": "linux/x64/hip", }, } +HIP_APT_INSTALL = ( + "DEBIAN_FRONTEND=noninteractive apt_get_install build-essential binutils ccache " + "ninja-build pkg-config python3 python3-pip rocblas-dev hipblas-dev" +) + def workflow_steps(text: str, name: str) -> list[str]: found = [] @@ -61,6 +66,29 @@ def workflow_steps(text: str, name: str) -> list[str]: start = text.find(marker, start + 1) return found + +def workflow_jobs(text: str) -> dict[str, str]: + body = text[text.index("\njobs:\n") :] + starts = [(m.group(1), m.start()) for m in re.finditer(r"\n ([a-z][\w-]*):\n", body)] + return { + name: body[pos : starts[i + 1][1] if i + 1 < len(starts) else len(body)] + for i, (name, pos) in enumerate(starts) + } + + +def steps_by_job(text: str, name: str) -> dict[str, list[str]]: + found = {} + for job, block in workflow_jobs(text).items(): + steps = workflow_steps(block, name) + if steps: + found[job] = steps + return found + + +def step_input(step: str, key: str) -> str | None: + match = re.search(rf"^ *{re.escape(key)}: (.+)$", step, re.MULTILINE) + return match.group(1) if match else None + LIST_COMMAND = "python3 tools/build.py list --android-arm64-cpu-variant-libs" SINGLED_OUT_VARIANT_LIBS = frozenset( @@ -209,34 +237,28 @@ def test_summary_heading_comes_from_the_label_input(self) -> None: self.assertEqual(2, self.stats_action.count("ccache --show-stats || true")) def test_every_lane_passes_the_cache_key_it_used_before(self) -> None: - setups = workflow_steps(self.workflow, "Set up ccache") - self.assertEqual(len(CCACHE_LANES), len(setups)) - for step in setups: + setups = steps_by_job(self.workflow, "Set up ccache") + self.assertEqual(set(CCACHE_LANES), set(setups)) + for job, lane in CCACHE_LANES.items(): + self.assertEqual(1, len(setups[job]), job) + step = setups[job][0] self.assertIn("uses: ./.github/actions/setup-ccache", step) - self.assertIn("cache-dir: ${{ env.CCACHE_DIR }}", step) - self.assertIn("max-size: ${{ env.CCACHE_MAXSIZE }}", step) - prefixes = {lane["key-prefix"] for lane in CCACHE_LANES.values()} - self.assertEqual( - prefixes, - {re.search(r"key-prefix: (.+)\n", step).group(1) for step in setups}, - ) - extras = {lane["extra-restore-keys"] for lane in CCACHE_LANES.values()} - found_extras = set() - for step in setups: - match = re.search(r"extra-restore-keys: (.+)\n", step) - found_extras.add(match.group(1) if match else None) - self.assertEqual(extras, found_extras) + self.assertEqual("${{ env.CCACHE_DIR }}", step_input(step, "cache-dir"), job) + self.assertEqual("${{ env.CCACHE_MAXSIZE }}", step_input(step, "max-size"), job) + self.assertEqual(lane["key-prefix"], step_input(step, "key-prefix"), job) + self.assertEqual( + lane["extra-restore-keys"], step_input(step, "extra-restore-keys"), job + ) def test_every_lane_reports_stats_under_its_old_heading(self) -> None: - reports = workflow_steps(self.workflow, "Report ccache stats") - self.assertEqual(len(CCACHE_LANES), len(reports)) - for step in reports: + reports = steps_by_job(self.workflow, "Report ccache stats") + self.assertEqual(set(CCACHE_LANES), set(reports)) + for job, lane in CCACHE_LANES.items(): + self.assertEqual(1, len(reports[job]), job) + step = reports[job][0] self.assertIn("if: always()", step) self.assertIn("uses: ./.github/actions/ccache-stats", step) - self.assertEqual( - {lane["label"] for lane in CCACHE_LANES.values()}, - {re.search(r"label: (.+)\n", step).group(1) for step in reports}, - ) + self.assertEqual(lane["label"], step_input(step, "label"), job) def test_no_lane_keeps_its_own_copy_of_the_scaffolding(self) -> None: self.assertNotIn("ccache --zero-stats", self.workflow) @@ -280,10 +302,18 @@ def test_only_the_pre_checkout_step_still_declares_the_helpers(self) -> None: self.workflow.index("uses: ./.github/actions/checkout-llama-ref\n", self.workflow.index("build-linux-hip:")), ) - def test_container_step_still_installs_noninteractively(self) -> None: - hip_deps = workflow_steps(self.workflow, "Install build deps")[-1] - self.assertIn("DEBIAN_FRONTEND: noninteractive", hip_deps) - self.assertIn("apt_get_install build-essential binutils ccache", hip_deps) + def test_container_step_installs_noninteractively_without_exporting_it(self) -> None: + hip_deps = steps_by_job(self.workflow, "Install build deps")["build-linux-hip"][0] + self.assertEqual( + [HIP_APT_INSTALL], + [line.strip() for line in hip_deps.splitlines() if "DEBIAN_FRONTEND" in line], + ) + self.assertIn("python3 -m pip install --upgrade cmake", hip_deps) + + def test_the_hosted_linux_lane_sets_no_apt_frontend(self) -> None: + linux_deps = steps_by_job(self.workflow, "Install build deps")["build-linux"][0] + self.assertNotIn("DEBIAN_FRONTEND", linux_deps) + self.assertNotIn("DEBIAN_FRONTEND", APT_RETRY.read_text()) class UbuntuSourcesTemplateTests(unittest.TestCase): From 4966ed1cb9f546af3495c735361e38d91eede00e Mon Sep 17 00:00:00 2001 From: Jhin Lee Date: Tue, 22 Sep 2026 10:43:36 -0400 Subject: [PATCH 3/3] test: pin the shell and container of every ccache lane The extracted steps declare `shell: bash`; build-linux-hip's inline originals had no `shell:` and, being in a container, ran under `sh -e {0}`. Record each lane's container and pre-extraction default shell so that delta cannot widen unnoticed, and assert no extracted script grows a pipeline, which is what would make the added `-o pipefail` load-bearing. --- tests/test_native_release_dedup.py | 45 ++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/test_native_release_dedup.py b/tests/test_native_release_dedup.py index f97855e..827e52b 100644 --- a/tests/test_native_release_dedup.py +++ b/tests/test_native_release_dedup.py @@ -31,24 +31,36 @@ "key-prefix": "ccache-android-${{ matrix.android_abi }}-${{ matrix.backend }}", "extra-restore-keys": "ccache-android-${{ matrix.android_abi }}-", "label": "android/${{ matrix.android_abi }}/${{ matrix.backend }}", + "container": None, + "runner-default-shell": "bash", }, "build-apple": { "key-prefix": "ccache-apple-${{ matrix.target }}", "extra-restore-keys": None, "label": "apple/${{ matrix.target }}", + "container": None, + "runner-default-shell": "bash", }, "build-linux": { "key-prefix": "ccache-linux-${{ matrix.arch }}-${{ matrix.backend }}", "extra-restore-keys": "ccache-linux-${{ matrix.arch }}-", "label": "linux/${{ matrix.arch }}/${{ matrix.backend }}", + "container": None, + "runner-default-shell": "bash", }, "build-linux-hip": { "key-prefix": "ccache-linux-x64-hip", "extra-restore-keys": None, "label": "linux/x64/hip", + "container": "rocm/dev-ubuntu-22.04:6.1.2", + "runner-default-shell": "sh", }, } +CCACHE_ACTION_SHELL = "bash" + +LONE_PIPE = re.compile(r"(? None: self.assertNotIn("sccache", self.setup_action) +class CcacheStepShellTests(unittest.TestCase): + def setUp(self) -> None: + self.workflow = WORKFLOW.read_text() + self.actions = { + "setup-ccache": SETUP_CCACHE.read_text(), + "ccache-stats": CCACHE_STATS.read_text(), + } + + def test_every_extracted_step_declares_one_shell(self) -> None: + for name, text in self.actions.items(): + shells = re.findall(r"^ *shell: (.+)$", text, re.MULTILINE) + runs = len(re.findall(r"^ *run: ", text, re.MULTILINE)) + self.assertEqual([CCACHE_ACTION_SHELL] * runs, shells, name) + + def test_each_lane_container_is_pinned(self) -> None: + jobs = workflow_jobs(self.workflow) + for job, lane in CCACHE_LANES.items(): + match = re.search(r"^ container: (.+)$", jobs[job], re.MULTILINE) + self.assertEqual(lane["container"], match.group(1) if match else None, job) + + def test_containerised_lanes_are_the_ones_that_defaulted_to_sh(self) -> None: + for job, lane in CCACHE_LANES.items(): + expected = "sh" if lane["container"] else "bash" + self.assertEqual(expected, lane["runner-default-shell"], job) + + def test_no_extracted_script_contains_a_pipeline(self) -> None: + for name, text in self.actions.items(): + for number, line in enumerate(text.splitlines(), 1): + self.assertIsNone( + LONE_PIPE.search(line.rstrip()), f"{name}:{number}: {line}" + ) + + class AptRetryScriptTests(unittest.TestCase): def setUp(self) -> None: self.workflow = WORKFLOW.read_text()