From 403198b8710620fbbc1c9d454955f2f571d306b3 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Thu, 5 Mar 2026 23:50:39 -0500 Subject: [PATCH 01/11] Test ccache --- .github/workflows/matrix.yml | 14 +++ Dockerfile | 26 ++++- hack/build/generate-docker-script.py | 164 +++++++++++++++++++++++---- 3 files changed, 181 insertions(+), 23 deletions(-) diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 2087eeb2..33f8982e 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -82,6 +82,14 @@ jobs: registry: ghcr.io username: "${{github.actor}}" password: "${{secrets.GITHUB_TOKEN}}" + - name: restore ccache + uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: ~/.cache/kernel-ccache + key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}" + restore-keys: | + ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}- + ccache-${{ matrix.builds.flavor }}- - name: generate docker script run: "./hack/build/generate-docker-script.sh" - name: upload docker script @@ -92,3 +100,9 @@ jobs: compression-level: 0 - name: run docker script run: sh -x docker.sh + - name: save ccache + if: always() + uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + with: + path: ~/.cache/kernel-ccache + key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}" diff --git a/Dockerfile b/Dockerfile index ae7da6eb..4e421cdb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,14 +13,15 @@ RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install - build-essential squashfs-tools python3-yaml \ patch diffutils sed mawk findutils zstd \ python3 python3-packaging curl rsync cpio gpg grep \ - flex bison pahole libssl-dev libelf-dev bc kmod && \ + flex bison pahole libssl-dev libelf-dev bc kmod ccache && \ rm -rf /var/lib/apt/lists/* ARG BUILDPLATFORM RUN if [ "${BUILDPLATFORM}" = "linux/amd64" ]; then \ apt-get update && apt-get install -y linux-headers-amd64 g++-aarch64-linux-gnu gcc-aarch64-linux-gnu && rm -rf /var/lib/apt/lists/*; fi RUN if [ "${BUILDPLATFORM}" = "linux/arm64" ] || [ "${BUILDPLATFORM}" = "linux/aarch64" ]; then \ apt-get update && apt-get install -y linux-headers-arm64 g++-x86-64-linux-gnu gcc-x86-64-linux-gnu && rm -rf /var/lib/apt/lists/*; fi -RUN useradd -ms /bin/sh build +ENV PATH="/usr/lib/ccache:${PATH}" +RUN useradd -u 1000 -ms /bin/sh build COPY --chown=build:build . /build USER build WORKDIR /build @@ -65,3 +66,24 @@ COPY --from=build /build/target/metadata /kernel/metadata FROM scratch AS kernel COPY --from=kernelcopy /kernel /kernel + +# Prebuilt packaging stages: use --build-context prebuilt=./target after a +# docker run compile phase (with ccache bind-mounted from the host). +FROM scratch AS kernel-prebuilt +COPY --from=prebuilt kernel /kernel/image +COPY --from=prebuilt config.gz /kernel/config.gz +COPY --from=prebuilt addons.squashfs /kernel/addons.squashfs +COPY --from=prebuilt metadata /kernel/metadata + +FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS sdkbuild-prebuilt +ARG KERNEL_VERSION= +ARG KERNEL_FLAVOR=zone +COPY --from=prebuilt sdk.tar.gz /sdk.tar.gz +RUN mkdir -p /usr/src/kernel-sdk-${KERNEL_VERSION}-${KERNEL_FLAVOR} && \ + tar -zx -C /usr/src/kernel-sdk-${KERNEL_VERSION}-${KERNEL_FLAVOR} -f /sdk.tar.gz && \ + mkdir -p /lib/modules/${KERNEL_VERSION} && \ + ln -sf /usr/src/kernel-sdk-${KERNEL_VERSION}-${KERNEL_FLAVOR} /lib/modules/${KERNEL_VERSION}/build && \ + rm -rf /sdk.tar.gz + +FROM scratch AS sdk-prebuilt +COPY --from=sdkbuild-prebuilt /usr/src /usr/src diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index 0fd0f94a..1b59a2d6 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -9,6 +9,15 @@ from matrix import CONFIG from util import format_image_name, maybe, smart_script_split, parse_text_bool +# Targets that are handled via docker run + prebuilt packaging stages. +PREBUILT_TARGET_MAP = { + "kernel": "kernel-prebuilt", + "sdk": "sdk-prebuilt", +} + +# Targets skipped during the packaging phase (handled separately or not needed). +SKIP_PACKAGING_TARGETS = {"kernelsrc", "buildenv"} + def is_publish_enabled() -> bool: root_publish = os.getenv("KERNEL_PUBLISH", "false") @@ -18,6 +27,7 @@ def is_publish_enabled() -> bool: def quoted(text: str) -> str: return '"%s"' % text + def dockerify_version(version_string: str) -> str: # "+" is valid for both python versions and semver, # but docker rejects it for tags, so sanitize @@ -27,18 +37,79 @@ def dockerify_version(version_string: str) -> str: def docker_platforms(architectures: list[str]) -> list[str]: platforms = [] for arch in architectures: - platform = "" - if arch == "aarch64": - platform = "linux/aarch64" - elif arch == "x86_64": - platform = "linux/amd64" - if len(platform) == 0: - print("unknown platform %s" % arch, file=sys.stderr) - sys.exit(1) + platform = arch_to_platform(arch) platforms.append(platform) return platforms +def arch_to_platform(arch: str) -> str: + if arch == "aarch64": + return "linux/aarch64" + elif arch == "x86_64": + return "linux/amd64" + print("unknown arch %s" % arch, file=sys.stderr) + sys.exit(1) + + +def docker_compile( + version: str, + flavor: str, + architectures: list[str], + src_url: str, + firmware_url: str, + firmware_sig_url: str, +) -> list[str]: + """Generate docker run commands to compile the kernel with ccache.""" + lines = [] + version = dockerify_version(version) + buildenv_iidfile = "image-id-local-%s-buildenv" % flavor + + lines += ["", "rm -rf target && mkdir -p target"] + lines += ['mkdir -p "${HOME}/.cache/kernel-ccache"'] + + for arch in architectures: + platform = arch_to_platform(arch) + compile_command = [ + "docker", + "run", + "--rm", + "--platform", quoted(platform), + "-e", quoted("KERNEL_VERSION=%s" % version), + "-e", quoted("KERNEL_FLAVOR=%s" % flavor), + "-e", quoted("KERNEL_SRC_URL=%s" % src_url), + "-e", quoted("CCACHE_DIR=/home/build/.cache/ccache"), + "-e", quoted("CCACHE_COMPRESS=1"), + "-v", quoted("${HOME}/.cache/kernel-ccache:/home/build/.cache/ccache"), + "-v", quoted("${PWD}/target:/build/target"), + ] + if firmware_url and firmware_sig_url: + compile_command += [ + "-e", quoted("FIRMWARE_URL=/build/firmware-staging/firmware.tar.xz"), + "-e", quoted("FIRMWARE_SIG_URL=/build/firmware-staging/firmware.tar.sign"), + "-v", quoted("${PWD}/firmware-staging:/build/firmware-staging"), + ] + compile_command += [ + '"$(cat %s)"' % buildenv_iidfile, + "./hack/build/docker-build-internal.sh", + ] + lines += [""] + lines += smart_script_split( + compile_command, + "stage=compile flavor=%s version=%s arch=%s" % (flavor, version, arch), + ) + return lines + + +def docker_firmware_download(firmware_url: str, firmware_sig_url: str) -> list[str]: + """Pre-download firmware files to the host so they can be bind-mounted into docker run.""" + return [ + "", + "mkdir -p firmware-staging", + "curl -fsSL -o firmware-staging/firmware.tar.xz %s" % quoted(firmware_url), + "curl -fsSL -o firmware-staging/firmware.tar.sign %s" % quoted(firmware_sig_url), + ] + + def docker_build( target: str, name: str, @@ -53,10 +124,13 @@ def docker_build( mark_format: Optional[str], firmware_url: str, firmware_sig_url: str, + use_prebuilt: bool = False, ) -> list[str]: lines = [] version = dockerify_version(version) + actual_target = PREBUILT_TARGET_MAP.get(target, target) if use_prebuilt else target + root = format_image_name( image_name_format=CONFIG["imageNameFormat"], flavor=flavor, @@ -75,14 +149,17 @@ def docker_build( "-f", quoted("Dockerfile"), "--target", - quoted(target), + quoted(actual_target), "--iidfile", - quoted("image-id-%s-%s-%s" % (version, flavor, target)), + quoted("image-id-%s-%s-%s" % (version, flavor, actual_target)), ] for build_platform in docker_platforms(architectures): image_build_command += ["--platform", quoted(build_platform)] + if use_prebuilt: + image_build_command += ["--build-context", quoted("prebuilt=target")] + if mark_format is not None: image_build_command += [ "--annotation", @@ -96,14 +173,17 @@ def docker_build( "--build-arg", quoted("KERNEL_VERSION=%s" % version), "--build-arg", - quoted("KERNEL_SRC_URL=%s" % src_url), - "--build-arg", quoted("KERNEL_FLAVOR=%s" % flavor), - "--build-arg", - quoted("FIRMWARE_URL=%s" % firmware_url), - "--build-arg", - quoted("FIRMWARE_SIG_URL=%s" % firmware_sig_url), ] + if not use_prebuilt: + image_build_command += [ + "--build-arg", + quoted("KERNEL_SRC_URL=%s" % src_url), + "--build-arg", + quoted("FIRMWARE_URL=%s" % firmware_url), + "--build-arg", + quoted("FIRMWARE_SIG_URL=%s" % firmware_sig_url), + ] if mark_format is not None: image_build_command += [ @@ -149,7 +229,7 @@ def docker_build( "sign", "--yes", quoted( - '%s@$(cat "image-id-%s-%s-%s")' % (tag, version, flavor, target) + '%s@$(cat "image-id-%s-%s-%s")' % (tag, version, flavor, actual_target) ), ] lines += [""] @@ -180,9 +260,51 @@ def generate_builds( lines = [] kernel_version_info = parse(kernel_version) image_configs = CONFIG["images"] + + # Phase 1: Build the buildenv image (provides the compile environment). + for image_config in image_configs: + if image_config["target"] != "buildenv": + continue + image_name = image_config["name"] + image_version = maybe(image_config, "version", kernel_version) + image_tags = maybe(image_config, "tags", kernel_tags) + lines += docker_build( + target="buildenv", + name=image_name, + version=image_version, + version_info=kernel_version_info, + tags=image_tags, + publish=False, + pass_build_args=False, + mark_format=None, + flavor=kernel_flavor, + src_url=kernel_src_url, + architectures=kernel_architectures, + firmware_url=firmware_url, + firmware_sig_url=firmware_sig_url, + ) + break + + # Phase 1b: Pre-download firmware to the host if needed (zone-amdgpu only). + if firmware_url and firmware_sig_url: + lines += docker_firmware_download(firmware_url, firmware_sig_url) + + # Phase 2: Compile kernel via docker run with ccache bind-mounted from host. + lines += docker_compile( + version=kernel_version, + flavor=kernel_flavor, + architectures=kernel_architectures, + src_url=kernel_src_url, + firmware_url=firmware_url, + firmware_sig_url=firmware_sig_url, + ) + + # Phase 3: Package kernel and SDK images from the prebuilt artifacts. for image_config in image_configs: + target = image_config["target"] + if target in SKIP_PACKAGING_TARGETS: + continue image_name = image_config["name"] - image_target = image_config["target"] image_version = maybe(image_config, "version", kernel_version) image_tags = maybe(image_config, "tags", kernel_tags) image_format = maybe(image_config, "format") @@ -190,8 +312,8 @@ def generate_builds( if not is_publish_enabled(): should_publish = False should_pass_build_args = maybe(image_config, "passBuildArgs", True) - image_lines = docker_build( - target=image_target, + lines += docker_build( + target=target, name=image_name, version=image_version, version_info=kernel_version_info, @@ -204,8 +326,8 @@ def generate_builds( architectures=kernel_architectures, firmware_url=firmware_url, firmware_sig_url=firmware_sig_url, + use_prebuilt=True, ) - lines += image_lines return lines From 512dba75e2fa8243f61a2c3d7c9c89d8bc6d6379 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 15:22:51 -0500 Subject: [PATCH 02/11] fixup --- .github/workflows/matrix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 7070ca16..0332d074 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -83,7 +83,7 @@ jobs: username: "${{github.actor}}" password: "${{secrets.GITHUB_TOKEN}}" - name: restore ccache - uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.2 with: path: ~/.cache/kernel-ccache key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}" @@ -102,7 +102,7 @@ jobs: run: sh -x docker.sh - name: save ccache if: always() - uses: actions/cache/save@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 + uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.2 with: path: ~/.cache/kernel-ccache key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}" From 4ab8940168198608dcca631dfdada65496535f1e Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 15:33:24 -0500 Subject: [PATCH 03/11] perms --- hack/build/generate-docker-script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index 1b59a2d6..af0dfa26 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -65,7 +65,7 @@ def docker_compile( buildenv_iidfile = "image-id-local-%s-buildenv" % flavor lines += ["", "rm -rf target && mkdir -p target"] - lines += ['mkdir -p "${HOME}/.cache/kernel-ccache"'] + lines += ['mkdir -p "${HOME}/.cache/kernel-ccache" && chmod a+rwx "${HOME}/.cache/kernel-ccache"'] for arch in architectures: platform = arch_to_platform(arch) From 5c94ecb178bdb96ff30f9c8f910088765a3006dc Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 15:47:41 -0500 Subject: [PATCH 04/11] cleanup dockerfile --- Dockerfile | 59 +++--------- hack/build/generate-docker-script.py | 128 ++++++++++++--------------- 2 files changed, 70 insertions(+), 117 deletions(-) diff --git a/Dockerfile b/Dockerfile index ce0718d5..5d6cf659 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,63 +27,28 @@ USER build WORKDIR /build RUN chmod +x hack/build/docker-build-internal.sh -FROM buildenv AS build -ARG KERNEL_VERSION= -ARG KERNEL_FLAVOR=zone -ARG BUILDPLATFORM -ARG TARGETPLATFORM +FROM buildenv AS build-staged COPY --from=kernelsrc --chown=build:build /src.tar.xz /build/override-kernel-src.tar.xz + +FROM build-staged AS build-staged-amdgpu COPY --from=firmware --chown=build:build /firmware.tar.xz /build/override-firmware.tar.xz COPY --from=firmware --chown=build:build /firmware.tar.sign /build/override-firmware.tar.sign -RUN if [ "${KERNEL_FLAVOR}" = "zone-amdgpu" ]; then \ - FIRMWARE_SIG_URL="/build/override-firmware.tar.sign" \ - FIRMWARE_URL="/build/override-firmware.tar.xz" \ - KERNEL_SRC_URL="/build/override-kernel-src.tar.xz" \ - ./hack/build/docker-build-internal.sh; \ - else \ - KERNEL_SRC_URL="/build/override-kernel-src.tar.xz" \ - ./hack/build/docker-build-internal.sh; \ - fi - -FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS sdkbuild -ARG KERNEL_VERSION= -ARG KERNEL_FLAVOR=zone -COPY --from=build /build/target/sdk.tar.gz /sdk.tar.gz -RUN mkdir -p /usr/src/kernel-sdk-${KERNEL_VERSION}-${KERNEL_FLAVOR} && \ - tar -zx -C /usr/src/kernel-sdk-${KERNEL_VERSION}-${KERNEL_FLAVOR} -f /sdk.tar.gz && \ - mkdir -p /lib/modules/${KERNEL_VERSION} && \ - ln -sf /usr/src/kernel-sdk-${KERNEL_VERSION}-${KERNEL_FLAVOR} /lib/modules/${KERNEL_VERSION}/build && \ - rm -rf /sdk.tar.gz - -FROM scratch AS sdk -COPY --from=sdkbuild /usr/src /usr/src - -FROM scratch AS kernelcopy -COPY --from=build /build/target/kernel /kernel/image -COPY --from=build /build/target/config.gz /kernel/config.gz -COPY --from=build /build/target/addons.squashfs /kernel/addons.squashfs -COPY --from=build /build/target/metadata /kernel/metadata - -FROM scratch AS kernel -COPY --from=kernelcopy /kernel /kernel -# Prebuilt packaging stages: use --build-context prebuilt=./target after a -# docker run compile phase (with ccache bind-mounted from the host). -FROM scratch AS kernel-prebuilt -COPY --from=prebuilt kernel /kernel/image -COPY --from=prebuilt config.gz /kernel/config.gz -COPY --from=prebuilt addons.squashfs /kernel/addons.squashfs -COPY --from=prebuilt metadata /kernel/metadata +FROM scratch AS kernel-ccachebuild +COPY --from=ccachebuild kernel /kernel/image +COPY --from=ccachebuild config.gz /kernel/config.gz +COPY --from=ccachebuild addons.squashfs /kernel/addons.squashfs +COPY --from=ccachebuild metadata /kernel/metadata -FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS sdkbuild-prebuilt +FROM alpine:3.23@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS sdkbuild-ccachebuild ARG KERNEL_VERSION= ARG KERNEL_FLAVOR=zone -COPY --from=prebuilt sdk.tar.gz /sdk.tar.gz +COPY --from=ccachebuild sdk.tar.gz /sdk.tar.gz RUN mkdir -p /usr/src/kernel-sdk-${KERNEL_VERSION}-${KERNEL_FLAVOR} && \ tar -zx -C /usr/src/kernel-sdk-${KERNEL_VERSION}-${KERNEL_FLAVOR} -f /sdk.tar.gz && \ mkdir -p /lib/modules/${KERNEL_VERSION} && \ ln -sf /usr/src/kernel-sdk-${KERNEL_VERSION}-${KERNEL_FLAVOR} /lib/modules/${KERNEL_VERSION}/build && \ rm -rf /sdk.tar.gz -FROM scratch AS sdk-prebuilt -COPY --from=sdkbuild-prebuilt /usr/src /usr/src +FROM scratch AS sdk-ccachebuild +COPY --from=sdkbuild-ccachebuild /usr/src /usr/src diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index af0dfa26..66eeba55 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -9,10 +9,10 @@ from matrix import CONFIG from util import format_image_name, maybe, smart_script_split, parse_text_bool -# Targets that are handled via docker run + prebuilt packaging stages. -PREBUILT_TARGET_MAP = { - "kernel": "kernel-prebuilt", - "sdk": "sdk-prebuilt", +# Targets that are handled via docker run + host CCACHE packaging stages. +CCACHE_TARGET_MAP = { + "kernel": "kernel-ccachebuild", + "sdk": "sdk-ccachebuild", } # Targets skipped during the packaging phase (handled separately or not needed). @@ -51,18 +51,52 @@ def arch_to_platform(arch: str) -> str: sys.exit(1) -def docker_compile( +def docker_build_staged( version: str, flavor: str, architectures: list[str], src_url: str, firmware_url: str, firmware_sig_url: str, +) -> list[str]: + """Build the build-staged image: buildenv with kernel source (and firmware for amdgpu) baked in.""" + version = dockerify_version(version) + has_firmware = bool(firmware_url and firmware_sig_url) + target = "build-staged-amdgpu" if has_firmware else "build-staged" + iidfile = "image-id-%s-%s-%s" % (version, flavor, target) + command = [ + "docker", "buildx", "build", + "--builder", "edera", + "--load", + "-f", quoted("Dockerfile"), + "--target", quoted(target), + "--iidfile", quoted(iidfile), + ] + for platform in docker_platforms(architectures): + command += ["--platform", quoted(platform)] + command += ["--build-arg", quoted("KERNEL_SRC_URL=%s" % src_url)] + if has_firmware: + command += [ + "--build-arg", quoted("FIRMWARE_URL=%s" % firmware_url), + "--build-arg", quoted("FIRMWARE_SIG_URL=%s" % firmware_sig_url), + ] + command += ["."] + return [""] + smart_script_split(command, "stage=stage flavor=%s version=%s" % (flavor, version)) + + +def docker_compile( + version: str, + flavor: str, + architectures: list[str], + firmware_url: str, + firmware_sig_url: str, ) -> list[str]: """Generate docker run commands to compile the kernel with ccache.""" lines = [] version = dockerify_version(version) - buildenv_iidfile = "image-id-local-%s-buildenv" % flavor + has_firmware = bool(firmware_url and firmware_sig_url) + stage_target = "build-staged-amdgpu" if has_firmware else "build-staged" + staged_iidfile = "image-id-%s-%s-%s" % (version, flavor, stage_target) lines += ["", "rm -rf target && mkdir -p target"] lines += ['mkdir -p "${HOME}/.cache/kernel-ccache" && chmod a+rwx "${HOME}/.cache/kernel-ccache"'] @@ -76,20 +110,19 @@ def docker_compile( "--platform", quoted(platform), "-e", quoted("KERNEL_VERSION=%s" % version), "-e", quoted("KERNEL_FLAVOR=%s" % flavor), - "-e", quoted("KERNEL_SRC_URL=%s" % src_url), + "-e", quoted("KERNEL_SRC_URL=/build/override-kernel-src.tar.xz"), "-e", quoted("CCACHE_DIR=/home/build/.cache/ccache"), "-e", quoted("CCACHE_COMPRESS=1"), "-v", quoted("${HOME}/.cache/kernel-ccache:/home/build/.cache/ccache"), "-v", quoted("${PWD}/target:/build/target"), ] - if firmware_url and firmware_sig_url: + if has_firmware: compile_command += [ - "-e", quoted("FIRMWARE_URL=/build/firmware-staging/firmware.tar.xz"), - "-e", quoted("FIRMWARE_SIG_URL=/build/firmware-staging/firmware.tar.sign"), - "-v", quoted("${PWD}/firmware-staging:/build/firmware-staging"), + "-e", quoted("FIRMWARE_URL=/build/override-firmware.tar.xz"), + "-e", quoted("FIRMWARE_SIG_URL=/build/override-firmware.tar.sign"), ] compile_command += [ - '"$(cat %s)"' % buildenv_iidfile, + '"$(cat %s)"' % staged_iidfile, "./hack/build/docker-build-internal.sh", ] lines += [""] @@ -100,15 +133,6 @@ def docker_compile( return lines -def docker_firmware_download(firmware_url: str, firmware_sig_url: str) -> list[str]: - """Pre-download firmware files to the host so they can be bind-mounted into docker run.""" - return [ - "", - "mkdir -p firmware-staging", - "curl -fsSL -o firmware-staging/firmware.tar.xz %s" % quoted(firmware_url), - "curl -fsSL -o firmware-staging/firmware.tar.sign %s" % quoted(firmware_sig_url), - ] - def docker_build( target: str, @@ -118,18 +142,14 @@ def docker_build( version_info: Version, tags: list[str], architectures: list[str], - src_url: str, publish: bool, pass_build_args: bool, mark_format: Optional[str], - firmware_url: str, - firmware_sig_url: str, - use_prebuilt: bool = False, ) -> list[str]: lines = [] version = dockerify_version(version) - actual_target = PREBUILT_TARGET_MAP.get(target, target) if use_prebuilt else target + actual_target = CCACHE_TARGET_MAP.get(target, target) root = format_image_name( image_name_format=CONFIG["imageNameFormat"], @@ -157,8 +177,8 @@ def docker_build( for build_platform in docker_platforms(architectures): image_build_command += ["--platform", quoted(build_platform)] - if use_prebuilt: - image_build_command += ["--build-context", quoted("prebuilt=target")] + if actual_target != target: + image_build_command += ["--build-context", quoted("ccachebuild=target")] if mark_format is not None: image_build_command += [ @@ -175,15 +195,6 @@ def docker_build( "--build-arg", quoted("KERNEL_FLAVOR=%s" % flavor), ] - if not use_prebuilt: - image_build_command += [ - "--build-arg", - quoted("KERNEL_SRC_URL=%s" % src_url), - "--build-arg", - quoted("FIRMWARE_URL=%s" % firmware_url), - "--build-arg", - quoted("FIRMWARE_SIG_URL=%s" % firmware_sig_url), - ] if mark_format is not None: image_build_command += [ @@ -261,45 +272,26 @@ def generate_builds( kernel_version_info = parse(kernel_version) image_configs = CONFIG["images"] - # Phase 1: Build the buildenv image (provides the compile environment). - for image_config in image_configs: - if image_config["target"] != "buildenv": - continue - image_name = image_config["name"] - image_version = maybe(image_config, "version", kernel_version) - image_tags = maybe(image_config, "tags", kernel_tags) - lines += docker_build( - target="buildenv", - name=image_name, - version=image_version, - version_info=kernel_version_info, - tags=image_tags, - publish=False, - pass_build_args=False, - mark_format=None, - flavor=kernel_flavor, - src_url=kernel_src_url, - architectures=kernel_architectures, - firmware_url=firmware_url, - firmware_sig_url=firmware_sig_url, - ) - break - - # Phase 1b: Pre-download firmware to the host if needed (zone-amdgpu only). - if firmware_url and firmware_sig_url: - lines += docker_firmware_download(firmware_url, firmware_sig_url) + # Phase 1: Build the build-staged image (buildenv + kernel source + firmware baked in). + lines += docker_build_staged( + version=kernel_version, + flavor=kernel_flavor, + architectures=kernel_architectures, + src_url=kernel_src_url, + firmware_url=firmware_url, + firmware_sig_url=firmware_sig_url, + ) # Phase 2: Compile kernel via docker run with ccache bind-mounted from host. lines += docker_compile( version=kernel_version, flavor=kernel_flavor, architectures=kernel_architectures, - src_url=kernel_src_url, firmware_url=firmware_url, firmware_sig_url=firmware_sig_url, ) - # Phase 3: Package kernel and SDK images from the prebuilt artifacts. + # Phase 3: Package kernel and SDK images from the ccache-built artifacts. for image_config in image_configs: target = image_config["target"] if target in SKIP_PACKAGING_TARGETS: @@ -322,11 +314,7 @@ def generate_builds( pass_build_args=should_pass_build_args, mark_format=image_format, flavor=kernel_flavor, - src_url=kernel_src_url, architectures=kernel_architectures, - firmware_url=firmware_url, - firmware_sig_url=firmware_sig_url, - use_prebuilt=True, ) return lines From 0ea820f0d453f9dea6a220fbe42cfea616ec7361 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 16:38:21 -0500 Subject: [PATCH 05/11] recursively chown + fix nvidia stripping --- hack/build/generate-docker-script.py | 2 +- hack/build/patchlist.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index 66eeba55..3127c144 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -99,7 +99,7 @@ def docker_compile( staged_iidfile = "image-id-%s-%s-%s" % (version, flavor, stage_target) lines += ["", "rm -rf target && mkdir -p target"] - lines += ['mkdir -p "${HOME}/.cache/kernel-ccache" && chmod a+rwx "${HOME}/.cache/kernel-ccache"'] + lines += ['mkdir -p "${HOME}/.cache/kernel-ccache" && chmod -R a+rwX "${HOME}/.cache/kernel-ccache"'] for arch in architectures: platform = arch_to_platform(arch) diff --git a/hack/build/patchlist.py b/hack/build/patchlist.py index e9e9b9de..f617be76 100644 --- a/hack/build/patchlist.py +++ b/hack/build/patchlist.py @@ -9,7 +9,10 @@ print("Usage: patchlist ") exit(1) -target_version = parse(sys.argv[1]) +try: + target_version = parse(sys.argv[1]) +except Exception: + target_version = parse(sys.argv[1].split('-')[0]) kernel_flavor = sys.argv[2] series = "%s.%s" % (target_version.major, target_version.minor) From 83ed18b8d632372ad233860a94a9758cd9c071c6 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 16:52:38 -0500 Subject: [PATCH 06/11] Don't queue CI jobs, fix flavor --- .github/workflows/test.yml | 3 +++ hack/build/generate-docker-script.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60653d58..ed17fa2e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,9 @@ on: permissions: contents: read packages: read +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: test: uses: ./.github/workflows/matrix.yml diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index 3127c144..689e10cd 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -61,7 +61,7 @@ def docker_build_staged( ) -> list[str]: """Build the build-staged image: buildenv with kernel source (and firmware for amdgpu) baked in.""" version = dockerify_version(version) - has_firmware = bool(firmware_url and firmware_sig_url) + has_firmware = flavor == "zone-amdgpu" target = "build-staged-amdgpu" if has_firmware else "build-staged" iidfile = "image-id-%s-%s-%s" % (version, flavor, target) command = [ @@ -94,7 +94,7 @@ def docker_compile( """Generate docker run commands to compile the kernel with ccache.""" lines = [] version = dockerify_version(version) - has_firmware = bool(firmware_url and firmware_sig_url) + has_firmware = flavor == "zone-amdgpu" stage_target = "build-staged-amdgpu" if has_firmware else "build-staged" staged_iidfile = "image-id-%s-%s-%s" % (version, flavor, stage_target) From 6e81d0550ac532be8c7aa79c2a8d5c8027bdf6b6 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 17:07:22 -0500 Subject: [PATCH 07/11] One cache per flavor+arch should be enough --- .github/workflows/matrix.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 0332d074..ce5bea8d 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -86,9 +86,8 @@ jobs: uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.2 with: path: ~/.cache/kernel-ccache - key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}" + key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}" restore-keys: | - ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}- ccache-${{ matrix.builds.flavor }}- - name: generate docker script run: "./hack/build/generate-docker-script.sh" @@ -105,4 +104,4 @@ jobs: uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.2 with: path: ~/.cache/kernel-ccache - key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}" + key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}" From fc079866fc72937c6403b90eaa5e0d23a894eae4 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 17:10:06 -0500 Subject: [PATCH 08/11] more perms --- Dockerfile | 2 +- hack/build/generate-docker-script.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5d6cf659..45b5b846 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ RUN if [ "${BUILDPLATFORM}" = "linux/amd64" ]; then \ RUN if [ "${BUILDPLATFORM}" = "linux/arm64" ] || [ "${BUILDPLATFORM}" = "linux/aarch64" ]; then \ apt-get update && apt-get install -y linux-headers-arm64 g++-x86-64-linux-gnu gcc-x86-64-linux-gnu && rm -rf /var/lib/apt/lists/*; fi ENV PATH="/usr/lib/ccache:${PATH}" -RUN useradd -u 1000 -ms /bin/sh build +RUN useradd -ms /bin/sh build COPY --chown=build:build . /build USER build WORKDIR /build diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index 689e10cd..cfa8896c 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -98,7 +98,7 @@ def docker_compile( stage_target = "build-staged-amdgpu" if has_firmware else "build-staged" staged_iidfile = "image-id-%s-%s-%s" % (version, flavor, stage_target) - lines += ["", "rm -rf target && mkdir -p target"] + lines += ["", "rm -rf target && mkdir -p target && chmod a+rwX target"] lines += ['mkdir -p "${HOME}/.cache/kernel-ccache" && chmod -R a+rwX "${HOME}/.cache/kernel-ccache"'] for arch in architectures: From bfdbca5e0a5eaf932eb5eae75b82b1508e34cded Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 17:11:27 -0500 Subject: [PATCH 09/11] don't cache save on build fail --- .github/workflows/matrix.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index ce5bea8d..83dcd974 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -100,7 +100,6 @@ jobs: - name: run docker script run: sh -x docker.sh - name: save ccache - if: always() uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.2 with: path: ~/.cache/kernel-ccache From c62c054e8c2029963593cb71ca1f9181be9bda65 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 18:11:11 -0500 Subject: [PATCH 10/11] Cache nvidia bits as a layer to avoid DL issues --- Dockerfile | 7 +++++++ hack/build/generate-docker-script.py | 31 +++++++++++++++++++++++----- hack/build/nvidiagpu-common.sh | 27 +++++++++++++----------- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index 45b5b846..0c897189 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,10 @@ ARG FIRMWARE_SIG_URL= ADD ${FIRMWARE_URL} /firmware.tar.xz ADD ${FIRMWARE_SIG_URL} /firmware.tar.sign +FROM --platform=$BUILDPLATFORM scratch AS nvidia-modules +ARG NV_MODULES_TARBALL_URL= +ADD ${NV_MODULES_TARBALL_URL} /nvidia-modules.tar.gz + FROM --platform=$BUILDPLATFORM debian:bookworm@sha256:0a5bf4ecacfc050bad0131c8e1401063fd1e8343a418723f6dbd3cd13a7b9e33 AS buildenv RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y \ build-essential squashfs-tools python3-yaml \ @@ -34,6 +38,9 @@ FROM build-staged AS build-staged-amdgpu COPY --from=firmware --chown=build:build /firmware.tar.xz /build/override-firmware.tar.xz COPY --from=firmware --chown=build:build /firmware.tar.sign /build/override-firmware.tar.sign +FROM build-staged AS build-staged-nvidiagpu +COPY --from=nvidia-modules --chown=build:build /nvidia-modules.tar.gz /build/override-nvidia-modules.tar.gz + FROM scratch AS kernel-ccachebuild COPY --from=ccachebuild kernel /kernel/image COPY --from=ccachebuild config.gz /kernel/config.gz diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index cfa8896c..9ea5c011 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -59,10 +59,19 @@ def docker_build_staged( firmware_url: str, firmware_sig_url: str, ) -> list[str]: - """Build the build-staged image: buildenv with kernel source (and firmware for amdgpu) baked in.""" - version = dockerify_version(version) + """Build the build-staged image: buildenv with kernel source (and firmware/nvidia-modules for gpu flavors) baked in.""" has_firmware = flavor == "zone-amdgpu" - target = "build-staged-amdgpu" if has_firmware else "build-staged" + has_nvidia = flavor == "zone-nvidiagpu" + if has_nvidia: + nv_version = version.split("+nvidia-")[1] if "+nvidia-" in version else version.split("-nvidia-")[1] + nv_modules_url = "https://github.com/NVIDIA/open-gpu-kernel-modules/archive/refs/tags/%s.tar.gz" % nv_version + version = dockerify_version(version) + if has_nvidia: + target = "build-staged-nvidiagpu" + elif has_firmware: + target = "build-staged-amdgpu" + else: + target = "build-staged" iidfile = "image-id-%s-%s-%s" % (version, flavor, target) command = [ "docker", "buildx", "build", @@ -80,6 +89,8 @@ def docker_build_staged( "--build-arg", quoted("FIRMWARE_URL=%s" % firmware_url), "--build-arg", quoted("FIRMWARE_SIG_URL=%s" % firmware_sig_url), ] + if has_nvidia: + command += ["--build-arg", quoted("NV_MODULES_TARBALL_URL=%s" % nv_modules_url)] command += ["."] return [""] + smart_script_split(command, "stage=stage flavor=%s version=%s" % (flavor, version)) @@ -93,9 +104,15 @@ def docker_compile( ) -> list[str]: """Generate docker run commands to compile the kernel with ccache.""" lines = [] - version = dockerify_version(version) has_firmware = flavor == "zone-amdgpu" - stage_target = "build-staged-amdgpu" if has_firmware else "build-staged" + has_nvidia = flavor == "zone-nvidiagpu" + version = dockerify_version(version) + if has_nvidia: + stage_target = "build-staged-nvidiagpu" + elif has_firmware: + stage_target = "build-staged-amdgpu" + else: + stage_target = "build-staged" staged_iidfile = "image-id-%s-%s-%s" % (version, flavor, stage_target) lines += ["", "rm -rf target && mkdir -p target && chmod a+rwX target"] @@ -121,6 +138,10 @@ def docker_compile( "-e", quoted("FIRMWARE_URL=/build/override-firmware.tar.xz"), "-e", quoted("FIRMWARE_SIG_URL=/build/override-firmware.tar.sign"), ] + if has_nvidia: + compile_command += [ + "-e", quoted("NVIDIA_MODULES_PATH=/build/override-nvidia-modules.tar.gz"), + ] compile_command += [ '"$(cat %s)"' % staged_iidfile, "./hack/build/docker-build-internal.sh", diff --git a/hack/build/nvidiagpu-common.sh b/hack/build/nvidiagpu-common.sh index 558d57a6..7a9c7356 100644 --- a/hack/build/nvidiagpu-common.sh +++ b/hack/build/nvidiagpu-common.sh @@ -22,25 +22,28 @@ fi echo "Fetching nvidia module release: $NV_VERSION" -RELEASE_JSON=$(curl -s --retry 5 --retry-delay 2 --retry-max-time 30 --retry-all-errors "https://api.github.com/repos/${NV_KMOD_REPO_OWNER}/${NV_KMOD_REPO_NAME}/releases/tags/${NV_VERSION}") -TARBALL_URL=$(echo "$RELEASE_JSON" | grep -o '"tarball_url": *"[^"]*"' | sed 's/"tarball_url": *"\(.*\)"/\1/') -if [ -z "$TARBALL_URL" ]; then - echo "Failed to fetch release information for version $NV_VERSION" - exit 1 +NV_WORKDIR="$(mktemp -d)/nvidia-modules/${NV_VERSION}" +mkdir -p "$NV_WORKDIR" + +if [ -n "${NVIDIA_MODULES_PATH}" ] && [ -f "${NVIDIA_MODULES_PATH}" ]; then + ARCHIVE="${NVIDIA_MODULES_PATH}" +else + RELEASE_JSON=$(curl -s --retry 5 --retry-delay 2 --retry-max-time 30 --retry-all-errors "https://api.github.com/repos/${NV_KMOD_REPO_OWNER}/${NV_KMOD_REPO_NAME}/releases/tags/${NV_VERSION}") + TARBALL_URL=$(echo "$RELEASE_JSON" | grep -o '"tarball_url": *"[^"]*"' | sed 's/"tarball_url": *"\(.*\)"/\1/') + if [ -z "$TARBALL_URL" ]; then + echo "Failed to fetch release information for version $NV_VERSION" + exit 1 + fi + ARCHIVE="$NV_WORKDIR/driver-src.tar.gz" + curl -L -o "$ARCHIVE" "$TARBALL_URL" fi echo "Building NVIDIA driver version: $NV_VERSION" -NV_WORKDIR="$(mktemp -d)/nvidia-modules/${NV_VERSION}" -ARCHIVE="$NV_WORKDIR/driver-src.tar.gz" - -mkdir -p "$NV_WORKDIR" - -curl -L -o "$ARCHIVE" "$TARBALL_URL" tar -xzf "$ARCHIVE" -C "$NV_WORKDIR" OLDPWD=$(pwd) -cd "$NV_WORKDIR"/"$NV_KMOD_REPO_OWNER"-* +cd "$(find "$NV_WORKDIR" -mindepth 1 -maxdepth 1 -type d | head -1)" # Apply nvidia hackpatches if we have them for patch in "${OLDPWD}"/patches-nvidia/*.patch; do patch -p1 < "$patch"; done From 074aec7731fa0e457ecdd84e879ff07e42543131 Mon Sep 17 00:00:00 2001 From: Benjamin Leggett Date: Fri, 6 Mar 2026 18:22:16 -0500 Subject: [PATCH 11/11] Github caching madness --- .github/workflows/matrix.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 83dcd974..f0ab5a6a 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -86,9 +86,12 @@ jobs: uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.2 with: path: ~/.cache/kernel-ccache - key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}" + # restore-keys is important here - it lets us restore the most recent cache key, + # *ignoring* the specific run ID, as a fuzzy match. So we can use previous build's + # caches for this flavor/arch even if the runid is not the same + key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}" restore-keys: | - ccache-${{ matrix.builds.flavor }}- + ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}- - name: generate docker script run: "./hack/build/generate-docker-script.sh" - name: upload docker script @@ -103,4 +106,7 @@ jobs: uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.2 with: path: ~/.cache/kernel-ccache - key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}" + # The run_id here is just for write-key uniqueness, as GH doesn't allow overwriting + # existing cache keys - the `restore` action will fuzzy-match and ignore the run_id + # for subsequent runs. + key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}"