fix(arc): force field conflicts when Helm 4 upgrades the controller and scale sets

.github/workflows/helm-apply.yml

@@ -0,0 +1,73 @@
+name: Helm apply
+
+# Runs the ARC role's Helm version read and scale-set helm task in a kind cluster with Helm 4 and Helm 3 (tests/helm_apply/run.sh): a release whose maxRunners kubectl has patched, as the autoscaler does, is upgraded to a different value, a release first installed client-side moves to server-side apply, and Helm 3 is passed neither server-side option.
+#
+# GitHub-hosted ubuntu-latest has Docker, kind and kubectl already. Both Helm releases are downloaded and checked against SHA-256 digests pinned here, taken from Helm's GitHub release notes and matching get.helm.sh's .sha256sum files, so the test depends neither on the image's own Helm nor on the download host vouching for its own archive. Each phase also runs with the oldest kubernetes.core galaxy.yml accepts, as well as the newest.
+
+on:
+  pull_request:
+    paths:
+      - roles/github_runner_arc/tasks/main.yml
+      - roles/github_runner_arc/tasks/install_controller.yml
+      - roles/github_runner_arc/tasks/install_scale_set_profile.yml
+      - plugins/filter/arc.py
+      - tests/helm_apply/**
+      - .github/workflows/helm-apply.yml
+      - galaxy.yml
+  workflow_dispatch:
+
+concurrency:
+  group: helm-apply-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  helm-apply:
+    name: Helm apply in kind
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      HELM3_VERSION: v3.18.6
+      HELM3_SHA256: 3f43c0aa57243852dd542493a0f54f1396c0bc8ec7296bbb2c01e802010819ce
+      HELM4_VERSION: v4.3.0
+      HELM4_SHA256: 86584a54def73570558f66f5111cc53dfed56689637ae32c1201205d494f54fb
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Install Ansible and the Kubernetes collection
+        run: |
+          set -euo pipefail
+          if ! python3 -m venv "${RUNNER_TEMP}/venv" 2>/dev/null; then
+            sudo apt-get update -qq
+            sudo apt-get install -y -qq python3-venv
+            python3 -m venv "${RUNNER_TEMP}/venv"
+          fi
+          "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core kubernetes pyyaml
+          echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH"
+          # The newest kubernetes.core in the default path, and exactly the lowest one galaxy.yml accepts in a path of its own.
+          floor="$(PYTHON="${RUNNER_TEMP}/venv/bin/python" tests/helm_apply/kubernetes-core-floor.sh)"
+          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install "kubernetes.core>=${floor}"
+          # --force: otherwise ansible-galaxy counts the newest already in the default path as satisfying ==floor when the two are the same version, and installs nothing into the floor path.
+          "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --force "kubernetes.core==${floor}" -p "${RUNNER_TEMP}/kubernetes-core-floor"
+
+      - name: Download Helm 3 and Helm 4
+        run: |
+          set -euo pipefail
+          for pair in "${HELM3_VERSION}:${HELM3_SHA256}" "${HELM4_VERSION}:${HELM4_SHA256}"; do
+            version="${pair%%:*}"
+            archive="helm-${version}-linux-amd64.tar.gz"
+            curl -fsSLo "${RUNNER_TEMP}/${archive}" "https://get.helm.sh/${archive}"
+            # sha256sum -c exits non-zero on a mismatch, which fails the step under set -e.
+            echo "${pair#*:}  ${RUNNER_TEMP}/${archive}" | sha256sum -c -
+            mkdir -p "${RUNNER_TEMP}/${version}"
+            tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}/${version}"
+          done
+
+      - name: Run the Helm apply test
+        run: |
+          HELM3="${RUNNER_TEMP}/${HELM3_VERSION}/linux-amd64/helm" \
+          HELM4="${RUNNER_TEMP}/${HELM4_VERSION}/linux-amd64/helm" \
+          KUBERNETES_CORE_FLOOR="${RUNNER_TEMP}/kubernetes-core-floor" \
+            tests/helm_apply/run.sh

README.md

@@ -154,7 +154,7 @@ Each poll (`AUTOSCALER_POLL_SECONDS`, default 45s):
 
 Ships with `AUTOSCALER_DRY_RUN=true` by default: it computes and logs the target and writes its own status, but never patches. Generate real concurrent traffic to watch it react: `gh workflow run test-autoscaler.yml`, then watch `kubectl logs -n github-runner-platform deploy/autoscaler -f` and `kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.status\.json}'`. Only set `AUTOSCALER_DRY_RUN=false` after watching real dry-run output across genuine CI traffic. Status and its raise-confirm counter live in a shared Kubernetes ConfigMap (`autoscaler-status`, pre-created empty so neither pod's own ServiceAccount ever needs `create` RBAC on it, only `get`/`patch`) rather than a bind-mounted directory, since the two pods can land on different nodes now — `scripts/heartbeat.sh` reads that same ConfigMap back out and republishes it as a second file in the heartbeat gist, reusing its existing gist-write credential rather than giving the autoscaler its own.
 
-Every Helm upgrade above reverts `maxRunners` to the values file's safe floor. The role's `install_org.yml` triggers one immediate autoscaler poll after every such upgrade (`kubectl exec deploy/autoscaler -- ...`, reaching the pod through the API server regardless of which node it's on), so the safe-floor window after a redeploy is seconds, not a full poll interval.
+A Helm upgrade of a scale set (the role runs one when the release's rendered values or chart differ from what is installed) reverts `maxRunners` to the values file's safe floor. On Helm 4, which applies releases server-side, the autoscaler's patches make it the manager of `maxRunners`, so the role forces conflicts on the controller and scale-set releases (`force_conflicts`, only over the fields the chart sets); without that the upgrade would fail instead of reverting. The role's `install_org.yml` triggers one immediate autoscaler poll after every such upgrade (`kubectl exec deploy/autoscaler -- ...`, reaching the pod through the API server regardless of which node it's on), so the safe-floor window after a redeploy is seconds, not a full poll interval.
 
 ### Node recovery
 

ansible/requirements.yml

@@ -1,5 +1,6 @@
 ---
 collections:
-  - name: kubernetes.core   # k8s, k8s_info, helm
+  - name: kubernetes.core   # k8s, k8s_info, helm; 6.5.0 adds helm's server_side and force_conflicts
+    version: ">=6.5.0"
   - name: community.docker  # docker_compose_v2
   - name: community.general  # homebrew (host prerequisites: helm, kubectl, docker)

galaxy.yml

@@ -17,7 +17,7 @@ tags:
   - kubernetes
   - k3s
 dependencies:
-  kubernetes.core: ">=5.0.0"
+  kubernetes.core: ">=6.5.0"
   community.docker: ">=4.0.0"
   community.general: ">=9.0.0"
 repository: https://github.com/ExaDev/github-runner

plugins/filter/arc.py

@@ -1155,6 +1155,26 @@ def arc_upgrade_blocker(deployed_chart: str, desired_version: str, running_runne
     )
 
 
+def arc_helm_server_side(version: str) -> bool:
+    """Report whether the Helm client applies releases with server-side apply, which decides whether the role forces field conflicts.
+
+    Helm 4 upgrades a release with server-side apply when it was installed that way (its default), and fails on any field another field manager has since set to a different value, such as a scale set's maxRunners after the autoscaler has patched it. Helm 3 has no server-side apply; below Helm 4 the helm module refuses server_side in any form and force_conflicts when it is true, so the role passes neither there.
+
+    Args:
+        version: the client version as `helm version --short` prints it, for example v4.3.0+gbec5b06 or v3.16.2+g13654a5.
+
+    Returns:
+        True for Helm 4 or later, False for an earlier major version.
+
+    Raises:
+        ValueError: when the version has no leading major number.
+    """
+    match = re.match(r"^v?(\d+)\.", str(version).strip())
+    if match is None:
+        raise ValueError(f"cannot read a Helm major version from {version!r}")
+    return int(match.group(1)) >= 4
+
+
 class FilterModule:
     """Registers the ARC role's filters with Ansible."""
 
@@ -1165,6 +1185,7 @@ def filters(self) -> dict[str, Any]:
             "arc_max_runners": arc_max_runners,
             "arc_measured_max_runners": arc_measured_max_runners,
             "arc_upgrade_blocker": arc_upgrade_blocker,
+            "arc_helm_server_side": arc_helm_server_side,
             "arc_image_ref": arc_image_ref,
             "arc_runner_placement": arc_runner_placement,
             "arc_capability_errors": arc_capability_errors,

roles/github_runner_arc/README.md

@@ -30,7 +30,10 @@ Before touching the cluster the role then checks the secrets it is about to writ
 - `github_runner_arc_kubeconfig_path`: the kubeconfig on the host the role runs against. Defaults to `github_runner_cluster`'s kubeconfig; empty means `KUBECONFIG` or `~/.kube/config`.
 - `github_runner_arc_controller_chart_version`, `github_runner_arc_scaleset_chart_version`: chart pins; unset installs the latest chart. The controller chart's CRDs are applied (server-side) from `github_runner_arc_controller_chart_ref` at the controller's version before every controller install or upgrade, because Helm never upgrades a chart's CRDs itself; without that, a newer scale-set chart is rejected for fields the cluster's CRDs predate. Move the two pins together, and the CRDs follow. `tests/arc_upgrade/run.sh` covers the upgrade in a kind cluster.
 
-Upgrading ARC on a live fleet. The new controller deletes every scale set whose version differs from its own (its log says "Autoscaling runner set version doesn't match the build version. Deleting the resource."), together with the set's listener and running ephemeral runners, until Helm upgrades that scale set. A controller version change therefore kills the jobs on running runners, so the role refuses it while any ephemeral runner is running (`github_runner_arc_allow_busy_controller_upgrade` accepts that), and a fleet with no running runner loses nothing. The check needs the controller chart version pinned, since an unpinned run cannot know what it will install. Dry-run (`--check`) cannot show a CRD or ownership failure, so it is not a rehearsal. Note the scale sets that exist before (`kubectl get autoscalingrunnerset -A`) and check the same set exists afterwards. Never apply a release's manifests by hand with `kubectl apply --server-side` to recover one: Helm's server-side apply then conflicts with `kubectl` on the labels a chart upgrade changes, and the ServiceAccount, Role and RoleBinding must be deleted for Helm to recreate them. The heartbeat reports a missing scale set within a few minutes (`fleet-health.json` in its gist), and jobs fall back to GitHub-hosted runners meanwhile.
+Upgrading ARC on a live fleet. The new controller deletes every scale set whose version differs from its own (its log says "Autoscaling runner set version doesn't match the build version. Deleting the resource."), together with the set's listener and running ephemeral runners, until Helm upgrades that scale set. A controller version change therefore kills the jobs on running runners, so the role refuses it while any ephemeral runner is running (`github_runner_arc_allow_busy_controller_upgrade` accepts that), and a fleet with no running runner loses nothing. The check needs the controller chart version pinned, since an unpinned run cannot know what it will install. Dry-run (`--check`) cannot show a CRD or ownership failure, so it is not a rehearsal. Note the scale sets that exist before (`kubectl get autoscalingrunnerset -A`) and check the same set exists afterwards. Never apply a release's manifests by hand with `kubectl apply --server-side` to recover one: the ServiceAccount, Role and RoleBinding must then be deleted for Helm to recreate them. Such an apply also makes `kubectl` the manager of the labels a chart upgrade changes; on Helm 4 the role's forced conflicts (below) take those back, but that is no reason to do it. The heartbeat reports a missing scale set within a few minutes (`fleet-health.json` in its gist), and jobs fall back to GitHub-hosted runners meanwhile.
+
+Helm 4 and server-side apply. The role reads `helm version --short` and, on Helm 4 or later, installs and upgrades the controller and scale-set releases with `server_side: true` and `force_conflicts: true`, so a field another manager changed since the last upgrade, such as `maxRunners` after the autoscaler's patch, takes the chart's value instead of failing the upgrade. Only the conflicting fields the chart sets are taken over; a manager's other fields stay its own. A release first installed with client-side apply moves to server-side apply on its next upgrade. On Helm 3 neither option reaches the module. Both options need `kubernetes.core` 6.5.0 or later, which `galaxy.yml` requires. `tests/helm_apply/run.sh` covers it in a kind cluster with Helm 3 and Helm 4.
+
 - `github_runner_arc_controller_release_name`, `github_runner_arc_controller_namespace`: the controller's Helm release and namespace, default `arc` in `actions-runner-controller`. The stale-listener check, the heartbeat's RBAC and the heartbeat's controller check use them too.
 - `github_runner_arc_controller_replicas`: above 1, the chart turns on leader election and the role adds a preferred pod anti-affinity across nodes. `github_runner_arc_controller_extra_values` is merged over the role's controller values.
 - `github_runner_arc_metrics_enabled` (and the `_addr`/`_endpoint` settings): Prometheus metrics for the controller and every listener; the 0.14 chart only enables them together.

roles/github_runner_arc/tasks/install_controller.yml

@@ -35,3 +35,6 @@
     release_namespace: "{{ github_runner_arc_controller_namespace }}"
     create_namespace: true
     values: "{{ lookup('ansible.builtin.template', 'controller-values.yaml.j2') | from_yaml | default({}, true) | combine(github_runner_arc_controller_extra_values, recursive=true) }}"
+    # On Helm 4, server-side apply with conflicts forced: the role's values win over a field another manager changed since the last upgrade, and only for the fields the chart sets, so a manager's other fields stay its own. Without forcing, the upgrade fails on such a field. server_side is set explicitly because Helm refuses --force-conflicts when its default (auto) picks client-side apply for a release first installed that way, so such a release moves to server-side apply on its next upgrade. On Helm 3 neither option is passed: below Helm 4 the module refuses server_side in any form and force_conflicts when it is true, and both are omitted rather than passed as false.
+    server_side: "{{ 'true' if github_runner_arc_helm_server_side | bool else omit }}"
+    force_conflicts: "{{ true if github_runner_arc_helm_server_side | bool else omit }}"

roles/github_runner_arc/tasks/install_scale_set_profile.yml

@@ -119,6 +119,9 @@
     values: "{{ github_runner_arc_scale_set_values }}"
     # vars/main.yml says what is set this way and why.
     set_values: "{{ github_runner_arc_scale_set_set_values }}"
+    # On Helm 4, server-side apply with conflicts forced: the role's values win over a field another manager changed since the last upgrade, and only for the fields the chart sets, so a manager's other fields stay its own. Without forcing, the upgrade fails on such a field: the autoscaler patches maxRunners and so becomes its manager, and an upgrade setting a different value would fail rather than restore the values file's floor. server_side is set explicitly because Helm refuses --force-conflicts when its default (auto) picks client-side apply for a release first installed that way, so such a release moves to server-side apply on its next upgrade. On Helm 3 neither option is passed: below Helm 4 the module refuses server_side in any form and force_conflicts when it is true, and both are omitted rather than passed as false.
+    server_side: "{{ 'true' if github_runner_arc_helm_server_side | bool else omit }}"
+    force_conflicts: "{{ true if github_runner_arc_helm_server_side | bool else omit }}"
 
 # A Helm upgrade that changes the pod template renames the EphemeralRunnerSet the release owns (it carries a spec-hash suffix). The AutoscalingListener should follow the rename but can be left pointing at the deleted name, crash-looping on every patch while the scale set's queue goes unserviced; see the README's Non-obvious behaviour section. Deleting the stale listener makes the controller recreate it against the EphemeralRunnerSet that exists now.
 - name: Grant the runner pods the permission to record job peaks for {{ profile.label }}

roles/github_runner_arc/tasks/main.yml

@@ -66,6 +66,20 @@
     - name: Apply the ARC controller chart's CRDs, which Helm never upgrades
       ansible.builtin.include_tasks: install_crds.yml
 
+    # Helm 4 applies a release server-side and, without forcing, fails an upgrade on any field another manager changed since, so the controller and scale-set releases force conflicts on Helm 4 (see the helm tasks there). Read in check mode too, since the helm tasks' arguments depend on it.
+    - name: Read the Helm client's version
+      ansible.builtin.command:
+        argv: [helm, version, --short]
+      register: github_runner_arc_helm_version
+      changed_when: false
+      check_mode: false
+      when: github_runner_arc_orgs | length > 0
+
+    - name: Decide whether the helm releases are applied server-side
+      ansible.builtin.set_fact:
+        github_runner_arc_helm_server_side: "{{ github_runner_arc_helm_version.stdout | exadev.github_runner.arc_helm_server_side }}"
+      when: github_runner_arc_orgs | length > 0
+
     - name: Install/upgrade the ARC controller (shared by every org on this host)
       ansible.builtin.include_tasks: install_controller.yml
       when: github_runner_arc_orgs | length > 0

tests/helm_apply/kubernetes-core-floor.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Prints the lowest kubernetes.core version galaxy.yml accepts, read from its plain ">=X.Y.Z" requirement, for tests/helm_apply/run.sh and the Helm apply workflow, so both take the floor from one place. Exits non-zero with a message when the requirement is not such a bound.
+#
+# Usage: tests/helm_apply/kubernetes-core-floor.sh. Needs python3 with PyYAML (PYTHON overrides which python).
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "$0")/../.." && pwd)"
+"${PYTHON:-python3}" - "$repo_root/galaxy.yml" <<'PY'
+import re, sys
+import yaml
+spec = str(yaml.safe_load(open(sys.argv[1]))["dependencies"]["kubernetes.core"]).strip()
+match = re.fullmatch(r">=\s*(\d+\.\d+\.\d+)", spec)
+if match is None:
+    sys.exit(f"galaxy.yml's kubernetes.core requirement {spec!r} is not a plain >= lower bound, so the floor version cannot be read from it")
+print(match.group(1))
+PY