diff --git a/.github/workflows/helm-apply.yml b/.github/workflows/helm-apply.yml new file mode 100644 index 0000000..c9879f9 --- /dev/null +++ b/.github/workflows/helm-apply.yml @@ -0,0 +1,73 @@ +name: Helm apply + +# Runs the ARC role's Helm version read and scale-set helm task in a kind cluster with Helm 4 and Helm 3 (tests/helm_apply/run.sh): a release whose maxRunners kubectl has patched, as the autoscaler does, is upgraded to a different value, a release first installed client-side moves to server-side apply, and Helm 3 is passed neither server-side option. +# +# GitHub-hosted ubuntu-latest has Docker, kind and kubectl already. Both Helm releases are downloaded and checked against SHA-256 digests pinned here, taken from Helm's GitHub release notes and matching get.helm.sh's .sha256sum files, so the test depends neither on the image's own Helm nor on the download host vouching for its own archive. Each phase also runs with the oldest kubernetes.core galaxy.yml accepts, as well as the newest. + +on: + pull_request: + paths: + - roles/github_runner_arc/tasks/main.yml + - roles/github_runner_arc/tasks/install_controller.yml + - roles/github_runner_arc/tasks/install_scale_set_profile.yml + - plugins/filter/arc.py + - tests/helm_apply/** + - .github/workflows/helm-apply.yml + - galaxy.yml + workflow_dispatch: + +concurrency: + group: helm-apply-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + helm-apply: + name: Helm apply in kind + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + HELM3_VERSION: v3.18.6 + HELM3_SHA256: 3f43c0aa57243852dd542493a0f54f1396c0bc8ec7296bbb2c01e802010819ce + HELM4_VERSION: v4.3.0 + HELM4_SHA256: 86584a54def73570558f66f5111cc53dfed56689637ae32c1201205d494f54fb + steps: + - uses: actions/checkout@v7 + + - name: Install Ansible and the Kubernetes collection + run: | + set -euo pipefail + if ! python3 -m venv "${RUNNER_TEMP}/venv" 2>/dev/null; then + sudo apt-get update -qq + sudo apt-get install -y -qq python3-venv + python3 -m venv "${RUNNER_TEMP}/venv" + fi + "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core kubernetes pyyaml + echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" + # The newest kubernetes.core in the default path, and exactly the lowest one galaxy.yml accepts in a path of its own. + floor="$(PYTHON="${RUNNER_TEMP}/venv/bin/python" tests/helm_apply/kubernetes-core-floor.sh)" + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install "kubernetes.core>=${floor}" + # --force: otherwise ansible-galaxy counts the newest already in the default path as satisfying ==floor when the two are the same version, and installs nothing into the floor path. + "${RUNNER_TEMP}/venv/bin/ansible-galaxy" collection install --force "kubernetes.core==${floor}" -p "${RUNNER_TEMP}/kubernetes-core-floor" + + - name: Download Helm 3 and Helm 4 + run: | + set -euo pipefail + for pair in "${HELM3_VERSION}:${HELM3_SHA256}" "${HELM4_VERSION}:${HELM4_SHA256}"; do + version="${pair%%:*}" + archive="helm-${version}-linux-amd64.tar.gz" + curl -fsSLo "${RUNNER_TEMP}/${archive}" "https://get.helm.sh/${archive}" + # sha256sum -c exits non-zero on a mismatch, which fails the step under set -e. + echo "${pair#*:} ${RUNNER_TEMP}/${archive}" | sha256sum -c - + mkdir -p "${RUNNER_TEMP}/${version}" + tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}/${version}" + done + + - name: Run the Helm apply test + run: | + HELM3="${RUNNER_TEMP}/${HELM3_VERSION}/linux-amd64/helm" \ + HELM4="${RUNNER_TEMP}/${HELM4_VERSION}/linux-amd64/helm" \ + KUBERNETES_CORE_FLOOR="${RUNNER_TEMP}/kubernetes-core-floor" \ + tests/helm_apply/run.sh diff --git a/README.md b/README.md index b50d00c..d319e5f 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ Each poll (`AUTOSCALER_POLL_SECONDS`, default 45s): Ships with `AUTOSCALER_DRY_RUN=true` by default: it computes and logs the target and writes its own status, but never patches. Generate real concurrent traffic to watch it react: `gh workflow run test-autoscaler.yml`, then watch `kubectl logs -n github-runner-platform deploy/autoscaler -f` and `kubectl get configmap autoscaler-status -n github-runner-platform -o jsonpath='{.data.status\.json}'`. Only set `AUTOSCALER_DRY_RUN=false` after watching real dry-run output across genuine CI traffic. Status and its raise-confirm counter live in a shared Kubernetes ConfigMap (`autoscaler-status`, pre-created empty so neither pod's own ServiceAccount ever needs `create` RBAC on it, only `get`/`patch`) rather than a bind-mounted directory, since the two pods can land on different nodes now — `scripts/heartbeat.sh` reads that same ConfigMap back out and republishes it as a second file in the heartbeat gist, reusing its existing gist-write credential rather than giving the autoscaler its own. -Every Helm upgrade above reverts `maxRunners` to the values file's safe floor. The role's `install_org.yml` triggers one immediate autoscaler poll after every such upgrade (`kubectl exec deploy/autoscaler -- ...`, reaching the pod through the API server regardless of which node it's on), so the safe-floor window after a redeploy is seconds, not a full poll interval. +A Helm upgrade of a scale set (the role runs one when the release's rendered values or chart differ from what is installed) reverts `maxRunners` to the values file's safe floor. On Helm 4, which applies releases server-side, the autoscaler's patches make it the manager of `maxRunners`, so the role forces conflicts on the controller and scale-set releases (`force_conflicts`, only over the fields the chart sets); without that the upgrade would fail instead of reverting. The role's `install_org.yml` triggers one immediate autoscaler poll after every such upgrade (`kubectl exec deploy/autoscaler -- ...`, reaching the pod through the API server regardless of which node it's on), so the safe-floor window after a redeploy is seconds, not a full poll interval. ### Node recovery diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 92c99cd..2985148 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,5 +1,6 @@ --- collections: - - name: kubernetes.core # k8s, k8s_info, helm + - name: kubernetes.core # k8s, k8s_info, helm; 6.5.0 adds helm's server_side and force_conflicts + version: ">=6.5.0" - name: community.docker # docker_compose_v2 - name: community.general # homebrew (host prerequisites: helm, kubectl, docker) diff --git a/galaxy.yml b/galaxy.yml index cb7a64f..da13003 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -17,7 +17,7 @@ tags: - kubernetes - k3s dependencies: - kubernetes.core: ">=5.0.0" + kubernetes.core: ">=6.5.0" community.docker: ">=4.0.0" community.general: ">=9.0.0" repository: https://github.com/ExaDev/github-runner diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index b87cb58..3a82372 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -1155,6 +1155,26 @@ def arc_upgrade_blocker(deployed_chart: str, desired_version: str, running_runne ) +def arc_helm_server_side(version: str) -> bool: + """Report whether the Helm client applies releases with server-side apply, which decides whether the role forces field conflicts. + + Helm 4 upgrades a release with server-side apply when it was installed that way (its default), and fails on any field another field manager has since set to a different value, such as a scale set's maxRunners after the autoscaler has patched it. Helm 3 has no server-side apply; below Helm 4 the helm module refuses server_side in any form and force_conflicts when it is true, so the role passes neither there. + + Args: + version: the client version as `helm version --short` prints it, for example v4.3.0+gbec5b06 or v3.16.2+g13654a5. + + Returns: + True for Helm 4 or later, False for an earlier major version. + + Raises: + ValueError: when the version has no leading major number. + """ + match = re.match(r"^v?(\d+)\.", str(version).strip()) + if match is None: + raise ValueError(f"cannot read a Helm major version from {version!r}") + return int(match.group(1)) >= 4 + + class FilterModule: """Registers the ARC role's filters with Ansible.""" @@ -1165,6 +1185,7 @@ def filters(self) -> dict[str, Any]: "arc_max_runners": arc_max_runners, "arc_measured_max_runners": arc_measured_max_runners, "arc_upgrade_blocker": arc_upgrade_blocker, + "arc_helm_server_side": arc_helm_server_side, "arc_image_ref": arc_image_ref, "arc_runner_placement": arc_runner_placement, "arc_capability_errors": arc_capability_errors, diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index f5b71f1..256f046 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -30,7 +30,10 @@ Before touching the cluster the role then checks the secrets it is about to writ - `github_runner_arc_kubeconfig_path`: the kubeconfig on the host the role runs against. Defaults to `github_runner_cluster`'s kubeconfig; empty means `KUBECONFIG` or `~/.kube/config`. - `github_runner_arc_controller_chart_version`, `github_runner_arc_scaleset_chart_version`: chart pins; unset installs the latest chart. The controller chart's CRDs are applied (server-side) from `github_runner_arc_controller_chart_ref` at the controller's version before every controller install or upgrade, because Helm never upgrades a chart's CRDs itself; without that, a newer scale-set chart is rejected for fields the cluster's CRDs predate. Move the two pins together, and the CRDs follow. `tests/arc_upgrade/run.sh` covers the upgrade in a kind cluster. -Upgrading ARC on a live fleet. The new controller deletes every scale set whose version differs from its own (its log says "Autoscaling runner set version doesn't match the build version. Deleting the resource."), together with the set's listener and running ephemeral runners, until Helm upgrades that scale set. A controller version change therefore kills the jobs on running runners, so the role refuses it while any ephemeral runner is running (`github_runner_arc_allow_busy_controller_upgrade` accepts that), and a fleet with no running runner loses nothing. The check needs the controller chart version pinned, since an unpinned run cannot know what it will install. Dry-run (`--check`) cannot show a CRD or ownership failure, so it is not a rehearsal. Note the scale sets that exist before (`kubectl get autoscalingrunnerset -A`) and check the same set exists afterwards. Never apply a release's manifests by hand with `kubectl apply --server-side` to recover one: Helm's server-side apply then conflicts with `kubectl` on the labels a chart upgrade changes, and the ServiceAccount, Role and RoleBinding must be deleted for Helm to recreate them. The heartbeat reports a missing scale set within a few minutes (`fleet-health.json` in its gist), and jobs fall back to GitHub-hosted runners meanwhile. +Upgrading ARC on a live fleet. The new controller deletes every scale set whose version differs from its own (its log says "Autoscaling runner set version doesn't match the build version. Deleting the resource."), together with the set's listener and running ephemeral runners, until Helm upgrades that scale set. A controller version change therefore kills the jobs on running runners, so the role refuses it while any ephemeral runner is running (`github_runner_arc_allow_busy_controller_upgrade` accepts that), and a fleet with no running runner loses nothing. The check needs the controller chart version pinned, since an unpinned run cannot know what it will install. Dry-run (`--check`) cannot show a CRD or ownership failure, so it is not a rehearsal. Note the scale sets that exist before (`kubectl get autoscalingrunnerset -A`) and check the same set exists afterwards. Never apply a release's manifests by hand with `kubectl apply --server-side` to recover one: the ServiceAccount, Role and RoleBinding must then be deleted for Helm to recreate them. Such an apply also makes `kubectl` the manager of the labels a chart upgrade changes; on Helm 4 the role's forced conflicts (below) take those back, but that is no reason to do it. The heartbeat reports a missing scale set within a few minutes (`fleet-health.json` in its gist), and jobs fall back to GitHub-hosted runners meanwhile. + +Helm 4 and server-side apply. The role reads `helm version --short` and, on Helm 4 or later, installs and upgrades the controller and scale-set releases with `server_side: true` and `force_conflicts: true`, so a field another manager changed since the last upgrade, such as `maxRunners` after the autoscaler's patch, takes the chart's value instead of failing the upgrade. Only the conflicting fields the chart sets are taken over; a manager's other fields stay its own. A release first installed with client-side apply moves to server-side apply on its next upgrade. On Helm 3 neither option reaches the module. Both options need `kubernetes.core` 6.5.0 or later, which `galaxy.yml` requires. `tests/helm_apply/run.sh` covers it in a kind cluster with Helm 3 and Helm 4. + - `github_runner_arc_controller_release_name`, `github_runner_arc_controller_namespace`: the controller's Helm release and namespace, default `arc` in `actions-runner-controller`. The stale-listener check, the heartbeat's RBAC and the heartbeat's controller check use them too. - `github_runner_arc_controller_replicas`: above 1, the chart turns on leader election and the role adds a preferred pod anti-affinity across nodes. `github_runner_arc_controller_extra_values` is merged over the role's controller values. - `github_runner_arc_metrics_enabled` (and the `_addr`/`_endpoint` settings): Prometheus metrics for the controller and every listener; the 0.14 chart only enables them together. diff --git a/roles/github_runner_arc/tasks/install_controller.yml b/roles/github_runner_arc/tasks/install_controller.yml index ac50d02..a23d5eb 100644 --- a/roles/github_runner_arc/tasks/install_controller.yml +++ b/roles/github_runner_arc/tasks/install_controller.yml @@ -35,3 +35,6 @@ release_namespace: "{{ github_runner_arc_controller_namespace }}" create_namespace: true values: "{{ lookup('ansible.builtin.template', 'controller-values.yaml.j2') | from_yaml | default({}, true) | combine(github_runner_arc_controller_extra_values, recursive=true) }}" + # On Helm 4, server-side apply with conflicts forced: the role's values win over a field another manager changed since the last upgrade, and only for the fields the chart sets, so a manager's other fields stay its own. Without forcing, the upgrade fails on such a field. server_side is set explicitly because Helm refuses --force-conflicts when its default (auto) picks client-side apply for a release first installed that way, so such a release moves to server-side apply on its next upgrade. On Helm 3 neither option is passed: below Helm 4 the module refuses server_side in any form and force_conflicts when it is true, and both are omitted rather than passed as false. + server_side: "{{ 'true' if github_runner_arc_helm_server_side | bool else omit }}" + force_conflicts: "{{ true if github_runner_arc_helm_server_side | bool else omit }}" diff --git a/roles/github_runner_arc/tasks/install_scale_set_profile.yml b/roles/github_runner_arc/tasks/install_scale_set_profile.yml index ef58134..c39dc69 100644 --- a/roles/github_runner_arc/tasks/install_scale_set_profile.yml +++ b/roles/github_runner_arc/tasks/install_scale_set_profile.yml @@ -119,6 +119,9 @@ values: "{{ github_runner_arc_scale_set_values }}" # vars/main.yml says what is set this way and why. set_values: "{{ github_runner_arc_scale_set_set_values }}" + # On Helm 4, server-side apply with conflicts forced: the role's values win over a field another manager changed since the last upgrade, and only for the fields the chart sets, so a manager's other fields stay its own. Without forcing, the upgrade fails on such a field: the autoscaler patches maxRunners and so becomes its manager, and an upgrade setting a different value would fail rather than restore the values file's floor. server_side is set explicitly because Helm refuses --force-conflicts when its default (auto) picks client-side apply for a release first installed that way, so such a release moves to server-side apply on its next upgrade. On Helm 3 neither option is passed: below Helm 4 the module refuses server_side in any form and force_conflicts when it is true, and both are omitted rather than passed as false. + server_side: "{{ 'true' if github_runner_arc_helm_server_side | bool else omit }}" + force_conflicts: "{{ true if github_runner_arc_helm_server_side | bool else omit }}" # A Helm upgrade that changes the pod template renames the EphemeralRunnerSet the release owns (it carries a spec-hash suffix). The AutoscalingListener should follow the rename but can be left pointing at the deleted name, crash-looping on every patch while the scale set's queue goes unserviced; see the README's Non-obvious behaviour section. Deleting the stale listener makes the controller recreate it against the EphemeralRunnerSet that exists now. - name: Grant the runner pods the permission to record job peaks for {{ profile.label }} diff --git a/roles/github_runner_arc/tasks/main.yml b/roles/github_runner_arc/tasks/main.yml index 5087702..c2036db 100644 --- a/roles/github_runner_arc/tasks/main.yml +++ b/roles/github_runner_arc/tasks/main.yml @@ -66,6 +66,20 @@ - name: Apply the ARC controller chart's CRDs, which Helm never upgrades ansible.builtin.include_tasks: install_crds.yml + # Helm 4 applies a release server-side and, without forcing, fails an upgrade on any field another manager changed since, so the controller and scale-set releases force conflicts on Helm 4 (see the helm tasks there). Read in check mode too, since the helm tasks' arguments depend on it. + - name: Read the Helm client's version + ansible.builtin.command: + argv: [helm, version, --short] + register: github_runner_arc_helm_version + changed_when: false + check_mode: false + when: github_runner_arc_orgs | length > 0 + + - name: Decide whether the helm releases are applied server-side + ansible.builtin.set_fact: + github_runner_arc_helm_server_side: "{{ github_runner_arc_helm_version.stdout | exadev.github_runner.arc_helm_server_side }}" + when: github_runner_arc_orgs | length > 0 + - name: Install/upgrade the ARC controller (shared by every org on this host) ansible.builtin.include_tasks: install_controller.yml when: github_runner_arc_orgs | length > 0 diff --git a/tests/helm_apply/kubernetes-core-floor.sh b/tests/helm_apply/kubernetes-core-floor.sh new file mode 100755 index 0000000..28dd39c --- /dev/null +++ b/tests/helm_apply/kubernetes-core-floor.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Prints the lowest kubernetes.core version galaxy.yml accepts, read from its plain ">=X.Y.Z" requirement, for tests/helm_apply/run.sh and the Helm apply workflow, so both take the floor from one place. Exits non-zero with a message when the requirement is not such a bound. +# +# Usage: tests/helm_apply/kubernetes-core-floor.sh. Needs python3 with PyYAML (PYTHON overrides which python). +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +"${PYTHON:-python3}" - "$repo_root/galaxy.yml" <<'PY' +import re, sys +import yaml +spec = str(yaml.safe_load(open(sys.argv[1]))["dependencies"]["kubernetes.core"]).strip() +match = re.fullmatch(r">=\s*(\d+\.\d+\.\d+)", spec) +if match is None: + sys.exit(f"galaxy.yml's kubernetes.core requirement {spec!r} is not a plain >= lower bound, so the floor version cannot be read from it") +print(match.group(1)) +PY diff --git a/tests/helm_apply/run.sh b/tests/helm_apply/run.sh new file mode 100755 index 0000000..cc06d2e --- /dev/null +++ b/tests/helm_apply/run.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# Integration test for how the github_runner_arc role upgrades a scale-set release once something else has changed its maxRunners, as the autoscaler's kubectl patch does. Runs the role's own Helm version read and scale-set helm task (copied verbatim from roles/github_runner_arc/tasks into a task file) with Helm 4 and with Helm 3: +# +# - Helm 4: a release the role installed, then patched by kubectl, is upgraded to a different maxRunners. A plain helm upgrade without --force-conflicts is shown to fail on that field first, so the test reproduces the conflict it guards against. +# - Helm 4: a release first installed with client-side apply, then patched, is upgraded by the role, which needs server_side true and not auto. +# - Helm 3: the role installs and upgrades a patched release. kubernetes.core.helm refuses server_side in any form, and force_conflicts only when it is true, below Helm 4, so this phase fails if server_side or a true force_conflicts reaches the module. A false force_conflicts is accepted there, so this phase cannot show that the role omits it rather than passing false; only tests/unit/test_helm_apply.py's render test guards that. +# +# Every phase runs twice: with the kubernetes.core the environment provides, and with the oldest version galaxy.yml accepts, installed in KUBERNETES_CORE_FLOOR. +# +# No controller is installed, only the CRDs, so nothing reconciles the scale sets and no GitHub credential is needed. +# +# The floor version comes from tests/helm_apply/kubernetes-core-floor.sh, which the workflow uses too. +# +# Usage: HELM3= HELM4= KUBERNETES_CORE_FLOOR= tests/helm_apply/run.sh. Needs Docker, kind, kubectl, python3 with PyYAML, and ansible-playbook (ANSIBLE_PLAYBOOK overrides which) with the kubernetes.core collection (the version galaxy.yml accepts) and the kubernetes Python package; KUBERNETES_CORE_FLOOR is a collections directory holding kubernetes.core at exactly the lowest version galaxy.yml accepts. Creates a kind cluster named grtest-helm-apply and removes it on exit; GRTEST_KEEP=1 keeps it. +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +ansible_playbook="${ANSIBLE_PLAYBOOK:-ansible-playbook}" +# Indexed by Helm major version. +helm_binary=() +helm_binary[3]="${HELM3:?HELM3 must name a Helm 3 binary}" +helm_binary[4]="${HELM4:?HELM4 must name a Helm 4 binary}" +cluster=grtest-helm-apply +charts=oci://ghcr.io/actions/actions-runner-controller-charts +# The chart pair the fleet runs; any version with the AutoscalingRunnerSet maxRunners field would do. +chart_version=0.15.0 +controller_namespace=grtest-arc-systems +work="$(mktemp -d "${TMPDIR:-/tmp}/grtest-helm-apply.XXXXXX")" +export KUBECONFIG="$work/kubeconfig" +kubernetes_core_floor="${KUBERNETES_CORE_FLOOR:?KUBERNETES_CORE_FLOOR must name a collections directory holding the lowest kubernetes.core galaxy.yml accepts}" +default_collections="${ANSIBLE_COLLECTIONS_PATH:-$HOME/.ansible/collections:/usr/share/ansible/collections}" +# Without this, a phase whose collections path holds no kubernetes.core would fall back to one in Python's own site-packages (the full ansible package ships one) and test that version instead of failing; a kubernetes.core in a configured path already wins over site-packages. +export ANSIBLE_LOCALHOST_WARNING=false ANSIBLE_INVENTORY_UNPARSED_WARNING=false ANSIBLE_COLLECTIONS_SCAN_SYS_PATH=false + +log() { echo "==> $*"; } +# Set once the kind cluster exists, so a failure before then does not try to list resources from a cluster that is not there. +cluster_created=0 +fail() { + echo "FAIL: $*" >&2 + if [ "$cluster_created" = 1 ]; then + kubectl get autoscalingrunnersets -A -o wide >&2 || true + fi + exit 1 +} +cleanup() { + if [ "${GRTEST_KEEP:-0}" != 1 ]; then + kind delete cluster --name "$cluster" >/dev/null 2>&1 || true + rm -rf "$work" + fi +} +trap cleanup EXIT + +# One directory per Helm major holding a helm that is that binary, so a step chooses its Helm through PATH, the way the module and the version read find it. +for major in 3 4; do + mkdir -p "$work/helm$major" + ln -s "${helm_binary[$major]}" "$work/helm$major/helm" + "$work/helm$major/helm" version --short | grep -q "^v$major\." || fail "HELM$major is not a Helm $major binary" +done + +log "Copying the role's version read and scale-set helm task" +python3 - "$repo_root" "$work/helm-tasks.yml" <<'PY' +import pathlib, sys +import yaml + +root, out = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) + +def walk(tasks): + for task in tasks or []: + yield task + for key in ("block", "rescue", "always"): + yield from walk(task.get(key)) + +main = list(walk(yaml.safe_load((root / "roles/github_runner_arc/tasks/main.yml").read_text()))) +version = [task for task in main if task.get("register") == "github_runner_arc_helm_version" or "github_runner_arc_helm_server_side" in task.get("ansible.builtin.set_fact", {})] +helm = [task for task in walk(yaml.safe_load((root / "roles/github_runner_arc/tasks/install_scale_set_profile.yml").read_text())) if "kubernetes.core.helm" in task] +assert len(version) == 2, f"expected the version read and the fact in main.yml, found {len(version)}" +assert len(helm) == 1, f"expected one helm task in install_scale_set_profile.yml, found {len(helm)}" +out.write_text(yaml.safe_dump(version + helm, sort_keys=False)) +PY + +# Runs the role's tasks with the Helm of the given major on PATH, for one release at one maxRunners, resolving kubernetes.core from the calling run_phases' collections path; prints the play's output. +role_upgrade() { + local major="$1" release="$2" namespace="$3" max_runners="$4" + printf '%s' "{\"helm_tasks_file\": \"$work/helm-tasks.yml\", \"release\": \"$release\", \"namespace\": \"$namespace\", \"chart_version\": \"$chart_version\", \"kubeconfig\": \"$KUBECONFIG\", \"scale_set_values\": $(scale_set_values "$max_runners")}" > "$work/extra.json" + PATH="$work/helm$major:$PATH" ANSIBLE_COLLECTIONS_PATH="$repo_root/playbooks/collections:$collections" \ + "$ansible_playbook" "$repo_root/tests/helm_apply/upgrade.yml" -e "@$work/extra.json" 2>&1 +} + +# The kubernetes.core version a collections path resolves to, as ansible-galaxy reports it. +kubernetes_core_version() { + ANSIBLE_COLLECTIONS_PATH="$1" ansible-galaxy collection list kubernetes.core 2>/dev/null | awk '$1 == "kubernetes.core" {print $2; exit}' +} + +# The scale set's Helm values: no GitHub credential is used, since no controller runs, and the controller's service account is named because there is no controller to look it up from. +scale_set_values() { + printf '%s' "{\"maxRunners\": $1, \"minRunners\": 0, \"githubConfigUrl\": \"https://github.com/example\", \"githubConfigSecret\": {\"github_token\": \"unused\"}, \"controllerServiceAccount\": {\"name\": \"arc-gha-rs-controller\", \"namespace\": \"$controller_namespace\"}}" +} + +live_max_runners() { kubectl get autoscalingrunnerset "$1" -n "$2" -o jsonpath='{.spec.maxRunners}'; } + +# Changes maxRunners the way the autoscaler does, which makes kubectl-patch that field's manager. +autoscaler_patch() { kubectl patch autoscalingrunnerset "$1" -n "$2" --type merge -p "{\"spec\":{\"maxRunners\":$3}}" >/dev/null; } + +# Before the cluster is created, so a missing or wrong floor installation fails first. +floor="$("$repo_root/tests/helm_apply/kubernetes-core-floor.sh")" || fail "could not read the lowest kubernetes.core galaxy.yml accepts (the reason is above)" +floor_installed="$(kubernetes_core_version "$kubernetes_core_floor")" +[ "$floor_installed" = "$floor" ] || fail "KUBERNETES_CORE_FLOOR ($kubernetes_core_floor) holds ${floor_installed:+kubernetes.core }${floor_installed:-no kubernetes.core}, not galaxy.yml's lowest accepted $floor" +[ "$(kubernetes_core_version "$default_collections")" ] || fail "no kubernetes.core in $default_collections" + +log "Creating the kind cluster and applying the ARC CRDs (no controller)" +kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 120s >/dev/null +cluster_created=1 +"$work/helm4/helm" show crds "$charts/gha-runner-scale-set-controller" --version "$chart_version" | kubectl apply --server-side -f - >/dev/null + +# Runs every phase with the role's tasks resolving kubernetes.core from the given collections path; tag keeps each run's namespaces and logs apart. +run_phases() { + local tag="$1" collections="$2" version + local ns4="grtest-helm4-$tag" ns4_csa="grtest-helm4-csa-$tag" ns3="grtest-helm3-$tag" + version="$(kubernetes_core_version "$collections")" + [ -n "$version" ] || fail "no kubernetes.core in $collections" + log "[kubernetes.core $version] Helm 4: the role installs a release, kubectl patches its maxRunners" + kubectl create namespace "$ns4" >/dev/null + role_upgrade 4 grtest-runners "$ns4" 3 > "$work/helm4-install-$tag.log" || fail "the role could not install the release with Helm 4: $(tail -20 "$work/helm4-install-$tag.log")" + [ "$(live_max_runners grtest-runners "$ns4")" = 3 ] || fail "maxRunners after install is not 3" + "$work/helm4/helm" get metadata grtest-runners -n "$ns4" | grep -q "APPLY_METHOD: server-side apply" || fail "Helm 4 did not install the release server-side" + autoscaler_patch grtest-runners "$ns4" 5 + + log "[kubernetes.core $version] Helm 4: a plain upgrade without --force-conflicts fails on maxRunners" + if output="$(printf '%s' "$(scale_set_values 2)" | "$work/helm4/helm" upgrade grtest-runners "$charts/gha-runner-scale-set" --version "$chart_version" -n "$ns4" -f - 2>&1)"; then + fail "a plain Helm 4 upgrade succeeded against the patched maxRunners, so this test no longer reproduces the conflict: $output" + fi + echo "$output" | grep -q "conflict" || fail "the plain upgrade failed, but not on a conflict: $output" + + log "[kubernetes.core $version] Helm 4: the role upgrades the patched release to a different maxRunners" + role_upgrade 4 grtest-runners "$ns4" 2 > "$work/helm4-upgrade-$tag.log" || fail "the role's upgrade failed with Helm 4: $(tail -20 "$work/helm4-upgrade-$tag.log")" + [ "$(live_max_runners grtest-runners "$ns4")" = 2 ] || fail "maxRunners after the role's upgrade is $(live_max_runners grtest-runners "$ns4"), not 2" + + log "[kubernetes.core $version] Helm 4: a release first installed with client-side apply, then patched, is upgraded by the role" + kubectl create namespace "$ns4_csa" >/dev/null + printf '%s' "$(scale_set_values 3)" | "$work/helm4/helm" install grtest-runners "$charts/gha-runner-scale-set" --version "$chart_version" -n "$ns4_csa" --server-side=false -f - >/dev/null + autoscaler_patch grtest-runners "$ns4_csa" 5 + role_upgrade 4 grtest-runners "$ns4_csa" 2 > "$work/helm4-csa-$tag.log" || fail "the role could not upgrade a client-side release with Helm 4: $(tail -20 "$work/helm4-csa-$tag.log")" + [ "$(live_max_runners grtest-runners "$ns4_csa")" = 2 ] || fail "maxRunners after upgrading the client-side release is not 2" + "$work/helm4/helm" get metadata grtest-runners -n "$ns4_csa" | grep -q "APPLY_METHOD: server-side apply" || fail "the client-side release did not move to server-side apply" + + log "[kubernetes.core $version] Helm 3: the role installs and upgrades a patched release, passing neither option" + kubectl create namespace "$ns3" >/dev/null + role_upgrade 3 grtest-runners "$ns3" 3 > "$work/helm3-install-$tag.log" || fail "the role could not install the release with Helm 3: $(tail -20 "$work/helm3-install-$tag.log")" + autoscaler_patch grtest-runners "$ns3" 5 + role_upgrade 3 grtest-runners "$ns3" 2 > "$work/helm3-upgrade-$tag.log" || fail "the role's upgrade failed with Helm 3: $(tail -20 "$work/helm3-upgrade-$tag.log")" + [ "$(live_max_runners grtest-runners "$ns3")" = 2 ] || fail "maxRunners after the Helm 3 upgrade is not 2" +} + +run_phases current "$default_collections" +run_phases floor "$kubernetes_core_floor" + +log "PASS" diff --git a/tests/helm_apply/upgrade.yml b/tests/helm_apply/upgrade.yml new file mode 100644 index 0000000..2ad6a72 --- /dev/null +++ b/tests/helm_apply/upgrade.yml @@ -0,0 +1,22 @@ +--- +# Runs the github_runner_arc role's own Helm version read and its scale-set helm task against a test cluster, for tests/helm_apply/run.sh, which copies those tasks verbatim from the role into helm_tasks_file. Takes helm_tasks_file, release, namespace, chart_version, scale_set_values (the release's Helm values) and kubeconfig as extra vars. The role's defaults are not loaded: they would outrank these play variables, and the tasks need nothing from them. +- name: Install or upgrade a scale set with the role's helm task + hosts: localhost + connection: local + gather_facts: false + vars: + # Any non-empty list: the version read runs only when the role installs at least one org. + github_runner_arc_orgs: [{name: example}] + github_runner_arc_scaleset_chart_version: "{{ chart_version }}" + profile: + release: "{{ release }}" + namespace: "{{ namespace }}" + label: "{{ release }}" + github_runner_arc_scale_set_values: "{{ scale_set_values }}" + github_runner_arc_scale_set_set_values: [] + module_defaults: + group/kubernetes.core.helm: + kubeconfig: "{{ kubeconfig }}" + tasks: + - name: Include the role's version read and scale-set helm task + ansible.builtin.include_tasks: "{{ helm_tasks_file }}" diff --git a/tests/unit/test_helm_apply.py b/tests/unit/test_helm_apply.py new file mode 100644 index 0000000..9195237 --- /dev/null +++ b/tests/unit/test_helm_apply.py @@ -0,0 +1,146 @@ +"""Tests for how the github_runner_arc role applies its Helm releases: the Helm version filter (plugins/filter/arc.py's arc_helm_server_side) and the options every kubernetes.core.helm task in the roles passes, so a release the autoscaler has patched since its last upgrade can still be upgraded on Helm 4, and a Helm 3 host is passed neither option. The options are rendered by ansible-playbook itself, against a stand-in for the helm module that only reports the arguments it receives.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import tempfile +import textwrap +import unittest +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import yaml +from ansible.module_utils.parsing.convert_bool import boolean + +ROOT = Path(__file__).resolve().parents[2] +PLUGIN = ROOT / "plugins" / "filter" / "arc.py" +_spec = importlib.util.spec_from_file_location("arc_filters", PLUGIN) +assert _spec is not None and _spec.loader is not None +arc = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(arc) + +FACT = "github_runner_arc_helm_server_side" +HELM_MODULES = ("kubernetes.core.helm", "helm") +OPTIONS = ("server_side", "force_conflicts") + +# Stands in for kubernetes.core.helm: an old-style module (WANT_JSON) that accepts any arguments and reports the ones Ansible passed it, so an omitted option is absent from the result. +FAKE_HELM = textwrap.dedent( + """\ + #!/usr/bin/python + # WANT_JSON + import json, sys + args = json.load(open(sys.argv[1])) + print(json.dumps({"changed": False, "received": {key: value for key, value in args.items() if not key.startswith("_ansible")}})) + """ +) + + +def tasks(items: Any) -> Iterator[dict[str, Any]]: + """Yield every task in a task list, descending into block, rescue and always.""" + for task in items or []: + yield task + for key in ("block", "rescue", "always"): + yield from tasks(task.get(key)) + + +def helm_tasks() -> list[tuple[str, dict[str, Any]]]: + """Return every task in the roles that calls the helm module, with the file it is in.""" + found = [] + for path in sorted(ROOT.glob("roles/*/tasks/**/*.yml")): + for task in tasks(yaml.safe_load(path.read_text())): + for module in HELM_MODULES: + if module in task: + found.append((f"{path.relative_to(ROOT)}: {task.get('name')}", task[module])) + return found + + +def render_options(options: dict[str, Any], server_side: bool) -> dict[str, Any]: + """Run a task with the given helm options, exactly as the role writes them, through ansible-playbook with the version fact set, and return the arguments the module received.""" + with tempfile.TemporaryDirectory() as work: + root = Path(work) + module = root / "collections" / "ansible_collections" / "kubernetes" / "core" / "plugins" / "modules" / "helm.py" + module.parent.mkdir(parents=True) + module.write_text(FAKE_HELM) + (module.parents[2] / "galaxy.yml").write_text("namespace: kubernetes\nname: core\nversion: 6.6.0\n") + output = root / "received.json" + (root / "helm-task.yml").write_text(yaml.safe_dump([{"name": "Run the helm task's options", "kubernetes.core.helm": options, "register": "helm_result"}])) + (root / "play.yml").write_text(yaml.safe_dump([{ + "hosts": "localhost", "connection": "local", "gather_facts": False, + "vars": {FACT: server_side}, + "tasks": [ + {"ansible.builtin.include_tasks": str(root / "helm-task.yml")}, + {"ansible.builtin.copy": {"content": "{{ helm_result.received | to_json }}", "dest": str(output), "mode": "0644"}}, + ], + }])) + env = {**os.environ, "ANSIBLE_COLLECTIONS_PATH": str(root / "collections"), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false"} + subprocess.run([os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook"), str(root / "play.yml")], check=True, env=env, cwd=work, capture_output=True, text=True) + received: dict[str, Any] = json.loads(output.read_text()) + return received + + +class HelmServerSideTest(unittest.TestCase): + def test_helm_4_and_later_apply_server_side(self) -> None: + for version in ("v4.3.0+gbec5b06", "v4.0.0", "v5.1.2", "4.3.0"): + with self.subTest(version=version): + self.assertTrue(arc.arc_helm_server_side(version)) + + def test_helm_3_does_not(self) -> None: + for version in ("v3.16.2+g13654a5", "v3.0.0"): + with self.subTest(version=version): + self.assertFalse(arc.arc_helm_server_side(version)) + + def test_an_unreadable_version_is_an_error_not_a_guess(self) -> None: + for version in ("", "unknown", "vX.1"): + with self.subTest(version=version), self.assertRaises(ValueError): + arc.arc_helm_server_side(version) + + +class HelmTasksTest(unittest.TestCase): + def test_the_role_installs_helm_releases(self) -> None: + names = [name for name, _ in helm_tasks()] + self.assertTrue(any("install_controller.yml" in name for name in names), names) + self.assertTrue(any("install_scale_set_profile.yml" in name for name in names), names) + + def test_every_helm_release_forces_conflicts_on_helm_4(self) -> None: + for name, args in helm_tasks(): + with self.subTest(task=name): + self.assertIn(FACT, str(args.get("force_conflicts"))) + self.assertIn(FACT, str(args.get("server_side"))) + self.assertNotIn("force", args, "force means a client-side replace, which cannot be combined with force_conflicts") + + def test_helm_4_gets_server_side_apply_with_forced_conflicts(self) -> None: + for name, args in helm_tasks(): + with self.subTest(task=name): + received = render_options({key: args[key] for key in OPTIONS if key in args}, True) + self.assertEqual(set(received), set(OPTIONS)) + # "true", not "auto": with auto Helm picks client-side apply for a release first installed that way, and then refuses to force conflicts. + self.assertEqual(str(received["server_side"]), "true") + self.assertTrue(boolean(received["force_conflicts"])) + + def test_helm_3_is_passed_neither_option(self) -> None: + # Below Helm 4 kubernetes.core.helm refuses server_side in any form and force_conflicts when it is true. Both are omitted rather than passed as false, so the role does not depend on how the module treats a false force_conflicts (and on Helm 3 needs no module version that knows the option). Only this test guards that: tests/helm_apply/run.sh's Helm 3 phase passes with a false force_conflicts. + for name, args in helm_tasks(): + with self.subTest(task=name): + self.assertEqual(render_options({key: args[key] for key in OPTIONS if key in args}, False), {}) + + def test_the_version_is_read_before_any_release_is_installed(self) -> None: + main = list(tasks(yaml.safe_load((ROOT / "roles" / "github_runner_arc" / "tasks" / "main.yml").read_text()))) + names = [task.get("name", "") for task in main] + decide = next(index for index, task in enumerate(main) if FACT in str(task.get("ansible.builtin.set_fact", {}))) + controller = next(index for index, task in enumerate(main) if task.get("ansible.builtin.include_tasks") == "install_controller.yml") + orgs = next(index for index, task in enumerate(main) if task.get("ansible.builtin.include_tasks") == "install_org.yml") + self.assertLess(decide, controller, names) + self.assertLess(decide, orgs, names) + read = main[decide - 1] + self.assertEqual(read["ansible.builtin.command"]["argv"], ["helm", "version", "--short"]) + self.assertFalse(read["check_mode"], "check mode must still read the version, since the helm tasks' arguments depend on it") + self.assertEqual(main[decide]["when"], read["when"]) + self.assertEqual(main[controller]["when"], read["when"]) + + +if __name__ == "__main__": + unittest.main()