|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Integration test for the autoscaler's ceiling (scripts/autoscaler.sh, which runs scripts/autoscaler-ceiling.jq) on a real cluster. Two scale sets are installed from the chart the fleet runs, with no controller, so nothing reconciles them and no GitHub credential is needed. The autoscaler polls as its own service account would, through kubectl impersonation, holding exactly the RBAC the role's install_platform.yml grants (rendered from that file), and patches the scale sets for real: |
| 3 | +# |
| 4 | +# - the ceiling it works out equals the role's install-time measurement (plugins/filter/arc.py's arc_measured_max_runners) of the same nodes and pods, and a pool above it is lowered to it; |
| 5 | +# - a pod of another workload taking a runner's worth of cpu lowers the ceiling and the pool, and once it is gone the pool is raised back after the confirmation polls; |
| 6 | +# - a runner template the ceiling cannot be worked out from keeps the last ceiling and never raises the pool; |
| 7 | +# - a cordoned node is not counted, so the pool comes down to its running count. |
| 8 | +# |
| 9 | +# The runner pod's cpu request is chosen from the node's free cpu so that cpu decides the ceiling, whatever size of node the cluster has. |
| 10 | +# |
| 11 | +# Usage: tests/autoscaler_ceiling/run.sh. Needs Docker, kind, kubectl, helm, jq, and python3 (PYTHON overrides which) with PyYAML, Jinja2 and ansible-core (plugins/filter/arc.py imports it). Creates a kind cluster named grtest-ceiling with metrics-server and removes it on exit; GRTEST_KEEP=1 keeps it. With GRTEST_KUBECONFIG and GRTEST_DISPOSABLE=1 it uses that cluster instead, which must have exactly one node and a working metrics API (k3s's own metrics-server, for one); the test labels and cordons that node, so it refuses a cluster not declared disposable. |
| 12 | +set -euo pipefail |
| 13 | + |
| 14 | +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" |
| 15 | +python="${PYTHON:-python3}" |
| 16 | +cluster=grtest-ceiling |
| 17 | +charts=oci://ghcr.io/actions/actions-runner-controller-charts |
| 18 | +# The chart pair the fleet runs; any version with the AutoscalingRunnerSet maxRunners field would do. |
| 19 | +chart_version=0.15.0 |
| 20 | +# metrics-server for kind, which ships none, pinned by digest: the release's own components.yaml. |
| 21 | +metrics_server_version=v0.7.2 |
| 22 | +metrics_server_sha256=f103539a54ed72efe66616afc74a8bfaed651703cb3918797599046af5617441 |
| 23 | +platform_namespace=grtest-ceiling-platform |
| 24 | +pooled_namespaces=(grtest-ceiling-a grtest-ceiling-b) |
| 25 | +other_namespace=grtest-ceiling-other |
| 26 | +eligibility_label=grtest.example/eligible |
| 27 | +initial_max_runners=3 |
| 28 | +# The cpu request is the node's free cpu divided by this, so that cpu alone places this many runners on the node. |
| 29 | +runners_by_cpu=4 |
| 30 | +# A memory request small enough that memory never decides the ceiling on any node this test runs on. |
| 31 | +runner_memory=64Mi |
| 32 | +work="$(mktemp -d "${TMPDIR:-/tmp}/grtest-ceiling.XXXXXX")" |
| 33 | + |
| 34 | +log() { echo "==> $*"; } |
| 35 | +cluster_created=0 |
| 36 | +fail() { |
| 37 | + echo "FAIL: $*" >&2 |
| 38 | + if [ -s "$work/poll.log" ]; then |
| 39 | + echo "--- last poll" >&2 |
| 40 | + cat "$work/poll.log" >&2 |
| 41 | + fi |
| 42 | + exit 1 |
| 43 | +} |
| 44 | +cleanup() { |
| 45 | + if [ "${GRTEST_KEEP:-0}" != 1 ]; then |
| 46 | + if [ "$cluster_created" = 1 ]; then |
| 47 | + kind delete cluster --name "$cluster" >/dev/null 2>&1 || true |
| 48 | + elif [ -n "${GRTEST_KUBECONFIG:-}" ]; then |
| 49 | + kubectl uncordon "${node:-}" >/dev/null 2>&1 || true |
| 50 | + # The chart puts the controller's cleanup finalizer on what it installs, and no controller runs here to remove it, so the namespaces would never finish terminating. |
| 51 | + for namespace in "${pooled_namespaces[@]}"; do |
| 52 | + kubectl get autoscalingrunnersets,roles,rolebindings,serviceaccounts,secrets -n "$namespace" -o name 2>/dev/null | while read -r object; do |
| 53 | + kubectl patch "$object" -n "$namespace" --type merge -p '{"metadata":{"finalizers":null}}' >/dev/null 2>&1 || true |
| 54 | + done |
| 55 | + done |
| 56 | + kubectl delete namespace "$platform_namespace" "${pooled_namespaces[@]}" "$other_namespace" --ignore-not-found --wait=false >/dev/null 2>&1 || true |
| 57 | + fi |
| 58 | + rm -rf "$work" |
| 59 | + fi |
| 60 | +} |
| 61 | +trap cleanup EXIT |
| 62 | + |
| 63 | +if [ -n "${GRTEST_KUBECONFIG:-}" ]; then |
| 64 | + [ "${GRTEST_DISPOSABLE:-0}" = 1 ] || fail "GRTEST_KUBECONFIG names a cluster this test will label and cordon a node of; set GRTEST_DISPOSABLE=1 to say it is disposable" |
| 65 | + export KUBECONFIG="$GRTEST_KUBECONFIG" |
| 66 | +else |
| 67 | + export KUBECONFIG="$work/kubeconfig" |
| 68 | + log "Creating the kind cluster and installing metrics-server" |
| 69 | + kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 120s >/dev/null |
| 70 | + cluster_created=1 |
| 71 | + curl -fsSLo "$work/metrics-server.yaml" "https://github.com/kubernetes-sigs/metrics-server/releases/download/${metrics_server_version}/components.yaml" |
| 72 | + echo "${metrics_server_sha256} $work/metrics-server.yaml" | sha256sum -c - >/dev/null || fail "metrics-server's components.yaml does not match its pinned digest" |
| 73 | + kubectl apply -f "$work/metrics-server.yaml" >/dev/null |
| 74 | + # kind's kubelets serve certificates the cluster's CA did not sign. |
| 75 | + kubectl patch deployment metrics-server -n kube-system --type json -p '[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--kubelet-insecure-tls"}]' >/dev/null |
| 76 | +fi |
| 77 | + |
| 78 | +mapfile -t nodes < <(kubectl get nodes -o name) |
| 79 | +[ "${#nodes[@]}" -eq 1 ] || fail "the cluster has ${#nodes[@]} nodes; this test needs exactly one" |
| 80 | +node="${nodes[0]#node/}" |
| 81 | + |
| 82 | +log "Waiting for the metrics API" |
| 83 | +for _ in $(seq 1 60); do |
| 84 | + kubectl top nodes --no-headers >/dev/null 2>&1 && break |
| 85 | + sleep 5 |
| 86 | +done |
| 87 | +kubectl top nodes --no-headers >/dev/null || fail "the metrics API never answered" |
| 88 | + |
| 89 | +kubectl label node "$node" "${eligibility_label}=true" --overwrite >/dev/null |
| 90 | +helm show crds "$charts/gha-runner-scale-set-controller" --version "$chart_version" | kubectl apply --server-side -f - >/dev/null |
| 91 | + |
| 92 | +# The role's install-time measurement of the node as it is now, with a runner pod of the given cpu request, and the node's free cpu in millicores: a reference worked out in Python's exact decimals, apart from the program under test. |
| 93 | +reference() { |
| 94 | + kubectl get nodes -o json > "$work/nodes.json" |
| 95 | + kubectl get pods --all-namespaces -o json > "$work/pods.json" |
| 96 | + "$python" - "$repo_root" "$work" "$1" "$eligibility_label" "${pooled_namespaces[@]}" <<'PY' |
| 97 | +import importlib.util, json, pathlib, sys |
| 98 | +from decimal import Decimal |
| 99 | +
|
| 100 | +root, work, cpu, label, pooled = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), sys.argv[3], sys.argv[4], sys.argv[5:] |
| 101 | +spec = importlib.util.spec_from_file_location("arc_filters", root / "plugins/filter/arc.py") |
| 102 | +arc = importlib.util.module_from_spec(spec) |
| 103 | +spec.loader.exec_module(arc) |
| 104 | +nodes = json.loads((work / "nodes.json").read_text())["items"] |
| 105 | +pods = json.loads((work / "pods.json").read_text())["items"] |
| 106 | +sizing = {"measured": True, "pod_cpu_request": str(arc.arc_quantity(cpu)), "pod_memory_request_gib": "0.0625"} |
| 107 | +measured = arc.arc_measured_max_runners(sizing, nodes, pods, {label: "true"}, pooled) |
| 108 | +assert not measured["errors"], measured["errors"] |
| 109 | +node = measured["nodes"][0] |
| 110 | +free = (Decimal(str(node["allocatable_cpu"])) - Decimal(str(node["requested_cpu"]))) * 1000 |
| 111 | +print(measured["max_runners"], int(free)) |
| 112 | +PY |
| 113 | +} |
| 114 | + |
| 115 | +read -r _ free_millicores < <(reference 1) |
| 116 | +runner_cpu="$(( free_millicores / runners_by_cpu ))m" |
| 117 | +read -r expected _ < <(reference "$runner_cpu") |
| 118 | +[ "$expected" -eq "$runners_by_cpu" ] || fail "a ${runner_cpu} runner should place ${runners_by_cpu} on the node by cpu, but the reference places ${expected}" |
| 119 | +log "The node has ${free_millicores}m of cpu free; a ${runner_cpu} runner places ${expected}" |
| 120 | + |
| 121 | +log "Installing two scale sets with no controller" |
| 122 | +for namespace in "${pooled_namespaces[@]}"; do |
| 123 | + kubectl create namespace "$namespace" >/dev/null |
| 124 | + jq -n --arg cpu "$runner_cpu" --arg memory "$runner_memory" --arg label "$eligibility_label" --argjson max "$initial_max_runners" '{ |
| 125 | + maxRunners: $max, minRunners: 0, githubConfigUrl: "https://github.com/example", githubConfigSecret: {github_token: "unused"}, |
| 126 | + controllerServiceAccount: {name: "arc-gha-rs-controller", namespace: "grtest-arc-systems"}, |
| 127 | + template: {spec: {nodeSelector: {($label): "true"}, containers: [{name: "runner", image: "ghcr.io/actions/actions-runner:latest", command: ["/home/runner/run.sh"], |
| 128 | + resources: {requests: {cpu: $cpu, memory: $memory}, limits: {cpu: $cpu, memory: $memory}}}]}} |
| 129 | + }' > "$work/values-$namespace.json" |
| 130 | + helm install grtest-runners "$charts/gha-runner-scale-set" --version "$chart_version" -n "$namespace" -f "$work/values-$namespace.json" >/dev/null |
| 131 | +done |
| 132 | + |
| 133 | +log "Granting the autoscaler's service account exactly the role's RBAC" |
| 134 | +kubectl create namespace "$platform_namespace" >/dev/null |
| 135 | +kubectl create namespace "$other_namespace" >/dev/null |
| 136 | +kubectl create serviceaccount autoscaler -n "$platform_namespace" >/dev/null |
| 137 | +kubectl create configmap autoscaler-status -n "$platform_namespace" >/dev/null |
| 138 | +"$python" - "$repo_root" "$platform_namespace" "${pooled_namespaces[@]}" > "$work/rbac.yaml" <<'PY' |
| 139 | +import pathlib, sys |
| 140 | +import jinja2, yaml |
| 141 | +
|
| 142 | +root, platform, pooled = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3:] |
| 143 | +tasks = yaml.safe_load((root / "roles/github_runner_arc/tasks/install_platform.yml").read_text()) |
| 144 | +blocks = [task for task in tasks if task.get("name") == "Platform: autoscaler RBAC"] |
| 145 | +assert len(blocks) == 1, f"expected one autoscaler RBAC block in install_platform.yml, found {len(blocks)}" |
| 146 | +environment = jinja2.Environment(undefined=jinja2.StrictUndefined) |
| 147 | +documents = [] |
| 148 | +for task in blocks[0]["block"]: |
| 149 | + template = environment.from_string(yaml.safe_dump(task["kubernetes.core.k8s"]["definition"])) |
| 150 | + items = [{"namespace": namespace, "release": "grtest-runners", "label": namespace} for namespace in pooled] if "loop" in task else [None] |
| 151 | + for item in items: |
| 152 | + documents.append(yaml.safe_load(template.render(github_runner_arc_platform_namespace=platform, item=item))) |
| 153 | +print(yaml.safe_dump_all(documents)) |
| 154 | +PY |
| 155 | +kubectl apply -f "$work/rbac.yaml" >/dev/null |
| 156 | + |
| 157 | +# The autoscaler's kubectl, as its service account. |
| 158 | +mkdir -p "$work/bin" |
| 159 | +real_kubectl="$(command -v kubectl)" |
| 160 | +printf '#!/bin/sh\nexec %s --as=system:serviceaccount:%s:autoscaler "$@"\n' "$real_kubectl" "$platform_namespace" > "$work/bin/kubectl" |
| 161 | +chmod +x "$work/bin/kubectl" |
| 162 | + |
| 163 | +targets="" |
| 164 | +for namespace in "${pooled_namespaces[@]}"; do targets="$targets $namespace/grtest-runners"; done |
| 165 | + |
| 166 | +# One poll, for real (not dry run), its output kept for the checks and for a failure's report. |
| 167 | +poll() { |
| 168 | + PATH="$work/bin:$PATH" \ |
| 169 | + AUTOSCALER_TARGETS="${targets# }" \ |
| 170 | + AUTOSCALER_DRY_RUN=false \ |
| 171 | + AUTOSCALER_USABLE_BUDGET_GI=1024 \ |
| 172 | + AUTOSCALER_MAX_CEILING="" \ |
| 173 | + AUTOSCALER_FLOOR=1 \ |
| 174 | + AUTOSCALER_NODE_SELECTOR="${eligibility_label}=true" \ |
| 175 | + AUTOSCALER_STATE_NAMESPACE="$platform_namespace" \ |
| 176 | + AUTOSCALER_STATUS_CONFIGMAP=autoscaler-status \ |
| 177 | + "$repo_root/scripts/autoscaler.sh" > "$work/poll.log" 2>&1 || fail "the poll exited non-zero" |
| 178 | +} |
| 179 | +pooled_total() { |
| 180 | + local namespace total=0 |
| 181 | + for namespace in "${pooled_namespaces[@]}"; do |
| 182 | + total=$(( total + $(kubectl get autoscalingrunnerset grtest-runners -n "$namespace" -o jsonpath='{.spec.maxRunners}') )) |
| 183 | + done |
| 184 | + echo "$total" |
| 185 | +} |
| 186 | +status() { kubectl get configmap autoscaler-status -n "$platform_namespace" -o jsonpath='{.data.status\.json}'; } |
| 187 | +expect_log() { grep -qF -- "$1" "$work/poll.log" || fail "the poll did not print '$1'"; } |
| 188 | +# What the poll recorded as its decision, which a poll that changes nothing reports only there. |
| 189 | +expect_mode() { [ "$(status | jq -r .mode)" = "$1" ] || fail "the poll recorded '$(status | jq -r .mode)', not '$1'"; } |
| 190 | +expect_total() { [ "$(pooled_total)" -eq "$1" ] || fail "the pooled maxRunners total is $(pooled_total), not $1"; } |
| 191 | + |
| 192 | +log "A pool above the ceiling comes down to it" |
| 193 | +poll |
| 194 | +expect_log "ceiling=${expected} measured_ceiling=true" |
| 195 | +expect_log "above the ceiling ${expected}" |
| 196 | +expect_total "$expected" |
| 197 | +[ "$(status | jq -c --arg node "$node" '[.ceiling, .ceiling_measured, (.ceiling_nodes[] | select(.name == $node) | .runners)]')" = "[${expected},true,${expected}]" ] || fail "the status does not record the ceiling: $(status)" |
| 198 | + |
| 199 | +log "Another workload's pod taking a runner's cpu lowers the ceiling and the pool" |
| 200 | +kubectl run consumer -n "$other_namespace" --image=registry.k8s.io/pause:3.10 --restart=Never --overrides "{\"spec\": {\"containers\": [{\"name\": \"consumer\", \"image\": \"registry.k8s.io/pause:3.10\", \"resources\": {\"requests\": {\"cpu\": \"${runner_cpu}\"}}}]}}" >/dev/null |
| 201 | +kubectl wait --for=condition=Ready pod/consumer -n "$other_namespace" --timeout=180s >/dev/null || fail "the consumer pod never started" |
| 202 | +read -r lowered _ < <(reference "$runner_cpu") |
| 203 | +[ "$lowered" -eq $(( expected - 1 )) ] || fail "the reference places ${lowered} beside the consumer, not $(( expected - 1 ))" |
| 204 | +poll |
| 205 | +expect_log "ceiling=${lowered} measured_ceiling=true" |
| 206 | +expect_total "$lowered" |
| 207 | + |
| 208 | +log "Once it is gone the pool is raised back after the confirmation polls" |
| 209 | +kubectl delete pod consumer -n "$other_namespace" --wait=true >/dev/null |
| 210 | +poll |
| 211 | +expect_mode "watching: headroom confirmed for 1/2 polls" |
| 212 | +expect_total "$lowered" |
| 213 | +poll |
| 214 | +expect_log "raise:" |
| 215 | +expect_total "$expected" |
| 216 | + |
| 217 | +log "A template the ceiling cannot be worked out from keeps the last ceiling and never raises" |
| 218 | +# No cpu request or limit, so how many fit cannot be worked out; the memory limit stays, since without it the poll fails safe before the ceiling is read. |
| 219 | +for namespace in "${pooled_namespaces[@]}"; do |
| 220 | + kubectl patch autoscalingrunnerset grtest-runners -n "$namespace" --type json -p '[{"op": "remove", "path": "/spec/template/spec/containers/0/resources/requests/cpu"}, {"op": "remove", "path": "/spec/template/spec/containers/0/resources/limits/cpu"}, {"op": "replace", "path": "/spec/maxRunners", "value": 1}]' >/dev/null |
| 221 | +done |
| 222 | +for _ in 1 2 3; do |
| 223 | + poll |
| 224 | + expect_log "ERROR: could not work out the ceiling from the nodes; keeping the last one (${expected}) and not raising" |
| 225 | + expect_log "ceiling=${expected} measured_ceiling=false" |
| 226 | + expect_mode "steady: the ceiling could not be worked out from the nodes, so the pool is not raised" |
| 227 | + expect_total 2 |
| 228 | +done |
| 229 | +grep -qF "the runner pod requests no cpu or no memory" "$work/poll.log" || fail "the poll did not say why the ceiling could not be worked out" |
| 230 | +for namespace in "${pooled_namespaces[@]}"; do |
| 231 | + kubectl patch autoscalingrunnerset grtest-runners -n "$namespace" --type json -p "[{\"op\": \"add\", \"path\": \"/spec/template/spec/containers/0/resources/requests/cpu\", \"value\": \"${runner_cpu}\"}, {\"op\": \"add\", \"path\": \"/spec/template/spec/containers/0/resources/limits/cpu\", \"value\": \"${runner_cpu}\"}]" >/dev/null |
| 232 | +done |
| 233 | +poll |
| 234 | +expect_log "ceiling=${expected} measured_ceiling=true" |
| 235 | + |
| 236 | +log "A cordoned node is not counted, so the pool comes down to its running count" |
| 237 | +kubectl cordon "$node" >/dev/null |
| 238 | +poll |
| 239 | +expect_log "ceiling=0 measured_ceiling=true" |
| 240 | +expect_total 0 |
| 241 | +[ "$(status | jq -r --arg node "$node" '.ceiling_nodes[] | select(.name == $node) | .not_counted')" = "is cordoned" ] || fail "the status does not say why the node is not counted: $(status)" |
| 242 | +kubectl uncordon "$node" >/dev/null |
| 243 | + |
| 244 | +log "PASS" |
0 commit comments