From 3f6c0d5e61f69d17acdc8987f71fee6212d72f7f Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 8 Oct 2026 14:11:12 +0100 Subject: [PATCH 1/8] refactor(arc): share the App installation token minting between jobs Move the JWT signing and token minting out of the pull Secret renewal into a sourced file in the role's files, so another in-cluster job can mint a token restricted to its own permissions the same way. The renewer image carries it beside the renewal script. --- pull-secret-renewer/Dockerfile | 1 + .../files/github-app-token.sh | 59 +++++++++++++++++++ scripts/renew-pull-secret.sh | 48 ++------------- tests/unit/test_renew_pull_secret.py | 11 +++- 4 files changed, 74 insertions(+), 45 deletions(-) create mode 100644 roles/github_runner_arc/files/github-app-token.sh diff --git a/pull-secret-renewer/Dockerfile b/pull-secret-renewer/Dockerfile index cac21c3..e0877cf 100644 --- a/pull-secret-renewer/Dockerfile +++ b/pull-secret-renewer/Dockerfile @@ -4,6 +4,7 @@ FROM alpine:3.20 RUN apk add --no-cache bash curl jq kubectl openssl COPY scripts/renew-pull-secret.sh /app/renew-pull-secret.sh +COPY roles/github_runner_arc/files/github-app-token.sh /app/github-app-token.sh RUN chmod +x /app/renew-pull-secret.sh # A non-root user by number, so the CronJob's runAsNonRoot check can verify it without reading /etc/passwd. diff --git a/roles/github_runner_arc/files/github-app-token.sh b/roles/github_runner_arc/files/github-app-token.sh new file mode 100644 index 0000000..3cb7016 --- /dev/null +++ b/roles/github_runner_arc/files/github-app-token.sh @@ -0,0 +1,59 @@ +# shellcheck shell=bash +# Sourced, not run: mints a GitHub App installation token restricted to the permissions a caller names. Shared by the pull Secret renewal (scripts/renew-pull-secret.sh, which its image carries beside this file) and the runner reaper (files/reap-runners.sh, which its ConfigMap carries beside this file). +# +# The caller defines fail MESSAGE, which must not return, and sets umask 077 before calling. The JWT never appears in a command's arguments: curl reads it from a config file under the caller's private working directory. + +# GitHub rejects an App JWT whose exp is more than ten minutes after iat; iat is backdated a minute to absorb clock drift. +GITHUB_APP_JWT_BACKDATE_SECONDS=60 +GITHUB_APP_JWT_LIFETIME_SECONDS=540 + +github_app_b64url() { + openssl base64 -A | tr '+/' '-_' | tr -d '=' +} + +# Writes a curl config line for a header or user value. curl config values are double-quoted strings in which a backslash escapes the next character. +curl_config() { + local option="$1" value="$2" + value="${value//\\/\\\\}" + value="${value//\"/\\\"}" + printf '%s = "%s"\n' "$option" "$value" +} + +# github_app_mint_token APP_DIR API_URL PERMISSIONS WORK +# +# APP_DIR holds the App Secret's keys as files (github_app_id, github_app_installation_id, github_app_private_key). PERMISSIONS is the JSON object of permissions to restrict the token to, such as {"packages":"read"}. On success it writes the token, on one line, to WORK/token and its expiry to WORK/expires_at. Every refusal calls fail, naming the permissions the App needs when GitHub refuses them. +github_app_mint_token() { + local app_dir="$1" api_url="$2" permissions="$3" work="$4" + local field app_id installation_id now header claims signature status wanted + for field in github_app_id github_app_installation_id github_app_private_key; do + [ -s "$app_dir/$field" ] || fail "the App Secret has no $field" + done + app_id="$(tr -d '[:space:]' < "$app_dir/github_app_id")" + installation_id="$(tr -d '[:space:]' < "$app_dir/github_app_installation_id")" + [[ "$app_id" =~ ^[0-9]+$ ]] || fail "the App Secret's github_app_id is not a number" + [[ "$installation_id" =~ ^[0-9]+$ ]] || fail "the App Secret's github_app_installation_id is not a number" + wanted="$(jq -r 'to_entries | map("\(.key): \(.value)") | join(", ")' <<< "$permissions")" || fail "the permissions to request are not a JSON object" + + now="$(date +%s)" + header="$(printf '{"alg":"RS256","typ":"JWT"}' | github_app_b64url)" + claims="$(printf '{"iat":%d,"exp":%d,"iss":"%s"}' "$((now - GITHUB_APP_JWT_BACKDATE_SECONDS))" "$((now + GITHUB_APP_JWT_LIFETIME_SECONDS))" "$app_id" | github_app_b64url)" + signature="$(printf '%s.%s' "$header" "$claims" | openssl dgst -sha256 -sign "$app_dir/github_app_private_key" -binary | github_app_b64url)" \ + || fail "could not sign the App JWT with the App Secret's private key" + curl_config header "Authorization: Bearer ${header}.${claims}.${signature}" > "$work/github-app.curl" + + status="$(curl -sS -o "$work/mint.json" -w '%{http_code}' -K "$work/github-app.curl" -X POST \ + -H 'Accept: application/vnd.github+json' -H 'X-GitHub-Api-Version: 2022-11-28' \ + --data "$(jq -c '{permissions: .}' <<< "$permissions")" \ + "${api_url}/app/installations/${installation_id}/access_tokens")" || fail "could not reach the GitHub API" + rm -f "$work/github-app.curl" + if [ "$status" != "201" ]; then + fail "GitHub refused to mint a ${wanted} installation token (HTTP ${status}: $(jq -r '.message // empty' "$work/mint.json" 2>/dev/null)). The App needs the ${wanted} permission, approved on this installation" + fi + jq -r '.token // empty' "$work/mint.json" > "$work/token" + jq -r '.expires_at // empty' "$work/mint.json" > "$work/expires_at" + rm -f "$work/mint.json" + if [ ! -s "$work/token" ] || [ "$(wc -l < "$work/token")" -gt 1 ]; then + fail "GitHub's response held no token" + fi + [ -s "$work/expires_at" ] || fail "GitHub's response held no expiry" +} diff --git a/scripts/renew-pull-secret.sh b/scripts/renew-pull-secret.sh index 5747060..77f66d0 100755 --- a/scripts/renew-pull-secret.sh +++ b/scripts/renew-pull-secret.sh @@ -23,9 +23,6 @@ EXPIRY_ANNOTATION="${EXPIRY_ANNOTATION:?EXPIRY_ANNOTATION must be set}" FIELD_MANAGER="${FIELD_MANAGER:?FIELD_MANAGER must be set}" GITHUB_API_URL="${GITHUB_API_URL:-https://api.github.com}" -# GitHub rejects an App JWT whose exp is more than ten minutes after iat; iat is backdated a minute to absorb clock drift. -JWT_BACKDATE_SECONDS=60 -JWT_LIFETIME_SECONDS=540 # The username GitHub documents for authenticating to its registries with a token that belongs to no user. TOKEN_USERNAME="x-access-token" MANIFEST_ACCEPT="application/vnd.oci.image.index.v1+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json" @@ -39,47 +36,12 @@ umask 077 work="$(mktemp -d)" trap 'rm -rf "$work"' EXIT -b64url() { - openssl base64 -A | tr '+/' '-_' | tr -d '=' -} - -# Writes a curl config line for a header or user value. curl config values are double-quoted strings in which a backslash escapes the next character. -curl_config() { - local option="$1" value="$2" - value="${value//\\/\\\\}" - value="${value//\"/\\\"}" - printf '%s = "%s"\n' "$option" "$value" -} - -for field in github_app_id github_app_installation_id github_app_private_key; do - [ -s "$GITHUB_APP_DIR/$field" ] || fail "the App Secret has no $field" -done -app_id="$(tr -d '[:space:]' < "$GITHUB_APP_DIR/github_app_id")" -installation_id="$(tr -d '[:space:]' < "$GITHUB_APP_DIR/github_app_installation_id")" -[[ "$app_id" =~ ^[0-9]+$ ]] || fail "the App Secret's github_app_id is not a number" -[[ "$installation_id" =~ ^[0-9]+$ ]] || fail "the App Secret's github_app_installation_id is not a number" - -now="$(date +%s)" -header="$(printf '{"alg":"RS256","typ":"JWT"}' | b64url)" -claims="$(printf '{"iat":%d,"exp":%d,"iss":"%s"}' "$((now - JWT_BACKDATE_SECONDS))" "$((now + JWT_LIFETIME_SECONDS))" "$app_id" | b64url)" -signature="$(printf '%s.%s' "$header" "$claims" | openssl dgst -sha256 -sign "$GITHUB_APP_DIR/github_app_private_key" -binary | b64url)" \ - || fail "could not sign the App JWT with the App Secret's private key" -curl_config header "Authorization: Bearer ${header}.${claims}.${signature}" > "$work/github.curl" +# The image carries the shared token minting beside this script (see pull-secret-renewer/Dockerfile). +# shellcheck source=roles/github_runner_arc/files/github-app-token.sh +source "$(dirname "${BASH_SOURCE[0]}")/github-app-token.sh" -status="$(curl -sS -o "$work/mint.json" -w '%{http_code}' -K "$work/github.curl" -X POST \ - -H 'Accept: application/vnd.github+json' -H 'X-GitHub-Api-Version: 2022-11-28' \ - --data '{"permissions":{"packages":"read"}}' \ - "${GITHUB_API_URL}/app/installations/${installation_id}/access_tokens")" || fail "could not reach the GitHub API" -if [ "$status" != "201" ]; then - fail "GitHub refused to mint a packages: read installation token (HTTP ${status}: $(jq -r '.message // empty' "$work/mint.json" 2>/dev/null)). The App needs the packages: read permission, approved on this installation" -fi -jq -r '.token // empty' "$work/mint.json" > "$work/token" -expires_at="$(jq -r '.expires_at // empty' "$work/mint.json")" -rm -f "$work/mint.json" -if [ ! -s "$work/token" ] || [ "$(wc -l < "$work/token")" -gt 1 ]; then - fail "GitHub's response held no token" -fi -[ -n "$expires_at" ] || fail "GitHub's response held no expiry" +github_app_mint_token "$GITHUB_APP_DIR" "$GITHUB_API_URL" '{"packages":"read"}' "$work" +expires_at="$(cat "$work/expires_at")" token="$(cat "$work/token")" # The registry's own two-step handshake against each real image: a bearer token scoped to pull it, then a manifest read. A token can authenticate to the registry and still be denied the package, which is only visible at the second step. diff --git a/tests/unit/test_renew_pull_secret.py b/tests/unit/test_renew_pull_secret.py index 4152197..23d0486 100644 --- a/tests/unit/test_renew_pull_secret.py +++ b/tests/unit/test_renew_pull_secret.py @@ -12,7 +12,10 @@ import unittest from pathlib import Path -SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "renew-pull-secret.sh" +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "renew-pull-secret.sh" +# The image carries the shared token minting beside the script, so the tests run the two from one directory as the image does. +TOKEN_LIB = ROOT / "roles" / "github_runner_arc" / "files" / "github-app-token.sh" # A stand-in for curl that answers from a scenario file and records each call's arguments and config file. It understands only the options the script uses. FAKE_CURL = textwrap.dedent( @@ -96,6 +99,10 @@ def setUp(self) -> None: root = Path(self.directory.name) self.bin = root / "bin" self.bin.mkdir() + self.script = root / "app-scripts" / SCRIPT.name + self.script.parent.mkdir() + shutil.copy(SCRIPT, self.script) + shutil.copy(TOKEN_LIB, self.script.parent / TOKEN_LIB.name) for name, body in (("curl", FAKE_CURL), ("kubectl", FAKE_KUBECTL)): path = self.bin / name path.write_text(body) @@ -133,7 +140,7 @@ def run_script(self, images: list[dict[str, str]] | None = None, kubectl_exit: i "EXPIRY_ANNOTATION": "example.com/token-expires-at", "FIELD_MANAGER": "example-renewer", } - return subprocess.run(["bash", str(SCRIPT)], env=env, capture_output=True, text=True) + return subprocess.run(["bash", str(self.script)], env=env, capture_output=True, text=True) def calls(self) -> list[dict]: return [json.loads(line) for line in self.log.read_text().splitlines()] if self.log.exists() else [] From 4bd10ce5d1d1f0855847542e4d0484013336bc30 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 8 Oct 2026 14:17:07 +0100 Subject: [PATCH 2/8] feat(arc): add a runner reaper pass for runners that outlive their job One pass lists a namespace's EphemeralRunners and, for each one ARC has assigned a job, reads the job's run from GitHub with an actions: read installation token. It deletes the runner when the run or its own job is completed, or when the job has run past a configured bound, and never touches an idle runner or one it could not judge. --- roles/github_runner_arc/files/reap-runners.sh | 181 +++++++++ tests/unit/test_reap_runners.py | 376 ++++++++++++++++++ 2 files changed, 557 insertions(+) create mode 100755 roles/github_runner_arc/files/reap-runners.sh create mode 100644 tests/unit/test_reap_runners.py diff --git a/roles/github_runner_arc/files/reap-runners.sh b/roles/github_runner_arc/files/reap-runners.sh new file mode 100755 index 0000000..1d06be7 --- /dev/null +++ b/roles/github_runner_arc/files/reap-runners.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Runs as the runner reaper CronJob the github_runner_arc role installs in each runner namespace when github_runner_arc_runner_reaper_enabled is on (see roles/github_runner_arc/tasks/install_runner_reaper.yml). One pass: it mints a GitHub App installation token restricted to actions: read, lists the namespace's EphemeralRunners, and for each runner that ARC has recorded as running a job reads that job's workflow run from GitHub. It deletes the runner when the run is completed (which includes cancelled), when the runner's own job is completed, or when that job has been running for longer than MAX_JOB_SECONDS; ARC's scale set replaces a deleted runner. A completed run or job is only acted on once it has been completed for COMPLETED_GRACE_SECONDS, which keeps the reaper out of the way of a runner that is exiting normally. +# +# It never deletes a runner ARC has not assigned a job (no workflow run recorded on it), a runner already being deleted, or a runner whose run or job it could not read. A runner is deleted only on a positive answer from GitHub. If the token cannot be minted or the runners cannot be listed it deletes nothing and exits non-zero; if any one runner's run cannot be read it leaves that runner alone, carries on with the rest, and exits non-zero at the end. +# +# Environment (provided by the CronJob): +# - GITHUB_APP_DIR: directory holding the App Secret's three keys as files, mounted from the App Secret +# - RUNNER_NAMESPACE: the namespace whose EphemeralRunners it checks +# - MAX_JOB_SECONDS: a job running for longer than this has its runner deleted +# - COMPLETED_GRACE_SECONDS: how long a run or job must have been completed before its runner is deleted +# - DRY_RUN: true logs what it would delete and deletes nothing +# - GITHUB_API_URL: the GitHub API, default https://api.github.com +# The ConfigMap carries the shared token minting (github-app-token.sh) beside this script. kubectl needs no KUBECONFIG: it uses the pod's ServiceAccount token. The token never appears in a command's arguments: curl reads it from a config file under a private temporary directory that is removed on exit. +set -euo pipefail + +GITHUB_APP_DIR="${GITHUB_APP_DIR:-/var/run/github-app}" +RUNNER_NAMESPACE="${RUNNER_NAMESPACE:?RUNNER_NAMESPACE must be set}" +MAX_JOB_SECONDS="${MAX_JOB_SECONDS:?MAX_JOB_SECONDS must be set}" +COMPLETED_GRACE_SECONDS="${COMPLETED_GRACE_SECONDS:?COMPLETED_GRACE_SECONDS must be set}" +DRY_RUN="${DRY_RUN:-false}" +GITHUB_API_URL="${GITHUB_API_URL:-https://api.github.com}" + +# The largest page GitHub's list endpoints return. +JOBS_PER_PAGE=100 +RESOURCE="ephemeralrunners.actions.github.com" + +log() { + echo "reap-runners: $*" >&2 +} + +# Before any runner is looked at, so nothing has been deleted when it is called. +fail() { + log "$*; deleted nothing" + exit 1 +} + +[[ "$MAX_JOB_SECONDS" =~ ^[1-9][0-9]*$ ]] || fail "MAX_JOB_SECONDS must be a positive whole number of seconds" +[[ "$COMPLETED_GRACE_SECONDS" =~ ^[0-9]+$ ]] || fail "COMPLETED_GRACE_SECONDS must be a whole number of seconds" +case "$DRY_RUN" in + true | false) ;; + *) fail "DRY_RUN must be true or false" ;; +esac + +umask 077 +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# shellcheck source=roles/github_runner_arc/files/github-app-token.sh +source "$(dirname "${BASH_SOURCE[0]}")/github-app-token.sh" + +github_app_mint_token "$GITHUB_APP_DIR" "$GITHUB_API_URL" '{"actions":"read"}' "$work" +curl_config header "Authorization: Bearer $(cat "$work/token")" > "$work/api.curl" +rm -f "$work/token" "$work/expires_at" + +kubectl get "$RESOURCE" -n "$RUNNER_NAMESPACE" -o json > "$work/runners.json" \ + || fail "could not list the EphemeralRunners in ${RUNNER_NAMESPACE}" +jq -e '.items | type == "array"' "$work/runners.json" > /dev/null || fail "the EphemeralRunner list in ${RUNNER_NAMESPACE} held no items" + +# Prints the HTTP status of a GET on the API path, writing the body to the file. +github_get() { + curl -sS -o "$2" -w '%{http_code}' -K "$work/api.curl" \ + -H 'Accept: application/vnd.github+json' -H 'X-GitHub-Api-Version: 2022-11-28' \ + "${GITHUB_API_URL}$1" +} + +# Seconds since a GitHub timestamp, ignoring any fractional seconds, which fromdateiso8601 does not parse. +JQ_AGE='def age(t): now - (t | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) | floor;' + +checked=0 +deleted=0 +idle=0 +kept=0 +errors=0 + +error() { + log "$*; left alone" + errors=$((errors + 1)) +} + +delete_runner() { + local name="$1" detail="$2" + if [ "$DRY_RUN" = true ]; then + log "would delete ${RUNNER_NAMESPACE}/${name}: ${detail} (dry run)" + deleted=$((deleted + 1)) + return + fi + if kubectl delete "$RESOURCE" "$name" -n "$RUNNER_NAMESPACE" --wait=false >&2; then + log "deleted ${RUNNER_NAMESPACE}/${name}: ${detail}" + deleted=$((deleted + 1)) + else + error "${name}: could not delete it (${detail})" + fi +} + +keep_runner() { + log "kept ${RUNNER_NAMESPACE}/${1}: ${2}" + kept=$((kept + 1)) +} + +# Finds the job of the run that names the runner, across every attempt and page, writing it to WORK/job.json. Returns 1 when no job names it and 2 when GitHub could not be read. +find_job() { + local name="$1" repository="$2" run_id="$3" page=1 status + while :; do + status="$(github_get "/repos/${repository}/actions/runs/${run_id}/jobs?filter=all&per_page=${JOBS_PER_PAGE}&page=${page}" "$work/jobs.json")" || return 2 + [ "$status" = 200 ] || { log "listing the jobs of run ${run_id} returned HTTP ${status}"; return 2; } + jq -e --arg name "$name" '[.jobs[] | select(.runner_name == $name)] | first // empty' "$work/jobs.json" > "$work/job.json" 2> /dev/null && return 0 + jq -e --argjson page "$page" --argjson size "$JOBS_PER_PAGE" '(.jobs | length) > 0 and $page * $size < .total_count' "$work/jobs.json" > /dev/null 2>&1 || return 1 + page=$((page + 1)) + done +} + +while IFS= read -r runner <&3; do + checked=$((checked + 1)) + name="$(jq -r '.metadata.name' <<< "$runner")" + if [ "$(jq -r '.metadata.deletionTimestamp // empty' <<< "$runner")" != "" ]; then + keep_runner "$name" "already being deleted" + continue + fi + run_id="$(jq -r '.status.workflowRunId // 0' <<< "$runner")" + repository="$(jq -r '.status.jobRepositoryName // empty' <<< "$runner")" + # ARC records the run and repository on a runner only when the listener assigns it a job, and an ephemeral runner never takes a second one, so a runner without them is idle. + if [ "$run_id" = 0 ] || [ -z "$repository" ]; then + idle=$((idle + 1)) + continue + fi + if ! [[ "$run_id" =~ ^[0-9]+$ ]] || ! [[ "$repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]]; then + error "${name}: its recorded run '${run_id}' in '${repository}' is not a run GitHub could name" + continue + fi + run="run ${run_id} in ${repository}" + + status="$(github_get "/repos/${repository}/actions/runs/${run_id}" "$work/run.json")" || { error "${name}: could not reach GitHub to read ${run}"; continue; } + if [ "$status" != 200 ]; then + error "${name}: reading ${run} returned HTTP ${status} ($(jq -r '.message // empty' "$work/run.json" 2> /dev/null)); the App's installation may not include ${repository}" + continue + fi + verdict="$(jq -r --argjson grace "$COMPLETED_GRACE_SECONDS" "$JQ_AGE"' + if .status == "completed" then + age(.updated_at) as $age + | if $age >= $grace then "delete\t\(.conclusion // "no conclusion"), completed \($age)s ago" + else "keep\t\(.conclusion // "no conclusion"), completed \($age)s ago, inside the grace of \($grace)s" end + else "jobs\t\(.status)" end' "$work/run.json")" || { error "${name}: could not read the state of ${run}"; continue; } + IFS=$'\t' read -r action detail <<< "$verdict" + case "$action" in + delete) delete_runner "$name" "${run} is ${detail}"; continue ;; + keep) keep_runner "$name" "${run} is ${detail}"; continue ;; + esac + + # The run is still going: judge the runner by its own job. + outcome=0 + find_job "$name" "$repository" "$run_id" || outcome=$? + case "$outcome" in + 1) keep_runner "$name" "${run} is ${detail} and none of its jobs names this runner"; continue ;; + 2) error "${name}: could not read the jobs of ${run}"; continue ;; + esac + verdict="$(jq -r --argjson grace "$COMPLETED_GRACE_SECONDS" --argjson bound "$MAX_JOB_SECONDS" "$JQ_AGE"' + if .status == "completed" then + age(.completed_at) as $age + | if $age >= $grace then "delete\tjob \(.id) is \(.conclusion // "no conclusion"), completed \($age)s ago" + else "keep\tjob \(.id) is \(.conclusion // "no conclusion"), completed \($age)s ago, inside the grace of \($grace)s" end + elif .started_at == null then "keep\tjob \(.id) is \(.status) and has not started" + else + age(.started_at) as $age + | if $age > $bound then "delete\tjob \(.id) has been \(.status) for \($age)s, over the bound of \($bound)s" + else "keep\tjob \(.id) has been \(.status) for \($age)s, within the bound of \($bound)s" end + end' "$work/job.json")" || { error "${name}: could not read the state of its job in ${run}"; continue; } + IFS=$'\t' read -r action detail <<< "$verdict" + if [ "$action" = delete ]; then + delete_runner "$name" "${run}: ${detail}" + else + keep_runner "$name" "${run}: ${detail}" + fi +done 3< <(jq -c '.items[]' "$work/runners.json") + +deleted_word=deleted +[ "$DRY_RUN" = false ] || deleted_word="to delete (dry run)" +log "checked ${checked} runner(s) in ${RUNNER_NAMESPACE}: ${deleted} ${deleted_word}, ${kept} kept, ${idle} idle, ${errors} not readable" +if [ "$errors" -gt 0 ]; then + log "exiting non-zero because ${errors} runner(s) could not be judged; they were left alone" + exit 1 +fi diff --git a/tests/unit/test_reap_runners.py b/tests/unit/test_reap_runners.py new file mode 100644 index 0000000..0b38ea9 --- /dev/null +++ b/tests/unit/test_reap_runners.py @@ -0,0 +1,376 @@ +"""Tests for roles/github_runner_arc/files/reap-runners.sh, one pass of the runner reaper, run against scripted stand-ins for curl (GitHub's API) and kubectl with the real bash, jq and openssl.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +FILES = Path(__file__).resolve().parents[2] / "roles" / "github_runner_arc" / "files" +SCRIPT = FILES / "reap-runners.sh" +TOKEN_LIB = FILES / "github-app-token.sh" + +NAMESPACE = "example-runners" +REPO = "example-org/example-repo" +TOKEN = "ghs_examplereapertoken0123456789" +BOUND = 3600 +GRACE = 300 +API = "https://api.github.com" + +# A stand-in for curl that answers from a scenario file and records each call. The scenario maps an API path (with its query) to [status, body], or to "unreachable" for a connection failure; a path it does not list answers 404. +FAKE_CURL = textwrap.dedent( + """\ + #!/usr/bin/env python3 + import json, os, sys + args = sys.argv[1:] + scenario = json.load(open(os.environ["FAKE_SCENARIO"])) + output = None + config = "" + data = None + url = None + i = 0 + while i < len(args): + arg = args[i] + if arg in ("-o", "-w", "-K", "-X", "-H", "--data"): + value = args[i + 1] + if arg == "-o": + output = value + elif arg == "-K": + config = open(value).read() + elif arg == "--data": + data = value + i += 2 + continue + if not arg.startswith("-"): + url = arg + i += 1 + with open(os.environ["FAKE_LOG"], "a") as log: + log.write(json.dumps({"argv": args, "config": config, "data": data, "url": url}) + "\\n") + path = url.removeprefix(os.environ["FAKE_API"]) + if path.endswith("/access_tokens"): + answer = scenario["mint"] + else: + answer = scenario["paths"].get(path, [404, {"message": "Not Found"}]) + if answer == "unreachable": + sys.stderr.write("curl: (7) Failed to connect\\n") + sys.exit(7) + status, body = answer + if output: + with open(output, "w") as handle: + handle.write(json.dumps(body)) + sys.stdout.write(str(status)) + """ +) + +# Answers `kubectl get ephemeralrunners...` from a file and records every `kubectl delete`; FAKE_GET_EXIT and FAKE_DELETE_EXIT set their exit statuses. +FAKE_KUBECTL = textwrap.dedent( + """\ + #!/usr/bin/env python3 + import json, os, sys + args = sys.argv[1:] + with open(os.environ["FAKE_KUBECTL_LOG"], "a") as log: + log.write(json.dumps(args) + "\\n") + if args[:2] == ["get", "ephemeralrunners.actions.github.com"]: + code = int(os.environ.get("FAKE_GET_EXIT", "0")) + if code == 0: + sys.stdout.write(open(os.environ["FAKE_RUNNERS"]).read()) + sys.exit(code) + if args[:2] == ["delete", "ephemeralrunners.actions.github.com"]: + sys.exit(int(os.environ.get("FAKE_DELETE_EXIT", "0"))) + sys.stderr.write("unexpected kubectl call: %s\\n" % args) + sys.exit(2) + """ +) + + +def timestamp(seconds_ago: int) -> str: + return (datetime.now(timezone.utc) - timedelta(seconds=seconds_ago)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def runner(name: str, run_id: int | None = None, repository: str = REPO, deleting: bool = False) -> dict[str, Any]: + """An EphemeralRunner as ARC records it: a runner the listener assigned a job carries the job's run and repository in its status.""" + metadata: dict[str, Any] = {"name": name, "namespace": NAMESPACE} + if deleting: + metadata["deletionTimestamp"] = timestamp(10) + status: dict[str, Any] = {"phase": "Running", "runnerName": name} + if run_id is not None: + status.update({"workflowRunId": run_id, "jobRepositoryName": repository, "jobRequestId": run_id * 10}) + return {"apiVersion": "actions.github.com/v1alpha1", "kind": "EphemeralRunner", "metadata": metadata, "status": status} + + +def run(status: str, conclusion: str | None = None, updated_seconds_ago: int = 0) -> list[Any]: + return [200, {"status": status, "conclusion": conclusion, "updated_at": timestamp(updated_seconds_ago)}] + + +def job(job_id: int, runner_name: str, status: str, started_seconds_ago: int | None = None, completed_seconds_ago: int | None = None, conclusion: str | None = None) -> dict[str, Any]: + return { + "id": job_id, + "runner_name": runner_name, + "status": status, + "conclusion": conclusion, + "started_at": None if started_seconds_ago is None else timestamp(started_seconds_ago), + "completed_at": None if completed_seconds_ago is None else timestamp(completed_seconds_ago), + } + + +def run_path(run_id: int) -> str: + return f"/repos/{REPO}/actions/runs/{run_id}" + + +def jobs_path(run_id: int, page: int = 1) -> str: + return f"{run_path(run_id)}/jobs?filter=all&per_page=100&page={page}" + + +class ReapRunnersTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + for tool in ("bash", "jq", "openssl"): + if shutil.which(tool) is None: + raise unittest.SkipTest(f"{tool} is not installed") + cls.keys = tempfile.TemporaryDirectory() + cls.key = Path(cls.keys.name) / "key.pem" + subprocess.run(["openssl", "genrsa", "-out", str(cls.key), "2048"], check=True, capture_output=True) + + @classmethod + def tearDownClass(cls) -> None: + cls.keys.cleanup() + + def setUp(self) -> None: + self.directory = tempfile.TemporaryDirectory() + root = Path(self.directory.name) + self.bin = root / "bin" + self.bin.mkdir() + for name, body in (("curl", FAKE_CURL), ("kubectl", FAKE_KUBECTL)): + path = self.bin / name + path.write_text(body) + path.chmod(0o755) + # The ConfigMap mounts the script and the token minting side by side, so the tests run them from one directory too. + self.script = root / "reaper" / SCRIPT.name + self.script.parent.mkdir() + shutil.copy(SCRIPT, self.script) + shutil.copy(TOKEN_LIB, self.script.parent / TOKEN_LIB.name) + self.app = root / "app" + self.app.mkdir() + (self.app / "github_app_id").write_text("12345") + (self.app / "github_app_installation_id").write_text("67890\n") + (self.app / "github_app_private_key").write_text(self.key.read_text()) + self.curl_log = root / "curl.jsonl" + self.kubectl_log = root / "kubectl.jsonl" + self.runners_path = root / "runners.json" + self.scenario_path = root / "scenario.json" + self.runners: list[dict[str, Any]] = [] + self.scenario: dict[str, Any] = {"mint": [201, {"token": TOKEN, "expires_at": "2030-01-01T00:00:00Z"}], "paths": {}} + + def tearDown(self) -> None: + self.directory.cleanup() + + def reap(self, **env_overrides: str) -> subprocess.CompletedProcess[str]: + self.runners_path.write_text(json.dumps({"apiVersion": "v1", "kind": "List", "items": self.runners})) + self.scenario_path.write_text(json.dumps(self.scenario)) + env = { + "PATH": f"{self.bin}:{os.environ['PATH']}", + "FAKE_SCENARIO": str(self.scenario_path), + "FAKE_LOG": str(self.curl_log), + "FAKE_API": API, + "FAKE_KUBECTL_LOG": str(self.kubectl_log), + "FAKE_RUNNERS": str(self.runners_path), + "GITHUB_APP_DIR": str(self.app), + "RUNNER_NAMESPACE": NAMESPACE, + "MAX_JOB_SECONDS": str(BOUND), + "COMPLETED_GRACE_SECONDS": str(GRACE), + **env_overrides, + } + return subprocess.run(["bash", str(self.script)], env=env, capture_output=True, text=True) + + def curl_calls(self) -> list[dict[str, Any]]: + return [json.loads(line) for line in self.curl_log.read_text().splitlines()] if self.curl_log.exists() else [] + + def kubectl_calls(self) -> list[list[str]]: + return [json.loads(line) for line in self.kubectl_log.read_text().splitlines()] if self.kubectl_log.exists() else [] + + def deleted(self) -> list[str]: + return [call[2] for call in self.kubectl_calls() if call[0] == "delete"] + + def test_mints_a_token_restricted_to_actions_read(self) -> None: + self.reap() + mint = self.curl_calls()[0] + self.assertEqual(mint["url"], f"{API}/app/installations/67890/access_tokens") + self.assertEqual(json.loads(mint["data"]), {"permissions": {"actions": "read"}}) + + def test_an_idle_runner_is_never_deleted_or_looked_up(self) -> None: + self.runners = [runner("idle-runner")] + result = self.reap() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), []) + self.assertEqual([call["url"] for call in self.curl_calls()][1:], []) + self.assertIn("1 idle", result.stderr) + + def test_a_runner_whose_run_was_cancelled_is_deleted_and_logged(self) -> None: + self.runners = [runner("stuck-runner", run_id=101)] + self.scenario["paths"][run_path(101)] = run("completed", "cancelled", updated_seconds_ago=GRACE * 2) + result = self.reap() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), ["stuck-runner"]) + [delete] = [call for call in self.kubectl_calls() if call[0] == "delete"] + self.assertEqual(delete, ["delete", "ephemeralrunners.actions.github.com", "stuck-runner", "-n", NAMESPACE, "--wait=false"]) + self.assertIn(f"deleted {NAMESPACE}/stuck-runner: run 101 in {REPO} is cancelled", result.stderr) + + def test_a_run_completed_inside_the_grace_keeps_its_runner(self) -> None: + self.runners = [runner("exiting-runner", run_id=102)] + self.scenario["paths"][run_path(102)] = run("completed", "success", updated_seconds_ago=GRACE // 2) + result = self.reap() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), []) + self.assertIn("inside the grace", result.stderr) + + def test_a_job_running_past_the_bound_has_its_runner_deleted(self) -> None: + self.runners = [runner("hung-runner", run_id=103)] + self.scenario["paths"][run_path(103)] = run("in_progress") + self.scenario["paths"][jobs_path(103)] = [200, {"total_count": 1, "jobs": [job(9001, "hung-runner", "in_progress", started_seconds_ago=BOUND * 2)]}] + result = self.reap() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), ["hung-runner"]) + self.assertIn(f"run 103 in {REPO}: job 9001 has been in_progress for", result.stderr) + self.assertIn(f"over the bound of {BOUND}s", result.stderr) + + def test_a_job_within_the_bound_keeps_its_runner(self) -> None: + self.runners = [runner("busy-runner", run_id=104)] + self.scenario["paths"][run_path(104)] = run("in_progress") + self.scenario["paths"][jobs_path(104)] = [200, {"total_count": 1, "jobs": [job(9002, "busy-runner", "in_progress", started_seconds_ago=BOUND // 2)]}] + result = self.reap() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), []) + self.assertIn("within the bound", result.stderr) + + def test_a_completed_job_in_a_running_run_has_its_runner_deleted(self) -> None: + self.runners = [runner("finished-runner", run_id=105)] + self.scenario["paths"][run_path(105)] = run("in_progress") + self.scenario["paths"][jobs_path(105)] = [200, {"total_count": 2, "jobs": [ + job(9003, "other-runner", "in_progress", started_seconds_ago=10), + job(9004, "finished-runner", "completed", started_seconds_ago=GRACE * 4, completed_seconds_ago=GRACE * 2, conclusion="success"), + ]}] + result = self.reap() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), ["finished-runner"]) + self.assertIn("job 9004 is success", result.stderr) + + def test_the_runners_job_is_found_on_a_later_page(self) -> None: + self.runners = [runner("paged-runner", run_id=106)] + self.scenario["paths"][run_path(106)] = run("in_progress") + self.scenario["paths"][jobs_path(106, 1)] = [200, {"total_count": 101, "jobs": [job(n, f"runner-{n}", "in_progress", started_seconds_ago=10) for n in range(100)]}] + self.scenario["paths"][jobs_path(106, 2)] = [200, {"total_count": 101, "jobs": [job(9005, "paged-runner", "in_progress", started_seconds_ago=BOUND * 2)]}] + result = self.reap() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), ["paged-runner"]) + + def test_a_runner_no_job_names_is_kept_after_the_last_page(self) -> None: + self.runners = [runner("unnamed-runner", run_id=107)] + self.scenario["paths"][run_path(107)] = run("queued") + self.scenario["paths"][jobs_path(107)] = [200, {"total_count": 1, "jobs": [job(9006, "someone-else", "queued")]}] + result = self.reap() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), []) + self.assertIn("none of its jobs names this runner", result.stderr) + self.assertEqual([call["url"] for call in self.curl_calls()][-1], API + jobs_path(107)) + + def test_a_runner_already_being_deleted_is_left_alone(self) -> None: + self.runners = [runner("leaving-runner", run_id=108, deleting=True)] + self.scenario["paths"][run_path(108)] = run("completed", "cancelled", updated_seconds_ago=GRACE * 2) + result = self.reap() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), []) + + def test_a_refused_mint_deletes_nothing_and_names_the_permission(self) -> None: + self.runners = [runner("stuck-runner", run_id=101)] + self.scenario["paths"][run_path(101)] = run("completed", "cancelled", updated_seconds_ago=GRACE * 2) + self.scenario["mint"] = [422, {"message": "The permissions requested are not granted to this installation."}] + result = self.reap() + self.assertNotEqual(result.returncode, 0) + self.assertIn("The App needs the actions: read permission", result.stderr) + self.assertIn("deleted nothing", result.stderr) + self.assertEqual(self.kubectl_calls(), []) + + def test_unreachable_github_deletes_nothing(self) -> None: + self.runners = [runner("stuck-runner", run_id=101)] + self.scenario["mint"] = "unreachable" + result = self.reap() + self.assertNotEqual(result.returncode, 0) + self.assertIn("could not reach the GitHub API", result.stderr) + self.assertEqual(self.kubectl_calls(), []) + + def test_a_failed_listing_deletes_nothing(self) -> None: + result = self.reap(FAKE_GET_EXIT="1") + self.assertNotEqual(result.returncode, 0) + self.assertIn(f"could not list the EphemeralRunners in {NAMESPACE}; deleted nothing", result.stderr) + self.assertEqual(self.deleted(), []) + + def test_an_unreadable_run_leaves_its_runner_and_fails_after_judging_the_rest(self) -> None: + self.runners = [runner("unknown-runner", run_id=109), runner("offline-runner", run_id=110), runner("stuck-runner", run_id=101)] + self.scenario["paths"][run_path(110)] = "unreachable" + self.scenario["paths"][run_path(101)] = run("completed", "cancelled", updated_seconds_ago=GRACE * 2) + result = self.reap() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.deleted(), ["stuck-runner"]) + self.assertIn("unknown-runner: reading run 109 in", result.stderr) + self.assertIn("returned HTTP 404", result.stderr) + self.assertIn("offline-runner: could not reach GitHub", result.stderr) + self.assertIn("2 runner(s) could not be judged", result.stderr) + + def test_an_unreadable_job_list_leaves_its_runner(self) -> None: + self.runners = [runner("hung-runner", run_id=111)] + self.scenario["paths"][run_path(111)] = run("in_progress") + self.scenario["paths"][jobs_path(111)] = [500, {"message": "Server Error"}] + result = self.reap() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.deleted(), []) + + def test_a_dry_run_deletes_nothing_but_says_what_it_would(self) -> None: + self.runners = [runner("stuck-runner", run_id=101)] + self.scenario["paths"][run_path(101)] = run("completed", "cancelled", updated_seconds_ago=GRACE * 2) + result = self.reap(DRY_RUN="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.deleted(), []) + self.assertIn(f"would delete {NAMESPACE}/stuck-runner: run 101", result.stderr) + + def test_a_failed_delete_is_reported(self) -> None: + self.runners = [runner("stuck-runner", run_id=101)] + self.scenario["paths"][run_path(101)] = run("completed", "cancelled", updated_seconds_ago=GRACE * 2) + result = self.reap(FAKE_DELETE_EXIT="1") + self.assertNotEqual(result.returncode, 0) + self.assertIn("stuck-runner: could not delete it", result.stderr) + + def test_a_bad_recorded_repository_is_never_requested(self) -> None: + self.runners = [runner("odd-runner", run_id=112, repository="../../app/installations")] + result = self.reap() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(self.curl_calls()), 1) + self.assertEqual(self.deleted(), []) + + def test_a_bound_that_is_not_a_positive_number_fails_before_any_request(self) -> None: + for bound in ("0", "", "1h"): + with self.subTest(bound=bound): + result = self.reap(MAX_JOB_SECONDS=bound) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.curl_calls(), []) + + def test_the_token_never_reaches_an_argument_or_the_output(self) -> None: + self.runners = [runner("stuck-runner", run_id=101)] + self.scenario["paths"][run_path(101)] = run("completed", "cancelled", updated_seconds_ago=GRACE * 2) + result = self.reap() + for call in self.curl_calls(): + self.assertNotIn(TOKEN, " ".join(call["argv"])) + self.assertIn(f"Bearer {TOKEN}", self.curl_calls()[1]["config"]) + for call in self.kubectl_calls(): + self.assertNotIn(TOKEN, " ".join(call)) + self.assertNotIn(TOKEN, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() From 32c15680562a7a0ada14b88860200ca564b8504e Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 8 Oct 2026 14:24:43 +0100 Subject: [PATCH 3/8] feat(arc): install an optional runner reaper CronJob in each runner namespace Off by default. When on, each scale-set profile's namespace gets a CronJob that runs the reaper from a ConfigMap in the pull Secret renewal's image, with a Role that may only list and delete EphemeralRunners there and the App Secret mounted as a volume. Turning it off removes all of it. --- roles/github_runner_arc/defaults/main.yml | 17 ++- .../github_runner_arc/meta/argument_specs.yml | 29 ++++ .../tasks/install_runner_reaper.yml | 33 +++++ .../tasks/install_scale_set_profile.yml | 3 + roles/github_runner_arc/tasks/validate.yml | 16 ++ .../templates/runner-reaper.yaml.j2 | 139 ++++++++++++++++++ tests/runner_reaper/render.yml | 21 +++ tests/unit/test_runner_reaper_role.py | 132 +++++++++++++++++ 8 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 roles/github_runner_arc/tasks/install_runner_reaper.yml create mode 100644 roles/github_runner_arc/templates/runner-reaper.yaml.j2 create mode 100644 tests/runner_reaper/render.yml create mode 100644 tests/unit/test_runner_reaper_role.py diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index b532b1b..2f1b818 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -154,7 +154,22 @@ github_runner_arc_node_recovery_poll_seconds: 15 # Image the watcher runs. Pulled without a pull Secret, so it must be public. github_runner_arc_node_recovery_image: "ghcr.io/exadev/github-runner-node-recovery:1.25.0" -# GitHub App creation through the manifest flow (tasks/app_setup.yml, run by playbooks/github_app_setup.yml). The App needs only organisation self-hosted runner write access; add packages: read to the permissions if its installation token should also pull private packages. +# Runner reaper (see the role README's Runner reaper section, tasks/install_runner_reaper.yml and files/reap-runners.sh): a CronJob in each runner namespace that deletes an EphemeralRunner whose job's workflow run is completed or cancelled, whose own job is completed, or whose job has been running for longer than github_runner_arc_runner_reaper_max_job_seconds; the scale set replaces it. It reads the runs with an installation token from the org's GitHub App restricted to actions: read, so the App needs the Actions repository permission (read) on every repository whose jobs the runners take, approved on its installation. It never deletes a runner that has not been assigned a job, and deletes nothing when GitHub or the token is unavailable. false removes the CronJob, its RBAC and its ConfigMap. +github_runner_arc_runner_reaper_enabled: false +# A job running for longer than this has its runner deleted. The default is GitHub's own limit on a job's execution time on a self-hosted runner, five days, past which GitHub ends the job itself, so no job can legitimately run longer; a fleet whose jobs are all shorter sets it a little above its longest job's timeout-minutes. +github_runner_arc_runner_reaper_max_job_seconds: 432000 +# How long a run or job must have been completed before its runner is deleted. ARC removes a runner whose job finished normally within seconds, so this keeps the reaper clear of a runner on its way out, at the cost of at most one more pass at the default schedule. +github_runner_arc_runner_reaper_completed_grace_seconds: 300 +github_runner_arc_runner_reaper_schedule: "*/5 * * * *" +# true logs what each pass would delete and deletes nothing, to see the reaper's verdicts on a fleet before trusting it. +github_runner_arc_runner_reaper_dry_run: false +# Image the CronJob runs the reaper in. It needs bash, curl, jq, kubectl and openssl, which the pull Secret renewal's image has; the reaper itself comes from a ConfigMap, so the image need not be rebuilt when it changes. Pulled without a pull Secret, so it must be public. +github_runner_arc_runner_reaper_image: "{{ github_runner_arc_image_pull_secret_renewer_image }}" + +# A backstop that needs no GitHub call: activeDeadlineSeconds on every runner pod, after which the kubelet stops the pod whatever it is doing and ARC treats it as a failed runner. It counts from when the pod starts, including any time the runner waits idle for a job, so set it above the longest a runner may wait plus the longest job a fleet runs. Unset leaves the runner pods without one. +github_runner_arc_runner_active_deadline_seconds: + +# GitHub App creation through the manifest flow (tasks/app_setup.yml, run by playbooks/github_app_setup.yml). The App needs only organisation self-hosted runner write access; add packages: read to the permissions if its installation token should also pull private packages, and actions: read if the runner reaper is on. github_runner_arc_app_setup_org: "" github_runner_arc_app_setup_name: "" github_runner_arc_app_setup_description: "Registers self-hosted GitHub Actions runners through Actions Runner Controller." diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index df11aed..305a519 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -300,6 +300,35 @@ argument_specs: github_runner_arc_node_recovery_image: type: str description: Image the node recovery watcher runs, pulled without a pull Secret. + github_runner_arc_runner_reaper_enabled: + type: bool + default: false + description: + - Install the runner reaper in each runner namespace, a CronJob that deletes an EphemeralRunner whose job's run is completed or cancelled, whose own job is completed, or whose job has run past github_runner_arc_runner_reaper_max_job_seconds. + - The org's GitHub App needs the Actions repository permission (read), approved on its installation; without it the reaper deletes nothing and its Jobs fail saying so. + - false removes the CronJob, its RBAC and its ConfigMap. + github_runner_arc_runner_reaper_max_job_seconds: + type: int + default: 432000 + description: Seconds a job may run before the reaper deletes its runner. The default is GitHub's own five-day limit on a self-hosted runner's job. + github_runner_arc_runner_reaper_completed_grace_seconds: + type: int + default: 300 + description: Seconds a run or job must have been completed before the reaper deletes a runner still holding it. + github_runner_arc_runner_reaper_schedule: + type: str + default: "*/5 * * * *" + description: Cron schedule of the runner reaper. + github_runner_arc_runner_reaper_dry_run: + type: bool + default: false + description: Log what the reaper would delete and delete nothing. + github_runner_arc_runner_reaper_image: + type: str + description: Image the reaper CronJob runs, pulled without a pull Secret. Defaults to the pull Secret renewal's image. + github_runner_arc_runner_active_deadline_seconds: + type: int + description: activeDeadlineSeconds on every runner pod, counted from the pod's start. Unset leaves the runner pods without one. github_runner_arc_autoscaler_usable_budget_gi: type: int description: Memory runner pods may use across every node. Required for an autoscaled profile. diff --git a/roles/github_runner_arc/tasks/install_runner_reaper.yml b/roles/github_runner_arc/tasks/install_runner_reaper.yml new file mode 100644 index 0000000..de4caa0 --- /dev/null +++ b/roles/github_runner_arc/tasks/install_runner_reaper.yml @@ -0,0 +1,33 @@ +--- +# Included once per scale-set profile from install_scale_set_profile.yml, with `org` and `profile` set. With github_runner_arc_runner_reaper_enabled it installs the runner reaper in the profile's namespace (templates/runner-reaper.yaml.j2): a CronJob that runs files/reap-runners.sh from a ConfigMap and deletes runners that have outlived their job. Otherwise it removes the CronJob, its ConfigMap and its RBAC, so turning it off leaves nothing behind. +# +# The job reads the App Secret through a volume, which the kubelet mounts for it, so its ServiceAccount needs no access to Secrets; its one Role grants list and delete on the namespace's EphemeralRunners and nothing else. + +- name: Describe the runner reaper for {{ profile.label }} + ansible.builtin.set_fact: + github_runner_arc_runner_reaper_resources: "{{ lookup('ansible.builtin.template', 'runner-reaper.yaml.j2') | from_yaml_all | list }}" + vars: + github_runner_arc_runner_reaper_scripts: + reap-runners.sh: "{{ lookup('ansible.builtin.file', 'reap-runners.sh') }}" + github-app-token.sh: "{{ lookup('ansible.builtin.file', 'github-app-token.sh') }}" + +- name: Install the runner reaper for {{ profile.label }} + kubernetes.core.k8s: + definition: "{{ item }}" + loop: "{{ github_runner_arc_runner_reaper_resources }}" + loop_control: + label: "{{ item.kind }}/{{ item.metadata.name }}" + when: github_runner_arc_runner_reaper_enabled | bool + +# Deleted in the reverse order, so the CronJob goes before the script and permissions it runs with. +- name: Remove any runner reaper for {{ profile.label }} + kubernetes.core.k8s: + state: absent + api_version: "{{ item.apiVersion }}" + kind: "{{ item.kind }}" + namespace: "{{ item.metadata.namespace }}" + name: "{{ item.metadata.name }}" + loop: "{{ github_runner_arc_runner_reaper_resources | reverse | list }}" + loop_control: + label: "{{ item.kind }}/{{ item.metadata.name }}" + when: not (github_runner_arc_runner_reaper_enabled | bool) diff --git a/roles/github_runner_arc/tasks/install_scale_set_profile.yml b/roles/github_runner_arc/tasks/install_scale_set_profile.yml index 94c1fd8..ef58134 100644 --- a/roles/github_runner_arc/tasks/install_scale_set_profile.yml +++ b/roles/github_runner_arc/tasks/install_scale_set_profile.yml @@ -71,6 +71,9 @@ - name: Install or remove the pull Secret renewal for {{ profile.label }} ansible.builtin.include_tasks: install_pull_secret_renewal.yml +- name: Install or remove the runner reaper for {{ profile.label }} + ansible.builtin.include_tasks: install_runner_reaper.yml + # Only when the profile sets no max_runners of its own, which takes precedence; the autoscaled profile's measured sizing is its autoscaler's ceiling, measured in install_platform.yml. - name: Measure the eligible nodes' free capacity for {{ profile.label }} ansible.builtin.include_tasks: measure_sizing.yml diff --git a/roles/github_runner_arc/tasks/validate.yml b/roles/github_runner_arc/tasks/validate.yml index 4563e01..d5fd76e 100644 --- a/roles/github_runner_arc/tasks/validate.yml +++ b/roles/github_runner_arc/tasks/validate.yml @@ -95,6 +95,22 @@ and ([github_runner_arc_node_recovery_after_seconds, github_runner_arc_node_recovery_poll_seconds] | map('string') | reject('match', '^[1-9][0-9]*$') | list | length > 0) }} + - msg: github_runner_arc_runner_reaper_max_job_seconds must be a whole number of seconds, at least 1, and github_runner_arc_runner_reaper_completed_grace_seconds a whole number of seconds. + failed: >- + {{ + github_runner_arc_runner_reaper_enabled | bool + and (github_runner_arc_runner_reaper_max_job_seconds | string is not match('^[1-9][0-9]*$') + or github_runner_arc_runner_reaper_completed_grace_seconds | string is not match('^[0-9]+$')) + }} + - msg: github_runner_arc_runner_reaper_schedule must be a cron schedule, five fields or a macro such as @hourly. + failed: >- + {{ + github_runner_arc_runner_reaper_enabled | bool + and not (github_runner_arc_runner_reaper_schedule | string is match('^@[a-z]+$') + or github_runner_arc_runner_reaper_schedule | string | split | length == 5) + }} + - msg: github_runner_arc_runner_active_deadline_seconds must be a whole number of seconds, at least 1, or unset. + failed: "{{ github_runner_arc_runner_active_deadline_seconds is not none and github_runner_arc_runner_active_deadline_seconds | string is not match('^[1-9][0-9]*$') }}" - msg: github_runner_arc_labelled_nodes needs github_runner_arc_node_label_key. failed: "{{ github_runner_arc_labelled_nodes | length > 0 and github_runner_arc_node_label_key | length == 0 }}" - msg: github_runner_arc_node_label_value must be set when github_runner_arc_node_label_key is. diff --git a/roles/github_runner_arc/templates/runner-reaper.yaml.j2 b/roles/github_runner_arc/templates/runner-reaper.yaml.j2 new file mode 100644 index 0000000..c1c81fc --- /dev/null +++ b/roles/github_runner_arc/templates/runner-reaper.yaml.j2 @@ -0,0 +1,139 @@ +{# The runner reaper for one scale-set profile (see tasks/install_runner_reaper.yml): a ServiceAccount, a Role that may only list and delete the namespace's EphemeralRunners, its RoleBinding, the ConfigMap holding files/reap-runners.sh and the token minting it sources, and the CronJob running them. `profile` is set by the including task, and github_runner_arc_runner_reaper_scripts holds the ConfigMap's data. #} +{% set name = 'runner-reaper' %} +{% set labels = {'app.kubernetes.io/name': name, 'app.kubernetes.io/part-of': 'github-runner'} %} +{% set scripts_dir = '/etc/runner-reaper' %} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ name | to_json }} + namespace: {{ profile.namespace | to_json }} + labels: {{ labels | to_json }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ name | to_json }} + namespace: {{ profile.namespace | to_json }} + labels: {{ labels | to_json }} +rules: + # list to read every runner's recorded job, delete to remove one. Nothing else: no pods, no Secrets (the App Secret reaches the job as a volume the kubelet mounts) and no write to a runner's status. + - apiGroups: ["actions.github.com"] + resources: ["ephemeralrunners"] + verbs: ["list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ name | to_json }} + namespace: {{ profile.namespace | to_json }} + labels: {{ labels | to_json }} +subjects: + - kind: ServiceAccount + name: {{ name | to_json }} + namespace: {{ profile.namespace | to_json }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ name | to_json }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ name | to_json }} + namespace: {{ profile.namespace | to_json }} + labels: {{ labels | to_json }} +data: {{ github_runner_arc_runner_reaper_scripts | to_json }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ name | to_json }} + namespace: {{ profile.namespace | to_json }} + labels: {{ labels | to_json }} +spec: + schedule: {{ github_runner_arc_runner_reaper_schedule | to_json }} + # One pass at a time; a pass that misses its slot by more than this is skipped rather than run late, and the next one checks instead. + concurrencyPolicy: Forbid + startingDeadlineSeconds: 300 + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + # A pass that fails, because GitHub or the token was unavailable, is not retried: it deleted nothing it could not judge, the failed Job stays visible, and the next scheduled pass tries again. + backoffLimit: 0 + activeDeadlineSeconds: 300 + template: + metadata: + labels: {{ labels | to_json }} + spec: + serviceAccountName: {{ name | to_json }} + restartPolicy: Never +{% if github_runner_arc_node_label_key | length > 0 %} + nodeSelector: {{ {github_runner_arc_node_label_key: github_runner_arc_node_label_value | string} | to_json }} +{% endif %} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + # Makes the mounted App Secret readable by that group, and by nobody else. + fsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: reap + image: {{ github_runner_arc_runner_reaper_image | to_json }} + command: ["bash", {{ (scripts_dir ~ '/reap-runners.sh') | to_json }}] + env: + - name: RUNNER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MAX_JOB_SECONDS + value: {{ github_runner_arc_runner_reaper_max_job_seconds | string | to_json }} + - name: COMPLETED_GRACE_SECONDS + value: {{ github_runner_arc_runner_reaper_completed_grace_seconds | string | to_json }} + - name: DRY_RUN + value: {{ (github_runner_arc_runner_reaper_dry_run | bool) | string | lower | to_json }} + - name: GITHUB_APP_DIR + value: /var/run/github-app + # kubectl's cache and the script's working files go to the in-memory volume, since the root filesystem is read-only. + - name: HOME + value: /tmp + - name: TMPDIR + value: /tmp + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 200m + memory: 64Mi + volumeMounts: + - name: scripts + mountPath: {{ scripts_dir | to_json }} + readOnly: true + - name: github-app + mountPath: /var/run/github-app + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: scripts + configMap: + name: {{ name | to_json }} + # 0444: the scripts are run through bash, so nobody needs to execute or write them. + defaultMode: 292 + - name: github-app + secret: + secretName: {{ profile.app_secret | to_json }} + # 0440: owner and group read. + defaultMode: 288 + - name: tmp + emptyDir: + medium: Memory + sizeLimit: 8Mi diff --git a/tests/runner_reaper/render.yml b/tests/runner_reaper/render.yml new file mode 100644 index 0000000..8672aa6 --- /dev/null +++ b/tests/runner_reaper/render.yml @@ -0,0 +1,21 @@ +# Renders the github_runner_arc role's runner reaper manifests (templates/runner-reaper.yaml.j2) with the role's defaults and constants and the ConfigMap data install_runner_reaper.yml gives them, for tests/runner_reaper/run.sh to apply to a test cluster and tests/unit/test_runner_reaper_role.py to read back. Takes reaper_namespace and reaper_output as extra variables; a role variable such as the image or the bound is overridden with an extra variable of its own name, since the defaults file loaded here outranks play variables. +- name: Render the runner reaper manifests + hosts: localhost + connection: local + gather_facts: false + vars_files: + - ../../roles/github_runner_arc/defaults/main.yml + - ../../roles/github_runner_arc/vars/main.yml + vars: + profile: + namespace: "{{ reaper_namespace }}" + app_secret: example-org-github-app + github_runner_arc_runner_reaper_scripts: + reap-runners.sh: "{{ lookup('ansible.builtin.file', playbook_dir ~ '/../../roles/github_runner_arc/files/reap-runners.sh') }}" + github-app-token.sh: "{{ lookup('ansible.builtin.file', playbook_dir ~ '/../../roles/github_runner_arc/files/github-app-token.sh') }}" + tasks: + - name: Write the manifests + ansible.builtin.copy: + content: "{{ lookup('ansible.builtin.template', playbook_dir ~ '/../../roles/github_runner_arc/templates/runner-reaper.yaml.j2') }}" + dest: "{{ reaper_output }}" + mode: "0644" diff --git a/tests/unit/test_runner_reaper_role.py b/tests/unit/test_runner_reaper_role.py new file mode 100644 index 0000000..89623c3 --- /dev/null +++ b/tests/unit/test_runner_reaper_role.py @@ -0,0 +1,132 @@ +"""Tests for how the github_runner_arc role installs the runner reaper: the manifests templates/runner-reaper.yaml.j2 renders (through tests/runner_reaper/render.yml, with ansible-playbook), and the toggle in tasks/install_runner_reaper.yml that installs them or removes them.""" + +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +ROLE = ROOT / "roles" / "github_runner_arc" +RENDER = ROOT / "tests" / "runner_reaper" / "render.yml" +NAMESPACE = "example-runners" + + +def render(**role_vars: Any) -> dict[str, dict[str, Any]]: + """Render the reaper's manifests with ansible-playbook and return them by kind.""" + with tempfile.TemporaryDirectory() as work: + output = Path(work) / "reaper.yaml" + extra = Path(work) / "extra.json" + extra.write_text(json.dumps({"reaper_namespace": NAMESPACE, "reaper_output": str(output), **role_vars})) + env = {**os.environ, "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false"} + subprocess.run([os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook"), str(RENDER), "-e", f"@{extra}"], check=True, env=env, cwd=work, capture_output=True, text=True) + documents = [document for document in yaml.safe_load_all(output.read_text()) if document] + by_kind = {document["kind"]: document for document in documents} + assert len(by_kind) == len(documents), "two manifests of one kind" + return by_kind + + +class ManifestTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.manifests = render() + cls.pod = cls.manifests["CronJob"]["spec"]["jobTemplate"]["spec"]["template"]["spec"] + cls.container = cls.pod["containers"][0] + + def test_everything_is_in_the_runner_namespace(self) -> None: + self.assertEqual(set(self.manifests), {"ServiceAccount", "Role", "RoleBinding", "ConfigMap", "CronJob"}) + for manifest in self.manifests.values(): + self.assertEqual(manifest["metadata"]["namespace"], NAMESPACE) + + def test_the_role_may_only_list_and_delete_ephemeral_runners(self) -> None: + self.assertEqual(self.manifests["Role"]["rules"], [{"apiGroups": ["actions.github.com"], "resources": ["ephemeralrunners"], "verbs": ["list", "delete"]}]) + + def test_the_binding_gives_that_role_to_the_cronjobs_service_account(self) -> None: + binding = self.manifests["RoleBinding"] + account = self.manifests["ServiceAccount"]["metadata"]["name"] + self.assertEqual(binding["subjects"], [{"kind": "ServiceAccount", "name": account, "namespace": NAMESPACE}]) + self.assertEqual(binding["roleRef"], {"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": self.manifests["Role"]["metadata"]["name"]}) + self.assertEqual(self.pod["serviceAccountName"], account) + + def test_the_configmap_carries_the_script_and_the_token_minting_it_sources(self) -> None: + files = ROLE / "files" + self.assertEqual(self.manifests["ConfigMap"]["data"], { + "reap-runners.sh": (files / "reap-runners.sh").read_text().rstrip("\n"), + "github-app-token.sh": (files / "github-app-token.sh").read_text().rstrip("\n"), + }) + + def test_the_cronjob_runs_the_mounted_script_one_pass_at_a_time(self) -> None: + cronjob = self.manifests["CronJob"] + self.assertEqual(cronjob["spec"]["schedule"], "*/5 * * * *") + self.assertEqual(cronjob["spec"]["concurrencyPolicy"], "Forbid") + self.assertEqual(cronjob["spec"]["jobTemplate"]["spec"]["backoffLimit"], 0) + mounts = {mount["name"]: mount for mount in self.container["volumeMounts"]} + self.assertEqual(self.container["command"], ["bash", mounts["scripts"]["mountPath"] + "/reap-runners.sh"]) + self.assertTrue(mounts["scripts"]["readOnly"]) + volumes = {volume["name"]: volume for volume in self.pod["volumes"]} + self.assertEqual(volumes["scripts"]["configMap"]["name"], self.manifests["ConfigMap"]["metadata"]["name"]) + self.assertEqual(volumes["github-app"]["secret"]["secretName"], "example-org-github-app") + self.assertEqual(mounts["github-app"]["mountPath"], self.env()["GITHUB_APP_DIR"]) + + def test_the_bound_grace_and_dry_run_reach_the_script(self) -> None: + self.assertEqual({key: self.env()[key] for key in ("MAX_JOB_SECONDS", "COMPLETED_GRACE_SECONDS", "DRY_RUN")}, {"MAX_JOB_SECONDS": "432000", "COMPLETED_GRACE_SECONDS": "300", "DRY_RUN": "false"}) + container = render(github_runner_arc_runner_reaper_max_job_seconds=7200, github_runner_arc_runner_reaper_dry_run=True)["CronJob"]["spec"]["jobTemplate"]["spec"]["template"]["spec"]["containers"][0] + env = {item["name"]: item.get("value") for item in container["env"]} + self.assertEqual((env["MAX_JOB_SECONDS"], env["DRY_RUN"]), ("7200", "true")) + + def test_the_pod_runs_unprivileged_with_a_read_only_root(self) -> None: + self.assertTrue(self.pod["securityContext"]["runAsNonRoot"]) + self.assertFalse(self.container["securityContext"]["allowPrivilegeEscalation"]) + self.assertTrue(self.container["securityContext"]["readOnlyRootFilesystem"]) + self.assertEqual(self.container["securityContext"]["capabilities"], {"drop": ["ALL"]}) + + def test_the_image_defaults_to_the_pull_secret_renewers(self) -> None: + defaults = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) + self.assertEqual(self.container["image"], defaults["github_runner_arc_image_pull_secret_renewer_image"]) + + def env(self) -> dict[str, str]: + return {item["name"]: item["value"] for item in self.container["env"] if "value" in item} + + +class ToggleTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.tasks = yaml.safe_load((ROLE / "tasks" / "install_runner_reaper.yml").read_text()) + + def task(self, prefix: str) -> dict[str, Any]: + [task] = [task for task in self.tasks if task["name"].startswith(prefix)] + return task + + def test_it_is_off_by_default(self) -> None: + defaults = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) + self.assertIs(defaults["github_runner_arc_runner_reaper_enabled"], False) + specs = yaml.safe_load((ROLE / "meta" / "argument_specs.yml").read_text()) + options = specs["argument_specs"]["main"]["options"] + self.assertIs(options["github_runner_arc_runner_reaper_enabled"]["default"], False) + + def test_on_installs_every_manifest(self) -> None: + install = self.task("Install the runner reaper") + self.assertEqual(install["when"], "github_runner_arc_runner_reaper_enabled | bool") + self.assertEqual(install["loop"], "{{ github_runner_arc_runner_reaper_resources }}") + self.assertEqual(install["kubernetes.core.k8s"], {"definition": "{{ item }}"}) + + def test_off_removes_every_manifest_cronjob_first(self) -> None: + remove = self.task("Remove any runner reaper") + self.assertEqual(remove["when"], "not (github_runner_arc_runner_reaper_enabled | bool)") + self.assertEqual(remove["kubernetes.core.k8s"]["state"], "absent") + self.assertEqual(remove["loop"], "{{ github_runner_arc_runner_reaper_resources | reverse | list }}") + self.assertEqual(list(render())[-1], "CronJob") + + def test_each_profile_includes_it(self) -> None: + profile_tasks = yaml.safe_load((ROLE / "tasks" / "install_scale_set_profile.yml").read_text()) + self.assertIn("install_runner_reaper.yml", [task.get("ansible.builtin.include_tasks") for task in profile_tasks]) + + +if __name__ == "__main__": + unittest.main() From 3a9fdddb2f5602a90cdf27847b6cd6ac23beea43 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 8 Oct 2026 14:25:18 +0100 Subject: [PATCH 4/8] feat(arc): set activeDeadlineSeconds on runner pods when configured github_runner_arc_runner_active_deadline_seconds puts the deadline on every runner pod as a backstop that needs no GitHub call. Unset leaves the runner pod spec as it was. --- .../templates/scale-set-overlay.yaml.j2 | 4 ++++ tests/unit/test_scale_set_overlay.py | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/roles/github_runner_arc/templates/scale-set-overlay.yaml.j2 b/roles/github_runner_arc/templates/scale-set-overlay.yaml.j2 index f864626..03a611f 100644 --- a/roles/github_runner_arc/templates/scale-set-overlay.yaml.j2 +++ b/roles/github_runner_arc/templates/scale-set-overlay.yaml.j2 @@ -19,6 +19,10 @@ template: {% for field, value in (profile | exadev.github_runner.arc_runner_placement(eligibility)).items() %} {{ field }}: {{ value | to_json }} {% endfor %} +{% if github_runner_arc_runner_active_deadline_seconds is not none %} + # The backstop for a pod that outlives its job: the kubelet stops it this long after it starts. + activeDeadlineSeconds: {{ github_runner_arc_runner_active_deadline_seconds | int }} +{% endif %} {% if github_runner_arc_node_label_key | length > 0 or github_runner_arc_listener_probes_enabled | bool %} # The chart puts no nodeSelector and no probe on the listener. The AutoscalingRunnerSet needs containers whenever listenerTemplate.spec is set, and a container named listener is merged into the real listener rather than added as a sidecar. listenerTemplate: diff --git a/tests/unit/test_scale_set_overlay.py b/tests/unit/test_scale_set_overlay.py index 9f1e3da..7f1d6b8 100644 --- a/tests/unit/test_scale_set_overlay.py +++ b/tests/unit/test_scale_set_overlay.py @@ -81,5 +81,18 @@ def test_burst_runners_are_added_to_a_measured_ceiling(self) -> None: self.assertEqual(overlay["template"]["spec"]["tolerations"], BURST["tolerations"]) +class ActiveDeadlineTest(unittest.TestCase): + def test_unset_leaves_the_runner_pod_spec_exactly_as_without_it(self) -> None: + overlay = render(expanded({"sizing": SIZING}), github_runner_arc_node_label_key=ELIGIBLE, github_runner_arc_node_label_value="true") + self.assertEqual(overlay["template"], {"spec": {"nodeSelector": {ELIGIBLE: "true"}}}) + + def test_set_adds_it_to_the_runner_pod_spec_and_nothing_else(self) -> None: + profile = expanded({"sizing": SIZING}) + without = render(profile, github_runner_arc_node_label_key=ELIGIBLE, github_runner_arc_node_label_value="true") + overlay = render(profile, github_runner_arc_node_label_key=ELIGIBLE, github_runner_arc_node_label_value="true", github_runner_arc_runner_active_deadline_seconds=21600) + self.assertEqual(overlay["template"]["spec"].pop("activeDeadlineSeconds"), 21600) + self.assertEqual(overlay, without) + + if __name__ == "__main__": unittest.main() From 8a29a077e6fe238374ba34af66d29ca4620ed427 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 8 Oct 2026 14:29:08 +0100 Subject: [PATCH 5/8] refactor(arc): word a refused token mint for any permission --- roles/github_runner_arc/files/github-app-token.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/github_runner_arc/files/github-app-token.sh b/roles/github_runner_arc/files/github-app-token.sh index 3cb7016..16d64aa 100644 --- a/roles/github_runner_arc/files/github-app-token.sh +++ b/roles/github_runner_arc/files/github-app-token.sh @@ -47,7 +47,7 @@ github_app_mint_token() { "${api_url}/app/installations/${installation_id}/access_tokens")" || fail "could not reach the GitHub API" rm -f "$work/github-app.curl" if [ "$status" != "201" ]; then - fail "GitHub refused to mint a ${wanted} installation token (HTTP ${status}: $(jq -r '.message // empty' "$work/mint.json" 2>/dev/null)). The App needs the ${wanted} permission, approved on this installation" + fail "GitHub refused to mint an installation token restricted to ${wanted} (HTTP ${status}: $(jq -r '.message // empty' "$work/mint.json" 2>/dev/null)). The App needs the ${wanted} permission, approved on this installation" fi jq -r '.token // empty' "$work/mint.json" > "$work/token" jq -r '.expires_at // empty' "$work/mint.json" > "$work/expires_at" From 2f2f4b421d0bd6e8a71f0cf439fc099a75fdbef4 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 8 Oct 2026 14:29:18 +0100 Subject: [PATCH 6/8] test(arc): show the runner reaper deleting stuck runners in kind Applies ARC's CRDs and the role's reaper manifests to a kind cluster, records jobs on EphemeralRunners as ARC's listener does, and runs a pass against a stand-in for GitHub: the runner past the bound and the cancelled run's runner go, the idle and busy runners stay, the ServiceAccount can do nothing else, and a refused token deletes nothing. --- .github/workflows/runner-reaper.yml | 43 +++++++ tests/runner_reaper/run.sh | 169 ++++++++++++++++++++++++++++ tests/runner_reaper/stub_github.py | 78 +++++++++++++ 3 files changed, 290 insertions(+) create mode 100644 .github/workflows/runner-reaper.yml create mode 100755 tests/runner_reaper/run.sh create mode 100644 tests/runner_reaper/stub_github.py diff --git a/.github/workflows/runner-reaper.yml b/.github/workflows/runner-reaper.yml new file mode 100644 index 0000000..4d3d83e --- /dev/null +++ b/.github/workflows/runner-reaper.yml @@ -0,0 +1,43 @@ +name: Runner reaper + +# Runs the ARC role's runner reaper in a kind cluster (tests/runner_reaper/run.sh): ARC's EphemeralRunner CRD, the role's rendered CronJob, RBAC and ConfigMap, the pull Secret renewal's image built from this checkout, and a stand-in for GitHub's API. Shows a runner whose job ran past the bound and one whose run was cancelled being deleted, and an idle runner and a busy one being left alone. Only on changes that can affect the reaper. +# +# GitHub-hosted ubuntu-latest has Docker, kind, kubectl, helm and jq already. + +on: + pull_request: + paths: + - roles/github_runner_arc/** + - pull-secret-renewer/** + - tests/runner_reaper/** + - .github/workflows/runner-reaper.yml + workflow_dispatch: + +concurrency: + group: runner-reaper-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + reaper: + name: Reaper in kind + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v7 + + - name: Install Ansible + run: | + set -euo pipefail + if ! python3 -m venv "${RUNNER_TEMP}/venv" 2>/dev/null; then + sudo apt-get update -qq + sudo apt-get install -y -qq python3-venv + python3 -m venv "${RUNNER_TEMP}/venv" + fi + "${RUNNER_TEMP}/venv/bin/pip" install --quiet --prefer-binary ansible-core + echo "${RUNNER_TEMP}/venv/bin" >> "$GITHUB_PATH" + + - name: Run the reaper test + run: tests/runner_reaper/run.sh diff --git a/tests/runner_reaper/run.sh b/tests/runner_reaper/run.sh new file mode 100755 index 0000000..c4ffc0c --- /dev/null +++ b/tests/runner_reaper/run.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Integration test for the github_runner_arc role's runner reaper: renders the role's reaper manifests, applies them to a kind cluster that has ARC's EphemeralRunner CRD (no controller, so a deleted runner goes at once), and runs the CronJob's Job in the pull Secret renewal's image built from this checkout against a stand-in for GitHub (stub_github.py). Asserts the ServiceAccount can only list and delete EphemeralRunners in its namespace; that a pass deletes a runner whose job has run past the bound and one whose run was cancelled, and leaves an idle runner and one whose job is within the bound; and that a refused token fails the Job and deletes nothing. The script's every branch is covered by tests/unit/test_reap_runners.py; this shows the role's CronJob, RBAC and ConfigMap doing it in a cluster. +# +# Usage: tests/runner_reaper/run.sh. Needs Docker, kind, kubectl, helm, jq, openssl and ansible-playbook (ANSIBLE_PLAYBOOK overrides which). Creates a kind cluster named grtest-reaper and removes it on exit unless GRTEST_KEEP=1. +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +ansible_playbook="${ANSIBLE_PLAYBOOK:-ansible-playbook}" +cluster=grtest-reaper +namespace=grtest-runners +other_namespace=grtest-elsewhere +image=grtest/pull-secret-renewer:test +stub_image=python:3.13-alpine +controller_chart=oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller +reaper=runner-reaper +app_secret=example-org-github-app +# An hour: the stand-in's hung job started days ago and its busy job a minute ago, so the bound falls between them. +bound=3600 +job_timeout=180s +work="$(mktemp -d "${TMPDIR:-/tmp}/grtest-reaper.XXXXXX")" +export KUBECONFIG="$work/kubeconfig" + +log() { echo "==> $*"; } +fail() { + echo "FAIL: $*" >&2 + kubectl -n "$namespace" get jobs,pods,ephemeralrunners >&2 || true + kubectl -n "$namespace" logs -l app.kubernetes.io/name=runner-reaper --tail=40 --prefix >&2 || true + kubectl -n "$namespace" logs deploy/stub-github --tail=40 >&2 || true + exit 1 +} +cleanup() { + if [ "${GRTEST_KEEP:-0}" != 1 ]; then + kind delete cluster --name "$cluster" >/dev/null 2>&1 || true + rm -rf "$work" + fi +} +trap cleanup EXIT + +# Runs one Job from the CronJob and waits for it to finish either way, printing complete, failed or timeout. +run_job() { + local job="$1" + kubectl -n "$namespace" create job "$job" --from="cronjob/$reaper" >/dev/null + kubectl -n "$namespace" wait --for=condition=complete "job/$job" --timeout="$job_timeout" >/dev/null 2>&1 & + local complete=$! + kubectl -n "$namespace" wait --for=condition=failed "job/$job" --timeout="$job_timeout" >/dev/null 2>&1 & + local failed=$! + while kill -0 "$complete" 2>/dev/null && kill -0 "$failed" 2>/dev/null; do + sleep 2 + done + if wait "$complete" 2>/dev/null; then + kill "$failed" 2>/dev/null || true + echo complete + elif wait "$failed" 2>/dev/null; then + kill "$complete" 2>/dev/null || true + echo failed + else + echo timeout + fi +} + +# Creates an EphemeralRunner as ARC's controller would; with a run id it also records the job on its status, as ARC's listener does when it assigns the runner a job. +create_runner() { + local name="$1" run_id="${2:-}" + kubectl -n "$namespace" apply -f - >/dev/null </dev/null + fi +} + +runner_exists() { + kubectl -n "$namespace" get ephemeralrunner "$1" >/dev/null 2>&1 +} + +log "Creating kind cluster $cluster" +kind create cluster --name "$cluster" --kubeconfig "$KUBECONFIG" --wait 120s >/dev/null + +log "Applying ARC's CRDs" +helm show crds "$controller_chart" 2>/dev/null | kubectl apply --server-side -f - >/dev/null +kubectl wait --for=condition=established crd/ephemeralrunners.actions.github.com --timeout=60s >/dev/null + +log "Building and loading the image the reaper runs in" +docker build -q -f "$repo_root/pull-secret-renewer/Dockerfile" -t "$image" "$repo_root" >/dev/null +kind load docker-image "$image" --name "$cluster" >/dev/null + +log "Starting the GitHub stand-in" +kubectl create namespace "$namespace" >/dev/null +kubectl create namespace "$other_namespace" >/dev/null +kubectl -n "$namespace" create configmap stub-github --from-file="$repo_root/tests/runner_reaper/stub_github.py" >/dev/null +kubectl -n "$namespace" create deployment stub-github --image="$stub_image" --port=8080 -- python3 /stub/stub_github.py >/dev/null +kubectl -n "$namespace" patch deployment stub-github --type=json -p '[ + {"op": "add", "path": "/spec/template/spec/volumes", "value": [{"name": "stub", "configMap": {"name": "stub-github"}}]}, + {"op": "add", "path": "/spec/template/spec/containers/0/volumeMounts", "value": [{"name": "stub", "mountPath": "/stub"}]} +]' >/dev/null +kubectl -n "$namespace" expose deployment stub-github --port=8080 >/dev/null +kubectl -n "$namespace" rollout status deployment/stub-github --timeout=120s >/dev/null + +log "Creating the App Secret and the runners" +openssl genrsa -out "$work/app.pem" 2048 2>/dev/null +kubectl -n "$namespace" create secret generic "$app_secret" \ + --from-literal=github_app_id=12345 --from-literal=github_app_installation_id=67890 \ + --from-file=github_app_private_key="$work/app.pem" >/dev/null +create_runner idle-runner +create_runner hung-runner 201 +create_runner cancelled-runner 202 +create_runner busy-runner 203 + +log "Rendering and applying the role's reaper manifests" +ANSIBLE_COLLECTIONS_PATH="$repo_root/playbooks/collections${ANSIBLE_COLLECTIONS_PATH:+:$ANSIBLE_COLLECTIONS_PATH}" \ + "$ansible_playbook" "$repo_root/tests/runner_reaper/render.yml" \ + -e reaper_namespace="$namespace" -e github_runner_arc_runner_reaper_image="$image" \ + -e github_runner_arc_runner_reaper_max_job_seconds="$bound" -e reaper_output="$work/reaper.yaml" >/dev/null +kubectl apply -f "$work/reaper.yaml" >/dev/null +# The test points the Job at the stand-in; everything else is as the role renders it. +kubectl -n "$namespace" patch cronjob "$reaper" --type=json -p "[{\"op\": \"add\", \"path\": \"/spec/jobTemplate/spec/template/spec/containers/0/env/-\", \"value\": {\"name\": \"GITHUB_API_URL\", \"value\": \"http://stub-github.${namespace}.svc:8080\"}}]" >/dev/null + +log "Checking the ServiceAccount's permissions" +as="system:serviceaccount:${namespace}:${reaper}" +check_can() { + local expected="$1" + shift + local answer + answer="$(kubectl auth can-i --as="$as" "$@" 2>/dev/null || true)" + [ "$answer" = "$expected" ] || fail "can-i $* answered '$answer', expected '$expected'" +} +check_can yes list ephemeralrunners.actions.github.com -n "$namespace" +check_can yes delete ephemeralrunners.actions.github.com -n "$namespace" +check_can no patch ephemeralrunners.actions.github.com -n "$namespace" +check_can no create ephemeralrunners.actions.github.com -n "$namespace" +check_can no list ephemeralrunners.actions.github.com -n "$other_namespace" +check_can no delete ephemeralrunners.actions.github.com -n "$other_namespace" +check_can no delete pods -n "$namespace" +check_can no get "secret/$app_secret" -n "$namespace" +check_can no list secrets -n "$namespace" + +log "Reaping: the hung and cancelled runners should go, the idle and busy ones stay" +[ "$(run_job reap-1)" = complete ] || fail "the reaper Job did not complete" +runner_exists hung-runner && fail "the runner whose job ran past the bound is still there" +runner_exists cancelled-runner && fail "the runner whose run was cancelled is still there" +runner_exists idle-runner || fail "the idle runner was deleted" +runner_exists busy-runner || fail "the runner whose job is within the bound was deleted" +kubectl -n "$namespace" logs job/reap-1 > "$work/reap-1.log" +grep -q "deleted ${namespace}/hung-runner: run 201 in example-org/example-repo: job 9201 has been in_progress for .*over the bound of ${bound}s" "$work/reap-1.log" \ + || fail "the log does not say why the hung runner was deleted: $(cat "$work/reap-1.log")" +grep -q "deleted ${namespace}/cancelled-runner: run 202 in example-org/example-repo is cancelled" "$work/reap-1.log" \ + || fail "the log does not say why the cancelled runner was deleted: $(cat "$work/reap-1.log")" +kubectl -n "$namespace" logs deploy/stub-github > "$work/stub.log" +[ "$(jq -sc 'map(select(.method == "POST"))[0].body' "$work/stub.log")" = '{"permissions":{"actions":"read"}}' ] || fail "the mint did not ask for actions: read only" +jq -se 'map(select(.method == "GET")) | all(.authorization == "Bearer ghs_stubreapertoken")' "$work/stub.log" >/dev/null || fail "a run was read without the minted token" +jq -se 'map(select(.method == "GET" and (.path | contains("/runs/")))) | length == 3' "$work/stub.log" >/dev/null || fail "the reaper looked up something other than the three assigned runners' runs" + +log "Refusing the token: the Job should fail and delete nothing" +create_runner second-hung-runner 201 +kubectl -n "$namespace" patch secret "$app_secret" --type=merge -p "{\"data\": {\"github_app_installation_id\": \"$(printf 999 | base64)\"}}" >/dev/null +# The kubelet refreshes a mounted Secret on its own sync period; a new pod reads it fresh, which is what each Job starts. +[ "$(run_job reap-refused)" = failed ] || fail "a refused token did not fail the Job" +runner_exists second-hung-runner || fail "a pass whose token was refused deleted a runner" +kubectl -n "$namespace" logs job/reap-refused 2>/dev/null | grep -q "The App needs the actions: read permission.*deleted nothing" || fail "the refused Job did not name the permission it needs" + +log "PASS" diff --git a/tests/runner_reaper/stub_github.py b/tests/runner_reaper/stub_github.py new file mode 100644 index 0000000..81b851e --- /dev/null +++ b/tests/runner_reaper/stub_github.py @@ -0,0 +1,78 @@ +"""A stand-in for the parts of GitHub's API the runner reaper reads, run inside the test cluster by tests/runner_reaper/run.sh. + +POST /app/installations//access_tokens answers 201 with a token, unless the installation id is the refused one, when it answers 422 as GitHub does for a permission the installation lacks. GET /repos//actions/runs/ and its /jobs answer from RUNS below, with timestamps relative to the moment of the request; any other path answers 404. Each request is logged as one JSON line on stdout, so the test reads what the job asked for from the pod's log. +""" + +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import urlsplit + +REFUSED_INSTALLATION = "999" +REPOSITORY = "example-org/example-repo" +PORT = 8080 +DAY = 86400 + +# Run id: (run status, conclusion, seconds since the run was last updated, jobs as (id, runner name, status, seconds since it started)). +RUNS = { + 201: ("in_progress", None, 0, [(9201, "hung-runner", "in_progress", 10 * DAY)]), + 202: ("completed", "cancelled", DAY, [(9202, "cancelled-runner", "completed", 2 * DAY)]), + 203: ("in_progress", None, 0, [(9203, "busy-runner", "in_progress", 60)]), +} + + +def ago(seconds: int) -> str: + return (datetime.now(timezone.utc) - timedelta(seconds=seconds)).strftime("%Y-%m-%dT%H:%M:%SZ") + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 (the name http.server calls) + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length", "0"))) or b"{}") + segments = self.path.strip("/").split("/") + installation = segments[2] if len(segments) == 4 and segments[3] == "access_tokens" else "" + print(json.dumps({"method": "POST", "path": self.path, "body": body}), flush=True) + if not installation: + self.reply(404, {"message": "Not Found"}) + elif installation == REFUSED_INSTALLATION: + self.reply(422, {"message": "The permissions requested are not granted to this installation."}) + else: + self.reply(201, {"token": "ghs_stubreapertoken", "expires_at": ago(-3600)}) + + def do_GET(self) -> None: # noqa: N802 (the name http.server calls) + print(json.dumps({"method": "GET", "path": self.path, "authorization": self.headers.get("Authorization", "")}), flush=True) + path = urlsplit(self.path).path + prefix = f"/repos/{REPOSITORY}/actions/runs/" + segments = path.removeprefix(prefix).split("/") if path.startswith(prefix) else [] + run = RUNS.get(int(segments[0])) if segments and segments[0].isdigit() else None + if run is None: + self.reply(404, {"message": "Not Found"}) + return + status, conclusion, updated, jobs = run + if segments[1:] == []: + self.reply(200, {"id": int(segments[0]), "status": status, "conclusion": conclusion, "updated_at": ago(updated)}) + elif segments[1:] == ["jobs"]: + listed = [ + {"id": job_id, "runner_name": runner, "status": job_status, "conclusion": conclusion if job_status == "completed" else None, + "started_at": ago(started), "completed_at": ago(updated) if job_status == "completed" else None} + for job_id, runner, job_status, started in jobs + ] + self.reply(200, {"total_count": len(listed), "jobs": listed}) + else: + self.reply(404, {"message": "Not Found"}) + + def reply(self, status: int, payload: dict) -> None: + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *_args: object) -> None: + return + + +if __name__ == "__main__": + HTTPServer(("", PORT), Handler).serve_forever() From 4bb5aef1e07ef987f16024ecb3a9d67826b60413 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 8 Oct 2026 14:30:09 +0100 Subject: [PATCH 7/8] docs(arc): document the runner reaper, its bound, permission and safety --- roles/github_runner_arc/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index a80c623..01c9dec 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -44,6 +44,8 @@ Upgrading ARC on a live fleet. The new controller deletes every scale set whose - `github_runner_arc_heartbeat_gist_id`: install the fleet-health platform from this host, refreshing this gist. `github_runner_arc_heartbeat_bootstrap_gist`, `github_runner_arc_heartbeat_gist_description` and `github_runner_arc_heartbeat_gist_consumer` control the bootstrap described above. - `github_runner_arc_autoscaler_usable_budget_gi`, `github_runner_arc_autoscaler_max_ceiling`, `github_runner_arc_autoscaler_floor`: required when a profile sets `autoscale: true` and the platform is installed. The floor must equal the autoscaled profile's `maxRunners`, which every Helm upgrade reverts to. The other `github_runner_arc_autoscaler_*` and `github_runner_arc_heartbeat_*` settings have defaults; see `defaults/main.yml`. - `github_runner_arc_node_recovery_enabled` (default `true`), `github_runner_arc_node_recovery_after_seconds` (default 120), `github_runner_arc_node_recovery_poll_seconds` (default 15), `github_runner_arc_node_recovery_image`: the node recovery watcher (see [Node recovery](#node-recovery)). +- `github_runner_arc_runner_reaper_enabled` (default `false`), `github_runner_arc_runner_reaper_max_job_seconds` (default 432000, five days), `github_runner_arc_runner_reaper_completed_grace_seconds` (default 300), `github_runner_arc_runner_reaper_schedule` (default every 5 minutes), `github_runner_arc_runner_reaper_dry_run` (default `false`), `github_runner_arc_runner_reaper_image` (default the renewal's image): the runner reaper (see [Runner reaper](#runner-reaper)). +- `github_runner_arc_runner_active_deadline_seconds` (default unset): `activeDeadlineSeconds` on every runner pod (see [Runner reaper](#runner-reaper)). - `github_runner_arc_app_setup_*`, `github_runner_arc_app_manifest_code`, `github_runner_arc_app_private_key_path`: inputs to `playbooks/github_app_setup.yml` (see below), including `github_runner_arc_app_setup_write_secret`, `github_runner_arc_app_setup_secret_namespaces`, `github_runner_arc_app_setup_secret_name`, `github_runner_arc_app_setup_keep_private_key` and `github_runner_arc_app_setup_command`. ## Sizing @@ -332,6 +334,30 @@ github_runner_arc_orgs: Every host that installs anything also installs a small watcher, the `node-recovery` Deployment in `github_runner_arc_platform_namespace`, so that a node that stops does not leave its pods `Terminating` for ever. ARC waits for a listener's old pod to go before starting a new one, so a listener on a stopped node otherwise stalls its scale set until the pod is force-deleted by hand. Once a node's `Ready` condition has been `Unknown` for `github_runner_arc_node_recovery_after_seconds`, the watcher gives it the `node.kubernetes.io/out-of-service=nodeshutdown:NoExecute` taint of Kubernetes' non-graceful node shutdown, which makes the control plane evict the node's pods and force-delete the terminating ones; it removes the taint when the node reports `Ready` again. It never acts on a node reporting `NotReady`, never on its own node, and never removes a taint it did not apply. Its ClusterRole allows `get`, `list` and `patch` on nodes and nothing else. `github_runner_arc_node_recovery_enabled: false` removes the watcher, its RBAC and any taint it left behind. Its image is pulled without a pull Secret, so it must be public. The repository README's Node recovery section explains the default threshold. +## Runner reaper + +A runner pod can outlive its job: a step hangs and the run is cancelled but the runner never exits, or the run ends while the pod keeps its slot. With `github_runner_arc_runner_reaper_enabled: true` each scale-set profile's namespace gets a `runner-reaper` CronJob that finds such runners and deletes their EphemeralRunner, which the scale set then replaces. It is off by default. + +Each pass mints an installation token from the profile's App Secret restricted to `actions: read`, lists the namespace's EphemeralRunners, and reads the workflow run that ARC's listener recorded on each runner when it assigned it a job (`status.workflowRunId` and `status.jobRepositoryName`). It deletes the runner when: + +- the run is completed, whatever its conclusion, cancelled included; +- the run is still going but the runner's own job (the job of any attempt of the run whose `runner_name` is the runner's name) is completed; or +- that job has been running for longer than `github_runner_arc_runner_reaper_max_job_seconds`, counted from the job's `started_at`. + +A run or job only counts as completed once it has been completed for `github_runner_arc_runner_reaper_completed_grace_seconds`, since ARC removes a runner that finished normally within seconds and the reaper should not race it. The bound is the longest a job may legitimately run: its default is GitHub's own limit on a job's execution time on a self-hosted runner, five days, which nothing can exceed, so a fleet whose jobs are shorter should lower it to a little above its longest job's `timeout-minutes`. Every deletion is logged on the Job's pod with the runner, the run, the repository and the reason, and each pass ends with a count of runners deleted, kept, idle and not readable. `github_runner_arc_runner_reaper_dry_run: true` logs what a pass would delete and deletes nothing. + +Its safety properties: + +- A runner with no recorded run has not been assigned a job, and an ephemeral runner never takes a second one, so an idle runner is never looked up or deleted, and neither is a runner already being deleted. A runner whose job has been assigned but has not started is kept. +- A runner is deleted only on a positive answer from GitHub. When the token cannot be minted (GitHub unreachable, the App Secret unusable or the permission missing) or the runners cannot be listed, the pass deletes nothing and its Job fails with the reason. When one runner's run or jobs cannot be read, that runner is left alone, the rest are still judged, and the Job fails at the end. A failed pass is not retried; the next scheduled one tries again. +- Its Role allows `list` and `delete` on `ephemeralrunners` in its own namespace and nothing else. The App Secret reaches the pod as a volume, the token never appears in a command's arguments or the log, and the pod runs as a non-root user with a read-only root filesystem. + +The reaper needs the org's GitHub App to have the **Actions** repository permission set to **Read**, approved on its installation, and the installation must include every repository whose jobs the runners take. The App the role's [setup flow](#github-app-setup) creates asks only for organisation self-hosted runners (write), so add `actions: read` to `github_runner_arc_app_setup_permissions` for a new App, or add the permission to an existing App in its settings and have an organisation owner approve it. Until then every pass fails saying the App needs the `actions: read` permission, and deletes nothing; a run in a repository the installation does not include is reported as unreadable, and its runner is left alone. + +The script (`files/reap-runners.sh`, with the token minting it shares with the pull Secret renewal) is mounted from a ConfigMap and runs in the renewal's image, which already has bash, curl, jq, kubectl and openssl, so a change to it reaches the cluster on the next run of the role without a new image. Its image is pulled without a pull Secret, so it must be public. `github_runner_arc_runner_reaper_enabled: false` removes the CronJob, its ConfigMap and its RBAC. + +ARC keeps the pod of a runner that GitHub still reports as running a job, even once its EphemeralRunner is being deleted, until GitHub releases the job. A runner whose run is completed is therefore removed straight away, while one whose job is still running past the bound is removed once GitHub ends that job, by its `timeout-minutes` or a cancellation. `github_runner_arc_runner_active_deadline_seconds` is a backstop that needs no GitHub call: it sets `activeDeadlineSeconds` on every runner pod, after which the kubelet stops the pod whatever it is doing and ARC handles it as a failed runner. It counts from when the pod starts, including any time the runner waits idle for a job, so set it above the longest a runner may wait plus the longest job the fleet runs. Unset, the runner pods are rendered without it. + ## GitHub App setup `playbooks/github_app_setup.yml` creates the App the scale sets authenticate as, through GitHub's manifest flow. The first run renders a local form that posts the manifest (organisation self-hosted runner write access, no webhook, private) to GitHub; an organisation owner submits it and copies the one-time code GitHub returns. The second run, with that code and a path for the key, exchanges the code, writes the private key there, waits while the App is installed on the organisation, and prints the `app_id` and `installation_id` to record. From 782611a2c29e38448cdc2ea4f87cf7edd7137c07 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Thu, 8 Oct 2026 14:35:23 +0100 Subject: [PATCH 8/8] test(arc): check the reaper reads only the assigned runners' runs by run id --- tests/runner_reaper/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/runner_reaper/run.sh b/tests/runner_reaper/run.sh index c4ffc0c..a8fb77e 100755 --- a/tests/runner_reaper/run.sh +++ b/tests/runner_reaper/run.sh @@ -156,7 +156,7 @@ grep -q "deleted ${namespace}/cancelled-runner: run 202 in example-org/example-r kubectl -n "$namespace" logs deploy/stub-github > "$work/stub.log" [ "$(jq -sc 'map(select(.method == "POST"))[0].body' "$work/stub.log")" = '{"permissions":{"actions":"read"}}' ] || fail "the mint did not ask for actions: read only" jq -se 'map(select(.method == "GET")) | all(.authorization == "Bearer ghs_stubreapertoken")' "$work/stub.log" >/dev/null || fail "a run was read without the minted token" -jq -se 'map(select(.method == "GET" and (.path | contains("/runs/")))) | length == 3' "$work/stub.log" >/dev/null || fail "the reaper looked up something other than the three assigned runners' runs" +jq -se '[.[] | select(.method == "GET") | .path | capture("/runs/(?[0-9]+)").id] | unique == ["201", "202", "203"]' "$work/stub.log" >/dev/null || fail "the reaper read runs other than those of the three assigned runners" log "Refusing the token: the Job should fail and delete nothing" create_runner second-hung-runner 201