From 2bc7dc30694bfe6b814ea36a1414bad26da873cd Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 19:45:43 +0100 Subject: [PATCH 01/15] feat(cluster): ship the platform-neutral regrade core and its tests The core steps a k3s server down through levels of reduced function under memory pressure, serialised across servers by a Lease and gated on the other etcd voters being Ready. It calls only the three functions of an actuator (levels, current_level, apply_level), so a Docker node and a NixOS node can drive the same decision logic. The files are the fleet image's core and tests unchanged; a .shellcheckrc names bash for them, since they carry no shebang. --- .shellcheckrc | 2 + galaxy.yml | 1 + .../files/regrade/fleet-regrade-core.sh | 347 ++++++++++++++++++ tests/regrade/fake-kubectl.sh | 55 +++ tests/regrade/regrade-core.sh | 207 +++++++++++ tests/regrade/regrade-lease.sh | 162 ++++++++ tests/regrade/testlib.sh | 34 ++ tests/unit/test_regrade_shell.py | 33 ++ 8 files changed, 841 insertions(+) create mode 100644 .shellcheckrc create mode 100644 roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh create mode 100755 tests/regrade/fake-kubectl.sh create mode 100644 tests/regrade/regrade-core.sh create mode 100644 tests/regrade/regrade-lease.sh create mode 100644 tests/regrade/testlib.sh create mode 100644 tests/unit/test_regrade_shell.py diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..0111886 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,2 @@ +# The scripts shared with other repositories are sourced or run as bash without a shebang line. +shell=bash diff --git a/galaxy.yml b/galaxy.yml index 119fb2a..4b5ac72 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -29,6 +29,7 @@ build_ignore: - .env* - .gitignore - .yamllint + - .shellcheckrc - AGENTS.md - CLAUDE.md - .ansible diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh new file mode 100644 index 0000000..7115eed --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh @@ -0,0 +1,347 @@ +# The platform-neutral core of fleet-regrade: it steps a k3s server down through levels of reduced function when memory runs short, and, by hand, back up. Nothing here knows how a level is entered; that is the actuator's job, so the same core drives a NixOS node (modules/fleet-regrade-nix.sh) and, later, a server that runs in Docker. It has no Nix store paths and no NixOS assumptions, so it can move to another repository unchanged. +# +# THE ACTUATOR CONTRACT. The file that is concatenated after this one, or sources it, defines exactly three functions and then calls `regrade_main "$@"`: +# +# levels prints the level names, one per line, from the most capable to the most reduced. The first is the level the node starts in and is restored to; a reboot returns to it. +# +# current_level prints the name of the level the node is running at now, read from the system itself, or "unknown" if it is none of them. +# +# apply_level NAME makes the node run at NAME, by whatever means the platform has, and returns non-zero if it could not. It is called with the node's workloads already drained when stepping down, and it is the actuator's business to restart whatever has to restart. +# +# The core owns the coordination between servers too: a step starts only while the node holds a Lease and every other etcd-voting node is Ready, so two servers never step at once and a step never starts while a peer is down. +# +# The core owns the signals (memory pressure and available memory), the decision (over the limit for long enough, one step at a time, a cooldown between steps), shedding and restoring workloads with kubectl, waiting for the API server, and the command line. The sustain period is the hysteresis: a reading under the limit ends a streak, so a node has to be over for the whole period, with nothing under it, before a step. +# +# It needs bash, awk, jq and kubectl. Configuration comes from the environment file named by FLEET_REGRADE_ENV; the paths below default to the real system and are overridden only by tests. + +FLEET_REGRADE_ENV=${FLEET_REGRADE_ENV:-/etc/fleet-regrade.env} +PSI_FILE=${PSI_FILE:-/proc/pressure/memory} +MEMINFO_FILE=${MEMINFO_FILE:-/proc/meminfo} +UPTIME_FILE=${UPTIME_FILE:-/proc/uptime} + +# Where the cluster keeps its own leases, and the name that identifies this protocol among them. +LEASE_NAMESPACE=${LEASE_NAMESPACE:-kube-system} +LEASE_NAME=${LEASE_NAME:-fleet-regrade} + +# The label k3s puts on the nodes that are etcd members, which are the nodes whose health a step must not stack on top of. +VOTER_LABEL=${VOTER_LABEL:-node-role.kubernetes.io/etcd=true} + +# shellcheck source=/dev/null +[ -f "$FLEET_REGRADE_ENV" ] && . "$FLEET_REGRADE_ENV" + +# Everything goes to stdout, which the service manager keeps as the log. +log() { + printf '%s\n' "$*" +} + +die() { + printf '%s\n' "$*" >&2 + exit 1 +} + +# The 300-second average of the share of time some task was stalled on memory, in percent. +psi_avg300() { + awk '$1 == "some" { for (i = 2; i <= NF; i++) if ($i ~ /^avg300=/) { sub(/^avg300=/, "", $i); print $i } }' "$PSI_FILE" +} + +meminfo_kb() { + awk -v key="$1:" '$1 == key { print $2 }' "$MEMINFO_FILE" +} + +uptime_seconds() { + awk '{ print int($1) }' "$UPTIME_FILE" +} + +# Succeeds when pressure is at or above the threshold and MemAvailable is below the floor, a share of MemTotal. Both are needed: pressure alone also rises on reclaim while plenty is available, and a low MemAvailable alone is normal once the page cache has filled. +is_over() { + local pressure=$1 available_kb=$2 total_kb=$3 + awk -v p="$pressure" -v t="$PRESSURE_THRESHOLD" -v a="$available_kb" -v m="$total_kb" -v f="$AVAILABLE_FLOOR_PERCENT" \ + 'BEGIN { exit !(p + 0 >= t + 0 && a * 100 < m * f) }' +} + +# Succeeds when the reading has been over for the sustain period and the cooldown since the last step has passed. above_since and last_step are uptime seconds, empty when unset. +may_step() { + local now=$1 above_since=$2 last_step=$3 + [ -n "$above_since" ] || return 1 + [ $((now - above_since)) -ge "$SUSTAIN_SECONDS" ] || return 1 + [ -z "$last_step" ] || [ $((now - last_step)) -ge "$COOLDOWN_SECONDS" ] +} + +# The actuator's levels as an array, most capable first. +load_levels() { + local level + LEVELS=() + while IFS= read -r level; do + [ -z "$level" ] || LEVELS+=("$level") + done < <(levels) + [ "${#LEVELS[@]}" -gt 0 ] || die "The actuator reports no levels." +} + +level_position() { + local i + for i in "${!LEVELS[@]}"; do + if [ "${LEVELS[$i]}" = "$1" ]; then + echo "$i" + return 0 + fi + done + return 1 +} + +next_level_down() { + local position + position=$(level_position "$1") || return 1 + [ $((position + 1)) -lt "${#LEVELS[@]}" ] || return 1 + echo "${LEVELS[$((position + 1))]}" +} + +next_level_up() { + local position + position=$(level_position "$1") || return 1 + [ "$position" -gt 0 ] || return 1 + echo "${LEVELS[$((position - 1))]}" +} + +# kubectl against the cluster, as an administrator. KUBECTL may be more than one word (`k3s kubectl`). KUBE_SERVER, when set, names the API server to use instead of the one in the kubeconfig: a node that has switched its own API server off still has to reach the cluster's. +kubectl_() { + # shellcheck disable=SC2086 # KUBECTL is a command that may take a subcommand, so it is split on purpose. + $KUBECTL --kubeconfig "$KUBECONFIG_PATH" ${KUBE_SERVER:+--server "$KUBE_SERVER"} "$@" +} + +# Taints, cordons and evicts everything but daemonsets, while the API server is still up. Both steps are idempotent, so each step down runs it again before anything is switched off. The taint is applied here and not left to the node's own configuration, which takes effect only when a node registers. --force evicts pods that no controller owns too: the node is shedding load, and a pod that blocks the drain would stop the step. +shed_workloads() { + kubectl_ taint nodes "$NODE_NAME" "$CONTROL_PLANE_TAINT" --overwrite + kubectl_ drain "$NODE_NAME" --ignore-daemonsets --delete-emptydir-data --force --timeout="${OPERATION_TIMEOUT_SECONDS}s" +} + +restore_workloads() { + kubectl_ taint nodes "$NODE_NAME" "${CONTROL_PLANE_TAINT%%=*}:NoSchedule-" || true + kubectl_ uncordon "$NODE_NAME" +} + +wait_for_api() { + local deadline=$((SECONDS + OPERATION_TIMEOUT_SECONDS)) + until kubectl_ get --raw=/readyz > /dev/null 2>&1; do + [ "$SECONDS" -lt "$deadline" ] || die "The API server did not become ready within ${OPERATION_TIMEOUT_SECONDS}s." + sleep 2 + done +} + +# The time as epoch seconds, replaced by tests. +now_epoch() { + date +%s +} + +# A Lease timestamp (RFC 3339 with microseconds) for an epoch time, built by jq so that it is the same on every platform. +micro_time() { + jq -nr --argjson t "$1" '$t | todate | sub("Z$"; ".000000Z")' +} + +# The Lease as the API server stores it: held by $1, acquired at epoch $2, last renewed at epoch $3, for LEASE_DURATION_SECONDS. A fourth argument is the resourceVersion of the Lease being replaced, which makes the replacement fail if anyone else has changed it since it was read. +build_lease() { + jq -n --arg name "$LEASE_NAME" --arg namespace "$LEASE_NAMESPACE" --arg holder "$1" \ + --arg acquired "$(micro_time "$2")" --arg renewed "$(micro_time "$3")" \ + --argjson duration "$LEASE_DURATION_SECONDS" --arg version "${4:-}" ' + { + apiVersion: "coordination.k8s.io/v1", + kind: "Lease", + metadata: ({ name: $name, namespace: $namespace } + (if $version == "" then {} else { resourceVersion: $version } end)), + spec: { holderIdentity: $holder, leaseDurationSeconds: $duration, acquireTime: $acquired, renewTime: $renewed } + }' +} + +# What a Lease, given as JSON, means to node $2 at epoch time $3: free (nobody holds it), expired (a holder whose time has run out), mine (unexpired and held by $2) or held (unexpired and held by another). A Lease lasts until its renew time plus its duration, and is expired from that moment. +lease_state() { + jq -r --arg me "$2" --argjson now "$3" ' + (.spec.holderIdentity // "") as $holder + | (.spec.renewTime // .spec.acquireTime // null) as $renewed + | (.spec.leaseDurationSeconds // 0) as $duration + | if $holder == "" then "free" + elif $renewed == null then "expired" + elif (($renewed | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) + $duration) <= $now then "expired" + elif $holder == $me then "mine" + else "held" end' <<< "$1" +} + +# Takes the Lease, or fails. Created when there is none, replaced when it is free, expired or already this node's, refused when another holds it, and refused when the API server cannot be asked, since a step must not start on a guess. Both writes fail if someone else got there first. +acquire_lease() { + : "${LEASE_DURATION_SECONDS:?LEASE_DURATION_SECONDS must be set}" + local current now state holder version + now=$(now_epoch) + if ! current=$(kubectl_ get lease "$LEASE_NAME" --namespace "$LEASE_NAMESPACE" --output json 2>&1); then + case $current in + *"(NotFound)"*) + if build_lease "$NODE_NAME" "$now" "$now" | kubectl_ create -f - > /dev/null; then + return 0 + fi + log "cannot step: another node created the lease first" + return 1 + ;; + *) + log "cannot step: the lease cannot be read: $current" + return 1 + ;; + esac + fi + state=$(lease_state "$current" "$NODE_NAME" "$now") + if [ "$state" = held ]; then + holder=$(jq -r '.spec.holderIdentity' <<< "$current") + log "cannot step: the lease is held by $holder" + return 1 + fi + version=$(jq -r '.metadata.resourceVersion' <<< "$current") + if build_lease "$NODE_NAME" "$now" "$now" "$version" | kubectl_ replace -f - > /dev/null; then + return 0 + fi + log "cannot step: the lease changed while it was being taken" + return 1 +} + +# Gives the Lease back, if it is still this node's: the holder is cleared, so the next node does not wait for it to expire. If it has meanwhile passed to another node it is left alone. +release_lease() { + local current + current=$(kubectl_ get lease "$LEASE_NAME" --namespace "$LEASE_NAMESPACE" --output json) || return 1 + [ "$(jq -r '.spec.holderIdentity // ""' <<< "$current")" = "$NODE_NAME" ] || return 0 + jq '.spec.holderIdentity = ""' <<< "$current" | kubectl_ replace -f - > /dev/null +} + +# Succeeds when every other etcd-voting node is Ready. A node whose state is unknown, that has no Ready condition, or that cannot be listed counts as not ready: a step never starts while a peer might be down. +peers_ready() { + local nodes not_ready + if ! nodes=$(kubectl_ get nodes --selector "$VOTER_LABEL" --output json); then + log "cannot step: the etcd-voting nodes cannot be listed" + return 1 + fi + not_ready=$(jq -r --arg me "$NODE_NAME" ' + [ .items[] | select(.metadata.name != $me) + | select(([.status.conditions[]? | select(.type == "Ready") | .status] | first // "Unknown") != "True") + | .metadata.name ] | join(" ")' <<< "$nodes") || return 1 + if [ -n "$not_ready" ]; then + log "cannot step: etcd-voting nodes not ready: $not_ready" + return 1 + fi +} + +# Runs a command only while this node may transition: its peers are Ready and it holds the Lease. The Lease is released afterwards whether or not the command succeeded, and the command's status is the result. +coordinated() { + local status=0 + peers_ready || return 1 + acquire_lease || return 1 + "$@" || status=$? + release_lease || log "the lease could not be released; it expires on its own" + return "$status" +} + +# One step down. Workloads are drained before the level is applied, and a failed drain stops the step, so nothing that needs the API server is switched off with pods still on the node. +step_down() { + local from=$1 to=$2 reading=$3 + log "step down $from -> $to: $reading" + if ! shed_workloads; then + log "drain failed, staying at $from" + return 1 + fi + if ! apply_level "$to"; then + log "applying $to failed, level is now $(current_level)" + return 1 + fi + log "now at $to" +} + +# One step up, by hand. Returning to the first level also lifts the taint and uncordons the node, once the API server answers. +step_up() { + local from=$1 to=$2 + log "step up $from -> $to" + if ! apply_level "$to"; then + log "applying $to failed, level is now $(current_level)" + return 1 + fi + if [ "$to" = "${LEVELS[0]}" ]; then + wait_for_api + restore_workloads + fi + log "now at $to" +} + +reading_text() { + printf 'memory some avg300=%s%% (threshold %s%%), MemAvailable %s kB of %s kB (floor %s%%)' \ + "$1" "$PRESSURE_THRESHOLD" "$2" "$3" "$AVAILABLE_FLOOR_PERCENT" +} + +# One look at the signals. The streak and the time of the last step are kept in WATCH_ABOVE_SINCE and WATCH_LAST_STEP, which a loop (or a test) carries from one sample to the next. +WATCH_ABOVE_SINCE='' +WATCH_LAST_STEP='' +WATCH_WARNED='' +watch_sample() { + local level next pressure available total now text + level=$(current_level) + if [ "$level" = unknown ]; then + [ -n "$WATCH_WARNED" ] || log "the running system is not one of the levels; not stepping" + WATCH_WARNED=1 + return 0 + fi + WATCH_WARNED='' + pressure=$(psi_avg300) + available=$(meminfo_kb MemAvailable) + total=$(meminfo_kb MemTotal) + now=$(uptime_seconds) + text=$(reading_text "$pressure" "$available" "$total") + if is_over "$pressure" "$available" "$total"; then + if [ -z "$WATCH_ABOVE_SINCE" ]; then + WATCH_ABOVE_SINCE=$now + log "level $level: over the limit: $text" + fi + elif [ -n "$WATCH_ABOVE_SINCE" ]; then + WATCH_ABOVE_SINCE='' + log "level $level: back under the limit: $text" + fi + if next=$(next_level_down "$level") && may_step "$now" "$WATCH_ABOVE_SINCE" "$WATCH_LAST_STEP"; then + coordinated step_down "$level" "$next" "$text" || true + WATCH_LAST_STEP=$(uptime_seconds) + WATCH_ABOVE_SINCE='' + fi +} + +cmd_watch() { + load_levels + while true; do + sleep "$SAMPLE_SECONDS" + watch_sample + done +} + +cmd_status() { + load_levels + log "level: $(current_level) (of: ${LEVELS[*]})" + log "$(reading_text "$(psi_avg300)" "$(meminfo_kb MemAvailable)" "$(meminfo_kb MemTotal)")" + log "sample ${SAMPLE_SECONDS}s, sustained for ${SUSTAIN_SECONDS}s, cooldown ${COOLDOWN_SECONDS}s" +} + +# One step down by hand, for trying a level without waiting for pressure. +cmd_down() { + local level next + load_levels + level=$(current_level) + [ "$level" != unknown ] || die "The running system is not one of the levels." + next=$(next_level_down "$level") || die "Already at $level, the most reduced level." + coordinated step_down "$level" "$next" "by hand" +} + +# One step up by hand: the only way back up. A reboot also returns to the first level. +cmd_up() { + local level next + load_levels + level=$(current_level) + [ "$level" != unknown ] || die "The running system is not one of the levels." + next=$(next_level_up "$level") || die "Already at $level, the first level." + coordinated step_up "$level" "$next" +} + +regrade_main() { + case ${1:-} in + watch) cmd_watch ;; + status) cmd_status ;; + up) cmd_up ;; + down) cmd_down ;; + *) die "Usage: fleet-regrade watch|status|up|down" ;; + esac +} diff --git a/tests/regrade/fake-kubectl.sh b/tests/regrade/fake-kubectl.sh new file mode 100755 index 0000000..327f996 --- /dev/null +++ b/tests/regrade/fake-kubectl.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# A stand-in for kubectl for the regrade tests, keeping a cluster in the directory FAKE_DIR: lease.json (the Lease, absent when there is none) and nodes.json (what listing the etcd-voting nodes returns). It understands only what the core asks of it, answers as the real one does for the cases that matter (a Lease that is not found, one that already exists, a replacement of a Lease that has changed), and records every call in FAKE_DIR/calls. The FAKE_* variables make a call fail: FAKE_LEASE_READ_ERROR, FAKE_NODES_ERROR, FAKE_CREATE_RACE, FAKE_REPLACE_CONFLICT. +set -u + +dir=${FAKE_DIR:?FAKE_DIR must be set} +args=() +while [ $# -gt 0 ]; do + case $1 in + --kubeconfig | --server) shift 2 ;; + *) args+=("$1"); shift ;; + esac +done +echo "${args[*]}" >> "$dir/calls" + +lease="$dir/lease.json" +case "${args[0]:-} ${args[1]:-}" in + "get lease") + if [ -n "${FAKE_LEASE_READ_ERROR:-}" ]; then + echo "Unable to connect to the server: dial tcp: i/o timeout" >&2 + exit 1 + fi + if [ -f "$lease" ]; then + cat "$lease" + else + echo 'Error from server (NotFound): leases.coordination.k8s.io "fleet-regrade" not found' >&2 + exit 1 + fi + ;; + "create -f") + body=$(cat) + if [ -f "$lease" ] || [ -n "${FAKE_CREATE_RACE:-}" ]; then + echo 'Error from server (AlreadyExists): leases.coordination.k8s.io "fleet-regrade" already exists' >&2 + exit 1 + fi + jq '.metadata.resourceVersion = "1"' <<< "$body" > "$lease" + ;; + "replace -f") + body=$(cat) + current=$(jq -r '.metadata.resourceVersion' "$lease" 2> /dev/null || echo none) + sent=$(jq -r '.metadata.resourceVersion // "none"' <<< "$body") + if [ -n "${FAKE_REPLACE_CONFLICT:-}" ] || [ "$sent" != "$current" ]; then + echo 'Error from server (Conflict): Operation cannot be fulfilled on leases.coordination.k8s.io "fleet-regrade": the object has been modified' >&2 + exit 1 + fi + jq --arg v "$((current + 1))" '.metadata.resourceVersion = $v' <<< "$body" > "$lease" + ;; + "get nodes") + if [ -n "${FAKE_NODES_ERROR:-}" ]; then + echo "Unable to connect to the server" >&2 + exit 1 + fi + cat "$dir/nodes.json" + ;; + *) ;; +esac diff --git a/tests/regrade/regrade-core.sh b/tests/regrade/regrade-core.sh new file mode 100644 index 0000000..e5d93eb --- /dev/null +++ b/tests/regrade/regrade-core.sh @@ -0,0 +1,207 @@ +# Tests for the platform-neutral core of fleet-regrade (modules/fleet-regrade-core.sh): the signals, the decision, the order of a step, and that the core decides the same whatever actuator it drives. Run as `bash tests/regrade-core.sh `; the flake's checks do. The actuator is a fake in these tests (several, in fact), every path the core reads is a fixture, and the two functions that act on a cluster are recorders, so nothing here touches a cluster or the running system. +# The core calls the functions defined here (the fake actuator and the recorders) by name, and reads the variables set here, so shellcheck sees them as unused. +# shellcheck disable=SC2329,SC2034 +set -euo pipefail + +core=${1:?usage: regrade-core.sh } +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +cat > "$work/env" < "$FAKE_DIR/nodes.json" + +export FLEET_REGRADE_ENV=$work/env PSI_FILE=$work/psi MEMINFO_FILE=$work/meminfo UPTIME_FILE=$work/uptime + +# shellcheck source=/dev/null +. "$(dirname "$0")/testlib.sh" +# shellcheck source=/dev/null +. "$core" +now_epoch() { echo 1000000000; } + +total=1000000 # kB; the floor is 15 percent of this, 150000 kB + +printf 'some avg10=0.00 avg60=0.00 avg300=12.50 total=1\nfull avg10=0.00 avg60=0.00 avg300=3.00 total=1\n' > "$work/psi" +printf 'MemTotal: %s kB\nMemAvailable: %s kB\n' "$total" 100000 > "$work/meminfo" +check "reads the some avg300 line, not full" test "$(psi_avg300)" = 12.50 +check "reads MemAvailable" test "$(meminfo_kb MemAvailable)" = 100000 + +check "over: pressure at threshold and memory below the floor" is_over 10 100000 "$total" +check_not "not over: pressure just under the threshold" is_over 9.99 100000 "$total" +check_not "not over: pressure high but MemAvailable above the floor" is_over 50 150000 "$total" +check "over: MemAvailable just under the floor" is_over 50 149999 "$total" + +check_not "no step with no streak" may_step 1000 '' '' +check_not "no step before the sustain period" may_step 1119 1000 '' +check "step once sustained, with no earlier step" may_step 1120 1000 '' +check_not "no step inside the cooldown" may_step 2000 1000 1200 +check "step once the cooldown has passed" may_step 2100 1000 1200 + +# Three fake actuators that satisfy the contract in different ways: A keeps its level in a file under the usual names, B keeps it in memory under other names, C has only two levels. Each records what it was asked to apply. +applied="$work/applied" +actuator_a() { + echo full > "$work/level-a" + levels() { printf '%s\n' full control-plane etcd-only; } + current_level() { cat "$work/level-a"; } + apply_level() { echo "$1" > "$work/level-a"; echo "$1" >> "$applied"; } +} +actuator_b() { + B_LEVEL=alpha + levels() { printf '%s\n' alpha beta gamma; } + current_level() { echo "$B_LEVEL"; } + apply_level() { B_LEVEL=$1; echo "$1" >> "$applied"; } +} +actuator_c() { + C_LEVEL=one + levels() { printf '%s\n' one two; } + current_level() { echo "$C_LEVEL"; } + apply_level() { C_LEVEL=$1; echo "$1" >> "$applied"; } +} + +# Workloads and the API server are the cluster side of the core, replaced by recorders. +calls=() +shed_workloads() { calls+=(shed); } +restore_workloads() { calls+=(restore); } +wait_for_api() { calls+=(wait); } + +actuator_a +load_levels +check "the levels are the actuator's, in order" test "${LEVELS[*]}" = "full control-plane etcd-only" +check "the first step down" test "$(next_level_down full)" = control-plane +check "the second step down" test "$(next_level_down control-plane)" = etcd-only +check_not "nowhere to go from the last level" next_level_down etcd-only +check "the first step up" test "$(next_level_up etcd-only)" = control-plane +check "the second step up" test "$(next_level_up control-plane)" = full +check_not "nowhere to go from the first level" next_level_up full +check_not "a level the actuator does not have has no next" next_level_down unheard-of +actuator_b +load_levels +check "another actuator's levels are used as they are" test "$(next_level_down alpha)" = beta + +# Order of a step: workloads are shed before anything is applied, and a failed drain stops the step. +actuator_a +: > "$applied" +calls=() +# The fake actuator records into the same list as the cluster recorders, to show the order of a step. +apply_level() { calls+=("apply:$1"); } +step_down full control-plane "test reading" > /dev/null +check "a step sheds, then applies" test "${calls[*]}" = "shed apply:control-plane" + +calls=() +step_down control-plane etcd-only "test reading" > /dev/null +check "the step that switches the API server off sheds again first" test "${calls[*]}" = "shed apply:etcd-only" + +calls=() +shed_workloads() { calls+=(shed-failed); return 1; } +failing_step() { step_down control-plane etcd-only "test reading" > /dev/null; } +check_not "a failed drain fails the step" failing_step +check "a failed drain applies nothing" test "${calls[*]}" = "shed-failed" +shed_workloads() { calls+=(shed); } + +failing_apply() { step_down full control-plane "test reading" > /dev/null; } +apply_level() { calls+=("apply-failed:$1"); return 1; } +calls=() +check_not "a level that cannot be applied fails the step" failing_apply + +# Stepping up applies the level above and restores the node's workloads only on reaching the first level. +actuator_a +: > "$applied" +calls=() +echo etcd-only > "$work/level-a" +cmd_up > /dev/null +check "up from the last level applies the level above" test "$(cat "$applied")" = control-plane +check "up to a middle level does not restore workloads" test "${calls[*]}" = "" +cmd_up > /dev/null +check "up to the first level waits for the API and restores workloads" test "${calls[*]}" = "wait restore" +up_at_top() { (cmd_up > /dev/null 2>&1); } +check_not "up at the first level is refused" up_at_top +: > "$applied" +calls=() +cmd_down > /dev/null +check "down by hand sheds, then applies the level below" test "${calls[*]}-$(cat "$applied")" = "shed-control-plane" +cmd_down > /dev/null +down_at_bottom() { (cmd_down > /dev/null 2>&1); } +check_not "down at the last level is refused" down_at_bottom +unknown_level() { (current_level() { echo unknown; }; cmd_down > /dev/null 2>&1); } +check_not "down from an unknown level is refused" unknown_level + +# The core decides the same whatever it drives. The same readings are fed to each actuator, one sample every SAMPLE_SECONDS: one under the limit, then over it. Over for the sustain period gives the first step; the second needs both a fresh streak and the cooldown since the first; and there is nothing after the last level. The expected steps are worked out by hand from the settings above: the streak starts at the second sample (60 s), the first step is at 180 s, a new streak starts at 210 s, and 1080 s is the first moment 900 s have passed since the first step. +run_scenario() { + local sample uptime + : > "$applied" + calls=() + WATCH_ABOVE_SINCE='' + WATCH_LAST_STEP='' + WATCH_WARNED='' + for sample in $(seq 1 60); do + uptime=$((sample * 30)) + printf '%s 0.1\n' "$uptime" > "$UPTIME_FILE" + if [ "$sample" -eq 1 ]; then + printf 'some avg10=0 avg60=0 avg300=2.00 total=1\n' > "$PSI_FILE" + printf 'MemTotal: %s kB\nMemAvailable: 900000 kB\n' "$total" > "$MEMINFO_FILE" + else + printf 'some avg10=0 avg60=0 avg300=20.00 total=1\n' > "$PSI_FILE" + printf 'MemTotal: %s kB\nMemAvailable: 90000 kB\n' "$total" > "$MEMINFO_FILE" + fi + before=$(wc -l < "$applied" | tr -d ' ') + watch_sample > /dev/null + after=$(wc -l < "$applied" | tr -d ' ') + if [ "$after" -gt "$before" ]; then + printf '%s:%s ' "$sample" "$(position_of_last_applied)" + fi + done +} +position_of_last_applied() { + level_position "$(tail -n 1 "$applied")" +} +shed_workloads() { calls+=(shed); } + +actuator_a; load_levels; steps_a=$(run_scenario) +actuator_b; load_levels; steps_b=$(run_scenario) +actuator_c; load_levels; steps_c=$(run_scenario) +check "actuator A steps at the worked-out moments" test "$steps_a" = "6:1 36:2 " +check "actuator B, with other names and another way of keeping its level, decides identically" test "$steps_b" = "$steps_a" +check "actuator C, with only two levels, steps once at the same moment and then has nowhere to go" test "$steps_c" = "6:1 " + +# A step ends the streak that caused it: the next step needs a fresh sustained period of its own, not only the cooldown. That shows only when the sustain period is longer than the cooldown, so the settings are changed for this run: the streak starts at 60 s, the first step is at 660 s, a new streak starts at 690 s and the second step is at 1290 s, where without the reset it would come at 870 s. +SUSTAIN_SECONDS=600 +COOLDOWN_SECONDS=200 +actuator_a; load_levels; steps_long_sustain=$(run_scenario) +SUSTAIN_SECONDS=120 +COOLDOWN_SECONDS=900 +check "a step starts a fresh streak" test "$steps_long_sustain" = "22:1 43:2 " + +# The coordination reaches the watch loop: with another etcd-voting node down, the same readings that stepped before lead to no step at all, and with the lease held by another node likewise. +printf '{"items":[{"metadata":{"name":"peer-a"},"status":{"conditions":[{"type":"Ready","status":"False"}]}},{"metadata":{"name":"testnode"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}\n' > "$FAKE_DIR/nodes.json" +actuator_a; load_levels; steps_peer_down=$(run_scenario) +check "a peer down: the watch loop never steps" test "$steps_peer_down" = "" +printf '{"items":[{"metadata":{"name":"peer-a"},"status":{"conditions":[{"type":"Ready","status":"True"}]}},{"metadata":{"name":"testnode"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}\n' > "$FAKE_DIR/nodes.json" +build_lease other-node 1000000000 1000000000 3 > "$FAKE_DIR/lease.json" +actuator_a; load_levels; steps_lease_held=$(run_scenario) +check "the lease held elsewhere: the watch loop never steps" test "$steps_lease_held" = "" +rm -f "$FAKE_DIR/lease.json" + +# A system that is none of the levels is left alone. +actuator_a +echo unknown > "$work/level-a" +: > "$applied" +watch_sample > /dev/null +check "an unknown level is not stepped" test ! -s "$applied" + +finish diff --git a/tests/regrade/regrade-lease.sh b/tests/regrade/regrade-lease.sh new file mode 100644 index 0000000..fead934 --- /dev/null +++ b/tests/regrade/regrade-lease.sh @@ -0,0 +1,162 @@ +# Tests for the cluster-wide coordination in the core of fleet-regrade (modules/fleet-regrade-core.sh): a transition starts only while the node holds the Lease and the other etcd-voting nodes are Ready. Run as `bash tests/regrade-lease.sh `; the flake's checks do. kubectl is a fake that keeps a cluster in a directory (tests/fake-kubectl.sh), and time is a variable, so nothing here touches a cluster. +# The core calls the functions defined here by name, and reads the variables set here, so shellcheck sees them as unused. +# shellcheck disable=SC2329,SC2034 +set -euo pipefail + +core=${1:?usage: regrade-lease.sh } +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +cat > "$work/env" < "$FAKE_DIR/nodes.json" +} +holder() { jq -r '.spec.holderIdentity' "$lease"; } +# Writes a Lease held by $1 and last renewed at epoch $2, with the configured duration. +write_lease() { + build_lease "$1" "$2" "$2" 7 > "$lease" +} +acquires() { acquire_lease > /dev/null 2>&1; } + +# The Lease. +reset_cluster +check "free: taken when there is no lease" acquires +check "the lease is created in the name of this node" test "$(holder)" = testnode +check "the lease carries the configured duration" test "$(jq -r '.spec.leaseDurationSeconds' "$lease")" = 600 +check "the lease is renewed as of now" test "$(jq -r '.spec.renewTime' "$lease")" = "$(micro_time "$NOW")" + +reset_cluster +write_lease other-node "$((NOW - 100))" +check_not "held: refused when another holds it unexpired" acquires +check "a refusal leaves the holder alone" test "$(holder)" = other-node + +reset_cluster +write_lease other-node "$((NOW - 599))" +check_not "held: still refused one second before it expires" acquires + +reset_cluster +write_lease other-node "$((NOW - 600))" +check "expired: taken over at the moment it expires" acquires +check "the takeover is in the name of this node" test "$(holder)" = testnode + +reset_cluster +write_lease other-node "$((NOW - 100000))" +check "expired: taken over long after" acquires + +reset_cluster +write_lease testnode "$((NOW - 100))" +check "mine: taken again (a step that was cut short)" acquires + +reset_cluster +write_lease '' "$((NOW - 10))" +check "released: taken when nobody holds it" acquires +check "the release is replaced, not created" test "$(jq -r '.metadata.resourceVersion' "$lease")" = 8 + +reset_cluster +export FAKE_CREATE_RACE=1 +check_not "another node creating it first refuses the step" acquires + +reset_cluster +write_lease other-node "$((NOW - 1000))" +export FAKE_REPLACE_CONFLICT=1 +check_not "a lease changed while it was being taken refuses the step" acquires +check "and is left as the other node had it" test "$(holder)" = other-node + +reset_cluster +export FAKE_LEASE_READ_ERROR=1 +check_not "a lease that cannot be read refuses the step" acquires +check "and nothing is written" test ! -e "$lease" +writes_attempted() { grep -qE "^(create|replace) " "$FAKE_DIR/calls"; } +check_not "and no write is even attempted" writes_attempted + +# Releasing. +reset_cluster +acquire_lease > /dev/null +release_lease +check "release clears the holder" test "$(holder)" = "" +check "and the next node can take it at once" acquires + +reset_cluster +write_lease other-node "$((NOW - 10))" +release_lease +check "release leaves a lease that has passed to another node alone" test "$(holder)" = other-node + +# The other etcd-voting nodes. +reset_cluster +check "peers: all Ready" peers_ready +not_ready_peer() { (peers_ready > /dev/null 2>&1); } +nodes_json False True +check_not "peers: one NotReady refuses" not_ready_peer +nodes_json True Unknown +check_not "peers: one Unknown refuses" not_ready_peer +printf '{"items":[{"metadata":{"name":"peer-a"},"status":{"conditions":[]}},{"metadata":{"name":"testnode"},"status":{"conditions":[{"type":"Ready","status":"True"}]}}]}\n' > "$FAKE_DIR/nodes.json" +check_not "peers: no Ready condition at all refuses" not_ready_peer +printf '{"items":[{"metadata":{"name":"testnode"},"status":{"conditions":[{"type":"Ready","status":"False"}]}}]}\n' > "$FAKE_DIR/nodes.json" +check "peers: this node's own state is not a peer's" peers_ready +reset_cluster +export FAKE_NODES_ERROR=1 +check_not "peers: a list that cannot be read refuses" not_ready_peer +reset_cluster +peers_ready +check "peers: only etcd-voting nodes are asked about" grep -qF -- "--selector node-role.kubernetes.io/etcd=true" "$FAKE_DIR/calls" + +# Coordinated: the peers first, then the lease, then the command, then the release. +ran='' +step_recorder() { ran="$ran step"; } +failing_step() { ran="$ran failing"; return 3; } +run_coordinated() { ran=''; coordinated "$@" > /dev/null 2>&1; } + +reset_cluster +run_coordinated step_recorder +check "all clear: the command runs" test "$ran" = " step" +check "and the lease is given back" test "$(holder)" = "" + +reset_cluster +status=0 +run_coordinated failing_step || status=$? +check "a failing command runs" test "$ran" = " failing" +check "its status is the result" test "$status" = 3 +check "and the lease is still given back" test "$(holder)" = "" + +reset_cluster +nodes_json False True +run_coordinated step_recorder || true +check "a peer down: the command does not run" test "$ran" = "" +check "a peer down: no lease is taken" test ! -e "$lease" + +reset_cluster +write_lease other-node "$((NOW - 10))" +run_coordinated step_recorder || true +check "lease held elsewhere: the command does not run" test "$ran" = "" +check "lease held elsewhere: it is left as it was" test "$(holder)" = other-node + +# Two nodes never step at once: while one holds the lease, a second asking as another node is refused. +reset_cluster +acquire_lease > /dev/null +second_node() { (NODE_NAME=second-node; acquire_lease > /dev/null 2>&1); } +check_not "a second node is refused while the first holds the lease" second_node + +finish diff --git a/tests/regrade/testlib.sh b/tests/regrade/testlib.sh new file mode 100644 index 0000000..9f83dfd --- /dev/null +++ b/tests/regrade/testlib.sh @@ -0,0 +1,34 @@ +# Shared by the regrade tests: a counter of failures and the two assertions every test uses. Sourced, never run. + +failures=0 + +check() { + local name=$1 + shift + if "$@"; then + echo "ok $name" + else + echo "FAIL $name" + failures=$((failures + 1)) + fi +} + +check_not() { + local name=$1 + shift + if "$@"; then + echo "FAIL $name" + failures=$((failures + 1)) + else + echo "ok $name" + fi +} + +# Ends the test: a non-zero status if any assertion failed. +finish() { + if [ "$failures" -ne 0 ]; then + echo "$failures failed" + exit 1 + fi + echo "all passed" +} diff --git a/tests/unit/test_regrade_shell.py b/tests/unit/test_regrade_shell.py new file mode 100644 index 0000000..110716b --- /dev/null +++ b/tests/unit/test_regrade_shell.py @@ -0,0 +1,33 @@ +"""Runs the shell tests of fleet-regrade (tests/regrade) so that the unit-test job covers them: each script prints one line per assertion and exits non-zero when any failed.""" + +from __future__ import annotations + +import subprocess +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +FILES = ROOT / "roles" / "github_runner_cluster" / "files" / "regrade" +TESTS = ROOT / "tests" / "regrade" +CORE = FILES / "fleet-regrade-core.sh" + + +def run_script(script: str, *args: Path) -> subprocess.CompletedProcess[str]: + """Run one test script under bash from its own directory, which the scripts resolve their helpers from.""" + return subprocess.run(["bash", str(TESTS / script), *map(str, args)], capture_output=True, text=True, check=False, cwd=TESTS) + + +class CoreTest(unittest.TestCase): + def test_signals_decision_and_order_of_a_step(self) -> None: + result = run_script("regrade-core.sh", CORE) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("all passed", result.stdout) + + def test_lease_and_peer_coordination(self) -> None: + result = run_script("regrade-lease.sh", CORE) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("all passed", result.stdout) + + +if __name__ == "__main__": + unittest.main() From 932fcca5b1a4188bb614e131d34e88d05aed98e6 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 20:53:32 +0100 Subject: [PATCH 02/15] feat(cluster): let the regrade core step a server back up by itself With STEP_UP=automatic the watcher goes back up one level at a time once memory has been calm for the sustain period and the cooldown since the last step, in either direction, has passed. Calm is stricter than not being over the limit (pressure under the threshold and MemAvailable at or above the floor, both), so a reading between the two ends both streaks and a server does not step both ways on the same noise. A step up takes the same Lease and peer check as a step down, since it restarts etcd at the first level that leaves etcd-only. STEP_UP defaults to manual, which is what the core did before the setting existed, so a consumer that sets nothing is unchanged. --- .../files/regrade/fleet-regrade-core.sh | 47 +++-- tests/regrade/regrade-recover.sh | 180 ++++++++++++++++++ 2 files changed, 216 insertions(+), 11 deletions(-) create mode 100644 tests/regrade/regrade-recover.sh diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh index 7115eed..2257c47 100644 --- a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh @@ -1,4 +1,4 @@ -# The platform-neutral core of fleet-regrade: it steps a k3s server down through levels of reduced function when memory runs short, and, by hand, back up. Nothing here knows how a level is entered; that is the actuator's job, so the same core drives a NixOS node (modules/fleet-regrade-nix.sh) and, later, a server that runs in Docker. It has no Nix store paths and no NixOS assumptions, so it can move to another repository unchanged. +# The platform-neutral core of fleet-regrade: it steps a k3s server down through levels of reduced function when memory runs short, and back up: by hand, or by itself once memory has been calm for long enough (STEP_UP, below). Nothing here knows how a level is entered; that is the actuator's job, so the same core drives a NixOS node (modules/fleet-regrade-nix.sh) and, later, a server that runs in Docker. It has no Nix store paths and no NixOS assumptions, so it can move to another repository unchanged. # # THE ACTUATOR CONTRACT. The file that is concatenated after this one, or sources it, defines exactly three functions and then calls `regrade_main "$@"`: # @@ -10,7 +10,9 @@ # # The core owns the coordination between servers too: a step starts only while the node holds a Lease and every other etcd-voting node is Ready, so two servers never step at once and a step never starts while a peer is down. # -# The core owns the signals (memory pressure and available memory), the decision (over the limit for long enough, one step at a time, a cooldown between steps), shedding and restoring workloads with kubectl, waiting for the API server, and the command line. The sustain period is the hysteresis: a reading under the limit ends a streak, so a node has to be over for the whole period, with nothing under it, before a step. +# The core owns the signals (memory pressure and available memory), the decision (over the limit for long enough, one step at a time, a cooldown between steps, and, when STEP_UP is automatic, calm for long enough to go back up one level), shedding and restoring workloads with kubectl, waiting for the API server, and the command line. The sustain period is the hysteresis: a reading under the limit ends a streak, so a node has to be over for the whole period, with nothing under it, before a step down, and calm for the whole period, with nothing over it, before a step up. Calm is stricter than not being over: pressure under the threshold and MemAvailable at or above the floor, both, where over needs both to be bad, so a reading between the two states ends both streaks and a node does not step down and up on the same noise. The cooldown after a step, in either direction, is longer than the pressure window, so a node that cannot hold a higher level cycles no faster than once per cooldown and sustain period. +# +# STEP_UP is optional: `automatic` lets the watcher step up one level at a time, `manual` (the default when it is unset, which is what the core did before it existed) leaves stepping up to `fleet-regrade up`. Giving up an etcd vote is not a level, and nothing here ever does it. # # It needs bash, awk, jq and kubectl. Configuration comes from the environment file named by FLEET_REGRADE_ENV; the paths below default to the real system and are overridden only by tests. @@ -59,11 +61,18 @@ is_over() { 'BEGIN { exit !(p + 0 >= t + 0 && a * 100 < m * f) }' } -# Succeeds when the reading has been over for the sustain period and the cooldown since the last step has passed. above_since and last_step are uptime seconds, empty when unset. +# Succeeds when the reading is calm: pressure under the threshold and MemAvailable at or above the floor. It is not the opposite of is_over, which needs both signals to be bad: a node between the two states is neither, and a streak of either ends there. +is_calm() { + local pressure=$1 available_kb=$2 total_kb=$3 + awk -v p="$pressure" -v t="$PRESSURE_THRESHOLD" -v a="$available_kb" -v m="$total_kb" -v f="$AVAILABLE_FLOOR_PERCENT" \ + 'BEGIN { exit !(p + 0 < t + 0 && a * 100 >= m * f) }' +} + +# Succeeds when the reading has held (over, to step down; calm, to step up) for the sustain period and the cooldown since the last step has passed. since and last_step are uptime seconds, empty when unset. may_step() { - local now=$1 above_since=$2 last_step=$3 - [ -n "$above_since" ] || return 1 - [ $((now - above_since)) -ge "$SUSTAIN_SECONDS" ] || return 1 + local now=$1 since=$2 last_step=$3 + [ -n "$since" ] || return 1 + [ $((now - since)) -ge "$SUSTAIN_SECONDS" ] || return 1 [ -z "$last_step" ] || [ $((now - last_step)) -ge "$COOLDOWN_SECONDS" ] } @@ -247,7 +256,7 @@ step_down() { log "now at $to" } -# One step up, by hand. Returning to the first level also lifts the taint and uncordons the node, once the API server answers. +# One step up, by hand or by the watcher. Returning to the first level also lifts the taint and uncordons the node, once the API server answers. step_up() { local from=$1 to=$2 log "step up $from -> $to" @@ -267,12 +276,13 @@ reading_text() { "$1" "$PRESSURE_THRESHOLD" "$2" "$3" "$AVAILABLE_FLOOR_PERCENT" } -# One look at the signals. The streak and the time of the last step are kept in WATCH_ABOVE_SINCE and WATCH_LAST_STEP, which a loop (or a test) carries from one sample to the next. +# One look at the signals. The streaks and the time of the last step are kept in WATCH_ABOVE_SINCE, WATCH_CALM_SINCE and WATCH_LAST_STEP, which a loop (or a test) carries from one sample to the next. WATCH_ABOVE_SINCE='' +WATCH_CALM_SINCE='' WATCH_LAST_STEP='' WATCH_WARNED='' watch_sample() { - local level next pressure available total now text + local level next previous pressure available total now text level=$(current_level) if [ "$level" = unknown ]; then [ -n "$WATCH_WARNED" ] || log "the running system is not one of the levels; not stepping" @@ -294,10 +304,25 @@ watch_sample() { WATCH_ABOVE_SINCE='' log "level $level: back under the limit: $text" fi + if is_calm "$pressure" "$available" "$total"; then + if [ -z "$WATCH_CALM_SINCE" ]; then + WATCH_CALM_SINCE=$now + if [ "${STEP_UP:-manual}" = automatic ] && next_level_up "$level" > /dev/null; then + log "level $level: calm: $text" + fi + fi + else + WATCH_CALM_SINCE='' + fi if next=$(next_level_down "$level") && may_step "$now" "$WATCH_ABOVE_SINCE" "$WATCH_LAST_STEP"; then coordinated step_down "$level" "$next" "$text" || true WATCH_LAST_STEP=$(uptime_seconds) WATCH_ABOVE_SINCE='' + WATCH_CALM_SINCE='' + elif [ "${STEP_UP:-manual}" = automatic ] && previous=$(next_level_up "$level") && may_step "$now" "$WATCH_CALM_SINCE" "$WATCH_LAST_STEP"; then + coordinated step_up "$level" "$previous" || true + WATCH_LAST_STEP=$(uptime_seconds) + WATCH_CALM_SINCE='' fi } @@ -313,7 +338,7 @@ cmd_status() { load_levels log "level: $(current_level) (of: ${LEVELS[*]})" log "$(reading_text "$(psi_avg300)" "$(meminfo_kb MemAvailable)" "$(meminfo_kb MemTotal)")" - log "sample ${SAMPLE_SECONDS}s, sustained for ${SUSTAIN_SECONDS}s, cooldown ${COOLDOWN_SECONDS}s" + log "sample ${SAMPLE_SECONDS}s, sustained for ${SUSTAIN_SECONDS}s, cooldown ${COOLDOWN_SECONDS}s, step up ${STEP_UP:-manual}" } # One step down by hand, for trying a level without waiting for pressure. @@ -326,7 +351,7 @@ cmd_down() { coordinated step_down "$level" "$next" "by hand" } -# One step up by hand: the only way back up. A reboot also returns to the first level. +# One step up by hand. cmd_up() { local level next load_levels diff --git a/tests/regrade/regrade-recover.sh b/tests/regrade/regrade-recover.sh new file mode 100644 index 0000000..83e4ad8 --- /dev/null +++ b/tests/regrade/regrade-recover.sh @@ -0,0 +1,180 @@ +# Tests for stepping up by itself in the core of fleet-regrade (files/regrade/fleet-regrade-core.sh): a node that has been calm for long enough goes back up one level at a time, through the same Lease and peer check as a step down, and never without STEP_UP=automatic. Run as `bash tests/regrade/regrade-recover.sh `. The actuator is a fake, every path the core reads is a fixture, and the two functions that act on a cluster are recorders, so nothing here touches a cluster or the running system. + +# The core calls the functions defined here by name, and reads the variables set here, so shellcheck sees them as unused. + +# shellcheck disable=SC2329,SC2034 +set -euo pipefail + +core=${1:?usage: regrade-recover.sh } +here=$(cd "$(dirname "$0")" && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +cat > "$work/env" < "$FAKE_DIR/nodes.json" +} +nodes_json True + +total=1000000 # kB; the floor is 15 percent of this, 150000 kB + +# The readings: calm is pressure under 10 and MemAvailable at or above 150000; over is pressure at or above 10 and MemAvailable under 150000; between is a reading that is neither, in each of the two ways. +calm_pressure=2.00 calm_available=900000 +over_pressure=20.00 over_available=90000 +check "calm: pressure under the threshold and memory at the floor" is_calm "$calm_pressure" 150000 "$total" +check_not "not calm: pressure at the threshold" is_calm 10 900000 "$total" +check "calm: pressure just under the threshold" is_calm 9.99 900000 "$total" +check_not "not calm: memory just under the floor" is_calm 2 149999 "$total" +check "calm: memory exactly at the floor" is_calm 2 150000 "$total" +check_not "high pressure with plenty of memory is neither over nor calm" bash -c ". '$core' 2>/dev/null; is_over 50 900000 $total || is_calm 50 900000 $total" +check_not "low pressure with little memory is neither over nor calm" bash -c ". '$core' 2>/dev/null; is_over 2 90000 $total || is_calm 2 90000 $total" + +applied="$work/applied" +levels() { printf '%s\n' full control-plane etcd-only; } +current_level() { cat "$work/level"; } +apply_level() { echo "$1" > "$work/level"; echo "$1" >> "$applied"; } +# What the core did to the cluster's workloads, in order; a file, since a scenario runs in a subshell. +calls="$work/calls" +shed_workloads() { echo shed >> "$calls"; } +restore_workloads() { echo restore >> "$calls"; } +wait_for_api() { echo wait >> "$calls"; } + +# Feeds the core one sample every SAMPLE_SECONDS, the reading for each given by the function named in $1 from the sample number, and returns what it did as ": ...". +run_scenario() { + local reading=$1 samples=$2 sample uptime before after + : > "$applied" + : > "$calls" + WATCH_ABOVE_SINCE='' + WATCH_CALM_SINCE='' + WATCH_LAST_STEP='' + WATCH_WARNED='' + echo "${3:-full}" > "$work/level" + for sample in $(seq 1 "$samples"); do + uptime=$((sample * 30)) + printf '%s 0.1\n' "$uptime" > "$UPTIME_FILE" + "$reading" "$sample" + before=$(wc -l < "$applied" | tr -d ' ') + watch_sample > /dev/null + after=$(wc -l < "$applied" | tr -d ' ') + if [ "$after" -gt "$before" ]; then + printf '%s:%s ' "$sample" "$(level_position "$(tail -n 1 "$applied")")" + fi + done +} +set_reading() { + printf 'some avg10=0 avg60=0 avg300=%s total=1\n' "$1" > "$PSI_FILE" + printf 'MemTotal: %s kB\nMemAvailable: %s kB\n' "$total" "$2" > "$MEMINFO_FILE" +} +load_levels + +# Over from the second sample to the 70th, then calm. The steps worked out by hand from the settings: the streak starts at 60 s and the first step down is at 180 s (sample 6); a new streak starts at 210 s and the second needs the cooldown, so it is at 1080 s (sample 36). Calm starts at sample 71 (2130 s) and has held for the sustain period at 2250 s, sample 75; the cooldown since the step at 1080 s ended long ago, so that is the first step up. The streak starts afresh at sample 76 (2280 s) and has held for the sustain period at 2400 s, but the cooldown since the step at 2250 s runs to 3150 s, sample 105, which is the second step up. +rise_and_fall() { + if [ "$1" -eq 1 ]; then set_reading "$calm_pressure" "$calm_available" + elif [ "$1" -le 70 ]; then set_reading "$over_pressure" "$over_available" + else set_reading "$calm_pressure" "$calm_available"; fi +} + +STEP_UP=automatic +steps=$(run_scenario rise_and_fall 110) +check "down twice under pressure, then up twice once calm, at the worked-out moments" test "$steps" = "6:1 36:2 75:1 105:0 " +check "stepping up to the first level waits for the API and restores the workloads" test "$(tr '\n' ' ' < "$calls")" = "shed shed wait restore " + +STEP_UP=manual +steps=$(run_scenario rise_and_fall 110) +check "manual: the same readings never step up" test "$steps" = "6:1 36:2 " +unset STEP_UP +steps=$(run_scenario rise_and_fall 110) +check "unset STEP_UP is manual, as the core was before it existed" test "$steps" = "6:1 36:2 " +STEP_UP=automatic + +# The cooldown after a step down holds a step up back: calm straight after the step still waits for it. Over at samples 2 to 6 (one step down, at sample 6, 180 s), calm from sample 7 (210 s); sustained at 330 s; the cooldown ends at 1080 s (sample 36). +quick_recovery() { + if [ "$1" -ge 2 ] && [ "$1" -le 6 ]; then set_reading "$over_pressure" "$over_available"; else set_reading "$calm_pressure" "$calm_available"; fi +} +steps=$(run_scenario quick_recovery 60) +check "the cooldown after a step down holds the step up back" test "$steps" = "6:1 36:0 " + +# A reading between calm and over ends the calm streak, and so cannot step up, but does not step down either. +noisy_recovery() { + if [ "$1" -eq 1 ]; then set_reading "$calm_pressure" "$calm_available" + elif [ "$1" -le 6 ]; then set_reading "$over_pressure" "$over_available" + elif [ $(($1 % 3)) -eq 0 ]; then set_reading 50 "$calm_available" + else set_reading "$calm_pressure" "$calm_available"; fi +} +steps=$(run_scenario noisy_recovery 80) +check "a reading in between keeps ending the calm streak: no step up, and no step down" test "$steps" = "6:1 " + +# The first level has nowhere to go up to, and calm there is not logged as a streak towards anything. +always_calm() { set_reading "$calm_pressure" "$calm_available"; } +steps=$(run_scenario always_calm 100) +check "calm at the first level does nothing" test "$steps" = "" + +# A node that starts at the last level steps up through the middle one. Calm from the first sample (30 s) has held for the sustain period at 150 s, sample 5; the cooldown after that step runs to 1050 s, sample 35. +steps=$(run_scenario always_calm 100 etcd-only) +check "from the last level it steps up one at a time" test "$steps" = "5:1 35:0 " + +# A step up ends the streak that caused it: the next one needs a fresh calm period of its own, not only the cooldown. That shows only when the sustain period is longer than the cooldown, so the settings are changed for this run: calm from 30 s has held for 600 s at 630 s (sample 21), the first step; the new streak starts at 660 s and holds at 1260 s (sample 42), where without the reset the cooldown, 200 s, would have allowed the second at 840 s (sample 28). +SUSTAIN_SECONDS=600 +COOLDOWN_SECONDS=200 +steps=$(run_scenario always_calm 60 etcd-only) +SUSTAIN_SECONDS=120 +COOLDOWN_SECONDS=900 +check "a step up starts a fresh calm streak" test "$steps" = "21:1 42:0 " + +# The coordination reaches the step up as it does the step down. +nodes_json False +steps=$(run_scenario always_calm 100 etcd-only) +check "a peer down: the watch loop never steps up" test "$steps" = "" +nodes_json True +build_lease other-node 1000000000 1000000000 3 > "$FAKE_DIR/lease.json" +steps=$(run_scenario always_calm 100 etcd-only) +check "the lease held elsewhere: the watch loop never steps up" test "$steps" = "" +rm -f "$FAKE_DIR/lease.json" + +# A level the actuator cannot name is left alone, and a failed step up still starts the cooldown, so it is tried again after it (samples 5, 35, 65 and 95 of 120) and not at every sample. +echo unknown > "$work/level" +: > "$applied" +watch_sample > /dev/null +check "an unknown level is not stepped" test ! -s "$applied" +apply_level() { echo "$1" >> "$applied"; return 1; } +steps=$(run_scenario always_calm 120 etcd-only) +check "a step up that fails is retried only after the cooldown" test "$(wc -l < "$applied" | tr -d ' ')" = 4 +apply_level() { echo "$1" > "$work/level"; echo "$1" >> "$applied"; } + +# The log says what the watcher is waiting for. +level_log() { + echo etcd-only > "$work/level" + printf '%s 0.1\n' 30 > "$UPTIME_FILE" + set_reading "$calm_pressure" "$calm_available" + WATCH_CALM_SINCE='' + watch_sample +} +check "calm is logged at a level that can go up" test "$(level_log | grep -c 'level etcd-only: calm')" = 1 +manual_log() { STEP_UP=manual level_log | grep -q 'level etcd-only: calm'; } +check_not "and not when stepping up is by hand" manual_log + +finish From 7f4fe0c8b7657416d3fcd77f8da0772ede83944e Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 20:53:46 +0100 Subject: [PATCH 03/15] feat(cluster): apply regrade levels to a Docker node The Docker actuator records a level in a file on the k3s data volume and restarts the container when the level changes between etcd-only and another; full and control-plane run the same k3s and differ only in the cordon and taint the core puts on the Node. The container's entrypoint reads the file at every start and starts k3s with its API server, controller manager, scheduler and agent switched off at etcd-only, which leaves the server's etcd member, and its vote, in place. It records the level k3s started at on the container's tmpfs, since k3s hides its arguments from /proc, and that is what the actuator reports as the node's level. A restart returns only when etcd serves again and, for a level that runs the agent, the node is Ready on a heartbeat from the new container, because the core releases its Lease as soon as a step returns. The Node carries a label naming the level from the first step down until it is back at the first level. The regrade Compose service runs the core and the actuator beside the k3s container, in a profile of its own, with the Docker socket and its scripts mounted. --- .../files/docker-compose.yml | 14 ++ .../files/k3s/node-entrypoint.sh | 23 +++ .../files/regrade/Dockerfile | 8 + .../files/regrade/fleet-regrade-docker.sh | 119 +++++++++++ .../files/regrade/fleet-regrade.sh | 13 ++ tests/regrade/fake-docker.sh | 65 ++++++ tests/regrade/regrade-docker.sh | 187 ++++++++++++++++++ tests/unit/test_cluster_etcd_timing.py | 1 + tests/unit/test_regrade_shell.py | 13 ++ 9 files changed, 443 insertions(+) create mode 100644 roles/github_runner_cluster/files/regrade/Dockerfile create mode 100644 roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh create mode 100755 roles/github_runner_cluster/files/regrade/fleet-regrade.sh create mode 100755 tests/regrade/fake-docker.sh create mode 100644 tests/regrade/regrade-docker.sh diff --git a/roles/github_runner_cluster/files/docker-compose.yml b/roles/github_runner_cluster/files/docker-compose.yml index 0e6d2ae..43232ee 100644 --- a/roles/github_runner_cluster/files/docker-compose.yml +++ b/roles/github_runner_cluster/files/docker-compose.yml @@ -126,6 +126,20 @@ services: sleep "$${DOCKER_PRUNE_INTERVAL_SECONDS}" done + # Steps this server down through levels of reduced function when memory runs short, so a small host keeps its place in etcd's quorum instead of being lost to the OOM killer, and back up again; see the role README's Memory-pressure regrading. The core (regrade/fleet-regrade-core.sh) reads memory pressure from the Docker VM's /proc and decides; the Docker actuator (regrade/fleet-regrade-docker.sh) applies a level by recording it in the k3s container and restarting that container, so this service needs the Docker socket, the same permission cost as docker-prune above and for the same reason: it acts on the host's other containers. The scripts and the settings (regrade/fleet-regrade.env, templated by the role) are mounted, not built in, so a collection upgrade reaches the service when the role restarts it, without an image Compose would not otherwise rebuild. Selected by its own profile, which the role adds only when github_runner_cluster_regrade is true. + regrade: + build: + context: ./regrade + restart: unless-stopped + profiles: ["regrade"] + # Keeps the watchdog alive when memory is short, which is when it is needed. Matches the NixOS module's watchdog; -1000 would exempt it from the OOM killer altogether. + oom_score_adj: -900 + environment: + - FLEET_REGRADE_ENV=/usr/local/lib/fleet-regrade/fleet-regrade.env + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./regrade:/usr/local/lib/fleet-regrade:ro + volumes: k3s-data: {} k3s-agent-data: {} diff --git a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh index 6d02a1c..4eb602e 100755 --- a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh +++ b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh @@ -37,6 +37,25 @@ set_api_port_flags() { } set_api_port_flags +# Sets regrade_flags to the arguments that put this server at the level the regrade service (docker-compose.yml, files/regrade) last recorded for it, and records the level k3s is about to start at. A server that has never been stepped down has no record and starts at full. Only etcd-only changes k3s's arguments: it turns off the API server, controller manager, scheduler and agent, so the node keeps its etcd member, and its vote, and little else. full and control-plane run the same k3s and differ in the cordon and taint the regrade service puts on the Node. +# +# The regrade service writes regrade_level_file, on the k3s data volume, before it restarts the container, so the level survives the container being recreated and the host restarting; a level is left only by stepping up. regrade_started_file is on the container's tmpfs, so it exists only while this container runs and says what its k3s started at, which the regrade service reads as the level the node is at: k3s hides its arguments from /proc, so they cannot be read back. The paths are the regrade service's too (fleet-regrade-docker.sh); the environment variables replace them only in tests. +regrade_level_file=${REGRADE_LEVEL_FILE:-/var/lib/rancher/k3s/regrade-level} +regrade_started_file=${REGRADE_STARTED_FILE:-/run/regrade-level} +# Empty unless set_regrade_flags says otherwise, which only the plain server start below calls: a bootstrap server that supervises k3s for an in-cluster Headscale server is never regraded, since etcd-only would stop the kubelet that runs it (the role refuses that combination). +regrade_flags="" + +set_regrade_flags() { + regrade_level=full + if [ -r "$regrade_level_file" ]; then + read -r regrade_level < "$regrade_level_file" + fi + if [ "$regrade_level" = etcd-only ]; then + regrade_flags="--disable-apiserver --disable-controller-manager --disable-scheduler --disable-agent" + fi + printf '%s\n' "$regrade_level" > "$regrade_started_file" +} + # k3s's own vpn-auth integration only runs `tailscale up`; it does not start the daemon itself (without this, k3s fails outright with "tailscale up failed: ... it doesn't appear to be running"). Waits for the control socket before anything talks to it. mkdir -p /var/run/tailscale /var/lib/tailscale tailscaled --state=/var/lib/tailscale/tailscaled.state --socket=/var/run/tailscale/tailscaled.sock & @@ -132,6 +151,8 @@ install_tailscale_up_wait() { # --tls-san is added once per entry of K3S_TLS_SAN_LIST: every server's certificate must cover every server's own mesh name, since a client reaching any server directly by that name needs the certificate to match it. # # --etcd-arg=heartbeat-interval and --etcd-arg=election-timeout are added when set (set_etcd_timing_flags above), for peers whose traffic goes through a relay and so exceeds etcd's default timing. +# +# regrade_flags (set_regrade_flags above) are added last, and are empty at every level but etcd-only. mesh_server_args() { set -- --disable=traefik --disable-network-policy --write-kubeconfig-mode=6443 \ --vpn-auth="$vpn_auth" \ @@ -145,6 +166,7 @@ mesh_server_args() { fi for san in ${K3S_TLS_SAN_LIST:-}; do set -- "$@" --tls-san "$san"; done for flag in $etcd_timing_flags; do set -- "$@" "$flag"; done + for flag in $regrade_flags; do set -- "$@" "$flag"; done server_args="$*" } @@ -207,6 +229,7 @@ case "$role" in wait_for_mesh_host "$server_join_host" install_tailscale_up_wait "$server_join_host" "$tailscale_up_wait_dir" /usr/local/bin/tailscale rm -f /output/kubeconfig.yaml + set_regrade_flags mesh_server_args # shellcheck disable=SC2086 # As above. PATH="$tailscale_up_wait_dir:$PATH" exec /bin/k3s server $server_args $node_taint_flags diff --git a/roles/github_runner_cluster/files/regrade/Dockerfile b/roles/github_runner_cluster/files/regrade/Dockerfile new file mode 100644 index 0000000..2256529 --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/Dockerfile @@ -0,0 +1,8 @@ +# The regrade service of the cluster role's Compose file (see docker-compose.yml): bash, jq and the Docker CLI, all from Alpine's apk. The scripts it runs are not copied in: Compose mounts this directory, so a host picks up a collection upgrade when the service restarts, without an image Compose would not otherwise rebuild. +FROM alpine:3.22 + +RUN apk add --no-cache bash jq docker-cli \ + && ln -s /usr/local/lib/fleet-regrade/fleet-regrade.sh /usr/local/bin/fleet-regrade + +ENTRYPOINT ["/usr/local/bin/fleet-regrade"] +CMD ["watch"] diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh new file mode 100644 index 0000000..03c222e --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh @@ -0,0 +1,119 @@ +# The Docker actuator for fleet-regrade, run by the regrade service of the cluster role's Compose file. The core, fleet-regrade-core.sh, is placed ahead of this file (concatenated, or sourced by the wrapper beside it); this file supplies the three functions of the actuator contract that the core describes, and runs it. +# +# The levels are ways of running the one k3s container of this server. full and control-plane run the same k3s and differ only in the cordon and taint the core puts on the Node, so entering either from the other restarts nothing. etcd-only runs k3s with its API server, controller manager, scheduler and agent turned off: the server keeps its etcd member, and its vote, and nothing else, which it needs the container's entrypoint to start differently, so entering or leaving it restarts the container. +# +# A level is recorded in the container, where the entrypoint (files/k3s/node-entrypoint.sh, which uses the same two paths) reads it at every start. LEVEL_FILE is on the k3s data volume: it survives the container being recreated and the host restarting, so a node stepped down stays stepped down until it is stepped up, and an Ansible run, which changes nothing the container is created from, cannot undo a step. STARTED_FILE is on the container's tmpfs: it exists only while the container runs and holds the level its k3s started at, which is what current_level reports, since k3s hides its arguments from /proc and a level cannot be read back from the process. The container is restarted, never created again, so the node password, kept in the container's writable layer or in a volume, is the one the cluster already holds. +# +# This differs from the NixOS actuator in one respect the core's contract allows: a reboot does not return the node to the first level. A server that restarts at etcd-only comes back at etcd-only, with the cordon the core put on its Node still in place, so the cluster's view and the node's agree. +# +# The Node carries a label, LEVEL_LABEL, with the level as its value, from the first step down until the node is back at the first level. Whatever judges nodes by their readiness can leave a stepped-down server out: at etcd-only it has no kubelet, so it is not Ready by design. +# +# It needs docker and the variables the core reads from the environment file, plus NODE_CONTAINER, the name of this server's k3s container, and LEVEL_LABEL. kubectl is the one in that container (KUBECTL in the environment file is `docker exec` of it), and KUBE_SERVER must name another server's API: the container's own, at etcd-only, answers `apiserver disabled` on some servers. + +LEVEL_FILE=${LEVEL_FILE:-/var/lib/rancher/k3s/regrade-level} +STARTED_FILE=${STARTED_FILE:-/run/regrade-level} + +# How often a restarted container is asked whether it is back, in seconds: the core's own pace for waiting on the API server. +POLL_SECONDS=2 + +# The line k3s logs once its embedded etcd is serving, at every level and on every start. Waiting for it is how a restart is known to have returned the server's vote, which the core's lease relies on: it is released as soon as apply_level returns, and another server that stepped before this one's etcd was back would stack on top of the outage. +ETCD_READY_LOG='ETCD server is now running' + +levels() { + printf '%s\n' full control-plane etcd-only +} + +# Runs a command in the server's container. +in_node() { + docker exec -i "$NODE_CONTAINER" "$@" +} + +# The level the running k3s started at, as its entrypoint recorded it; "unknown" while the container is not running, or when the record is none of the levels (for example one written by another version, or a restart in progress). A running container with neither a record nor a requested level is one whose entrypoint is from before regrading, which could only start k3s whole, so it is at full; its next restart picks up the new entrypoint. +current_level() { + local level + # shellcheck disable=SC2016 # The expansions are for the shell inside the container, which is given the two paths as arguments. + level=$(in_node sh -c 'if [ -r "$1" ]; then cat "$1"; elif [ ! -e "$2" ]; then echo full; fi' sh "$STARTED_FILE" "$LEVEL_FILE" 2> /dev/null) || { + echo unknown + return 0 + } + if levels | grep -Fqx -- "$level"; then + echo "$level" + else + echo unknown + fi +} + +# Puts the label for the level $1 on the Node, or takes it off for the first level. Done before the level is entered, so the label is never missing from a node that has stepped down; the cluster API is reached through another server, which is up at every level. +label_node() { + if [ "$1" = "$(levels | head -n 1)" ]; then + kubectl_ label node "$NODE_NAME" "${LEVEL_LABEL}-" + else + kubectl_ label node "$NODE_NAME" "${LEVEL_LABEL}=$1" --overwrite + fi +} + +# Replaces the file at $1, inside the container, with the level $2, atomically, so a reader never sees half a name. +write_level() { + # shellcheck disable=SC2016 # The expansions are for the shell inside the container, which is given the level and the path as arguments. + in_node sh -c 'umask 077 && printf "%s\n" "$1" > "$2.new" && mv "$2.new" "$2"' sh "$2" "$1" +} + +# How k3s runs at a level: as a server (every level but etcd-only), or as an etcd member alone. Only a change between the two changes the arguments k3s starts with, so only that needs a restart. +k3s_mode() { + if [ "$1" = etcd-only ]; then + echo etcd + else + echo server + fi +} + +# Whether the node has registered again after the container started at $1: it reports Ready and its kubelet has renewed the node lease since. Ready alone is not proof, since the API keeps the old container's Ready condition until the node controller's grace period runs out. Both are UTC, so their first nineteen characters, to the second, order as text. +node_ready_since() { + local started=$1 ready renewed + ready=$(kubectl_ get node "$NODE_NAME" --output 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null) || return 1 + renewed=$(kubectl_ get lease --namespace kube-node-lease "$NODE_NAME" --output 'jsonpath={.spec.renewTime}' 2> /dev/null) || return 1 + [ "$ready" = True ] && [[ ${renewed:0:19} > "${started:0:19}" ]] +} + +# Restarts the container and returns once the server is back as the level $1 runs it: etcd serving again, and, when the level runs the agent, the node Ready on a heartbeat from the new container. Fails when that takes longer than the operation timeout, which leaves the node as it is, for current_level to report. +restart_node() { + local to=$1 started deadline + docker restart --time "$OPERATION_TIMEOUT_SECONDS" "$NODE_CONTAINER" > /dev/null || return 1 + started=$(docker inspect --format '{{.State.StartedAt}}' "$NODE_CONTAINER") || return 1 + deadline=$((SECONDS + OPERATION_TIMEOUT_SECONDS)) + until grep -Fq "$ETCD_READY_LOG" <<< "$(docker logs --since "$started" "$NODE_CONTAINER" 2>&1)"; do + if [ "$SECONDS" -ge "$deadline" ]; then + log "etcd did not come back within ${OPERATION_TIMEOUT_SECONDS}s" + return 1 + fi + sleep "$POLL_SECONDS" + done + [ "$(k3s_mode "$to")" = server ] || return 0 + until node_ready_since "$started"; do + if [ "$SECONDS" -ge "$deadline" ]; then + log "the node did not report Ready within ${OPERATION_TIMEOUT_SECONDS}s" + return 1 + fi + sleep "$POLL_SECONDS" + done +} + +apply_level() { + local to=$1 from + from=$(current_level) + if [ "$from" = unknown ]; then + log "cannot apply $to: the node is not running at a known level" + return 1 + fi + label_node "$to" || return 1 + write_level "$LEVEL_FILE" "$to" || return 1 + if [ "$(k3s_mode "$from")" = "$(k3s_mode "$to")" ]; then + write_level "$STARTED_FILE" "$to" + else + restart_node "$to" + fi +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + regrade_main "$@" +fi diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade.sh new file mode 100755 index 0000000..4e5c3c8 --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Runs fleet-regrade in the regrade service of the cluster role's Compose file: the platform-neutral core, then the Docker actuator, both from the directory the Compose file mounts beside this script, so a collection upgrade reaches the service when it restarts without the image being built again. The settings are in fleet-regrade.env in the same directory, which the role templates. +set -euo pipefail + +lib=$(dirname "$(readlink -f "$0")") +FLEET_REGRADE_ENV=${FLEET_REGRADE_ENV:-$lib/fleet-regrade.env} + +# shellcheck source=/dev/null +. "$lib/fleet-regrade-core.sh" +# shellcheck source=/dev/null +. "$lib/fleet-regrade-docker.sh" + +regrade_main "$@" diff --git a/tests/regrade/fake-docker.sh b/tests/regrade/fake-docker.sh new file mode 100755 index 0000000..85f9eb5 --- /dev/null +++ b/tests/regrade/fake-docker.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# A stand-in for docker for the Docker actuator's tests, keeping one container in the directory FAKE_NODE: `running` (present while it runs), the files the actuator reads and writes inside it (the test points LEVEL_FILE and STARTED_FILE into this directory, so `docker exec` runs a command on the local filesystem), `started-at` (what `docker inspect` prints), `log` (lines of "