From 40e91db711ab93be48d6892b9bc4252e14da2a83 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 22:38:25 +0100 Subject: [PATCH 1/4] test(mesh): count only a known reduced level as a server having stepped --- tests/mesh/run.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 5ea35eb..3fff85b 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -562,11 +562,13 @@ peer_refusal() { at_level() { [ "$(level_of "$1")" = "$2" ]; } node_is() { [ "$(node_status "$1" "$2")" = "$3" ]; } -# The numbers of the nodes whose regrade service reports a level below full. +# The numbers of the nodes whose regrade service reports a level below full. A read that gives no level, or unknown, is not a step: a test node's k3s container can restart by itself under the strain, and its status then reads unknown or fails for a moment, which says nothing about whether the server stepped. stepped_nodes() { local index for index in $(seq 1 "$node_count"); do - [ "$(level_of "$index")" = full ] || printf '%s ' "$index" + case "$(level_of "$index" || true)" in + control-plane | etcd-only) printf '%s ' "$index" ;; + esac done } From f0059c9e4d5893c5f1a8bca7b56edd593561fbef Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 22:52:57 +0100 Subject: [PATCH 2/4] test(mesh): wait for a level to read instead of asserting one read --- tests/mesh/run.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 3fff85b..6aa9b2b 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -556,7 +556,9 @@ recreate_node() { write_marker() { kubectl_on 1 create configmap "grtest-vote-${RANDOM}" --from-literal=a=1 >/dev/null; } # A step by hand on node 3 is refused, and the refusal names the stepped-down server. Retried because node 3's own API may still be settling after it was stopped, which is a refusal too but for another reason; a step that goes ahead is a failure, not a retry. peer_refusal() { - [ "$(level_of 3)" = full ] || fail "$(node_name 3) stepped while $(node_name 2) was stepped down" + case "$(level_of 3 || true)" in + control-plane | etcd-only) fail "$(node_name 3) stepped while $(node_name 2) was stepped down" ;; + esac ! regrade_on 3 fleet-regrade down >"${work}/peer.log" 2>&1 && grep -F "etcd-voting nodes not ready: $(node_name 2)" "${work}/peer.log" >/dev/null } at_level() { [ "$(level_of "$1")" = "$2" ]; } @@ -606,7 +608,7 @@ spec: {holderIdentity: grtest-other, leaseDurationSeconds: 3600, acquireTime: "$ EOF if regrade_on 2 fleet-regrade down >"${work}/held.log" 2>&1; then cat "${work}/held.log"; fail "a step ran with the lease held by another server"; fi grep -F 'the lease is held by grtest-other' "${work}/held.log" >/dev/null || { cat "${work}/held.log"; fail "the refusal did not name the lease holder"; } - [ "$(level_of 2)" = full ] || fail "$(node_name 2) changed level without the lease" + wait_for "$(node_name 2) to read full, since a step without the lease is a failure" 180 at_level 2 full kubectl_on 1 delete lease fleet-regrade --namespace kube-system >/dev/null log "Memory pressure on $(node_name 2) steps it down a level at a time, and no other server steps with it" @@ -642,13 +644,13 @@ EOF log "While $(node_name 2) is stepped down no other server steps" wait_for "$(node_name 3)'s refusal to name the stepped-down server" 300 peer_refusal cat "${work}/peer.log" - [ "$(level_of 3)" = full ] || fail "$(node_name 3) changed level" + wait_for "$(node_name 3) to read full, since a step while another was stepped down is a failure" 180 at_level 3 full log "Rerunning the role with a server stepped down changes nothing, and turning regrading off is refused" # Judged by the role's own recap, not by the containers' start times: after a server was stopped and started a container can restart by itself under the strain, which says nothing about the role. output=$(run_role 2>&1) || { echo "$output"; fail "the role failed with $(node_name 2) stepped down"; } [ "$(grep -cE '^grtest-node[0-9]+ +: .* changed=0 ' <<< "$output")" = "$node_count" ] || { echo "$output" | tail -n 20; fail "rerunning the role changed something while $(node_name 2) was stepped down"; } - [ "$(level_of 2)" = etcd-only ] || fail "the role moved $(node_name 2) off etcd-only" + wait_for "$(node_name 2) to read etcd-only after the role ran" 180 at_level 2 etcd-only regrade=false write_inventory hosted if output=$(run_role 2>&1); then echo "$output"; fail "the role removed the service of a stepped-down server"; fi @@ -681,11 +683,11 @@ EOF log "By hand, a server goes down to etcd-only, stays there through a restart and a recreate, and up is the way back" password=$(node_password_hash 3) regrade_on 3 fleet-regrade down - [ "$(level_of 3)" = control-plane ] || fail "$(node_name 3) did not step down by hand" + wait_for "$(node_name 3) to read control-plane after a step down by hand" 180 at_level 3 control-plane wait_for "$(node_name 3) to be cordoned" 60 node_is 1 3 Ready,SchedulingDisabled [ "$(level_label 3)" = control-plane ] || fail "the Node does not carry the level label at control-plane" regrade_on 3 fleet-regrade down - [ "$(level_of 3)" = etcd-only ] || fail "$(node_name 3) did not reach etcd-only by hand" + wait_for "$(node_name 3) to read etcd-only after a second step down by hand" 180 at_level 3 etcd-only # The level is a file on the data volume that the entrypoint reads at every start, so neither a restart by hand nor a recreate, which is what Compose does when the role changes the container's configuration, brings the server back at full. docker restart "$(container 3)" >/dev/null wait_for "$(node_name 3) to be at etcd-only after a restart by hand" 300 at_level 3 etcd-only @@ -700,7 +702,7 @@ EOF wait_for "$(node_name 3) to be Ready at control-plane" 300 node_is 1 3 Ready,SchedulingDisabled regrade_on 3 fleet-regrade up wait_for "$(node_name 3) to be schedulable again" 300 node_is 1 3 Ready - [ "$(level_of 3)" = full ] || fail "$(node_name 3) did not step up by hand" + wait_for "$(node_name 3) to read full after stepping up by hand" 180 at_level 3 full [ -z "$(level_label 3)" ] || fail "the level label is still on the Node at full" log "With every server at full, turning regrading off removes the services and leaves the nodes running" From 14ce981539e49b19fc9531b3b296eb49097c6740 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 23:05:58 +0100 Subject: [PATCH 3/4] test(mesh): print what regrading recorded for each node when a scenario fails --- tests/mesh/run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 6aa9b2b..51bc844 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -45,6 +45,11 @@ diagnose() { docker exec "$(container "$index")" tailscale status &1 | head -n 20 || true echo "----- $(container "$index"): last log lines" docker logs --tail 60 "$(container "$index")" 2>&1 | cut -c1-400 || true + # Where regrading has the node: how often and when the container started, what it recorded, and which credentials exist (an etcd-only server writes no kubeconfig). + echo "----- $(container "$index"): regrade state" + docker inspect --format 'restarts={{.RestartCount}} started={{.State.StartedAt}} running={{.State.Running}}' "$(container "$index")" || true + docker exec "$(container "$index")" sh -c 'echo "started: $(cat /run/regrade-level 2>&1)"; echo "requested: $(cat /var/lib/rancher/k3s/regrade-level 2>&1)"; ls /etc/rancher/k3s /var/lib/rancher/k3s/server/cred 2>&1' &1 | head -n 40 || true + docker exec "grtest-node${index}-regrade-1" fleet-regrade status &1 | head -n 5 || true done if docker inspect grtest-headscale >/dev/null 2>&1; then echo "----- grtest-headscale: nodes and routes" From 91697b9ede3b2aa68dbfa63d9ba2542c5560f0e8 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 23:28:26 +0100 Subject: [PATCH 4/4] fix(cluster): leave a level someone else changed for the cooldown The watcher's calm streak and cooldown belonged to its own steps only. A server stepped down by hand, or by another watcher, while the watcher had been calm since its first sample was therefore stepped straight back up on the next sample, undoing the step. A level that is not the one the last sample saw, and that this watcher did not step to, now ends both streaks and starts the cooldown. --- .../files/regrade/fleet-regrade-core.sh | 12 +++++++++++- tests/regrade/regrade-recover.sh | 13 +++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh index 2257c47..33b5752 100644 --- a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh @@ -276,10 +276,11 @@ reading_text() { "$1" "$PRESSURE_THRESHOLD" "$2" "$3" "$AVAILABLE_FLOOR_PERCENT" } -# One look at the signals. The streaks and the time of the last step are kept in WATCH_ABOVE_SINCE, WATCH_CALM_SINCE and WATCH_LAST_STEP, which a loop (or a test) carries from one sample to the next. +# One look at the signals. The streaks, the time of the last step and the level the last sample saw are kept in WATCH_ABOVE_SINCE, WATCH_CALM_SINCE, WATCH_LAST_STEP and WATCH_LEVEL, which a loop (or a test) carries from one sample to the next. A level that is not the one the last sample saw, and that this watcher did not step to, was changed by someone else (a step by hand, another watcher, a restart): the streaks end and the cooldown starts, so the watcher does not undo it on a streak that began before it. WATCH_ABOVE_SINCE='' WATCH_CALM_SINCE='' WATCH_LAST_STEP='' +WATCH_LEVEL='' WATCH_WARNED='' watch_sample() { local level next previous pressure available total now text @@ -294,6 +295,13 @@ watch_sample() { available=$(meminfo_kb MemAvailable) total=$(meminfo_kb MemTotal) now=$(uptime_seconds) + if [ -n "$WATCH_LEVEL" ] && [ "$level" != "$WATCH_LEVEL" ]; then + log "level $level: changed from $WATCH_LEVEL by something other than this watcher; waiting out the cooldown" + WATCH_ABOVE_SINCE='' + WATCH_CALM_SINCE='' + WATCH_LAST_STEP=$now + fi + WATCH_LEVEL=$level text=$(reading_text "$pressure" "$available" "$total") if is_over "$pressure" "$available" "$total"; then if [ -z "$WATCH_ABOVE_SINCE" ]; then @@ -317,11 +325,13 @@ watch_sample() { if next=$(next_level_down "$level") && may_step "$now" "$WATCH_ABOVE_SINCE" "$WATCH_LAST_STEP"; then coordinated step_down "$level" "$next" "$text" || true WATCH_LAST_STEP=$(uptime_seconds) + WATCH_LEVEL=$(current_level) WATCH_ABOVE_SINCE='' WATCH_CALM_SINCE='' elif [ "${STEP_UP:-manual}" = automatic ] && previous=$(next_level_up "$level") && may_step "$now" "$WATCH_CALM_SINCE" "$WATCH_LAST_STEP"; then coordinated step_up "$level" "$previous" || true WATCH_LAST_STEP=$(uptime_seconds) + WATCH_LEVEL=$(current_level) WATCH_CALM_SINCE='' fi } diff --git a/tests/regrade/regrade-recover.sh b/tests/regrade/regrade-recover.sh index 83e4ad8..3d86eb5 100644 --- a/tests/regrade/regrade-recover.sh +++ b/tests/regrade/regrade-recover.sh @@ -71,6 +71,7 @@ run_scenario() { WATCH_ABOVE_SINCE='' WATCH_CALM_SINCE='' WATCH_LAST_STEP='' + WATCH_LEVEL='' WATCH_WARNED='' echo "${3:-full}" > "$work/level" for sample in $(seq 1 "$samples"); do @@ -145,6 +146,18 @@ SUSTAIN_SECONDS=120 COOLDOWN_SECONDS=900 check "a step up starts a fresh calm streak" test "$steps" = "21:1 42:0 " +# A level changed by someone else, a step by hand say, is not undone on a calm streak that began before it: the watcher has been calm since its first sample, and the level becomes control-plane at sample 10 (300 s), which it notices at once; the cooldown starts there and ends at 1200 s, sample 40, so that is the step up. Without noticing, it would step up at sample 10. +changed_by_hand() { + set_reading "$calm_pressure" "$calm_available" + if [ "$1" -eq 10 ]; then echo control-plane > "$work/level"; fi +} +steps=$(run_scenario changed_by_hand 60) +check "a level changed by someone else is left for the cooldown, not undone at once" test "$steps" = "40:0 " + +# The same for a level this watcher stepped to and that is then read back as it left it: no change is seen. +steps=$(run_scenario rise_and_fall 110) +check "its own steps are not mistaken for someone else's" test "$steps" = "6:1 36:2 75:1 105:0 " + # The coordination reaches the step up as it does the step down. nodes_json False steps=$(run_scenario always_calm 100 etcd-only)