fix(cluster): leave a level someone else changed for the cooldown

roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh

@@ -276,10 +276,11 @@ reading_text() {
     "$1" "$PRESSURE_THRESHOLD" "$2" "$3" "$AVAILABLE_FLOOR_PERCENT"
 }
 
-# One look at the signals. The streaks and the time of the last step are kept in WATCH_ABOVE_SINCE, WATCH_CALM_SINCE and WATCH_LAST_STEP, which a loop (or a test) carries from one sample to the next.
+# One look at the signals. The streaks, the time of the last step and the level the last sample saw are kept in WATCH_ABOVE_SINCE, WATCH_CALM_SINCE, WATCH_LAST_STEP and WATCH_LEVEL, which a loop (or a test) carries from one sample to the next. A level that is not the one the last sample saw, and that this watcher did not step to, was changed by someone else (a step by hand, another watcher, a restart): the streaks end and the cooldown starts, so the watcher does not undo it on a streak that began before it.
 WATCH_ABOVE_SINCE=''
 WATCH_CALM_SINCE=''
 WATCH_LAST_STEP=''
+WATCH_LEVEL=''
 WATCH_WARNED=''
 watch_sample() {
   local level next previous pressure available total now text
@@ -294,6 +295,13 @@ watch_sample() {
   available=$(meminfo_kb MemAvailable)
   total=$(meminfo_kb MemTotal)
   now=$(uptime_seconds)
+  if [ -n "$WATCH_LEVEL" ] && [ "$level" != "$WATCH_LEVEL" ]; then
+    log "level $level: changed from $WATCH_LEVEL by something other than this watcher; waiting out the cooldown"
+    WATCH_ABOVE_SINCE=''
+    WATCH_CALM_SINCE=''
+    WATCH_LAST_STEP=$now
+  fi
+  WATCH_LEVEL=$level
   text=$(reading_text "$pressure" "$available" "$total")
   if is_over "$pressure" "$available" "$total"; then
     if [ -z "$WATCH_ABOVE_SINCE" ]; then
@@ -317,11 +325,13 @@ watch_sample() {
   if next=$(next_level_down "$level") && may_step "$now" "$WATCH_ABOVE_SINCE" "$WATCH_LAST_STEP"; then
     coordinated step_down "$level" "$next" "$text" || true
     WATCH_LAST_STEP=$(uptime_seconds)
+    WATCH_LEVEL=$(current_level)
     WATCH_ABOVE_SINCE=''
     WATCH_CALM_SINCE=''
   elif [ "${STEP_UP:-manual}" = automatic ] && previous=$(next_level_up "$level") && may_step "$now" "$WATCH_CALM_SINCE" "$WATCH_LAST_STEP"; then
     coordinated step_up "$level" "$previous" || true
     WATCH_LAST_STEP=$(uptime_seconds)
+    WATCH_LEVEL=$(current_level)
     WATCH_CALM_SINCE=''
   fi
 }

tests/mesh/run.sh

@@ -45,6 +45,11 @@ diagnose() {
     docker exec "$(container "$index")" tailscale status </dev/null 2>&1 | head -n 20 || true
     echo "----- $(container "$index"): last log lines"
     docker logs --tail 60 "$(container "$index")" 2>&1 | cut -c1-400 || true
+    # Where regrading has the node: how often and when the container started, what it recorded, and which credentials exist (an etcd-only server writes no kubeconfig).
+    echo "----- $(container "$index"): regrade state"
+    docker inspect --format 'restarts={{.RestartCount}} started={{.State.StartedAt}} running={{.State.Running}}' "$(container "$index")" || true
+    docker exec "$(container "$index")" sh -c 'echo "started: $(cat /run/regrade-level 2>&1)"; echo "requested: $(cat /var/lib/rancher/k3s/regrade-level 2>&1)"; ls /etc/rancher/k3s /var/lib/rancher/k3s/server/cred 2>&1' </dev/null 2>&1 | head -n 40 || true
+    docker exec "grtest-node${index}-regrade-1" fleet-regrade status </dev/null 2>&1 | head -n 5 || true
   done
   if docker inspect grtest-headscale >/dev/null 2>&1; then
     echo "----- grtest-headscale: nodes and routes"
@@ -556,17 +561,21 @@ recreate_node() {
 write_marker() { kubectl_on 1 create configmap "grtest-vote-${RANDOM}" --from-literal=a=1 >/dev/null; }
 # A step by hand on node 3 is refused, and the refusal names the stepped-down server. Retried because node 3's own API may still be settling after it was stopped, which is a refusal too but for another reason; a step that goes ahead is a failure, not a retry.
 peer_refusal() {
-  [ "$(level_of 3)" = full ] || fail "$(node_name 3) stepped while $(node_name 2) was stepped down"
+  case "$(level_of 3 || true)" in
+    control-plane | etcd-only) fail "$(node_name 3) stepped while $(node_name 2) was stepped down" ;;
+  esac
   ! regrade_on 3 fleet-regrade down >"${work}/peer.log" 2>&1 && grep -F "etcd-voting nodes not ready: $(node_name 2)" "${work}/peer.log" >/dev/null
 }
 at_level() { [ "$(level_of "$1")" = "$2" ]; }
 node_is() { [ "$(node_status "$1" "$2")" = "$3" ]; }
 
-# The numbers of the nodes whose regrade service reports a level below full.
+# The numbers of the nodes whose regrade service reports a level below full. A read that gives no level, or unknown, is not a step: a test node's k3s container can restart by itself under the strain, and its status then reads unknown or fails for a moment, which says nothing about whether the server stepped.
 stepped_nodes() {
   local index
   for index in $(seq 1 "$node_count"); do
-    [ "$(level_of "$index")" = full ] || printf '%s ' "$index"
+    case "$(level_of "$index" || true)" in
+      control-plane | etcd-only) printf '%s ' "$index" ;;
+    esac
   done
 }
 
@@ -604,7 +613,7 @@ spec: {holderIdentity: grtest-other, leaseDurationSeconds: 3600, acquireTime: "$
 EOF
   if regrade_on 2 fleet-regrade down >"${work}/held.log" 2>&1; then cat "${work}/held.log"; fail "a step ran with the lease held by another server"; fi
   grep -F 'the lease is held by grtest-other' "${work}/held.log" >/dev/null || { cat "${work}/held.log"; fail "the refusal did not name the lease holder"; }
-  [ "$(level_of 2)" = full ] || fail "$(node_name 2) changed level without the lease"
+  wait_for "$(node_name 2) to read full, since a step without the lease is a failure" 180 at_level 2 full
   kubectl_on 1 delete lease fleet-regrade --namespace kube-system >/dev/null
 
   log "Memory pressure on $(node_name 2) steps it down a level at a time, and no other server steps with it"
@@ -640,13 +649,13 @@ EOF
   log "While $(node_name 2) is stepped down no other server steps"
   wait_for "$(node_name 3)'s refusal to name the stepped-down server" 300 peer_refusal
   cat "${work}/peer.log"
-  [ "$(level_of 3)" = full ] || fail "$(node_name 3) changed level"
+  wait_for "$(node_name 3) to read full, since a step while another was stepped down is a failure" 180 at_level 3 full
 
   log "Rerunning the role with a server stepped down changes nothing, and turning regrading off is refused"
   # Judged by the role's own recap, not by the containers' start times: after a server was stopped and started a container can restart by itself under the strain, which says nothing about the role.
   output=$(run_role 2>&1) || { echo "$output"; fail "the role failed with $(node_name 2) stepped down"; }
   [ "$(grep -cE '^grtest-node[0-9]+ +: .* changed=0 ' <<< "$output")" = "$node_count" ] || { echo "$output" | tail -n 20; fail "rerunning the role changed something while $(node_name 2) was stepped down"; }
-  [ "$(level_of 2)" = etcd-only ] || fail "the role moved $(node_name 2) off etcd-only"
+  wait_for "$(node_name 2) to read etcd-only after the role ran" 180 at_level 2 etcd-only
   regrade=false
   write_inventory hosted
   if output=$(run_role 2>&1); then echo "$output"; fail "the role removed the service of a stepped-down server"; fi
@@ -679,11 +688,11 @@ EOF
   log "By hand, a server goes down to etcd-only, stays there through a restart and a recreate, and up is the way back"
   password=$(node_password_hash 3)
   regrade_on 3 fleet-regrade down
-  [ "$(level_of 3)" = control-plane ] || fail "$(node_name 3) did not step down by hand"
+  wait_for "$(node_name 3) to read control-plane after a step down by hand" 180 at_level 3 control-plane
   wait_for "$(node_name 3) to be cordoned" 60 node_is 1 3 Ready,SchedulingDisabled
   [ "$(level_label 3)" = control-plane ] || fail "the Node does not carry the level label at control-plane"
   regrade_on 3 fleet-regrade down
-  [ "$(level_of 3)" = etcd-only ] || fail "$(node_name 3) did not reach etcd-only by hand"
+  wait_for "$(node_name 3) to read etcd-only after a second step down by hand" 180 at_level 3 etcd-only
   # The level is a file on the data volume that the entrypoint reads at every start, so neither a restart by hand nor a recreate, which is what Compose does when the role changes the container's configuration, brings the server back at full.
   docker restart "$(container 3)" >/dev/null
   wait_for "$(node_name 3) to be at etcd-only after a restart by hand" 300 at_level 3 etcd-only
@@ -698,7 +707,7 @@ EOF
   wait_for "$(node_name 3) to be Ready at control-plane" 300 node_is 1 3 Ready,SchedulingDisabled
   regrade_on 3 fleet-regrade up
   wait_for "$(node_name 3) to be schedulable again" 300 node_is 1 3 Ready
-  [ "$(level_of 3)" = full ] || fail "$(node_name 3) did not step up by hand"
+  wait_for "$(node_name 3) to read full after stepping up by hand" 180 at_level 3 full
   [ -z "$(level_label 3)" ] || fail "the level label is still on the Node at full"
 
   log "With every server at full, turning regrading off removes the services and leaves the nodes running"

tests/regrade/regrade-recover.sh

@@ -71,6 +71,7 @@ run_scenario() {
   WATCH_ABOVE_SINCE=''
   WATCH_CALM_SINCE=''
   WATCH_LAST_STEP=''
+  WATCH_LEVEL=''
   WATCH_WARNED=''
   echo "${3:-full}" > "$work/level"
   for sample in $(seq 1 "$samples"); do
@@ -145,6 +146,18 @@ SUSTAIN_SECONDS=120
 COOLDOWN_SECONDS=900
 check "a step up starts a fresh calm streak" test "$steps" = "21:1 42:0 "
 
+# A level changed by someone else, a step by hand say, is not undone on a calm streak that began before it: the watcher has been calm since its first sample, and the level becomes control-plane at sample 10 (300 s), which it notices at once; the cooldown starts there and ends at 1200 s, sample 40, so that is the step up. Without noticing, it would step up at sample 10.
+changed_by_hand() {
+  set_reading "$calm_pressure" "$calm_available"
+  if [ "$1" -eq 10 ]; then echo control-plane > "$work/level"; fi
+}
+steps=$(run_scenario changed_by_hand 60)
+check "a level changed by someone else is left for the cooldown, not undone at once" test "$steps" = "40:0 "
+
+# The same for a level this watcher stepped to and that is then read back as it left it: no change is seen.
+steps=$(run_scenario rise_and_fall 110)
+check "its own steps are not mistaken for someone else's" test "$steps" = "6:1 36:2 75:1 105:0 "
+
 # The coordination reaches the step up as it does the step down.
 nodes_json False
 steps=$(run_scenario always_calm 100 etcd-only)