diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh index 2257c47..33b5752 100644 --- a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh @@ -276,10 +276,11 @@ reading_text() { "$1" "$PRESSURE_THRESHOLD" "$2" "$3" "$AVAILABLE_FLOOR_PERCENT" } -# One look at the signals. The streaks and the time of the last step are kept in WATCH_ABOVE_SINCE, WATCH_CALM_SINCE and WATCH_LAST_STEP, which a loop (or a test) carries from one sample to the next. +# One look at the signals. The streaks, the time of the last step and the level the last sample saw are kept in WATCH_ABOVE_SINCE, WATCH_CALM_SINCE, WATCH_LAST_STEP and WATCH_LEVEL, which a loop (or a test) carries from one sample to the next. A level that is not the one the last sample saw, and that this watcher did not step to, was changed by someone else (a step by hand, another watcher, a restart): the streaks end and the cooldown starts, so the watcher does not undo it on a streak that began before it. WATCH_ABOVE_SINCE='' WATCH_CALM_SINCE='' WATCH_LAST_STEP='' +WATCH_LEVEL='' WATCH_WARNED='' watch_sample() { local level next previous pressure available total now text @@ -294,6 +295,13 @@ watch_sample() { available=$(meminfo_kb MemAvailable) total=$(meminfo_kb MemTotal) now=$(uptime_seconds) + if [ -n "$WATCH_LEVEL" ] && [ "$level" != "$WATCH_LEVEL" ]; then + log "level $level: changed from $WATCH_LEVEL by something other than this watcher; waiting out the cooldown" + WATCH_ABOVE_SINCE='' + WATCH_CALM_SINCE='' + WATCH_LAST_STEP=$now + fi + WATCH_LEVEL=$level text=$(reading_text "$pressure" "$available" "$total") if is_over "$pressure" "$available" "$total"; then if [ -z "$WATCH_ABOVE_SINCE" ]; then @@ -317,11 +325,13 @@ watch_sample() { if next=$(next_level_down "$level") && may_step "$now" "$WATCH_ABOVE_SINCE" "$WATCH_LAST_STEP"; then coordinated step_down "$level" "$next" "$text" || true WATCH_LAST_STEP=$(uptime_seconds) + WATCH_LEVEL=$(current_level) WATCH_ABOVE_SINCE='' WATCH_CALM_SINCE='' elif [ "${STEP_UP:-manual}" = automatic ] && previous=$(next_level_up "$level") && may_step "$now" "$WATCH_CALM_SINCE" "$WATCH_LAST_STEP"; then coordinated step_up "$level" "$previous" || true WATCH_LAST_STEP=$(uptime_seconds) + WATCH_LEVEL=$(current_level) WATCH_CALM_SINCE='' fi } diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 5ea35eb..51bc844 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -45,6 +45,11 @@ diagnose() { docker exec "$(container "$index")" tailscale status &1 | head -n 20 || true echo "----- $(container "$index"): last log lines" docker logs --tail 60 "$(container "$index")" 2>&1 | cut -c1-400 || true + # Where regrading has the node: how often and when the container started, what it recorded, and which credentials exist (an etcd-only server writes no kubeconfig). + echo "----- $(container "$index"): regrade state" + docker inspect --format 'restarts={{.RestartCount}} started={{.State.StartedAt}} running={{.State.Running}}' "$(container "$index")" || true + docker exec "$(container "$index")" sh -c 'echo "started: $(cat /run/regrade-level 2>&1)"; echo "requested: $(cat /var/lib/rancher/k3s/regrade-level 2>&1)"; ls /etc/rancher/k3s /var/lib/rancher/k3s/server/cred 2>&1' &1 | head -n 40 || true + docker exec "grtest-node${index}-regrade-1" fleet-regrade status &1 | head -n 5 || true done if docker inspect grtest-headscale >/dev/null 2>&1; then echo "----- grtest-headscale: nodes and routes" @@ -556,17 +561,21 @@ recreate_node() { write_marker() { kubectl_on 1 create configmap "grtest-vote-${RANDOM}" --from-literal=a=1 >/dev/null; } # A step by hand on node 3 is refused, and the refusal names the stepped-down server. Retried because node 3's own API may still be settling after it was stopped, which is a refusal too but for another reason; a step that goes ahead is a failure, not a retry. peer_refusal() { - [ "$(level_of 3)" = full ] || fail "$(node_name 3) stepped while $(node_name 2) was stepped down" + case "$(level_of 3 || true)" in + control-plane | etcd-only) fail "$(node_name 3) stepped while $(node_name 2) was stepped down" ;; + esac ! regrade_on 3 fleet-regrade down >"${work}/peer.log" 2>&1 && grep -F "etcd-voting nodes not ready: $(node_name 2)" "${work}/peer.log" >/dev/null } at_level() { [ "$(level_of "$1")" = "$2" ]; } node_is() { [ "$(node_status "$1" "$2")" = "$3" ]; } -# The numbers of the nodes whose regrade service reports a level below full. +# The numbers of the nodes whose regrade service reports a level below full. A read that gives no level, or unknown, is not a step: a test node's k3s container can restart by itself under the strain, and its status then reads unknown or fails for a moment, which says nothing about whether the server stepped. stepped_nodes() { local index for index in $(seq 1 "$node_count"); do - [ "$(level_of "$index")" = full ] || printf '%s ' "$index" + case "$(level_of "$index" || true)" in + control-plane | etcd-only) printf '%s ' "$index" ;; + esac done } @@ -604,7 +613,7 @@ spec: {holderIdentity: grtest-other, leaseDurationSeconds: 3600, acquireTime: "$ EOF if regrade_on 2 fleet-regrade down >"${work}/held.log" 2>&1; then cat "${work}/held.log"; fail "a step ran with the lease held by another server"; fi grep -F 'the lease is held by grtest-other' "${work}/held.log" >/dev/null || { cat "${work}/held.log"; fail "the refusal did not name the lease holder"; } - [ "$(level_of 2)" = full ] || fail "$(node_name 2) changed level without the lease" + wait_for "$(node_name 2) to read full, since a step without the lease is a failure" 180 at_level 2 full kubectl_on 1 delete lease fleet-regrade --namespace kube-system >/dev/null log "Memory pressure on $(node_name 2) steps it down a level at a time, and no other server steps with it" @@ -640,13 +649,13 @@ EOF log "While $(node_name 2) is stepped down no other server steps" wait_for "$(node_name 3)'s refusal to name the stepped-down server" 300 peer_refusal cat "${work}/peer.log" - [ "$(level_of 3)" = full ] || fail "$(node_name 3) changed level" + wait_for "$(node_name 3) to read full, since a step while another was stepped down is a failure" 180 at_level 3 full log "Rerunning the role with a server stepped down changes nothing, and turning regrading off is refused" # Judged by the role's own recap, not by the containers' start times: after a server was stopped and started a container can restart by itself under the strain, which says nothing about the role. output=$(run_role 2>&1) || { echo "$output"; fail "the role failed with $(node_name 2) stepped down"; } [ "$(grep -cE '^grtest-node[0-9]+ +: .* changed=0 ' <<< "$output")" = "$node_count" ] || { echo "$output" | tail -n 20; fail "rerunning the role changed something while $(node_name 2) was stepped down"; } - [ "$(level_of 2)" = etcd-only ] || fail "the role moved $(node_name 2) off etcd-only" + wait_for "$(node_name 2) to read etcd-only after the role ran" 180 at_level 2 etcd-only regrade=false write_inventory hosted if output=$(run_role 2>&1); then echo "$output"; fail "the role removed the service of a stepped-down server"; fi @@ -679,11 +688,11 @@ EOF log "By hand, a server goes down to etcd-only, stays there through a restart and a recreate, and up is the way back" password=$(node_password_hash 3) regrade_on 3 fleet-regrade down - [ "$(level_of 3)" = control-plane ] || fail "$(node_name 3) did not step down by hand" + wait_for "$(node_name 3) to read control-plane after a step down by hand" 180 at_level 3 control-plane wait_for "$(node_name 3) to be cordoned" 60 node_is 1 3 Ready,SchedulingDisabled [ "$(level_label 3)" = control-plane ] || fail "the Node does not carry the level label at control-plane" regrade_on 3 fleet-regrade down - [ "$(level_of 3)" = etcd-only ] || fail "$(node_name 3) did not reach etcd-only by hand" + wait_for "$(node_name 3) to read etcd-only after a second step down by hand" 180 at_level 3 etcd-only # The level is a file on the data volume that the entrypoint reads at every start, so neither a restart by hand nor a recreate, which is what Compose does when the role changes the container's configuration, brings the server back at full. docker restart "$(container 3)" >/dev/null wait_for "$(node_name 3) to be at etcd-only after a restart by hand" 300 at_level 3 etcd-only @@ -698,7 +707,7 @@ EOF wait_for "$(node_name 3) to be Ready at control-plane" 300 node_is 1 3 Ready,SchedulingDisabled regrade_on 3 fleet-regrade up wait_for "$(node_name 3) to be schedulable again" 300 node_is 1 3 Ready - [ "$(level_of 3)" = full ] || fail "$(node_name 3) did not step up by hand" + wait_for "$(node_name 3) to read full after stepping up by hand" 180 at_level 3 full [ -z "$(level_label 3)" ] || fail "the level label is still on the Node at full" log "With every server at full, turning regrading off removes the services and leaves the nodes running" diff --git a/tests/regrade/regrade-recover.sh b/tests/regrade/regrade-recover.sh index 83e4ad8..3d86eb5 100644 --- a/tests/regrade/regrade-recover.sh +++ b/tests/regrade/regrade-recover.sh @@ -71,6 +71,7 @@ run_scenario() { WATCH_ABOVE_SINCE='' WATCH_CALM_SINCE='' WATCH_LAST_STEP='' + WATCH_LEVEL='' WATCH_WARNED='' echo "${3:-full}" > "$work/level" for sample in $(seq 1 "$samples"); do @@ -145,6 +146,18 @@ SUSTAIN_SECONDS=120 COOLDOWN_SECONDS=900 check "a step up starts a fresh calm streak" test "$steps" = "21:1 42:0 " +# A level changed by someone else, a step by hand say, is not undone on a calm streak that began before it: the watcher has been calm since its first sample, and the level becomes control-plane at sample 10 (300 s), which it notices at once; the cooldown starts there and ends at 1200 s, sample 40, so that is the step up. Without noticing, it would step up at sample 10. +changed_by_hand() { + set_reading "$calm_pressure" "$calm_available" + if [ "$1" -eq 10 ]; then echo control-plane > "$work/level"; fi +} +steps=$(run_scenario changed_by_hand 60) +check "a level changed by someone else is left for the cooldown, not undone at once" test "$steps" = "40:0 " + +# The same for a level this watcher stepped to and that is then read back as it left it: no change is seen. +steps=$(run_scenario rise_and_fall 110) +check "its own steps are not mistaken for someone else's" test "$steps" = "6:1 36:2 75:1 105:0 " + # The coordination reaches the step up as it does the step down. nodes_json False steps=$(run_scenario always_calm 100 etcd-only)