|
173 | 173 | echo " github_runner_cluster_etcd_heartbeat_interval: ${etcd_heartbeat_ms}" |
174 | 174 | echo " github_runner_cluster_etcd_election_timeout: ${etcd_election_ms}" |
175 | 175 | echo " github_runner_cluster_regrade: ${regrade}" |
| 176 | + # The test cluster is throwaway: the role must not install a login item that starts the host's Colima profile. |
| 177 | + echo " github_runner_cluster_runtime_autostart: false" |
176 | 178 | if [ "$pin_context" = 1 ]; then |
177 | 179 | echo " github_runner_cluster_docker_context: ${grtest_pinned_context}" |
178 | 180 | fi |
@@ -575,6 +577,15 @@ peer_refusal() { |
575 | 577 | ! regrade_on 3 fleet-regrade down >"${work}/peer.log" 2>&1 && grep -F "etcd-voting nodes not ready: $(node_name 2)" "${work}/peer.log" >/dev/null |
576 | 578 | } |
577 | 579 | at_level() { [ "$(level_of "$1")" = "$2" ]; } |
| 580 | +# Whether the regrade Lease exists and nobody holds it, as a release leaves it, read through node 1's API. A Lease left to expire keeps its holder, so this tells a release from an expiry at any time. |
| 581 | +lease_free() { [ "$(kubectl_on 1 get lease fleet-regrade --namespace kube-system -o 'jsonpath={.spec.holderIdentity}' 2>/dev/null; echo "/$?")" = /0 ]; } |
| 582 | +# Whether the regrade service of node $1 has logged, in the file $2 inside it, that it reached the level $3. |
| 583 | +logged_level() { regrade_on "$1" cat "$2" | grep -Fx "now at $3" >/dev/null; } |
| 584 | +# A step down to etcd-only gives the Lease back from another server's API, since the stepped-down server has none: once the step has finished the Lease is free, and the service never logged that it could not release it. |
| 585 | +assert_released() { |
| 586 | + wait_for "the Lease to be released after $(node_name "$1")'s step down to etcd-only" 120 lease_free |
| 587 | + if regrade_on "$1" cat "$2" | grep -F 'lease could not be released' >/dev/null; then regrade_on "$1" cat "$2"; fail "$(node_name "$1") could not release the Lease"; fi |
| 588 | +} |
578 | 589 | node_is() { [ "$(node_status "$1" "$2")" = "$3" ]; } |
579 | 590 |
|
580 | 591 | # The numbers of the nodes whose regrade service reports a level below full. A read that gives no level, or unknown, is not a step: a test node's k3s container can restart by itself under the strain, and its status then reads unknown or fails for a moment, which says nothing about whether the server stepped. |
|
636 | 647 | [ "$SECONDS" -lt "$deadline" ] || { regrade_on 2 cat /tmp/watch.log || true; fail "$(node_name 2) did not reach etcd-only"; } |
637 | 648 | sleep 2 |
638 | 649 | done |
| 650 | + wait_for "the watcher on $(node_name 2) to finish its step to etcd-only" 300 logged_level 2 /tmp/watch.log etcd-only |
| 651 | + assert_released 2 /tmp/watch.log |
639 | 652 | regrade_on 2 cat /tmp/watch.log |
640 | 653 | regrade_on 2 cat /tmp/watch.log | grep -F 'step down full -> control-plane' >/dev/null || fail "the watcher did not step down through control-plane first" |
641 | 654 |
|
|
699 | 712 | wait_for "$(node_name 3) to read control-plane after a step down by hand" 180 at_level 3 control-plane |
700 | 713 | wait_for "$(node_name 3) to be cordoned" 60 node_is 1 3 Ready,SchedulingDisabled |
701 | 714 | [ "$(level_label 3)" = control-plane ] || fail "the Node does not carry the level label at control-plane" |
702 | | - regrade_on 3 fleet-regrade down |
| 715 | + regrade_on 3 sh -c 'fleet-regrade down >/tmp/down.log 2>&1' || { regrade_on 3 cat /tmp/down.log; fail "the second step down by hand on $(node_name 3) failed"; } |
| 716 | + regrade_on 3 cat /tmp/down.log |
703 | 717 | wait_for "$(node_name 3) to read etcd-only after a second step down by hand" 180 at_level 3 etcd-only |
| 718 | + assert_released 3 /tmp/down.log |
704 | 719 | # The level is a file on the data volume that the entrypoint reads at every start, so neither a restart by hand nor a recreate, which is what Compose does when the role changes the container's configuration, brings the server back at full. |
705 | 720 | docker restart "$(container 3)" >/dev/null |
706 | 721 | wait_for "$(node_name 3) to be at etcd-only after a restart by hand" 300 at_level 3 etcd-only |
|
0 commit comments