Skip to content

Commit d976df7

Browse files
committed
test(mesh): check a server stepped down to etcd-only gives the Lease back
The stepped-down server has no API server, so the release goes through another server's. The scenario now waits for the Lease to be free rather than expired, and fails if the service logged that it could not release it. The harness inventory also turns the runtime autostart off, so a run against a Colima profile installs no login item.
1 parent 6a5752d commit d976df7

1 file changed

Lines changed: 16 additions & 1 deletion

File tree

‎tests/mesh/run.sh‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ EOF
173173
echo " github_runner_cluster_etcd_heartbeat_interval: ${etcd_heartbeat_ms}"
174174
echo " github_runner_cluster_etcd_election_timeout: ${etcd_election_ms}"
175175
echo " github_runner_cluster_regrade: ${regrade}"
176+
# The test cluster is throwaway: the role must not install a login item that starts the host's Colima profile.
177+
echo " github_runner_cluster_runtime_autostart: false"
176178
if [ "$pin_context" = 1 ]; then
177179
echo " github_runner_cluster_docker_context: ${grtest_pinned_context}"
178180
fi
@@ -575,6 +577,15 @@ peer_refusal() {
575577
! regrade_on 3 fleet-regrade down >"${work}/peer.log" 2>&1 && grep -F "etcd-voting nodes not ready: $(node_name 2)" "${work}/peer.log" >/dev/null
576578
}
577579
at_level() { [ "$(level_of "$1")" = "$2" ]; }
580+
# Whether the regrade Lease exists and nobody holds it, as a release leaves it, read through node 1's API. A Lease left to expire keeps its holder, so this tells a release from an expiry at any time.
581+
lease_free() { [ "$(kubectl_on 1 get lease fleet-regrade --namespace kube-system -o 'jsonpath={.spec.holderIdentity}' 2>/dev/null; echo "/$?")" = /0 ]; }
582+
# Whether the regrade service of node $1 has logged, in the file $2 inside it, that it reached the level $3.
583+
logged_level() { regrade_on "$1" cat "$2" | grep -Fx "now at $3" >/dev/null; }
584+
# A step down to etcd-only gives the Lease back from another server's API, since the stepped-down server has none: once the step has finished the Lease is free, and the service never logged that it could not release it.
585+
assert_released() {
586+
wait_for "the Lease to be released after $(node_name "$1")'s step down to etcd-only" 120 lease_free
587+
if regrade_on "$1" cat "$2" | grep -F 'lease could not be released' >/dev/null; then regrade_on "$1" cat "$2"; fail "$(node_name "$1") could not release the Lease"; fi
588+
}
578589
node_is() { [ "$(node_status "$1" "$2")" = "$3" ]; }
579590

580591
# The numbers of the nodes whose regrade service reports a level below full. A read that gives no level, or unknown, is not a step: a test node's k3s container can restart by itself under the strain, and its status then reads unknown or fails for a moment, which says nothing about whether the server stepped.
@@ -636,6 +647,8 @@ EOF
636647
[ "$SECONDS" -lt "$deadline" ] || { regrade_on 2 cat /tmp/watch.log || true; fail "$(node_name 2) did not reach etcd-only"; }
637648
sleep 2
638649
done
650+
wait_for "the watcher on $(node_name 2) to finish its step to etcd-only" 300 logged_level 2 /tmp/watch.log etcd-only
651+
assert_released 2 /tmp/watch.log
639652
regrade_on 2 cat /tmp/watch.log
640653
regrade_on 2 cat /tmp/watch.log | grep -F 'step down full -> control-plane' >/dev/null || fail "the watcher did not step down through control-plane first"
641654

@@ -699,8 +712,10 @@ EOF
699712
wait_for "$(node_name 3) to read control-plane after a step down by hand" 180 at_level 3 control-plane
700713
wait_for "$(node_name 3) to be cordoned" 60 node_is 1 3 Ready,SchedulingDisabled
701714
[ "$(level_label 3)" = control-plane ] || fail "the Node does not carry the level label at control-plane"
702-
regrade_on 3 fleet-regrade down
715+
regrade_on 3 sh -c 'fleet-regrade down >/tmp/down.log 2>&1' || { regrade_on 3 cat /tmp/down.log; fail "the second step down by hand on $(node_name 3) failed"; }
716+
regrade_on 3 cat /tmp/down.log
703717
wait_for "$(node_name 3) to read etcd-only after a second step down by hand" 180 at_level 3 etcd-only
718+
assert_released 3 /tmp/down.log
704719
# The level is a file on the data volume that the entrypoint reads at every start, so neither a restart by hand nor a recreate, which is what Compose does when the role changes the container's configuration, brings the server back at full.
705720
docker restart "$(container 3)" >/dev/null
706721
wait_for "$(node_name 3) to be at etcd-only after a restart by hand" 300 at_level 3 etcd-only

0 commit comments

Comments
 (0)