From 5ce89f75556f8be15860772c0de8568af7b08165 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Tue, 20 Oct 2026 22:15:07 +0100 Subject: [PATCH] test(mesh): keep a stepped-down server at its level through a restart and a recreate --- roles/github_runner_cluster/README.md | 2 +- tests/mesh/run.sh | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 7dad84c..31de629 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -203,7 +203,7 @@ With `github_runner_cluster_regrade_step_up: automatic` (the default) the servic Steps are serialised across the whole cluster. A server starts a step only while it holds a Lease named `fleet-regrade` in `kube-system` and every other etcd-voting Node is Ready, so two servers never step at once, and a step never starts while a peer is down. A server at etcd-only is not Ready (it has no kubelet), which also means no other server steps down while one is etcd-only. The Lease expires on its own after `github_runner_cluster_regrade_lease_duration_seconds` if its holder never gives it back. Every call the service makes to the cluster goes to another server's API and never to the stepping server's own: at etcd-only the server has no API, and its own kubeconfig answers `apiserver disabled` on some servers. -A level is applied by the Docker actuator, which records it in a file on the k3s data volume, which the container's entrypoint reads at every start, and restarts the container when the change is between etcd-only and another level. Between `full` and `control-plane` nothing restarts, because both run the same k3s; the difference is the taint and cordon the service puts on the Node. The container is restarted, never created again, so the node's password and mesh identity are the ones the cluster already holds, and an Ansible run, which changes nothing a container is created from, cannot undo a step. A level survives the container being recreated and the Docker host restarting: a server stepped down stays down, with its cordon, until it is stepped up, whether by hand or, in `automatic`, by the service once memory has been calm. The entrypoint also records, on the container's tmpfs, the level k3s started at; that is what the service reads as the level the node is at, since k3s hides its arguments from `/proc`. A server whose entrypoint predates this feature is at `full` until it next restarts. +A level is applied by the Docker actuator, which records it in a file on the k3s data volume, which the container's entrypoint reads at every start, and restarts the container when the change is between etcd-only and another level. Between `full` and `control-plane` nothing restarts, because both run the same k3s; the difference is the taint and cordon the service puts on the Node. The container is restarted, never created again, so the node's password and mesh identity are the ones the cluster already holds, and an Ansible run, which changes nothing a container is created from, cannot undo a step. A level survives the container being recreated and the Docker host restarting: a server stepped down stays down, with its cordon, until it is stepped up, whether by hand or, in `automatic`, by the service once memory has been calm. That includes a `docker restart` by hand and an Ansible run that recreates the container (a changed image or Compose setting): the server comes back at the level it was left at, and `fleet-regrade up` is the way back to `full`. The entrypoint also records, on the container's tmpfs, the level k3s started at; that is what the service reads as the level the node is at, since k3s hides its arguments from `/proc`. A server whose entrypoint predates this feature is at `full` until it next restarts. A step that restarts the container returns only when etcd serves again, read from the container's log since it started, and, for a level that runs the agent, when the node reports Ready on a heartbeat from the new container. The Lease is released as soon as a step returns, so a step that returned early would let another server step on top of an etcd that has not come back. diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index f870c64..5ea35eb 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -547,6 +547,11 @@ wait_for() { done } +# Recreates a node's k3s container from its own Compose project, as the role does when it changes the container's configuration; the project's .env names the Compose files and the project. +recreate_node() { + docker compose --project-directory "${work}/$(node_name "$1")" --profile server up -d --force-recreate --no-deps k3s >/dev/null +} + # A write to the cluster, under a name of its own each time: a write the API server committed but did not answer is not repeated under the same name. write_marker() { kubectl_on 1 create configmap "grtest-vote-${RANDOM}" --from-literal=a=1 >/dev/null; } # A step by hand on node 3 is refused, and the refusal names the stepped-down server. Retried because node 3's own API may still be settling after it was stopped, which is a refusal too but for another reason; a step that goes ahead is a failure, not a retry. @@ -671,13 +676,28 @@ EOF [ "$(node_password_hash 2)" = "$password" ] || fail "$(node_name 2)'s node password changed" assert_cluster - log "By hand, a server goes down a level and up again through the same lease" + log "By hand, a server goes down to etcd-only, stays there through a restart and a recreate, and up is the way back" + password=$(node_password_hash 3) regrade_on 3 fleet-regrade down [ "$(level_of 3)" = control-plane ] || fail "$(node_name 3) did not step down by hand" wait_for "$(node_name 3) to be cordoned" 60 node_is 1 3 Ready,SchedulingDisabled [ "$(level_label 3)" = control-plane ] || fail "the Node does not carry the level label at control-plane" + regrade_on 3 fleet-regrade down + [ "$(level_of 3)" = etcd-only ] || fail "$(node_name 3) did not reach etcd-only by hand" + # The level is a file on the data volume that the entrypoint reads at every start, so neither a restart by hand nor a recreate, which is what Compose does when the role changes the container's configuration, brings the server back at full. + docker restart "$(container 3)" >/dev/null + wait_for "$(node_name 3) to be at etcd-only after a restart by hand" 300 at_level 3 etcd-only + before=$(docker inspect --format '{{.Id}}' "$(container 3)") + recreate_node 3 + [ "$before" != "$(docker inspect --format '{{.Id}}' "$(container 3)")" ] || fail "$(node_name 3)'s container was not recreated" + wait_for "$(node_name 3) to be at etcd-only after a recreate" 300 at_level 3 etcd-only + [ "$(level_label 3)" = etcd-only ] || fail "the Node lost its level label across the recreate" + [ "$(node_password_hash 3)" = "$password" ] || fail "$(node_name 3)'s node password changed across the recreate" + [ "$(etcd_member_ids 1)" = "$ids" ] || fail "the etcd members changed across the restart and the recreate" + regrade_on 3 fleet-regrade up + wait_for "$(node_name 3) to be Ready at control-plane" 300 node_is 1 3 Ready,SchedulingDisabled regrade_on 3 fleet-regrade up - wait_for "$(node_name 3) to be schedulable again" 120 node_is 1 3 Ready + wait_for "$(node_name 3) to be schedulable again" 300 node_is 1 3 Ready [ "$(level_of 3)" = full ] || fail "$(node_name 3) did not step up by hand" [ -z "$(level_label 3)" ] || fail "the level label is still on the Node at full"