diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b2a3eb8..46d4d05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,9 @@ jobs: ansible-galaxy collection build --output-path "${RUNNER_TEMP}/collection" tar -tzf "${RUNNER_TEMP}"/collection/*.tar.gz | grep -Fx roles/github_runner_cluster/files/docker-compose.yml tar -tzf "${RUNNER_TEMP}"/collection/*.tar.gz | grep -Fx roles/github_runner_cluster/files/k3s/node-entrypoint.sh + for file in Dockerfile fleet-regrade.sh fleet-regrade-core.sh fleet-regrade-docker.sh; do + tar -tzf "${RUNNER_TEMP}"/collection/*.tar.gz | grep -Fx "roles/github_runner_cluster/files/regrade/${file}" + done unit-tests: name: Unit tests (shard ${{ matrix.shard }}) diff --git a/.github/workflows/mesh-integration.yml b/.github/workflows/mesh-integration.yml index 824bbe0..8455c20 100644 --- a/.github/workflows/mesh-integration.yml +++ b/.github/workflows/mesh-integration.yml @@ -1,6 +1,6 @@ name: Mesh integration -# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. Only on changes that can affect how nodes join the mesh. +# Brings up three k3s nodes in Docker and joins them through Headscale with the cluster role (tests/mesh/run.sh), once per scenario: headscale_hosted, headscale_in_cluster, and the headscale_existing policy check, plus the role's refusal of a Docker Compose release that recreates the containers it has just built. headscale_in_cluster also runs with github_runner_cluster_docker_context pinned while the current Docker context points at nothing, which covers the cluster role's own docker calls and the recovery playbook's, and context_checks checks the role refuses a pin it cannot honour. regrade turns memory-pressure regrading on and steps a server down to etcd-only under (simulated) pressure and back up, checking that it keeps its etcd seat and vote and that no other server steps meanwhile. Only on changes that can affect how nodes join the mesh. # # Runs on GitHub-hosted ubuntu-latest, whose Docker daemon runs the privileged containers the test needs (k3s, and Tailscale inside it). The work directory sits under runner.temp, which the job always has write access to. @@ -37,6 +37,7 @@ jobs: - {scenario: existing, compose: v5.5.1} - {scenario: hosted, compose: v5.5.1} - {scenario: in_cluster, compose: v5.5.1} + - {scenario: regrade, compose: v5.5.1} - {scenario: hosted, compose: v2.39.0} - {scenario: compose_faulty, compose: v2.38.2} - {scenario: in_cluster, compose: v5.5.1, pin: true} diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..0111886 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,2 @@ +# The scripts shared with other repositories are sourced or run as bash without a shebang line. +shell=bash diff --git a/README.md b/README.md index e184096..4290dd3 100644 --- a/README.md +++ b/README.md @@ -137,9 +137,9 @@ Confirm it refreshes: `gh gist view "$HEARTBEAT_GIST_ID"` must show a timestamp #### Fleet-down notification -`runner-fallback-action` reroutes silently, so nothing otherwise tells a person that the fleet has no runners. `fleet-health.json` carries what a notifier needs to judge how long that has lasted: `at` (when the heartbeat last wrote), `unhealthy_since` (when it stopped vouching for the fleet, `null` while it does) and `not_ready_nodes` (each node whose Ready condition is not True, whether it is a control-plane server, and since when). The heartbeat carries each onset over from the document its previous tick published, read back from the gist, so a tick that cannot read it restarts the onsets at now, which delays an alert and never raises a false one. +`runner-fallback-action` reroutes silently, so nothing otherwise tells a person that the fleet has no runners. `fleet-health.json` carries what a notifier needs to judge how long that has lasted: `at` (when the heartbeat last wrote), `unhealthy_since` (when it stopped vouching for the fleet, `null` while it does) and `not_ready_nodes` (each node whose Ready condition is not True, whether it is a control-plane server, the `regrade_level` memory-pressure regrading has stepped it to if it has, and since when). The heartbeat carries each onset over from the document its previous tick published, read back from the gist, so a tick that cannot read it restarts the onsets at now, which delays an alert and never raises a false one. -`scripts/fleet-alert.sh` reads that document and announces an outage when any of these has held for at least `FLEET_ALERT_AFTER_SECONDS` (default 900, longer than the heartbeat's own 600 second window): the heartbeat has not written for that long, a control-plane server has not been Ready for that long, or the heartbeat has not vouched for the fleet for that long (a scale set missing or without a running listener, no ready controller, no Ready node, runner pods that cannot be scheduled; the reasons are in the notification). A worker node that is not Ready is not an outage by itself. An outage is announced once, and a recovery note follows when every condition has cleared. A heartbeat image older than the onset fields makes the script fail with a message saying so, rather than never alerting. +`scripts/fleet-alert.sh` reads that document and announces an outage when any of these has held for at least `FLEET_ALERT_AFTER_SECONDS` (default 900, longer than the heartbeat's own 600 second window): the heartbeat has not written for that long, a control-plane server has not been Ready for that long (except one regrading has stepped down, which is not Ready on purpose), or the heartbeat has not vouched for the fleet for that long (a scale set missing or without a running listener, no ready controller, no Ready node, runner pods that cannot be scheduled; the reasons are in the notification). A worker node that is not Ready is not an outage by itself. An outage is announced once, and a recovery note follows when every condition has cleared. A heartbeat image older than the onset fields makes the script fail with a message saying so, rather than never alerting. Delivery is independent per channel, and with none configured the script only prints what it found: @@ -221,6 +221,7 @@ Commits follow Conventional Commits (`fix:`, `feat:`, `tune:`, and so on). - **A global `--etcd-arg` override is not scoped to the "real" server's own etcd instance - it also applies to k3s's separate, internal "temporary etcd", spun up on every *restart* (never on first boot) to reconcile with an already-initialized on-disk datastore before the real instance starts.** An earlier `--etcd-arg="listen-peer-urls=https://0.0.0.0:2380"` (kept as a "harmless" belt-and-braces bind override after fixing the peer-URL issue above) forced that temporary instance onto the same TLS `0.0.0.0:2380` bind it has no certificates for by design (it normally listens on a local, ephemeral, non-TLS port). Confirmed live: this only broke on a restart against an already-initialized datastore, never on a fresh `--cluster-init`, which is exactly why it wasn't caught until well after the first working deploy - fatal: `"cannot listen on TLS for [::]:2380: KeyFile and CertFile are not presented"`. `--node-ip` alone (above) is sufficient; etcd's own default `listen-peer-urls` already derives from the same `PrivateIP` correctly, so there's nothing left for an explicit override to add. - **Calling `tailscale up --authkey=...` unconditionally on every container restart re-registers a brand new tailnet device each time, instead of reconnecting the persisted one - even with the join key's own state volume intact.** A reusable Tailscale auth key is designed to authorize provisioning *new* devices, and a naive `tailscale up` call on every restart (needed here to learn this node's own Tailscale IP before `exec`ing k3s - see above) doesn't check whether the daemon is already authenticated first, so it keeps minting new devices under names like `node1`, `node1-1`, `node1-2`... k3s's own `--vpn-auth` integration avoids exactly this by checking the Tailscale backend state first and skipping its own `tailscale up` call entirely once already `"Running"` (confirmed against `pkg/vpn/vpn.go`'s `StartVPN()`) - the manual pre-empt call needs the identical guard: check `tailscale ip -4` succeeds *before* calling `tailscale up` at all, only calling it when not already authenticated. - **Tailscale assigns an IP and wires up its own MagicDNS resolver as two separate steps, not atomically - a node having a real IP is not proof DNS resolution against another tailnet member is working yet.** A joining server needs to resolve the bootstrap host's own MagicDNS hostname (`K3S_JOIN_SERVER_URL`) before it can bootstrap against it at all. Confirmed live: k3s failed outright with `"dial tcp: lookup k3s-server.example.ts.net: no such host"` immediately after `tailscale ip -4` had already returned a real address for the *local* node. Fix: wait for a real `nslookup` success against the join server's own hostname specifically, not just local Tailscale readiness, before invoking k3s (the `k3s` service's entrypoint only does this when `K3S_JOIN_SERVER_URL` is actually set - the bootstrap host has nothing else to resolve). +- **A server that memory-pressure regrading has stepped to `etcd-only` is `NotReady` by design, and must stay registered.** With `github_runner_cluster_regrade` on, a server under sustained memory pressure steps down (`full`, `control-plane`, `etcd-only`) while it keeps its etcd member and vote; at `etcd-only` k3s runs with `--disable-apiserver --disable-controller-manager --disable-scheduler --disable-agent`, so there is no kubelet to keep its Node Ready, and the Node goes `NotReady` with `Unknown` conditions and stays so. That is not a failure to clear. Its Node object must not be deleted (k3s removes that server's etcd member with it), so anything that culls long-`NotReady` nodes must leave a Node carrying the `github-runner.exadev.com/regrade-level` label alone, and the role's wait for every node to be Ready already does. `kubectl` on such a server's own container is unreliable (its API server is off); the regrade service always reaches the cluster through another server. See the cluster role's README, Memory-pressure regrading. Confirmed against a three-server k3s cluster: the same etcd member ids before, during and after, the cluster taking writes with the stepped-down server as one of the two live voters, and the same Node and node password after stepping back up. - **A single remaining healthy etcd server can get stuck indefinitely if another member is registered but permanently unreachable (e.g. a partial join that never completed), even though that member is a non-voting learner and shouldn't block quorum.** Symptom: `"failed to publish local member to cluster through raft"` with `context deadline exceeded`, repeating forever, alongside `kubectl` returning `503 the server is currently unable to handle the request`. The fix is k3s's own documented recovery mechanism, run once against the affected node: stop the container, then `k3s server --cluster-reset` (with the SAME flags the real service uses, including `--node-ip` set to the node's own persisted Tailscale IP - getting this wrong re-registers the single remaining member under the *wrong* address, e.g. the invoking container's own ad-hoc bridge IP, and the real service then fails to start with `"this server is a not a member of the etcd cluster"`) - this resets *etcd cluster membership* back to a single member using the *existing local data*, it does not wipe anything. It's a one-shot operation: k3s writes a guard file after a successful reset and refuses to run `--cluster-reset` again until it's manually deleted (`rm /var/lib/rancher/k3s/server/db/reset-flag`) if a second reset is genuinely needed. After a reset, restart the service normally (without the flag) and clean up the stale member's own now-orphaned Kubernetes `Node` object and `node-password` Secret before it rejoins fresh. ## References diff --git a/galaxy.yml b/galaxy.yml index 119fb2a..4b5ac72 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -29,6 +29,7 @@ build_ignore: - .env* - .gitignore - .yamllint + - .shellcheckrc - AGENTS.md - CLAUDE.md - .ansible diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index 4c20bfa..7dad84c 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -185,6 +185,46 @@ A change reaches a server when the role recreates its container, which restarts An `--etcd-arg` is global: k3s also gives it to the temporary etcd it starts on every restart against an already-initialised datastore, never on a first boot. An earlier `listen-peer-urls` override broke exactly that restart, and only that one (see the first of the known issues in the [repository README](../../README.md) about `--etcd-arg`). These two arguments do not change what etcd binds to, but test a restart of an already-initialised datastore with them set before relying on them: `tests/mesh/run.sh` does that for the `hosted` scenario, restarting one server after the cluster is up. A timing that is too slow for the relay also lengthens how long a real leader failure takes to notice, so raise them only as far as the observed round trips need. +## Memory-pressure regrading + +A server on a laptop or a small board can run short of memory while it still holds an etcd vote. Without this the choices are to leave it running whole and risk the node, or to stop it and lose the vote. `github_runner_cluster_regrade: true` lets a server step down while it keeps its etcd seat, and step back up. It is off by default and changes nothing on a running node until it is turned on. + +There are three levels, from the most capable down: + +- `full`: the server as it runs: control plane, etcd and workloads. +- `control-plane`: still a server and an etcd voter, tainted, cordoned and drained, so it runs no workloads. +- `etcd-only`: k3s runs with its API server, controller manager, scheduler and agent switched off (`--disable-apiserver`, `--disable-controller-manager`, `--disable-scheduler` and `--disable-agent`), so the server holds its etcd member and little else. + +Giving up the vote is never part of it: a server that leaves etcd would have to be removed from the member list, which on a partition is unsafe, and k3s removes an etcd member when its Node object is deleted. The Node object of a stepped-down server stays, with its etcd role label. + +A `regrade` service runs beside the server's k3s container (in its own Compose profile, which the role adds when the variable is true). It reads the memory pressure of the Docker host's kernel, the `some` line's `avg300` of `/proc/pressure/memory`, and `MemAvailable`; both are needed, because pressure alone also rises on reclaim while plenty is available, and a low `MemAvailable` alone is normal once the page cache has filled. A server counts as over the limit when the pressure is at least `github_runner_cluster_regrade_pressure_threshold` percent and `MemAvailable` is under `github_runner_cluster_regrade_available_floor_percent` of `MemTotal`. It steps down one level after being over for `github_runner_cluster_regrade_sustain_seconds` without a reading under the limit in between, and waits `github_runner_cluster_regrade_cooldown_seconds` after any step before the next. The cooldown has to exceed the pressure window, or the next reading still holds data from before the step; the role refuses a shorter one. The defaults are multiples of the window and the sample period, so they follow if either changes. The threshold and the floor are reasoned starting points, not measurements: read the service's log with `docker compose --profile regrade logs regrade` on a real host and tune them. + +With `github_runner_cluster_regrade_step_up: automatic` (the default) the service also steps back up by itself, one level at a time. A server counts as calm when the pressure is under the threshold and `MemAvailable` is at or above the floor, both, which is stricter than not being over the limit: a reading between the two ends the streak towards stepping down and the streak towards stepping up alike, so the server does not step both ways on the same noise. It steps up after being calm for `github_runner_cluster_regrade_sustain_seconds`, once the cooldown since the last step, in either direction, has passed, through the same Lease and peer check as a step down; stepping up restarts the server's etcd at the first level that leaves etcd-only, so it never starts while a peer is down. A server that cannot hold the higher level cycles down and up no faster than once per cooldown and sustain period, which is the sign it is too small for the workloads it is given. `manual` leaves stepping up to `fleet-regrade up`. + +Steps are serialised across the whole cluster. A server starts a step only while it holds a Lease named `fleet-regrade` in `kube-system` and every other etcd-voting Node is Ready, so two servers never step at once, and a step never starts while a peer is down. A server at etcd-only is not Ready (it has no kubelet), which also means no other server steps down while one is etcd-only. The Lease expires on its own after `github_runner_cluster_regrade_lease_duration_seconds` if its holder never gives it back. Every call the service makes to the cluster goes to another server's API and never to the stepping server's own: at etcd-only the server has no API, and its own kubeconfig answers `apiserver disabled` on some servers. + +A level is applied by the Docker actuator, which records it in a file on the k3s data volume, which the container's entrypoint reads at every start, and restarts the container when the change is between etcd-only and another level. Between `full` and `control-plane` nothing restarts, because both run the same k3s; the difference is the taint and cordon the service puts on the Node. The container is restarted, never created again, so the node's password and mesh identity are the ones the cluster already holds, and an Ansible run, which changes nothing a container is created from, cannot undo a step. A level survives the container being recreated and the Docker host restarting: a server stepped down stays down, with its cordon, until it is stepped up, whether by hand or, in `automatic`, by the service once memory has been calm. The entrypoint also records, on the container's tmpfs, the level k3s started at; that is what the service reads as the level the node is at, since k3s hides its arguments from `/proc`. A server whose entrypoint predates this feature is at `full` until it next restarts. + +A step that restarts the container returns only when etcd serves again, read from the container's log since it started, and, for a level that runs the agent, when the node reports Ready on a heartbeat from the new container. The Lease is released as soon as a step returns, so a step that returned early would let another server step on top of an etcd that has not come back. + +The Node carries the label `github-runner.exadev.com/regrade-level` (`github_runner_cluster_regrade_level_label`) with the level as its value, from the first step down until the server is back at `full`. The role's wait for every node to be Ready leaves such a node out, so running the role against a fleet with a stepped-down server neither fails nor clears its node-password Secret. The heartbeat publishes the level beside each not-Ready node in `fleet-health.json`, and `scripts/fleet-alert.sh` does not count a stepped-down server as an outage; a heartbeat image that predates the field alerts on it as it would on any server that is not Ready, so update that image before turning regrading on. + +By hand, in the project directory (the `.env` the role writes names the Compose files and project, so these act on the right one): + +```sh +docker compose --profile regrade exec regrade fleet-regrade status +docker compose --profile regrade exec regrade fleet-regrade down # one step down +docker compose --profile regrade exec regrade fleet-regrade up # one step up +``` + +Both go through the same Lease and peer check as the watcher. + +The role refuses regrading where it could not work, before it changes anything: on an agent; where the server is the only one in the group, since a server at etcd-only needs another server's API; and on the bootstrap server of a `headscale_in_cluster` mesh, whose Headscale server is a static pod on the kubelet that etcd-only switches off. It also refuses to turn regrading off, or to remove the service, while the server is below `full`, because nothing else would step it up: step it up first, then run the role again. + +The service mounts the Docker socket, which is a permission cost: whatever runs in that container can start containers on the host with any mount. `docker-prune` has the same mount for the same reason, and the regrade service's scripts are mounted read-only from the project directory, not fetched at run time. Its Compose profile is separate from `server` and `agent`, so a host that does not enable regrading never builds or starts it. The image is built locally by Compose from `regrade/Dockerfile` (Alpine with bash, jq and the Docker CLI), the first time the service starts. + +The decision logic lives in `files/regrade/fleet-regrade-core.sh` and calls only three functions of an actuator: `levels`, `current_level` and `apply_level NAME`. The Docker actuator is `files/regrade/fleet-regrade-docker.sh`. Another platform supplies its own three functions and uses the core unchanged; a NixOS image can take the core from a pinned release of this collection. + ## Multiple sites Each site is its own cluster: its own inventory group, bootstrap server and mesh provider, run by its own play with `github_runner_cluster_group` set to that group (see `examples/two-site/`). One cluster spanning two meshes is not supported: flannel's tailscale backend carries pod traffic over the single mesh every node belongs to, so every node of a cluster must be on the same tailnet or Headscale server. An inventory host belongs to at most one cluster group; to run a second cluster on the same machines, give each machine a second inventory host (see [Several clusters on one host](#several-clusters-on-one-host)). @@ -218,6 +258,7 @@ Each Compose project runs its own `docker-prune` service against the same daemon - `github_runner_cluster_node_name`: the k3s node name and mesh hostname, defaulting to the inventory name. - `github_runner_cluster_node_taints`: Kubernetes taints (`key=value:Effect`, no spaces) the host's node registers with, empty by default. `NoSchedule` keeps pods without a matching toleration off it and `NoExecute` also evicts the ones already there. k3s applies them on every registration, so they come back if the node object is deleted, which a taint set with `kubectl` does not. - `github_runner_cluster_etcd_heartbeat_interval`, `github_runner_cluster_etcd_election_timeout`: etcd's timing in milliseconds for every server, empty by default (k3s's own). Set together, the same on every server, the election timeout at least five times the heartbeat interval; see [etcd timing](#etcd-timing). +- `github_runner_cluster_regrade` (default false): run the `regrade` service, which steps this server down under memory pressure while it keeps its etcd vote (see [Memory-pressure regrading](#memory-pressure-regrading)). Servers only. `github_runner_cluster_regrade_pressure_threshold` (percent, default 10) and `github_runner_cluster_regrade_available_floor_percent` (default 15) are the limit; `github_runner_cluster_regrade_step_up` (`automatic`, the default, or `manual`), `github_runner_cluster_regrade_sample_seconds`, `github_runner_cluster_regrade_sustain_seconds`, `github_runner_cluster_regrade_cooldown_seconds`, `github_runner_cluster_regrade_operation_timeout_seconds` and `github_runner_cluster_regrade_lease_duration_seconds` are the timing, all derived from `github_runner_cluster_regrade_pressure_window_seconds` (the kernel's `avg300`, 300 seconds, not a setting) unless set. `github_runner_cluster_regrade_level_label` and `github_runner_cluster_regrade_taint` are derived, never set directly. - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`. The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). - `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-at-login-macos)). @@ -266,6 +307,8 @@ or read from 1Password by adding `exadev.github_runner.github_runner_secrets_one `tests/mesh/run.sh` brings up three k3s nodes in Docker on one machine and joins them through Headscale with this role, for the `headscale_hosted` and `headscale_in_cluster` providers, and checks the `headscale_existing` policy check against a real server. Each cluster scenario runs the role a second time and fails if any node container restarted. Both set the etcd timing and check each server's etcd config file carries it, and the `hosted` scenario then restarts a joined server against its initialised datastore and checks the cluster recovers with the timing still in place. The `compose_faulty` scenario checks the role refuses a Compose release that has that fault. With `GRTEST_PIN_CONTEXT=1` the cluster scenarios pin `github_runner_cluster_docker_context` while the current Docker context the playbooks see points at a socket that does not exist, so any docker call that ignored the pin fails, and they check the current context is unchanged afterwards; `tests/litestream/run.sh` takes the same setting for the Headscale hosts it delegates to and the promotion playbook. The `context_checks` scenario checks the role refuses a pinned context that does not exist, and one set alongside `DOCKER_HOST`, before writing anything. `.github/workflows/mesh-integration.yml` runs them on pull requests that touch the role, the hosted scenario on 2.39.0, the first fixed release, as well as on a current one. +`tests/regrade` tests the regrading: the core's decision, stepping up by itself, and the Lease and peer coordination (`regrade-core.sh`, `regrade-recover.sh`, `regrade-lease.sh`) and the Docker actuator (`regrade-docker.sh`), all against stand-ins for `docker` and `kubectl`, run by `tests/unit/test_regrade_shell.py`; `tests/unit/test_cluster_regrade.py` covers the role's checks, the settings file, the Compose service, the levels the entrypoint starts k3s at and the tasks that remove the service. + `tests/unit/test_runtime_autostart.py` covers the [container runtime tasks](#container-runtime-at-login-macos) without a Mac: it renders the launchd agent through `tests/runtime_autostart/render_plist.yml`, and runs `tasks/runtime.yml` and `tasks/end_checks.yml` through the role against stand-ins for `docker`, `colima`, `launchctl` and `plutil` on `PATH`. It checks the variable defaults, that Linux hosts are left alone, that a second run changes nothing, that `--check` writes and starts nothing, and that each way the runtime or the node's container can be down fails with its message. `tests/unit/test_container_runtime.py` covers the filter that works out the runtime from the Docker context. Nothing there touches a real Colima, launchd or Docker Desktop, so loading the agent into a real login session at a real reboot is the part only a run on a Mac can show. `tests/unit/test_node_password.py` covers [the node password volume and the servers' turns](#node-passwords-and-recreating-servers) without Docker: the filters in `plugins/filter/cluster_recreate.py`, the Compose file's volumes, and `tasks/node_password.yml`, `tasks/server_recreate_check.yml`, `tasks/server_ready.yml` and `tasks/start_servers.yml` run through the role (`tests/node_password/run.yml`) against a stand-in for `docker` that moves a real tar stream between its `docker cp` processes and keeps containers, volumes and node readiness as files, and a stand-in for the Compose module that recreates a container whose rendered hash differs and takes its node out of Ready until the role has waited for it. It checks the seeding (mode, ownership, labels, no copy of the password anywhere on the host, a failed copy leaving no volume), that `--check` changes nothing and reports each recreate, that a server is refused while another is not Ready and that agents are not, and from the order of the calls that the turns do not overlap. The `hosted` and `in_cluster` mesh scenarios recreate a joined server in Docker (`recreate_a_joined_server` in `tests/mesh/run.sh`) and check that its node rejoins with the same password and Secret, that no other container restarts, and that the role's `--check` report names exactly that server. diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 3f53ce3..d5e5ce9 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -28,6 +28,31 @@ github_runner_cluster_node_taints: [] github_runner_cluster_etcd_heartbeat_interval: "" github_runner_cluster_etcd_election_timeout: "" +# Memory-pressure regrading (see the role README): a server steps down through three levels when memory runs short, and back up, without ever leaving etcd. full is the server as it runs; control-plane is still a server and an etcd voter, cordoned and drained; etcd-only has its API server, controller manager, scheduler and agent switched off, and holds the node's etcd seat and nothing else. Giving up the seat is never automatic. false (the default) installs and changes nothing. true runs the `regrade` Compose service beside this server's k3s container, a server host only, with the Docker socket mounted (the same permission cost as the docker-prune service). The run refuses it where a step down could not work: an agent, a cluster with no other server in the inventory to keep the API up, and the bootstrap server of a headscale_in_cluster mesh, whose Headscale server runs on the agent that etcd-only switches off. +github_runner_cluster_regrade: false +# Whether the service also steps up by itself. automatic (the default) goes back up one level at a time once the server has been calm, that is the pressure under the threshold and MemAvailable at or above the floor, for the sustain period, and the cooldown since the last step has passed; it takes the same Lease and checks the same peers as a step down. manual leaves stepping up to `fleet-regrade up`. Stepping down is always automatic, and giving up the etcd vote never is. +github_runner_cluster_regrade_step_up: automatic +# The window of the longest pressure average the kernel reports (avg300 in /proc/pressure/memory), in seconds. A fact about the kernel, not a setting: every default below that is a time is a multiple of it or of the sample period, so that a reading always spans the level it describes. +github_runner_cluster_regrade_pressure_window_seconds: 300 +# The "some" avg300 of /proc/pressure/memory, in percent, at or above which the server counts as under pressure: for that share of the last five minutes some task was stalled waiting for memory, a rate at which etcd's heartbeat and election timers, which run in tenths of a second, are being missed. A reasoned starting point, not a measurement: read the service's log (`docker compose --profile regrade logs regrade`) on a real host and tune it. +github_runner_cluster_regrade_pressure_threshold: 10 +# MemAvailable, as a percentage of MemTotal, below which a pressure reading counts. It guards against stepping down on pressure caused by reclaim while plenty is available, and, as a share rather than a number of bytes, is right for any host. Chosen, not measured. +github_runner_cluster_regrade_available_floor_percent: 15 +# How often the pressure is read: a tenth of the pressure window, so a rising average is noticed well inside it and the watcher itself stays idle almost all the time. +github_runner_cluster_regrade_sample_seconds: "{{ (github_runner_cluster_regrade_pressure_window_seconds | int / 10) | int }}" +# How long the reading must stay over the limit before a step. The average already smooths five minutes, so this only has to ride out a single noisy sample: four consecutive samples. +github_runner_cluster_regrade_sustain_seconds: "{{ 4 * (github_runner_cluster_regrade_sample_seconds | int) }}" +# How long to wait after a step before another is allowed. It has to exceed the pressure window, or the next reading still holds data from before the step; three windows leave the whole of one to settle after the restart and another for the new level to be measured. The role refuses a value under one window. +github_runner_cluster_regrade_cooldown_seconds: "{{ 3 * (github_runner_cluster_regrade_pressure_window_seconds | int) }}" +# The longest a drain, a restart of the k3s container and the wait for it to serve again may each take before the step fails. A failed drain stops the step with the server where it was. +github_runner_cluster_regrade_operation_timeout_seconds: "{{ github_runner_cluster_regrade_pressure_window_seconds }}" +# How long the cluster-wide Lease that serialises steps lasts if its holder never gives it back. A step is a drain and then a restart that waits for etcd and the node, each bounded by the operation timeout, so a holder that is working never loses the Lease mid-step; a holder that crashed blocks the other servers for no longer than this. +github_runner_cluster_regrade_lease_duration_seconds: "{{ 3 * (github_runner_cluster_regrade_operation_timeout_seconds | int) }}" +# Derived, never set directly: the label a server's Node carries while regrading has it below full, with the level as its value. The role's wait for every node to be Ready leaves such a node out, since it runs no kubelet at etcd-only, and anything else that judges nodes by their readiness can do the same. The domain is the one the deployed-version annotation uses. +github_runner_cluster_regrade_level_label: "github-runner.exadev.com/regrade-level" +# Derived, never set directly: the standard control-plane taint a server carries while it runs no workloads. The core applies it with the cordon on the first step down and removes it on the way back to full; the NoSchedule effect is what it removes. +github_runner_cluster_regrade_taint: "node-role.kubernetes.io/control-plane=true:NoSchedule" + # The images every node's k3s container is built from: the rancher/k3s tag (k3s's version with "+" written as "-") and the tailscale/tailscale tag whose tailscale and tailscaled binaries are copied in. Pinned, never latest: Compose builds a node's image only when it is missing, so a floating tag would put whatever version is newest on the day a node is first built, rebuilt or moved into a running cluster, unannounced. Set both once for the whole cluster group, in inventory or group_vars (the check below reads every host's through hostvars). Before it changes anything on a host the role fails if either is not a pinned tag, if the group's hosts name different k3s minor versions, or if the k3s version would break k3s's upgrade path or Kubernetes' version skew policy from what this node and the cluster run now (see the role README's Versions section). Changing them changes no running node: its image is kept until it is rebuilt (see the role README's Versions section for upgrading). github_runner_cluster_k3s_version: "{{ github_runner_cluster_k3s_tested_version }}" github_runner_cluster_tailscale_version: "v1.102.4" diff --git a/roles/github_runner_cluster/files/docker-compose.yml b/roles/github_runner_cluster/files/docker-compose.yml index 0e6d2ae..43232ee 100644 --- a/roles/github_runner_cluster/files/docker-compose.yml +++ b/roles/github_runner_cluster/files/docker-compose.yml @@ -126,6 +126,20 @@ services: sleep "$${DOCKER_PRUNE_INTERVAL_SECONDS}" done + # Steps this server down through levels of reduced function when memory runs short, so a small host keeps its place in etcd's quorum instead of being lost to the OOM killer, and back up again; see the role README's Memory-pressure regrading. The core (regrade/fleet-regrade-core.sh) reads memory pressure from the Docker VM's /proc and decides; the Docker actuator (regrade/fleet-regrade-docker.sh) applies a level by recording it in the k3s container and restarting that container, so this service needs the Docker socket, the same permission cost as docker-prune above and for the same reason: it acts on the host's other containers. The scripts and the settings (regrade/fleet-regrade.env, templated by the role) are mounted, not built in, so a collection upgrade reaches the service when the role restarts it, without an image Compose would not otherwise rebuild. Selected by its own profile, which the role adds only when github_runner_cluster_regrade is true. + regrade: + build: + context: ./regrade + restart: unless-stopped + profiles: ["regrade"] + # Keeps the watchdog alive when memory is short, which is when it is needed. Matches the NixOS module's watchdog; -1000 would exempt it from the OOM killer altogether. + oom_score_adj: -900 + environment: + - FLEET_REGRADE_ENV=/usr/local/lib/fleet-regrade/fleet-regrade.env + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./regrade:/usr/local/lib/fleet-regrade:ro + volumes: k3s-data: {} k3s-agent-data: {} diff --git a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh index 6d02a1c..4eb602e 100755 --- a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh +++ b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh @@ -37,6 +37,25 @@ set_api_port_flags() { } set_api_port_flags +# Sets regrade_flags to the arguments that put this server at the level the regrade service (docker-compose.yml, files/regrade) last recorded for it, and records the level k3s is about to start at. A server that has never been stepped down has no record and starts at full. Only etcd-only changes k3s's arguments: it turns off the API server, controller manager, scheduler and agent, so the node keeps its etcd member, and its vote, and little else. full and control-plane run the same k3s and differ in the cordon and taint the regrade service puts on the Node. +# +# The regrade service writes regrade_level_file, on the k3s data volume, before it restarts the container, so the level survives the container being recreated and the host restarting; a level is left only by stepping up. regrade_started_file is on the container's tmpfs, so it exists only while this container runs and says what its k3s started at, which the regrade service reads as the level the node is at: k3s hides its arguments from /proc, so they cannot be read back. The paths are the regrade service's too (fleet-regrade-docker.sh); the environment variables replace them only in tests. +regrade_level_file=${REGRADE_LEVEL_FILE:-/var/lib/rancher/k3s/regrade-level} +regrade_started_file=${REGRADE_STARTED_FILE:-/run/regrade-level} +# Empty unless set_regrade_flags says otherwise, which only the plain server start below calls: a bootstrap server that supervises k3s for an in-cluster Headscale server is never regraded, since etcd-only would stop the kubelet that runs it (the role refuses that combination). +regrade_flags="" + +set_regrade_flags() { + regrade_level=full + if [ -r "$regrade_level_file" ]; then + read -r regrade_level < "$regrade_level_file" + fi + if [ "$regrade_level" = etcd-only ]; then + regrade_flags="--disable-apiserver --disable-controller-manager --disable-scheduler --disable-agent" + fi + printf '%s\n' "$regrade_level" > "$regrade_started_file" +} + # k3s's own vpn-auth integration only runs `tailscale up`; it does not start the daemon itself (without this, k3s fails outright with "tailscale up failed: ... it doesn't appear to be running"). Waits for the control socket before anything talks to it. mkdir -p /var/run/tailscale /var/lib/tailscale tailscaled --state=/var/lib/tailscale/tailscaled.state --socket=/var/run/tailscale/tailscaled.sock & @@ -132,6 +151,8 @@ install_tailscale_up_wait() { # --tls-san is added once per entry of K3S_TLS_SAN_LIST: every server's certificate must cover every server's own mesh name, since a client reaching any server directly by that name needs the certificate to match it. # # --etcd-arg=heartbeat-interval and --etcd-arg=election-timeout are added when set (set_etcd_timing_flags above), for peers whose traffic goes through a relay and so exceeds etcd's default timing. +# +# regrade_flags (set_regrade_flags above) are added last, and are empty at every level but etcd-only. mesh_server_args() { set -- --disable=traefik --disable-network-policy --write-kubeconfig-mode=6443 \ --vpn-auth="$vpn_auth" \ @@ -145,6 +166,7 @@ mesh_server_args() { fi for san in ${K3S_TLS_SAN_LIST:-}; do set -- "$@" --tls-san "$san"; done for flag in $etcd_timing_flags; do set -- "$@" "$flag"; done + for flag in $regrade_flags; do set -- "$@" "$flag"; done server_args="$*" } @@ -207,6 +229,7 @@ case "$role" in wait_for_mesh_host "$server_join_host" install_tailscale_up_wait "$server_join_host" "$tailscale_up_wait_dir" /usr/local/bin/tailscale rm -f /output/kubeconfig.yaml + set_regrade_flags mesh_server_args # shellcheck disable=SC2086 # As above. PATH="$tailscale_up_wait_dir:$PATH" exec /bin/k3s server $server_args $node_taint_flags diff --git a/roles/github_runner_cluster/files/regrade/Dockerfile b/roles/github_runner_cluster/files/regrade/Dockerfile new file mode 100644 index 0000000..2256529 --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/Dockerfile @@ -0,0 +1,8 @@ +# The regrade service of the cluster role's Compose file (see docker-compose.yml): bash, jq and the Docker CLI, all from Alpine's apk. The scripts it runs are not copied in: Compose mounts this directory, so a host picks up a collection upgrade when the service restarts, without an image Compose would not otherwise rebuild. +FROM alpine:3.22 + +RUN apk add --no-cache bash jq docker-cli \ + && ln -s /usr/local/lib/fleet-regrade/fleet-regrade.sh /usr/local/bin/fleet-regrade + +ENTRYPOINT ["/usr/local/bin/fleet-regrade"] +CMD ["watch"] diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh new file mode 100644 index 0000000..2257c47 --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-core.sh @@ -0,0 +1,372 @@ +# The platform-neutral core of fleet-regrade: it steps a k3s server down through levels of reduced function when memory runs short, and back up: by hand, or by itself once memory has been calm for long enough (STEP_UP, below). Nothing here knows how a level is entered; that is the actuator's job, so the same core drives a NixOS node (modules/fleet-regrade-nix.sh) and, later, a server that runs in Docker. It has no Nix store paths and no NixOS assumptions, so it can move to another repository unchanged. +# +# THE ACTUATOR CONTRACT. The file that is concatenated after this one, or sources it, defines exactly three functions and then calls `regrade_main "$@"`: +# +# levels prints the level names, one per line, from the most capable to the most reduced. The first is the level the node starts in and is restored to; a reboot returns to it. +# +# current_level prints the name of the level the node is running at now, read from the system itself, or "unknown" if it is none of them. +# +# apply_level NAME makes the node run at NAME, by whatever means the platform has, and returns non-zero if it could not. It is called with the node's workloads already drained when stepping down, and it is the actuator's business to restart whatever has to restart. +# +# The core owns the coordination between servers too: a step starts only while the node holds a Lease and every other etcd-voting node is Ready, so two servers never step at once and a step never starts while a peer is down. +# +# The core owns the signals (memory pressure and available memory), the decision (over the limit for long enough, one step at a time, a cooldown between steps, and, when STEP_UP is automatic, calm for long enough to go back up one level), shedding and restoring workloads with kubectl, waiting for the API server, and the command line. The sustain period is the hysteresis: a reading under the limit ends a streak, so a node has to be over for the whole period, with nothing under it, before a step down, and calm for the whole period, with nothing over it, before a step up. Calm is stricter than not being over: pressure under the threshold and MemAvailable at or above the floor, both, where over needs both to be bad, so a reading between the two states ends both streaks and a node does not step down and up on the same noise. The cooldown after a step, in either direction, is longer than the pressure window, so a node that cannot hold a higher level cycles no faster than once per cooldown and sustain period. +# +# STEP_UP is optional: `automatic` lets the watcher step up one level at a time, `manual` (the default when it is unset, which is what the core did before it existed) leaves stepping up to `fleet-regrade up`. Giving up an etcd vote is not a level, and nothing here ever does it. +# +# It needs bash, awk, jq and kubectl. Configuration comes from the environment file named by FLEET_REGRADE_ENV; the paths below default to the real system and are overridden only by tests. + +FLEET_REGRADE_ENV=${FLEET_REGRADE_ENV:-/etc/fleet-regrade.env} +PSI_FILE=${PSI_FILE:-/proc/pressure/memory} +MEMINFO_FILE=${MEMINFO_FILE:-/proc/meminfo} +UPTIME_FILE=${UPTIME_FILE:-/proc/uptime} + +# Where the cluster keeps its own leases, and the name that identifies this protocol among them. +LEASE_NAMESPACE=${LEASE_NAMESPACE:-kube-system} +LEASE_NAME=${LEASE_NAME:-fleet-regrade} + +# The label k3s puts on the nodes that are etcd members, which are the nodes whose health a step must not stack on top of. +VOTER_LABEL=${VOTER_LABEL:-node-role.kubernetes.io/etcd=true} + +# shellcheck source=/dev/null +[ -f "$FLEET_REGRADE_ENV" ] && . "$FLEET_REGRADE_ENV" + +# Everything goes to stdout, which the service manager keeps as the log. +log() { + printf '%s\n' "$*" +} + +die() { + printf '%s\n' "$*" >&2 + exit 1 +} + +# The 300-second average of the share of time some task was stalled on memory, in percent. +psi_avg300() { + awk '$1 == "some" { for (i = 2; i <= NF; i++) if ($i ~ /^avg300=/) { sub(/^avg300=/, "", $i); print $i } }' "$PSI_FILE" +} + +meminfo_kb() { + awk -v key="$1:" '$1 == key { print $2 }' "$MEMINFO_FILE" +} + +uptime_seconds() { + awk '{ print int($1) }' "$UPTIME_FILE" +} + +# Succeeds when pressure is at or above the threshold and MemAvailable is below the floor, a share of MemTotal. Both are needed: pressure alone also rises on reclaim while plenty is available, and a low MemAvailable alone is normal once the page cache has filled. +is_over() { + local pressure=$1 available_kb=$2 total_kb=$3 + awk -v p="$pressure" -v t="$PRESSURE_THRESHOLD" -v a="$available_kb" -v m="$total_kb" -v f="$AVAILABLE_FLOOR_PERCENT" \ + 'BEGIN { exit !(p + 0 >= t + 0 && a * 100 < m * f) }' +} + +# Succeeds when the reading is calm: pressure under the threshold and MemAvailable at or above the floor. It is not the opposite of is_over, which needs both signals to be bad: a node between the two states is neither, and a streak of either ends there. +is_calm() { + local pressure=$1 available_kb=$2 total_kb=$3 + awk -v p="$pressure" -v t="$PRESSURE_THRESHOLD" -v a="$available_kb" -v m="$total_kb" -v f="$AVAILABLE_FLOOR_PERCENT" \ + 'BEGIN { exit !(p + 0 < t + 0 && a * 100 >= m * f) }' +} + +# Succeeds when the reading has held (over, to step down; calm, to step up) for the sustain period and the cooldown since the last step has passed. since and last_step are uptime seconds, empty when unset. +may_step() { + local now=$1 since=$2 last_step=$3 + [ -n "$since" ] || return 1 + [ $((now - since)) -ge "$SUSTAIN_SECONDS" ] || return 1 + [ -z "$last_step" ] || [ $((now - last_step)) -ge "$COOLDOWN_SECONDS" ] +} + +# The actuator's levels as an array, most capable first. +load_levels() { + local level + LEVELS=() + while IFS= read -r level; do + [ -z "$level" ] || LEVELS+=("$level") + done < <(levels) + [ "${#LEVELS[@]}" -gt 0 ] || die "The actuator reports no levels." +} + +level_position() { + local i + for i in "${!LEVELS[@]}"; do + if [ "${LEVELS[$i]}" = "$1" ]; then + echo "$i" + return 0 + fi + done + return 1 +} + +next_level_down() { + local position + position=$(level_position "$1") || return 1 + [ $((position + 1)) -lt "${#LEVELS[@]}" ] || return 1 + echo "${LEVELS[$((position + 1))]}" +} + +next_level_up() { + local position + position=$(level_position "$1") || return 1 + [ "$position" -gt 0 ] || return 1 + echo "${LEVELS[$((position - 1))]}" +} + +# kubectl against the cluster, as an administrator. KUBECTL may be more than one word (`k3s kubectl`). KUBE_SERVER, when set, names the API server to use instead of the one in the kubeconfig: a node that has switched its own API server off still has to reach the cluster's. +kubectl_() { + # shellcheck disable=SC2086 # KUBECTL is a command that may take a subcommand, so it is split on purpose. + $KUBECTL --kubeconfig "$KUBECONFIG_PATH" ${KUBE_SERVER:+--server "$KUBE_SERVER"} "$@" +} + +# Taints, cordons and evicts everything but daemonsets, while the API server is still up. Both steps are idempotent, so each step down runs it again before anything is switched off. The taint is applied here and not left to the node's own configuration, which takes effect only when a node registers. --force evicts pods that no controller owns too: the node is shedding load, and a pod that blocks the drain would stop the step. +shed_workloads() { + kubectl_ taint nodes "$NODE_NAME" "$CONTROL_PLANE_TAINT" --overwrite + kubectl_ drain "$NODE_NAME" --ignore-daemonsets --delete-emptydir-data --force --timeout="${OPERATION_TIMEOUT_SECONDS}s" +} + +restore_workloads() { + kubectl_ taint nodes "$NODE_NAME" "${CONTROL_PLANE_TAINT%%=*}:NoSchedule-" || true + kubectl_ uncordon "$NODE_NAME" +} + +wait_for_api() { + local deadline=$((SECONDS + OPERATION_TIMEOUT_SECONDS)) + until kubectl_ get --raw=/readyz > /dev/null 2>&1; do + [ "$SECONDS" -lt "$deadline" ] || die "The API server did not become ready within ${OPERATION_TIMEOUT_SECONDS}s." + sleep 2 + done +} + +# The time as epoch seconds, replaced by tests. +now_epoch() { + date +%s +} + +# A Lease timestamp (RFC 3339 with microseconds) for an epoch time, built by jq so that it is the same on every platform. +micro_time() { + jq -nr --argjson t "$1" '$t | todate | sub("Z$"; ".000000Z")' +} + +# The Lease as the API server stores it: held by $1, acquired at epoch $2, last renewed at epoch $3, for LEASE_DURATION_SECONDS. A fourth argument is the resourceVersion of the Lease being replaced, which makes the replacement fail if anyone else has changed it since it was read. +build_lease() { + jq -n --arg name "$LEASE_NAME" --arg namespace "$LEASE_NAMESPACE" --arg holder "$1" \ + --arg acquired "$(micro_time "$2")" --arg renewed "$(micro_time "$3")" \ + --argjson duration "$LEASE_DURATION_SECONDS" --arg version "${4:-}" ' + { + apiVersion: "coordination.k8s.io/v1", + kind: "Lease", + metadata: ({ name: $name, namespace: $namespace } + (if $version == "" then {} else { resourceVersion: $version } end)), + spec: { holderIdentity: $holder, leaseDurationSeconds: $duration, acquireTime: $acquired, renewTime: $renewed } + }' +} + +# What a Lease, given as JSON, means to node $2 at epoch time $3: free (nobody holds it), expired (a holder whose time has run out), mine (unexpired and held by $2) or held (unexpired and held by another). A Lease lasts until its renew time plus its duration, and is expired from that moment. +lease_state() { + jq -r --arg me "$2" --argjson now "$3" ' + (.spec.holderIdentity // "") as $holder + | (.spec.renewTime // .spec.acquireTime // null) as $renewed + | (.spec.leaseDurationSeconds // 0) as $duration + | if $holder == "" then "free" + elif $renewed == null then "expired" + elif (($renewed | sub("\\.[0-9]+Z$"; "Z") | fromdateiso8601) + $duration) <= $now then "expired" + elif $holder == $me then "mine" + else "held" end' <<< "$1" +} + +# Takes the Lease, or fails. Created when there is none, replaced when it is free, expired or already this node's, refused when another holds it, and refused when the API server cannot be asked, since a step must not start on a guess. Both writes fail if someone else got there first. +acquire_lease() { + : "${LEASE_DURATION_SECONDS:?LEASE_DURATION_SECONDS must be set}" + local current now state holder version + now=$(now_epoch) + if ! current=$(kubectl_ get lease "$LEASE_NAME" --namespace "$LEASE_NAMESPACE" --output json 2>&1); then + case $current in + *"(NotFound)"*) + if build_lease "$NODE_NAME" "$now" "$now" | kubectl_ create -f - > /dev/null; then + return 0 + fi + log "cannot step: another node created the lease first" + return 1 + ;; + *) + log "cannot step: the lease cannot be read: $current" + return 1 + ;; + esac + fi + state=$(lease_state "$current" "$NODE_NAME" "$now") + if [ "$state" = held ]; then + holder=$(jq -r '.spec.holderIdentity' <<< "$current") + log "cannot step: the lease is held by $holder" + return 1 + fi + version=$(jq -r '.metadata.resourceVersion' <<< "$current") + if build_lease "$NODE_NAME" "$now" "$now" "$version" | kubectl_ replace -f - > /dev/null; then + return 0 + fi + log "cannot step: the lease changed while it was being taken" + return 1 +} + +# Gives the Lease back, if it is still this node's: the holder is cleared, so the next node does not wait for it to expire. If it has meanwhile passed to another node it is left alone. +release_lease() { + local current + current=$(kubectl_ get lease "$LEASE_NAME" --namespace "$LEASE_NAMESPACE" --output json) || return 1 + [ "$(jq -r '.spec.holderIdentity // ""' <<< "$current")" = "$NODE_NAME" ] || return 0 + jq '.spec.holderIdentity = ""' <<< "$current" | kubectl_ replace -f - > /dev/null +} + +# Succeeds when every other etcd-voting node is Ready. A node whose state is unknown, that has no Ready condition, or that cannot be listed counts as not ready: a step never starts while a peer might be down. +peers_ready() { + local nodes not_ready + if ! nodes=$(kubectl_ get nodes --selector "$VOTER_LABEL" --output json); then + log "cannot step: the etcd-voting nodes cannot be listed" + return 1 + fi + not_ready=$(jq -r --arg me "$NODE_NAME" ' + [ .items[] | select(.metadata.name != $me) + | select(([.status.conditions[]? | select(.type == "Ready") | .status] | first // "Unknown") != "True") + | .metadata.name ] | join(" ")' <<< "$nodes") || return 1 + if [ -n "$not_ready" ]; then + log "cannot step: etcd-voting nodes not ready: $not_ready" + return 1 + fi +} + +# Runs a command only while this node may transition: its peers are Ready and it holds the Lease. The Lease is released afterwards whether or not the command succeeded, and the command's status is the result. +coordinated() { + local status=0 + peers_ready || return 1 + acquire_lease || return 1 + "$@" || status=$? + release_lease || log "the lease could not be released; it expires on its own" + return "$status" +} + +# One step down. Workloads are drained before the level is applied, and a failed drain stops the step, so nothing that needs the API server is switched off with pods still on the node. +step_down() { + local from=$1 to=$2 reading=$3 + log "step down $from -> $to: $reading" + if ! shed_workloads; then + log "drain failed, staying at $from" + return 1 + fi + if ! apply_level "$to"; then + log "applying $to failed, level is now $(current_level)" + return 1 + fi + log "now at $to" +} + +# One step up, by hand or by the watcher. Returning to the first level also lifts the taint and uncordons the node, once the API server answers. +step_up() { + local from=$1 to=$2 + log "step up $from -> $to" + if ! apply_level "$to"; then + log "applying $to failed, level is now $(current_level)" + return 1 + fi + if [ "$to" = "${LEVELS[0]}" ]; then + wait_for_api + restore_workloads + fi + log "now at $to" +} + +reading_text() { + printf 'memory some avg300=%s%% (threshold %s%%), MemAvailable %s kB of %s kB (floor %s%%)' \ + "$1" "$PRESSURE_THRESHOLD" "$2" "$3" "$AVAILABLE_FLOOR_PERCENT" +} + +# One look at the signals. The streaks and the time of the last step are kept in WATCH_ABOVE_SINCE, WATCH_CALM_SINCE and WATCH_LAST_STEP, which a loop (or a test) carries from one sample to the next. +WATCH_ABOVE_SINCE='' +WATCH_CALM_SINCE='' +WATCH_LAST_STEP='' +WATCH_WARNED='' +watch_sample() { + local level next previous pressure available total now text + level=$(current_level) + if [ "$level" = unknown ]; then + [ -n "$WATCH_WARNED" ] || log "the running system is not one of the levels; not stepping" + WATCH_WARNED=1 + return 0 + fi + WATCH_WARNED='' + pressure=$(psi_avg300) + available=$(meminfo_kb MemAvailable) + total=$(meminfo_kb MemTotal) + now=$(uptime_seconds) + text=$(reading_text "$pressure" "$available" "$total") + if is_over "$pressure" "$available" "$total"; then + if [ -z "$WATCH_ABOVE_SINCE" ]; then + WATCH_ABOVE_SINCE=$now + log "level $level: over the limit: $text" + fi + elif [ -n "$WATCH_ABOVE_SINCE" ]; then + WATCH_ABOVE_SINCE='' + log "level $level: back under the limit: $text" + fi + if is_calm "$pressure" "$available" "$total"; then + if [ -z "$WATCH_CALM_SINCE" ]; then + WATCH_CALM_SINCE=$now + if [ "${STEP_UP:-manual}" = automatic ] && next_level_up "$level" > /dev/null; then + log "level $level: calm: $text" + fi + fi + else + WATCH_CALM_SINCE='' + fi + if next=$(next_level_down "$level") && may_step "$now" "$WATCH_ABOVE_SINCE" "$WATCH_LAST_STEP"; then + coordinated step_down "$level" "$next" "$text" || true + WATCH_LAST_STEP=$(uptime_seconds) + WATCH_ABOVE_SINCE='' + WATCH_CALM_SINCE='' + elif [ "${STEP_UP:-manual}" = automatic ] && previous=$(next_level_up "$level") && may_step "$now" "$WATCH_CALM_SINCE" "$WATCH_LAST_STEP"; then + coordinated step_up "$level" "$previous" || true + WATCH_LAST_STEP=$(uptime_seconds) + WATCH_CALM_SINCE='' + fi +} + +cmd_watch() { + load_levels + while true; do + sleep "$SAMPLE_SECONDS" + watch_sample + done +} + +cmd_status() { + load_levels + log "level: $(current_level) (of: ${LEVELS[*]})" + log "$(reading_text "$(psi_avg300)" "$(meminfo_kb MemAvailable)" "$(meminfo_kb MemTotal)")" + log "sample ${SAMPLE_SECONDS}s, sustained for ${SUSTAIN_SECONDS}s, cooldown ${COOLDOWN_SECONDS}s, step up ${STEP_UP:-manual}" +} + +# One step down by hand, for trying a level without waiting for pressure. +cmd_down() { + local level next + load_levels + level=$(current_level) + [ "$level" != unknown ] || die "The running system is not one of the levels." + next=$(next_level_down "$level") || die "Already at $level, the most reduced level." + coordinated step_down "$level" "$next" "by hand" +} + +# One step up by hand. +cmd_up() { + local level next + load_levels + level=$(current_level) + [ "$level" != unknown ] || die "The running system is not one of the levels." + next=$(next_level_up "$level") || die "Already at $level, the first level." + coordinated step_up "$level" "$next" +} + +regrade_main() { + case ${1:-} in + watch) cmd_watch ;; + status) cmd_status ;; + up) cmd_up ;; + down) cmd_down ;; + *) die "Usage: fleet-regrade watch|status|up|down" ;; + esac +} diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh new file mode 100644 index 0000000..03c222e --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade-docker.sh @@ -0,0 +1,119 @@ +# The Docker actuator for fleet-regrade, run by the regrade service of the cluster role's Compose file. The core, fleet-regrade-core.sh, is placed ahead of this file (concatenated, or sourced by the wrapper beside it); this file supplies the three functions of the actuator contract that the core describes, and runs it. +# +# The levels are ways of running the one k3s container of this server. full and control-plane run the same k3s and differ only in the cordon and taint the core puts on the Node, so entering either from the other restarts nothing. etcd-only runs k3s with its API server, controller manager, scheduler and agent turned off: the server keeps its etcd member, and its vote, and nothing else, which it needs the container's entrypoint to start differently, so entering or leaving it restarts the container. +# +# A level is recorded in the container, where the entrypoint (files/k3s/node-entrypoint.sh, which uses the same two paths) reads it at every start. LEVEL_FILE is on the k3s data volume: it survives the container being recreated and the host restarting, so a node stepped down stays stepped down until it is stepped up, and an Ansible run, which changes nothing the container is created from, cannot undo a step. STARTED_FILE is on the container's tmpfs: it exists only while the container runs and holds the level its k3s started at, which is what current_level reports, since k3s hides its arguments from /proc and a level cannot be read back from the process. The container is restarted, never created again, so the node password, kept in the container's writable layer or in a volume, is the one the cluster already holds. +# +# This differs from the NixOS actuator in one respect the core's contract allows: a reboot does not return the node to the first level. A server that restarts at etcd-only comes back at etcd-only, with the cordon the core put on its Node still in place, so the cluster's view and the node's agree. +# +# The Node carries a label, LEVEL_LABEL, with the level as its value, from the first step down until the node is back at the first level. Whatever judges nodes by their readiness can leave a stepped-down server out: at etcd-only it has no kubelet, so it is not Ready by design. +# +# It needs docker and the variables the core reads from the environment file, plus NODE_CONTAINER, the name of this server's k3s container, and LEVEL_LABEL. kubectl is the one in that container (KUBECTL in the environment file is `docker exec` of it), and KUBE_SERVER must name another server's API: the container's own, at etcd-only, answers `apiserver disabled` on some servers. + +LEVEL_FILE=${LEVEL_FILE:-/var/lib/rancher/k3s/regrade-level} +STARTED_FILE=${STARTED_FILE:-/run/regrade-level} + +# How often a restarted container is asked whether it is back, in seconds: the core's own pace for waiting on the API server. +POLL_SECONDS=2 + +# The line k3s logs once its embedded etcd is serving, at every level and on every start. Waiting for it is how a restart is known to have returned the server's vote, which the core's lease relies on: it is released as soon as apply_level returns, and another server that stepped before this one's etcd was back would stack on top of the outage. +ETCD_READY_LOG='ETCD server is now running' + +levels() { + printf '%s\n' full control-plane etcd-only +} + +# Runs a command in the server's container. +in_node() { + docker exec -i "$NODE_CONTAINER" "$@" +} + +# The level the running k3s started at, as its entrypoint recorded it; "unknown" while the container is not running, or when the record is none of the levels (for example one written by another version, or a restart in progress). A running container with neither a record nor a requested level is one whose entrypoint is from before regrading, which could only start k3s whole, so it is at full; its next restart picks up the new entrypoint. +current_level() { + local level + # shellcheck disable=SC2016 # The expansions are for the shell inside the container, which is given the two paths as arguments. + level=$(in_node sh -c 'if [ -r "$1" ]; then cat "$1"; elif [ ! -e "$2" ]; then echo full; fi' sh "$STARTED_FILE" "$LEVEL_FILE" 2> /dev/null) || { + echo unknown + return 0 + } + if levels | grep -Fqx -- "$level"; then + echo "$level" + else + echo unknown + fi +} + +# Puts the label for the level $1 on the Node, or takes it off for the first level. Done before the level is entered, so the label is never missing from a node that has stepped down; the cluster API is reached through another server, which is up at every level. +label_node() { + if [ "$1" = "$(levels | head -n 1)" ]; then + kubectl_ label node "$NODE_NAME" "${LEVEL_LABEL}-" + else + kubectl_ label node "$NODE_NAME" "${LEVEL_LABEL}=$1" --overwrite + fi +} + +# Replaces the file at $1, inside the container, with the level $2, atomically, so a reader never sees half a name. +write_level() { + # shellcheck disable=SC2016 # The expansions are for the shell inside the container, which is given the level and the path as arguments. + in_node sh -c 'umask 077 && printf "%s\n" "$1" > "$2.new" && mv "$2.new" "$2"' sh "$2" "$1" +} + +# How k3s runs at a level: as a server (every level but etcd-only), or as an etcd member alone. Only a change between the two changes the arguments k3s starts with, so only that needs a restart. +k3s_mode() { + if [ "$1" = etcd-only ]; then + echo etcd + else + echo server + fi +} + +# Whether the node has registered again after the container started at $1: it reports Ready and its kubelet has renewed the node lease since. Ready alone is not proof, since the API keeps the old container's Ready condition until the node controller's grace period runs out. Both are UTC, so their first nineteen characters, to the second, order as text. +node_ready_since() { + local started=$1 ready renewed + ready=$(kubectl_ get node "$NODE_NAME" --output 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' 2> /dev/null) || return 1 + renewed=$(kubectl_ get lease --namespace kube-node-lease "$NODE_NAME" --output 'jsonpath={.spec.renewTime}' 2> /dev/null) || return 1 + [ "$ready" = True ] && [[ ${renewed:0:19} > "${started:0:19}" ]] +} + +# Restarts the container and returns once the server is back as the level $1 runs it: etcd serving again, and, when the level runs the agent, the node Ready on a heartbeat from the new container. Fails when that takes longer than the operation timeout, which leaves the node as it is, for current_level to report. +restart_node() { + local to=$1 started deadline + docker restart --time "$OPERATION_TIMEOUT_SECONDS" "$NODE_CONTAINER" > /dev/null || return 1 + started=$(docker inspect --format '{{.State.StartedAt}}' "$NODE_CONTAINER") || return 1 + deadline=$((SECONDS + OPERATION_TIMEOUT_SECONDS)) + until grep -Fq "$ETCD_READY_LOG" <<< "$(docker logs --since "$started" "$NODE_CONTAINER" 2>&1)"; do + if [ "$SECONDS" -ge "$deadline" ]; then + log "etcd did not come back within ${OPERATION_TIMEOUT_SECONDS}s" + return 1 + fi + sleep "$POLL_SECONDS" + done + [ "$(k3s_mode "$to")" = server ] || return 0 + until node_ready_since "$started"; do + if [ "$SECONDS" -ge "$deadline" ]; then + log "the node did not report Ready within ${OPERATION_TIMEOUT_SECONDS}s" + return 1 + fi + sleep "$POLL_SECONDS" + done +} + +apply_level() { + local to=$1 from + from=$(current_level) + if [ "$from" = unknown ]; then + log "cannot apply $to: the node is not running at a known level" + return 1 + fi + label_node "$to" || return 1 + write_level "$LEVEL_FILE" "$to" || return 1 + if [ "$(k3s_mode "$from")" = "$(k3s_mode "$to")" ]; then + write_level "$STARTED_FILE" "$to" + else + restart_node "$to" + fi +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + regrade_main "$@" +fi diff --git a/roles/github_runner_cluster/files/regrade/fleet-regrade.sh b/roles/github_runner_cluster/files/regrade/fleet-regrade.sh new file mode 100755 index 0000000..8b46f54 --- /dev/null +++ b/roles/github_runner_cluster/files/regrade/fleet-regrade.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Runs fleet-regrade in the regrade service of the cluster role's Compose file: the platform-neutral core, then the Docker actuator, both from the directory the Compose file mounts beside this script, so a collection upgrade reaches the service when it restarts without the image being built again. The settings are in fleet-regrade.env in the same directory, which the role templates. +set -euo pipefail + +lib=$(dirname "$(readlink -f "$0")") +FLEET_REGRADE_ENV=${FLEET_REGRADE_ENV:-$lib/fleet-regrade.env} + +# shellcheck source=/dev/null +. "$lib/fleet-regrade-core.sh" +# shellcheck source=/dev/null +. "$lib/fleet-regrade-docker.sh" + +# Memory pressure is what the core decides on. A kernel without it (built without CONFIG_PSI, or booted with psi=0) would fail on every sample, so the service stops at once and says why. +case ${1:-} in + watch | status) + if [ ! -r "${PSI_FILE:-/proc/pressure/memory}" ]; then + echo "The kernel of this Docker host exposes no memory pressure (${PSI_FILE:-/proc/pressure/memory} is not readable): it needs CONFIG_PSI and no psi=0 on its command line. Regrading cannot decide without it." >&2 + exit 1 + fi + ;; +esac + +regrade_main "$@" diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index d99fee7..f9d8875 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -52,6 +52,9 @@ - name: Check etcd's heartbeat interval and election timeout ansible.builtin.include_tasks: etcd_timing.yml +- name: Check the memory-pressure regrading settings + ansible.builtin.include_tasks: regrade.yml + - name: Fail if the k3s token is not set ansible.builtin.fail: msg: >- @@ -124,6 +127,9 @@ # See tasks/start_servers.yml for why, and for what a server's turn checks before Compose may recreate its container. ansible.builtin.include_tasks: start_servers.yml + - name: Run the regrade service beside this server, or remove it + ansible.builtin.include_tasks: regrade_service.yml + - name: Check the runtime and this node's container are up ansible.builtin.include_tasks: end_checks.yml when: ansible_facts['system'] == 'Darwin' @@ -179,8 +185,8 @@ when: github_runner_cluster_wait_for_nodes | bool block: - name: Wait for every node to report Ready - # --all, not a --limit-scoped single node: once a server/agent has joined, this must wait for it too, not just this host's own control-plane node. - ansible.builtin.command: kubectl wait --for=condition=Ready node --all --timeout=120s + # Every node, not a --limit-scoped single node: once a server/agent has joined, this must wait for it too, not just this host's own control-plane node. Except a server that memory-pressure regrading has stepped down (see the README), which carries the level label and does not run a kubelet at etcd-only, so it is not Ready by design. + ansible.builtin.command: kubectl wait --for=condition=Ready node --selector '!{{ github_runner_cluster_regrade_level_label }}' --timeout=120s environment: KUBECONFIG: "{{ github_runner_cluster_kubeconfig_path }}" changed_when: false @@ -193,12 +199,21 @@ changed_when: false register: github_runner_cluster_node_readiness + - name: Read which nodes memory-pressure regrading has stepped down + # Not Ready by design at etcd-only, and nothing a stale registration explains: clearing its node-password Secret would be a change nobody asked for. + ansible.builtin.command: + argv: [kubectl, get, nodes, --selector, "{{ github_runner_cluster_regrade_level_label }}", -o, 'jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}'] + environment: + KUBECONFIG: "{{ github_runner_cluster_kubeconfig_path }}" + changed_when: false + register: github_runner_cluster_nodes_stepped_down + # Only the hosts whose node is missing or not Ready: clearing a Ready node's Node object makes the cluster evict every pod on it, so a single unreachable host must not take the registrations of healthy ones with it. - name: Pick the non-bootstrap hosts whose node has not rejoined ansible.builtin.set_fact: github_runner_cluster_rejoin_hosts: >- {{ github_runner_cluster_topology.hosts | dict2items | rejectattr('value.bootstrap') - | rejectattr('value.node_name', 'in', github_runner_cluster_node_readiness.stdout_lines | select('search', ' True$') | map('split', ' ') | map('first') | list) + | rejectattr('value.node_name', 'in', github_runner_cluster_node_readiness.stdout_lines | select('search', ' True$') | map('split', ' ') | map('first') | list + github_runner_cluster_nodes_stepped_down.stdout_lines) | map(attribute='key') | list }} - name: Clear the stale node-password secret of each host that has not rejoined @@ -220,7 +235,7 @@ loop: "{{ github_runner_cluster_rejoin_hosts | select('in', github_runner_cluster_topology.hosts | dict2items | selectattr('value.role', 'equalto', 'agent') | map(attribute='key') | list) | list }}" - name: Retry waiting for every node to report Ready now that any stale registration is cleared - ansible.builtin.command: kubectl wait --for=condition=Ready node --all --timeout=120s + ansible.builtin.command: kubectl wait --for=condition=Ready node --selector '!{{ github_runner_cluster_regrade_level_label }}' --timeout=120s environment: KUBECONFIG: "{{ github_runner_cluster_kubeconfig_path }}" changed_when: false diff --git a/roles/github_runner_cluster/tasks/regrade.yml b/roles/github_runner_cluster/tasks/regrade.yml new file mode 100644 index 0000000..a124a9c --- /dev/null +++ b/roles/github_runner_cluster/tasks/regrade.yml @@ -0,0 +1,101 @@ +--- +# Checks the memory-pressure regrading settings (github_runner_cluster_regrade and the settings beside it) before the role changes anything on the host. Run after the topology is known: whether a step down can work depends on the other servers. Off is the default and passes. + +- name: Fail if regrading is not true or false + ansible.builtin.assert: + that: github_runner_cluster_regrade is boolean + quiet: true + fail_msg: github_runner_cluster_regrade is '{{ github_runner_cluster_regrade }}'; it must be true or false. Nothing was changed. + +- name: Check the settings of a host that regrades + when: github_runner_cluster_regrade | bool + block: + - name: Fail if this host is not a server + # An agent has no control plane to shed and no etcd vote to keep. + ansible.builtin.assert: + that: github_runner_cluster_effective_node_role == 'server' + quiet: true + fail_msg: >- + github_runner_cluster_regrade is true, but {{ inventory_hostname }} is an agent. Regrading steps a server down while it + keeps its etcd vote; an agent has neither. Set it for the servers only. Nothing was changed. + + - name: Fail if no other server of the group could keep the cluster's API up + # At etcd-only this server has no API server of its own, and the core drains it, takes its Lease and brings it back through another server's. + ansible.builtin.assert: + that: github_runner_cluster_topology.servers | reject('equalto', inventory_hostname) | list | length > 0 + quiet: true + fail_msg: >- + github_runner_cluster_regrade is true, but {{ inventory_hostname }} is the only server in {{ github_runner_cluster_group }}. + A server at etcd-only has no API server, and needs another server's to be drained, coordinated and brought back. Nothing + was changed. + + - name: Fail if this host runs the in-cluster Headscale server + # headscale_in_cluster runs Headscale as a static pod on the bootstrap server's kubelet, which etcd-only switches off, taking the mesh's control server with it. + ansible.builtin.assert: + that: not (github_runner_cluster_mesh == 'headscale_in_cluster' and github_runner_cluster_effective_bootstrap | bool) + quiet: true + fail_msg: >- + github_runner_cluster_regrade is true, but {{ inventory_hostname }} is the bootstrap server of a headscale_in_cluster mesh, + whose Headscale server is a static pod on this server's kubelet; etcd-only would switch it off and take the mesh's control + server with it. Regrade the other servers. Nothing was changed. + + - name: Fail if stepping up is neither automatic nor manual + ansible.builtin.assert: + that: github_runner_cluster_regrade_step_up in ['automatic', 'manual'] + quiet: true + fail_msg: >- + github_runner_cluster_regrade_step_up is '{{ github_runner_cluster_regrade_step_up }}'; it must be automatic or manual. + Nothing was changed. + + - name: Fail if a time or the available-memory floor is not a positive whole number + ansible.builtin.assert: + that: (lookup('ansible.builtin.vars', item) | string) is match('^[1-9][0-9]*\Z') + quiet: true + fail_msg: >- + {{ item }} is '{{ lookup('ansible.builtin.vars', item) }}'; it must be a positive whole number. Nothing was changed. + loop: + - github_runner_cluster_regrade_pressure_window_seconds + - github_runner_cluster_regrade_available_floor_percent + - github_runner_cluster_regrade_sample_seconds + - github_runner_cluster_regrade_sustain_seconds + - github_runner_cluster_regrade_cooldown_seconds + - github_runner_cluster_regrade_operation_timeout_seconds + - github_runner_cluster_regrade_lease_duration_seconds + + - name: Fail if the pressure threshold is not a number from 0 to 100 + ansible.builtin.assert: + that: + - (github_runner_cluster_regrade_pressure_threshold | string) is match('^[0-9]+(\.[0-9]+)?\Z') + - (github_runner_cluster_regrade_pressure_threshold | float) <= 100 + quiet: true + fail_msg: >- + github_runner_cluster_regrade_pressure_threshold is '{{ github_runner_cluster_regrade_pressure_threshold }}'; it must be a + percentage, a number from 0 to 100. Nothing was changed. + + - name: Fail if the available-memory floor is over 100 percent + ansible.builtin.assert: + that: (github_runner_cluster_regrade_available_floor_percent | int) <= 100 + quiet: true + fail_msg: >- + github_runner_cluster_regrade_available_floor_percent is {{ github_runner_cluster_regrade_available_floor_percent }}; it is a + share of MemTotal, so it must be at most 100. Nothing was changed. + + - name: Fail if the cooldown is shorter than the pressure window + # The average a reading is judged by still holds the data from before the step for a whole window afterwards. + ansible.builtin.assert: + that: (github_runner_cluster_regrade_cooldown_seconds | int) >= (github_runner_cluster_regrade_pressure_window_seconds | int) + quiet: true + fail_msg: >- + github_runner_cluster_regrade_cooldown_seconds ({{ github_runner_cluster_regrade_cooldown_seconds }}) must be at least + github_runner_cluster_regrade_pressure_window_seconds ({{ github_runner_cluster_regrade_pressure_window_seconds }}), or a + reading after a step still holds data from before it. Nothing was changed. + + - name: Fail if the Lease can run out while a step is still working + # A step is a drain, a stop that waits for k3s to shut down, and the wait for the server to serve again, each bounded by the operation timeout. + ansible.builtin.assert: + that: (github_runner_cluster_regrade_lease_duration_seconds | int) >= 3 * (github_runner_cluster_regrade_operation_timeout_seconds | int) + quiet: true + fail_msg: >- + github_runner_cluster_regrade_lease_duration_seconds ({{ github_runner_cluster_regrade_lease_duration_seconds }}) must be at least + three times github_runner_cluster_regrade_operation_timeout_seconds ({{ github_runner_cluster_regrade_operation_timeout_seconds }}), + or another server could take the Lease while this one is still stepping. Nothing was changed. diff --git a/roles/github_runner_cluster/tasks/regrade_service.yml b/roles/github_runner_cluster/tasks/regrade_service.yml new file mode 100644 index 0000000..269efd0 --- /dev/null +++ b/roles/github_runner_cluster/tasks/regrade_service.yml @@ -0,0 +1,79 @@ +--- +# Runs the regrade service beside this server's k3s container when github_runner_cluster_regrade is true, and removes it when it is not. Run for a server host, after the servers have been started; tasks/regrade.yml has already checked the settings. + +- name: Run the regrade service + when: github_runner_cluster_regrade | bool + block: + - name: Create the regrade service's directory + ansible.builtin.file: + path: "{{ github_runner_cluster_dir }}/regrade" + state: directory + mode: "0755" + + - name: Place the regrade service's image build context and scripts + # Copied on every run, so a collection upgrade reaches the host. The directory is mounted into the service rather than built into its image, so a changed script takes effect when the service is created again, below. + ansible.builtin.copy: + src: "regrade/{{ item.name }}" + dest: "{{ github_runner_cluster_dir }}/regrade/{{ item.name }}" + mode: "{{ item.mode }}" + loop: + - {name: Dockerfile, mode: "0644"} + - {name: fleet-regrade.sh, mode: "0755"} + - {name: fleet-regrade-core.sh, mode: "0644"} + - {name: fleet-regrade-docker.sh, mode: "0644"} + register: github_runner_cluster_regrade_scripts + + - name: Template the regrade service's settings + ansible.builtin.template: + src: regrade.env.j2 + dest: "{{ github_runner_cluster_dir }}/regrade/fleet-regrade.env" + mode: "0644" + register: github_runner_cluster_regrade_settings + + - name: Start the regrade service + # recreate: always when a script or the settings changed, since the service reads both once, at its start, and Compose sees no change in a mounted directory's contents. A service nothing changed for is left running. + community.docker.docker_compose_v2: + project_src: "{{ github_runner_cluster_dir }}" + project_name: "{{ github_runner_cluster_compose_project }}" + profiles: ["regrade"] + services: ["regrade"] + cli_context: "{{ github_runner_cluster_docker_cli_context or omit }}" + files: "{{ github_runner_cluster_compose_file_list if github_runner_cluster_compose_file_list | length > 1 else omit }}" + recreate: "{{ 'always' if github_runner_cluster_regrade_scripts.changed or github_runner_cluster_regrade_settings.changed else 'auto' }}" + state: present + +- name: Stop regrading + when: not (github_runner_cluster_regrade | bool) + block: + - name: Read the level this server's k3s started at + # Read-only, so it runs under --check too. Absent while the container is not running, or has never been regraded and so has an older entrypoint. + ansible.builtin.command: + argv: [docker, exec, "{{ github_runner_cluster_node_container }}", cat, /run/regrade-level] + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_regrade_level + changed_when: false + failed_when: false + check_mode: false + + - name: Fail if the server is stepped down + # Without the service nothing steps it back up, and the entrypoint would keep starting it at the level it was left at. Fatal for the whole play, so that no other host removes its service while one server is still stepped down: the refusal is then the same on every host. + any_errors_fatal: true + ansible.builtin.fail: + msg: >- + github_runner_cluster_regrade is false, but {{ inventory_hostname }} is running at the level + {{ github_runner_cluster_regrade_level.stdout | trim }}, below full, and the regrade service is what steps it back up. Run + `docker compose --profile regrade exec regrade fleet-regrade up` in {{ github_runner_cluster_dir }} until it reports full (status shows the + level), then run again. Nothing was changed. + when: + - github_runner_cluster_regrade_level.rc == 0 + - github_runner_cluster_regrade_level.stdout | trim != 'full' + + - name: Remove the regrade service's container + ansible.builtin.command: + argv: [docker, rm, --force, "{{ github_runner_cluster_compose_project }}-regrade-1"] + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_regrade_removed + changed_when: github_runner_cluster_regrade_removed.rc == 0 + failed_when: + - github_runner_cluster_regrade_removed.rc != 0 + - "'No such container' not in github_runner_cluster_regrade_removed.stderr" diff --git a/roles/github_runner_cluster/templates/regrade.env.j2 b/roles/github_runner_cluster/templates/regrade.env.j2 new file mode 100644 index 0000000..b148423 --- /dev/null +++ b/roles/github_runner_cluster/templates/regrade.env.j2 @@ -0,0 +1,17 @@ +# Generated by the exadev.github_runner collection's github_runner_cluster role on every run. Do not edit by hand: change this host's ansible/host_vars file and rerun ansible-playbook. Read by the regrade service (docker-compose.yml, regrade/); see the role README's Memory-pressure regrading. +PRESSURE_THRESHOLD={{ github_runner_cluster_regrade_pressure_threshold }} +AVAILABLE_FLOOR_PERCENT={{ github_runner_cluster_regrade_available_floor_percent }} +SAMPLE_SECONDS={{ github_runner_cluster_regrade_sample_seconds }} +SUSTAIN_SECONDS={{ github_runner_cluster_regrade_sustain_seconds }} +COOLDOWN_SECONDS={{ github_runner_cluster_regrade_cooldown_seconds }} +OPERATION_TIMEOUT_SECONDS={{ github_runner_cluster_regrade_operation_timeout_seconds }} +LEASE_DURATION_SECONDS={{ github_runner_cluster_regrade_lease_duration_seconds }} +STEP_UP={{ github_runner_cluster_regrade_step_up }} +NODE_NAME={{ github_runner_cluster_effective_node_name }} +NODE_CONTAINER={{ github_runner_cluster_node_container }} +CONTROL_PLANE_TAINT={{ github_runner_cluster_regrade_taint }} +LEVEL_LABEL={{ github_runner_cluster_regrade_level_label }} +KUBECTL="docker exec -i {{ github_runner_cluster_node_container }} kubectl" +KUBECONFIG_PATH=/etc/rancher/k3s/k3s.yaml +{# Another server's API, never this one's: at etcd-only this server has none, and its own answers "apiserver disabled" on some servers. The first other server in inventory order, by the address the servers' certificates carry. #} +KUBE_SERVER=https://{{ github_runner_cluster_topology.hosts[github_runner_cluster_topology.servers | reject('equalto', inventory_hostname) | first].address }}:{{ github_runner_cluster_api_port }} diff --git a/scripts/fleet-alert.sh b/scripts/fleet-alert.sh index 39760b4..a55ecb6 100755 --- a/scripts/fleet-alert.sh +++ b/scripts/fleet-alert.sh @@ -3,7 +3,7 @@ # # An outage is any of these holding for at least FLEET_ALERT_AFTER_SECONDS: # - the heartbeat has not published for that long (the heartbeat pod, the cluster or the gist write is down); -# - a control-plane server node has not been Ready for that long; +# - a control-plane server node has not been Ready for that long, other than one memory-pressure regrading has stepped down on purpose (the heartbeat publishes its regrade_level), which is not Ready by design; # - the heartbeat has not vouched for the fleet for that long (a scale set missing or without a listener, no ready ARC controller, no Ready node, runner pods that cannot be scheduled). # The onset of the last two is recorded by the heartbeat itself (unhealthy_since, and since on each entry of not_ready_nodes), so this script keeps no clock of its own. An alert fires once per outage, not on every pass, and a recovery note follows when every condition has cleared. # @@ -86,7 +86,7 @@ conditions="$(jq -c --argjson now "$now" --argjson after "$after" ' if (.at | age) >= $after then [{kind: "heartbeat_stale", detail: "the heartbeat has not published since \(.at)", since: .at}] else - [ (.not_ready_nodes[] | select(.control_plane and (.since | age) >= $after) + [ (.not_ready_nodes[] | select(.control_plane and (.regrade_level == null) and (.since | age) >= $after) | {kind: "server_not_ready", detail: "server node \(.node) is not Ready", since: .since}), (select((.healthy | not) and (.unhealthy_since | age) >= $after) | {kind: "fleet_not_vouched", detail: "the heartbeat is not vouching for the fleet: \(.reasons | join("; "))", since: .unhealthy_since}) ] diff --git a/scripts/heartbeat.sh b/scripts/heartbeat.sh index 835ea35..65e0420 100755 --- a/scripts/heartbeat.sh +++ b/scripts/heartbeat.sh @@ -10,6 +10,7 @@ # - HEALTH_GIST_FILE: the gist file the health reason (healthy, why not, nodes under pressure) is written to every tick (default fleet-health.json) # - HEARTBEAT_STATE_NAMESPACE: namespace holding the autoscaler's own status ConfigMap (see scripts/autoscaler.sh) # - AUTOSCALER_STATUS_CONFIGMAP: name of that ConfigMap +# - HEARTBEAT_REGRADE_LEVEL_LABEL: the label memory-pressure regrading puts on a server it has stepped down, with the level as its value (default github-runner.exadev.com/regrade-level, the cluster role's github_runner_cluster_regrade_level_label) # - HEARTBEAT_DEPLOYED_STATE_CONFIGMAP: name of the ConfigMap, in the state namespace, holding the collection version the ARC role last applied (default deployed-state) # - HEARTBEAT_DEPLOYED_VERSION_ANNOTATION: the node annotation holding the collection version the cluster role last applied on that node's host # - HEARTBEAT_CONTROLLER_NAMESPACE: namespace of the ARC controller's Deployment, when it is not the default actions-runner-controller @@ -27,6 +28,7 @@ AUTOSCALER_STATUS_CONFIGMAP="${AUTOSCALER_STATUS_CONFIGMAP:-autoscaler-status}" AUTOSCALER_STATUS_GIST_FILE="${AUTOSCALER_STATUS_GIST_FILE:-autoscaler-status.json}" DEPLOYED_STATE_CONFIGMAP="${HEARTBEAT_DEPLOYED_STATE_CONFIGMAP:-deployed-state}" DEPLOYED_VERSION_ANNOTATION="${HEARTBEAT_DEPLOYED_VERSION_ANNOTATION:-github-runner.exadev.com/deployed-version}" +REGRADE_LEVEL_LABEL="${HEARTBEAT_REGRADE_LEVEL_LABEL:-github-runner.exadev.com/regrade-level}" # How far in the future to set the timestamp: comfortably longer than the loop interval, so a single missed/slow tick doesn't look like an outage, but short enough that a real outage is detected promptly. HEARTBEAT_WINDOW_SECONDS="${HEARTBEAT_WINDOW_SECONDS:-600}" HEARTBEAT_CONTROLLER_NAMESPACE="${HEARTBEAT_CONTROLLER_NAMESPACE:-actions-runner-controller}" @@ -98,12 +100,12 @@ previous_health="$(curl -fsS -H "Authorization: Bearer ${HEARTBEAT_GH_TOKEN}" -H "https://api.github.com/gists/${HEARTBEAT_GIST_ID}" 2>/dev/null \ | jq -c --arg file "$HEALTH_GIST_FILE" '.files[$file].content // empty | fromjson' 2>/dev/null || true)" -# A server node is one carrying the control-plane role label, which k3s sets on every server. A node that is not Ready is listed whether or not it is a server, but only a server's absence endangers etcd quorum and the API, which is why the flag is published. -not_ready_json="$(jq -c --arg at "$at" --argjson previous "${previous_health:-null}" ' +# A server node is one carrying the control-plane role label, which k3s sets on every server. A node that is not Ready is listed whether or not it is a server, but only a server's absence endangers etcd quorum and the API, which is why the flag is published. A server that regrading has stepped down carries the level label and is not Ready on purpose (at etcd-only it runs no kubelet), so its level is published beside it, for a reader to tell it from a server that has failed. +not_ready_json="$(jq -c --arg at "$at" --arg level_label "$REGRADE_LEVEL_LABEL" --argjson previous "${previous_health:-null}" ' ($previous.not_ready_nodes // []) as $before | [.items[] | select(any(.status.conditions[]?; .type == "Ready" and .status == "True") | not) - | {node: .metadata.name, control_plane: ((.metadata.labels // {}) | has("node-role.kubernetes.io/control-plane"))} as $n + | {node: .metadata.name, control_plane: ((.metadata.labels // {}) | has("node-role.kubernetes.io/control-plane")), regrade_level: ((.metadata.labels // {})[$level_label] // null)} as $n | $n + {since: (first($before[] | select(.node == $n.node) | .since) // $at)}]' <<< "${nodes_json:-null}" 2>/dev/null || echo '[]')" unhealthy_since="$(jq -nc --arg at "$at" --argjson healthy "$healthy" --argjson previous "${previous_health:-null}" \ 'if $healthy then null else (if $previous.healthy == false then ($previous.unhealthy_since // $at) else $at end) end')" diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index 0f8f88f..f870c64 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -15,6 +15,8 @@ network=grtest-mesh subnet_prefix=172.31.250 headscale_ip="${subnet_prefix}.10" node_count=3 +# Whether the inventory turns memory-pressure regrading on; the regrade scenario sets it. +regrade=false etcd_image=gcr.io/etcd-development/etcd:v3.5.21 busybox_image=busybox:1.37 # Tries per pod pair, 5 seconds apart plus each try's own timeout: about two minutes for the mesh to converge after a restart. @@ -65,7 +67,7 @@ container() { echo "grtest-node$1-k3s"; } teardown() { for index in $(seq 1 "$node_count"); do if [ -f "${work}/$(node_name "$index")/docker-compose.yml" ]; then - docker compose --project-directory "${work}/$(node_name "$index")" -f "${work}/$(node_name "$index")/docker-compose.yml" --profile server down -v --rmi local >/dev/null 2>&1 || true + docker compose --project-directory "${work}/$(node_name "$index")" -f "${work}/$(node_name "$index")/docker-compose.yml" --profile server --profile regrade down -v --rmi local >/dev/null 2>&1 || true fi done if [ -f "${work}/headscale/docker-compose.yml" ]; then @@ -165,6 +167,7 @@ EOF echo " github_runner_cluster_k3s_token: ${k3s_token}" echo " github_runner_cluster_etcd_heartbeat_interval: ${etcd_heartbeat_ms}" echo " github_runner_cluster_etcd_election_timeout: ${etcd_election_ms}" + echo " github_runner_cluster_regrade: ${regrade}" if [ "$pin_context" = 1 ]; then echo " github_runner_cluster_docker_context: ${grtest_pinned_context}" fi @@ -510,6 +513,185 @@ EOF log "It did" } +# --- Memory-pressure regrading (github_runner_cluster_regrade) --- + +regrade_container() { echo "grtest-node$1-regrade-1"; } +# Runs a command in a node's regrade service. +regrade_on() { docker exec -i "$(regrade_container "$1")" "${@:2}"; } +# The level a node's regrade service reports. +level_of() { regrade_on "$1" fleet-regrade status | sed -n 's/^level: \([^ ]*\) .*/\1/p'; } +node_status() { kubectl_on "$1" get node "$(node_name "$2")" --no-headers 2>/dev/null | awk '{print $2}'; } +node_password_hash() { docker exec "$(container "$1")" sha256sum /etc/rancher/node/password /dev/null 2>&1 || true + docker run --rm --name grtest-etcdctl --network "container:$(container "$1")" --volumes-from "$(container "$1")" --entrypoint etcdctl "$etcd_image" \ + --endpoints https://127.0.0.1:2379 --cacert /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \ + --cert /var/lib/rancher/k3s/server/tls/etcd/client.crt --key /var/lib/rancher/k3s/server/tls/etcd/client.key \ + member list -w simple /dev/null; } +# A step by hand on node 3 is refused, and the refusal names the stepped-down server. Retried because node 3's own API may still be settling after it was stopped, which is a refusal too but for another reason; a step that goes ahead is a failure, not a retry. +peer_refusal() { + [ "$(level_of 3)" = full ] || fail "$(node_name 3) stepped while $(node_name 2) was stepped down" + ! regrade_on 3 fleet-regrade down >"${work}/peer.log" 2>&1 && grep -F "etcd-voting nodes not ready: $(node_name 2)" "${work}/peer.log" >/dev/null +} +at_level() { [ "$(level_of "$1")" = "$2" ]; } +node_is() { [ "$(node_status "$1" "$2")" = "$3" ]; } + +# The numbers of the nodes whose regrade service reports a level below full. +stepped_nodes() { + local index + for index in $(seq 1 "$node_count"); do + [ "$(level_of "$index")" = full ] || printf '%s ' "$index" + done +} + +scenario_regrade() { + local index ids password before after stepped deadline output + regrade=true + reset_environment + prepare_nodes hosted + write_inventory hosted + run_role + assert_cluster + + log "Every server runs the regrade service, at full" + for index in $(seq 1 "$node_count"); do + [ "$(docker inspect --format '{{.State.Running}}' "$(regrade_container "$index")")" = true ] || fail "$(regrade_container "$index") is not running" + wait_for "$(node_name "$index")'s regrade service to see its level" 120 at_level "$index" full + done + + log "Rerunning the role must restart neither the nodes nor the services" + before="$(started_at)$(service_started_at)" + run_role + after="$(started_at)$(service_started_at)" + [ "$before" = "$after" ] || fail "rerunning the role restarted a container: ${before} -> ${after}" + + ids=$(etcd_member_ids 1) + password=$(node_password_hash 2) + echo "etcd members: ${ids}" + + log "A step is refused while another server holds the cluster-wide lease" + kubectl_on 1 apply -f - <"${work}/held.log" 2>&1; then cat "${work}/held.log"; fail "a step ran with the lease held by another server"; fi + grep -F 'the lease is held by grtest-other' "${work}/held.log" >/dev/null || { cat "${work}/held.log"; fail "the refusal did not name the lease holder"; } + [ "$(level_of 2)" = full ] || fail "$(node_name 2) changed level without the lease" + kubectl_on 1 delete lease fleet-regrade --namespace kube-system >/dev/null + + log "Memory pressure on $(node_name 2) steps it down a level at a time, and no other server steps with it" + # The watcher reads its pressure from files in place of the kernel's, so the test controls it. It runs in the service container, with the service's own settings but for the times, which are seconds here, so it has the service's Lease, Docker access and kubectl. + regrade_on 2 sh -c 'cp /usr/local/lib/fleet-regrade/fleet-regrade.env /tmp/pressure.env && printf "SAMPLE_SECONDS=1\nSUSTAIN_SECONDS=3\nCOOLDOWN_SECONDS=5\n" >> /tmp/pressure.env && printf "some avg10=0.00 avg60=0.00 avg300=50.00 total=1\n" > /tmp/psi && printf "MemTotal: 1000000 kB\nMemAvailable: 1000 kB\n" > /tmp/meminfo' + # shellcheck disable=SC2016 # The expansion is for the shell in the service container, whose process becomes the watcher through exec, so the pid it records is the watcher's. + docker exec -d "$(regrade_container 2)" sh -c 'echo $$ > /tmp/watch.pid; FLEET_REGRADE_ENV=/tmp/pressure.env PSI_FILE=/tmp/psi MEMINFO_FILE=/tmp/meminfo exec fleet-regrade watch >/tmp/watch.log 2>&1' + deadline=$((SECONDS + 900)) + until at_level 2 etcd-only; do + stepped=$(stepped_nodes) + [ "$stepped" = "" ] || [ "$stepped" = "2 " ] || fail "another server stepped while $(node_name 2) did: ${stepped}" + [ "$SECONDS" -lt "$deadline" ] || { regrade_on 2 cat /tmp/watch.log || true; fail "$(node_name 2) did not reach etcd-only"; } + sleep 2 + done + regrade_on 2 cat /tmp/watch.log + regrade_on 2 cat /tmp/watch.log | grep -F 'step down full -> control-plane' >/dev/null || fail "the watcher did not step down through control-plane first" + + log "At etcd-only the server keeps its seat: the same members, healthy, and its Node still registered" + wait_for "$(node_name 2)'s Node to report NotReady" 240 node_is 1 2 NotReady,SchedulingDisabled + [ "$(etcd_member_ids 1)" = "$ids" ] || fail "the etcd members changed: $(etcd_member_ids 1) (was ${ids})" + [ "$(level_label 2)" = etcd-only ] || fail "the Node does not carry the level label" + kubectl_on 1 get node "$(node_name 2)" -o jsonpath='{.metadata.labels}' | grep -F 'node-role.kubernetes.io/etcd' >/dev/null || fail "the Node lost its etcd role label" + if docker exec "$(container 2)" sh -c 'for p in /proc/[0-9]*; do cat "$p/comm"; done' | grep -Fx containerd >/dev/null; then fail "$(node_name 2) still runs containerd at etcd-only"; fi + [ "$(node_password_hash 2)" = "$password" ] || fail "$(node_name 2)'s node password changed" + + log "The stepped-down server still votes: the cluster takes writes with the third server stopped" + docker stop "$(container 3)" >/dev/null + wait_for "a write through $(node_name 1) with $(node_name 3) stopped" 180 write_marker + docker start "$(container 3)" >/dev/null + # Through its own API, not the others' view of it: for the grace period of the node controller the others still show the Ready it had before it was stopped. + wait_for "$(node_name 3) to be Ready again" 600 node_is 3 3 Ready + + log "While $(node_name 2) is stepped down no other server steps" + wait_for "$(node_name 3)'s refusal to name the stepped-down server" 300 peer_refusal + cat "${work}/peer.log" + [ "$(level_of 3)" = full ] || fail "$(node_name 3) changed level" + + log "Rerunning the role with a server stepped down changes nothing, and turning regrading off is refused" + # Judged by the role's own recap, not by the containers' start times: after a server was stopped and started a container can restart by itself under the strain, which says nothing about the role. + output=$(run_role 2>&1) || { echo "$output"; fail "the role failed with $(node_name 2) stepped down"; } + [ "$(grep -cE '^grtest-node[0-9]+ +: .* changed=0 ' <<< "$output")" = "$node_count" ] || { echo "$output" | tail -n 20; fail "rerunning the role changed something while $(node_name 2) was stepped down"; } + [ "$(level_of 2)" = etcd-only ] || fail "the role moved $(node_name 2) off etcd-only" + regrade=false + write_inventory hosted + if output=$(run_role 2>&1); then echo "$output"; fail "the role removed the service of a stepped-down server"; fi + echo "$output" | grep -F 'is running at the level etcd-only' >/dev/null || { echo "$output"; fail "the refusal did not name the level"; } + for index in $(seq 1 "$node_count"); do + docker inspect "$(regrade_container "$index")" >/dev/null 2>&1 || fail "$(regrade_container "$index") was removed in spite of the refusal, which is the whole play's" + done + regrade=true + write_inventory hosted + + log "The pressure passes: the watcher steps back up by itself, one level at a time, to the same node" + regrade_on 2 sh -c 'printf "some avg10=0.00 avg60=0.00 avg300=0.00 total=1\n" > /tmp/psi && printf "MemTotal: 1000000 kB\nMemAvailable: 900000 kB\n" > /tmp/meminfo' + deadline=$((SECONDS + 900)) + until at_level 2 full; do + stepped=$(stepped_nodes) + [ "$stepped" = "" ] || [ "$stepped" = "2 " ] || fail "another server stepped while $(node_name 2) stepped up: ${stepped}" + [ "$SECONDS" -lt "$deadline" ] || { regrade_on 2 cat /tmp/watch.log || true; fail "$(node_name 2) did not step back up to full"; } + sleep 2 + done + regrade_on 2 cat /tmp/watch.log + regrade_on 2 cat /tmp/watch.log | grep -F 'step up etcd-only -> control-plane' >/dev/null || fail "the watcher did not step up through control-plane first" + # shellcheck disable=SC2016 # The substitution is for the shell in the service container, which holds the watcher. + regrade_on 2 sh -c 'kill "$(cat /tmp/watch.pid)"' || fail "the watcher on $(node_name 2) could not be stopped" + wait_for "$(node_name 2) to be Ready and schedulable" 300 node_is 1 2 Ready + [ -z "$(level_label 2)" ] || fail "the level label is still on the Node at full" + [ "$(etcd_member_ids 1)" = "$ids" ] || fail "the etcd members changed after stepping up" + [ "$(node_password_hash 2)" = "$password" ] || fail "$(node_name 2)'s node password changed" + assert_cluster + + log "By hand, a server goes down a level and up again through the same lease" + regrade_on 3 fleet-regrade down + [ "$(level_of 3)" = control-plane ] || fail "$(node_name 3) did not step down by hand" + wait_for "$(node_name 3) to be cordoned" 60 node_is 1 3 Ready,SchedulingDisabled + [ "$(level_label 3)" = control-plane ] || fail "the Node does not carry the level label at control-plane" + regrade_on 3 fleet-regrade up + wait_for "$(node_name 3) to be schedulable again" 120 node_is 1 3 Ready + [ "$(level_of 3)" = full ] || fail "$(node_name 3) did not step up by hand" + [ -z "$(level_label 3)" ] || fail "the level label is still on the Node at full" + + log "With every server at full, turning regrading off removes the services and leaves the nodes running" + before=$(started_at) + regrade=false + write_inventory hosted + run_role + [ "$before" = "$(started_at)" ] || fail "turning regrading off restarted a node" + for index in $(seq 1 "$node_count"); do + if docker inspect "$(regrade_container "$index")" >/dev/null 2>&1; then fail "$(regrade_container "$index") is still there"; fi + done +} + scenarios=("$@") [ "${#scenarios[@]}" -gt 0 ] || scenarios=(existing hosted in_cluster) for scenario in "${scenarios[@]}"; do @@ -517,6 +699,7 @@ for scenario in "${scenarios[@]}"; do case "$scenario" in hosted | in_cluster) scenario_cluster "$scenario" ;; existing) scenario_existing ;; + regrade) scenario_regrade ;; compose_faulty) scenario_compose_faulty ;; context_checks) scenario_context_checks ;; # Re-checks a cluster a previous run kept with GRTEST_KEEP=1 and the same GRTEST_WORK, without rebuilding it. diff --git a/tests/regrade/.shellcheckrc b/tests/regrade/.shellcheckrc new file mode 100644 index 0000000..c80f8cc --- /dev/null +++ b/tests/regrade/.shellcheckrc @@ -0,0 +1,3 @@ +# The regrade tests define functions only the sourced core calls, which older shellcheck releases report as unreachable (SC2317; newer ones as SC2329, which the files already disable). A shellcheckrc replaces the one above it rather than adding to it, so the shell is named again. +shell=bash +disable=SC2317 diff --git a/tests/regrade/fake-docker.sh b/tests/regrade/fake-docker.sh new file mode 100755 index 0000000..85f9eb5 --- /dev/null +++ b/tests/regrade/fake-docker.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# A stand-in for docker for the Docker actuator's tests, keeping one container in the directory FAKE_NODE: `running` (present while it runs), the files the actuator reads and writes inside it (the test points LEVEL_FILE and STARTED_FILE into this directory, so `docker exec` runs a command on the local filesystem), `started-at` (what `docker inspect` prints), `log` (lines of "