diff --git a/plugins/filter/cluster_instance.py b/plugins/filter/cluster_instance.py new file mode 100644 index 0000000..90a28aa --- /dev/null +++ b/plugins/filter/cluster_instance.py @@ -0,0 +1,76 @@ +"""Filters the github_runner_cluster role uses before it changes anything on a host, to tell whether the cluster it is about to set up there would collide with another Compose project on the same Docker daemon: a container name, the published API port or the project directory that another project already uses.""" + +from __future__ import annotations + +import json +from collections.abc import Sequence +from typing import Any + +from ansible.errors import AnsibleFilterError + +PROJECT_LABEL = "com.docker.compose.project" +WORKING_DIR_LABEL = "com.docker.compose.project.working_dir" + + +def _owner(labels: dict[str, Any]) -> str: + """Name the Compose project a container belongs to, for a message.""" + project = labels.get(PROJECT_LABEL) or "" + return f"the Compose project {project}" if project else "no Compose project" + + +def _host_ports(bindings: Any) -> set[str]: + """Every host port a container's port bindings publish, as strings, whatever the address or protocol.""" + ports: set[str] = set() + if not isinstance(bindings, dict): + return ports + for published in bindings.values(): + for binding in published or []: + if isinstance(binding, dict) and binding.get("HostPort"): + ports.add(str(binding["HostPort"])) + return ports + + +def cluster_instance_conflicts(lines: Any, project: Any, names: Any, port: Any, directory: Any) -> list[str]: + """Say what on a Docker daemon another Compose project already uses that this cluster's project would need. + + :param lines: one JSON object per container on the daemon, as ``docker inspect`` prints with the format in tasks/instance_check.yml: ``name`` (with Docker's leading slash), ``labels`` (the container's labels, or null) and ``ports`` (its HostConfig.PortBindings, or null). :param project: this cluster's Compose project name on the host. :param names: the container names this project creates (the server's and the agent's). :param port: the host port this project publishes, or an empty string when it publishes none (an agent). :param directory: this project's directory on the host. :returns: one sentence per container of another project (or of none) that has one of the names, publishes the port or runs from the directory; empty when there is no collision. Containers of this project itself are never a collision, since Compose recreates or keeps them. :raises AnsibleFilterError: if the arguments have the wrong types or a line is not such an object. + """ + if not isinstance(lines, Sequence) or isinstance(lines, str) or not isinstance(project, str) or project == "": + raise AnsibleFilterError("The lines must be a list and the project a non-empty string.") + if not isinstance(names, Sequence) or isinstance(names, str) or not all(isinstance(name, str) and name for name in names): + raise AnsibleFilterError("The names must be a list of non-empty strings.") + if not isinstance(directory, str) or directory == "": + raise AnsibleFilterError("The directory must be a non-empty string.") + wanted_port = str(port).strip() + conflicts: list[str] = [] + for line in lines: + if not isinstance(line, str) or line.strip() == "": + continue + try: + container = json.loads(line) + except json.JSONDecodeError as exc: + raise AnsibleFilterError(f"A container line is not JSON: {exc}") from exc + if not isinstance(container, dict) or not isinstance(container.get("name"), str): + raise AnsibleFilterError("A container line has no name.") + labels = container.get("labels") or {} + if not isinstance(labels, dict): + raise AnsibleFilterError(f"The container {container['name']} has labels that are not a mapping.") + if labels.get(PROJECT_LABEL) == project: + continue + name = container["name"].lstrip("/") + owner = _owner(labels) + if name in names: + conflicts.append(f"a container named {name} already exists and belongs to {owner}") + if wanted_port and wanted_port in _host_ports(container.get("ports")): + conflicts.append(f"the container {name}, of {owner}, already publishes host port {wanted_port}") + if labels.get(WORKING_DIR_LABEL) == directory: + conflicts.append(f"the container {name}, of {owner}, runs from the same project directory {directory}") + return conflicts + + +class FilterModule: + """Registers the cluster instance filter with Ansible.""" + + def filters(self) -> dict[str, Any]: + """Return the filters this plugin provides.""" + return {"cluster_instance_conflicts": cluster_instance_conflicts} diff --git a/roles/github_runner_cluster/README.md b/roles/github_runner_cluster/README.md index e6be3d4..4c20bfa 100644 --- a/roles/github_runner_cluster/README.md +++ b/roles/github_runner_cluster/README.md @@ -187,7 +187,22 @@ An `--etcd-arg` is global: k3s also gives it to the temporary etcd it starts on ## Multiple sites -Each site is its own cluster: its own inventory group, bootstrap server and mesh provider, run by its own play with `github_runner_cluster_group` set to that group (see `examples/two-site/`). One cluster spanning two meshes is not supported: flannel's tailscale backend carries pod traffic over the single mesh every node belongs to, so every node of a cluster must be on the same tailnet or Headscale server. A host belongs to at most one cluster group. +Each site is its own cluster: its own inventory group, bootstrap server and mesh provider, run by its own play with `github_runner_cluster_group` set to that group (see `examples/two-site/`). One cluster spanning two meshes is not supported: flannel's tailscale backend carries pod traffic over the single mesh every node belongs to, so every node of a cluster must be on the same tailnet or Headscale server. An inventory host belongs to at most one cluster group; to run a second cluster on the same machines, give each machine a second inventory host (see [Several clusters on one host](#several-clusters-on-one-host)). + +## Several clusters on one host + +A machine can run a node of a second, independent cluster beside a node of the first, for example to rebuild a cluster on another mesh (a different tailnet, a new k3s token, a fresh etcd) and move the runners to it before the first is removed. Each cluster is its own inventory group with its own play (`github_runner_cluster_group`), and the machine appears in each group as its own inventory host, both with the same `ansible_host`. The two nodes share the machine's Docker daemon, so the second cluster's hosts need their own values of everything that names something on it: + +- `github_runner_cluster_container_name` and `github_runner_cluster_agent_container_name`: container names are unique on a daemon. +- `github_runner_cluster_api_port`: the port the server's API listens on inside its container, publishes on the Docker host and writes into its kubeconfig (`https://127.0.0.1:`), and that joining nodes dial, so it is set once for the whole group. On a Mac every VM's published ports are forwarded to the Mac's own loopback, so it must differ from the first cluster's even when the second runs in a VM of its own. A port other than k3s's own 6443 adds `docker-compose.api-port.yml`, which starts k3s with `--https-listen-port`; the default adds nothing, so a cluster on the defaults keeps exactly the container configuration it had before the port was configurable and upgrading recreates none of its servers. +- `github_runner_cluster_compose_project`: Compose names the node's volumes (datastore, mesh identity, node password) and its image after the project. +- `github_runner_cluster_dir`: the directory holds the project's `.env` and the kubeconfig. + +Before it writes anything to a host the role reads every container on its daemon and refuses to go on when a container of another Compose project, or of none, already has one of this cluster's container names, publishes this server's API port, or runs from this cluster's directory, naming each one. It also refuses an API port that is not a port number, hosts of one group that use different ports, and container names Docker would refuse or that are the same. Under `--check` the same read runs, so a dry run against hosts that run the first cluster shows whether the second fits. + +The two nodes share the Docker host's memory, CPU and kernel limits. Each k3s server needs its own memory beside the pods scheduled on it, and both nodes' kubelets, containerd and pods draw on the same `fs.inotify.max_user_instances`, which a small VM may set low; measure both on the host before adding the second node. Each mesh device is the node's own container, so the two nodes are separate devices even when they are on the same tailnet and share a node name, and a name that is already a device there registers with a suffix (see `github_runner_cluster_node_name`). Each cluster's ARC role installs into the cluster its own kubeconfig reaches; a runner scale set's name can belong to only one controller at a time, so a scale set is moved by uninstalling it from one cluster before installing it in the other. + +Each Compose project runs its own `docker-prune` service against the same daemon. Two of them do the same work and are harmless. ## Variables @@ -195,7 +210,7 @@ Each site is its own cluster: its own inventory group, bootstrap server and mesh - `github_runner_cluster_compose_project` (default `github-runner`): the Compose project name. Compose prefixes the node's named volumes with it, and those volumes hold the k3s datastore and the node's mesh identity, so changing it on a running host brings up a new, empty node. Because it is pinned rather than taken from the directory's name, `github_runner_cluster_dir` can move without losing the cluster. - `github_runner_cluster_k3s_token` (required), `github_runner_cluster_tailscale_join_key`, `github_runner_cluster_tailscale_api_token`, `github_runner_cluster_headscale_api_key`, `github_runner_cluster_headscale_join_key`: secrets, as plain variables from any source Ansible reads. - `github_runner_cluster_join_key_path`, `github_runner_cluster_join_key_listener`: where a join key the role creates is kept (see [Join keys](#join-keys)). -- `github_runner_cluster_group` (default `github_runner_cluster`), `github_runner_cluster_max_servers` (default 5), `github_runner_cluster_api_port` (default 6443), `github_runner_cluster_datastore` (`etcd`, the default, or `sqlite`, which makes the cluster a single server with every other host an agent). +- `github_runner_cluster_group` (default `github_runner_cluster`), `github_runner_cluster_max_servers` (default 5), `github_runner_cluster_api_port` (default 6443, k3s's own; the same on every host of the group, see [Several clusters on one host](#several-clusters-on-one-host)), `github_runner_cluster_datastore` (`etcd`, the default, or `sqlite`, which makes the cluster a single server with every other host an agent). - `github_runner_cluster_mesh` (default `tailscale`), `github_runner_cluster_mesh_tag` (default `tag:github-runner-node`), `github_runner_cluster_pod_cidr` (default k3s's `10.42.0.0/16`). - `github_runner_cluster_tailnet_domain`: the mesh's MagicDNS domain. Each node's address is `.`, and join URLs and TLS SANs use it. - `github_runner_cluster_headscale_*`: the Headscale providers' settings, documented in `defaults/main.yml`. @@ -206,7 +221,8 @@ Each site is its own cluster: its own inventory group, bootstrap server and mesh - Per-host overrides: `github_runner_cluster_node_role` (`server` or `agent`), `github_runner_cluster_bootstrap` (true picks the bootstrap host, false excludes a host), `github_runner_cluster_server_url` (join this server instead, for a cluster whose servers are outside the inventory), `github_runner_cluster_tls_sans`, `github_runner_cluster_node_address`. The run fails if the result has an even number of servers, no servers, or more than one bootstrap host. - `github_runner_cluster_docker_context` (default empty): pins the Docker context this host's node runs in (see [Docker contexts](#docker-contexts)). - `github_runner_cluster_runtime_autostart` (default true), `github_runner_cluster_runtime` (default `auto`; `colima`, `docker_desktop` or `other`), `github_runner_cluster_colima_profile` (default empty, taken from the Docker context), `github_runner_cluster_runtime_path` (default both Homebrew prefixes and the system directories): on a macOS host, start Colima at login through a launchd agent and check the runtime (see [Container runtime at login](#container-runtime-at-login-macos)). -- `github_runner_cluster_compose_files`: extra Compose files for the host's k3s project, merged after `docker-compose.yml`, as paths relative to `github_runner_cluster_dir` or absolute. Several nodes on one Docker host each need their own `github_runner_cluster_dir` and `github_runner_cluster_compose_project`. +- `github_runner_cluster_container_name` (default `github-runner-k3s`) and `github_runner_cluster_agent_container_name` (default `github-runner-k3s-agent`): the names of the node's server and agent containers, which a second cluster on the same Docker hosts sets to names of its own (see [Several clusters on one host](#several-clusters-on-one-host)). +- `github_runner_cluster_compose_files`: extra Compose files for the host's k3s project, merged after `docker-compose.yml`, as paths relative to `github_runner_cluster_dir` or absolute. Several nodes on one Docker host each need their own `github_runner_cluster_dir`, `github_runner_cluster_compose_project` and container names; nodes of one cluster share its API port, so all but one of them on a host also need an extra file that publishes none (`ports: !reset []`, as `tests/mesh` does). - `github_runner_cluster_wait_for_nodes`: wait for every node to be Ready afterwards, clearing the node-password registration of a replaced node that fails to rejoin (installs the kubernetes Python client through `github_runner_k8s_client`). - `github_runner_cluster_server_ready_timeout` (default 600): the most seconds, about, the role waits for a recreated server's node to report Ready before the next server's turn (see [Node passwords and recreating servers](#node-passwords-and-recreating-servers)). diff --git a/roles/github_runner_cluster/defaults/main.yml b/roles/github_runner_cluster/defaults/main.yml index 8b01488..23dcac6 100644 --- a/roles/github_runner_cluster/defaults/main.yml +++ b/roles/github_runner_cluster/defaults/main.yml @@ -4,7 +4,10 @@ # Server and agent selection is automatic. The hosts in github_runner_cluster_group, in inventory order, are the cluster: the first N are servers, where N is the largest odd number no greater than both the number of hosts and github_runner_cluster_max_servers (1 or 2 hosts give 1 server, 3 or 4 give 3, 5 or more give 5), since etcd needs a majority and an even member count tolerates no more failures than the odd count below it. The first server bootstraps the cluster (k3s --cluster-init, embedded etcd); every other server joins it as a further control-plane/etcd member, and the remaining hosts join it as agents (pure workers). The group rather than the play's hosts, because --limit narrows the play but must never change which host bootstraps or how many servers there are. See tasks/topology.yml and the collection's plugins/filter/cluster_topology.py. github_runner_cluster_group: github_runner_cluster github_runner_cluster_max_servers: 5 +# The port every server's k3s API listens on inside its container, publishes on its Docker host and writes into its kubeconfig (https://127.0.0.1:), and the port joining nodes dial. The default is k3s's own. Another value is for a second cluster on Docker hosts that already run one on the default port, alongside its own github_runner_cluster_container_name, github_runner_cluster_agent_container_name, github_runner_cluster_compose_project and github_runner_cluster_dir (see the role README's Several clusters on one host); on a Mac it must also differ from the port of any other VM there, since each VM's published ports are forwarded to the Mac's own loopback. A whole number from 1 to 65535, the same on every host of the group: it is part of every joining node's server URL. github_runner_cluster_api_port: 6443 +# k3s's own default for --https-listen-port, a constant: a server on this port needs no flag, so the role adds docker-compose.api-port.yml only for another port. Never set it. +github_runner_cluster_k3s_default_api_port: 6443 # The bootstrap server's datastore. etcd (the default) is embedded etcd, which any number of servers share. sqlite is k3s's default single-node datastore, lighter for a cluster with one server: the cluster then has exactly one server, whatever github_runner_cluster_max_servers says, and every other host joins it as an agent. Switching from sqlite to etcd later is k3s's own supported migration (the server restarts with --cluster-init); the reverse is not. github_runner_cluster_datastore: etcd @@ -119,17 +122,19 @@ github_runner_cluster_headscale_accept_node_local_state: false # Extra Compose files merged after docker-compose.yml for this host's k3s project, as paths relative to github_runner_cluster_dir or absolute. Nothing needs them in normal use; the mesh integration tests use one to put several nodes on one Docker host. github_runner_cluster_compose_files: [] -# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then github_runner_cluster_compose_files. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. +# Derived, never set directly: every Compose file for this host's k3s project, in merge order: docker-compose.yml, then the override the headscale_in_cluster provider writes for the bootstrap server, then docker-compose.api-port.yml when github_runner_cluster_api_port is not the default, then github_runner_cluster_compose_files. With no files beyond docker-compose.yml the role passes none at all and Compose reads it by itself, exactly as a plain `docker compose up` in github_runner_cluster_dir would. github_runner_cluster_compose_file_list: >- {{ ['docker-compose.yml'] + (['docker-compose.mesh.yml'] if github_runner_cluster_mesh == 'headscale_in_cluster' and github_runner_cluster_effective_bootstrap | default(false) | bool else []) + + (['docker-compose.api-port.yml'] if github_runner_cluster_api_port | int != github_runner_cluster_k3s_default_api_port | int else []) + github_runner_cluster_compose_files }} # The Docker context this host's node runs in, pinned by name (for example "colima"). Empty, the default, follows the host's current Docker context, or DOCKER_HOST when that is set. When set, every community.docker module call in the role passes it as cli_context and every docker CLI command runs with DOCKER_CONTEXT set to it, so both always reach the same daemon, whatever the host's current context is at the time. The role fails before touching the host if the context does not exist there, or if DOCKER_HOST is set as well, and it never changes the host's current context. Set it on any host with more than one Docker engine installed: on a Mac running its node in Colima with Docker Desktop also installed, merely launching Docker Desktop makes desktop-linux the current context, and an unpinned run would then build a second, empty node in Docker Desktop instead of managing the real one. Per host, like every value here that names something on the host; the Headscale tasks read it from the inventory of the host they act on. github_runner_cluster_docker_context: "" -# The node's k3s container, as docker-compose.yml names it. The headscale_in_cluster provider runs commands inside it; override it only alongside an extra Compose file that renames the container. +# The names of this host's k3s server container and k3s agent container, which docker-compose.yml takes from .env. Container names are unique on a Docker daemon, so a second cluster on the same Docker hosts needs both set to names of its own (see github_runner_cluster_api_port); the defaults are the names every existing node runs under, so leaving them unset changes nothing. Changing one on a running host makes Compose recreate that container under the new name, on the same volumes, which a server's turn treats as any other recreate. github_runner_cluster_container_name: github-runner-k3s -# The node's k3s container as this host runs it: the server's, or the agent's, which docker-compose.yml names differently. Derived, never set directly. -github_runner_cluster_node_container: "{{ github_runner_cluster_container_name if github_runner_cluster_effective_node_role == 'server' else 'github-runner-k3s-agent' }}" +github_runner_cluster_agent_container_name: github-runner-k3s-agent +# The node's k3s container as this host runs it: the server's, or the agent's. Derived, never set directly. +github_runner_cluster_node_container: "{{ github_runner_cluster_container_name if github_runner_cluster_effective_node_role == 'server' else github_runner_cluster_agent_container_name }}" # Derived, never set directly: the Compose service and the named volume that holds /etc/rancher/node (the node's password) for this host's role, as docker-compose.yml names them, and the volume's name in Docker, which Compose forms from the project and the volume's key. The role seeds the volume from the running container before the first recreate that mounts it (tasks/node_password.yml). github_runner_cluster_node_service: "{{ 'k3s' if github_runner_cluster_effective_node_role == 'server' else 'k3s-agent' }}" github_runner_cluster_node_password_volume_key: "{{ 'k3s-node' if github_runner_cluster_effective_node_role == 'server' else 'k3s-agent-node' }}" diff --git a/roles/github_runner_cluster/files/docker-compose.api-port.yml b/roles/github_runner_cluster/files/docker-compose.api-port.yml new file mode 100644 index 0000000..ca37cd4 --- /dev/null +++ b/roles/github_runner_cluster/files/docker-compose.api-port.yml @@ -0,0 +1,5 @@ +# Merged after docker-compose.yml by the github_runner_cluster role when github_runner_cluster_api_port is not k3s's own default (see github_runner_cluster_compose_file_list in defaults/main.yml): tells the server's entrypoint to start k3s with --https-listen-port at that port, which is also the port docker-compose.yml publishes, the port joining nodes dial and the port in the kubeconfig k3s writes. A separate file rather than an entry in docker-compose.yml, because any entry there, even an empty one, changes the configuration hash Compose compares and so would recreate every running server on the default port when the collection is upgraded. +services: + k3s: + environment: + - K3S_API_PORT=${K3S_API_PORT:?K3S_API_PORT must be set in .env - the port k3s listens on, publishes and writes into the kubeconfig} diff --git a/roles/github_runner_cluster/files/docker-compose.yml b/roles/github_runner_cluster/files/docker-compose.yml index 4fe9289..0e6d2ae 100644 --- a/roles/github_runner_cluster/files/docker-compose.yml +++ b/roles/github_runner_cluster/files/docker-compose.yml @@ -23,7 +23,8 @@ services: entrypoint: ["/bin/sh", "/usr/local/bin/github-runner-node", "server"] # Stable node name across container recreations. Without this k3s derives the node name from the container's hostname (its ID), so every recreate (e.g. on an image pull) would register a NEW node and leave the old one stale in etcd. Pinning it keeps `kubectl get nodes` to one entry. This is also the hostname Tailscale registers the container's own new device under, giving it a stable MagicDNS name too. Every server host sets its own K3S_NODE_NAME (Ansible defaults it to the host's own inventory name; a host that joined under a different name, such as a bootstrap host first brought up as "k3s-server", keeps that name through github_runner_cluster_node_name rather than being renamed disruptively). hostname: ${K3S_NODE_NAME:-k3s-server} - container_name: github-runner-k3s + # From .env (github_runner_cluster_container_name), so that a second cluster's Compose project on the same Docker daemon names its container differently; the default is the name every existing node runs under. + container_name: ${K3S_CONTAINER_NAME:-github-runner-k3s} privileged: true restart: unless-stopped profiles: ["server"] @@ -63,7 +64,8 @@ services: - ./kubeconfig:/output - ./k3s/node-entrypoint.sh:/usr/local/bin/github-runner-node:ro ports: - - "6443:6443" + # The API port, published on the Docker host at the same number k3s listens on inside the container, so that the kubeconfig k3s writes (server https://127.0.0.1:) works on the host as it is. From .env (github_runner_cluster_api_port); the default is k3s's own. k3s is told to listen on another port only through docker-compose.api-port.yml, which the role adds when the port is not the default, so the default renders the same container configuration as before the port was configurable. + - "${K3S_API_PORT:-6443}:${K3S_API_PORT:-6443}" # No 8472/udp or 10250/tcp published: Tailscale establishes outbound connections and NAT-traverses itself (DERP relay/STUN), so no inbound Docker port-forwarding is needed for cross-node traffic at all - confirmed live, unlike the VXLAN-over-Docker- port-publish approach this replaced. k3s-agent: @@ -75,7 +77,8 @@ services: # The same k3s/node-entrypoint.sh as the k3s service; see that service's comment. K3S_URL points at a server's own mesh name, not the host machine's own Tailscale identity. entrypoint: ["/bin/sh", "/usr/local/bin/github-runner-node", "agent"] hostname: ${K3S_AGENT_NODE_NAME:-k3s-agent} - container_name: github-runner-k3s-agent + # From .env (github_runner_cluster_agent_container_name), for the same reason as the server's. + container_name: ${K3S_AGENT_CONTAINER_NAME:-github-runner-k3s-agent} privileged: true restart: unless-stopped profiles: ["agent"] diff --git a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh index 79febd6..67a8afa 100755 --- a/roles/github_runner_cluster/files/k3s/node-entrypoint.sh +++ b/roles/github_runner_cluster/files/k3s/node-entrypoint.sh @@ -1,7 +1,7 @@ #!/bin/sh # Entry point for the k3s and k3s-agent services in docker-compose.yml: starts Tailscale inside the node's own container, then k3s with its --vpn-auth integration, so each node joins the mesh as its own device. Bind-mounted from the Compose project directory the role manages (github_runner_cluster_dir) rather than baked into the k3s image, so a host picks up a change to it with the Compose file that uses it, without rebuilding an image Compose would not otherwise rebuild. # -# Usage: node-entrypoint.sh server|agent. Reads, from the container's environment (docker-compose.yml, from .env): K3S_VPN_AUTH_JOIN_KEY, K3S_VPN_AUTH_CONTROL_SERVER_URL (empty for Tailscale's own control plane, a Headscale URL otherwise), K3S_JOIN_SERVER_URL, K3S_TLS_SAN_LIST, K3S_DATASTORE, K3S_ETCD_HEARTBEAT_INTERVAL and K3S_ETCD_ELECTION_TIMEOUT (server), and K3S_MESH_SELF_HOSTED_HEALTH_URL (the bootstrap server of a cluster that runs its own Headscale, see supervise_self_hosted below). +# Usage: node-entrypoint.sh server|agent. Reads, from the container's environment (docker-compose.yml, from .env): K3S_VPN_AUTH_JOIN_KEY, K3S_VPN_AUTH_CONTROL_SERVER_URL (empty for Tailscale's own control plane, a Headscale URL otherwise), K3S_JOIN_SERVER_URL, K3S_TLS_SAN_LIST, K3S_DATASTORE, K3S_API_PORT, K3S_ETCD_HEARTBEAT_INTERVAL and K3S_ETCD_ELECTION_TIMEOUT (server), and K3S_MESH_SELF_HOSTED_HEALTH_URL (the bootstrap server of a cluster that runs its own Headscale, see supervise_self_hosted below). # No set -e: a failed `tailscale up` falls through to waiting for an address, as it always has, rather than exiting and restarting the container. set -u @@ -28,6 +28,15 @@ set_etcd_timing_flags() { } set_etcd_timing_flags +# Sets api_port_flags from K3S_API_PORT, for the server only: --https-listen-port at that port, or nothing when it is unset, which leaves k3s on its own 6443. docker-compose.api-port.yml sets it only for a port other than the default, and docker-compose.yml publishes the same port on the Docker host, so the kubeconfig k3s writes for 127.0.0.1 at its listen port works on the host. Joining nodes dial it too (github_runner_cluster_api_port in their join URL). +set_api_port_flags() { + api_port_flags="" + if [ -n "${K3S_API_PORT:-}" ]; then + api_port_flags="--https-listen-port=$K3S_API_PORT" + fi +} +set_api_port_flags + # k3s's own vpn-auth integration only runs `tailscale up`; it does not start the daemon itself (without this, k3s fails outright with "tailscale up failed: ... it doesn't appear to be running"). Waits for the control socket before anything talks to it. mkdir -p /var/run/tailscale /var/lib/tailscale tailscaled --state=/var/lib/tailscale/tailscaled.state --socket=/var/run/tailscale/tailscaled.sock & @@ -94,6 +103,8 @@ wait_for_join_server() { # # --cluster-init (only when K3S_JOIN_SERVER_URL is unset, and K3S_DATASTORE is not sqlite) bootstraps embedded etcd, or converts an existing single-node SQLite datastore to it, the one migration k3s documents; once initialised, k3s ignores it on restart, so it stays in place permanently. A host with K3S_JOIN_SERVER_URL set passes --server instead and joins as another control-plane/etcd member. # +# --https-listen-port is added when K3S_API_PORT is set (set_api_port_flags above), for a second cluster on a Docker host whose default port another cluster already publishes. +# # --tls-san is added once per entry of K3S_TLS_SAN_LIST: every server's certificate must cover every server's own mesh name, since a client reaching any server directly by that name needs the certificate to match it. # # --etcd-arg=heartbeat-interval and --etcd-arg=election-timeout are added when set (set_etcd_timing_flags above), for peers whose traffic goes through a relay and so exceeds etcd's default timing. @@ -102,6 +113,7 @@ mesh_server_args() { --vpn-auth="$vpn_auth" \ --resolv-conf=/etc/k3s-resolv.conf \ --node-ip="$mesh_ip" + for flag in $api_port_flags; do set -- "$@" "$flag"; done if [ -n "${K3S_JOIN_SERVER_URL:-}" ]; then set -- "$@" --server "$K3S_JOIN_SERVER_URL" elif [ "${K3S_DATASTORE:-etcd}" != sqlite ]; then @@ -115,6 +127,7 @@ mesh_server_args() { # Sets server_args to the arguments of `k3s server` off the mesh, for a self-hosted bootstrap server whose Headscale server is not answering yet. No --vpn-auth, no --node-ip and no mesh resolver, so k3s and the Headscale static pod come up on the container's own network. No --cluster-init either: a fresh server starts on SQLite, and restarting it on the mesh with --cluster-init converts that to etcd with the mesh IP as etcd's peer address from the start, while an already-initialised server finds its etcd datastore regardless. direct_server_args() { set -- --disable=traefik --disable-network-policy --write-kubeconfig-mode=6443 + for flag in $api_port_flags; do set -- "$@" "$flag"; done for san in ${K3S_TLS_SAN_LIST:-}; do set -- "$@" --tls-san "$san"; done for flag in $etcd_timing_flags; do set -- "$@" "$flag"; done server_args="$*" diff --git a/roles/github_runner_cluster/tasks/instance_check.yml b/roles/github_runner_cluster/tasks/instance_check.yml new file mode 100644 index 0000000..c9e9c12 --- /dev/null +++ b/roles/github_runner_cluster/tasks/instance_check.yml @@ -0,0 +1,65 @@ +--- +# Checks, before the role writes anything to the host, that this cluster's Compose project has the host's Docker daemon to itself for what it needs: its container names (github_runner_cluster_container_name, github_runner_cluster_agent_container_name), the API port a server publishes (github_runner_cluster_api_port) and its project directory (github_runner_cluster_dir). A second cluster on the same hosts needs its own of each, and Docker would otherwise refuse the name or the port only when Compose starts the container, after the role has rewritten the project's files. Read-only, so it runs under --check too. + +- name: Fail if the API port is not a port number + ansible.builtin.fail: + msg: >- + github_runner_cluster_api_port is '{{ github_runner_cluster_api_port }}'; it must be a whole number from 1 to 65535. + Nothing was changed. + when: >- + (github_runner_cluster_api_port | string) is not match('^[1-9][0-9]*\\Z') + or github_runner_cluster_api_port | int > 65535 + +- name: Fail if the hosts of the cluster group do not all use the same API port + # Every joining node dials the bootstrap server at the port in its own join URL, which tasks/topology.yml builds from this host's value, so a host with another value would dial a port nothing listens on. Read for every host through hostvars, which never sees role defaults, so a host that sets nothing counts as the default. + vars: + github_runner_cluster_group_api_ports: >- + {{ groups[github_runner_cluster_group] | map('extract', hostvars) + | map(attribute='github_runner_cluster_api_port', default=github_runner_cluster_k3s_default_api_port) | map('string') | list }} + ansible.builtin.fail: + msg: >- + The hosts of {{ github_runner_cluster_group }} do not all use the same github_runner_cluster_api_port + ({{ groups[github_runner_cluster_group] | zip(github_runner_cluster_group_api_ports) | map('join', ': ') | join('; ') }}). + It is the port every server listens on and every joining node dials, so set it once for the whole group. Nothing was changed. + when: github_runner_cluster_group_api_ports | unique | length > 1 + +- name: Fail if a container name is not one Docker accepts, or both are the same + ansible.builtin.fail: + msg: >- + github_runner_cluster_container_name is '{{ github_runner_cluster_container_name }}' and github_runner_cluster_agent_container_name + is '{{ github_runner_cluster_agent_container_name }}'. Each must be a Docker container name (a letter or digit, then letters, digits, + '_', '.' or '-'), and they must differ, since docker-compose.yml names the server's and the agent's containers with them. Nothing was changed. + when: >- + github_runner_cluster_container_name is not match('^[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z') + or github_runner_cluster_agent_container_name is not match('^[a-zA-Z0-9][a-zA-Z0-9_.-]*\\Z') + or github_runner_cluster_container_name == github_runner_cluster_agent_container_name + +- name: Read every container on this host's Docker daemon, with its Compose project, project directory and published ports + # docker inspect rather than docker ps, since docker ps shows a stopped container's ports as none, and a stopped container still claims its host port the next time it starts. + ansible.builtin.shell: | + set -eu + ids=$(docker ps --all --quiet) + if [ -n "$ids" ]; then + # shellcheck disable=SC2086 # One argument per container id. + docker inspect --format '{% raw %}{"name":{{json .Name}},"labels":{{json .Config.Labels}},"ports":{{json .HostConfig.PortBindings}}}{% endraw %}' $ids + fi + environment: "{{ github_runner_cluster_docker_cli_environment }}" + register: github_runner_cluster_instance_containers + changed_when: false + check_mode: false + +- name: Fail if another Compose project on this daemon already uses this cluster's container names, API port or directory + vars: + github_runner_cluster_instance_conflicts: >- + {{ github_runner_cluster_instance_containers.stdout_lines | exadev.github_runner.cluster_instance_conflicts( + github_runner_cluster_compose_project, + [github_runner_cluster_container_name, github_runner_cluster_agent_container_name], + github_runner_cluster_api_port if github_runner_cluster_effective_node_role == 'server' else '', + github_runner_cluster_dir) }} + ansible.builtin.fail: + msg: >- + Not setting up the Compose project {{ github_runner_cluster_compose_project }} on {{ inventory_hostname }}: + {{ github_runner_cluster_instance_conflicts | join('; ') }}. A second cluster on this Docker daemon needs its own + github_runner_cluster_container_name, github_runner_cluster_agent_container_name, github_runner_cluster_api_port and + github_runner_cluster_dir (see the role README's Several clusters on one host). Nothing was changed. + when: github_runner_cluster_instance_conflicts | length > 0 diff --git a/roles/github_runner_cluster/tasks/main.yml b/roles/github_runner_cluster/tasks/main.yml index f19c879..d99fee7 100644 --- a/roles/github_runner_cluster/tasks/main.yml +++ b/roles/github_runner_cluster/tasks/main.yml @@ -63,6 +63,10 @@ - name: Check the Compose files the node's .env names ansible.builtin.include_tasks: compose_files.yml +- name: Check this cluster's container names, API port and directory are its own on this host's Docker daemon + # Before anything is written to the host, so that a second cluster set up with a name, port or directory another Compose project already uses fails here rather than halfway through. See tasks/instance_check.yml. + ansible.builtin.include_tasks: instance_check.yml + - name: Create this host's Compose project directory ansible.builtin.file: path: "{{ github_runner_cluster_dir }}/{{ item }}" @@ -78,6 +82,7 @@ mode: "{{ item.mode }}" loop: - {src: docker-compose.yml, mode: "0644"} + - {src: docker-compose.api-port.yml, mode: "0644"} - {src: k3s/Dockerfile, mode: "0644"} - {src: k3s/node-entrypoint.sh, mode: "0755"} diff --git a/roles/github_runner_cluster/templates/env-agent.j2 b/roles/github_runner_cluster/templates/env-agent.j2 index 04ab1e2..1a5ea85 100644 --- a/roles/github_runner_cluster/templates/env-agent.j2 +++ b/roles/github_runner_cluster/templates/env-agent.j2 @@ -2,6 +2,10 @@ # The Compose files, as absolute paths, and the project the role runs this node with, so that a docker compose command run by hand in this directory, or with --project-directory from another, acts on the same project with the same files (each command still needs its --profile, and the host's pinned Docker context when it has one). COMPOSE_FILE={{ github_runner_cluster_compose_file_list | map('regex_replace', '^(?!/)', github_runner_cluster_dir ~ '/') | join(':') }} COMPOSE_PROJECT_NAME={{ github_runner_cluster_compose_project }} +K3S_AGENT_CONTAINER_NAME={{ github_runner_cluster_agent_container_name }} +# The server's container name and API port, which docker-compose.yml interpolates for the k3s service: unused by the agent, but set so that the agent's .env renders the same names for both services as the role uses. +K3S_CONTAINER_NAME={{ github_runner_cluster_container_name }} +K3S_API_PORT={{ github_runner_cluster_api_port }} K3S_TOKEN={{ github_runner_cluster_k3s_token }} K3S_VERSION={{ github_runner_cluster_k3s_version }} TAILSCALE_VERSION={{ github_runner_cluster_tailscale_version }} diff --git a/roles/github_runner_cluster/templates/env.j2 b/roles/github_runner_cluster/templates/env.j2 index 66af1e0..d636243 100644 --- a/roles/github_runner_cluster/templates/env.j2 +++ b/roles/github_runner_cluster/templates/env.j2 @@ -2,6 +2,8 @@ # The Compose files, as absolute paths, and the project the role runs this node with, so that a docker compose command run by hand in this directory, or with --project-directory from another, acts on the same project with the same files (each command still needs its --profile, and the host's pinned Docker context when it has one). COMPOSE_FILE={{ github_runner_cluster_compose_file_list | map('regex_replace', '^(?!/)', github_runner_cluster_dir ~ '/') | join(':') }} COMPOSE_PROJECT_NAME={{ github_runner_cluster_compose_project }} +K3S_CONTAINER_NAME={{ github_runner_cluster_container_name }} +K3S_API_PORT={{ github_runner_cluster_api_port }} K3S_TOKEN={{ github_runner_cluster_k3s_token }} K3S_VERSION={{ github_runner_cluster_k3s_version }} TAILSCALE_VERSION={{ github_runner_cluster_tailscale_version }} diff --git a/tests/cluster_versions/render_env.yml b/tests/cluster_versions/render_env.yml index a806eab..b397b11 100644 --- a/tests/cluster_versions/render_env.yml +++ b/tests/cluster_versions/render_env.yml @@ -1,5 +1,5 @@ --- -# Renders the cluster role's env.j2 and env-agent.j2 with the role's defaults and placeholder values for the settings this test does not look at, writing them to env_output and env_agent_output, for tests/unit/test_cluster_versions.py. Role variables are passed as extra variables, which outrank the defaults loaded here. +# Renders the cluster role's env.j2 and env-agent.j2 with the role's defaults and placeholder values for the settings this test does not look at, writing them to env_output and env_agent_output, and, when compose_files_output is set, the Compose file list the node's project would use, for tests/unit/test_cluster_versions.py, test_cluster_etcd_timing.py and test_cluster_instance.py. Role variables are passed as extra variables, which outrank the defaults loaded here. - name: Render the node .env templates hosts: localhost connection: local @@ -25,3 +25,10 @@ content: "{{ lookup('ansible.builtin.template', playbook_dir ~ '/../../roles/github_runner_cluster/templates/env-agent.j2') }}" dest: "{{ env_agent_output }}" mode: "0600" + + - name: Write the Compose files the node's project would use + ansible.builtin.copy: + content: "{{ github_runner_cluster_compose_file_list | to_json }}" + dest: "{{ compose_files_output }}" + mode: "0600" + when: compose_files_output is defined diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh index faa7cbf..0f8f88f 100755 --- a/tests/mesh/run.sh +++ b/tests/mesh/run.sh @@ -103,7 +103,7 @@ reset_environment() { fi } -# Each node gets its own cluster directory, which the role fills with the Compose file and k3s build context, plus an override that renames its container, drops the host port every node would otherwise publish, and puts it on the shared test network at a fixed address. +# Each node gets its own cluster directory, which the role fills with the Compose file and k3s build context, plus an override that drops the host port every node would otherwise publish and puts it on the shared test network at a fixed address. Its container is named by github_runner_cluster_container_name in the inventory. prepare_nodes() { local scenario="$1" index dir for index in $(seq 1 "$node_count"); do @@ -112,7 +112,6 @@ prepare_nodes() { { echo "services:" echo " k3s:" - echo " container_name: $(container "$index")" if [ "$scenario" = in_cluster ] && [ "$index" = 1 ] && [ "$api_on_host" = true ]; then echo " ports: !override [\"127.0.0.1:18080:8080\"]" else diff --git a/tests/unit/test_cluster_etcd_timing.py b/tests/unit/test_cluster_etcd_timing.py index cdc7afd..5adc865 100644 --- a/tests/unit/test_cluster_etcd_timing.py +++ b/tests/unit/test_cluster_etcd_timing.py @@ -185,8 +185,10 @@ def server_args(self, builder: str, **env: str) -> list[str]: 'vpn_auth="name=tailscale,joinKey=example"', 'mesh_ip="100.64.0.1"', function_source("set_etcd_timing_flags"), + function_source("set_api_port_flags"), function_source(builder), "set_etcd_timing_flags", + "set_api_port_flags", f"{builder}", 'printf "%s\\n" "$server_args"', ] diff --git a/tests/unit/test_cluster_instance.py b/tests/unit/test_cluster_instance.py new file mode 100644 index 0000000..5f94a6d --- /dev/null +++ b/tests/unit/test_cluster_instance.py @@ -0,0 +1,402 @@ +"""Tests for running a second cluster beside another on the same Docker hosts: the container names and API port the Compose files take from .env, the override that moves k3s's listen port, the entrypoint flag, the role's defaults and .env templates, and the check in tasks/instance_check.yml (its filter in plugins/filter/cluster_instance.py, and the task file run through the role against a stand-in for docker).""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import Any + +import yaml + +ROOT = Path(__file__).resolve().parents[2] +_spec = importlib.util.spec_from_file_location("cluster_instance", ROOT / "plugins" / "filter" / "cluster_instance.py") +assert _spec is not None and _spec.loader is not None +cluster_instance = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(cluster_instance) +AnsibleFilterError = cluster_instance.AnsibleFilterError +ROLE = ROOT / "roles" / "github_runner_cluster" +COMPOSE = ROLE / "files" / "docker-compose.yml" +API_PORT_COMPOSE = ROLE / "files" / "docker-compose.api-port.yml" +ENTRYPOINT = ROLE / "files" / "k3s" / "node-entrypoint.sh" +ANSIBLE_PLAYBOOK = os.environ.get("ANSIBLE_PLAYBOOK", "ansible-playbook") +DEFAULTS: dict[str, Any] = yaml.safe_load((ROLE / "defaults" / "main.yml").read_text()) + +# The values every node ran under before they were configurable, written out here rather than read from the defaults, so that a changed default fails these tests instead of moving with them. +SERVER_CONTAINER = "github-runner-k3s" +AGENT_CONTAINER = "github-runner-k3s-agent" +K3S_PORT = 6443 + + +def ansible_env() -> dict[str, str]: + return {**os.environ, "ANSIBLE_COLLECTIONS_PATH": str(ROOT / "playbooks" / "collections"), "ANSIBLE_LOCALHOST_WARNING": "false", "ANSIBLE_INVENTORY_UNPARSED_WARNING": "false", "ANSIBLE_RETRY_FILES_ENABLED": "false"} + + +def container_line(name: str, project: str | None = None, directory: str | None = None, ports: dict[str, Any] | None = None) -> str: + """One container as tasks/instance_check.yml has docker inspect print it.""" + labels = {key: value for key, value in (("com.docker.compose.project", project), ("com.docker.compose.project.working_dir", directory)) if value is not None} + return json.dumps({"name": f"/{name}", "labels": labels or None, "ports": ports}) + + +def published(port: int, protocol: str = "tcp") -> dict[str, Any]: + """Port bindings that publish ``port`` on every address of the host, as Docker records "port:port".""" + return {f"{port}/{protocol}": [{"HostIp": "", "HostPort": str(port)}, {"HostIp": "::", "HostPort": str(port)}]} + + +class ConflictFilterTest(unittest.TestCase): + def conflicts(self, *lines: str, port: Any = K3S_PORT, directory: str = "/home/u/.github-runner") -> list[str]: + return cluster_instance.cluster_instance_conflicts(list(lines), "github-runner", [SERVER_CONTAINER, AGENT_CONTAINER], port, directory) + + def test_an_empty_daemon_has_no_conflict(self) -> None: + self.assertEqual(self.conflicts(), []) + self.assertEqual(self.conflicts(""), []) + + def test_this_project_s_own_containers_never_conflict(self) -> None: + own = container_line(SERVER_CONTAINER, "github-runner", "/home/u/.github-runner", published(K3S_PORT)) + prune = container_line("github-runner-docker-prune-1", "github-runner", "/home/u/.github-runner") + self.assertEqual(self.conflicts(own, prune), []) + + def test_a_container_name_another_project_holds_is_named_with_its_owner(self) -> None: + found = self.conflicts(container_line(SERVER_CONTAINER, "other", "/srv/other")) + self.assertEqual(found, [f"a container named {SERVER_CONTAINER} already exists and belongs to the Compose project other"]) + found = self.conflicts(container_line(AGENT_CONTAINER)) + self.assertEqual(found, [f"a container named {AGENT_CONTAINER} already exists and belongs to no Compose project"]) + + def test_the_published_port_of_another_project_conflicts_whatever_the_address_or_protocol(self) -> None: + for bindings in (published(K3S_PORT), {f"{K3S_PORT}/tcp": [{"HostIp": "127.0.0.1", "HostPort": str(K3S_PORT)}]}, {"80/udp": [{"HostIp": "", "HostPort": str(K3S_PORT)}]}): + with self.subTest(bindings=bindings): + found = self.conflicts(container_line("old-k3s", "old", "/srv/old", bindings)) + self.assertEqual(found, [f"the container old-k3s, of the Compose project old, already publishes host port {K3S_PORT}"]) + + def test_another_port_or_an_unpublished_container_port_does_not_conflict(self) -> None: + self.assertEqual(self.conflicts(container_line("web", "web", "/srv/web", published(3000))), []) + self.assertEqual(self.conflicts(container_line("web", "web", "/srv/web", {f"{K3S_PORT}/tcp": None})), []) + self.assertEqual(self.conflicts(container_line("web", "web", "/srv/web", {f"{K3S_PORT}/tcp": [{"HostIp": "", "HostPort": ""}]})), []) + + def test_an_agent_publishes_no_port_so_none_is_compared(self) -> None: + self.assertEqual(self.conflicts(container_line("old-k3s", "old", "/srv/old", published(K3S_PORT)), port=""), []) + + def test_another_project_in_the_same_directory_conflicts(self) -> None: + found = self.conflicts(container_line("x-1", "x", "/home/u/.github-runner")) + self.assertEqual(found, ["the container x-1, of the Compose project x, runs from the same project directory /home/u/.github-runner"]) + + def test_every_collision_of_one_container_is_reported(self) -> None: + found = self.conflicts(container_line(SERVER_CONTAINER, "old", "/home/u/.github-runner", published(K3S_PORT))) + self.assertEqual(len(found), 3, found) + + def test_malformed_input_is_refused(self) -> None: + for lines in ("not a list", ["not json"], [json.dumps({"labels": {}})], [json.dumps({"name": "/a", "labels": ["x"]})]): + with self.subTest(lines=lines), self.assertRaises(AnsibleFilterError): + cluster_instance.cluster_instance_conflicts(lines, "p", ["a"], K3S_PORT, "/d") + for project, names, directory in (("", ["a"], "/d"), ("p", "a", "/d"), ("p", [""], "/d"), ("p", ["a"], "")): + with self.subTest(project=project, names=names, directory=directory), self.assertRaises(AnsibleFilterError): + cluster_instance.cluster_instance_conflicts([], project, names, K3S_PORT, directory) + + +class DefaultsTest(unittest.TestCase): + def test_the_defaults_are_the_names_and_port_every_node_already_runs_under(self) -> None: + self.assertEqual(DEFAULTS["github_runner_cluster_container_name"], SERVER_CONTAINER) + self.assertEqual(DEFAULTS["github_runner_cluster_agent_container_name"], AGENT_CONTAINER) + self.assertEqual(DEFAULTS["github_runner_cluster_api_port"], K3S_PORT) + self.assertEqual(DEFAULTS["github_runner_cluster_k3s_default_api_port"], K3S_PORT) + + def test_the_node_container_is_the_agent_s_own_variable_on_an_agent(self) -> None: + template = DEFAULTS["github_runner_cluster_node_container"] + self.assertIn("github_runner_cluster_agent_container_name", template) + self.assertNotIn(AGENT_CONTAINER, template) + + def test_the_role_copies_the_api_port_override_to_the_host(self) -> None: + tasks = yaml.safe_load((ROLE / "tasks" / "main.yml").read_text()) + copy = next(task for task in tasks if task.get("name") == "Place the collection's Compose file and k3s build context in it") + self.assertIn("docker-compose.api-port.yml", [item["src"] for item in copy["loop"]]) + + def test_the_check_runs_before_the_role_writes_to_the_host(self) -> None: + names = [task.get("name", "") for task in yaml.safe_load((ROLE / "tasks" / "main.yml").read_text())] + check = next(index for index, name in enumerate(names) if "container names, API port and directory" in name) + self.assertLess(check, names.index("Create this host's Compose project directory")) + self.assertLess(check, names.index("Keep this node's password across container recreation")) + self.assertGreater(check, names.index("Decide this host's part in the cluster")) + + def test_every_new_variable_is_in_the_readme(self) -> None: + readme = (ROLE / "README.md").read_text() + for name in ("github_runner_cluster_container_name", "github_runner_cluster_agent_container_name", "github_runner_cluster_api_port"): + with self.subTest(name=name): + self.assertIn(f"`{name}`", readme) + self.assertIn("## Several clusters on one host", readme) + + +@unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") +class RenderTest(unittest.TestCase): + """Renders the .env templates and the derived Compose file list with the role's defaults.""" + + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + + def render(self, *extra: str) -> tuple[list[str], list[str], list[str]]: + env, agent, files = self.work / "env", self.work / "env-agent", self.work / "files.json" + args = [ANSIBLE_PLAYBOOK, str(ROOT / "tests" / "cluster_versions" / "render_env.yml"), "-e", f"env_output={env}", "-e", f"env_agent_output={agent}", "-e", f"compose_files_output={files}", "-e", "github_runner_cluster_dir=/srv/github-runner", *extra] + result = subprocess.run(args, capture_output=True, text=True, check=False, env=ansible_env(), cwd=self.work) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + compose_files: list[str] = json.loads(files.read_text()) + return env.read_text().splitlines(), agent.read_text().splitlines(), compose_files + + def test_the_defaults_write_the_existing_names_and_port_and_add_no_compose_file(self) -> None: + server, agent, files = self.render() + self.assertIn(f"K3S_CONTAINER_NAME={SERVER_CONTAINER}", server) + self.assertIn(f"K3S_API_PORT={K3S_PORT}", server) + self.assertIn(f"K3S_AGENT_CONTAINER_NAME={AGENT_CONTAINER}", agent) + self.assertIn(f"K3S_CONTAINER_NAME={SERVER_CONTAINER}", agent) + self.assertIn(f"K3S_API_PORT={K3S_PORT}", agent) + self.assertEqual(files, ["docker-compose.yml"]) + self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml", server) + + def test_a_second_cluster_s_values_reach_both_templates_and_the_port_adds_the_override(self) -> None: + server, agent, files = self.render("-e", "github_runner_cluster_container_name=second-k3s", "-e", "github_runner_cluster_agent_container_name=second-k3s-agent", "-e", "github_runner_cluster_api_port=6543") + self.assertIn("K3S_CONTAINER_NAME=second-k3s", server) + self.assertIn("K3S_API_PORT=6543", server) + self.assertIn("K3S_AGENT_CONTAINER_NAME=second-k3s-agent", agent) + self.assertEqual(files, ["docker-compose.yml", "docker-compose.api-port.yml"]) + self.assertIn("COMPOSE_FILE=/srv/github-runner/docker-compose.yml:/srv/github-runner/docker-compose.api-port.yml", server) + + def test_the_default_port_written_as_a_string_adds_no_override(self) -> None: + _, _, files = self.render("-e", '{"github_runner_cluster_api_port": "6443"}') + self.assertEqual(files, ["docker-compose.yml"]) + + +def has_compose() -> bool: + if shutil.which("docker") is None: + return False + return subprocess.run(["docker", "compose", "version"], capture_output=True, check=False).returncode == 0 + + +@unittest.skipUnless(has_compose(), "docker compose is not installed") +class ComposeRenderTest(unittest.TestCase): + """Renders the Compose files with docker compose config, which needs no daemon.""" + + def compose(self, *arguments: str, files: tuple[Path, ...] = (COMPOSE,), **env: str) -> str: + clean = {key: value for key, value in os.environ.items() if not key.startswith(("DOCKER_", "COMPOSE_", "K3S_", "TAILSCALE_"))} + base = {"K3S_TOKEN": "t", "K3S_TLS_SAN_LIST": "n", "K3S_VERSION": "v1.34.1-k3s1", "TAILSCALE_VERSION": "v1.102.4"} + file_args = [argument for path in files for argument in ("-f", str(path))] + args = ["docker", "compose", "--project-directory", tempfile.gettempdir(), "--project-name", "example", *file_args, "--profile", "server", "--profile", "agent", *arguments] + result = subprocess.run(args, capture_output=True, text=True, check=False, env=clean | base | env) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout + + def services(self, files: tuple[Path, ...] = (COMPOSE,), **env: str) -> dict[str, Any]: + services: dict[str, Any] = json.loads(self.compose("config", "--format", "json", files=files, **env))["services"] + return services + + def hashes(self, files: tuple[Path, ...] = (COMPOSE,), **env: str) -> dict[str, str]: + return dict(line.split() for line in self.compose("config", "--hash", "*", files=files, **env).splitlines()) + + def test_with_nothing_set_the_containers_and_port_are_the_ones_every_node_runs_under(self) -> None: + services = self.services() + self.assertEqual(services["k3s"]["container_name"], SERVER_CONTAINER) + self.assertEqual(services["k3s-agent"]["container_name"], AGENT_CONTAINER) + self.assertEqual([(port["published"], port["target"]) for port in services["k3s"]["ports"]], [(str(K3S_PORT), K3S_PORT)]) + self.assertNotIn("K3S_API_PORT", services["k3s"]["environment"]) + + def test_the_values_the_role_writes_by_default_render_the_same_configuration_hash_as_none(self) -> None: + # Compose recreates a container whose configuration hash changes, so a default .env that rendered any differently would restart every running server on upgrade. + written = {"K3S_CONTAINER_NAME": SERVER_CONTAINER, "K3S_AGENT_CONTAINER_NAME": AGENT_CONTAINER, "K3S_API_PORT": str(K3S_PORT)} + self.assertEqual(self.hashes(**written), self.hashes()) + + def test_a_second_cluster_gets_its_own_names_and_publishes_and_listens_on_its_own_port(self) -> None: + services = self.services(files=(COMPOSE, API_PORT_COMPOSE), K3S_CONTAINER_NAME="second-k3s", K3S_AGENT_CONTAINER_NAME="second-k3s-agent", K3S_API_PORT="6543") + self.assertEqual(services["k3s"]["container_name"], "second-k3s") + self.assertEqual(services["k3s-agent"]["container_name"], "second-k3s-agent") + self.assertEqual([(port["published"], port["target"]) for port in services["k3s"]["ports"]], [("6543", 6543)]) + self.assertEqual(services["k3s"]["environment"]["K3S_API_PORT"], "6543") + self.assertNotIn("K3S_API_PORT", services["k3s-agent"]["environment"]) + self.assertNotEqual(self.hashes(files=(COMPOSE, API_PORT_COMPOSE), K3S_API_PORT="6543")["k3s"], self.hashes()["k3s"]) + + def test_the_override_refuses_to_render_without_a_port(self) -> None: + clean = {key: value for key, value in os.environ.items() if not key.startswith(("DOCKER_", "COMPOSE_", "K3S_", "TAILSCALE_"))} + base = {"K3S_TOKEN": "t", "K3S_TLS_SAN_LIST": "n", "K3S_VERSION": "v1.34.1-k3s1", "TAILSCALE_VERSION": "v1.102.4"} + args = ["docker", "compose", "--project-directory", tempfile.gettempdir(), "--project-name", "example", "-f", str(COMPOSE), "-f", str(API_PORT_COMPOSE), "--profile", "server", "config"] + result = subprocess.run(args, capture_output=True, text=True, check=False, env=clean | base) + self.assertNotEqual(result.returncode, 0) + self.assertIn("K3S_API_PORT must be set", result.stderr) + + +def function_source(name: str) -> str: + """The definition of a shell function in the entrypoint, from its first line to the closing brace at the start of a line.""" + match = re.search(rf"^{name}\(\) \{{\n.*?^\}}\n", ENTRYPOINT.read_text(), re.MULTILINE | re.DOTALL) + assert match is not None, f"{name} is not defined in node-entrypoint.sh" + return match.group(0) + + +class EntrypointTest(unittest.TestCase): + """Runs the entrypoint's argument-list functions in a shell, as test_cluster_etcd_timing.py does.""" + + def server_args(self, builder: str, **env: str) -> list[str]: + script = "\n".join( + [ + "set -u", + 'vpn_auth="name=tailscale,joinKey=example"', + 'mesh_ip="100.64.0.1"', + function_source("set_etcd_timing_flags"), + function_source("set_api_port_flags"), + function_source(builder), + "set_etcd_timing_flags", + "set_api_port_flags", + builder, + 'printf "%s\\n" "$server_args"', + ] + ) + clean = {key: value for key, value in os.environ.items() if not key.startswith("K3S_")} + result = subprocess.run(["/bin/sh", "-c", script], capture_output=True, text=True, check=False, env=clean | env) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout.split() + + def test_the_script_calls_the_function_that_sets_the_flag(self) -> None: + self.assertRegex(ENTRYPOINT.read_text(), re.compile(r"^set_api_port_flags$", re.MULTILINE)) + + def test_a_port_makes_k3s_listen_on_it_on_the_mesh_and_off_it(self) -> None: + for builder in ("mesh_server_args", "direct_server_args"): + with self.subTest(builder=builder): + args = self.server_args(builder, K3S_API_PORT="6543", K3S_JOIN_SERVER_URL="https://bootstrap.example.internal:6543") + self.assertEqual([arg for arg in args if arg.startswith("--https-listen-port")], ["--https-listen-port=6543"]) + + def test_no_port_leaves_k3s_on_its_own_default(self) -> None: + for builder in ("mesh_server_args", "direct_server_args"): + for env in ({}, {"K3S_API_PORT": ""}): + with self.subTest(builder=builder, env=env): + self.assertEqual([arg for arg in self.server_args(builder, **env) if "listen-port" in arg], []) + + def test_the_agent_command_line_does_not_take_it(self) -> None: + agent_line = next(line for line in ENTRYPOINT.read_text().splitlines() if "k3s agent" in line) + self.assertNotIn("listen-port", agent_line) + + +# Stands in for the docker CLI for tasks/instance_check.yml: `ps --all --quiet` prints one id per line of $DOCKER_STUB/containers, and `inspect --format ... ` prints those lines, as the task's format has docker print them. +DOCKER_STUB = """#!{python} +import os, sys +stub = os.environ["DOCKER_STUB"] +args = sys.argv[1:] +with open(os.path.join(stub, "calls.log"), "a") as log: + log.write(" ".join(args) + "\\n") +path = os.path.join(stub, "containers") +lines = open(path).read().splitlines() if os.path.exists(path) else [] +if args == ["ps", "--all", "--quiet"]: + print("\\n".join("c%d" % index for index in range(len(lines)))) +elif args[0] == "inspect" and args[1] == "--format": + ids = args[3:] + if ids != ["c%d" % index for index in range(len(lines))]: + sys.exit("unexpected ids: " + " ".join(ids)) + print("\\n".join(lines)) +else: + sys.exit("unexpected docker call: " + " ".join(args)) +""" + + +@unittest.skipIf(shutil.which(ANSIBLE_PLAYBOOK) is None, "ansible-playbook is not installed") +class RoleTest(unittest.TestCase): + """Runs tasks/instance_check.yml through the role (tests/cluster_versions/checks.yml) against the stand-in docker.""" + + def setUp(self) -> None: + self.work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.work) + self.stub = self.work / "stub" + self.stub.mkdir() + (self.work / "bin").mkdir() + docker = self.work / "bin" / "docker" + docker.write_text(DOCKER_STUB.replace("{python}", sys.executable)) + docker.chmod(0o755) + + def containers(self, *lines: str) -> None: + (self.stub / "containers").write_text("\n".join(lines) + "\n") + + def run_check(self, *extra: str, role: str = "server", group_vars: dict[str, Any] | None = None, hosts: dict[str, dict[str, Any]] | None = None) -> subprocess.CompletedProcess[str]: + node = {"github_runner_cluster_effective_node_role": role, "github_runner_cluster_docker_cli_environment": {}, "github_runner_cluster_dir": "/srv/second"} + group = {"vars": {"github_runner_cluster_compose_project": "second", **(group_vars or {})}, "hosts": {name: node | values for name, values in (hosts or {"node1": {}}).items()}} + inventory = self.work / "inventory.yml" + inventory.write_text(yaml.safe_dump({"all": {"children": {"github_runner_cluster": group}}})) + args = [ANSIBLE_PLAYBOOK, "-i", str(inventory), str(ROOT / "tests" / "cluster_versions" / "checks.yml"), "-e", "check_tasks=instance_check", "--limit", "node1", *extra] + env = ansible_env() | {"PATH": f"{self.work / 'bin'}{os.pathsep}{os.environ['PATH']}", "DOCKER_STUB": str(self.stub)} + return subprocess.run(args, capture_output=True, text=True, check=False, env=env, cwd=self.work) + + def first_cluster(self) -> list[str]: + """The containers a cluster on the defaults runs on a server host.""" + return [ + container_line(SERVER_CONTAINER, "github-runner", "/srv/github-runner", published(K3S_PORT)), + container_line("github-runner-docker-prune-1", "github-runner", "/srv/github-runner"), + ] + + def test_an_empty_daemon_passes(self) -> None: + result = self.run_check() + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual((self.stub / "calls.log").read_text().splitlines(), ["ps --all --quiet"]) + + def test_a_second_cluster_with_its_own_names_port_and_directory_passes_beside_the_first(self) -> None: + self.containers(*self.first_cluster()) + result = self.run_check(group_vars={"github_runner_cluster_container_name": "second-k3s", "github_runner_cluster_agent_container_name": "second-k3s-agent", "github_runner_cluster_api_port": 6543}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_second_cluster_on_the_default_names_and_port_is_refused_with_every_collision(self) -> None: + self.containers(*self.first_cluster()) + result = self.run_check() + self.assertNotEqual(result.returncode, 0) + output = result.stdout.replace("\n", " ") + self.assertIn(f"a container named {SERVER_CONTAINER} already exists and belongs to the Compose project github-runner", output) + self.assertIn(f"the container {SERVER_CONTAINER}, of the Compose project github-runner, already publishes host port {K3S_PORT}", output) + self.assertIn("Nothing was changed.", output) + + def test_the_check_runs_under_check_mode_too(self) -> None: + self.containers(*self.first_cluster()) + result = self.run_check("--check") + self.assertNotEqual(result.returncode, 0) + self.assertIn("already publishes host port", result.stdout) + + def test_an_agent_is_not_refused_for_a_port_it_does_not_publish(self) -> None: + self.containers(*self.first_cluster()) + result = self.run_check(role="agent", group_vars={"github_runner_cluster_container_name": "second-k3s", "github_runner_cluster_agent_container_name": "second-k3s-agent"}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_the_first_cluster_s_own_run_passes_beside_a_second(self) -> None: + second = container_line("second-k3s", "second", "/srv/second", published(6543)) + self.containers(*self.first_cluster(), second) + result = self.run_check(group_vars={"github_runner_cluster_compose_project": "github-runner"}, hosts={"node1": {"github_runner_cluster_dir": "/srv/github-runner"}}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_a_directory_another_project_runs_from_is_refused(self) -> None: + self.containers(*self.first_cluster()) + result = self.run_check(group_vars={"github_runner_cluster_container_name": "second-k3s", "github_runner_cluster_agent_container_name": "second-k3s-agent", "github_runner_cluster_api_port": 6543}, hosts={"node1": {"github_runner_cluster_dir": "/srv/github-runner"}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("runs from the same project directory /srv/github-runner", result.stdout.replace("\n", " ")) + + def test_a_port_that_is_not_a_port_number_is_refused_before_docker_is_read(self) -> None: + for port in ("0", "65536", "abc", "-1", "6443x"): + with self.subTest(port=port): + result = self.run_check("-e", json.dumps({"github_runner_cluster_api_port": port})) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must be a whole number from 1 to 65535", result.stdout.replace("\n", " ")) + self.assertFalse((self.stub / "calls.log").exists()) + + def test_hosts_of_the_group_with_different_ports_are_refused(self) -> None: + result = self.run_check(hosts={"node1": {"github_runner_cluster_api_port": 6543}, "node2": {}, "node3": {"github_runner_cluster_api_port": 6543}}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("node1: 6543; node2: 6443; node3: 6543", result.stdout.replace("\n", " ")) + + def test_the_same_port_set_once_for_the_group_passes(self) -> None: + result = self.run_check(group_vars={"github_runner_cluster_api_port": 6543}, hosts={"node1": {}, "node2": {}, "node3": {}}) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_container_names_docker_refuses_or_that_are_the_same_are_refused(self) -> None: + for names in ({"github_runner_cluster_container_name": "-bad"}, {"github_runner_cluster_agent_container_name": "has space"}, {"github_runner_cluster_container_name": "same", "github_runner_cluster_agent_container_name": "same"}): + with self.subTest(names=names): + result = self.run_check(group_vars=names) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must be a Docker container name", result.stdout.replace("\n", " ")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_runtime_autostart.py b/tests/unit/test_runtime_autostart.py index 25c9338..f4be4b3 100644 --- a/tests/unit/test_runtime_autostart.py +++ b/tests/unit/test_runtime_autostart.py @@ -66,7 +66,7 @@ def test_the_agent_is_per_profile_and_lives_in_the_connecting_users_launch_agent def test_the_node_container_follows_the_node_s_role(self) -> None: template = DEFAULTS["github_runner_cluster_node_container"] self.assertIn("github_runner_cluster_container_name", template) - self.assertIn("'github-runner-k3s-agent'", template) + self.assertIn("github_runner_cluster_agent_container_name", template) def test_every_new_variable_is_in_the_readme(self) -> None: readme = (ROLE / "README.md").read_text()