diff --git a/README.md b/README.md index d1a2598..9937b52 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ An Unraid host runs the same cluster role as any other Docker host (it ships Doc ### Capabilities -A profile can also add tools, libraries and setup to its runner pods when they start, rather than baking them into a runner image someone has to build, publish and patch: `capabilities` names entries of `github_runner_arc_capabilities`. A tool-cache capability is a public, pinned image holding a tool in the Actions tool-cache layout; an init container copies it into the runner's `RUNNER_TOOL_CACHE`, where `actions/setup-node`, `setup-tflint` and other setup actions that consult the cache find it and skip their download, and it can also be put on `PATH` for a tool no setup action looks up. A system-library capability is a public, pinned image holding a relocatable install prefix (shared libraries, headers, pkg-config files, programs) for what cannot live in the tool cache, such as an MPI implementation, a solver library or a database client library; an init container copies it into a sysroot (`/opt/sysroot` by default), and the job-started hook puts the sysroot on `PATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, `CPATH`, `PKG_CONFIG_PATH` and `CMAKE_PREFIX_PATH` and sets any relocation variable the library needs. A hook capability is a script run before each job's first step, through a job-started hook mounted from a ConfigMap. A profile with its own `runs_on_label` then works as a named runner, for example one with Terraform, TFLint and the AWS CLI, or one with Open MPI and a solver library, that workflows select with `runs-on`, all from one values file and the stock `actions-runner` image. `tool-images/` has Dockerfiles for reference tool images (Node.js, Terraform, TFLint, the AWS CLI) reference system-library images (Open MPI, HiGHS, GLPK, libpq, MariaDB Connector/C) and a reference build runner image (the stock runner image with a C toolchain, for jobs that compile against those libraries), which `.github/workflows/publish-tool-images.yml` publishes when started by hand or by a `tool-images-*` tag; `roles/github_runner_arc/README.md`'s Capabilities section has the tool-image and payload contracts, how each reference library is declared and proven to work from a relocated prefix, how to build an image of your own, and where capabilities apply. +A profile can also add tools, libraries and setup to its runner pods when they start, rather than baking them into a runner image someone has to build, publish and patch: `capabilities` names entries of `github_runner_arc_capabilities`. A tool-cache capability is a public, pinned image holding a tool in the Actions tool-cache layout; an init container copies it into the runner's `RUNNER_TOOL_CACHE`, where `actions/setup-node`, `setup-tflint` and other setup actions that consult the cache find it and skip their download, and it can also be put on `PATH` for a tool no setup action looks up. A system-library capability is a public, pinned image holding a relocatable install prefix (shared libraries, headers, pkg-config files, programs) for what cannot live in the tool cache, such as an MPI implementation, a solver library or a database client library; an init container copies it into a sysroot (`/opt/sysroot` by default), and the job-started hook puts the sysroot on `PATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, `CPATH`, `PKG_CONFIG_PATH` and `CMAKE_PREFIX_PATH` and sets any relocation variable the library needs. A hook capability is a script run before each job's first step, through a job-started hook mounted from a ConfigMap. A node-cache capability is a directory of the node shared by the profile's runner pods there, for a package manager's store, which a pruner keeps under a size limit. A profile with its own `runs_on_label` then works as a named runner, for example one with Terraform, TFLint and the AWS CLI, or one with Open MPI and a solver library, that workflows select with `runs-on`, all from one values file and the stock `actions-runner` image. `tool-images/` has Dockerfiles for reference tool images (Node.js, Terraform, TFLint, the AWS CLI) reference system-library images (Open MPI, HiGHS, GLPK, libpq, MariaDB Connector/C) and a reference build runner image (the stock runner image with a C toolchain, for jobs that compile against those libraries), which `.github/workflows/publish-tool-images.yml` publishes when started by hand or by a `tool-images-*` tag; `roles/github_runner_arc/README.md`'s Capabilities section has the tool-image and payload contracts, how each reference library is declared and proven to work from a relocated prefix, how to build an image of your own, and where capabilities apply. ### Registry cache diff --git a/autoscaler/Dockerfile b/autoscaler/Dockerfile index fc8ebff..4a00bac 100644 --- a/autoscaler/Dockerfile +++ b/autoscaler/Dockerfile @@ -14,7 +14,8 @@ RUN apk add --no-cache bash curl jq kubectl COPY autoscaler/loop.sh /app/loop.sh COPY scripts/autoscaler.sh /app/scripts/autoscaler.sh COPY scripts/psi-reporter.sh /app/scripts/psi-reporter.sh -RUN chmod +x /app/loop.sh /app/scripts/autoscaler.sh /app/scripts/psi-reporter.sh +COPY scripts/node-cache-pruner.sh /app/scripts/node-cache-pruner.sh +RUN chmod +x /app/loop.sh /app/scripts/autoscaler.sh /app/scripts/psi-reporter.sh /app/scripts/node-cache-pruner.sh WORKDIR /app ENTRYPOINT ["/app/loop.sh"] diff --git a/plugins/filter/arc.py b/plugins/filter/arc.py index 25cccaf..1630a9c 100644 --- a/plugins/filter/arc.py +++ b/plugins/filter/arc.py @@ -75,10 +75,12 @@ # The highest weight a preferred node affinity term may carry (1-100 in the Kubernetes API), so keeping runner pods on the fixed nodes outweighs any other preference the scheduler scores. _PREFER_FIXED_NODES_WEIGHT = 100 -# A capability's keys (see arc_capability_errors): a tool-cache capability names an image, and optionally a path put on PATH at job start; a system-library capability names a sysroot_image, and optionally env, extra variables its libraries need; a hook capability carries a job-started script. -_CAPABILITY_KEYS = frozenset({"image", "path", "sysroot_image", "env", "job_started_hook"}) +# A capability's keys (see arc_capability_errors): a tool-cache capability names an image, and optionally a path put on PATH at job start; a system-library capability names a sysroot_image, and optionally env, extra variables its libraries need; a hook capability carries a job-started script; a node-cache capability names a node_cache mapping, {init_image, dirs, env?}, and may also set env. +_CAPABILITY_KEYS = frozenset({"image", "path", "sysroot_image", "env", "job_started_hook", "node_cache"}) # The keys of which a capability sets exactly one, each naming its kind. -_CAPABILITY_KINDS = ("image", "sysroot_image", "job_started_hook") +_CAPABILITY_KINDS = ("image", "sysroot_image", "job_started_hook", "node_cache") +# The keys of a node-cache capability's node_cache mapping. +_NODE_CACHE_KEYS = frozenset({"init_image", "dirs"}) # Each tool-cache or system-library capability becomes an init container named with this prefix, and a container name is a DNS label, so a capability name is at most what is left of one. _CAPABILITY_CONTAINER_PREFIX = "capability-" _CAPABILITY_NAME_MAX_LENGTH = _DNS_LABEL_MAX_LENGTH - len(_CAPABILITY_CONTAINER_PREFIX) @@ -94,6 +96,8 @@ _TOOL_CACHE_VOLUME = "tool-cache" _SYSROOT_VOLUME = "sysroot" _HOOKS_VOLUME = "job-started-hooks" +# The node-local volume every node-cache capability of a profile shares: a hostPath directory of the node, one per profile, mounted once so that a hard link between two of its directories stays on one mount. +_NODE_CACHE_VOLUME = "node-cache" # The runner's own variables the capabilities set on the runner container: where it and the setup actions look for cached tools, and the hook it runs before a job's first step. _TOOL_CACHE_ENV = "RUNNER_TOOL_CACHE" _JOB_STARTED_HOOK_ENV = "ACTIONS_RUNNER_HOOK_JOB_STARTED" @@ -704,7 +708,10 @@ def _capability_entry_errors(name: Any, capability: Any) -> list[str]: sysroot_image = capability.get("sysroot_image") hook = capability.get("job_started_hook") if sum(capability.get(kind) is not None for kind in _CAPABILITY_KINDS) != 1: - errors.append(f"{where} must set exactly one of image (a tool-cache capability), sysroot_image (a system-library capability) and job_started_hook (a hook capability)") + errors.append(f"{where} must set exactly one of image (a tool-cache capability), sysroot_image (a system-library capability), job_started_hook (a hook capability) and node_cache (a node-cache capability)") + node_cache = capability.get("node_cache") + if node_cache is not None: + errors.extend(_node_cache_errors(f"{where}.node_cache", node_cache)) if image is not None: errors.extend(_pinned_image_errors(f"{where}.image", image)) if sysroot_image is not None: @@ -720,12 +727,34 @@ def _capability_entry_errors(name: Any, capability: Any) -> list[str]: errors.append(f"{where}.path must be /, optionally followed by /, each part letters, digits and '._+-' (got {path!r})") env = capability.get("env") if env is not None: - if sysroot_image is None: - errors.append(f"{where}.env is only for a system-library capability, one with a sysroot_image") + if sysroot_image is None and node_cache is None: + errors.append(f"{where}.env is only for a system-library capability (one with a sysroot_image) or a node-cache capability (one with a node_cache)") errors.extend(_env_errors(f"{where}.env", env)) return errors +def _node_cache_errors(where: str, node_cache: Any) -> list[str]: + """Return what is wrong with a node-cache capability's node_cache mapping: an init_image pinned to a version tag or digest, and dirs, a non-empty mapping of environment variable name to the subdirectory of the node cache that variable names, each subdirectory one path component and no two variables sharing one.""" + if not isinstance(node_cache, Mapping): + return [f"{where} must be a mapping with init_image and dirs"] + errors = [f"{where} has an unknown key '{key}'" for key in sorted(set(node_cache) - _NODE_CACHE_KEYS)] + errors.extend(_pinned_image_errors(f"{where}.init_image", node_cache.get("init_image"))) + dirs = node_cache.get("dirs") + if not isinstance(dirs, Mapping) or not dirs: + errors.append(f"{where}.dirs must be a non-empty mapping of environment variable name to a subdirectory of the node cache") + return errors + for name, subdirectory in dirs.items(): + if not isinstance(name, str) or not _ENV_NAME.match(name): + errors.append(f"{where}.dirs has a key that is not an environment variable name ({name!r})") + elif name in _RESERVED_ENV: + errors.append(f"{where}.dirs may not set {name}, which the capabilities set themselves") + if not isinstance(subdirectory, str) or not _PATH_COMPONENT.match(subdirectory): + errors.append(f"{where}.dirs.{name} must be one directory name, letters, digits and '._+-' (got {subdirectory!r})") + subdirectories = [value for value in dirs.values() if isinstance(value, str)] + errors.extend(f"{where}.dirs names the subdirectory {value} for more than one variable" for value in sorted({value for value in subdirectories if subdirectories.count(value) > 1})) + return errors + + def _sysroot_env_conflicts(label: str, names: Sequence[str], catalogue: Mapping[str, Any]) -> list[str]: """Return the variables that two of a profile's system-library capabilities set to different values, since a sysroot has one environment.""" first: dict[str, tuple[str, Any]] = {} @@ -768,19 +797,24 @@ def _runner_container(spec: Mapping[str, Any]) -> dict[str, Any] | None: return None -def _sysroot_path_errors(sysroot_path: str, others: Mapping[str, str]) -> list[str]: - """Return what is wrong with the sysroot's mount path: it must be an absolute path that can sit in a colon-separated search path, apart from the other directories the capabilities mount.""" - if not isinstance(sysroot_path, str) or not sysroot_path.startswith("/") or sysroot_path == "/" or any(character in sysroot_path for character in ":\n\r"): - return [f"the sysroot path must be an absolute directory with no ':' or line break, since it is joined into search paths such as LD_LIBRARY_PATH (got {sysroot_path!r})"] - root = sysroot_path.rstrip("/") +def _overlap_errors(subject: str, path: str, others: Mapping[str, str]) -> list[str]: + """Return a message for each of the other mounted directories that path equals, contains or sits inside.""" + root = path.rstrip("/") errors: list[str] = [] for what, other in others.items(): other_root = str(other).rstrip("/") if root == other_root or root.startswith(f"{other_root}/") or other_root.startswith(f"{root}/"): - errors.append(f"the sysroot path {sysroot_path} overlaps the {what} {other}; each needs a directory of its own") + errors.append(f"the {subject} {path} overlaps the {what} {other}; each needs a directory of its own") return errors +def _sysroot_path_errors(sysroot_path: str, others: Mapping[str, str]) -> list[str]: + """Return what is wrong with the sysroot's mount path: it must be an absolute path that can sit in a colon-separated search path, apart from the other directories the capabilities mount.""" + if not isinstance(sysroot_path, str) or not sysroot_path.startswith("/") or sysroot_path == "/" or any(character in sysroot_path for character in ":\n\r"): + return [f"the sysroot path must be an absolute directory with no ':' or line break, since it is joined into search paths such as LD_LIBRARY_PATH (got {sysroot_path!r})"] + return _overlap_errors("sysroot path", sysroot_path, others) + + def _sysroot_env(sysroot_path: str, capabilities: Sequence[tuple[str, Mapping[str, Any]]]) -> str: """Return the job-started hook's sysroot-env: a NAME+=directory line for each search path the sysroot extends (the hook prepends it to the variable, and PATH goes through GITHUB_PATH), then a NAME=value line for each variable the system-library capabilities set, in the profile's order.""" root = sysroot_path.rstrip("/") @@ -806,7 +840,47 @@ def _copy_init_container(name: str, image: str, payload: str, volume: str, mount } -def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, sysroot_path: str, runner_user: int, runner_group: int, hooks_dir: str, hooks_configmap: str) -> dict[str, Any]: +def _node_cache_path_errors(node_cache_path: str, node_cache_host_path: str, others: Mapping[str, str]) -> list[str]: + """Return what is wrong with the node cache's paths: the mount path must be an absolute directory apart from the other directories the capabilities mount, and the host path an absolute directory.""" + errors: list[str] = [] + if not isinstance(node_cache_path, str) or not node_cache_path.startswith("/") or node_cache_path == "/": + errors.append(f"the node cache path must be an absolute directory other than / (got {node_cache_path!r})") + else: + errors.extend(_overlap_errors("node cache path", node_cache_path, others)) + if not isinstance(node_cache_host_path, str) or not node_cache_host_path.startswith("/") or node_cache_host_path == "/": + errors.append(f"the node cache's host path must be an absolute directory other than / (got {node_cache_host_path!r})") + return errors + + +def _node_cache_env(node_cache_path: str, capabilities: Sequence[tuple[str, Mapping[str, Any]]], errors: list[str]) -> list[dict[str, str]]: + """Return the runner container's variables for a profile's node-cache capabilities: each dirs variable naming its subdirectory of the node cache, and each env variable as given. A variable that two capabilities set to different values is recorded in errors.""" + root = node_cache_path.rstrip("/") + values: dict[str, tuple[str, str]] = {} + for name, capability in capabilities: + entries = {variable: f"{root}/{subdirectory}" for variable, subdirectory in capability["node_cache"]["dirs"].items()} + entries.update(capability.get("env") or {}) + for variable, value in entries.items(): + if variable in values and values[variable][1] != value: + errors.append(f"node-cache capabilities {values[variable][0]} and {name} set {variable} to different values") + values.setdefault(variable, (name, value)) + return [{"name": variable, "value": value} for variable, (_, value) in values.items()] + + +def _node_cache_init_container(name: str, capability: Mapping[str, Any], node_cache_path: str, runner_user: int, runner_group: int) -> dict[str, Any]: + """Return the init container that creates a node-cache capability's subdirectories, owned by the runner's user. It runs as root, since the node's directory belongs to root until a pod claims it, and keeps only the one capability that needs: chown.""" + root = node_cache_path.rstrip("/") + directories = [f"{root}/{subdirectory}" for subdirectory in capability["node_cache"]["dirs"].values()] + return { + "name": f"{_CAPABILITY_CONTAINER_PREFIX}{name}", + "image": capability["node_cache"]["init_image"], + # The image contract: a POSIX sh, mkdir and chown. chown is not recursive: what a pod creates under a directory is already the runner user's. + "command": ["/bin/sh", "-c", 'owner=$1; shift; for directory in "$@"; do mkdir -p "$directory" && chown "$owner" "$directory"; done', "prepare-node-cache", f"{int(runner_user)}:{int(runner_group)}", *directories], + "securityContext": {"runAsUser": 0, "runAsNonRoot": False, "allowPrivilegeEscalation": False, "readOnlyRootFilesystem": True, "capabilities": {"drop": ["ALL"], "add": ["CHOWN"]}}, + "volumeMounts": [{"name": _NODE_CACHE_VOLUME, "mountPath": node_cache_path}], + } + + +def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogue: Mapping[str, Any], tool_cache_path: str, sysroot_path: str, node_cache_path: str, node_cache_host_path: str, runner_user: int, runner_group: int, hooks_dir: str, hooks_configmap: str) -> dict[str, Any]: """Add a profile's capabilities to its scale set's Helm values. Each tool-cache capability becomes an init container, running its image as the runner's user, that copies the image's /toolcache payload into an emptyDir mounted on the runner container at tool_cache_path, which RUNNER_TOOL_CACHE names; the files then belong to the runner user, so a setup action can still add a version beside them. Each system-library capability likewise becomes an init container that copies the image's /sysroot payload into an emptyDir mounted at sysroot_path; several share it, later ones overwriting a file an earlier one also wrote. Hook capabilities, the path of any tool-cache capability that sets one, and the sysroot's environment (its search paths and the capabilities' env) go into a ConfigMap mounted at hooks_dir, whose dispatcher ACTIONS_RUNNER_HOOK_JOB_STARTED names. Everything is appended after what the values already hold, so a values file's own init containers, volumes, environment and mounts are kept. Lists are replaced wholesale when Helm merges values, which is why this edits the merged values rather than adding an overlay. @@ -831,8 +905,11 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu tool_capabilities = [(name, catalogue[name]) for name in names if catalogue[name].get("image") is not None] sysroot_capabilities = [(name, catalogue[name]) for name in names if catalogue[name].get("sysroot_image") is not None] hook_capabilities = [(name, catalogue[name]) for name in names if catalogue[name].get("job_started_hook") is not None] + node_cache_capabilities = [(name, catalogue[name]) for name in names if catalogue[name].get("node_cache") is not None] if sysroot_capabilities: errors.extend(_sysroot_path_errors(sysroot_path, {"tool cache path": tool_cache_path, "job-started hooks directory": hooks_dir})) + if node_cache_capabilities: + errors.extend(_node_cache_path_errors(node_cache_path, node_cache_host_path, {"tool cache path": tool_cache_path, "sysroot path": sysroot_path, "job-started hooks directory": hooks_dir})) tool_paths = [capability["path"] for _, capability in tool_capabilities if capability.get("path") is not None] hooks: dict[str, str] = {} if tool_paths: @@ -853,6 +930,10 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu if sysroot_capabilities: volumes.append({"name": _SYSROOT_VOLUME, "emptyDir": {}}) mounts.append({"name": _SYSROOT_VOLUME, "mountPath": sysroot_path}) + if node_cache_capabilities: + volumes.append({"name": _NODE_CACHE_VOLUME, "hostPath": {"path": node_cache_host_path, "type": "DirectoryOrCreate"}}) + mounts.append({"name": _NODE_CACHE_VOLUME, "mountPath": node_cache_path}) + env.extend(_node_cache_env(node_cache_path, node_cache_capabilities, errors)) if hooks or scripts: items = [{"key": _HOOK_DISPATCHER, "path": _HOOK_DISPATCHER}] + [{"key": key, "path": key} for key in hooks] + [{"key": key, "path": f"{_HOOK_SCRIPTS_DIR}/{key}"} for key in scripts] volumes.append({"name": _HOOKS_VOLUME, "configMap": {"name": hooks_configmap, "defaultMode": _HOOK_FILE_MODE, "items": items}}) @@ -873,6 +954,8 @@ def arc_capability_pod(values: Mapping[str, Any], names: Sequence[str], catalogu _copy_init_container(name, catalogue[name]["image"], _TOOL_IMAGE_PAYLOAD, _TOOL_CACHE_VOLUME, tool_cache_path, runner_user, runner_group) if catalogue[name].get("image") is not None else _copy_init_container(name, catalogue[name]["sysroot_image"], _SYSROOT_IMAGE_PAYLOAD, _SYSROOT_VOLUME, sysroot_path, runner_user, runner_group) + if catalogue[name].get("sysroot_image") is not None + else _node_cache_init_container(name, catalogue[name], node_cache_path, runner_user, runner_group) for name in names if catalogue[name].get("job_started_hook") is None ] diff --git a/roles/github_runner_arc/README.md b/roles/github_runner_arc/README.md index b4158ee..f1f2881 100644 --- a/roles/github_runner_arc/README.md +++ b/roles/github_runner_arc/README.md @@ -27,6 +27,7 @@ Before touching the cluster the role then checks the secrets it is about to writ - `burst`: let the runner pods overflow onto nodes a cluster autoscaler adds on demand, and count those nodes' runners into `maxRunners` (see [Burst nodes](#burst-nodes)). Needs `sizing`, and no `max_runners` or `autoscale`; with `exclusive: true` the pods run on burst nodes only and `burst.max_runners` is the whole ceiling, with no `sizing`. - `github_runner_arc_values_dir`: the control-node directory relative `values_file` paths are read from. - `github_runner_arc_capabilities`: the capabilities profiles can name, as a mapping of name to capability (see [Capabilities](#capabilities)). `github_runner_arc_tool_cache_path` (default `/opt/hostedtoolcache`) is where the tool cache is mounted, `github_runner_arc_sysroot_path` (default `/opt/sysroot`) is where system-library capabilities are extracted, and `github_runner_arc_runner_uid` and `github_runner_arc_runner_gid` (default `1001`, the runner user of ARC's `actions-runner` image) are the ids the capabilities' init containers run as. +- `github_runner_arc_node_cache_path`, `github_runner_arc_node_cache_host_path`, `github_runner_arc_node_cache_size_limit_gib`, `github_runner_arc_node_cache_prune_seconds`: where a node-cache capability's volume is mounted on the runner container (`/node-cache`), the directory on each node under which each profile's cache lives (`/var/lib/github-runner/node-cache`), the most gibibytes a profile's cache may hold before the pruner empties it (20, and 0 for no pruner), and the seconds between the pruner's checks (300). See [Capabilities](#capabilities). - `github_runner_arc_registry_cache_enabled` (default `false`) and the other `github_runner_arc_registry_cache_*` variables: a pull-through registry cache every Docker-in-Docker profile pulls through, falling back to each registry when the cache is down (see [Registry cache](#registry-cache)). - `github_runner_arc_kubeconfig_path`: the kubeconfig on the host the role runs against. Defaults to `github_runner_cluster`'s kubeconfig; empty means `KUBECONFIG` or `~/.kube/config`. - `github_runner_arc_controller_chart_version`, `github_runner_arc_scaleset_chart_version`: chart pins; unset installs the latest chart. The controller chart's CRDs are applied (server-side) from `github_runner_arc_controller_chart_ref` at the controller's version before every controller install or upgrade, because Helm never upgrades a chart's CRDs itself; without that, a newer scale-set chart is rejected for fields the cluster's CRDs predate. Move the two pins together, and the CRDs follow. `tests/arc_upgrade/run.sh` covers the upgrade in a kind cluster. @@ -113,7 +114,7 @@ The runner pods then lose the eligibility `nodeSelector`, and get instead a requ ## Capabilities -A capability adds something to a profile's runner pods when they start, so the runner image itself can stay a public, stock image such as `ghcr.io/actions/actions-runner`. `github_runner_arc_capabilities` defines them by name, and each profile lists the ones it wants in `capabilities`. There are three kinds. +A capability adds something to a profile's runner pods when they start, so the runner image itself can stay a public, stock image such as `ghcr.io/actions/actions-runner`. `github_runner_arc_capabilities` defines them by name, and each profile lists the ones it wants in `capabilities`. There are four kinds. A tool-cache capability, `{image, path?}`, brings a tool. For each one the runner pod gets an init container, named `capability-`, that copies the tool from the image into an `emptyDir` mounted on the runner container at `github_runner_arc_tool_cache_path`, and `RUNNER_TOOL_CACHE` is set to that path. The runner hands `RUNNER_TOOL_CACHE` to every step, and setup actions that consult the Actions tool cache before downloading, such as `actions/setup-node` and `terraform-linters/setup-tflint`, then find the tool there and skip the download, with no change to the workflow. The init containers run as the runner's user, so the copies belong to it and a setup action can still add another version beside them. `path`, `/` optionally followed by `/`, also puts the tool on `PATH` before the job's first step, for a tool no setup action looks up (Terraform, whose `hashicorp/setup-terraform` always downloads, or the AWS CLI). @@ -121,6 +122,25 @@ A system-library capability, `{sysroot_image, env?}`, brings shared libraries, t A hook capability, `{job_started_hook}`, is a bash script the runner runs before each job's first step, for anything small: configuring git, writing a proxy setting to `GITHUB_ENV`, checking a mount. +A node-cache capability, `{node_cache: {init_image, dirs}, env?}`, gives the profile's runner pods on a node one directory of that node to share, for what is expensive to rebuild in every pod and safe to share: a package manager's content-addressed store. The runner pod gets a `hostPath` volume, `//` on the node (so profiles never share a cache), mounted once on the runner container at `github_runner_arc_node_cache_path` (`/node-cache` by default). An init container named `capability-`, run as root from the pinned `init_image` (it needs a POSIX `sh`, `mkdir` and `chown`; with only the `CHOWN` capability kept), creates each subdirectory `dirs` names and gives it to the runner's user. `dirs` maps an environment variable to a subdirectory, and the variable is set on the runner container to its path, so `pnpm_config_store_dir: pnpm` makes pnpm keep its store in `/node-cache/pnpm`; `env` sets further variables literally. All the node-cache capabilities of a profile share the one volume. + +The cache outlives the pod, so the next job on the node starts with it filled, and a job on a node whose cache is empty fills it as it goes. It is the workflow's own cache step that remains the fallback: a workflow checks that the variable's path exists before relying on it, so it still runs on a runner without the capability, such as a GitHub-hosted one. The role keeps each cache under `github_runner_arc_node_cache_size_limit_gib` with a pruner DaemonSet (`scripts/node-cache-pruner.sh`, from the autoscaler image) that empties a profile's whole cache, never part of it, and only while no runner pod of that profile runs on the node, since every running job may be linked into it. `github_runner_arc_node_cache_size_limit_gib: 0` installs no pruner. + +Two limits shape how a package manager uses it. First, Linux refuses a hard link between two mount points even of one filesystem, so a store in the node cache cannot be hard-linked into a workspace that sits on the pod's own volume (this was checked against two bind mounts of one directory, and against a subdirectory mounted separately, which is what a `subPath` is). The runner's workspace path is fixed by the runner, so it cannot be moved onto the shared mount, and a workspace shared by the pods would collide on the runner's own `_temp` and `_actions` directories. A store therefore has to be one that links with symbolic links, not hard links; pnpm's global virtual store (`enable-global-virtual-store`) is, and its `links` directory sits beside the store. This was checked with pnpm: with the global virtual store on, a project's `node_modules` holds symbolic links into `/v11/links` and a few metadata files, and a second project installing the same dependencies writes no package files. Second, the cache is shared writable state: a job that changes a file under it changes it for every later job on the node. pnpm checks the files it imports into its store (`verify-store-integrity`, on by default), but not the linked directories a global virtual store serves, so the capability is opt-in per profile for that reason, and a profile that runs untrusted workflows, such as pull requests from forks, should not have one. + +```yaml +github_runner_arc_capabilities: + pnpm-store: + node_cache: + init_image: docker.io/library/busybox:1.37.0 + dirs: + pnpm_config_store_dir: pnpm + env: + pnpm_config_enable_global_virtual_store: "true" +``` + +The `hostPath` volume needs a cluster that admits one: a namespace that enforces Pod Security's baseline or restricted level refuses it. + ```yaml github_runner_arc_capabilities: node-24: @@ -257,7 +277,7 @@ When a profile has a hook capability, a `path` or a system-library capability, t ### Where capabilities apply -The tool cache, the sysroot and the hook are on the runner container, so they serve the steps that run there: every step in the default container mode and in Docker-in-Docker mode (`container_mode: dind`). A job that runs in a `container:` reaches the tool cache only through the runner's own mapping of it, which in Docker-in-Docker mode points at the Docker daemon's filesystem, where the cache is not mounted, and it does not see the sysroot at all. The `kubernetes` container modes run every step in a separate pod, so the role refuses capabilities on a profile whose values use them. It also refuses them when the values define no runner container (the role's own template does), when the runner container already sets `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, when a volume already uses the name `tool-cache`, `sysroot` or `job-started-hooks`, or, for a profile with a system-library capability, when `github_runner_arc_sysroot_path` is not an absolute directory without a `:` or overlaps the tool cache or the hooks directory. During validation it refuses a system-library capability whose `env` sets one of the search paths, `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, a name that is not a variable name or a value that is not a single-line string (a line break would add a line to `GITHUB_ENV`), and two of a profile's system-library capabilities that set one variable to different values, since they share one sysroot. A values file's own init containers, volumes, environment and mounts are kept, and the capabilities' are added after them, by the `exadev.github_runner.arc_capability_pod` filter. `exadev.github_runner.arc_capability_errors` checks the catalogue and each profile's references during validation, before anything changes. +The tool cache, the sysroot, the node cache and the hook are on the runner container, so they serve the steps that run there: every step in the default container mode and in Docker-in-Docker mode (`container_mode: dind`). A job that runs in a `container:` reaches the tool cache only through the runner's own mapping of it, which in Docker-in-Docker mode points at the Docker daemon's filesystem, where the cache is not mounted, and it does not see the sysroot at all. The `kubernetes` container modes run every step in a separate pod, so the role refuses capabilities on a profile whose values use them. It also refuses them when the values define no runner container (the role's own template does), when the runner container already sets `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, when a volume already uses the name `tool-cache`, `sysroot` or `job-started-hooks`, or, for a profile with a system-library capability, when `github_runner_arc_sysroot_path` is not an absolute directory without a `:` or overlaps the tool cache or the hooks directory. During validation it refuses a system-library capability whose `env` sets one of the search paths, `RUNNER_TOOL_CACHE` or `ACTIONS_RUNNER_HOOK_JOB_STARTED`, a name that is not a variable name or a value that is not a single-line string (a line break would add a line to `GITHUB_ENV`), and two of a profile's system-library capabilities that set one variable to different values, since they share one sysroot. A values file's own init containers, volumes, environment and mounts are kept, and the capabilities' are added after them, by the `exadev.github_runner.arc_capability_pod` filter. `exadev.github_runner.arc_capability_errors` checks the catalogue and each profile's references during validation, before anything changes. Each tool-cache and system-library capability adds its copy to every pod's start, which is not yet measured. The `emptyDir` volumes live on the node's disk for the pod's lifetime, so a large tool set or sysroot counts against the node's ephemeral storage. diff --git a/roles/github_runner_arc/defaults/main.yml b/roles/github_runner_arc/defaults/main.yml index 6d023e6..8a488bd 100644 --- a/roles/github_runner_arc/defaults/main.yml +++ b/roles/github_runner_arc/defaults/main.yml @@ -19,6 +19,18 @@ github_runner_arc_tool_cache_path: "/opt/hostedtoolcache" # Where the sysroot is mounted on the runner container when a profile has a system-library capability: the prefix the capabilities' payloads are copied into and their search paths point at. A capability whose env names the prefix (OPAL_PREFIX, for example) should take it from this variable. github_runner_arc_sysroot_path: "/opt/sysroot" +# Where the node cache is mounted on the runner container when a profile has a node-cache capability: one directory of the node, shared by the profile's runner pods on that node, that holds each capability's subdirectories (a package manager's content-addressed store, say). It is one mount, which is what keeps a hard link between two of its directories possible, since Linux refuses one across mount points even of the same filesystem. +github_runner_arc_node_cache_path: "/node-cache" + +# Where on the node the cache lives, as the directory every profile's own directory (/) sits under, so profiles never share a cache, and each is a hostPath volume created when a pod first needs it. Its lifetime is the node's: it survives pods, and is emptied by the pruner below when it outgrows its limit. +github_runner_arc_node_cache_host_path: "/var/lib/github-runner/node-cache" + +# The most gibibytes one profile's cache may hold on a node before the pruner empties it. The pruner (scripts/node-cache-pruner.sh, a DaemonSet on the nodes runner pods may use) only ever empties a cache when no runner pod of the profile is running on the node, since a package manager's store is in use by every job that linked from it. 0 installs no pruner and leaves the cache to grow. +github_runner_arc_node_cache_size_limit_gib: 20 + +# Seconds between the pruner's checks of each cache's size. +github_runner_arc_node_cache_prune_seconds: 300 + # The runner container's user and group ids, which the capabilities' init containers run as so the copied tools and libraries belong to the runner user and a setup action can add another version beside them. The defaults are those of the runner user in ARC's actions-runner image; an image with a different runner user sets them to its own. github_runner_arc_runner_uid: 1001 github_runner_arc_runner_gid: 1001 diff --git a/roles/github_runner_arc/meta/argument_specs.yml b/roles/github_runner_arc/meta/argument_specs.yml index 5d67696..738dfed 100644 --- a/roles/github_runner_arc/meta/argument_specs.yml +++ b/roles/github_runner_arc/meta/argument_specs.yml @@ -25,6 +25,7 @@ argument_specs: - A tool-cache capability is {image, path?}, a public image pinned to a version tag or digest whose /toolcache holds a tool in the Actions tool-cache layout; path (/, optionally /) also puts it on PATH at job start. - A system-library capability is {sysroot_image, env?}, a public image pinned to a version tag or digest whose /sysroot is a relocatable prefix; it is copied into github_runner_arc_sysroot_path, whose bin, lib, include, lib/pkgconfig and the prefix itself are prepended to PATH, LD_LIBRARY_PATH, LIBRARY_PATH, CPATH, PKG_CONFIG_PATH and CMAKE_PREFIX_PATH at job start, and env is a mapping of further variables to single-line string values set at job start. - A hook capability is {job_started_hook}, a bash script the runner runs before each job's first step. + - A node-cache capability is {node_cache, env?}, where node_cache is {init_image, dirs}; init_image is a public image pinned to a version tag or digest with a POSIX sh, mkdir and chown, and dirs maps an environment variable name to a subdirectory of the node cache, which the variable then names on the runner container. env sets further variables literally. github_runner_arc_tool_cache_path: type: str default: /opt/hostedtoolcache @@ -33,6 +34,22 @@ argument_specs: type: str default: /opt/sysroot description: Where the sysroot is mounted on the runner container when a profile has a system-library capability, an absolute path with no colon. + github_runner_arc_node_cache_path: + type: str + default: /node-cache + description: Where the node cache is mounted on the runner container when a profile has a node-cache capability, an absolute path with no colon, apart from the tool cache, the sysroot and the job-started hooks directory. + github_runner_arc_node_cache_host_path: + type: str + default: /var/lib/github-runner/node-cache + description: The directory on each node under which every profile's node cache lives, in a directory of its own named /. + github_runner_arc_node_cache_size_limit_gib: + type: int + default: 20 + description: The most gibibytes one profile's node cache may hold on a node before the pruner empties it, which it does only while no runner pod of the profile runs on the node. 0 installs no pruner. + github_runner_arc_node_cache_prune_seconds: + type: int + default: 300 + description: Seconds between the pruner's checks of each node cache's size. github_runner_arc_runner_uid: type: int default: 1001 diff --git a/roles/github_runner_arc/tasks/install_node_cache_pruner.yml b/roles/github_runner_arc/tasks/install_node_cache_pruner.yml new file mode 100644 index 0000000..6dd362f --- /dev/null +++ b/roles/github_runner_arc/tasks/install_node_cache_pruner.yml @@ -0,0 +1,127 @@ +--- +# The node cache pruner (scripts/node-cache-pruner.sh): a DaemonSet that empties a profile's node cache when it outgrows github_runner_arc_node_cache_size_limit_gib and no runner pod of the profile runs on the node. Wanted when the limit is above zero and some profile carries a node-cache capability. + +- name: "Node cache: decide whether the pruner is wanted" + ansible.builtin.set_fact: + github_runner_arc_node_cache_pruner_wanted: >- + {{ + github_runner_arc_node_cache_size_limit_gib | int > 0 + and (github_runner_arc_fleet.profiles | map(attribute='capabilities') | flatten | map('extract', github_runner_arc_capabilities) + | selectattr('node_cache', 'defined') | list | length) > 0 + }} + +- name: "Node cache: remove the pruner when it is not wanted" + when: not github_runner_arc_node_cache_pruner_wanted + kubernetes.core.k8s: + state: absent + api_version: "{{ item.api_version }}" + kind: "{{ item.kind }}" + name: node-cache-pruner + namespace: "{{ item.namespaced | default(true) | ternary(github_runner_arc_platform_namespace, omit) }}" + loop: + - api_version: apps/v1 + kind: DaemonSet + - api_version: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + namespaced: false + - api_version: rbac.authorization.k8s.io/v1 + kind: ClusterRole + namespaced: false + - api_version: v1 + kind: ServiceAccount + +- name: "Node cache: install the pruner" + when: github_runner_arc_node_cache_pruner_wanted + block: + - name: "Node cache: the namespace the pruner runs in" + kubernetes.core.k8s: + definition: + apiVersion: v1 + kind: Namespace + metadata: + name: "{{ github_runner_arc_platform_namespace }}" + + - name: "Node cache: pruner service account and RBAC - it lists the pods on its node before it empties a cache" + kubernetes.core.k8s: + definition: "{{ item }}" + loop: + - apiVersion: v1 + kind: ServiceAccount + metadata: + name: node-cache-pruner + namespace: "{{ github_runner_arc_platform_namespace }}" + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: node-cache-pruner + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["list"] + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: node-cache-pruner + subjects: + - kind: ServiceAccount + name: node-cache-pruner + namespace: "{{ github_runner_arc_platform_namespace }}" + roleRef: + kind: ClusterRole + name: node-cache-pruner + apiGroup: rbac.authorization.k8s.io + + # The autoscaler's own image carries the script. It runs on every node runner pods may use, taints included, as root, since the files it removes belong to the runner's user. + - name: "Node cache: install/upgrade the pruner DaemonSet" + kubernetes.core.k8s: + definition: + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: node-cache-pruner + namespace: "{{ github_runner_arc_platform_namespace }}" + spec: + selector: + matchLabels: + app: node-cache-pruner + template: + metadata: + labels: + app: node-cache-pruner + spec: + serviceAccountName: node-cache-pruner + priorityClassName: "{{ github_runner_arc_platform_priority_class_name }}" + imagePullSecrets: "{{ github_runner_arc_platform_image_pull_secrets }}" + nodeSelector: "{{ {github_runner_arc_node_label_key: github_runner_arc_node_label_value | string} if github_runner_arc_node_label_key | length > 0 else {} }}" + tolerations: + - operator: Exists + containers: + - name: node-cache-pruner + image: "{{ github_runner_arc_autoscaler_image }}" + command: ["/app/scripts/node-cache-pruner.sh"] + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: LIMIT_GIB + value: "{{ github_runner_arc_node_cache_size_limit_gib | string }}" + - name: CHECK_SECONDS + value: "{{ github_runner_arc_node_cache_prune_seconds | string }}" + volumeMounts: + - name: node-cache + mountPath: /host-node-cache + resources: + requests: + cpu: "10m" + memory: "16Mi" + ephemeral-storage: "{{ github_runner_arc_platform_ephemeral_storage_request }}" + limits: + cpu: "200m" + memory: "64Mi" + ephemeral-storage: "{{ github_runner_arc_platform_ephemeral_storage_limit }}" + volumes: + - name: node-cache + hostPath: + path: "{{ github_runner_arc_node_cache_host_path }}" + type: DirectoryOrCreate diff --git a/roles/github_runner_arc/tasks/main.yml b/roles/github_runner_arc/tasks/main.yml index c2036db..993349e 100644 --- a/roles/github_runner_arc/tasks/main.yml +++ b/roles/github_runner_arc/tasks/main.yml @@ -84,6 +84,9 @@ ansible.builtin.include_tasks: install_controller.yml when: github_runner_arc_orgs | length > 0 + - name: Install the node cache pruner, or remove it when no profile has a node cache + ansible.builtin.include_tasks: install_node_cache_pruner.yml + # Before the scale sets, so the cache is serving when their pods first pull; on a host with no scale sets there is nothing for it to serve. - name: Install the registry cache, or remove it when turned off ansible.builtin.include_tasks: install_registry_cache.yml diff --git a/roles/github_runner_arc/tasks/render_scale_set_values.yml b/roles/github_runner_arc/tasks/render_scale_set_values.yml index f17fc4c..bf1c803 100644 --- a/roles/github_runner_arc/tasks/render_scale_set_values.yml +++ b/roles/github_runner_arc/tasks/render_scale_set_values.yml @@ -10,6 +10,7 @@ | combine(lookup('ansible.builtin.template', 'scale-set-overlay.yaml.j2') | from_yaml, recursive=true)) | exadev.github_runner.arc_capability_pod( profile.capabilities, github_runner_arc_capabilities, github_runner_arc_tool_cache_path, github_runner_arc_sysroot_path, + github_runner_arc_node_cache_path, github_runner_arc_node_cache_host_path ~ '/' ~ profile.namespace ~ '/' ~ profile.release, github_runner_arc_runner_uid, github_runner_arc_runner_gid, github_runner_arc_job_started_hooks_dir, github_runner_arc_job_started_configmap) }} diff --git a/scripts/node-cache-pruner.sh b/scripts/node-cache-pruner.sh new file mode 100755 index 0000000..d62edb1 --- /dev/null +++ b/scripts/node-cache-pruner.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Keeps each profile's node cache (see the role README's Capabilities section) under its size limit. Runs in the `node-cache-pruner` DaemonSet, one pod per node a runner pod may use, with the node's cache root mounted read-write. +# +# A cache is empty-or-in-use: a package manager's store is linked from by every job that installed from it, and a job's dependency tree holds links into it, so removing part of it under a running job breaks that job. The pruner therefore never trims a cache. When a profile's cache exceeds the limit it empties the whole cache, and only while no runner pod of that profile runs on this node, so no job can be using it. The next job rebuilds what it needs. +# +# The cache root holds one directory per profile, /; the profile's runner pods all run in , so a pod listed there on this node is a pod that may be using that profile's cache. +# +# Env: +# - NODE_NAME: the node this pod runs on (the downward API's spec.nodeName). Required. +# - CACHE_ROOT: where the node's cache root is mounted (default: /host-node-cache). +# - LIMIT_GIB: the most gibibytes one profile's cache may hold, a positive integer. Required unless LIMIT_KIB is set. +# - LIMIT_KIB: the limit in kibibytes, which takes the place of LIMIT_GIB, so a test need not write gibibytes. +# - CHECK_SECONDS: seconds between checks (default: 300). +# - PRUNE_ONCE: when "true", check once and exit, for tests. +set -euo pipefail + +: "${NODE_NAME:?NODE_NAME is required}" +CACHE_ROOT="${CACHE_ROOT:-/host-node-cache}" +CHECK_SECONDS="${CHECK_SECONDS:-300}" +PRUNE_ONCE="${PRUNE_ONCE:-false}" +LIMIT_KIB="${LIMIT_KIB:-$(( ${LIMIT_GIB:?LIMIT_GIB or LIMIT_KIB is required} * 1024 * 1024 ))}" + +# Empties one profile's cache directory when it is over the limit and no runner pod of the profile runs on this node. +prune() { + local directory=$1 namespace=$2 release=$3 size_kib pods + size_kib="$(du -sk "$directory" | cut -f1)" + [ "$size_kib" -le "$LIMIT_KIB" ] && return 0 + pods="$(kubectl get pods -n "$namespace" --field-selector "spec.nodeName=${NODE_NAME}" -o name)" + if [ -n "$pods" ]; then + echo "node cache ${namespace}/${release} holds ${size_kib} KiB, over the limit of ${LIMIT_KIB} KiB, but a runner pod is running on this node: left alone" + return 0 + fi + find "$directory" -mindepth 1 -delete + echo "node cache ${namespace}/${release} held ${size_kib} KiB, over the limit of ${LIMIT_KIB} KiB: emptied" +} + +check() { + local directory namespace release + for directory in "$CACHE_ROOT"/*/*/; do + [ -d "$directory" ] || continue + directory="${directory%/}" + release="$(basename "$directory")" + namespace="$(basename "$(dirname "$directory")")" + prune "$directory" "$namespace" "$release" + done +} + +while true; do + check + [ "$PRUNE_ONCE" = "true" ] && exit 0 + sleep "$CHECK_SECONDS" +done diff --git a/tests/unit/test_capabilities.py b/tests/unit/test_capabilities.py index 4d6e11d..f9d93ec 100644 --- a/tests/unit/test_capabilities.py +++ b/tests/unit/test_capabilities.py @@ -23,6 +23,8 @@ TOOL_CACHE = "/opt/hostedtoolcache" SYSROOT = "/opt/sysroot" +NODE_CACHE = "/node-cache" +NODE_CACHE_HOST = "/var/lib/example/node-cache/arc-runners/example-runners" HOOKS_DIR = "/etc/github-runner/hooks" CONFIGMAP = "github-runner-job-started" NODE_IMAGE = "ghcr.io/example/toolcache-node:22.11.0" @@ -32,6 +34,10 @@ "mpi": {"sysroot_image": "ghcr.io/example/sysroot-openmpi:5.0.11", "env": {"OPAL_PREFIX": SYSROOT}}, "libpq": {"sysroot_image": "ghcr.io/example/sysroot-libpq@sha256:" + "1" * 64}, "git-identity": {"job_started_hook": "git config --global user.name example\n"}, + "pnpm-store": { + "node_cache": {"init_image": "docker.io/library/busybox:1.37.0", "dirs": {"pnpm_config_store_dir": "pnpm"}}, + "env": {"pnpm_config_enable_global_virtual_store": "true"}, + }, "proxy": {"job_started_hook": "echo HTTPS_PROXY=http://proxy.example:3128 >> \"$GITHUB_ENV\"\n"}, } # The role's default values template's runner container, as it renders for a profile with no values_file. @@ -47,7 +53,7 @@ def expanded(profile: dict[str, Any]) -> dict[str, Any]: def pod(values: dict[str, Any], names: list[str]) -> dict[str, Any]: """Add names' capabilities from CATALOGUE to values with the role's default settings.""" - result: dict[str, Any] = arc.arc_capability_pod(values, names, CATALOGUE, TOOL_CACHE, SYSROOT, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result: dict[str, Any] = arc.arc_capability_pod(values, names, CATALOGUE, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) return result @@ -132,6 +138,28 @@ def test_two_system_libraries_setting_one_variable_differently_in_a_profile_is_a self.assertEqual(self.errors(catalogue, ["a", "c"]), []) self.assertEqual(self.errors(catalogue, ["b"]), []) + def test_bad_node_cache_entries_are_errors(self) -> None: + init = "docker.io/library/busybox:1.37.0" + cases = { + "a node_cache that is not a mapping": ({"store": {"node_cache": init}}, "must be a mapping with init_image and dirs"), + "an unpinned init image": ({"store": {"node_cache": {"init_image": "docker.io/library/busybox:latest", "dirs": {"STORE": "pnpm"}}}}, "must be pinned"), + "no init image": ({"store": {"node_cache": {"dirs": {"STORE": "pnpm"}}}}, "init_image"), + "no dirs": ({"store": {"node_cache": {"init_image": init}}}, "non-empty mapping"), + "an empty dirs": ({"store": {"node_cache": {"init_image": init, "dirs": {}}}}, "non-empty mapping"), + "an unknown key": ({"store": {"node_cache": {"init_image": init, "dirs": {"STORE": "pnpm"}, "size": 1}}}, "unknown key 'size'"), + "a variable name that is not one": ({"store": {"node_cache": {"init_image": init, "dirs": {"my-store": "pnpm"}}}}, "not an environment variable name"), + "a runner variable": ({"store": {"node_cache": {"init_image": init, "dirs": {"RUNNER_TOOL_CACHE": "pnpm"}}}}, "may not set RUNNER_TOOL_CACHE"), + "a subdirectory that leaves the cache": ({"store": {"node_cache": {"init_image": init, "dirs": {"STORE": "../x"}}}}, "must be one directory name"), + "a subdirectory with a slash": ({"store": {"node_cache": {"init_image": init, "dirs": {"STORE": "a/b"}}}}, "must be one directory name"), + "one subdirectory for two variables": ({"store": {"node_cache": {"init_image": init, "dirs": {"A": "pnpm", "B": "pnpm"}}}}, "more than one variable"), + "a node cache and a tool image": ({"store": {"image": NODE_IMAGE, "node_cache": {"init_image": init, "dirs": {"A": "pnpm"}}}}, "exactly one of image"), + "env on a tool image": ({"node": {"image": NODE_IMAGE, "env": {"A": "b"}}}, "only for a system-library capability"), + } + for case, (catalogue, message) in cases.items(): + with self.subTest(case): + errors = self.errors(catalogue) + self.assertTrue(any(message in error for error in errors), errors) + def test_a_digest_pins_an_image(self) -> None: self.assertEqual(self.errors({"node": {"image": "ghcr.io/example/node@sha256:" + "a" * 64}}), []) @@ -274,7 +302,7 @@ def test_every_kind_together_keeps_the_profiles_order_and_shares_one_sysroot(sel self.assertTrue(result["hooks"]["sysroot-env"].endswith("CMAKE_PREFIX_PATH+=/opt/sysroot\nOPAL_PREFIX=/opt/sysroot\n")) def test_the_sysroot_follows_its_configured_path(self) -> None: - result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], {"mpi": {"sysroot_image": "ghcr.io/example/m:1"}}, TOOL_CACHE, "/srv/libs/", 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], {"mpi": {"sysroot_image": "ghcr.io/example/m:1"}}, TOOL_CACHE, "/srv/libs/", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) self.assertEqual(result["errors"], []) self.assertEqual(result["values"]["template"]["spec"]["initContainers"][0]["command"][-1], "/srv/libs/") self.assertEqual(result["hooks"]["sysroot-env"].splitlines()[:2], ["PATH+=/srv/libs/bin", "LD_LIBRARY_PATH+=/srv/libs/lib"]) @@ -290,18 +318,76 @@ def test_a_sysroot_path_that_cannot_carry_a_sysroot_is_an_error(self) -> None: } for case, (path, message) in cases.items(): with self.subTest(case): - result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], CATALOGUE, TOOL_CACHE, path, 1001, 1001, HOOKS_DIR, CONFIGMAP) + result = arc.arc_capability_pod(DEFAULT_VALUES, ["mpi"], CATALOGUE, TOOL_CACHE, path, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) self.assertTrue(any(message in error for error in result["errors"]), result["errors"]) self.assertIs(result["values"], DEFAULT_VALUES) def test_the_sysroot_path_is_not_checked_for_a_profile_without_a_system_library(self) -> None: - self.assertEqual(arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, "relative", 1001, 1001, HOOKS_DIR, CONFIGMAP)["errors"], []) + self.assertEqual(arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, "relative", NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP)["errors"], []) def test_a_volume_already_named_sysroot_is_an_error(self) -> None: values = {"template": {"spec": {"volumes": [{"name": "sysroot", "emptyDir": {}}], "containers": [{"name": "runner"}]}}} result = pod(values, ["libpq"]) self.assertIn("the values already define a volume named sysroot, which capabilities add", result["errors"]) + def test_a_node_cache_capability_mounts_one_node_directory_and_prepares_its_subdirectories_as_the_runner_user(self) -> None: + result = pod(DEFAULT_VALUES, ["pnpm-store"]) + self.assertEqual(result["errors"], []) + spec = result["values"]["template"]["spec"] + self.assertEqual(spec["volumes"], [{"name": "node-cache", "hostPath": {"path": NODE_CACHE_HOST, "type": "DirectoryOrCreate"}}]) + [init] = spec["initContainers"] + self.assertEqual(init["name"], "capability-pnpm-store") + self.assertEqual(init["image"], "docker.io/library/busybox:1.37.0") + self.assertEqual(init["command"][-2:], ["1001:1001", "/node-cache/pnpm"]) + self.assertEqual(init["securityContext"]["runAsUser"], 0) + self.assertEqual(init["securityContext"]["capabilities"], {"drop": ["ALL"], "add": ["CHOWN"]}) + self.assertEqual(init["volumeMounts"], [{"name": "node-cache", "mountPath": NODE_CACHE}]) + container = runner(result["values"]) + self.assertEqual(container["volumeMounts"], [{"name": "node-cache", "mountPath": NODE_CACHE}]) + self.assertEqual({variable["name"]: variable["value"] for variable in container["env"]}, {"pnpm_config_store_dir": "/node-cache/pnpm", "pnpm_config_enable_global_virtual_store": "true"}) + self.assertEqual(result["hooks"], {}) + + def test_a_node_cache_capability_shares_one_volume_with_every_capability_of_the_profile(self) -> None: + catalogue = {**CATALOGUE, "yarn-cache": {"node_cache": {"init_image": "docker.io/library/busybox:1.37.0", "dirs": {"YARN_CACHE_FOLDER": "yarn"}}}} + result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store", "yarn-cache", "node"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + self.assertEqual(result["errors"], []) + spec = result["values"]["template"]["spec"] + self.assertEqual([volume["name"] for volume in spec["volumes"]], ["tool-cache", "node-cache"]) + self.assertEqual([container["name"] for container in spec["initContainers"]], ["capability-pnpm-store", "capability-yarn-cache", "capability-node"]) + self.assertEqual({variable["name"] for variable in runner(result["values"])["env"]}, {"RUNNER_TOOL_CACHE", "pnpm_config_store_dir", "pnpm_config_enable_global_virtual_store", "YARN_CACHE_FOLDER"}) + + def test_a_profile_without_a_node_cache_capability_gets_no_node_volume_and_its_path_is_not_checked(self) -> None: + result = arc.arc_capability_pod(DEFAULT_VALUES, ["node"], CATALOGUE, TOOL_CACHE, SYSROOT, "relative", "also/relative", 1001, 1001, HOOKS_DIR, CONFIGMAP) + self.assertEqual(result["errors"], []) + self.assertNotIn("node-cache", [volume["name"] for volume in result["values"]["template"]["spec"]["volumes"]]) + + def test_node_cache_paths_that_cannot_carry_a_cache_are_errors(self) -> None: + cases = { + "a relative mount path": ("relative", NODE_CACHE_HOST, "absolute directory other than /"), + "the root as mount path": ("/", NODE_CACHE_HOST, "absolute directory other than /"), + "the tool cache's path": (TOOL_CACHE, NODE_CACHE_HOST, "overlaps the tool cache path"), + "inside the sysroot": (f"{SYSROOT}/cache", NODE_CACHE_HOST, "overlaps the sysroot path"), + "a relative host path": (NODE_CACHE, "var/lib/cache", "host path must be an absolute directory"), + "the root as host path": (NODE_CACHE, "/", "host path must be an absolute directory"), + } + for case, (mount, host, message) in cases.items(): + with self.subTest(case): + result = arc.arc_capability_pod(DEFAULT_VALUES, ["pnpm-store"], CATALOGUE, TOOL_CACHE, SYSROOT, mount, host, 1001, 1001, HOOKS_DIR, CONFIGMAP) + self.assertTrue(any(message in error for error in result["errors"]), result["errors"]) + + def test_two_node_cache_capabilities_setting_one_variable_differently_is_an_error(self) -> None: + catalogue = { + "a": {"node_cache": {"init_image": "docker.io/library/busybox:1.37.0", "dirs": {"STORE": "one"}}}, + "b": {"node_cache": {"init_image": "docker.io/library/busybox:1.37.0", "dirs": {"STORE": "two"}}}, + } + result = arc.arc_capability_pod(DEFAULT_VALUES, ["a", "b"], catalogue, TOOL_CACHE, SYSROOT, NODE_CACHE, NODE_CACHE_HOST, 1001, 1001, HOOKS_DIR, CONFIGMAP) + self.assertEqual(result["errors"], ["node-cache capabilities a and b set STORE to different values"]) + + def test_a_volume_already_named_node_cache_is_an_error(self) -> None: + values = {"template": {"spec": {"containers": [{"name": "runner"}], "volumes": [{"name": "node-cache", "emptyDir": {}}]}}} + result = pod(values, ["pnpm-store"]) + self.assertEqual(result["errors"], ["the values already define a volume named node-cache, which capabilities add"]) + def test_dind_mode_keeps_the_capabilities_beside_the_charts_own_containers(self) -> None: # In dind mode the chart builds the runner and dind containers itself but keeps the values' init containers, volumes (other than work) and the runner's env and mounts, so the capabilities are added the same way. result = pod({**DEFAULT_VALUES, "containerMode": {"type": "dind"}}, ["node"]) @@ -344,6 +430,14 @@ def test_the_role_renders_a_system_library_capability_at_its_sysroot_path(self) self.assertEqual(set(rendered["job_started"]), {"job-started.sh", "sysroot-env"}) self.assertIn("LD_LIBRARY_PATH+=/opt/sysroot/lib\n", rendered["job_started"]["sysroot-env"]) + def test_the_role_renders_a_node_cache_capability_into_a_per_profile_node_directory(self) -> None: + rendered = self.render({"max_runners": 1, "capabilities": ["pnpm-store"]}) + profile = expanded({"max_runners": 1}) + [volume] = rendered["values"]["template"]["spec"]["volumes"] + self.assertEqual(volume["hostPath"], {"path": f"/var/lib/github-runner/node-cache/{profile['namespace']}/{profile['release']}", "type": "DirectoryOrCreate"}) + self.assertEqual(runner(rendered["values"])["volumeMounts"], [{"name": "node-cache", "mountPath": "/node-cache"}]) + self.assertEqual(rendered["job_started"], {}) + def test_a_profile_without_capabilities_renders_as_before(self) -> None: rendered = self.render({"max_runners": 3, "container_mode": "dind"}) self.assertEqual(rendered["job_started"], {}) diff --git a/tests/unit/test_node_cache_pruner.py b/tests/unit/test_node_cache_pruner.py new file mode 100644 index 0000000..b1539a8 --- /dev/null +++ b/tests/unit/test_node_cache_pruner.py @@ -0,0 +1,92 @@ +"""Tests for scripts/node-cache-pruner.sh: a cache over its limit is emptied only when no runner pod runs on the node in its profile's namespace, with a stand-in for kubectl that answers from a scripted pod list.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "node-cache-pruner.sh" + +# Answers `get pods` for a namespace in FAKE_BUSY_NAMESPACES, when asked about this node, with one pod; every other question gets no pods. +FAKE_KUBECTL = textwrap.dedent( + """\ + #!/usr/bin/env python3 + import os, sys + args = sys.argv[1:] + if args[:2] == ["get", "pods"] and "-n" in args and "--field-selector" in args: + open(os.environ["FAKE_CALLS_OUT"], "a").write(" ".join(args) + "\\n") + namespace = args[args.index("-n") + 1] + selector = args[args.index("--field-selector") + 1] + if namespace in os.environ.get("FAKE_BUSY_NAMESPACES", "").split(",") and selector == "spec.nodeName=" + os.environ["NODE_NAME"]: + print("pod/runner-abc") + sys.exit(0) + sys.stderr.write("unexpected kubectl call: %s\\n" % args) + sys.exit(2) + """ +) + +# The limit, in KiB, every case runs against: a cache of OVER_KIB exceeds it and one of UNDER_KIB does not. +LIMIT_KIB = 64 +OVER_KIB = 128 +UNDER_KIB = 16 + + +class PrunerTest(unittest.TestCase): + def run_pruner(self, caches: dict[str, int], busy: list[str]) -> tuple[Path, str, str]: + """Run one check over caches (/ mapped to the KiB of one file under a pnpm directory), with runner pods running on the node in the busy namespaces; return the cache root, the script's output and the pod lists it asked for.""" + work = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, work, True) + root = work / "root" + for name, size_kib in caches.items(): + (root / name / "pnpm").mkdir(parents=True) + (root / name / "pnpm" / "blob").write_bytes(b"x" * (size_kib * 1024)) + bin_dir = work / "bin" + bin_dir.mkdir() + kubectl = bin_dir / "kubectl" + kubectl.write_text(FAKE_KUBECTL) + kubectl.chmod(0o755) + calls = work / "calls.txt" + calls.write_text("") + env = {**os.environ, "PATH": f"{bin_dir}:{os.environ['PATH']}", "NODE_NAME": "node-a", "CACHE_ROOT": str(root), "LIMIT_KIB": str(LIMIT_KIB), "PRUNE_ONCE": "true", "FAKE_CALLS_OUT": str(calls), "FAKE_BUSY_NAMESPACES": ",".join(busy)} + completed = subprocess.run(["bash", str(SCRIPT)], check=True, env=env, capture_output=True, text=True) + return root, completed.stdout, calls.read_text() + + def test_a_cache_over_the_limit_with_no_runner_pod_on_the_node_is_emptied_and_its_directory_kept(self) -> None: + root, output, _ = self.run_pruner({"arc-a/main": OVER_KIB}, busy=[]) + self.assertTrue((root / "arc-a" / "main").is_dir()) + self.assertEqual(list((root / "arc-a" / "main").iterdir()), []) + self.assertIn("arc-a/main", output) + self.assertIn("emptied", output) + + def test_a_cache_over_the_limit_is_left_alone_while_a_runner_pod_of_its_profile_runs_on_the_node(self) -> None: + root, output, _ = self.run_pruner({"arc-a/main": OVER_KIB}, busy=["arc-a"]) + self.assertTrue((root / "arc-a" / "main" / "pnpm" / "blob").exists()) + self.assertIn("left alone", output) + + def test_a_runner_pod_of_another_profile_does_not_protect_a_cache(self) -> None: + root, _, _ = self.run_pruner({"arc-a/main": OVER_KIB, "arc-b/main": OVER_KIB}, busy=["arc-b"]) + self.assertEqual(list((root / "arc-a" / "main").iterdir()), []) + self.assertTrue((root / "arc-b" / "main" / "pnpm" / "blob").exists()) + + def test_a_cache_under_the_limit_is_left_alone_and_no_pod_is_listed(self) -> None: + root, output, calls = self.run_pruner({"arc-a/main": UNDER_KIB}, busy=[]) + self.assertTrue((root / "arc-a" / "main" / "pnpm" / "blob").exists()) + self.assertEqual(calls, "") + self.assertEqual(output, "") + + def test_each_profile_is_asked_about_in_its_own_namespace_on_this_node(self) -> None: + _, _, calls = self.run_pruner({"arc-a/main": OVER_KIB, "arc-b/other": OVER_KIB}, busy=[]) + self.assertEqual(sorted(calls.splitlines()), ["get pods -n arc-a --field-selector spec.nodeName=node-a -o name", "get pods -n arc-b --field-selector spec.nodeName=node-a -o name"]) + + def test_a_root_with_no_caches_is_not_an_error(self) -> None: + _, output, calls = self.run_pruner({}, busy=[]) + self.assertEqual((output, calls), ("", "")) + + +if __name__ == "__main__": + unittest.main()