feat(arc): bound each node's measured runners by its pod capacity

Measured sizing counted what a node's cpu and memory would hold and nothing else, so a
node whose kubelet max-pods was set lower, to keep a node with a slow disk from running more
jobs at once than it can serve, was still measured at its cpu and memory. The node's
allocatable pods, less the pods on it that are not runner pods, now bound its runners
as well. A node that reports no pod capacity is left as before.

Author: Mearman

plugins/filter/arc.py

@@ -324,7 +324,7 @@ def arc_measured_max_runners(sizing: Mapping[str, Any], nodes: Sequence[Mapping[
         tolerations: the runner pods' tolerations; a node whose only repelling taints they match is eligible, as it is for the pods.
 
     Returns:
-        A dict with ``errors`` (messages, empty when it could be worked out), ``nodes`` (per eligible node: name, allocatable_cpu, allocatable_memory_gib, requested_cpu, requested_memory_gib, by_cpu, by_memory, runners, and condition, naming any passing condition it was counted despite), ``skipped_nodes`` (name and reason for each node that is not eligible), ``theoretical`` and ``max_runners``, which is None when there are errors.
+        A dict with ``errors`` (messages, empty when it could be worked out), ``nodes`` (per eligible node: name, allocatable_cpu, allocatable_memory_gib, requested_cpu, requested_memory_gib, by_cpu, by_memory, by_pods (None when the node reports no pod capacity), runners, and condition, naming any passing condition it was counted despite), ``skipped_nodes`` (name and reason for each node that is not eligible), ``theoretical`` and ``max_runners``, which is None when there are errors.
     """
     checked = arc_max_runners(sizing)
     errors = list(checked["errors"])
@@ -339,6 +339,8 @@ def arc_measured_max_runners(sizing: Mapping[str, Any], nodes: Sequence[Mapping[
     reserve_cpu = Decimal(str(checked["reserve_cpu"]))
     reserve_memory = Decimal(str(checked["reserve_memory_gib"])) * _GIB
     requested: dict[str, list[Decimal]] = {}
+    # Pods other than runner pods on each node, which hold slots of the node's pod capacity the runners cannot have.
+    other_pods: dict[str, int] = {}
     for pod in pods:
         metadata = pod.get("metadata") or {}
         node_name = (pod.get("spec") or {}).get("nodeName")
@@ -352,6 +354,7 @@ def arc_measured_max_runners(sizing: Mapping[str, Any], nodes: Sequence[Mapping[
         totals = requested.setdefault(node_name, [Decimal(0), Decimal(0)])
         totals[0] += cpu
         totals[1] += memory
+        other_pods[node_name] = other_pods.get(node_name, 0) + 1
     eligible: list[dict[str, Any]] = []
     skipped: list[dict[str, str]] = []
     for node in sorted(nodes, key=lambda item: (item.get("metadata") or {}).get("name", "")):
@@ -370,6 +373,14 @@ def arc_measured_max_runners(sizing: Mapping[str, Any], nodes: Sequence[Mapping[
         used_cpu, used_memory = requested.get(name, [Decimal(0), Decimal(0)])
         by_cpu = max(0, _floor((allocatable_cpu - used_cpu - reserve_cpu) / pod_cpu))
         by_memory = max(0, _floor((allocatable_memory - used_memory - reserve_memory) / pod_memory))
+        # A node's pod capacity (the kubelet's max-pods) can be set below what its cpu and memory would hold, to keep a node whose disk is the limit from running more jobs than it can serve. Kubernetes always reports it, so a node without it is a test fixture or an unusual one, and is not limited by it.
+        by_pods = None
+        if "pods" in allocatable:
+            try:
+                by_pods = max(0, int(arc_quantity(allocatable["pods"])) - other_pods.get(name, 0))
+            except ValueError as error:
+                errors.append(f"node {name}: {error}")
+                continue
         eligible.append({
             "name": name,
             "allocatable_cpu": float(allocatable_cpu),
@@ -378,7 +389,8 @@ def arc_measured_max_runners(sizing: Mapping[str, Any], nodes: Sequence[Mapping[
             "requested_memory_gib": float(used_memory / _GIB),
             "by_cpu": by_cpu,
             "by_memory": by_memory,
-            "runners": min(by_cpu, by_memory),
+            "by_pods": by_pods,
+            "runners": min(by_cpu, by_memory) if by_pods is None else min(by_cpu, by_memory, by_pods),
             "condition": condition,
         })
     if not eligible and not errors:

roles/github_runner_arc/README.md

@@ -72,7 +72,7 @@ sizing:
   safety_margin: 0.8                # optional, default 1
 ```
 
-That assumes the nodes are alike. Where their free capacity differs a lot, `measured: true` works it out from the cluster instead, each time the role runs. It reads every node the runner pods may be placed on: every node carrying the eligibility label and the profile's `node_selector`, with no `NoSchedule` or `NoExecute` taint other than those Kubernetes adds for a passing condition. The ceiling lasts until the next run, so a node that is not ready, unreachable or cordoned when it is measured still counts, as it will once it is back. For each one it takes the node's allocatable CPU and memory, subtracts what the pods already on it request and the per-node reserve, and divides what is left by one runner pod's requests; the tighter of the two gives the runners that fit there. The sum over the nodes, scaled by the safety margin and rounded down, is the ceiling. Pods in any scale set's namespace are left out, so running runners never lower the ceiling they are sized by, as are pods that have finished. The reserve covers what requests do not show, such as a workload that uses more than it requests. The kubeconfig needs cluster-wide read access to nodes and pods. The role prints each node's figures when it runs.
+That assumes the nodes are alike. Where their free capacity differs a lot, `measured: true` works it out from the cluster instead, each time the role runs. It reads every node the runner pods may be placed on: every node carrying the eligibility label and the profile's `node_selector`, with no `NoSchedule` or `NoExecute` taint other than those Kubernetes adds for a passing condition. The ceiling lasts until the next run, so a node that is not ready, unreachable or cordoned when it is measured still counts, as it will once it is back. For each one it takes the node's allocatable CPU and memory, subtracts what the pods already on it request and the per-node reserve, and divides what is left by one runner pod's requests; the tighter of the two gives the runners that fit there. The sum over the nodes, scaled by the safety margin and rounded down, is the ceiling. Pods in any scale set's namespace are left out, so running runners never lower the ceiling they are sized by, as are pods that have finished. The reserve covers what requests do not show, such as a workload that uses more than it requests. The kubeconfig needs cluster-wide read access to nodes and pods. The role prints each node's figures when it runs. A node's pod capacity also bounds it: the kubelet's `max-pods` (`allocatable.pods`) less the pods already on it that are not runner pods, so a node whose pod limit is set below what its CPU and memory would hold, to keep a node with a slow disk from running more jobs at once than it can serve, is measured at that limit.
 
 ```yaml
 sizing:

roles/github_runner_arc/meta/argument_specs.yml

@@ -14,7 +14,7 @@ argument_specs:
           - max_runners, when set, is the release's maxRunners and takes precedence over its values file and its sizing.
           - scale_set_labels replaces runs_on_label with the whole list of scaleSetLabels; an empty list sets none, so jobs target the release name.
           - tolerations is a list of Kubernetes tolerations for the profile's runner pods, for nodes the eligibility label selects that also carry a taint to keep other workloads off. Measured sizing counts a node whose only repelling taints these match, as the pods can be placed there; without them it leaves such a node out. With burst they come before burst.tolerations.
-          - sizing derives a runner ceiling from uniform node figures, or with measured true from each eligible node's allocatable CPU and memory less its current requests and reserve_cpu and reserve_memory_gib.
+          - sizing derives a runner ceiling from uniform node figures, or with measured true from each eligible node's allocatable CPU and memory (and pod capacity, the kubelet's max-pods) less its current requests and reserve_cpu and reserve_memory_gib.
           - capabilities names entries of github_runner_arc_capabilities that the profile's runner pods carry, in order.
           - burst, on a profile with sizing and without max_runners or autoscale, lets its runner pods overflow onto nodes carrying burst.node_label_key (any value, or one of burst.node_label_values), with burst.tolerations for their taint, and adds burst.max_runners to the sized maxRunners. With burst.exclusive true (and no sizing, max_runners or autoscale) the pods run on burst nodes only and burst.max_runners is the whole maxRunners.
       github_runner_arc_capabilities:

tests/unit/test_arc_filters.py

@@ -343,6 +343,29 @@ def test_invalid_input_is_reported(self) -> None:
         bad = arc.arc_measured_max_runners(MEASURED, [node("a", cpu="lots")], [])
         self.assertEqual(bad["errors"], ["node a: not a Kubernetes quantity: 'lots'"])
 
+    def test_a_nodes_pod_capacity_bounds_its_runners_below_what_cpu_and_memory_would_hold(self) -> None:
+        capped = node("capped", "16", "64Gi")
+        capped["status"]["allocatable"]["pods"] = "2"
+        result = arc.arc_measured_max_runners(MEASURED, [capped, node("free", "4", "8Gi")], [])
+        by_name = {entry["name"]: entry for entry in result["nodes"]}
+        self.assertEqual((by_name["capped"]["by_cpu"], by_name["capped"]["by_pods"], by_name["capped"]["runners"]), (16, 2, 2))
+        self.assertIsNone(by_name["free"]["by_pods"])
+        self.assertEqual(by_name["free"]["runners"], 4)
+
+    def test_pods_that_are_not_runners_use_up_a_nodes_pod_capacity(self) -> None:
+        capped = node("capped", "16", "64Gi")
+        capped["status"]["allocatable"]["pods"] = "3"
+        pods = [pod("capped", "10m", "10Mi", namespace="kube-system"), pod("capped", "1", "1Gi", namespace="arc-runners")]
+        result = arc.arc_measured_max_runners(MEASURED, [capped], pods, None, ["arc-runners"])
+        # Three slots, one held by the kube-system pod; the runner pod is left out, so running jobs never lower the ceiling.
+        self.assertEqual(result["nodes"][0]["by_pods"], 2)
+
+    def test_a_node_with_no_pod_slots_left_contributes_nothing(self) -> None:
+        full = node("full", "16", "64Gi")
+        full["status"]["allocatable"]["pods"] = "1"
+        result = arc.arc_measured_max_runners(MEASURED, [full, node("other", "2", "4Gi")], [pod("full", "10m", "10Mi", namespace="kube-system")])
+        self.assertEqual([entry["runners"] for entry in result["nodes"]], [0, 2])
+
     def test_a_node_whose_taint_the_profile_tolerates_counts_and_one_it_does_not_still_does_not(self) -> None:
         nodes = [
             node("home", labels={"ci": "true"}, taints=[{"key": "example.com/ci-runner-only", "value": "true", "effect": "NoSchedule"}]),