NUMA trap-node cordoner for TopologyAffinityError livelock (#696)

osdc/modules/zombie-cleanup/deploy.sh

@@ -49,6 +49,9 @@ RUNNING_MAX_AGE=$(uv run "$CFG" "$CLUSTER" zombie_cleanup.running_max_age_hours
 DRY_RUN=$(uv run "$CFG" "$CLUSTER" zombie_cleanup.dry_run "false")
 PUSHGATEWAY_URL=$(uv run "$CFG" "$CLUSTER" zombie_cleanup.pushgateway_url "http://prometheus-pushgateway.monitoring.svc.cluster.local:9091")
 
+NUMA_CORDON_DRY_RUN=$(uv run "$CFG" "$CLUSTER" zombie_cleanup.numa_cordon_dry_run "false")
+NUMA_CORDON_UNCORDON=$(uv run "$CFG" "$CLUSTER" zombie_cleanup.numa_cordon_uncordon "true")
+
 # --- Compute content-based image tag ---
 TAG=$(find "$MODULE_DIR/docker" "$MODULE_DIR/scripts/python" \
   \( -name '*.py' -o -name 'Dockerfile' -o -name 'pyproject.toml' \) \
@@ -131,6 +134,7 @@ echo "Using ${IMAGE}:${TAG}"
 # --- Apply RBAC ---
 echo "  Applying RBAC..."
 kubectl_apply_if_changed -f "$MODULE_DIR/kubernetes/rbac.yaml"
+kubectl_apply_if_changed -f "$MODULE_DIR/kubernetes/numa-cordon-rbac.yaml"
 
 # --- Apply CronJob with config substitution ---
 echo "  Applying CronJob..."
@@ -142,4 +146,11 @@ sed \
   -e "s|PUSHGATEWAY_URL_PLACEHOLDER|${PUSHGATEWAY_URL}|" \
   "$MODULE_DIR/kubernetes/cronjob.yaml" | kubectl_apply_if_changed -f -
 
+echo "  Applying numa-cordon CronJob..."
+sed \
+  -e "s|ZOMBIE_CLEANUP_IMAGE_PLACEHOLDER|${IMAGE}:${TAG}|" \
+  -e "s|NUMA_CORDON_DRY_RUN_PLACEHOLDER|${NUMA_CORDON_DRY_RUN}|" \
+  -e "s|NUMA_CORDON_UNCORDON_PLACEHOLDER|${NUMA_CORDON_UNCORDON}|" \
+  "$MODULE_DIR/kubernetes/numa-cordon-cronjob.yaml" | kubectl_apply_if_changed -f -
+
 echo "  zombie-cleanup deployed."

osdc/modules/zombie-cleanup/kubernetes/numa-cordon-cronjob.yaml

@@ -0,0 +1,66 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: numa-cordon
+  namespace: arc-runners
+  labels:
+    app.kubernetes.io/name: numa-cordon
+    app.kubernetes.io/part-of: osdc-zombie-cleanup
+spec:
+  # Run every 5 minutes to break the fragmentation livelock quickly.
+  # The zombie-cleanup CronJob runs hourly; this needs to be faster
+  # because 4 consecutive failures can happen in ~90s on a single node.
+  schedule: "*/5 * * * *"
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 0
+      activeDeadlineSeconds: 120
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: numa-cordon
+            app.kubernetes.io/part-of: osdc-zombie-cleanup
+        spec:
+          serviceAccountName: numa-cordon
+          restartPolicy: Never
+          nodeSelector:
+            role: base-infrastructure
+          tolerations:
+            - key: CriticalAddonsOnly
+              operator: Exists
+              effect: NoSchedule
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65534
+            runAsGroup: 65534
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: numa-cordon
+              image: ZOMBIE_CLEANUP_IMAGE_PLACEHOLDER
+              command: ["python", "numa_cordon.py"]
+              env:
+                - name: PYTHONDONTWRITEBYTECODE
+                  value: "1"
+                - name: TARGET_NAMESPACE
+                  value: "arc-runners"
+                - name: DRY_RUN
+                  value: "NUMA_CORDON_DRY_RUN_PLACEHOLDER"
+                - name: UNCORDON_ENABLED
+                  value: "NUMA_CORDON_UNCORDON_PLACEHOLDER"
+              resources:
+                requests:
+                  cpu: 100m
+                  memory: 128Mi
+                limits:
+                  cpu: 500m
+                  memory: 256Mi
+              securityContext:
+                readOnlyRootFilesystem: true
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop:
+                    - ALL

osdc/modules/zombie-cleanup/kubernetes/numa-cordon-rbac.yaml

@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: numa-cordon
+  namespace: arc-runners
+  labels:
+    app.kubernetes.io/name: numa-cordon
+    app.kubernetes.io/part-of: osdc-zombie-cleanup
+---
+# Namespace-scoped: list/delete Failed pods in arc-runners.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: numa-cordon
+  namespace: arc-runners
+  labels:
+    app.kubernetes.io/name: numa-cordon
+    app.kubernetes.io/part-of: osdc-zombie-cleanup
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: numa-cordon
+  namespace: arc-runners
+  labels:
+    app.kubernetes.io/name: numa-cordon
+    app.kubernetes.io/part-of: osdc-zombie-cleanup
+subjects:
+  - kind: ServiceAccount
+    name: numa-cordon
+    namespace: arc-runners
+roleRef:
+  kind: Role
+  name: numa-cordon
+  apiGroup: rbac.authorization.k8s.io
+---
+# Cluster-scoped: get/list/patch nodes for cordon/uncordon.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: numa-cordon
+  labels:
+    app.kubernetes.io/name: numa-cordon
+    app.kubernetes.io/part-of: osdc-zombie-cleanup
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: numa-cordon
+  labels:
+    app.kubernetes.io/name: numa-cordon
+    app.kubernetes.io/part-of: osdc-zombie-cleanup
+subjects:
+  - kind: ServiceAccount
+    name: numa-cordon
+    namespace: arc-runners
+roleRef:
+  kind: ClusterRole
+  name: numa-cordon
+  apiGroup: rbac.authorization.k8s.io