Add numa-scheduler module (#696)

osdc/modules/numa-scheduler/deploy.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -euo pipefail
+#
+# Deploy NUMA-aware secondary scheduler (scheduler-plugins).
+# Called by: just deploy-module <cluster> numa-scheduler
+#
+# Args: $1=cluster-id  $2=cluster-name  $3=region
+#
+# Downloads the scheduler-plugins Helm chart from the GitHub release
+# and installs it as a secondary scheduler named "numa-scheduler" with
+# NodeResourceTopologyMatch enabled. Pods that set
+# schedulerName: numa-scheduler get NUMA-aware placement.
+
+CLUSTER="$1"
+export CNAME="$2"
+export REGION="$3"
+MODULE_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="${OSDC_ROOT:-$(cd "$MODULE_DIR/../.." && pwd)}"
+UPSTREAM_ROOT="${OSDC_UPSTREAM:-$REPO_ROOT}"
+# shellcheck source=/dev/null
+source "$UPSTREAM_ROOT/scripts/mise-activate.sh"
+# shellcheck source=/dev/null
+source "$UPSTREAM_ROOT/scripts/helm-upgrade.sh"
+CFG="$UPSTREAM_ROOT/scripts/cluster-config.py"
+
+CHART_VERSION=$(uv run "$CFG" "$CLUSTER" numa_scheduler.chart_version "0.34.7")
+SCHEDULER_REPLICAS=$(uv run "$CFG" "$CLUSTER" numa_scheduler.replicas "2")
+
+# --- Download chart from GitHub release ---
+CHART_TGZ="scheduler-plugins-${CHART_VERSION}.tgz"
+CHART_DIR=$(mktemp -d)
+trap 'rm -rf "$CHART_DIR"' EXIT
+
+CHART_URL="https://github.com/kubernetes-sigs/scheduler-plugins/releases/download/v${CHART_VERSION}/${CHART_TGZ}"
+echo "Downloading scheduler-plugins chart v${CHART_VERSION}..."
+curl -fsSL "$CHART_URL" -o "${CHART_DIR}/${CHART_TGZ}"
+
+echo "Installing numa-scheduler (scheduler-plugins v${CHART_VERSION})..."
+helm_upgrade_if_changed numa-scheduler numa-scheduler \
+  --create-namespace \
+  --history-max 3 \
+  -f "$MODULE_DIR/helm/values.yaml" \
+  --set scheduler.replicaCount="${SCHEDULER_REPLICAS}" \
+  --timeout 5m \
+  --wait \
+  "${CHART_DIR}/${CHART_TGZ}"
+
+echo "numa-scheduler deployed."

osdc/modules/numa-scheduler/helm/values.yaml

@@ -0,0 +1,57 @@
+# Scheduler-plugins — NUMA-aware secondary scheduler
+# Chart source: https://github.com/kubernetes-sigs/scheduler-plugins
+#   (vendored from manifests/install/charts/as-a-second-scheduler/)
+#
+# Deploys a second kube-scheduler named "numa-scheduler" with the
+# NodeResourceTopologyMatch plugin enabled. Pods that set
+# schedulerName: numa-scheduler will be filtered/scored by per-NUMA-zone
+# resource availability (read from NodeResourceTopology CRDs published
+# by the NFD topology-updater).
+#
+# Nothing uses this scheduler until runner definitions set
+# scheduler_name: numa-scheduler — deploying it is a no-op.
+
+scheduler:
+  name: numa-scheduler
+  image: registry.k8s.io/scheduler-plugins/kube-scheduler:v0.34.7
+  replicaCount: 2
+  leaderElect: true
+
+  # Run on base-infrastructure nodes alongside other control plane components.
+  nodeSelector:
+    role: base-infrastructure
+
+  tolerations:
+    - key: CriticalAddonsOnly
+      operator: Equal
+      value: "true"
+      effect: NoSchedule
+
+  resources:
+    requests:
+      cpu: 200m
+      memory: 256Mi
+    limits:
+      cpu: 500m
+      memory: 512Mi
+
+# The controller is needed for Coscheduling/CapacityScheduling plugins.
+# We only use NodeResourceTopologyMatch, so disable it.
+controller:
+  replicaCount: 0
+
+# Only enable NodeResourceTopologyMatch — we don't need Coscheduling,
+# CapacityScheduling, or other plugins.
+plugins:
+  enabled:
+    - NodeResourceTopologyMatch
+  disabled: []
+
+pluginConfig:
+  - name: NodeResourceTopologyMatch
+    args:
+      scoringStrategy:
+        # Pack pods onto already-busy NUMA zones so other zones stay
+        # fully free for large (4-GPU) requests. Without this, small
+        # pods spread across sockets and fragment both.
+        type: MostAllocated

osdc/modules/numa-scheduler/tests/smoke/conftest.py

@@ -0,0 +1 @@
+from smoke_conftest import *  # noqa: F403

osdc/modules/numa-scheduler/tests/smoke/test_numa_scheduler.py

@@ -0,0 +1,101 @@
+"""Smoke tests for the NUMA-aware secondary scheduler.
+
+Validates that the numa-scheduler namespace exists, the Helm release is
+deployed, the scheduler Deployment is ready, and pods are running on
+base-infrastructure nodes.
+"""
+
+from __future__ import annotations
+
+import pytest
+from helpers import (
+    assert_deployment_ready,
+    filter_pods,
+    find_helm_release,
+)
+
+pytestmark = [pytest.mark.live]
+
+NAMESPACE = "numa-scheduler"
+
+
+# ============================================================================
+# Namespace
+# ============================================================================
+
+
+class TestNUMASchedulerNamespace:
+    """Verify numa-scheduler namespace exists."""
+
+    def test_namespace_exists(self, all_namespaces: dict) -> None:
+        ns_names = [ns["metadata"]["name"] for ns in all_namespaces.get("items", [])]
+        assert NAMESPACE in ns_names, f"Namespace '{NAMESPACE}' not found"
+
+
+# ============================================================================
+# Helm Release
+# ============================================================================
+
+
+class TestNUMASchedulerHelm:
+    """Verify numa-scheduler Helm release is deployed."""
+
+    def test_helm_release_deployed(self, all_helm_releases: list[dict]) -> None:
+        release = find_helm_release(all_helm_releases, "numa-scheduler", namespace=NAMESPACE)
+        assert release is not None, "Helm release 'numa-scheduler' not found in 'numa-scheduler' namespace"
+        status = release.get("status", "")
+        assert status == "deployed", f"numa-scheduler Helm release status is '{status}', expected 'deployed'"
+
+
+# ============================================================================
+# Scheduler Deployment
+# ============================================================================
+
+
+class TestNUMASchedulerDeployment:
+    """Verify the numa-scheduler Deployment is ready."""
+
+    def test_deployment_ready(self, all_deployments: dict) -> None:
+        assert_deployment_ready(all_deployments, NAMESPACE, "numa-scheduler")
+
+
+# ============================================================================
+# Scheduler Pods
+# ============================================================================
+
+
+class TestNUMASchedulerPods:
+    """Verify scheduler pods are running on base-infrastructure nodes."""
+
+    def test_pods_running(self, all_pods: dict) -> None:
+        pods = filter_pods(all_pods, namespace=NAMESPACE)
+        running = [p for p in pods if p.get("status", {}).get("phase") == "Running"]
+        assert len(running) >= 1, f"Expected at least 1 Running numa-scheduler pod, found {len(running)}"
+
+    def test_pods_on_base_infrastructure_nodes(self, all_pods: dict, all_nodes: dict) -> None:
+        """Verify scheduler pods run on base-infrastructure nodes.
+
+        The numa-scheduler values.yaml sets nodeSelector: role: base-infrastructure
+        to colocate with other control plane components. This test catches
+        misconfigurations that would schedule the secondary scheduler onto
+        GPU worker nodes.
+        """
+        pods = filter_pods(all_pods, namespace=NAMESPACE)
+        running = [p for p in pods if p.get("status", {}).get("phase") == "Running"]
+        if not running:
+            pytest.skip("No running numa-scheduler pods to verify node placement")
+
+        node_labels: dict[str, dict] = {}
+        for node in all_nodes.get("items", []):
+            name = node.get("metadata", {}).get("name", "")
+            labels = node.get("metadata", {}).get("labels", {})
+            node_labels[name] = labels
+
+        for pod in running:
+            node_name = pod.get("spec", {}).get("nodeName", "")
+            labels = node_labels.get(node_name, {})
+            role = labels.get("role", "")
+            assert role == "base-infrastructure", (
+                f"Pod '{pod['metadata']['name']}' is on node '{node_name}' "
+                f"with role='{role}', expected 'base-infrastructure'"
+            )