diff --git a/osdc/modules/numa-scheduler/deploy.sh b/osdc/modules/numa-scheduler/deploy.sh new file mode 100755 index 00000000..a5ca18e1 --- /dev/null +++ b/osdc/modules/numa-scheduler/deploy.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +set -euo pipefail +# +# Deploy NUMA-aware secondary scheduler (scheduler-plugins). +# Called by: just deploy-module numa-scheduler +# +# Args: $1=cluster-id $2=cluster-name $3=region +# +# Downloads the scheduler-plugins Helm chart from the GitHub release +# and installs it as a secondary scheduler named "numa-scheduler" with +# NodeResourceTopologyMatch enabled. Pods that set +# schedulerName: numa-scheduler get NUMA-aware placement. + +CLUSTER="$1" +export CNAME="$2" +export REGION="$3" +MODULE_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="${OSDC_ROOT:-$(cd "$MODULE_DIR/../.." && pwd)}" +UPSTREAM_ROOT="${OSDC_UPSTREAM:-$REPO_ROOT}" +# shellcheck source=/dev/null +source "$UPSTREAM_ROOT/scripts/mise-activate.sh" +# shellcheck source=/dev/null +source "$UPSTREAM_ROOT/scripts/helm-upgrade.sh" +CFG="$UPSTREAM_ROOT/scripts/cluster-config.py" + +CHART_VERSION=$(uv run "$CFG" "$CLUSTER" numa_scheduler.chart_version "0.34.7") +SCHEDULER_REPLICAS=$(uv run "$CFG" "$CLUSTER" numa_scheduler.replicas "2") + +# --- Download chart from GitHub release --- +CHART_TGZ="scheduler-plugins-${CHART_VERSION}.tgz" +CHART_DIR=$(mktemp -d) +trap 'rm -rf "$CHART_DIR"' EXIT + +CHART_URL="https://github.com/kubernetes-sigs/scheduler-plugins/releases/download/v${CHART_VERSION}/${CHART_TGZ}" +echo "Downloading scheduler-plugins chart v${CHART_VERSION}..." +curl -fsSL "$CHART_URL" -o "${CHART_DIR}/${CHART_TGZ}" + +echo "Installing numa-scheduler (scheduler-plugins v${CHART_VERSION})..." +helm_upgrade_if_changed numa-scheduler numa-scheduler \ + --create-namespace \ + --history-max 3 \ + -f "$MODULE_DIR/helm/values.yaml" \ + --set scheduler.replicaCount="${SCHEDULER_REPLICAS}" \ + --timeout 5m \ + --wait \ + "${CHART_DIR}/${CHART_TGZ}" + +echo "numa-scheduler deployed." diff --git a/osdc/modules/numa-scheduler/helm/values.yaml b/osdc/modules/numa-scheduler/helm/values.yaml new file mode 100644 index 00000000..c5fb3858 --- /dev/null +++ b/osdc/modules/numa-scheduler/helm/values.yaml @@ -0,0 +1,57 @@ +# Scheduler-plugins — NUMA-aware secondary scheduler +# Chart source: https://github.com/kubernetes-sigs/scheduler-plugins +# (vendored from manifests/install/charts/as-a-second-scheduler/) +# +# Deploys a second kube-scheduler named "numa-scheduler" with the +# NodeResourceTopologyMatch plugin enabled. Pods that set +# schedulerName: numa-scheduler will be filtered/scored by per-NUMA-zone +# resource availability (read from NodeResourceTopology CRDs published +# by the NFD topology-updater). +# +# Nothing uses this scheduler until runner definitions set +# scheduler_name: numa-scheduler — deploying it is a no-op. + +scheduler: + name: numa-scheduler + image: registry.k8s.io/scheduler-plugins/kube-scheduler:v0.34.7 + replicaCount: 2 + leaderElect: true + + # Run on base-infrastructure nodes alongside other control plane components. + nodeSelector: + role: base-infrastructure + + tolerations: + - key: CriticalAddonsOnly + operator: Equal + value: "true" + effect: NoSchedule + + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +# The controller is needed for Coscheduling/CapacityScheduling plugins. +# We only use NodeResourceTopologyMatch, so disable it. +controller: + replicaCount: 0 + +# Only enable NodeResourceTopologyMatch — we don't need Coscheduling, +# CapacityScheduling, or other plugins. +plugins: + enabled: + - NodeResourceTopologyMatch + disabled: [] + +pluginConfig: + - name: NodeResourceTopologyMatch + args: + scoringStrategy: + # Pack pods onto already-busy NUMA zones so other zones stay + # fully free for large (4-GPU) requests. Without this, small + # pods spread across sockets and fragment both. + type: MostAllocated diff --git a/osdc/modules/numa-scheduler/tests/smoke/conftest.py b/osdc/modules/numa-scheduler/tests/smoke/conftest.py new file mode 100644 index 00000000..8dfd070c --- /dev/null +++ b/osdc/modules/numa-scheduler/tests/smoke/conftest.py @@ -0,0 +1 @@ +from smoke_conftest import * # noqa: F403 diff --git a/osdc/modules/numa-scheduler/tests/smoke/test_numa_scheduler.py b/osdc/modules/numa-scheduler/tests/smoke/test_numa_scheduler.py new file mode 100644 index 00000000..c0067707 --- /dev/null +++ b/osdc/modules/numa-scheduler/tests/smoke/test_numa_scheduler.py @@ -0,0 +1,101 @@ +"""Smoke tests for the NUMA-aware secondary scheduler. + +Validates that the numa-scheduler namespace exists, the Helm release is +deployed, the scheduler Deployment is ready, and pods are running on +base-infrastructure nodes. +""" + +from __future__ import annotations + +import pytest +from helpers import ( + assert_deployment_ready, + filter_pods, + find_helm_release, +) + +pytestmark = [pytest.mark.live] + +NAMESPACE = "numa-scheduler" + + +# ============================================================================ +# Namespace +# ============================================================================ + + +class TestNUMASchedulerNamespace: + """Verify numa-scheduler namespace exists.""" + + def test_namespace_exists(self, all_namespaces: dict) -> None: + ns_names = [ns["metadata"]["name"] for ns in all_namespaces.get("items", [])] + assert NAMESPACE in ns_names, f"Namespace '{NAMESPACE}' not found" + + +# ============================================================================ +# Helm Release +# ============================================================================ + + +class TestNUMASchedulerHelm: + """Verify numa-scheduler Helm release is deployed.""" + + def test_helm_release_deployed(self, all_helm_releases: list[dict]) -> None: + release = find_helm_release(all_helm_releases, "numa-scheduler", namespace=NAMESPACE) + assert release is not None, "Helm release 'numa-scheduler' not found in 'numa-scheduler' namespace" + status = release.get("status", "") + assert status == "deployed", f"numa-scheduler Helm release status is '{status}', expected 'deployed'" + + +# ============================================================================ +# Scheduler Deployment +# ============================================================================ + + +class TestNUMASchedulerDeployment: + """Verify the numa-scheduler Deployment is ready.""" + + def test_deployment_ready(self, all_deployments: dict) -> None: + assert_deployment_ready(all_deployments, NAMESPACE, "numa-scheduler") + + +# ============================================================================ +# Scheduler Pods +# ============================================================================ + + +class TestNUMASchedulerPods: + """Verify scheduler pods are running on base-infrastructure nodes.""" + + def test_pods_running(self, all_pods: dict) -> None: + pods = filter_pods(all_pods, namespace=NAMESPACE) + running = [p for p in pods if p.get("status", {}).get("phase") == "Running"] + assert len(running) >= 1, f"Expected at least 1 Running numa-scheduler pod, found {len(running)}" + + def test_pods_on_base_infrastructure_nodes(self, all_pods: dict, all_nodes: dict) -> None: + """Verify scheduler pods run on base-infrastructure nodes. + + The numa-scheduler values.yaml sets nodeSelector: role: base-infrastructure + to colocate with other control plane components. This test catches + misconfigurations that would schedule the secondary scheduler onto + GPU worker nodes. + """ + pods = filter_pods(all_pods, namespace=NAMESPACE) + running = [p for p in pods if p.get("status", {}).get("phase") == "Running"] + if not running: + pytest.skip("No running numa-scheduler pods to verify node placement") + + node_labels: dict[str, dict] = {} + for node in all_nodes.get("items", []): + name = node.get("metadata", {}).get("name", "") + labels = node.get("metadata", {}).get("labels", {}) + node_labels[name] = labels + + for pod in running: + node_name = pod.get("spec", {}).get("nodeName", "") + labels = node_labels.get(node_name, {}) + role = labels.get("role", "") + assert role == "base-infrastructure", ( + f"Pod '{pod['metadata']['name']}' is on node '{node_name}' " + f"with role='{role}', expected 'base-infrastructure'" + )