From b4792dff9a5da3495e070c4a9867151c7c599d0f Mon Sep 17 00:00:00 2001 From: George Hong Date: Thu, 11 Jun 2026 16:35:48 -0700 Subject: [PATCH] Update [ghstack-poisoned] --- osdc/clusters.yaml | 6 ++---- .../arc-runners/defs/l-bx86iavx512-88-1000-a100-8.yaml | 1 + .../arc-runners/defs/l-x86iavx512-11-125-a100.yaml | 1 + .../arc-runners/defs/l-x86iavx512-22-250-a100-2.yaml | 1 + .../arc-runners/defs/l-x86iavx512-44-500-a100-4.yaml | 2 ++ osdc/modules/nfd/helm/values.yaml | 8 +++----- osdc/modules/nfd/kubernetes/nfd-taint-remover.yaml | 5 +++-- osdc/modules/nodepools/defs/p4d.yaml | 4 ++-- .../nodepools/scripts/python/generate_nodepools.py | 7 +++---- 9 files changed, 18 insertions(+), 17 deletions(-) diff --git a/osdc/clusters.yaml b/osdc/clusters.yaml index 7b19983b..9b2cdb15 100644 --- a/osdc/clusters.yaml +++ b/osdc/clusters.yaml @@ -162,6 +162,8 @@ clusters: - karpenter - arc - nodepools + - nfd # TESTING: NUMA topology data for p4d nodes + - numa-scheduler # TESTING: NUMA-aware secondary scheduler - arc-runners - buildkit - pypi-cache @@ -205,8 +207,6 @@ clusters: - arc - nodepools - nodepools-h100 # H100 only — B200 has no capacity reservation in us-west-1 - - nfd # NUMA topology data for p5 nodes - - numa-scheduler # NUMA-aware secondary scheduler - arc-runners-h100 - pypi-cache - cache-enforcer @@ -287,8 +287,6 @@ clusters: - nodepools - nodepools-b200 - nodepools-h100 - - nfd # NUMA topology data for p5 nodes - - numa-scheduler # NUMA-aware secondary scheduler - arc-runners - arc-runners-b200 - arc-runners-h100 diff --git a/osdc/modules/arc-runners/defs/l-bx86iavx512-88-1000-a100-8.yaml b/osdc/modules/arc-runners/defs/l-bx86iavx512-88-1000-a100-8.yaml index c1dc6cd9..77a64721 100644 --- a/osdc/modules/arc-runners/defs/l-bx86iavx512-88-1000-a100-8.yaml +++ b/osdc/modules/arc-runners/defs/l-bx86iavx512-88-1000-a100-8.yaml @@ -8,3 +8,4 @@ runner: vcpu: 88 memory: 1000Gi gpu: 8 + max_runners: 0 # TESTING: disabled for arc-staging NUMA test diff --git a/osdc/modules/arc-runners/defs/l-x86iavx512-11-125-a100.yaml b/osdc/modules/arc-runners/defs/l-x86iavx512-11-125-a100.yaml index a19f2da1..61b3db6f 100644 --- a/osdc/modules/arc-runners/defs/l-x86iavx512-11-125-a100.yaml +++ b/osdc/modules/arc-runners/defs/l-x86iavx512-11-125-a100.yaml @@ -8,3 +8,4 @@ runner: vcpu: 11 memory: 125Gi gpu: 1 + max_runners: 4 # TESTING: limited for arc-staging NUMA test diff --git a/osdc/modules/arc-runners/defs/l-x86iavx512-22-250-a100-2.yaml b/osdc/modules/arc-runners/defs/l-x86iavx512-22-250-a100-2.yaml index a416d880..d49f3bf6 100644 --- a/osdc/modules/arc-runners/defs/l-x86iavx512-22-250-a100-2.yaml +++ b/osdc/modules/arc-runners/defs/l-x86iavx512-22-250-a100-2.yaml @@ -7,3 +7,4 @@ runner: vcpu: 22 memory: 250Gi gpu: 2 + max_runners: 2 # TESTING: limited for arc-staging NUMA test diff --git a/osdc/modules/arc-runners/defs/l-x86iavx512-44-500-a100-4.yaml b/osdc/modules/arc-runners/defs/l-x86iavx512-44-500-a100-4.yaml index e86bb5f5..b59b87eb 100644 --- a/osdc/modules/arc-runners/defs/l-x86iavx512-44-500-a100-4.yaml +++ b/osdc/modules/arc-runners/defs/l-x86iavx512-44-500-a100-4.yaml @@ -7,3 +7,5 @@ runner: vcpu: 44 memory: 500Gi gpu: 4 + max_runners: 2 # TESTING: cap to 1 p4d node (2 NUMA zones × 4 GPUs) + scheduler_name: numa-scheduler # TESTING: NUMA-aware scheduling diff --git a/osdc/modules/nfd/helm/values.yaml b/osdc/modules/nfd/helm/values.yaml index 6d221f72..0e64e7cc 100644 --- a/osdc/modules/nfd/helm/values.yaml +++ b/osdc/modules/nfd/helm/values.yaml @@ -23,12 +23,10 @@ topologyUpdater: # evaluated by the NUMA-aware scheduler. updateInterval: 15s - # Scoped to H100 nodes (p5 fleet) where the packed pool uses - # single-numa-node topology policy. Only these multi-NUMA nodes - # need NRT data for the numa-scheduler to prevent - # TopologyAffinityError on 4-GPU jobs. + # TESTING: p4d for arc-staging validation. + # Production: change to node-fleet: p5. nodeSelector: - node-fleet: p5 + node-fleet: p4d # Tolerate every taint so the topology-updater schedules on p5 # nodes regardless of their taint set (node-fleet, instance-type, diff --git a/osdc/modules/nfd/kubernetes/nfd-taint-remover.yaml b/osdc/modules/nfd/kubernetes/nfd-taint-remover.yaml index 371e89e0..bc315125 100644 --- a/osdc/modules/nfd/kubernetes/nfd-taint-remover.yaml +++ b/osdc/modules/nfd/kubernetes/nfd-taint-remover.yaml @@ -67,9 +67,10 @@ spec: spec: priorityClassName: system-node-critical - # Match the same nodes as the NFD topology-updater. + # TESTING: p4d for arc-staging validation. + # Production: change to node-fleet: p5. nodeSelector: - node-fleet: p5 + node-fleet: p4d # Tolerate every taint — same rationale as node-performance-tuning: # must coexist with all node-init.osdc.io/* startup taints without diff --git a/osdc/modules/nodepools/defs/p4d.yaml b/osdc/modules/nodepools/defs/p4d.yaml index a2d91bda..47f3fc18 100644 --- a/osdc/modules/nodepools/defs/p4d.yaml +++ b/osdc/modules/nodepools/defs/p4d.yaml @@ -11,8 +11,8 @@ fleet: name: p4d arch: amd64 gpu: true - exclude_regions: - - us-west-1 + # exclude_regions: # TESTING: temporarily allowing us-west-1 for arc-staging + # - us-west-1 instances: - type: p4d.24xlarge weight: 100 diff --git a/osdc/modules/nodepools/scripts/python/generate_nodepools.py b/osdc/modules/nodepools/scripts/python/generate_nodepools.py index 2e9cf0c1..f21c64f2 100755 --- a/osdc/modules/nodepools/scripts/python/generate_nodepools.py +++ b/osdc/modules/nodepools/scripts/python/generate_nodepools.py @@ -98,10 +98,9 @@ "key": "node-init.osdc.io/nfd-topology", "value": "true", "effect": "NoSchedule", - # NFD topology-updater only targets p5 nodes (nodeSelector: node-fleet: p5). - # Only emit the taint on nodepools where NFD actually runs — otherwise the - # node would be tainted with nothing to remove it. - "applies_when": lambda d: d.get("fleet_name") == "p5", + # TESTING: broadened to include p4d for arc-staging validation. + # Production: restrict to fleet_name == "p5" only. + "applies_when": lambda d: d.get("fleet_name") in ("p5", "p4d"), }, ]