From 556703e56ca23d8e8c847e01a917f78147b4014e Mon Sep 17 00:00:00 2001 From: james Date: Wed, 10 Jun 2026 17:38:49 +0800 Subject: [PATCH 1/2] feat: support hami-vnpu-core Signed-off-by: james --- cmd/webhook-manager/app/server.go | 2 +- .../admission/pods/mutate/device_mutator.go | 116 ++++++++++++++++++ .../admission/pods/mutate/mutate_pod.go | 17 +++ pkg/webhooks/config/config.go | 10 +- 4 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 pkg/webhooks/admission/pods/mutate/device_mutator.go diff --git a/cmd/webhook-manager/app/server.go b/cmd/webhook-manager/app/server.go index 9f2334166fe..b15490b951e 100644 --- a/cmd/webhook-manager/app/server.go +++ b/cmd/webhook-manager/app/server.go @@ -67,7 +67,7 @@ func Run(config *options.Config) error { if admissionConf == nil { klog.Errorf("loadAdmissionConf failed.") } else { - klog.V(2).Infof("loadAdmissionConf:%v", admissionConf.ResGroupsConfig) + klog.V(2).Infof("loadAdmissionConf:%v", admissionConf) } vClient := getVolcanoClient(restConfig) diff --git a/pkg/webhooks/admission/pods/mutate/device_mutator.go b/pkg/webhooks/admission/pods/mutate/device_mutator.go new file mode 100644 index 00000000000..4a3faa80957 --- /dev/null +++ b/pkg/webhooks/admission/pods/mutate/device_mutator.go @@ -0,0 +1,116 @@ +/* +Copyright 2026 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutate + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + devconfig "volcano.sh/volcano/pkg/scheduler/api/devices/config" + wkconfig "volcano.sh/volcano/pkg/webhooks/config" +) + +const ( + ascendHAMiMutatorName = "ascend" + jsonPatchOpAdd = "add" + VNPUModeAnnotation = "huawei.com/vnpu-mode" + VNPUModeHamiCore = "hami-core" +) + +type DeviceMutator interface { + Name() string + MutateAdmission(pod *v1.Pod, deviceMutatorConfig wkconfig.DeviceMutatorConfig) []patchOperation +} + +var registeredDeviceMutators = []DeviceMutator{ + newAscendMutator(), +} + +type AscendMutator struct{} + +func newAscendMutator() DeviceMutator { + return &AscendMutator{} +} + +func (m *AscendMutator) Name() string { + return ascendHAMiMutatorName +} + +func (m *AscendMutator) MutateAdmission(pod *v1.Pod, deviceMutatorConfig wkconfig.DeviceMutatorConfig) []patchOperation { + if pod == nil || pod.Annotations == nil { + return nil + } + if vnpuMode, ok := pod.Annotations[VNPUModeAnnotation]; !ok || vnpuMode != VNPUModeHamiCore { + return nil + } + klog.V(5).Infof("the hami-core annotation exists in pod %s/%s", pod.Namespace, pod.Name) + devconfig.InitDevicesConfig(deviceMutatorConfig.KnownGeometriesCMName, deviceMutatorConfig.KnownGeometriesCMNamespace) + config := devconfig.GetConfig() + if config == nil { + klog.V(5).Infof("device configuration is empty.") + return nil + } + var patch []patchOperation + for idx, container := range pod.Spec.Containers { + if container.Lifecycle != nil && container.Lifecycle.PostStart != nil { + continue + } + hasAscendResource := false + for _, vnpuConfig := range config.VNPUs { + if val, ok := container.Resources.Limits[v1.ResourceName(vnpuConfig.ResourceName)]; ok && !val.IsZero() { + hasAscendResource = true + break + } + if val, ok := container.Resources.Requests[v1.ResourceName(vnpuConfig.ResourceName)]; ok && !val.IsZero() { + hasAscendResource = true + break + } + } + if !hasAscendResource { + continue + } + klog.V(3).Infof("Ascend core resource detected, injecting postStart lifecycle for container %s", container.Name) + handler := v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: []string{ + "bash", + "-c", + "export RUST_LOG=info\n/hami-vnpu-core/limiter > /tmp/limiter_manager.log 2>&1 &", + }, + }, + } + if container.Lifecycle == nil { + patch = append(patch, patchOperation{ + Op: jsonPatchOpAdd, + Path: fmt.Sprintf("/spec/containers/%d/lifecycle", idx), + Value: v1.Lifecycle{ + PostStart: &handler, + }, + }) + continue + } + + patch = append(patch, patchOperation{ + Op: jsonPatchOpAdd, + Path: fmt.Sprintf("/spec/containers/%d/lifecycle/postStart", idx), + Value: handler, + }) + } + + return patch +} diff --git a/pkg/webhooks/admission/pods/mutate/mutate_pod.go b/pkg/webhooks/admission/pods/mutate/mutate_pod.go index c4d94667b9c..5bcc925f580 100644 --- a/pkg/webhooks/admission/pods/mutate/mutate_pod.go +++ b/pkg/webhooks/admission/pods/mutate/mutate_pod.go @@ -120,6 +120,10 @@ func createPatch(pod *v1.Pod) ([]byte, error) { patch = append(patch, *patchGates) } + if config.ConfigData.DeviceMutatorConf.Enable { + patch = append(patch, patchDeviceMutation(pod, config.ConfigData.DeviceMutatorConf)...) + } + for _, resourceGroup := range config.ConfigData.ResGroupsConfig { klog.V(3).Infof("resourceGroup %s", resourceGroup.ResourceGroup) group := GetResGroup(resourceGroup) @@ -260,3 +264,16 @@ func patchSchedulerName(resGroupConfig wkconfig.ResGroupConfig) *patchOperation return &patchOperation{Op: "add", Path: "/spec/schedulerName", Value: resGroupConfig.SchedulerName} } + +func patchDeviceMutation(pod *v1.Pod, deviceMutatorConf wkconfig.DeviceMutatorConfig) []patchOperation { + var patch []patchOperation + for _, mutator := range registeredDeviceMutators { + patches := mutator.MutateAdmission(pod, deviceMutatorConf) + if len(patches) == 0 { + continue + } + patch = append(patch, patches...) + + } + return patch +} diff --git a/pkg/webhooks/config/config.go b/pkg/webhooks/config/config.go index fe57212792f..2eba5445767 100644 --- a/pkg/webhooks/config/config.go +++ b/pkg/webhooks/config/config.go @@ -45,10 +45,17 @@ type ResGroupConfig struct { Affinity string `yaml:"affinity"` } +type DeviceMutatorConfig struct { + Enable bool `yaml:"enable"` + KnownGeometriesCMName string `yaml:"knownGeometriesCMName"` + KnownGeometriesCMNamespace string `yaml:"knownGeometriesCMNamespace"` +} + // AdmissionConfiguration defines the configuration of admission. type AdmissionConfiguration struct { sync.Mutex - ResGroupsConfig []ResGroupConfig `yaml:"resourceGroups"` + DeviceMutatorConf DeviceMutatorConfig `yaml:"deviceMutator"` + ResGroupsConfig []ResGroupConfig `yaml:"resourceGroups"` } var admissionConf AdmissionConfiguration @@ -73,6 +80,7 @@ func LoadAdmissionConf(confPath string) *AdmissionConfiguration { admissionConf.Lock() admissionConf.ResGroupsConfig = data.ResGroupsConfig + admissionConf.DeviceMutatorConf = data.DeviceMutatorConf admissionConf.Unlock() return &admissionConf } From 9e59a50971eedf8a227bcfb95872744eb60e17ad Mon Sep 17 00:00:00 2001 From: james Date: Mon, 15 Jun 2026 17:20:18 +0800 Subject: [PATCH 2/2] support hami-core in ascend device info Signed-off-by: james --- cmd/webhook-manager/app/server.go | 4 + docs/user-guide/how_to_use_vnpu.md | 122 ++++- .../chart/volcano/templates/admission.yaml | 10 + installer/helm/chart/volcano/values.yaml | 4 + .../api/devices/ascend/hami/constants.go | 23 + .../api/devices/ascend/hami/device_info.go | 93 ++-- .../devices/ascend/hami/device_info_test.go | 176 +++++-- pkg/scheduler/api/devices/config/config.go | 4 +- pkg/scheduler/api/devices/util.go | 11 +- .../admission/pods/mutate/device_mutator.go | 38 +- .../pods/mutate/device_mutator_test.go | 444 ++++++++++++++++++ .../admission/pods/mutate/mutate_pod.go | 7 +- 12 files changed, 847 insertions(+), 89 deletions(-) create mode 100644 pkg/scheduler/api/devices/ascend/hami/constants.go create mode 100644 pkg/webhooks/admission/pods/mutate/device_mutator_test.go diff --git a/cmd/webhook-manager/app/server.go b/cmd/webhook-manager/app/server.go index b15490b951e..e368d487fbd 100644 --- a/cmd/webhook-manager/app/server.go +++ b/cmd/webhook-manager/app/server.go @@ -34,6 +34,7 @@ import ( informers "volcano.sh/apis/pkg/client/informers/externalversions" "volcano.sh/volcano/cmd/webhook-manager/app/options" "volcano.sh/volcano/pkg/kube" + devconfig "volcano.sh/volcano/pkg/scheduler/api/devices/config" "volcano.sh/volcano/pkg/signals" commonutil "volcano.sh/volcano/pkg/util" wkconfig "volcano.sh/volcano/pkg/webhooks/config" @@ -68,6 +69,9 @@ func Run(config *options.Config) error { klog.Errorf("loadAdmissionConf failed.") } else { klog.V(2).Infof("loadAdmissionConf:%v", admissionConf) + if admissionConf.DeviceMutatorConf.Enable { + devconfig.InitDevicesConfig(admissionConf.DeviceMutatorConf.KnownGeometriesCMName, admissionConf.DeviceMutatorConf.KnownGeometriesCMNamespace) + } } vClient := getVolcanoClient(restConfig) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index fd6dabbb91c..b39b5c84090 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -25,7 +25,7 @@ with support for more chip types to come **Description**: -This mode is developed by a third-party community 'HAMi', which is the developer of [volcano-vgpu](./how_to_use_volcano_vgpu.md) feature. It supports vNPU feature for both Ascend 310 and Ascend 910. It also supports managing heterogeneous Ascend cluster(Cluster with multiple Ascend types, i.e. 910A,910B2,910B3,310P) +This mode is developed by a third-party community 'HAMi', which is the developer of [volcano-vgpu](./how_to_use_volcano_vgpu.md) feature. It supports vNPU feature for both Ascend 310 and Ascend 910. It also supports managing heterogeneous Ascend cluster(Cluster with multiple Ascend types, i.e. 910A,910B2,910B3,310P). Starting with version 1.16, we support soft device partitioning through [hami-vnpu-core](https://github.com/Project-HAMi/hami-vnpu-core). This feature is known as the `hami-core` mode. **Use case**: @@ -43,7 +43,7 @@ To enable vNPU scheduling, the following components must be set up based on the **Prerequisites**: Kubernetes >= 1.16 -Volcano >= 1.14 +Volcano >= 1.14 (1.16 for `hami-core` mode) [ascend-docker-runtime](https://gitcode.com/Ascend/mind-cluster/tree/master/component/ascend-docker-runtime) (for HAMi Mode) ### Install Volcano: @@ -115,19 +115,21 @@ data: #### HAMi mode -##### Label the Node with `ascend=on` +##### Template vNPU mode + +###### Label the Node with `ascend=on` ``` kubectl label node {ascend-node} ascend=on ``` -##### Deploy `hami-scheduler-device` ConfigMap +###### Deploy `hami-scheduler-device` ConfigMap ``` kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-configmap.yaml ``` -##### Deploy ascend-device-plugin +###### Deploy ascend-device-plugin ``` kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-plugin.yaml @@ -135,7 +137,76 @@ kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-pl For more information, refer to the [ascend-device-plugin documentation](https://github.com/Project-HAMi/ascend-device-plugin). -##### Scheduler Config Update +###### Update Scheduler Config +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: volcano-scheduler-configmap + namespace: volcano-system +data: + volcano-scheduler.conf: | + actions: "enqueue, allocate, backfill" + tiers: + - plugins: + - name: predicates + - name: deviceshare + arguments: + deviceshare.AscendHAMiVNPUEnable: true # enable ascend vnpu + deviceshare.SchedulePolicy: binpack # scheduling policy. binpack / spread + deviceshare.KnownGeometriesCMNamespace: kube-system + deviceshare.KnownGeometriesCMName: hami-scheduler-device +``` + + **Note:** You may notice that, 'volcano-vgpu' has its own GeometriesCMName and GeometriesCMNamespace, which means if you want to use both vNPU and vGPU in a same volcano cluster, you need to merge the configMap from both sides and set it here. + +##### `hami-core` mode + +###### Label the Node with `ascend=on` + +``` +kubectl label node {ascend-node} ascend=on +``` + +###### Deploy `hami-scheduler-device` ConfigMap + +1. Download file +``` +curl -O https://github.com/Project-HAMi/ascend-device-plugin/blob/main/ascend-device-configmap.yaml +``` +2. Set `hamiVnpuCore` to `true` +3. Deploy the yaml + +``` +kubectl apply -f ascend-device-configmap.yaml +``` + +###### Deploy `hami-device-node-config` ConfigMap +1. Download file +``` +curl -O https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/main/ascend-device-node-configmap.yaml +``` +2. Set the `name` field under the `nodes` block to the actual node name. +3. Deploy the yaml +``` +kubectl apply -f ascend-device-node-configmap.yaml +``` + +###### Deploy ascend-device-plugin + +``` +kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-plugin.yaml +``` + +###### Upgrade volcano-adminssion +``` +helm upgrade [RELEASE_NAME] [CHART_PATH] --set custom.device_mutator_enable=true \ + --set custom.enabled_admissions='/jobs/mutate\,/jobs/validate\,/podgroups/validate\,/queues/mutate\,/queues/validate\,/hypernodes/validate\,/cronjobs/validate\,/pods/mutate' +kubectl rollout restart deployment -n [VOLCANO_NAMESPACE] volcano-admission +``` +**Note**: If you deployed Volcano using a Deployment YAML, you need to either generate a new YAML with Helm using the parameters above and deploy it, or uninstall the existing Volcano and redeploy it with Helm. + +###### Update Scheduler Config ```yaml kind: ConfigMap apiVersion: v1 @@ -262,6 +333,8 @@ For detailed information, please consult the official [Ascend MindCluster docume ### HAMi mode +#### Template vNPU mode + ```yaml apiVersion: v1 kind: Pod @@ -289,6 +362,7 @@ The supported Ascend chips and their `ResourceNames` are shown in the following | 910B3 | huawei.com/Ascend910B3 | huawei.com/Ascend910B3-memory | | 910B4 | huawei.com/Ascend910B4 | huawei.com/Ascend910B4-memory | | 910B4-1 | huawei.com/Ascend910B4-1 | huawei.com/Ascend910B4-1-memory | +| 910C | huawei.com/Ascend910C | huawei.com/Ascend910C-memory | | 310P3 | huawei.com/Ascend310P | huawei.com/Ascend310P-memory | #### Hami vNPU scene memory allocation restrictions @@ -299,3 +373,39 @@ The supported Ascend chips and their `ResourceNames` are shown in the following - Pod requests 1 vNPU device with a requested memory of 20480, resulting in an actual allocated memory of 32768 - When a Pod requests multiple vNPU devices, the memory resource request can be left unspecified or filled with the maximum value, The memory allocated to Pod is the actual memory of the entire card + +#### `hami-core` mode + +``` +apiVersion: v1 +kind: Pod +metadata: + name: ascend-pod + annotations: + huawei.com/vnpu-mode: hami-core +spec: + schedulerName: volcano + containers: + - name: ubuntu-container + image: quay.io/ascend/vllm-ascend:v0.18.0-310p + command: ["sleep"] + args: ["100000"] + resources: + limits: + huawei.com/Ascend310P: "1" + huawei.com/Ascend310P-memory: "4096" + huawei.com/Ascend310P-core: "90" +``` +The supported Ascend chips and their `ResourceNames` are shown in the following table: + +| ChipName | ResourceName | ResourceMemoryName | ResourceCoreName +|-------|-------|-------| +| 910A | huawei.com/Ascend910A | huawei.com/Ascend910A-memory | huawei.com/Ascend910A-core | +| 910B2 | huawei.com/Ascend910B2 | huawei.com/Ascend910B2-memory | huawei.com/Ascend910B2-core | +| 910B3 | huawei.com/Ascend910B3 | huawei.com/Ascend910B3-memory | huawei.com/Ascend910B3-core | +| 910B4 | huawei.com/Ascend910B4 | huawei.com/Ascend910B4-memory | huawei.com/Ascend910B4-core | +| 910B4-1 | huawei.com/Ascend910B4-1 | huawei.com/Ascend910B4-1-memory | huawei.com/Ascend910B4-1-core | +| 910C | huawei.com/Ascend910C | huawei.com/Ascend910C-memory | huawei.com/Ascend910C-core | +| 310P3 | huawei.com/Ascend310P | huawei.com/Ascend310P-memory | huawei.com/Ascend310P-core | + +**Note**: If the pod's annotations do not specify `hami-core`, the device will be allocated in the template vNPU mode even if the `hami-core` feature is enabled in the configuration file. \ No newline at end of file diff --git a/installer/helm/chart/volcano/templates/admission.yaml b/installer/helm/chart/volcano/templates/admission.yaml index 72e72cc5d2a..927e253b324 100644 --- a/installer/helm/chart/volcano/templates/admission.yaml +++ b/installer/helm/chart/volcano/templates/admission.yaml @@ -20,6 +20,16 @@ data: {{- .Values.custom.admission_config_override | nindent 4 }} {{- else }} {{- (.Files.Glob .Values.basic.admission_config_file).AsConfig | nindent 2}} + {{- if .Values.custom.device_mutator_enable }} + deviceMutator: + enable: true + {{- if .Values.custom.known_geometries_cm_name }} + knownGeometriesCMName: {{ .Values.custom.known_geometries_cm_name }} + {{- end }} + {{- if .Values.custom.known_geometries_cm_namespace }} + knownGeometriesCMNamespace: {{ .Values.custom.known_geometries_cm_namespace}} + {{- end }} + {{- end }} {{- end }} --- apiVersion: v1 diff --git a/installer/helm/chart/volcano/values.yaml b/installer/helm/chart/volcano/values.yaml index 395548c29d5..b7a2343e1aa 100644 --- a/installer/helm/chart/volcano/values.yaml +++ b/installer/helm/chart/volcano/values.yaml @@ -69,6 +69,10 @@ custom: agent_extend_resource_memory_name: ~ agent_kube_cgroup_root: ~ + device_mutator_enable: false + known_geometries_cm_namespace: "kube-system" + known_geometries_cm_name: "hami-scheduler-device" + # Override the configuration for admission, controller or scheduler. # For example: # diff --git a/pkg/scheduler/api/devices/ascend/hami/constants.go b/pkg/scheduler/api/devices/ascend/hami/constants.go new file mode 100644 index 00000000000..ef510d07e62 --- /dev/null +++ b/pkg/scheduler/api/devices/ascend/hami/constants.go @@ -0,0 +1,23 @@ +/* +Copyright 2026 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hami + +const ( + VNPUModeAnnotation = "huawei.com/vnpu-mode" + VNPUModeHamiCore = "hami-core" + VNPUNodeSelectorAnnotation = "hami-vnpu-core" +) diff --git a/pkg/scheduler/api/devices/ascend/hami/device_info.go b/pkg/scheduler/api/devices/ascend/hami/device_info.go index 1e791e8bfd1..16a1b58a67b 100644 --- a/pkg/scheduler/api/devices/ascend/hami/device_info.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info.go @@ -60,6 +60,7 @@ type AscendDevice struct { DeviceUsage *devices.DeviceUsage Score float64 PodMap map[string]*devices.DeviceUsage + hamiVnpuCore bool } type AscendDevices struct { @@ -70,8 +71,10 @@ type AscendDevices struct { } type RuntimeInfo struct { - UUID string `json:"UUID,omitempty"` - Temp string `json:"temp,omitempty"` + UUID string `json:"UUID,omitempty"` + Temp string `json:"temp,omitempty"` + Memory int64 `json:"memory,omitempty"` + Core int32 `json:"core,omitempty"` } var ( @@ -102,7 +105,7 @@ func NewAscendDevices(name string, node *v1.Node) map[string]*AscendDevices { klog.V(3).Infof("Node %s does not have allocatable %s resource or value is 0", node.Name, resourceName) continue } - nodeDevices, err := dev.GetNodeDevices(*node) + nodeDevices, nodeSupportHamiCore, err := dev.getNodeDevices(node, curConfig.VNPUs) if err != nil { klog.Warningf("Failed to get node devices. nodeName %s, deviceType %s, error %v", node.Name, dev.CommonWord(), err) continue @@ -124,7 +127,8 @@ func NewAscendDevices(name string, node *v1.Node) map[string]*AscendDevices { Usedmem: 0, Usedcores: 0, }, - PodMap: make(map[string]*devices.DeviceUsage), + PodMap: make(map[string]*devices.DeviceUsage), + hamiVnpuCore: nodeSupportHamiCore, } asDevices.Devices[nd.ID] = cur_dev klog.V(5).Infof("add device. ID %s dev_info %+v", cur_dev.DeviceInfo.ID, cur_dev.DeviceInfo) @@ -191,7 +195,7 @@ func (ads *AscendDevices) SubResource(pod *v1.Pod) { if _, ok := dev.PodMap[string(pod.UID)]; ok { delete(dev.PodMap, string(pod.UID)) ads.SubResourceUsage(dev, cono_dev.Usedcores, cono_dev.Usedmem) - klog.V(5).Infof("sub resource usage for pod %s. device %s usedmem %d", pod.Name, dev.DeviceInfo.ID, cono_dev.Usedmem) + klog.V(5).Infof("sub resource usage for pod %s. device %s usedmem %d usedcore %d", pod.Name, dev.DeviceInfo.ID, cono_dev.Usedmem, cono_dev.Usedcores) } } } @@ -220,7 +224,7 @@ func (ads *AscendDevices) addResource(annotations map[string]string, pod *v1.Pod Usedmem: cono_dev.Usedmem, } ads.AddResourceUsage(dev, cono_dev.Usedcores, cono_dev.Usedmem) - klog.V(5).Infof("add resource usage for pod %s. device %s usedmem %d", pod.Name, dev.DeviceInfo.ID, cono_dev.Usedmem) + klog.V(5).Infof("add resource usage for pod %s. device %s usedmem %d usedcore %d", pod.Name, dev.DeviceInfo.ID, cono_dev.Usedmem, cono_dev.Usedcores) } } } @@ -379,7 +383,7 @@ func (ads *AscendDevices) GetIgnoredDevices() []string { return []string{""} } vnpuConfig := randDev.config - return []string{vnpuConfig.ResourceMemoryName} + return []string{vnpuConfig.ResourceMemoryName, fmt.Sprintf("%s-core", vnpuConfig.ResourceName)} } func (ads *AscendDevices) GetStatus() string { @@ -412,6 +416,7 @@ func (ads *AscendDevices) DeepCopy() interface{} { DeviceUsage: newUsage, Score: dev.Score, PodMap: make(map[string]*devices.DeviceUsage), + hamiVnpuCore: dev.hamiVnpuCore, } for k, v := range dev.PodMap { newDev.PodMap[k] = &devices.DeviceUsage{ @@ -446,10 +451,13 @@ func (ads *AscendDevices) selectDevices(pod *v1.Pod, schedulePolicy string) (dev usedDevs := make([]*AscendDevice, 0) for _, req := range reqs { klog.V(5).Infof("req %+v", req) - err := verifyReq(req, dupDevs[0]) - if err != nil { - return nil, err + if !isHAMiCoreMode(pod) { + err := verifyReq(req, dupDevs[0]) + if err != nil { + return nil, err + } } + availableDevs := make([]*AscendDevice, 0) for _, dev := range dupDevs { selected := false @@ -467,7 +475,7 @@ func (ads *AscendDevices) selectDevices(pod *v1.Pod, schedulePolicy string) (dev selectedDevs := make([]*AscendDevice, 0) for _, dev := range availableDevs { klog.V(5).Infof("check fit. req %+v dev_info %+v dev_usage %+v", req, dev.DeviceInfo, dev.DeviceUsage) - if !fit(&req, dev) { + if !fit(&req, dev, isHAMiCoreMode(pod)) { klog.V(5).Infof("fit false. dev ID %s", dev.DeviceInfo.ID) continue } @@ -512,10 +520,13 @@ func hasNetworkID(devices []*AscendDevice) bool { return true } -func fit(req *devices.ContainerDeviceRequest, dev *AscendDevice) bool { +func fit(req *devices.ContainerDeviceRequest, dev *AscendDevice, isHAMiCore bool) bool { if req.Type != dev.config.CommonWord { return false } + if isHAMiCore && !dev.hamiVnpuCore { + return false + } deviceUsage := dev.DeviceUsage deviceInfo := dev.DeviceInfo if deviceInfo.Count <= deviceUsage.Used { @@ -524,13 +535,17 @@ func fit(req *devices.ContainerDeviceRequest, dev *AscendDevice) bool { if deviceInfo.Devmem-deviceUsage.Usedmem < req.Memreq { return false } - if deviceInfo.Devcore-deviceUsage.Usedcores < req.Coresreq { + effectiveTotalCore := dev.DeviceInfo.Devcore + if isHAMiCore { + effectiveTotalCore = 100 + } + if effectiveTotalCore-deviceUsage.Usedcores < req.Coresreq { return false } - if deviceInfo.Devcore == 100 && req.Coresreq == 100 && deviceUsage.Used > 0 { + if effectiveTotalCore == 100 && req.Coresreq == 100 && deviceUsage.Used > 0 { return false } - if deviceInfo.Devcore != 0 && deviceUsage.Usedcores == deviceInfo.Devcore && req.Coresreq == 0 { + if effectiveTotalCore != 0 && deviceUsage.Usedcores == effectiveTotalCore && req.Coresreq == 0 { return false } return true @@ -550,7 +565,8 @@ func getDeviceSnapshot(ads *AscendDevices) []*AscendDevice { Usedmem: dev.DeviceUsage.Usedmem, Usedcores: dev.DeviceUsage.Usedcores, }, - PodMap: make(map[string]*devices.DeviceUsage), + PodMap: make(map[string]*devices.DeviceUsage), + hamiVnpuCore: dev.hamiVnpuCore, } for k, v := range dev.PodMap { dupDev.PodMap[k] = &devices.DeviceUsage{ @@ -656,32 +672,42 @@ func (dev *AscendDevice) CommonWord() string { return dev.config.CommonWord } -func (dev *AscendDevice) GetNodeDevices(n v1.Node) ([]*devices.DeviceInfo, error) { +func (dev *AscendDevice) getNodeDevices(n *v1.Node, conf config.VNPUsConfig) ([]*devices.DeviceInfo, bool, error) { + nodeSupportHamiCore := conf.HamiVnpuCore + if val, ok := n.Annotations[VNPUNodeSelectorAnnotation]; ok { + klog.V(5).Infof("hami-vnpu-core is %s in node %s annotation", val, n.Name) + if val == "true" { + nodeSupportHamiCore = true + } else { + nodeSupportHamiCore = false + } + } anno, ok := n.Annotations[dev.nodeRegisterAnno] if !ok { - return []*devices.DeviceInfo{}, fmt.Errorf("annos not found %s", dev.nodeRegisterAnno) + return []*devices.DeviceInfo{}, nodeSupportHamiCore, fmt.Errorf("annos not found %s", dev.nodeRegisterAnno) } nodeDevices, err := devices.UnMarshalNodeDevices(anno) if err != nil { klog.ErrorS(err, "failed to unmarshal node devices", "node", n.Name, "device annotation", anno) - return []*devices.DeviceInfo{}, err + return []*devices.DeviceInfo{}, nodeSupportHamiCore, err } if len(nodeDevices) == 0 { klog.InfoS("no ascend device found", "node", n.Name, "device annotation", anno) - return []*devices.DeviceInfo{}, errors.New("no device found on node") + return []*devices.DeviceInfo{}, nodeSupportHamiCore, errors.New("no device found on node") } - return nodeDevices, nil + return nodeDevices, nodeSupportHamiCore, nil } func (dev *AscendDevice) ResourceReqs(pod *v1.Pod) []devices.ContainerDeviceRequest { - reqs := devices.ExtractResourceRequest(pod, dev.CommonWord(), dev.config.ResourceName, dev.config.ResourceMemoryName, "", "") + resourceCoreName := fmt.Sprintf("%s-core", dev.config.ResourceName) + reqs := devices.ExtractResourceRequest(pod, dev.CommonWord(), dev.config.ResourceName, dev.config.ResourceMemoryName, "", resourceCoreName) for i := range reqs { req := &reqs[i] if req.Memreq == 0 && req.MemPercentagereq != 0 { req.Memreq = int32(dev.DeviceInfo.Devmem * req.MemPercentagereq / 100) klog.V(5).Infof("new memreq %d totalmem %d mempercentage %d", req.Memreq, dev.DeviceInfo.Devmem, req.MemPercentagereq) } - if req.Memreq > 0 { + if req.Memreq > 0 && !isHAMiCoreMode(pod) { m, _ := dev.trimMemory(int64(req.Memreq)) klog.V(5).Infof("raw mem %d, trimed mem %d", req.Memreq, m) req.Memreq = int32(m) @@ -705,11 +731,15 @@ func (ads *AscendDevices) CreateAnnotations(pod *v1.Pod, devList devices.PodSing var rtInfo []RuntimeInfo for _, dp := range devList { for _, val := range dp { - _, temp := dev.trimMemory(int64(val.Usedmem)) - rtInfo = append(rtInfo, RuntimeInfo{ - UUID: val.UUID, - Temp: temp, - }) + info := RuntimeInfo{UUID: val.UUID} + if isHAMiCoreMode(pod) { + info.Memory = int64(val.Usedmem) + info.Core = val.Usedcores + } else { + _, temp := dev.trimMemory(int64(val.Usedmem)) + info.Temp = temp + } + rtInfo = append(rtInfo, info) } } s, err := json.Marshal(rtInfo) @@ -754,3 +784,10 @@ func verifyReq(req devices.ContainerDeviceRequest, dev *AscendDevice) error { } return nil } + +func isHAMiCoreMode(pod *v1.Pod) bool { + if pod == nil { + return false + } + return pod.Annotations[VNPUModeAnnotation] == VNPUModeHamiCore +} diff --git a/pkg/scheduler/api/devices/ascend/hami/device_info_test.go b/pkg/scheduler/api/devices/ascend/hami/device_info_test.go index 16814e2fc6d..4d09c6ddaaa 100644 --- a/pkg/scheduler/api/devices/ascend/hami/device_info_test.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info_test.go @@ -230,7 +230,7 @@ func Test_fit(t *testing.T) { conf, err := yamlStringToConfig(config_yaml) assert.Nil(t, err) ascend310PConfig := conf.VNPUConfigs()[len(conf.VNPUConfigs())-1] - device_info := &devices.DeviceInfo{ + deviceInfo := &devices.DeviceInfo{ ID: "68496E64-20E05477-92C31323-6E78030A-BD003019", Index: 0, Count: 7, @@ -238,107 +238,221 @@ func Test_fit(t *testing.T) { Devmem: 21527, } tests := []struct { - name string - req *devices.ContainerDeviceRequest - dev *AscendDevice - result bool + name string + req *devices.ContainerDeviceRequest + dev *AscendDevice + isHAMiCore bool + result bool }{ { - "test1", - &devices.ContainerDeviceRequest{ + name: "test1", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 1024, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 1, Usedmem: 3072, }, }, - true, + isHAMiCore: false, + result: true, }, { - "test2", - &devices.ContainerDeviceRequest{ + name: "test2", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 21527, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 1, Usedmem: 3072, }, }, - false, + isHAMiCore: false, + result: false, }, { - "test3", - &devices.ContainerDeviceRequest{ + name: "test3", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 6144, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 1, Usedmem: 12288, }, }, - true, + isHAMiCore: false, + result: true, }, { - "test4", - &devices.ContainerDeviceRequest{ + name: "test4", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 24576, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 0, Usedmem: 0, }, }, - false, + isHAMiCore: false, + result: false, }, { - "test5_core", - &devices.ContainerDeviceRequest{ + name: "test5_core", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 6144, Coresreq: 4, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 1, Usedmem: 12288, Usedcores: 6, }, }, - false, + isHAMiCore: false, + result: false, + }, + { + name: "hami_core_mode_should_fail_when_node_not_support_hami_vnpu_core", + req: &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", + Memreq: 1024, + Coresreq: 20, + }, + dev: &AscendDevice{ + config: ascend310PConfig, + DeviceInfo: deviceInfo, + DeviceUsage: &devices.DeviceUsage{}, + hamiVnpuCore: false, + }, + isHAMiCore: true, + result: false, + }, + { + name: "hami_core_mode_should_fail_when_core_is_insufficient", + req: &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", + Memreq: 1024, + Coresreq: 20, + }, + dev: &AscendDevice{ + config: ascend310PConfig, + DeviceInfo: deviceInfo, + DeviceUsage: &devices.DeviceUsage{ + Usedcores: 90, + }, + hamiVnpuCore: true, + }, + isHAMiCore: true, + result: false, + }, + { + name: "hami_core_mode_should_pass_when_node_support_hami_vnpu_core", + req: &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", + Memreq: 1024, + Coresreq: 20, + }, + dev: &AscendDevice{ + config: ascend310PConfig, + DeviceInfo: deviceInfo, + DeviceUsage: &devices.DeviceUsage{}, + hamiVnpuCore: true, + }, + isHAMiCore: true, + result: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ret := fit(tt.req, tt.dev) + ret := fit(tt.req, tt.dev, tt.isHAMiCore) assert.Equal(t, tt.result, ret) }) } } +func TestAscendDevice_GetNodeDevices_HAMiVnpuCore(t *testing.T) { + deviceRegisterAnno := fmt.Sprintf("%s/node-register-%s", util.HAMiAnnotationsPrefix, "Ascend910A") + testNodeDevicesJSON := `[{"id":"dev-1","index":0,"count":1,"devmem":32768,"devcore":30}]` + dev := &AscendDevice{ + nodeRegisterAnno: deviceRegisterAnno, + } + var conf config.VNPUsConfig + + tests := []struct { + name string + annotations map[string]string + expectSupport bool + }{ + { + name: "node_support_hami_vnpu_core", + annotations: map[string]string{ + deviceRegisterAnno: testNodeDevicesJSON, + VNPUNodeSelectorAnnotation: "true", + }, + expectSupport: true, + }, + { + name: "node_not_support_hami_vnpu_core", + annotations: map[string]string{ + deviceRegisterAnno: testNodeDevicesJSON, + VNPUNodeSelectorAnnotation: "false", + }, + expectSupport: false, + }, + { + name: "node_without_hami_vnpu_core_annotation", + annotations: map[string]string{ + deviceRegisterAnno: testNodeDevicesJSON, + }, + expectSupport: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: tt.annotations, + }, + } + + nodeDevices, nodeSupportHamiCore, err := dev.getNodeDevices(&node, conf) + assert.NoError(t, err) + assert.Len(t, nodeDevices, 1) + assert.Equal(t, tt.expectSupport, nodeSupportHamiCore) + }) + } +} + func Test_verifyReq(t *testing.T) { conf, err := yamlStringToConfig(config_yaml) assert.Nil(t, err) diff --git a/pkg/scheduler/api/devices/config/config.go b/pkg/scheduler/api/devices/config/config.go index 19fa3061094..49f61883616 100644 --- a/pkg/scheduler/api/devices/config/config.go +++ b/pkg/scheduler/api/devices/config/config.go @@ -151,8 +151,8 @@ func InitDevicesConfig(cmName, cmNamespace string) { var err error configs, err = loadConfigFromCM(devices.GetClient(), cmName, cmNamespace) if err != nil { - klog.V(3).InfoS("Volcano device config not found in namespace kube-system, using default config", - "name", cmName) + klog.V(3).Infof("failed to load config from ConfigMap %s in namespace %s: %v. using default config.", + cmName, cmNamespace, err) configs = GetDefaultDevicesConfig() } klog.V(3).InfoS("Initializing volcano device config", "device-configs", configs) diff --git a/pkg/scheduler/api/devices/util.go b/pkg/scheduler/api/devices/util.go index 6e62447ecef..d7434bc5e2b 100644 --- a/pkg/scheduler/api/devices/util.go +++ b/pkg/scheduler/api/devices/util.go @@ -67,19 +67,24 @@ const ( Skip ) -var kubeClient *kubernetes.Clientset +var kubeClient kubernetes.Interface func GetClient() kubernetes.Interface { - var err error if kubeClient == nil { - kubeClient, err = NewClient() + client, err := NewClient() if err != nil { klog.ErrorS(err, "deviceshare initClient failed") + } else { + kubeClient = client } } return kubeClient } +func SetClient(client kubernetes.Interface) { + kubeClient = client +} + // NewClient connects to an API server func NewClient() (*kubernetes.Clientset, error) { config, err := rest.InClusterConfig() diff --git a/pkg/webhooks/admission/pods/mutate/device_mutator.go b/pkg/webhooks/admission/pods/mutate/device_mutator.go index 4a3faa80957..292532a20ec 100644 --- a/pkg/webhooks/admission/pods/mutate/device_mutator.go +++ b/pkg/webhooks/admission/pods/mutate/device_mutator.go @@ -21,20 +21,19 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/klog/v2" + + hami "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/hami" devconfig "volcano.sh/volcano/pkg/scheduler/api/devices/config" - wkconfig "volcano.sh/volcano/pkg/webhooks/config" ) const ( ascendHAMiMutatorName = "ascend" jsonPatchOpAdd = "add" - VNPUModeAnnotation = "huawei.com/vnpu-mode" - VNPUModeHamiCore = "hami-core" ) type DeviceMutator interface { Name() string - MutateAdmission(pod *v1.Pod, deviceMutatorConfig wkconfig.DeviceMutatorConfig) []patchOperation + MutateAdmission(pod *v1.Pod) []patchOperation } var registeredDeviceMutators = []DeviceMutator{ @@ -51,15 +50,14 @@ func (m *AscendMutator) Name() string { return ascendHAMiMutatorName } -func (m *AscendMutator) MutateAdmission(pod *v1.Pod, deviceMutatorConfig wkconfig.DeviceMutatorConfig) []patchOperation { +func (m *AscendMutator) MutateAdmission(pod *v1.Pod) []patchOperation { if pod == nil || pod.Annotations == nil { return nil } - if vnpuMode, ok := pod.Annotations[VNPUModeAnnotation]; !ok || vnpuMode != VNPUModeHamiCore { + if vnpuMode, ok := pod.Annotations[hami.VNPUModeAnnotation]; !ok || vnpuMode != hami.VNPUModeHamiCore { return nil } klog.V(5).Infof("the hami-core annotation exists in pod %s/%s", pod.Namespace, pod.Name) - devconfig.InitDevicesConfig(deviceMutatorConfig.KnownGeometriesCMName, deviceMutatorConfig.KnownGeometriesCMNamespace) config := devconfig.GetConfig() if config == nil { klog.V(5).Infof("device configuration is empty.") @@ -68,16 +66,16 @@ func (m *AscendMutator) MutateAdmission(pod *v1.Pod, deviceMutatorConfig wkconfi var patch []patchOperation for idx, container := range pod.Spec.Containers { if container.Lifecycle != nil && container.Lifecycle.PostStart != nil { + klog.V(5).Infof("container %s already has a postStart lifecycle handler, skipping hami-vnpu-core injection", container.Name) continue } hasAscendResource := false - for _, vnpuConfig := range config.VNPUs { - if val, ok := container.Resources.Limits[v1.ResourceName(vnpuConfig.ResourceName)]; ok && !val.IsZero() { - hasAscendResource = true - break - } - if val, ok := container.Resources.Requests[v1.ResourceName(vnpuConfig.ResourceName)]; ok && !val.IsZero() { - hasAscendResource = true + for _, vnpuConfig := range config.VNPUs.Configs { + if checkResource(container.Resources, vnpuConfig.ResourceName) { + resourceCoreName := fmt.Sprintf("%s-core", vnpuConfig.ResourceName) + if checkResource(container.Resources, vnpuConfig.ResourceMemoryName) && checkResource(container.Resources, resourceCoreName) { + hasAscendResource = true + } break } } @@ -88,7 +86,7 @@ func (m *AscendMutator) MutateAdmission(pod *v1.Pod, deviceMutatorConfig wkconfi handler := v1.LifecycleHandler{ Exec: &v1.ExecAction{ Command: []string{ - "bash", + "/bin/sh", "-c", "export RUST_LOG=info\n/hami-vnpu-core/limiter > /tmp/limiter_manager.log 2>&1 &", }, @@ -114,3 +112,13 @@ func (m *AscendMutator) MutateAdmission(pod *v1.Pod, deviceMutatorConfig wkconfi return patch } + +func checkResource(req v1.ResourceRequirements, resourceName string) bool { + if val, ok := req.Limits[v1.ResourceName(resourceName)]; ok && !val.IsZero() { + return true + } + if val, ok := req.Requests[v1.ResourceName(resourceName)]; ok && !val.IsZero() { + return true + } + return false +} diff --git a/pkg/webhooks/admission/pods/mutate/device_mutator_test.go b/pkg/webhooks/admission/pods/mutate/device_mutator_test.go new file mode 100644 index 00000000000..8ab80dc517b --- /dev/null +++ b/pkg/webhooks/admission/pods/mutate/device_mutator_test.go @@ -0,0 +1,444 @@ +/* +Copyright 2026 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutate + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + devices "volcano.sh/volcano/pkg/scheduler/api/devices" + devconfig "volcano.sh/volcano/pkg/scheduler/api/devices/config" +) + +func TestAscendMutator_Name(t *testing.T) { + mutator := newAscendMutator() + if mutator.Name() != "ascend" { + t.Errorf("expected name 'ascend', got '%s'", mutator.Name()) + } +} + +func TestAscendMutator_MutateAdmission(t *testing.T) { + const ( + knownGeometriesCMName = "test-cm" + knownGeometriesCMNamespace = "test-ns" + deviceConfigDataKey = "device-config.yaml" + ) + + deviceConfigYAML := "vnpus:\n" + + " hamiVnpuCore: true\n" + + " configs:\n" + + " - chipName: cm-310P3\n" + + " commonWord: Ascend310P\n" + + " resourceName: huawei.com/Ascend310P\n" + + " resourceMemoryName: huawei.com/Ascend310P-memory\n" + + " memoryAllocatable: 21527\n" + + " memoryCapacity: 24576\n" + + " aiCore: 8\n" + + " aiCPU: 7\n" + + " templates:\n" + + " - name: vir01\n" + + " memory: 3072\n" + + " aiCore: 1\n" + + " aiCPU: 1\n" + + " - chipName: cm-910A\n" + + " commonWord: Ascend910A\n" + + " resourceName: huawei.com/Ascend910A\n" + + " resourceMemoryName: huawei.com/Ascend910A-memory\n" + + " memoryAllocatable: 32768\n" + + " memoryCapacity: 32768\n" + + " aiCore: 30\n" + + " templates:\n" + + " - name: vir02\n" + + " memory: 2184\n" + + " aiCore: 2\n" + + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: knownGeometriesCMName, + Namespace: knownGeometriesCMNamespace, + }, + Data: map[string]string{ + deviceConfigDataKey: deviceConfigYAML, + }, + } + + fakeClient := fake.NewSimpleClientset(configMap) + devices.SetClient(fakeClient) + defer devices.SetClient(nil) + devconfig.InitDevicesConfig(knownGeometriesCMName, knownGeometriesCMNamespace) + tests := []struct { + name string + pod *v1.Pod + expectedPatchCount int + expectedPatchPaths []string + expectedPatchOps []string + }{ + { + name: "nil pod returns nil", + pod: nil, + expectedPatchCount: 0, + }, + { + name: "pod without annotations returns nil", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 0, + }, + { + name: "pod without hami-core annotation returns nil", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "other-mode", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 0, + }, + { + name: "container without ascend resource returns nil", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 0, + }, + { + name: "container with lifecycle postStart already exists skip", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + Lifecycle: &v1.Lifecycle{ + PostStart: &v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: []string{"echo", "hello"}, + }, + }, + }, + }, + }, + }, + }, + expectedPatchCount: 0, + }, + { + name: "successfully add lifecycle to container without lifecycle", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 1, + expectedPatchPaths: []string{"/spec/containers/0/lifecycle"}, + expectedPatchOps: []string{"add"}, + }, + { + name: "successfully add postStart to container with lifecycle but no postStart", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + Lifecycle: &v1.Lifecycle{ + PreStop: &v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: []string{"echo", "goodbye"}, + }, + }, + }, + }, + }, + }, + }, + expectedPatchCount: 1, + expectedPatchPaths: []string{"/spec/containers/0/lifecycle/postStart"}, + expectedPatchOps: []string{"add"}, + }, + { + name: "multiple containers with ascend resource", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container-1", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + { + Name: "container-2", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("2"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + { + Name: "container-3", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 2, + expectedPatchPaths: []string{ + "/spec/containers/0/lifecycle", + "/spec/containers/1/lifecycle", + }, + expectedPatchOps: []string{"add", "add"}, + }, + { + name: "container with ascend resource in requests", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 1, + expectedPatchPaths: []string{"/spec/containers/0/lifecycle"}, + expectedPatchOps: []string{"add"}, + }, + { + name: "multiple resource types in config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 1, + expectedPatchPaths: []string{"/spec/containers/0/lifecycle"}, + expectedPatchOps: []string{"add"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mutator := newAscendMutator() + patches := mutator.MutateAdmission(tt.pod) + + if len(patches) != tt.expectedPatchCount { + t.Errorf("expected %d patches, got %d", tt.expectedPatchCount, len(patches)) + } + + for i, patch := range patches { + if i < len(tt.expectedPatchPaths) && patch.Path != tt.expectedPatchPaths[i] { + t.Errorf("expected patch path %s, got %s", tt.expectedPatchPaths[i], patch.Path) + } + if i < len(tt.expectedPatchOps) && patch.Op != tt.expectedPatchOps[i] { + t.Errorf("expected patch op %s, got %s", tt.expectedPatchOps[i], patch.Op) + } + if patch.Op == jsonPatchOpAdd { + if patch.Path == "/spec/containers/0/lifecycle" || patch.Path == "/spec/containers/1/lifecycle" { + lifecycle, ok := patch.Value.(v1.Lifecycle) + if !ok { + t.Errorf("patch value is not v1.Lifecycle") + } + if lifecycle.PostStart == nil { + t.Errorf("PostStart handler is nil") + } + if lifecycle.PostStart.Exec == nil { + t.Errorf("Exec action is nil") + } + if len(lifecycle.PostStart.Exec.Command) != 3 { + t.Errorf("expected command length 3, got %d", len(lifecycle.PostStart.Exec.Command)) + } + } + } + } + }) + } + + config := devconfig.GetConfig() + if config == nil { + t.Fatalf("expected devices config loaded from configmap, got nil") + } + if len(config.VNPUs.Configs) != 2 { + t.Fatalf("expected 2 vnpu configs from configmap, got %d", len(config.VNPUs.Configs)) + } + if config.VNPUs.Configs[0].ChipName != "cm-310P3" { + t.Fatalf("expected chipName loaded from configmap to be cm-310P3, got %s", config.VNPUs.Configs[0].ChipName) + } +} + +// TestAscendMutator_RegisteredDevicesMutators verifies that AscendMutator is registered +func TestAscendMutator_RegisteredDevicesMutators(t *testing.T) { + found := false + for _, mutator := range registeredDeviceMutators { + if mutator.Name() == "ascend" { + found = true + break + } + } + if !found { + t.Error("AscendMutator not found in registeredDeviceMutators") + } +} diff --git a/pkg/webhooks/admission/pods/mutate/mutate_pod.go b/pkg/webhooks/admission/pods/mutate/mutate_pod.go index 5bcc925f580..a7e63e7a1f4 100644 --- a/pkg/webhooks/admission/pods/mutate/mutate_pod.go +++ b/pkg/webhooks/admission/pods/mutate/mutate_pod.go @@ -121,7 +121,7 @@ func createPatch(pod *v1.Pod) ([]byte, error) { } if config.ConfigData.DeviceMutatorConf.Enable { - patch = append(patch, patchDeviceMutation(pod, config.ConfigData.DeviceMutatorConf)...) + patch = append(patch, patchDeviceMutation(pod)...) } for _, resourceGroup := range config.ConfigData.ResGroupsConfig { @@ -265,15 +265,14 @@ func patchSchedulerName(resGroupConfig wkconfig.ResGroupConfig) *patchOperation return &patchOperation{Op: "add", Path: "/spec/schedulerName", Value: resGroupConfig.SchedulerName} } -func patchDeviceMutation(pod *v1.Pod, deviceMutatorConf wkconfig.DeviceMutatorConfig) []patchOperation { +func patchDeviceMutation(pod *v1.Pod) []patchOperation { var patch []patchOperation for _, mutator := range registeredDeviceMutators { - patches := mutator.MutateAdmission(pod, deviceMutatorConf) + patches := mutator.MutateAdmission(pod) if len(patches) == 0 { continue } patch = append(patch, patches...) - } return patch }