From a2b51be6e1e4bfc1b1fb6a844cec36a4be281adf Mon Sep 17 00:00:00 2001 From: Sabbir Hasan Date: Mon, 18 Sep 2023 22:33:03 +0200 Subject: [PATCH] initial commit to include cpu frequency Signed-off-by: Sabbir Hasan --- .../tuned/openshift-node-performance | 10 ++++++++++ ...ance.openshift.io_performanceprofiles.yaml | 6 ++++++ .../performance_v2_performanceprofile.yaml | 2 ++ manifests/20-performance-profile.crd.yaml | 6 ++++++ .../v1/performanceprofile_types.go | 7 +++++++ .../v1/zz_generated.deepcopy.go | 10 ++++++++++ .../v2/performanceprofile_conversion.go | 9 +++++++++ .../v2/performanceprofile_types.go | 7 +++++++ .../v2/zz_generated.deepcopy.go | 10 ++++++++++ .../components/tuned/tuned.go | 19 ++++++++++++++++++- .../utils/testing/testing.go | 14 +++++++++++--- 11 files changed, 96 insertions(+), 4 deletions(-) diff --git a/assets/performanceprofile/tuned/openshift-node-performance b/assets/performanceprofile/tuned/openshift-node-performance index d6dcc120c5..02aaa6f5dd 100644 --- a/assets/performanceprofile/tuned/openshift-node-performance +++ b/assets/performanceprofile/tuned/openshift-node-performance @@ -162,4 +162,14 @@ cmdline_pstate=+intel_pstate=passive cmdline_pstate=+intel_pstate=disable {{end}} +{{ if .ReservedCpuMaxFreq }} +{{- "[sysfs]" -}} +{{ range .ReservedCpuList }} +/sys/devices/system/cpu/cpufreq/policy[{{.}}]/scaling_max_freq={{$.ReservedCpuMaxFreq}} +{{- if $.ReservedCpuMinFreq }} +/sys/devices/system/cpu/cpufreq/policy[{{.}}]/scaling_min_freq={{$.ReservedCpuMinFreq}} +{{- end -}} +{{ end }} +{{ end }} + [rtentsk] diff --git a/examples/performanceprofile/crd/bases/performance.openshift.io_performanceprofiles.yaml b/examples/performanceprofile/crd/bases/performance.openshift.io_performanceprofiles.yaml index 1028c6daf5..b6cf9e3cc1 100644 --- a/examples/performanceprofile/crd/bases/performance.openshift.io_performanceprofiles.yaml +++ b/examples/performanceprofile/crd/bases/performance.openshift.io_performanceprofiles.yaml @@ -74,6 +74,12 @@ spec: description: Reserved defines a set of CPUs that will not be used for any container workloads initiated by kubelet. type: string + reservedCpuMaxFreq: + description: ReservedCpuMaxFreq defines the maximum cpu frequency for reserved CPUs. + type: integer + reservedCpuMinFreq: + description: ReservedCpuMinFreq defines the maximum cpu frequency for reserved CPUs. + type: integer required: - isolated type: object diff --git a/examples/performanceprofile/samples/performance_v2_performanceprofile.yaml b/examples/performanceprofile/samples/performance_v2_performanceprofile.yaml index 498ea18f49..e995bad0aa 100644 --- a/examples/performanceprofile/samples/performance_v2_performanceprofile.yaml +++ b/examples/performanceprofile/samples/performance_v2_performanceprofile.yaml @@ -14,6 +14,8 @@ spec: isolated: "2" reserved: "0-1" offlined: "3" + reservedCpuMaxFreq: 2800000 + reservedCpuMinFreq: 2600000 hugepages: defaultHugepagesSize: "1G" pages: diff --git a/manifests/20-performance-profile.crd.yaml b/manifests/20-performance-profile.crd.yaml index acf73aeb9e..66a20000ce 100644 --- a/manifests/20-performance-profile.crd.yaml +++ b/manifests/20-performance-profile.crd.yaml @@ -91,6 +91,12 @@ spec: description: Reserved defines a set of CPUs that will not be used for any container workloads initiated by kubelet. type: string + reservedCpuMaxFreq: + description: ReservedCpuMaxFreq defines the maximum cpu frequency for reserved CPUs. + type: integer + reservedCpuMinFreq: + description: ReservedCpuMinFreq defines the maximum cpu frequency for reserved CPUs. + type: integer required: - isolated type: object diff --git a/pkg/apis/performanceprofile/v1/performanceprofile_types.go b/pkg/apis/performanceprofile/v1/performanceprofile_types.go index e352577435..6dbd43d2aa 100644 --- a/pkg/apis/performanceprofile/v1/performanceprofile_types.go +++ b/pkg/apis/performanceprofile/v1/performanceprofile_types.go @@ -78,6 +78,7 @@ type PerformanceProfileSpec struct { // CPUSet defines the set of CPUs(0-3,8-11). type CPUSet string +type CPUfrequency int // CPU defines a set of CPU related features. type CPU struct { @@ -102,6 +103,12 @@ type CPU struct { // Offline defines a set of CPUs that will be unused and set offline // +optional Offlined *CPUSet `json:"offlined,omitempty"` + // + // +optional + ReservedCpuMaxFreq *CPUfrequency `json:"reservedCpuMaxFreq,omitempty"` + // + // +optional + ReservedCpuMinFreq *CPUfrequency `json:"reservedCpuMinFreq,omitempty"` } // HugePageSize defines size of huge pages, can be 2M or 1G. diff --git a/pkg/apis/performanceprofile/v1/zz_generated.deepcopy.go b/pkg/apis/performanceprofile/v1/zz_generated.deepcopy.go index b92382c30c..afe826d395 100644 --- a/pkg/apis/performanceprofile/v1/zz_generated.deepcopy.go +++ b/pkg/apis/performanceprofile/v1/zz_generated.deepcopy.go @@ -33,6 +33,16 @@ func (in *CPU) DeepCopyInto(out *CPU) { *out = new(CPUSet) **out = **in } + if in.ReservedCpuMaxFreq != nil { + in, out := &in.ReservedCpuMaxFreq, &out.ReservedCpuMaxFreq + *out = new(CPUfrequency) + **out = **in + } + if in.ReservedCpuMinFreq != nil { + in, out := &in.ReservedCpuMinFreq, &out.ReservedCpuMinFreq + *out = new(CPUfrequency) + **out = **in + } return } diff --git a/pkg/apis/performanceprofile/v2/performanceprofile_conversion.go b/pkg/apis/performanceprofile/v2/performanceprofile_conversion.go index ac647944f3..65aa66e533 100644 --- a/pkg/apis/performanceprofile/v2/performanceprofile_conversion.go +++ b/pkg/apis/performanceprofile/v2/performanceprofile_conversion.go @@ -31,6 +31,15 @@ func (curr *PerformanceProfile) ConvertTo(dstRaw conversion.Hub) error { if curr.Spec.CPU.BalanceIsolated != nil { dst.Spec.CPU.BalanceIsolated = pointer.BoolPtr(*curr.Spec.CPU.BalanceIsolated) } + //test + if curr.Spec.CPU.ReservedCpuMaxFreq != nil { + maxCpuFrequency := v1.CPUfrequency(*curr.Spec.CPU.ReservedCpuMaxFreq) + dst.Spec.CPU.ReservedCpuMaxFreq = &maxCpuFrequency + } + if curr.Spec.CPU.ReservedCpuMinFreq != nil { + minCpuFrequency := v1.CPUfrequency(*curr.Spec.CPU.ReservedCpuMaxFreq) + dst.Spec.CPU.ReservedCpuMinFreq = &minCpuFrequency + } } if curr.Spec.HugePages != nil { diff --git a/pkg/apis/performanceprofile/v2/performanceprofile_types.go b/pkg/apis/performanceprofile/v2/performanceprofile_types.go index 420ed86506..1584eea01b 100644 --- a/pkg/apis/performanceprofile/v2/performanceprofile_types.go +++ b/pkg/apis/performanceprofile/v2/performanceprofile_types.go @@ -86,6 +86,7 @@ type PerformanceProfileSpec struct { // CPUSet defines the set of CPUs(0-3,8-11). type CPUSet string +type CPUfrequency int // CPU defines a set of CPU related features. type CPU struct { @@ -110,6 +111,12 @@ type CPU struct { // Offline defines a set of CPUs that will be unused and set offline // +optional Offlined *CPUSet `json:"offlined,omitempty"` + // + // +optional + ReservedCpuMaxFreq *CPUfrequency `json:"reservedCpuMaxFreq,omitempty"` + // + // +optional + ReservedCpuMinFreq *CPUfrequency `json:"reservedCpuMinFreq,omitempty"` } // HugePageSize defines size of huge pages, can be 2M or 1G. diff --git a/pkg/apis/performanceprofile/v2/zz_generated.deepcopy.go b/pkg/apis/performanceprofile/v2/zz_generated.deepcopy.go index eac53c002d..6a31c8b6a2 100644 --- a/pkg/apis/performanceprofile/v2/zz_generated.deepcopy.go +++ b/pkg/apis/performanceprofile/v2/zz_generated.deepcopy.go @@ -33,6 +33,16 @@ func (in *CPU) DeepCopyInto(out *CPU) { *out = new(CPUSet) **out = **in } + if in.ReservedCpuMaxFreq != nil { + in, out := &in.ReservedCpuMaxFreq, &out.ReservedCpuMaxFreq + *out = new(CPUfrequency) + **out = **in + } + if in.ReservedCpuMinFreq != nil { + in, out := &in.ReservedCpuMinFreq, &out.ReservedCpuMinFreq + *out = new(CPUfrequency) + **out = **in + } return } diff --git a/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go b/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go index ccca2205e6..8b8e70736f 100644 --- a/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go +++ b/pkg/performanceprofile/controller/performanceprofile/components/tuned/tuned.go @@ -32,6 +32,9 @@ const ( templateRealTimeHint = "RealTimeHint" templateHighPowerConsumption = "HighPowerConsumption" templatePerPodPowerManagement = "PerPodPowerManagement" + templateReservedCpuMaxFreq = "ReservedCpuMaxFreq" + templateReservedCpuMinFreq = "ReservedCpuMinFreq" + templateReservedCpuList = "ReservedCpuList" ) func new(name string, profiles []tunedv1.TunedProfile, recommends []tunedv1.TunedRecommend) *tunedv1.Tuned { @@ -53,7 +56,8 @@ func new(name string, profiles []tunedv1.TunedProfile, recommends []tunedv1.Tune // NewNodePerformance returns tuned profile for performance sensitive workflows func NewNodePerformance(profile *performancev2.PerformanceProfile) (*tunedv1.Tuned, error) { - templateArgs := make(map[string]string) + // templateArgs := make(map[string]string) + templateArgs := make(map[string]interface{}) if profile.Spec.CPU.Isolated != nil { templateArgs[templateIsolatedCpus] = string(*profile.Spec.CPU.Isolated) @@ -62,6 +66,19 @@ func NewNodePerformance(profile *performancev2.PerformanceProfile) (*tunedv1.Tun } } + if profile.Spec.CPU.Reserved != nil && profile.Spec.CPU.ReservedCpuMaxFreq != nil { + + reservedSet, err := cpuset.Parse(string(*profile.Spec.CPU.Reserved)) + if err != nil { + return nil, err + } + + reservedCPuList := reservedSet.List() + templateArgs[templateReservedCpuList] = reservedCPuList + templateArgs[templateReservedCpuMaxFreq] = int(*profile.Spec.CPU.ReservedCpuMaxFreq) + templateArgs[templateReservedCpuMinFreq] = int(*profile.Spec.CPU.ReservedCpuMinFreq) + } + if profile.Spec.HugePages != nil { var defaultHugepageSize performancev2.HugePageSize if profile.Spec.HugePages.DefaultHugePagesSize != nil { diff --git a/pkg/performanceprofile/utils/testing/testing.go b/pkg/performanceprofile/utils/testing/testing.go index 3a8c9c0ce8..77be35531d 100644 --- a/pkg/performanceprofile/utils/testing/testing.go +++ b/pkg/performanceprofile/utils/testing/testing.go @@ -20,6 +20,10 @@ const ( IsolatedCPUs = performancev2.CPUSet("4-5") // ReservedCPUs defines the reserved CPU set used for tests ReservedCPUs = performancev2.CPUSet("0-3") + //test + ReservedCpuMaxFreq = performancev2.CPUfrequency(2800000) + //test + ReservedCpuMinFreq = performancev2.CPUfrequency(2500000) // OfflinedCPUs defines the Offline CPU set used for tests OfflinedCPUs = performancev2.CPUSet("6-7") // SingleNUMAPolicy defines the topologyManager policy used for tests SingleNUMAPolicy = "single-numa-node" @@ -42,6 +46,8 @@ func NewPerformanceProfile(name string) *performancev2.PerformanceProfile { size := HugePageSize isolatedCPUs := IsolatedCPUs reservedCPUs := ReservedCPUs + reservedCpuMaxFreq := ReservedCpuMaxFreq + reservedCpuMinFreq := ReservedCpuMinFreq offlineCPUs := OfflinedCPUs numaPolicy := SingleNUMAPolicy additionalKernelArgs := AdditionalKernelArgs @@ -54,9 +60,11 @@ func NewPerformanceProfile(name string) *performancev2.PerformanceProfile { }, Spec: performancev2.PerformanceProfileSpec{ CPU: &performancev2.CPU{ - Isolated: &isolatedCPUs, - Reserved: &reservedCPUs, - Offlined: &offlineCPUs, + Isolated: &isolatedCPUs, + Reserved: &reservedCPUs, + Offlined: &offlineCPUs, + ReservedCpuMaxFreq: &reservedCpuMaxFreq, + ReservedCpuMinFreq: &reservedCpuMinFreq, }, HugePages: &performancev2.HugePages{ DefaultHugePagesSize: &size,