diff --git a/cmd/performance-profile-creator/cmd/root.go b/cmd/performance-profile-creator/cmd/root.go index e538f63886..03f448df03 100644 --- a/cmd/performance-profile-creator/cmd/root.go +++ b/cmd/performance-profile-creator/cmd/root.go @@ -99,6 +99,8 @@ type ProfileData struct { realtimeHint *bool highPowerConsumptionHint *bool perPodPowerManagementHint *bool + reservedCpuMinFreq int + reservedCpuMaxFreq int } // ClusterData collects the cluster wide information, each mcp points to a list of ghw node handlers @@ -193,6 +195,7 @@ func NewRootCommand() *cobra.Command { root.PersistentFlags().StringVar(&pcArgs.TMPolicy, "topology-manager-policy", kubeletconfig.RestrictedTopologyManagerPolicy, fmt.Sprintf("Kubelet Topology Manager Policy of the performance profile to be created. [Valid values: %s, %s, %s]", kubeletconfig.SingleNumaNodeTopologyManagerPolicy, kubeletconfig.BestEffortTopologyManagerPolicy, kubeletconfig.RestrictedTopologyManagerPolicy)) root.PersistentFlags().StringVar(&pcArgs.Info, "info", infoModeLog, fmt.Sprintf("Show cluster information; requires --must-gather-dir-path, ignore the other arguments. [Valid values: %s]", strings.Join(validInfoModes, ", "))) root.PersistentFlags().BoolVar(pcArgs.PerPodPowerManagement, "per-pod-power-management", false, "Enable Per Pod Power Management") + root.PersistentFlags().BoolVar(&pcArgs.BoostFrequency, "boost-frequency", false, "Enable frequency tuning for reserved cpus") return root } @@ -400,6 +403,11 @@ func getDataFromFlags(cmd *cobra.Command) (ProfileCreatorArgs, error) { return creatorArgs, fmt.Errorf("failed to parse disable-ht flag: %v", err) } + boostFrequency, err := strconv.ParseBool(cmd.Flag("boost-frequency").Value.String()) + if err != nil { + return creatorArgs, fmt.Errorf("failed to parse boost-frequency flag: %v", err) + } + creatorArgs = ProfileCreatorArgs{ MustGatherDirPath: mustGatherDirPath, ProfileName: profileName, @@ -411,6 +419,7 @@ func getDataFromFlags(cmd *cobra.Command) (ProfileCreatorArgs, error) { RTKernel: rtKernelEnabled, PowerConsumptionMode: powerConsumptionMode, DisableHT: htDisabled, + BoostFrequency: boostFrequency, } if cmd.Flag("user-level-networking").Changed { @@ -485,6 +494,7 @@ func getProfileData(args ProfileCreatorArgs, cluster ClusterData) (*ProfileData, } log.Infof("%d reserved CPUs allocated: %v ", reservedCPUs.Size(), reservedCPUs.String()) log.Infof("%d isolated CPUs allocated: %v", isolatedCPUs.Size(), isolatedCPUs.String()) + kernelArgs := profilecreator.GetAdditionalKernelArgs(args.DisableHT) profileData := &ProfileData{ reservedCPUs: reservedCPUs.String(), @@ -501,6 +511,15 @@ func getProfileData(args ProfileCreatorArgs, cluster ClusterData) (*ProfileData, perPodPowerManagementHint: args.PerPodPowerManagement, } + // test: set frequency value + if args.BoostFrequency { + profileData.reservedCpuMaxFreq, profileData.reservedCpuMinFreq, err = profilecreator.CalculateFrequency(args.MustGatherDirPath, profileData.reservedCPUs) + if err != nil { + return nil, fmt.Errorf("failed to compute the maximum and minimum frequencies for reserved CPUs: %v", err) + } + log.Infof("reserved CPUs max frequency:%v, min frequency:%v", profileData.reservedCpuMaxFreq, profileData.reservedCpuMinFreq) + } + // setting workload hints switch args.PowerConsumptionMode { case defaultLatency: @@ -558,6 +577,8 @@ type ProfileCreatorArgs struct { TMPolicy string `json:"topology-manager-policy"` Info string `json:"info"` PerPodPowerManagement *bool `json:"per-pod-power-management,omitempty"` + //test + BoostFrequency bool `json:"boost-frequency,omitempty"` } func createProfile(profileData ProfileData) error { @@ -593,7 +614,14 @@ func createProfile(profileData ProfileData) error { offlined := performancev2.CPUSet(profileData.offlinedCPUs) profile.Spec.CPU.Offlined = &offlined } - + if profileData.reservedCpuMaxFreq > 0 && profileData.reservedCpuMinFreq > 0 { + minFreq := performancev2.CPUfrequency(profileData.reservedCpuMinFreq) + maxFreq := performancev2.CPUfrequency(profileData.reservedCpuMaxFreq) + profile.Spec.HardwareTuning = &performancev2.HardwareTuning{ + ReservedCpuMinFreq: &minFreq, + ReservedCpuMaxFreq: &maxFreq, + } + } if len(profileData.additionalKernelArgs) > 0 { profile.Spec.AdditionalKernelArgs = profileData.additionalKernelArgs } diff --git a/examples/performanceprofile/crd/bases/performance.openshift.io_performanceprofiles.yaml b/examples/performanceprofile/crd/bases/performance.openshift.io_performanceprofiles.yaml index 1028c6daf5..b596e0008c 100644 --- a/examples/performanceprofile/crd/bases/performance.openshift.io_performanceprofiles.yaml +++ b/examples/performanceprofile/crd/bases/performance.openshift.io_performanceprofiles.yaml @@ -86,6 +86,17 @@ spec: per pod CPUs when using irq-load-balancing.crio.io/cpu-quota.crio.io annotations. Defaults to "false" type: boolean + hardwareTuning: + description: HardwareTuning + properties: + reservedCpuMinFreq: + description: ReservedCpuMinFreq defines the maximum cpu frequency for reserved CPUs. + format: int32 + type: string + reservedCpuMaxFreq: + description: ReservedCpuMaxFreq defines the maximum cpu frequency for reserved CPUs. + format: int32 + type: integer hugepages: description: HugePages defines a set of huge pages related parameters. It is possible to set huge pages with multiple size values at the diff --git a/pkg/apis/performanceprofile/v1/performanceprofile_types.go b/pkg/apis/performanceprofile/v1/performanceprofile_types.go index e352577435..a3d0ba6fd2 100644 --- a/pkg/apis/performanceprofile/v1/performanceprofile_types.go +++ b/pkg/apis/performanceprofile/v1/performanceprofile_types.go @@ -30,6 +30,9 @@ const PerformanceProfilePauseAnnotation = "performance.openshift.io/pause-reconc type PerformanceProfileSpec struct { // CPU defines a set of CPU related parameters. CPU *CPU `json:"cpu"` + // HardwareTuning defines maximum and minuimum cpu frequencies for reserved cpus + // +optional + HardwareTuning *HardwareTuning `json:"hardwareTuning,omitempty"` // HugePages defines a set of huge pages related parameters. // It is possible to set huge pages with multiple size values at the same time. // For example, hugepages can be set with 1G and 2M, both values will be set on the node by the performance-addon-operator. @@ -78,6 +81,7 @@ type PerformanceProfileSpec struct { // CPUSet defines the set of CPUs(0-3,8-11). type CPUSet string +type CPUfrequency int // CPU defines a set of CPU related features. type CPU struct { @@ -104,6 +108,16 @@ type CPU struct { Offlined *CPUSet `json:"offlined,omitempty"` } +// // HardwareTuning defines a set of CPU frequency related features. +type HardwareTuning struct { + // ReservedCpuMinFreq defines a minimum frequency to be set across reserved cpus + ReservedCpuMinFreq *CPUfrequency `json:"reservedCpuMinFreq,omitempty"` + // ReservedCpuMaxFreq defines a maximum frequency to be set across reserved cpus + ReservedCpuMaxFreq *CPUfrequency `json:"reservedCpuMaxFreq,omitempty"` +} + +// + // HugePageSize defines size of huge pages, can be 2M or 1G. type HugePageSize string diff --git a/pkg/apis/performanceprofile/v2/performanceprofile_conversion.go b/pkg/apis/performanceprofile/v2/performanceprofile_conversion.go index ac647944f3..33d01b7bcc 100644 --- a/pkg/apis/performanceprofile/v2/performanceprofile_conversion.go +++ b/pkg/apis/performanceprofile/v2/performanceprofile_conversion.go @@ -33,6 +33,18 @@ func (curr *PerformanceProfile) ConvertTo(dstRaw conversion.Hub) error { } } + if curr.Spec.HardwareTuning != nil { + dst.Spec.HardwareTuning = new(v1.HardwareTuning) + if curr.Spec.HardwareTuning.ReservedCpuMaxFreq != nil { + maxCpuFrequency := v1.CPUfrequency(*curr.Spec.HardwareTuning.ReservedCpuMaxFreq) + dst.Spec.HardwareTuning.ReservedCpuMaxFreq = &maxCpuFrequency + } + if curr.Spec.HardwareTuning.ReservedCpuMinFreq != nil { + minCpuFrequency := v1.CPUfrequency(*curr.Spec.HardwareTuning.ReservedCpuMaxFreq) + dst.Spec.HardwareTuning.ReservedCpuMinFreq = &minCpuFrequency + } + } + if curr.Spec.HugePages != nil { dst.Spec.HugePages = new(v1.HugePages) diff --git a/pkg/apis/performanceprofile/v2/performanceprofile_types.go b/pkg/apis/performanceprofile/v2/performanceprofile_types.go index 420ed86506..c3064eddca 100644 --- a/pkg/apis/performanceprofile/v2/performanceprofile_types.go +++ b/pkg/apis/performanceprofile/v2/performanceprofile_types.go @@ -38,6 +38,9 @@ const PerformanceProfileEnableRpsAnnotation = "performance.openshift.io/enable-r type PerformanceProfileSpec struct { // CPU defines a set of CPU related parameters. CPU *CPU `json:"cpu"` + // HardwareTuning defines maximum and minuimum cpu frequencies for reserved cpus + // +optional + HardwareTuning *HardwareTuning `json:"hardwareTuning,omitempty"` // HugePages defines a set of huge pages related parameters. // It is possible to set huge pages with multiple size values at the same time. // For example, hugepages can be set with 1G and 2M, both values will be set on the node by the Performance Profile Controller. @@ -86,6 +89,7 @@ type PerformanceProfileSpec struct { // CPUSet defines the set of CPUs(0-3,8-11). type CPUSet string +type CPUfrequency int // CPU defines a set of CPU related features. type CPU struct { @@ -112,6 +116,14 @@ type CPU struct { Offlined *CPUSet `json:"offlined,omitempty"` } +// // HardwareTuning defines a set of CPU frequency related features. +type HardwareTuning struct { + // ReservedCpuMinFreq defines a minimum frequency to be set across reserved cpus + ReservedCpuMinFreq *CPUfrequency `json:"reservedCpuMinFreq,omitempty"` + // ReservedCpuMaxFreq defines a maximum frequency to be set across reserved cpus + ReservedCpuMaxFreq *CPUfrequency `json:"reservedCpuMaxFreq,omitempty"` +} + // HugePageSize defines size of huge pages, can be 2M or 1G. type HugePageSize string diff --git a/pkg/performanceprofile/profilecreator/profilecreator.go b/pkg/performanceprofile/profilecreator/profilecreator.go index 2ccfe76a87..9f04e07f49 100644 --- a/pkg/performanceprofile/profilecreator/profilecreator.go +++ b/pkg/performanceprofile/profilecreator/profilecreator.go @@ -24,6 +24,7 @@ import ( "path/filepath" "reflect" "sort" + "strconv" "strings" "github.com/jaypipes/ghw" @@ -59,6 +60,12 @@ const ( noSMTKernelArg = "nosmt" // allCores correspond to the value when all the processorCores need to be added to the generated CPUset allCores = -1 + // + JSONSuffix = ".json" + // + listCPU = "lscpu" + // + percentage = 95 ) var ( @@ -835,3 +842,81 @@ func IsLogicalProcessorUsed(extCPUInfo *extendedCPUInfo, logicalProcessor int) b _, ok := extCPUInfo.LogicalProcessorsUsed[logicalProcessor] return ok } + +//structure to read lscpu.json file +type Response struct { + Cpu []CpuInfo `json:"cpus"` +} +type CpuInfo struct { + CpuNumber string `json:"cpu"` + MaxMhz string `json:"maxmhz"` + MinMhz string `json:"minmhz"` +} + +// CalculateFrequency calculates maximum and minimum frequency for the first reserved cpu +func CalculateFrequency(mustGatherDirPath string, rsCPU string) (int, int, error) { + var ( + maxFreq, minFreq int + err error + result Response + ) + nodeList, err := GetNodeList(mustGatherDirPath) + if err != nil { + return maxFreq, minFreq, fmt.Errorf("failed to optain node list, %v\n", err) + } + // check with first node, assuming all nodes have same number of cpu + if len(nodeList) > 0 { + nodeName := nodeList[0].Name + pathSuffix := path.Join(Nodes, nodeName, listCPU+JSONSuffix) + + cpuPath, err := getMustGatherFullPaths(mustGatherDirPath, pathSuffix) + if err != nil { + return maxFreq, minFreq, fmt.Errorf("%v\n", err) + } + + src, err := os.Open(cpuPath) + if err != nil { + return maxFreq, minFreq, fmt.Errorf("failed to open %q: %v", cpuPath, err) + } + defer src.Close() + + dec := k8syaml.NewYAMLOrJSONDecoder(src, 1024) + if err := dec.Decode(&result); err != nil { + return maxFreq, minFreq, fmt.Errorf("failed to decode %q: %v", cpuPath, err) + } + } + + reservedCpu, err := cpuset.Parse(rsCPU) + if err != nil { + return maxFreq, minFreq, fmt.Errorf("failed to parse reserved cpu list, %v\n", err) + } + + // filter frequency from the file + for _, val := range result.Cpu { + // assuming all cpu returns the same max frequency + if val.CpuNumber == strconv.Itoa(reservedCpu.List()[0]) { + //CpuFreq governor is not installed or problem during must-gather execution + if val.MaxMhz == "" || val.MinMhz == "" { + return maxFreq, minFreq, fmt.Errorf("can't obtain cpu frequency for %v, %v\n", val.CpuNumber, err) + } + maxFreq, err = strconv.Atoi(strings.Split(strings.Trim(val.MaxMhz, "\""), ".")[0]) + if err != nil { + return maxFreq, minFreq, fmt.Errorf("failed to compute maximum cpu frequency for cpu number %v, %v\n", val.CpuNumber, err) + } + minFreq, err = strconv.Atoi(strings.Split(strings.Trim(val.MinMhz, "\""), ".")[0]) + if err != nil { + return maxFreq, minFreq, fmt.Errorf("failed to compute minimum cpu frequency for cpu number %v, %v\n", val.CpuNumber, err) + } + } + } + return maxFreq, setMinFrequency(maxFreq, minFreq), nil +} + +// check minimum frequency +func setMinFrequency(max, min int) int { + computedMin := float32(max) * float32(percentage) / float32(100) + if int(computedMin) < min { + return min + } + return int(computedMin) +}