diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 26eafc724..baf50b695 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -25,6 +25,7 @@ import ( cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/balloons" "github.com/containers/nri-plugins/pkg/cpuallocator" + "github.com/containers/nri-plugins/pkg/irq" "github.com/containers/nri-plugins/pkg/kubernetes" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cache" @@ -254,6 +255,7 @@ func (p *balloons) Sync(add []cache.Container, del []cache.Container) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() log.Debugf("synchronizing state...") for _, c := range del { @@ -278,6 +280,7 @@ func (p *balloons) AllocateResources(c cache.Container) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() if c.PreserveCpuResources() { log.Infof("not handling resources of container %s, preserving CPUs %q and memory %q", c.PrettyName(), c.GetCpusetCpus(), c.GetCpusetMems()) @@ -332,6 +335,7 @@ func (p *balloons) ReleaseResources(c cache.Container) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() log.Debugf("releasing container %s...", c.PrettyName()) if bln := p.balloonByContainer(c); bln != nil { @@ -366,6 +370,7 @@ func (p *balloons) UpdateResources(c cache.Container) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() log.Debugf("(not) updating container %s...", c.PrettyName()) return nil @@ -931,6 +936,98 @@ func (p *balloons) forgetCpuClass(bln *Balloon) { } } +const ( + // irqModeSink adds CPUs of a balloon to the affinity of + // unclaimed IRQs. + irqModeSink = "sink" + // irqModeIsolate removes CPUs of a balloon from the affinity + // of IRQs that are neither claimed nor sinked. + irqModeIsolate = "isolate" +) + +// irqOptionsInUse returns whether any balloon type configures IRQ +// affinities. +func (p *balloons) irqOptionsInUse() bool { + for _, blnDef := range p.bpoptions.BalloonDefs { + if len(blnDef.IrqClaim) > 0 || blnDef.IrqMode != "" { + return true + } + } + return false +} + +// irqClaimCpus returns the union of CPUs of balloons that claim the +// given interrupt. +func (p *balloons) irqClaimCpus(hwIrq *irq.Irq) cpuset.CPUSet { + claimCpus := cpuset.New() + for _, bln := range p.balloons { + if bln.Cpus.IsEmpty() { + continue + } + for _, claim := range bln.Def.IrqClaim { + if hwIrq.Match(claim) { + claimCpus = claimCpus.Union(bln.Cpus) + break + } + } + } + return claimCpus +} + +// applyIrqAffinities sets CPU affinities of system IRQs according to +// the irqClaim and irqMode options of balloon types. +func (p *balloons) applyIrqAffinities() { + if !p.irqOptionsInUse() { + return + } + hwIrqs, err := irq.Interrupts() + if err != nil { + log.Warnf("failed to read interrupts for IRQ affinity update: %v", err) + return + } + sinkCpus := cpuset.New() + isolateCpus := cpuset.New() + for _, bln := range p.balloons { + if bln.Cpus.IsEmpty() { + continue + } + switch bln.Def.IrqMode { + case irqModeSink: + sinkCpus = sinkCpus.Union(bln.Cpus) + case irqModeIsolate: + isolateCpus = isolateCpus.Union(bln.Cpus) + } + } + for _, hwIrq := range hwIrqs { + newCpus := p.allowed + curCpus, err := hwIrq.AffinityCpus() + if err != nil { + continue + } + switch claimCpus := p.irqClaimCpus(hwIrq); { + case !claimCpus.IsEmpty(): + newCpus = claimCpus + case !sinkCpus.IsEmpty(): + newCpus = sinkCpus + case curCpus.Intersection(p.allowed).IsEmpty(): + // IRQ is out of our scope + continue + case !isolateCpus.IsEmpty(): + // Keep IRQs affinity outside allowed CPUs as is. + // Allow IRQ on any allowed CPUs except for isolateCpus. + newCpus = curCpus.Union(p.allowed).Difference(isolateCpus) + } + if curCpus.Equals(newCpus) { + continue + } + if err := hwIrq.SetAffinityCpus(newCpus); err != nil { + log.Debugf("failed to set affinity of %s to %q: %v", hwIrq, newCpus, err) + } else { + log.Debugf("set affinity of %s to %q", hwIrq, newCpus) + } + } +} + // updateLoadedVirtDevsInAllocatorOptions updates CPU allocator // options with virtual devices under load due to the given load // classes. Returns a set of virtual device names under load. @@ -1526,6 +1623,7 @@ func (p *balloons) Reconfigure(newCfg interface{}) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() balloonsOptions, ok := newCfg.(*BalloonsOptions) if !ok { diff --git a/config/crd/bases/config.nri_balloonspolicies.yaml b/config/crd/bases/config.nri_balloonspolicies.yaml index 545c73cd2..cc1e4438d 100644 --- a/config/crd/bases/config.nri_balloonspolicies.yaml +++ b/config/crd/bases/config.nri_balloonspolicies.yaml @@ -158,6 +158,25 @@ spec: will remain completely idle as they cannot be allocated to other balloons. type: boolean + irqClaim: + description: |- + IrqClaim lists IRQs whose CPU affinity is set to the CPUs + of balloons of this type. Each list item refers to IRQs + either by an exact number or by a shell-like wildcard + pattern matching a device or chip name in /proc/interrupts + items: + type: string + type: array + irqMode: + description: |- + IrqMode controls how CPU affinities of unclaimed (see + irqClaim) IRQs relate to the CPUs of balloons of this + type. "sink" adds the CPUs to the affinity of IRQs. If sink + balloons are present, they take unclaimed IRQs from other + balloons. "isolate" removes the CPUs of this balloon type + from the affinity of IRQs that are neither claimed nor + sinked. +kubebuilder:validation:Enum="";sink;isolate + type: string loads: description: |- Loads is a list of loadClasses that describe load generated diff --git a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml index 545c73cd2..cc1e4438d 100644 --- a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml +++ b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml @@ -158,6 +158,25 @@ spec: will remain completely idle as they cannot be allocated to other balloons. type: boolean + irqClaim: + description: |- + IrqClaim lists IRQs whose CPU affinity is set to the CPUs + of balloons of this type. Each list item refers to IRQs + either by an exact number or by a shell-like wildcard + pattern matching a device or chip name in /proc/interrupts + items: + type: string + type: array + irqMode: + description: |- + IrqMode controls how CPU affinities of unclaimed (see + irqClaim) IRQs relate to the CPUs of balloons of this + type. "sink" adds the CPUs to the affinity of IRQs. If sink + balloons are present, they take unclaimed IRQs from other + balloons. "isolate" removes the CPUs of this balloon type + from the affinity of IRQs that are neither claimed nor + sinked. +kubebuilder:validation:Enum="";sink;isolate + type: string loads: description: |- Loads is a list of loadClasses that describe load generated diff --git a/deployment/helm/balloons/templates/daemonset.yaml b/deployment/helm/balloons/templates/daemonset.yaml index 966780da1..75aa04cf3 100644 --- a/deployment/helm/balloons/templates/daemonset.yaml +++ b/deployment/helm/balloons/templates/daemonset.yaml @@ -121,6 +121,8 @@ spec: mountPath: /var/lib/nri-resource-policy - name: hostsysfs mountPath: /host/sys + - name: hostprocfs + mountPath: /host/proc - name: resource-policysockets mountPath: /var/run/nri-resource-policy - name: nrisockets @@ -145,6 +147,10 @@ spec: hostPath: path: /sys type: Directory + - name: hostprocfs + hostPath: + path: /proc + type: Directory - name: resource-policysockets hostPath: path: /var/run/nri-resource-policy diff --git a/docs/resource-policy/policy/balloons.md b/docs/resource-policy/policy/balloons.md index 1da518331..aa5fabbc7 100644 --- a/docs/resource-policy/policy/balloons.md +++ b/docs/resource-policy/policy/balloons.md @@ -848,7 +848,8 @@ memory-type.resource-policy.nri.io/container.CONTAINER_NAME: HBM,DRAM ### CPU Tuning -These options configure CPU behavior and power management. +These options configure CPU behavior, power management, and tuning +IRQs CPU affinity. **`cpuClass`** (string) - References a CPU class name defined in `cpuClasses` (preferred) or @@ -1059,6 +1060,51 @@ control: maxFreq: 1200000 ``` +#### IRQ CPU Affinity Tuning + +These balloon type options control the CPU affinity of hardware +interrupts (IRQs). By default the policy does not touch IRQ affinities. +Affinities are updated by writing +`/proc/irq//smp_affinity_list`, and IRQs are matched against +`/proc/interrupts`. + +**`irqClaim`** (list of strings) +- Lists IRQs handled by CPUs of balloons of this type. +- Each item refers to IRQs either by an exact number, or by a pattern + matching last columns in `/proc/interrupts`. Supports wildcards, for + instance "*nvme*". +- The affinity of a claimed IRQ is set to the union of CPUs of all + balloons that claim it. + +**`irqMode`** (string: `"sink"` or `"isolate"`) +- `sink`: CPUs of balloons of this type handle IRQs that no balloon + has claimed. The affinity of such an unclaimed IRQ is set to the + union of CPUs of all sink balloons. +- `isolate`: CPUs of balloons of this type are removed from the + affinity of IRQs that are neither claimed nor sinked. This keeps + such CPUs free from interrupts. + +The options can be combined. Common use cases: + +- *Latency-sensitive balloon*: use `irqMode: isolate` to keep the + CPUs of the balloon free from unrelated IRQs. +- *Hardware-specific balloon*: use `irqClaim` to direct IRQs of the + relevant devices to the CPUs of the balloon. Add `irqMode: isolate` + to additionally keep all other IRQs away from those CPUs. +- *IRQ sink balloon*: use `irqMode: sink` to gather all otherwise + unclaimed IRQs on the CPUs of the balloon. + +```yaml +balloonTypes: +- name: network + irqClaim: + - "*eth0 *" + - "42" + irqMode: isolate +- name: housekeeping + irqMode: sink +``` + ### Built-in Balloon Types The policy includes two built-in balloon types that can be customized diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go index a8d6230d6..49a2be909 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go @@ -307,6 +307,19 @@ type BalloonDef struct { // may generate a lot of traffic and large CR object updates // to Kubernetes API server. ShowContainersInNrt *bool `json:"showContainersInNrt,omitempty"` + // IrqClaim lists IRQs whose CPU affinity is set to the CPUs + // of balloons of this type. Each list item refers to IRQs + // either by an exact number or by a shell-like wildcard + // pattern matching a device or chip name in /proc/interrupts + IrqClaim []string `json:"irqClaim,omitempty"` + // IrqMode controls how CPU affinities of unclaimed (see + // irqClaim) IRQs relate to the CPUs of balloons of this + // type. "sink" adds the CPUs to the affinity of IRQs. If sink + // balloons are present, they take unclaimed IRQs from other + // balloons. "isolate" removes the CPUs of this balloon type + // from the affinity of IRQs that are neither claimed nor + // sinked. +kubebuilder:validation:Enum="";sink;isolate + IrqMode string `json:"irqMode,omitempty"` } // BalloonDefComponent contains a balloon component definition. diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go index e4b9ceae1..115f0d692 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go @@ -87,6 +87,11 @@ func (in *BalloonDef) DeepCopyInto(out *BalloonDef) { *out = new(bool) **out = **in } + if in.IrqClaim != nil { + in, out := &in.IrqClaim, &out.IrqClaim + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BalloonDef. diff --git a/pkg/irq/irq.go b/pkg/irq/irq.go new file mode 100644 index 000000000..357d05119 --- /dev/null +++ b/pkg/irq/irq.go @@ -0,0 +1,170 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// irq package provides low-level functions to read interrupts from +// /proc/interrupts and to read and write CPU affinities of interrupts +// through /proc/irq/NUMBER/smp_affinity_list. +package irq + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strconv" + "strings" + + logger "github.com/containers/nri-plugins/pkg/log" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +var ( + procRoot = "/" + proc = "/proc" + + log = logger.NewLogger("irq") +) + +// SetProcRoot sets the procfs root directory and proc mountpoint. +func SetProcRoot(root string) { + if root != "" { + procRoot = filepath.Clean(root) + if procRoot != "" && !filepath.IsAbs(procRoot) { + a, err := filepath.Abs(procRoot) + if err != nil { + panic(fmt.Errorf("failed to resolve procfs root %q to absolute path: %v", procRoot, err)) + } + procRoot = a + } + if procRoot == "/" { + procRoot = "" + } + } else { + procRoot = "" + } + proc = filepath.Join(procRoot, "/proc") +} + +// Irq represents a single numbered hardware interrupt. +type Irq struct { + num int + description string +} + +// Interrupts returns all numbered interrupts listed in +// /proc/interrupts. +func Interrupts() ([]*Irq, error) { + data, err := os.ReadFile(filepath.Join(proc, "interrupts")) + if err != nil { + return nil, fmt.Errorf("failed to read interrupts: %w", err) + } + irqs := []*Irq{} + for line := range strings.SplitSeq(string(data), "\n") { + if irq, ok := parseInterruptsLine(line); ok { + irqs = append(irqs, irq) + } + } + return irqs, nil +} + +// parseInterruptsLine returns the interrupt parsed from a +// /proc/interrupts line and whether the line described a numbered +// interrupt. +func parseInterruptsLine(line string) (*Irq, bool) { + fields := strings.Fields(line) + if len(fields) < 2 { + return nil, false + } + head, rest := fields[0], fields[1:] + if !strings.HasSuffix(head, ":") { + return nil, false + } + num, err := strconv.Atoi(strings.TrimSuffix(head, ":")) + if err != nil { + return nil, false + } + // Skip the leading per-CPU interrupt counters and keep the + // remaining chip and device description for matching. + descStart := len(rest) + for i, field := range rest { + if _, err := strconv.Atoi(field); err != nil { + descStart = i + break + } + } + return &Irq{ + num: num, + description: strings.Join(rest[descStart:], " "), + }, true +} + +// Num returns the number of the interrupt. +func (irq *Irq) Num() int { + return irq.num +} + +// Description returns the chip and device description of the interrupt. +func (irq *Irq) Description() string { + return irq.description +} + +// String returns the interrupt as a human readable string. +func (irq *Irq) String() string { + return fmt.Sprintf("irq %d (%s)", irq.num, irq.description) +} + +// Match returns whether the interrupt matches the pattern string, +// that is either the exact interrupt number or a shell-like wildcard +// pattern match of its description. +func (irq *Irq) Match(pattern string) bool { + if pattern == "" { + return false + } + if pattern == strconv.Itoa(irq.num) { + return true + } + match, err := path.Match(pattern, irq.description) + return err == nil && match +} + +// AffinityCpus returns the CPUs in the affinity of the interrupt. +func (irq *Irq) AffinityCpus() (cpuset.CPUSet, error) { + data, err := os.ReadFile(irq.smpAffinityListPath()) + if err != nil { + return cpuset.New(), fmt.Errorf("failed to read affinity of irq %d: %w", irq.num, err) + } + cpus, err := cpuset.Parse(strings.TrimSpace(string(data))) + if err != nil { + return cpuset.New(), fmt.Errorf("failed to parse affinity of irq %d: %w", irq.num, err) + } + return cpus, nil +} + +// SetAffinityCpus sets the CPUs in the affinity of the interrupt. +func (irq *Irq) SetAffinityCpus(cpus cpuset.CPUSet) error { + if cpus.IsEmpty() { + return fmt.Errorf("refusing to set empty affinity on irq %d", irq.num) + } + if err := os.WriteFile(irq.smpAffinityListPath(), []byte(cpus.String()), 0644); err != nil { + return fmt.Errorf("failed to set affinity of irq %d to %q: %w", irq.num, cpus, err) + } + log.Debugf("irq %s smp_affinity_list written: %s", irq, cpus) + return nil +} + +// smpAffinityListPath returns the path to the smp_affinity_list file +// of the interrupt. +func (irq *Irq) smpAffinityListPath() string { + return filepath.Join(proc, "irq", strconv.Itoa(irq.num), "smp_affinity_list") +} diff --git a/pkg/irq/irq_test.go b/pkg/irq/irq_test.go new file mode 100644 index 000000000..aef9e92eb --- /dev/null +++ b/pkg/irq/irq_test.go @@ -0,0 +1,117 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package irq + +import ( + "os" + "path/filepath" + "testing" + + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +const sampleInterrupts = ` CPU0 CPU1 CPU2 CPU3 + 1: 0 0 601604 0 IR-IO-APIC 1-edge i8042 + 9: 236650 0 0 0 IR-IO-APIC 9-fasteoi acpi + 16: 0 0 32 0 IR-IO-APIC 16-fasteoi i801_smbus, processor_thermal_device_pci +NMI: 0 0 0 0 Non-maskable interrupts +LOC: 123456 123456 123456 123456 Local timer interrupts +` + +func TestInterruptsAndMatch(t *testing.T) { + dir := t.TempDir() + SetProcRoot(dir) + if err := os.Mkdir(filepath.Join(dir, "proc"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "proc", "interrupts"), []byte(sampleInterrupts), 0644); err != nil { + t.Fatal(err) + } + + irqs, err := Interrupts() + if err != nil { + t.Fatalf("Interrupts() failed: %v", err) + } + if len(irqs) != 3 { + t.Fatalf("expected 3 numbered interrupts, got %d: %v", len(irqs), irqs) + } + + byNum := map[int]*Irq{} + for _, irq := range irqs { + byNum[irq.Num()] = irq + } + + if byNum[1] == nil || byNum[9] == nil || byNum[16] == nil { + t.Fatalf("missing expected interrupts, got %v", irqs) + } + if got := byNum[1].Description(); got != "IR-IO-APIC 1-edge i8042" { + t.Errorf("irq 1 descriptor = %q", got) + } + + cases := []struct { + num int + claim string + want bool + }{ + {1, "1", true}, + {1, "*i8042*", true}, + {1, "*acpi*", false}, + {9, "*acpi*", true}, + {16, "*processor_thermal_device_pci*", true}, + {16, "9", false}, + {1, "", false}, + } + for _, c := range cases { + if got := byNum[c.num].Match(c.claim); got != c.want { + t.Errorf("irq %d Match(%q) = %v, want %v", c.num, c.claim, got, c.want) + } + } +} + +func TestAffinityReadWrite(t *testing.T) { + dir := t.TempDir() + SetProcRoot(dir) + irqDir := filepath.Join(dir, "proc", "irq", "42") + if err := os.MkdirAll(irqDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(irqDir, "smp_affinity_list"), []byte("0-3\n"), 0644); err != nil { + t.Fatal(err) + } + + irq := &Irq{num: 42} + cpus, err := irq.AffinityCpus() + if err != nil { + t.Fatalf("AffinityCpus() failed: %v", err) + } + if !cpus.Equals(cpuset.MustParse("0-3")) { + t.Errorf("AffinityCpus() = %q, want 0-3", cpus) + } + + if err := irq.SetAffinityCpus(cpuset.MustParse("1,3")); err != nil { + t.Fatalf("SetAffinityCpus() failed: %v", err) + } + data, err := os.ReadFile(filepath.Join(irqDir, "smp_affinity_list")) + if err != nil { + t.Fatal(err) + } + if string(data) != "1,3" { + t.Errorf("written affinity = %q, want 1,3", string(data)) + } + + if err := irq.SetAffinityCpus(cpuset.New()); err == nil { + t.Errorf("SetAffinityCpus(empty) should fail") + } +} diff --git a/pkg/resmgr/resource-manager.go b/pkg/resmgr/resource-manager.go index 2012eceee..232c39f4d 100644 --- a/pkg/resmgr/resource-manager.go +++ b/pkg/resmgr/resource-manager.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/yaml" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1" + "github.com/containers/nri-plugins/pkg/irq" ) // ResourceManager is the interface we expose for controlling the CRI resource manager. @@ -78,6 +79,7 @@ func NewResourceManager(backend policy.Backend, agt *agent.Agent) (ResourceManag if opt.HostRoot != "" { sysfs.SetSysRoot(opt.HostRoot) topology.SetSysRoot(opt.HostRoot) + irq.SetProcRoot(opt.HostRoot) } m := &resmgr{ diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-claim.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-claim.cfg new file mode 100644 index 000000000..770e8274f --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-claim.cfg @@ -0,0 +1,18 @@ +config: + reservedResources: + cpu: 1000m + pinCPU: true + pinMemory: true + balloonTypes: + - name: claimer + minCPUs: 2 + maxCPUs: 2 + irqClaim: + - "*ttyS0*" + - "*rtc0*" + log: + debug: + - policy + - irq + klog: + skip_headers: true diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-dedicated-claim.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-dedicated-claim.cfg new file mode 100644 index 000000000..ea6e93b43 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-dedicated-claim.cfg @@ -0,0 +1,18 @@ +config: + reservedResources: + cpu: 1000m + pinCPU: true + pinMemory: true + balloonTypes: + - name: dedicated-claimer + minCPUs: 2 + maxCPUs: 2 + irqClaim: + - "ttyS" # should not match + - "*rtc0*" + irqMode: isolate + log: + debug: + - policy + klog: + skip_headers: true diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-isolate.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-isolate.cfg new file mode 100644 index 000000000..968ed2ab0 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-isolate.cfg @@ -0,0 +1,15 @@ +config: + reservedResources: + cpu: 1000m + pinCPU: true + pinMemory: true + balloonTypes: + - name: isolate + minCPUs: 2 + maxCPUs: 2 + irqMode: isolate + log: + debug: + - policy + klog: + skip_headers: true diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-sink.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-sink.cfg new file mode 100644 index 000000000..ae99ed088 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-sink.cfg @@ -0,0 +1,21 @@ +config: + reservedResources: + cpu: 1000m + pinCPU: true + pinMemory: true + balloonTypes: + - name: claimer + minCPUs: 2 + maxCPUs: 2 + irqClaim: + - "*ttyS0*" + - "*rtc0*" + - name: sink + minCPUs: 2 + maxCPUs: 2 + irqMode: sink + log: + debug: + - policy + klog: + skip_headers: true diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh new file mode 100644 index 000000000..b1ff0c923 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh @@ -0,0 +1,166 @@ +# Test balloons IRQ CPU affinity: irqClaim and irqMode (sink, isolate). + +# expand-cpulist "0-2,5" prints "0 1 2 5" +expand-cpulist() { + python3 -c ' +import sys +r = set() +for part in sys.argv[1].split(","): + if not part: + continue + if "-" in part: + a, b = part.split("-") + r.update(range(int(a), int(b) + 1)) + else: + r.add(int(part)) +print(" ".join(str(x) for x in sorted(r))) +' "$1" +} + +# ids-difference "1 2 3 4" "2 3" prints "1 4" +ids-difference() { + python3 -c 'import sys +allc = set(int(x) for x in sys.argv[1].split()) +iso = set(int(x) for x in sys.argv[2].split()) +print(" ".join(str(x) for x in sorted(allc - iso))) +' "$1" "$2" +} + +# ctr-cpu-ids podXcY prints sorted CPU ids allowed for the container, +# e.g. "0 1". Requires a preceding "verify" to refresh the state. +ctr-cpu-ids() { + pyexec "print(' '.join(str(i) for i in sorted(cpu_ids(cpus['$1']))))" +} + +# irq-cpu-ids IRQNUM prints sorted CPU ids in the affinity of the IRQ. +irq-cpu-ids() { + expand-cpulist "$(vm-command-q "cat /proc/irq/$1/smp_affinity_list" | tr -d '[:space:]')" +} + +# set-irq-cpus IRQNUM CPULIST sets the affinity of the IRQ, e.g. "0-15". +set-irq-cpus() { + vm-command "echo $2 > /proc/irq/$1/smp_affinity_list" || + command-error "failed to set affinity of irq $1" +} + +# verify-irq-cpus IRQNUM EXPECTED waits until the affinity of the IRQ +# equals EXPECTED (sorted space-separated CPU ids), or fails after a +# timeout. +verify-irq-cpus() { + local irqnum=$1 expected=$2 got tries=20 + while [ "$tries" -gt 0 ]; do + got=$(irq-cpu-ids "$irqnum") + if [ "$got" == "$expected" ]; then + echo "irq $irqnum affinity is '$got' as expected" + return 0 + fi + tries=$((tries - 1)) + sleep 1 + done + error "irq $irqnum affinity: expected CPUs '$expected', got '$got'" +} + +# Detect ttyS0 and rtc0 is IRQ numbers on vm. +vm-command "awk '/ ttyS0/{print \$1}' < /proc/interrupts | sed 's/://g' | head -n 1" +TTYS0_IRQ=$COMMAND_OUTPUT +vm-command "awk '/ rtc0/{print \$1}' < /proc/interrupts | sed 's/://g' | head -n 1" +RTC0_IRQ=$COMMAND_OUTPUT +vm-command "awk '/ acpi/{print \$1}' < /proc/interrupts | sed 's/://g' | head -n 1" +ACPI_IRQ=$COMMAND_OUTPUT + +ALL_CPUS="$(expand-cpulist 0-15)" + +cleanup() { + vm-command "kubectl delete pods --all --now" + vm-command "for f in /proc/irq/*/smp_affinity_list ; do echo 0-15 | tee $f >&/dev/null; done" +} + +cleanup + +# Test irqClaim. CPUs of the claimer balloon handle the claimed +# IRQs (ttyS0 and rtc0). +helm-terminate +helm_config=${TEST_DIR}/balloons-irq-claim.cfg helm-launch balloons + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: claimer" CONTCOUNT=1 create balloons-busybox +report allowed + +claimer_cpus=$(ctr-cpu-ids pod0c0) +echo "claimer CPUs: $claimer_cpus" +verify-irq-cpus "$TTYS0_IRQ" "$claimer_cpus" +verify-irq-cpus "$RTC0_IRQ" "$claimer_cpus" +verify-irq-cpus "$ACPI_IRQ" "$ALL_CPUS" + +# Test irqMode isolate. CPUs of the isolate balloon are removed from +# the affinity of unclaimed IRQs. Preset a full affinity so that the +# removal is observable. +helm-terminate +helm_config=${TEST_DIR}/balloons-irq-isolate.cfg helm-launch balloons +cleanup + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: isolate" CONTCOUNT=1 create balloons-busybox +report allowed + +isolate_cpus=$(ctr-cpu-ids pod1c0) +echo "isolate CPUs: $isolate_cpus" + +# expected_isolate=$(python3 -c 'import sys +# allc = set(int(x) for x in sys.argv[1].split()) +# iso = set(int(x) for x in sys.argv[2].split()) +# print(" ".join(str(x) for x in sorted(allc - iso))) +# ' "$ALL_CPUS" "$isolate_cpus") +expected_isolate=$(ids-difference "$ALL_CPUS" "$isolate_cpus") +verify-irq-cpus "$RTC0_IRQ" "$expected_isolate" + + +# Test irqMode sink. CPUs of the sink balloon handle unclaimed IRQs. +helm-terminate +helm_config=${TEST_DIR}/balloons-irq-sink.cfg helm-launch balloons +cleanup + +# no sink, no claimer present +verify-irq-cpus "$TTYS0_IRQ" "$ALL_CPUS" + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: sink" CONTCOUNT=1 create balloons-busybox +report allowed +verify +sink_cpus=$(ctr-cpu-ids pod2c0) +echo "sink CPUs: $sink_cpus" +verify-irq-cpus "$TTYS0_IRQ" "$sink_cpus" +verify-irq-cpus "$RTC0_IRQ" "$sink_cpus" +verify-irq-cpus "$ACPI_IRQ" "$sink_cpus" + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: claimer" CONTCOUNT=1 create balloons-busybox +report allowed +sink_cpus=$(ctr-cpu-ids pod2c0) +claimer_cpus=$(ctr-cpu-ids pod3c0) +echo "claimer CPUs: $claimer_cpus" +echo "sink CPUs: $sink_cpus" +verify-irq-cpus "$TTYS0_IRQ" "$claimer_cpus" +verify-irq-cpus "$RTC0_IRQ" "$claimer_cpus" +verify-irq-cpus "$ACPI_IRQ" "$sink_cpus" + +vm-command 'kubectl delete pod pod2 --now' +# no sink present anymore +verify-irq-cpus "$TTYS0_IRQ" "$claimer_cpus" +verify-irq-cpus "$RTC0_IRQ" "$claimer_cpus" +verify-irq-cpus "$ACPI_IRQ" "$ALL_CPUS" + +# Test irqMode sink. CPUs of the sink balloon handle unclaimed IRQs. +helm-terminate +helm_config=${TEST_DIR}/balloons-irq-dedicated-claim.cfg helm-launch balloons +cleanup + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: dedicated-claimer" CONTCOUNT=1 create balloons-busybox +report allowed + +d_claimer_cpus=$(ctr-cpu-ids pod4c0) +echo "dedicated claimer CPUs: $d_claimer_cpus" +verify-irq-cpus "$RTC0_IRQ" "$d_claimer_cpus" + +expected_isolate=$(ids-difference "$ALL_CPUS" "$d_claimer_cpus") +verify-irq-cpus "$TTYS0_IRQ" "$expected_isolate" +verify-irq-cpus "$ACPI_IRQ" "$expected_isolate" + +cleanup +helm-terminate