balloons: IRQ CPU affinity controls

cmd/plugins/balloons/policy/balloons-policy.go

@@ -25,6 +25,7 @@ import (
 
 	cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/balloons"
 	"github.com/containers/nri-plugins/pkg/cpuallocator"
+	"github.com/containers/nri-plugins/pkg/irq"
 	"github.com/containers/nri-plugins/pkg/kubernetes"
 	logger "github.com/containers/nri-plugins/pkg/log"
 	"github.com/containers/nri-plugins/pkg/resmgr/cache"
@@ -254,6 +255,7 @@ func (p *balloons) Sync(add []cache.Container, del []cache.Container) error {
 	p.BlockMeters()
 	defer p.UnblockMeters()
 	defer p.commitCpuClasses()
+	defer p.applyIrqAffinities()
 
 	log.Debugf("synchronizing state...")
 	for _, c := range del {
@@ -278,6 +280,7 @@ func (p *balloons) AllocateResources(c cache.Container) error {
 	p.BlockMeters()
 	defer p.UnblockMeters()
 	defer p.commitCpuClasses()
+	defer p.applyIrqAffinities()
 
 	if c.PreserveCpuResources() {
 		log.Infof("not handling resources of container %s, preserving CPUs %q and memory %q", c.PrettyName(), c.GetCpusetCpus(), c.GetCpusetMems())
@@ -332,6 +335,7 @@ func (p *balloons) ReleaseResources(c cache.Container) error {
 	p.BlockMeters()
 	defer p.UnblockMeters()
 	defer p.commitCpuClasses()
+	defer p.applyIrqAffinities()
 
 	log.Debugf("releasing container %s...", c.PrettyName())
 	if bln := p.balloonByContainer(c); bln != nil {
@@ -366,6 +370,7 @@ func (p *balloons) UpdateResources(c cache.Container) error {
 	p.BlockMeters()
 	defer p.UnblockMeters()
 	defer p.commitCpuClasses()
+	defer p.applyIrqAffinities()
 
 	log.Debugf("(not) updating container %s...", c.PrettyName())
 	return nil
@@ -931,6 +936,98 @@ func (p *balloons) forgetCpuClass(bln *Balloon) {
 	}
 }
 
+const (
+	// irqModeSink adds CPUs of a balloon to the affinity of
+	// unclaimed IRQs.
+	irqModeSink = "sink"
+	// irqModeIsolate removes CPUs of a balloon from the affinity
+	// of IRQs that are neither claimed nor sinked.
+	irqModeIsolate = "isolate"
+)
+
+// irqOptionsInUse returns whether any balloon type configures IRQ
+// affinities.
+func (p *balloons) irqOptionsInUse() bool {
+	for _, blnDef := range p.bpoptions.BalloonDefs {
+		if len(blnDef.IrqClaim) > 0 || blnDef.IrqMode != "" {
+			return true
+		}
+	}
+	return false
+}
+
+// irqClaimCpus returns the union of CPUs of balloons that claim the
+// given interrupt.
+func (p *balloons) irqClaimCpus(hwIrq *irq.Irq) cpuset.CPUSet {
+	claimCpus := cpuset.New()
+	for _, bln := range p.balloons {
+		if bln.Cpus.IsEmpty() {
+			continue
+		}
+		for _, claim := range bln.Def.IrqClaim {
+			if hwIrq.Match(claim) {
+				claimCpus = claimCpus.Union(bln.Cpus)
+				break
+			}
+		}
+	}
+	return claimCpus
+}
+
+// applyIrqAffinities sets CPU affinities of system IRQs according to
+// the irqClaim and irqMode options of balloon types.
+func (p *balloons) applyIrqAffinities() {
+	if !p.irqOptionsInUse() {
+		return
+	}
+	hwIrqs, err := irq.Interrupts()
+	if err != nil {
+		log.Warnf("failed to read interrupts for IRQ affinity update: %v", err)
+		return
+	}
+	sinkCpus := cpuset.New()
+	isolateCpus := cpuset.New()
+	for _, bln := range p.balloons {
+		if bln.Cpus.IsEmpty() {
+			continue
+		}
+		switch bln.Def.IrqMode {
+		case irqModeSink:
+			sinkCpus = sinkCpus.Union(bln.Cpus)
+		case irqModeIsolate:
+			isolateCpus = isolateCpus.Union(bln.Cpus)
+		}
+	}
+	for _, hwIrq := range hwIrqs {
+		newCpus := p.allowed
+		curCpus, err := hwIrq.AffinityCpus()
+		if err != nil {
+			continue
+		}
+		switch claimCpus := p.irqClaimCpus(hwIrq); {
+		case !claimCpus.IsEmpty():
+			newCpus = claimCpus
+		case !sinkCpus.IsEmpty():
+			newCpus = sinkCpus
+		case curCpus.Intersection(p.allowed).IsEmpty():
+			// IRQ is out of our scope
+			continue
+		case !isolateCpus.IsEmpty():
+			// Keep IRQs affinity outside allowed CPUs as is.
+			// Allow IRQ on any allowed CPUs except for isolateCpus.
+			newCpus = curCpus.Union(p.allowed).Difference(isolateCpus)
+		}
+		if curCpus.Equals(newCpus) {
+			continue
+		}
+		if err := hwIrq.SetAffinityCpus(newCpus); err != nil {
+			log.Debugf("failed to set affinity of %s to %q: %v", hwIrq, newCpus, err)
+		} else {
+			log.Debugf("set affinity of %s to %q", hwIrq, newCpus)
+		}
+	}
+}
+
 // updateLoadedVirtDevsInAllocatorOptions updates CPU allocator
 // options with virtual devices under load due to the given load
 // classes. Returns a set of virtual device names under load.
@@ -1526,6 +1623,7 @@ func (p *balloons) Reconfigure(newCfg interface{}) error {
 	p.BlockMeters()
 	defer p.UnblockMeters()
 	defer p.commitCpuClasses()
+	defer p.applyIrqAffinities()
 
 	balloonsOptions, ok := newCfg.(*BalloonsOptions)
 	if !ok {

config/crd/bases/config.nri_balloonspolicies.yaml

@@ -158,6 +158,25 @@ spec:
                         will remain completely idle as they cannot be allocated to
                         other balloons.
                       type: boolean
+                    irqClaim:
+                      description: |-
+                        IrqClaim lists IRQs whose CPU affinity is set to the CPUs
+                        of balloons of this type. Each list item refers to IRQs
+                        either by an exact number or by a shell-like wildcard
+                        pattern matching a device or chip name in /proc/interrupts
+                      items:
+                        type: string
+                      type: array
+                    irqMode:
+                      description: |-
+                        IrqMode controls how CPU affinities of unclaimed (see
+                        irqClaim) IRQs relate to the CPUs of balloons of this
+                        type. "sink" adds the CPUs to the affinity of IRQs. If sink
+                        balloons are present, they take unclaimed IRQs from other
+                        balloons. "isolate" removes the CPUs of this balloon type
+                        from the affinity of IRQs that are neither claimed nor
+                        sinked.  +kubebuilder:validation:Enum="";sink;isolate
+                      type: string
                     loads:
                       description: |-
                         Loads is a list of loadClasses that describe load generated

deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml

@@ -158,6 +158,25 @@ spec:
                         will remain completely idle as they cannot be allocated to
                         other balloons.
                       type: boolean
+                    irqClaim:
+                      description: |-
+                        IrqClaim lists IRQs whose CPU affinity is set to the CPUs
+                        of balloons of this type. Each list item refers to IRQs
+                        either by an exact number or by a shell-like wildcard
+                        pattern matching a device or chip name in /proc/interrupts
+                      items:
+                        type: string
+                      type: array
+                    irqMode:
+                      description: |-
+                        IrqMode controls how CPU affinities of unclaimed (see
+                        irqClaim) IRQs relate to the CPUs of balloons of this
+                        type. "sink" adds the CPUs to the affinity of IRQs. If sink
+                        balloons are present, they take unclaimed IRQs from other
+                        balloons. "isolate" removes the CPUs of this balloon type
+                        from the affinity of IRQs that are neither claimed nor
+                        sinked.  +kubebuilder:validation:Enum="";sink;isolate
+                      type: string
                     loads:
                       description: |-
                         Loads is a list of loadClasses that describe load generated

deployment/helm/balloons/templates/daemonset.yaml

@@ -121,6 +121,8 @@ spec:
             mountPath: /var/lib/nri-resource-policy
           - name: hostsysfs
             mountPath: /host/sys
+          - name: hostprocfs
+            mountPath: /host/proc
           - name: resource-policysockets
             mountPath: /var/run/nri-resource-policy
           - name: nrisockets
@@ -145,6 +147,10 @@ spec:
         hostPath:
           path: /sys
           type: Directory
+      - name: hostprocfs
+        hostPath:
+          path: /proc
+          type: Directory
       - name: resource-policysockets
         hostPath:
           path: /var/run/nri-resource-policy

docs/resource-policy/policy/balloons.md

@@ -848,7 +848,8 @@ memory-type.resource-policy.nri.io/container.CONTAINER_NAME: HBM,DRAM
 
 ### CPU Tuning
 
-These options configure CPU behavior and power management.
+These options configure CPU behavior, power management, and tuning
+IRQs CPU affinity.
 
 **`cpuClass`** (string)
 - References a CPU class name defined in `cpuClasses` (preferred) or
@@ -1059,6 +1060,51 @@ control:
         maxFreq: 1200000
 ```
 
+#### IRQ CPU Affinity Tuning
+
+These balloon type options control the CPU affinity of hardware
+interrupts (IRQs). By default the policy does not touch IRQ affinities.
+Affinities are updated by writing
+`/proc/irq/<NUMBER>/smp_affinity_list`, and IRQs are matched against
+`/proc/interrupts`.
+
+**`irqClaim`** (list of strings)
+- Lists IRQs handled by CPUs of balloons of this type.
+- Each item refers to IRQs either by an exact number, or by a pattern
+  matching last columns in `/proc/interrupts`. Supports wildcards, for
+  instance "*nvme*".
+- The affinity of a claimed IRQ is set to the union of CPUs of all
+  balloons that claim it.
+
+**`irqMode`** (string: `"sink"` or `"isolate"`)
+- `sink`: CPUs of balloons of this type handle IRQs that no balloon
+  has claimed. The affinity of such an unclaimed IRQ is set to the
+  union of CPUs of all sink balloons.
+- `isolate`: CPUs of balloons of this type are removed from the
+  affinity of IRQs that are neither claimed nor sinked. This keeps
+  such CPUs free from interrupts.
+
+The options can be combined. Common use cases:
+
+- *Latency-sensitive balloon*: use `irqMode: isolate` to keep the
+  CPUs of the balloon free from unrelated IRQs.
+- *Hardware-specific balloon*: use `irqClaim` to direct IRQs of the
+  relevant devices to the CPUs of the balloon. Add `irqMode: isolate`
+  to additionally keep all other IRQs away from those CPUs.
+- *IRQ sink balloon*: use `irqMode: sink` to gather all otherwise
+  unclaimed IRQs on the CPUs of the balloon.
+
+```yaml
+balloonTypes:
+- name: network
+  irqClaim:
+    - "*eth0 *"
+    - "42"
+  irqMode: isolate
+- name: housekeeping
+  irqMode: sink
+```
+
 ### Built-in Balloon Types
 
 The policy includes two built-in balloon types that can be customized

pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go

@@ -307,6 +307,19 @@ type BalloonDef struct {
 	// may generate a lot of traffic and large CR object updates
 	// to Kubernetes API server.
 	ShowContainersInNrt *bool `json:"showContainersInNrt,omitempty"`
+	// IrqClaim lists IRQs whose CPU affinity is set to the CPUs
+	// of balloons of this type. Each list item refers to IRQs
+	// either by an exact number or by a shell-like wildcard
+	// pattern matching a device or chip name in /proc/interrupts
+	IrqClaim []string `json:"irqClaim,omitempty"`
+	// IrqMode controls how CPU affinities of unclaimed (see
+	// irqClaim) IRQs relate to the CPUs of balloons of this
+	// type. "sink" adds the CPUs to the affinity of IRQs. If sink
+	// balloons are present, they take unclaimed IRQs from other
+	// balloons. "isolate" removes the CPUs of this balloon type
+	// from the affinity of IRQs that are neither claimed nor
+	// sinked.  +kubebuilder:validation:Enum="";sink;isolate
+	IrqMode string `json:"irqMode,omitempty"`
 }
 
 // BalloonDefComponent contains a balloon component definition.