From 59c1713e698eacb6b63aab2e99526abac7fd20da Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Thu, 23 Jul 2026 12:34:56 +0300 Subject: [PATCH 1/5] pkg/irq: add IRQ affinity helpers and host proc enablers Add pkg/irq for reading interrupts from /proc/interrupts and reading and writing CPU affinities through /proc/irq/NUMBER/smp_affinity_list. Wire the proc root to the plugin host root and mount the host proc filesystem into the balloons daemonset so that affinities of host IRQs can be updated (write access needed). Signed-off-by: Antti Kervinen --- .../helm/balloons/templates/daemonset.yaml | 6 + pkg/irq/irq.go | 170 ++++++++++++++++++ pkg/irq/irq_test.go | 117 ++++++++++++ pkg/resmgr/resource-manager.go | 2 + 4 files changed, 295 insertions(+) create mode 100644 pkg/irq/irq.go create mode 100644 pkg/irq/irq_test.go diff --git a/deployment/helm/balloons/templates/daemonset.yaml b/deployment/helm/balloons/templates/daemonset.yaml index 966780da1..75aa04cf3 100644 --- a/deployment/helm/balloons/templates/daemonset.yaml +++ b/deployment/helm/balloons/templates/daemonset.yaml @@ -121,6 +121,8 @@ spec: mountPath: /var/lib/nri-resource-policy - name: hostsysfs mountPath: /host/sys + - name: hostprocfs + mountPath: /host/proc - name: resource-policysockets mountPath: /var/run/nri-resource-policy - name: nrisockets @@ -145,6 +147,10 @@ spec: hostPath: path: /sys type: Directory + - name: hostprocfs + hostPath: + path: /proc + type: Directory - name: resource-policysockets hostPath: path: /var/run/nri-resource-policy diff --git a/pkg/irq/irq.go b/pkg/irq/irq.go new file mode 100644 index 000000000..357d05119 --- /dev/null +++ b/pkg/irq/irq.go @@ -0,0 +1,170 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// irq package provides low-level functions to read interrupts from +// /proc/interrupts and to read and write CPU affinities of interrupts +// through /proc/irq/NUMBER/smp_affinity_list. +package irq + +import ( + "fmt" + "os" + "path" + "path/filepath" + "strconv" + "strings" + + logger "github.com/containers/nri-plugins/pkg/log" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +var ( + procRoot = "/" + proc = "/proc" + + log = logger.NewLogger("irq") +) + +// SetProcRoot sets the procfs root directory and proc mountpoint. +func SetProcRoot(root string) { + if root != "" { + procRoot = filepath.Clean(root) + if procRoot != "" && !filepath.IsAbs(procRoot) { + a, err := filepath.Abs(procRoot) + if err != nil { + panic(fmt.Errorf("failed to resolve procfs root %q to absolute path: %v", procRoot, err)) + } + procRoot = a + } + if procRoot == "/" { + procRoot = "" + } + } else { + procRoot = "" + } + proc = filepath.Join(procRoot, "/proc") +} + +// Irq represents a single numbered hardware interrupt. +type Irq struct { + num int + description string +} + +// Interrupts returns all numbered interrupts listed in +// /proc/interrupts. +func Interrupts() ([]*Irq, error) { + data, err := os.ReadFile(filepath.Join(proc, "interrupts")) + if err != nil { + return nil, fmt.Errorf("failed to read interrupts: %w", err) + } + irqs := []*Irq{} + for line := range strings.SplitSeq(string(data), "\n") { + if irq, ok := parseInterruptsLine(line); ok { + irqs = append(irqs, irq) + } + } + return irqs, nil +} + +// parseInterruptsLine returns the interrupt parsed from a +// /proc/interrupts line and whether the line described a numbered +// interrupt. +func parseInterruptsLine(line string) (*Irq, bool) { + fields := strings.Fields(line) + if len(fields) < 2 { + return nil, false + } + head, rest := fields[0], fields[1:] + if !strings.HasSuffix(head, ":") { + return nil, false + } + num, err := strconv.Atoi(strings.TrimSuffix(head, ":")) + if err != nil { + return nil, false + } + // Skip the leading per-CPU interrupt counters and keep the + // remaining chip and device description for matching. + descStart := len(rest) + for i, field := range rest { + if _, err := strconv.Atoi(field); err != nil { + descStart = i + break + } + } + return &Irq{ + num: num, + description: strings.Join(rest[descStart:], " "), + }, true +} + +// Num returns the number of the interrupt. +func (irq *Irq) Num() int { + return irq.num +} + +// Description returns the chip and device description of the interrupt. +func (irq *Irq) Description() string { + return irq.description +} + +// String returns the interrupt as a human readable string. +func (irq *Irq) String() string { + return fmt.Sprintf("irq %d (%s)", irq.num, irq.description) +} + +// Match returns whether the interrupt matches the pattern string, +// that is either the exact interrupt number or a shell-like wildcard +// pattern match of its description. +func (irq *Irq) Match(pattern string) bool { + if pattern == "" { + return false + } + if pattern == strconv.Itoa(irq.num) { + return true + } + match, err := path.Match(pattern, irq.description) + return err == nil && match +} + +// AffinityCpus returns the CPUs in the affinity of the interrupt. +func (irq *Irq) AffinityCpus() (cpuset.CPUSet, error) { + data, err := os.ReadFile(irq.smpAffinityListPath()) + if err != nil { + return cpuset.New(), fmt.Errorf("failed to read affinity of irq %d: %w", irq.num, err) + } + cpus, err := cpuset.Parse(strings.TrimSpace(string(data))) + if err != nil { + return cpuset.New(), fmt.Errorf("failed to parse affinity of irq %d: %w", irq.num, err) + } + return cpus, nil +} + +// SetAffinityCpus sets the CPUs in the affinity of the interrupt. +func (irq *Irq) SetAffinityCpus(cpus cpuset.CPUSet) error { + if cpus.IsEmpty() { + return fmt.Errorf("refusing to set empty affinity on irq %d", irq.num) + } + if err := os.WriteFile(irq.smpAffinityListPath(), []byte(cpus.String()), 0644); err != nil { + return fmt.Errorf("failed to set affinity of irq %d to %q: %w", irq.num, cpus, err) + } + log.Debugf("irq %s smp_affinity_list written: %s", irq, cpus) + return nil +} + +// smpAffinityListPath returns the path to the smp_affinity_list file +// of the interrupt. +func (irq *Irq) smpAffinityListPath() string { + return filepath.Join(proc, "irq", strconv.Itoa(irq.num), "smp_affinity_list") +} diff --git a/pkg/irq/irq_test.go b/pkg/irq/irq_test.go new file mode 100644 index 000000000..aef9e92eb --- /dev/null +++ b/pkg/irq/irq_test.go @@ -0,0 +1,117 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package irq + +import ( + "os" + "path/filepath" + "testing" + + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +const sampleInterrupts = ` CPU0 CPU1 CPU2 CPU3 + 1: 0 0 601604 0 IR-IO-APIC 1-edge i8042 + 9: 236650 0 0 0 IR-IO-APIC 9-fasteoi acpi + 16: 0 0 32 0 IR-IO-APIC 16-fasteoi i801_smbus, processor_thermal_device_pci +NMI: 0 0 0 0 Non-maskable interrupts +LOC: 123456 123456 123456 123456 Local timer interrupts +` + +func TestInterruptsAndMatch(t *testing.T) { + dir := t.TempDir() + SetProcRoot(dir) + if err := os.Mkdir(filepath.Join(dir, "proc"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "proc", "interrupts"), []byte(sampleInterrupts), 0644); err != nil { + t.Fatal(err) + } + + irqs, err := Interrupts() + if err != nil { + t.Fatalf("Interrupts() failed: %v", err) + } + if len(irqs) != 3 { + t.Fatalf("expected 3 numbered interrupts, got %d: %v", len(irqs), irqs) + } + + byNum := map[int]*Irq{} + for _, irq := range irqs { + byNum[irq.Num()] = irq + } + + if byNum[1] == nil || byNum[9] == nil || byNum[16] == nil { + t.Fatalf("missing expected interrupts, got %v", irqs) + } + if got := byNum[1].Description(); got != "IR-IO-APIC 1-edge i8042" { + t.Errorf("irq 1 descriptor = %q", got) + } + + cases := []struct { + num int + claim string + want bool + }{ + {1, "1", true}, + {1, "*i8042*", true}, + {1, "*acpi*", false}, + {9, "*acpi*", true}, + {16, "*processor_thermal_device_pci*", true}, + {16, "9", false}, + {1, "", false}, + } + for _, c := range cases { + if got := byNum[c.num].Match(c.claim); got != c.want { + t.Errorf("irq %d Match(%q) = %v, want %v", c.num, c.claim, got, c.want) + } + } +} + +func TestAffinityReadWrite(t *testing.T) { + dir := t.TempDir() + SetProcRoot(dir) + irqDir := filepath.Join(dir, "proc", "irq", "42") + if err := os.MkdirAll(irqDir, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(irqDir, "smp_affinity_list"), []byte("0-3\n"), 0644); err != nil { + t.Fatal(err) + } + + irq := &Irq{num: 42} + cpus, err := irq.AffinityCpus() + if err != nil { + t.Fatalf("AffinityCpus() failed: %v", err) + } + if !cpus.Equals(cpuset.MustParse("0-3")) { + t.Errorf("AffinityCpus() = %q, want 0-3", cpus) + } + + if err := irq.SetAffinityCpus(cpuset.MustParse("1,3")); err != nil { + t.Fatalf("SetAffinityCpus() failed: %v", err) + } + data, err := os.ReadFile(filepath.Join(irqDir, "smp_affinity_list")) + if err != nil { + t.Fatal(err) + } + if string(data) != "1,3" { + t.Errorf("written affinity = %q, want 1,3", string(data)) + } + + if err := irq.SetAffinityCpus(cpuset.New()); err == nil { + t.Errorf("SetAffinityCpus(empty) should fail") + } +} diff --git a/pkg/resmgr/resource-manager.go b/pkg/resmgr/resource-manager.go index 2012eceee..232c39f4d 100644 --- a/pkg/resmgr/resource-manager.go +++ b/pkg/resmgr/resource-manager.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/yaml" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1" + "github.com/containers/nri-plugins/pkg/irq" ) // ResourceManager is the interface we expose for controlling the CRI resource manager. @@ -78,6 +79,7 @@ func NewResourceManager(backend policy.Backend, agt *agent.Agent) (ResourceManag if opt.HostRoot != "" { sysfs.SetSysRoot(opt.HostRoot) topology.SetSysRoot(opt.HostRoot) + irq.SetProcRoot(opt.HostRoot) } m := &resmgr{ From a5d2a74c2475ef4224a4b16bac695fd0bb5fd9ce Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Thu, 23 Jul 2026 12:35:04 +0300 Subject: [PATCH 2/5] balloons: add irqClaim and irqMode config options Add the balloon type options irqClaim and irqMode to the balloons policy configuration. Signed-off-by: Antti Kervinen --- .../bases/config.nri_balloonspolicies.yaml | 19 +++++++++++++++++++ .../crds/config.nri_balloonspolicies.yaml | 19 +++++++++++++++++++ .../v1alpha1/resmgr/policy/balloons/config.go | 13 +++++++++++++ .../policy/balloons/zz_generated.deepcopy.go | 5 +++++ 4 files changed, 56 insertions(+) diff --git a/config/crd/bases/config.nri_balloonspolicies.yaml b/config/crd/bases/config.nri_balloonspolicies.yaml index 545c73cd2..cc1e4438d 100644 --- a/config/crd/bases/config.nri_balloonspolicies.yaml +++ b/config/crd/bases/config.nri_balloonspolicies.yaml @@ -158,6 +158,25 @@ spec: will remain completely idle as they cannot be allocated to other balloons. type: boolean + irqClaim: + description: |- + IrqClaim lists IRQs whose CPU affinity is set to the CPUs + of balloons of this type. Each list item refers to IRQs + either by an exact number or by a shell-like wildcard + pattern matching a device or chip name in /proc/interrupts + items: + type: string + type: array + irqMode: + description: |- + IrqMode controls how CPU affinities of unclaimed (see + irqClaim) IRQs relate to the CPUs of balloons of this + type. "sink" adds the CPUs to the affinity of IRQs. If sink + balloons are present, they take unclaimed IRQs from other + balloons. "isolate" removes the CPUs of this balloon type + from the affinity of IRQs that are neither claimed nor + sinked. +kubebuilder:validation:Enum="";sink;isolate + type: string loads: description: |- Loads is a list of loadClasses that describe load generated diff --git a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml index 545c73cd2..cc1e4438d 100644 --- a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml +++ b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml @@ -158,6 +158,25 @@ spec: will remain completely idle as they cannot be allocated to other balloons. type: boolean + irqClaim: + description: |- + IrqClaim lists IRQs whose CPU affinity is set to the CPUs + of balloons of this type. Each list item refers to IRQs + either by an exact number or by a shell-like wildcard + pattern matching a device or chip name in /proc/interrupts + items: + type: string + type: array + irqMode: + description: |- + IrqMode controls how CPU affinities of unclaimed (see + irqClaim) IRQs relate to the CPUs of balloons of this + type. "sink" adds the CPUs to the affinity of IRQs. If sink + balloons are present, they take unclaimed IRQs from other + balloons. "isolate" removes the CPUs of this balloon type + from the affinity of IRQs that are neither claimed nor + sinked. +kubebuilder:validation:Enum="";sink;isolate + type: string loads: description: |- Loads is a list of loadClasses that describe load generated diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go index a8d6230d6..49a2be909 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/config.go @@ -307,6 +307,19 @@ type BalloonDef struct { // may generate a lot of traffic and large CR object updates // to Kubernetes API server. ShowContainersInNrt *bool `json:"showContainersInNrt,omitempty"` + // IrqClaim lists IRQs whose CPU affinity is set to the CPUs + // of balloons of this type. Each list item refers to IRQs + // either by an exact number or by a shell-like wildcard + // pattern matching a device or chip name in /proc/interrupts + IrqClaim []string `json:"irqClaim,omitempty"` + // IrqMode controls how CPU affinities of unclaimed (see + // irqClaim) IRQs relate to the CPUs of balloons of this + // type. "sink" adds the CPUs to the affinity of IRQs. If sink + // balloons are present, they take unclaimed IRQs from other + // balloons. "isolate" removes the CPUs of this balloon type + // from the affinity of IRQs that are neither claimed nor + // sinked. +kubebuilder:validation:Enum="";sink;isolate + IrqMode string `json:"irqMode,omitempty"` } // BalloonDefComponent contains a balloon component definition. diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go index e4b9ceae1..115f0d692 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/balloons/zz_generated.deepcopy.go @@ -87,6 +87,11 @@ func (in *BalloonDef) DeepCopyInto(out *BalloonDef) { *out = new(bool) **out = **in } + if in.IrqClaim != nil { + in, out := &in.IrqClaim, &out.IrqClaim + *out = make([]string, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BalloonDef. From 948889f027a5a4d22bcc3905ff2875fa1d9204ae Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Thu, 23 Jul 2026 12:35:15 +0300 Subject: [PATCH 3/5] balloons: implement IRQ affinity control Set CPU affinities of system IRQs according to the irqClaim and irqMode balloon type options. For each IRQ, the logic is: find balloons that want exactly this IRQ (irqClaim). If nobody, then find balloons that want all IRQs (irqMode: sink). If no volunteers, then list balloons that do not want any IRQs (irqMode: isolate), and set IRQs affinity to all those who do not care. The default is nobody cares (no irq options present), meaning that the balloons policy will not touch IRQ CPU affinities at all, keeping the original behavior. Affinities are updated after balloon changes. Signed-off-by: Antti Kervinen --- .../balloons/policy/balloons-policy.go | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 26eafc724..baf50b695 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -25,6 +25,7 @@ import ( cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/balloons" "github.com/containers/nri-plugins/pkg/cpuallocator" + "github.com/containers/nri-plugins/pkg/irq" "github.com/containers/nri-plugins/pkg/kubernetes" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cache" @@ -254,6 +255,7 @@ func (p *balloons) Sync(add []cache.Container, del []cache.Container) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() log.Debugf("synchronizing state...") for _, c := range del { @@ -278,6 +280,7 @@ func (p *balloons) AllocateResources(c cache.Container) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() if c.PreserveCpuResources() { log.Infof("not handling resources of container %s, preserving CPUs %q and memory %q", c.PrettyName(), c.GetCpusetCpus(), c.GetCpusetMems()) @@ -332,6 +335,7 @@ func (p *balloons) ReleaseResources(c cache.Container) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() log.Debugf("releasing container %s...", c.PrettyName()) if bln := p.balloonByContainer(c); bln != nil { @@ -366,6 +370,7 @@ func (p *balloons) UpdateResources(c cache.Container) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() log.Debugf("(not) updating container %s...", c.PrettyName()) return nil @@ -931,6 +936,98 @@ func (p *balloons) forgetCpuClass(bln *Balloon) { } } +const ( + // irqModeSink adds CPUs of a balloon to the affinity of + // unclaimed IRQs. + irqModeSink = "sink" + // irqModeIsolate removes CPUs of a balloon from the affinity + // of IRQs that are neither claimed nor sinked. + irqModeIsolate = "isolate" +) + +// irqOptionsInUse returns whether any balloon type configures IRQ +// affinities. +func (p *balloons) irqOptionsInUse() bool { + for _, blnDef := range p.bpoptions.BalloonDefs { + if len(blnDef.IrqClaim) > 0 || blnDef.IrqMode != "" { + return true + } + } + return false +} + +// irqClaimCpus returns the union of CPUs of balloons that claim the +// given interrupt. +func (p *balloons) irqClaimCpus(hwIrq *irq.Irq) cpuset.CPUSet { + claimCpus := cpuset.New() + for _, bln := range p.balloons { + if bln.Cpus.IsEmpty() { + continue + } + for _, claim := range bln.Def.IrqClaim { + if hwIrq.Match(claim) { + claimCpus = claimCpus.Union(bln.Cpus) + break + } + } + } + return claimCpus +} + +// applyIrqAffinities sets CPU affinities of system IRQs according to +// the irqClaim and irqMode options of balloon types. +func (p *balloons) applyIrqAffinities() { + if !p.irqOptionsInUse() { + return + } + hwIrqs, err := irq.Interrupts() + if err != nil { + log.Warnf("failed to read interrupts for IRQ affinity update: %v", err) + return + } + sinkCpus := cpuset.New() + isolateCpus := cpuset.New() + for _, bln := range p.balloons { + if bln.Cpus.IsEmpty() { + continue + } + switch bln.Def.IrqMode { + case irqModeSink: + sinkCpus = sinkCpus.Union(bln.Cpus) + case irqModeIsolate: + isolateCpus = isolateCpus.Union(bln.Cpus) + } + } + for _, hwIrq := range hwIrqs { + newCpus := p.allowed + curCpus, err := hwIrq.AffinityCpus() + if err != nil { + continue + } + switch claimCpus := p.irqClaimCpus(hwIrq); { + case !claimCpus.IsEmpty(): + newCpus = claimCpus + case !sinkCpus.IsEmpty(): + newCpus = sinkCpus + case curCpus.Intersection(p.allowed).IsEmpty(): + // IRQ is out of our scope + continue + case !isolateCpus.IsEmpty(): + // Keep IRQs affinity outside allowed CPUs as is. + // Allow IRQ on any allowed CPUs except for isolateCpus. + newCpus = curCpus.Union(p.allowed).Difference(isolateCpus) + } + if curCpus.Equals(newCpus) { + continue + } + if err := hwIrq.SetAffinityCpus(newCpus); err != nil { + log.Debugf("failed to set affinity of %s to %q: %v", hwIrq, newCpus, err) + } else { + log.Debugf("set affinity of %s to %q", hwIrq, newCpus) + } + } +} + // updateLoadedVirtDevsInAllocatorOptions updates CPU allocator // options with virtual devices under load due to the given load // classes. Returns a set of virtual device names under load. @@ -1526,6 +1623,7 @@ func (p *balloons) Reconfigure(newCfg interface{}) error { p.BlockMeters() defer p.UnblockMeters() defer p.commitCpuClasses() + defer p.applyIrqAffinities() balloonsOptions, ok := newCfg.(*BalloonsOptions) if !ok { From 25e89e6d5c73903cf2ab4b49b65b11bf431619cc Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Thu, 23 Jul 2026 12:35:24 +0300 Subject: [PATCH 4/5] e2e: add test25-irq for balloons IRQ affinity Add an end-to-end test that verifies irqClaim, irqMode sink and irqMode isolate by inspecting /proc/irq affinities on the test VM. Signed-off-by: Antti Kervinen --- .../n4c16/test25-irq/balloons-irq-claim.cfg | 18 ++ .../balloons-irq-dedicated-claim.cfg | 18 ++ .../n4c16/test25-irq/balloons-irq-isolate.cfg | 15 ++ .../n4c16/test25-irq/balloons-irq-sink.cfg | 21 +++ .../balloons/n4c16/test25-irq/code.var.sh | 166 ++++++++++++++++++ 5 files changed, 238 insertions(+) create mode 100644 test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-claim.cfg create mode 100644 test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-dedicated-claim.cfg create mode 100644 test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-isolate.cfg create mode 100644 test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-sink.cfg create mode 100644 test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-claim.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-claim.cfg new file mode 100644 index 000000000..770e8274f --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-claim.cfg @@ -0,0 +1,18 @@ +config: + reservedResources: + cpu: 1000m + pinCPU: true + pinMemory: true + balloonTypes: + - name: claimer + minCPUs: 2 + maxCPUs: 2 + irqClaim: + - "*ttyS0*" + - "*rtc0*" + log: + debug: + - policy + - irq + klog: + skip_headers: true diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-dedicated-claim.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-dedicated-claim.cfg new file mode 100644 index 000000000..ea6e93b43 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-dedicated-claim.cfg @@ -0,0 +1,18 @@ +config: + reservedResources: + cpu: 1000m + pinCPU: true + pinMemory: true + balloonTypes: + - name: dedicated-claimer + minCPUs: 2 + maxCPUs: 2 + irqClaim: + - "ttyS" # should not match + - "*rtc0*" + irqMode: isolate + log: + debug: + - policy + klog: + skip_headers: true diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-isolate.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-isolate.cfg new file mode 100644 index 000000000..968ed2ab0 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-isolate.cfg @@ -0,0 +1,15 @@ +config: + reservedResources: + cpu: 1000m + pinCPU: true + pinMemory: true + balloonTypes: + - name: isolate + minCPUs: 2 + maxCPUs: 2 + irqMode: isolate + log: + debug: + - policy + klog: + skip_headers: true diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-sink.cfg b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-sink.cfg new file mode 100644 index 000000000..ae99ed088 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/balloons-irq-sink.cfg @@ -0,0 +1,21 @@ +config: + reservedResources: + cpu: 1000m + pinCPU: true + pinMemory: true + balloonTypes: + - name: claimer + minCPUs: 2 + maxCPUs: 2 + irqClaim: + - "*ttyS0*" + - "*rtc0*" + - name: sink + minCPUs: 2 + maxCPUs: 2 + irqMode: sink + log: + debug: + - policy + klog: + skip_headers: true diff --git a/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh new file mode 100644 index 000000000..b1ff0c923 --- /dev/null +++ b/test/e2e/policies.test-suite/balloons/n4c16/test25-irq/code.var.sh @@ -0,0 +1,166 @@ +# Test balloons IRQ CPU affinity: irqClaim and irqMode (sink, isolate). + +# expand-cpulist "0-2,5" prints "0 1 2 5" +expand-cpulist() { + python3 -c ' +import sys +r = set() +for part in sys.argv[1].split(","): + if not part: + continue + if "-" in part: + a, b = part.split("-") + r.update(range(int(a), int(b) + 1)) + else: + r.add(int(part)) +print(" ".join(str(x) for x in sorted(r))) +' "$1" +} + +# ids-difference "1 2 3 4" "2 3" prints "1 4" +ids-difference() { + python3 -c 'import sys +allc = set(int(x) for x in sys.argv[1].split()) +iso = set(int(x) for x in sys.argv[2].split()) +print(" ".join(str(x) for x in sorted(allc - iso))) +' "$1" "$2" +} + +# ctr-cpu-ids podXcY prints sorted CPU ids allowed for the container, +# e.g. "0 1". Requires a preceding "verify" to refresh the state. +ctr-cpu-ids() { + pyexec "print(' '.join(str(i) for i in sorted(cpu_ids(cpus['$1']))))" +} + +# irq-cpu-ids IRQNUM prints sorted CPU ids in the affinity of the IRQ. +irq-cpu-ids() { + expand-cpulist "$(vm-command-q "cat /proc/irq/$1/smp_affinity_list" | tr -d '[:space:]')" +} + +# set-irq-cpus IRQNUM CPULIST sets the affinity of the IRQ, e.g. "0-15". +set-irq-cpus() { + vm-command "echo $2 > /proc/irq/$1/smp_affinity_list" || + command-error "failed to set affinity of irq $1" +} + +# verify-irq-cpus IRQNUM EXPECTED waits until the affinity of the IRQ +# equals EXPECTED (sorted space-separated CPU ids), or fails after a +# timeout. +verify-irq-cpus() { + local irqnum=$1 expected=$2 got tries=20 + while [ "$tries" -gt 0 ]; do + got=$(irq-cpu-ids "$irqnum") + if [ "$got" == "$expected" ]; then + echo "irq $irqnum affinity is '$got' as expected" + return 0 + fi + tries=$((tries - 1)) + sleep 1 + done + error "irq $irqnum affinity: expected CPUs '$expected', got '$got'" +} + +# Detect ttyS0 and rtc0 is IRQ numbers on vm. +vm-command "awk '/ ttyS0/{print \$1}' < /proc/interrupts | sed 's/://g' | head -n 1" +TTYS0_IRQ=$COMMAND_OUTPUT +vm-command "awk '/ rtc0/{print \$1}' < /proc/interrupts | sed 's/://g' | head -n 1" +RTC0_IRQ=$COMMAND_OUTPUT +vm-command "awk '/ acpi/{print \$1}' < /proc/interrupts | sed 's/://g' | head -n 1" +ACPI_IRQ=$COMMAND_OUTPUT + +ALL_CPUS="$(expand-cpulist 0-15)" + +cleanup() { + vm-command "kubectl delete pods --all --now" + vm-command "for f in /proc/irq/*/smp_affinity_list ; do echo 0-15 | tee $f >&/dev/null; done" +} + +cleanup + +# Test irqClaim. CPUs of the claimer balloon handle the claimed +# IRQs (ttyS0 and rtc0). +helm-terminate +helm_config=${TEST_DIR}/balloons-irq-claim.cfg helm-launch balloons + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: claimer" CONTCOUNT=1 create balloons-busybox +report allowed + +claimer_cpus=$(ctr-cpu-ids pod0c0) +echo "claimer CPUs: $claimer_cpus" +verify-irq-cpus "$TTYS0_IRQ" "$claimer_cpus" +verify-irq-cpus "$RTC0_IRQ" "$claimer_cpus" +verify-irq-cpus "$ACPI_IRQ" "$ALL_CPUS" + +# Test irqMode isolate. CPUs of the isolate balloon are removed from +# the affinity of unclaimed IRQs. Preset a full affinity so that the +# removal is observable. +helm-terminate +helm_config=${TEST_DIR}/balloons-irq-isolate.cfg helm-launch balloons +cleanup + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: isolate" CONTCOUNT=1 create balloons-busybox +report allowed + +isolate_cpus=$(ctr-cpu-ids pod1c0) +echo "isolate CPUs: $isolate_cpus" + +# expected_isolate=$(python3 -c 'import sys +# allc = set(int(x) for x in sys.argv[1].split()) +# iso = set(int(x) for x in sys.argv[2].split()) +# print(" ".join(str(x) for x in sorted(allc - iso))) +# ' "$ALL_CPUS" "$isolate_cpus") +expected_isolate=$(ids-difference "$ALL_CPUS" "$isolate_cpus") +verify-irq-cpus "$RTC0_IRQ" "$expected_isolate" + + +# Test irqMode sink. CPUs of the sink balloon handle unclaimed IRQs. +helm-terminate +helm_config=${TEST_DIR}/balloons-irq-sink.cfg helm-launch balloons +cleanup + +# no sink, no claimer present +verify-irq-cpus "$TTYS0_IRQ" "$ALL_CPUS" + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: sink" CONTCOUNT=1 create balloons-busybox +report allowed +verify +sink_cpus=$(ctr-cpu-ids pod2c0) +echo "sink CPUs: $sink_cpus" +verify-irq-cpus "$TTYS0_IRQ" "$sink_cpus" +verify-irq-cpus "$RTC0_IRQ" "$sink_cpus" +verify-irq-cpus "$ACPI_IRQ" "$sink_cpus" + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: claimer" CONTCOUNT=1 create balloons-busybox +report allowed +sink_cpus=$(ctr-cpu-ids pod2c0) +claimer_cpus=$(ctr-cpu-ids pod3c0) +echo "claimer CPUs: $claimer_cpus" +echo "sink CPUs: $sink_cpus" +verify-irq-cpus "$TTYS0_IRQ" "$claimer_cpus" +verify-irq-cpus "$RTC0_IRQ" "$claimer_cpus" +verify-irq-cpus "$ACPI_IRQ" "$sink_cpus" + +vm-command 'kubectl delete pod pod2 --now' +# no sink present anymore +verify-irq-cpus "$TTYS0_IRQ" "$claimer_cpus" +verify-irq-cpus "$RTC0_IRQ" "$claimer_cpus" +verify-irq-cpus "$ACPI_IRQ" "$ALL_CPUS" + +# Test irqMode sink. CPUs of the sink balloon handle unclaimed IRQs. +helm-terminate +helm_config=${TEST_DIR}/balloons-irq-dedicated-claim.cfg helm-launch balloons +cleanup + +POD_ANNOTATION="balloon.balloons.resource-policy.nri.io: dedicated-claimer" CONTCOUNT=1 create balloons-busybox +report allowed + +d_claimer_cpus=$(ctr-cpu-ids pod4c0) +echo "dedicated claimer CPUs: $d_claimer_cpus" +verify-irq-cpus "$RTC0_IRQ" "$d_claimer_cpus" + +expected_isolate=$(ids-difference "$ALL_CPUS" "$d_claimer_cpus") +verify-irq-cpus "$TTYS0_IRQ" "$expected_isolate" +verify-irq-cpus "$ACPI_IRQ" "$expected_isolate" + +cleanup +helm-terminate From a93ae5fdddd9e801c4a2cf6274c6fbfbfb97e806 Mon Sep 17 00:00:00 2001 From: Antti Kervinen Date: Thu, 23 Jul 2026 12:35:24 +0300 Subject: [PATCH 5/5] docs: document balloons IRQ affinity options Describe the irqClaim and irqMode balloon type options and their common use cases. Signed-off-by: Antti Kervinen --- docs/resource-policy/policy/balloons.md | 48 ++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/docs/resource-policy/policy/balloons.md b/docs/resource-policy/policy/balloons.md index 1da518331..aa5fabbc7 100644 --- a/docs/resource-policy/policy/balloons.md +++ b/docs/resource-policy/policy/balloons.md @@ -848,7 +848,8 @@ memory-type.resource-policy.nri.io/container.CONTAINER_NAME: HBM,DRAM ### CPU Tuning -These options configure CPU behavior and power management. +These options configure CPU behavior, power management, and tuning +IRQs CPU affinity. **`cpuClass`** (string) - References a CPU class name defined in `cpuClasses` (preferred) or @@ -1059,6 +1060,51 @@ control: maxFreq: 1200000 ``` +#### IRQ CPU Affinity Tuning + +These balloon type options control the CPU affinity of hardware +interrupts (IRQs). By default the policy does not touch IRQ affinities. +Affinities are updated by writing +`/proc/irq//smp_affinity_list`, and IRQs are matched against +`/proc/interrupts`. + +**`irqClaim`** (list of strings) +- Lists IRQs handled by CPUs of balloons of this type. +- Each item refers to IRQs either by an exact number, or by a pattern + matching last columns in `/proc/interrupts`. Supports wildcards, for + instance "*nvme*". +- The affinity of a claimed IRQ is set to the union of CPUs of all + balloons that claim it. + +**`irqMode`** (string: `"sink"` or `"isolate"`) +- `sink`: CPUs of balloons of this type handle IRQs that no balloon + has claimed. The affinity of such an unclaimed IRQ is set to the + union of CPUs of all sink balloons. +- `isolate`: CPUs of balloons of this type are removed from the + affinity of IRQs that are neither claimed nor sinked. This keeps + such CPUs free from interrupts. + +The options can be combined. Common use cases: + +- *Latency-sensitive balloon*: use `irqMode: isolate` to keep the + CPUs of the balloon free from unrelated IRQs. +- *Hardware-specific balloon*: use `irqClaim` to direct IRQs of the + relevant devices to the CPUs of the balloon. Add `irqMode: isolate` + to additionally keep all other IRQs away from those CPUs. +- *IRQ sink balloon*: use `irqMode: sink` to gather all otherwise + unclaimed IRQs on the CPUs of the balloon. + +```yaml +balloonTypes: +- name: network + irqClaim: + - "*eth0 *" + - "42" + irqMode: isolate +- name: housekeeping + irqMode: sink +``` + ### Built-in Balloon Types The policy includes two built-in balloon types that can be customized