From 5956f50ddc41fd344502aa4a47ef0aaf1dc93c09 Mon Sep 17 00:00:00 2001 From: itayherzberg Date: Wed, 21 Oct 2026 19:17:58 +0300 Subject: [PATCH] fix(server): spread background jobs across workers; make the pool size configurable TaskManager.getAvailableWorker scanned the pool from the first worker and returned the first one with queue capacity. Each worker's queue holds 10 jobs, so every job landed on worker 1 and ran serially while the other workers stayed idle: eight parallel virtual-media inserts completed one after another, 1.2-1.9s apart, instead of concurrently. Start the scan from an atomic round-robin cursor so consecutive jobs go to different workers, and replace the hardcoded pool size of 4 with server.workers (KUBEVIRT_REDFISH_SERVER_WORKERS), default 8, range 1-64, zero meaning the default. The pool size is read once at startup. Signed-off-by: itayherzberg --- config/config.yaml | 1 + helm/templates/secret.yaml | 3 ++ helm/values-example.yaml | 1 + helm/values.yaml | 6 +++ pkg/config/config.go | 17 ++++++ pkg/config/config_test.go | 91 +++++++++++++++++++++++++++++++++ pkg/server/server.go | 7 ++- pkg/server/task_manager.go | 14 ++++- pkg/server/task_manager_test.go | 46 +++++++++++++++++ 9 files changed, 184 insertions(+), 2 deletions(-) create mode 100644 pkg/server/task_manager_test.go diff --git a/config/config.yaml b/config/config.yaml index 7f65aea..ea42128 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -6,6 +6,7 @@ kubevirt: server: host: "0.0.0.0" # Host for the Redfish server (0.0.0.0 is default (typically unchanged unless testing) port: 8443 # Port for the Redfish server (8443 is default (typically unchanged unless testing) + workers: 8 # Background task pool size for virtual media and other async tasks (8 is default) tls: enabled: false # Disabled for development/testing (not required with OpenShift because of default route TLS) cert_file: "/etc/kubevirt-redfish/tls/tls.crt" # Path to the certificate file (not required with OpenShift, but may be leveraged for other platforms) diff --git a/helm/templates/secret.yaml b/helm/templates/secret.yaml index 2fafc1b..c39bb54 100644 --- a/helm/templates/secret.yaml +++ b/helm/templates/secret.yaml @@ -19,6 +19,9 @@ stringData: server: host: "{{ .Values.server.host }}" port: {{ .Values.server.port }} + {{- if .Values.server.workers }} + workers: {{ .Values.server.workers }} + {{- end }} tls: enabled: {{ .Values.server.tls.enabled }} diff --git a/helm/values-example.yaml b/helm/values-example.yaml index b93583d..7c5983a 100644 --- a/helm/values-example.yaml +++ b/helm/values-example.yaml @@ -121,6 +121,7 @@ kubevirt: server: host: "0.0.0.0" port: 8443 + workers: 8 tls: enabled: false diff --git a/helm/values.yaml b/helm/values.yaml index d4e7a8e..5179df6 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -204,6 +204,12 @@ server: # Server port port: 8443 + # Background task pool size (virtual media insert/eject and other async + # Redfish tasks). Each job is a short burst of Kubernetes API calls, so this + # bounds how many inserts start at once, not how many ISOs copy at once. + # Read at startup only; changing it needs a restart. + workers: 8 + # System ID convention for Redfish compliance # Options: "legacy" (vm-name only) or "enhanced" (namespace.vm-name) # Default: "legacy" for backward compatibility with existing Metal3/Ironic deployments diff --git a/pkg/config/config.go b/pkg/config/config.go index e755731..45f80b6 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -83,6 +83,10 @@ type Config struct { SystemIDConvention string `mapstructure:"system_id_convention"` // "legacy" or "enhanced" } +// DefaultWorkers is the size of the background task pool when server.workers +// is left unset. +const DefaultWorkers = 8 + // ServerConfig holds HTTP server configuration including host, port, and TLS settings. // The server configuration determines how the Redfish API server listens for requests. type ServerConfig struct { @@ -90,6 +94,9 @@ type ServerConfig struct { Port int `mapstructure:"port"` TLS TLSConfig `mapstructure:"tls"` TestMode bool `mapstructure:"test_mode"` // Disables background processes for testing + // Workers is the size of the background task pool that runs virtual media + // inserts/ejects and other async Redfish tasks. Read once at startup. + Workers int `mapstructure:"workers"` } // TLSConfig holds TLS configuration for secure HTTPS connections. @@ -293,6 +300,7 @@ func LoadConfig(configPath string) (*Config, error) { func setDefaults() { viper.SetDefault("server.host", "0.0.0.0") viper.SetDefault("server.port", 8443) + viper.SetDefault("server.workers", DefaultWorkers) viper.SetDefault("server.tls.enabled", true) viper.SetDefault("kubevirt.api_version", "v1") viper.SetDefault("kubevirt.timeout", 30) @@ -368,6 +376,15 @@ func validateServerConfig(server *ServerConfig) error { return errors.NewValidationError("Invalid server port", fmt.Sprintf("server.port must be between 1 and 65535, got %d", server.Port)) } + // Zero means "not set": apply the default so a Config built without viper + // (tests, programmatic use) validates the same way a loaded one does. + if server.Workers == 0 { + server.Workers = DefaultWorkers + } + if server.Workers < 1 || server.Workers > 64 { + return errors.NewValidationError("Invalid worker count", fmt.Sprintf("server.workers must be between 1 and 64, got %d", server.Workers)) + } + if server.TLS.Enabled { if server.TLS.CertFile == "" { return errors.NewValidationError("TLS certificate file is required when TLS is enabled", "server.tls.cert_file cannot be empty when server.tls.enabled is true") diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index a74626b..d963937 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -22,6 +22,7 @@ package config import ( "os" "path/filepath" + "strings" "testing" "github.com/kubevirt/redfish-controller/pkg/errors" @@ -104,6 +105,96 @@ kubevirt: } } +// minimalConfig is the smallest config that passes validation, with the +// kubevirt section supplied by the caller. +func minimalConfig(kubevirtSection string) string { + return ` +server: + host: "127.0.0.1" + port: 8443 + tls: + enabled: false + +chassis: + - name: "test-cluster" + namespace: "test-namespace" + service_account: "test-sa" + +authentication: + users: + - username: "testuser" + password: "testpass" + chassis: ["test-cluster"] + +kubevirt: + api_version: "v1" + timeout: 30 +` + kubevirtSection +} + +func writeConfig(t *testing.T, content string) string { + t.Helper() + configPath := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(configPath, []byte(content), 0600); err != nil { + t.Fatalf("Failed to write test config: %v", err) + } + return configPath +} + +func TestWorkersConfig(t *testing.T) { + t.Run("default", func(t *testing.T) { + config, err := LoadConfig(writeConfig(t, minimalConfig(""))) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + if config.Server.Workers != 8 { + t.Errorf("Expected default workers 8, got %d", config.Server.Workers) + } + }) + + t.Run("from config file", func(t *testing.T) { + content := strings.Replace(minimalConfig(""), " port: 8443\n", " port: 8443\n workers: 3\n", 1) + config, err := LoadConfig(writeConfig(t, content)) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + if config.Server.Workers != 3 { + t.Errorf("Expected workers 3 from config file, got %d", config.Server.Workers) + } + }) + + t.Run("from environment", func(t *testing.T) { + t.Setenv("KUBEVIRT_REDFISH_SERVER_WORKERS", "16") + config, err := LoadConfig(writeConfig(t, minimalConfig(""))) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + if config.Server.Workers != 16 { + t.Errorf("Expected workers 16 from env, got %d", config.Server.Workers) + } + }) + + t.Run("rejects out-of-range values", func(t *testing.T) { + for _, v := range []string{"-1", "65"} { + t.Setenv("KUBEVIRT_REDFISH_SERVER_WORKERS", v) + if _, err := LoadConfig(writeConfig(t, minimalConfig(""))); err == nil { + t.Errorf("Expected validation error for workers=%s, got nil", v) + } + } + }) + + t.Run("zero means default", func(t *testing.T) { + t.Setenv("KUBEVIRT_REDFISH_SERVER_WORKERS", "0") + config, err := LoadConfig(writeConfig(t, minimalConfig(""))) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + if config.Server.Workers != DefaultWorkers { + t.Errorf("Expected default workers %d for zero, got %d", DefaultWorkers, config.Server.Workers) + } + }) +} + func TestLoadConfigWithDefaults(t *testing.T) { // Create a temporary directory and config file tempDir := t.TempDir() diff --git a/pkg/server/server.go b/pkg/server/server.go index d6cd801..72053ec 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -129,7 +129,12 @@ func NewServer(config *config.Config, kubevirtClient *kubevirt.Client) *Server { // Initialize background components only if not in test mode if !config.Server.TestMode { // Initialize enhanced task manager - taskManager = NewTaskManager(4, kubevirtClient) // 4 workers for background processing + workers := config.Server.Workers + if workers < 1 { + workers = 4 // config validation normally guarantees >= 1; keep the historical default otherwise + } + taskManager = NewTaskManager(workers, kubevirtClient) + logger.Info("Task manager started with %d workers", workers) // Initialize job scheduler jobScheduler = NewJobScheduler() diff --git a/pkg/server/task_manager.go b/pkg/server/task_manager.go index 026248c..cd377e9 100644 --- a/pkg/server/task_manager.go +++ b/pkg/server/task_manager.go @@ -23,6 +23,7 @@ import ( "context" "fmt" "sync" + "sync/atomic" "time" "github.com/kubevirt/redfish-controller/pkg/kubevirt" @@ -77,6 +78,7 @@ type TaskManager struct { // Worker pool workers []*Worker workerCount int + nextWorker uint64 // round-robin cursor for worker selection workerMutex sync.RWMutex // KubeVirt client for actual operations @@ -503,7 +505,17 @@ func (tm *TaskManager) getAvailableWorker() *Worker { tm.workerMutex.RLock() defer tm.workerMutex.RUnlock() - for _, worker := range tm.workers { + if len(tm.workers) == 0 { + return nil + } + + // Advance the round-robin cursor so consecutive jobs land on different + // workers. Without this the dispatcher always returns the first worker + // with channel capacity, which serializes all jobs onto worker 1 while + // the remaining workers stay idle. + start := int(atomic.AddUint64(&tm.nextWorker, 1) % uint64(len(tm.workers))) + for i := range len(tm.workers) { + worker := tm.workers[(start+i)%len(tm.workers)] // Check if worker's job channel has capacity if len(worker.jobChan) < cap(worker.jobChan) { return worker diff --git a/pkg/server/task_manager_test.go b/pkg/server/task_manager_test.go new file mode 100644 index 0000000..c2b0c2f --- /dev/null +++ b/pkg/server/task_manager_test.go @@ -0,0 +1,46 @@ +/* + * This file is part of the KubeVirt Redfish project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2025 KubeVirt Redfish project and its authors. + * + */ + +package server + +import "testing" + +// With every worker idle, consecutive jobs must be spread across the pool. +// A dispatcher that always returns the first worker with queue capacity +// serializes every job onto worker 1 while the others stay idle. +func TestGetAvailableWorkerRoundRobin(t *testing.T) { + const workers = 4 + tm := NewTaskManager(workers, nil) + defer tm.cancel() + + picks := make(map[int]int) + for i := 0; i < 2*workers; i++ { + w := tm.getAvailableWorker() + if w == nil { + t.Fatalf("pick %d: no worker available although all queues are empty", i) + } + picks[w.ID]++ + } + + for id := 1; id <= workers; id++ { + if picks[id] != 2 { + t.Errorf("worker %d was picked %d times, want 2 (picks per worker: %v)", id, picks[id], picks) + } + } +}