diff --git a/cmd/main.go b/cmd/main.go index 6e1f17b..398c048 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -103,13 +103,7 @@ func main() { log.Fatalf("Failed to load configuration: %v", err) } - // Create KubeVirt client - kubevirtClient, err := kubevirt.NewClient(*kubeconfig, 30*time.Second, cfg) - if err != nil { - log.Fatalf("Failed to create KubeVirt client: %v", err) - } - - // Initialize logger + // Initialize the logger before the client, which logs its settings logLevel := logger.GetLogLevelFromEnv() if logger.IsLoggingEnabled() { logger.Init(logLevel) @@ -118,6 +112,12 @@ func main() { logger.Info("Logging disabled via REDFISH_LOGGING_ENABLED=false") } + // Create KubeVirt client + kubevirtClient, err := kubevirt.NewClient(*kubeconfig, 30*time.Second, cfg) + if err != nil { + log.Fatalf("Failed to create KubeVirt client: %v", err) + } + // Create and start server srv := server.NewServer(cfg, kubevirtClient) diff --git a/config/config.yaml b/config/config.yaml index 2864a81..ec86357 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -2,6 +2,8 @@ kubevirt: api_version: "v1" # Kubevirt API version (v1 is default (typically unchanged unless testing) timeout: 30 # Timeout for Kubevirt API calls (30 seconds is default (typically unchanged unless testing) + qps: 5 # client-go rate limit for Kubernetes API calls, sustained requests/second (5, the client-go default, is default; env KUBEVIRT_REDFISH_KUBEVIRT_QPS) + burst: 10 # client-go rate limit burst, max requests above qps in a spike (10, the client-go default, is default; env KUBEVIRT_REDFISH_KUBEVIRT_BURST) server: host: "0.0.0.0" # Host for the Redfish server (0.0.0.0 is default (typically unchanged unless testing) diff --git a/helm/templates/secret.yaml b/helm/templates/secret.yaml index c39bb54..fff40c1 100644 --- a/helm/templates/secret.yaml +++ b/helm/templates/secret.yaml @@ -50,6 +50,12 @@ stringData: kubevirt: api_version: "{{ .Values.kubevirt.api_version }}" timeout: {{ .Values.kubevirt.timeout }} + {{- if .Values.kubevirt.qps }} + qps: {{ .Values.kubevirt.qps }} + {{- end }} + {{- if .Values.kubevirt.burst }} + burst: {{ .Values.kubevirt.burst }} + {{- end }} {{- if .Values.kubevirt.allow_insecure_tls }} allow_insecure_tls: {{ .Values.kubevirt.allow_insecure_tls }} {{- end }} diff --git a/helm/values-example.yaml b/helm/values-example.yaml index 7c5983a..363b262 100644 --- a/helm/values-example.yaml +++ b/helm/values-example.yaml @@ -112,6 +112,8 @@ rbac: kubevirt: api_version: "v1" timeout: 30 + qps: 100 # raised from the client-go default (5) for many concurrent BMHs + burst: 200 # raised from the client-go default (10) allow_insecure_tls: true # ============================================================================= diff --git a/helm/values.yaml b/helm/values.yaml index 5179df6..32a2e1f 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -263,6 +263,19 @@ kubevirt: # Timeout for KubeVirt API operations (in seconds) timeout: 30 + # client-go rate limiter for calls to the Kubernetes API. 0 (unset) keeps + # client-go's defaults of 5 / 10. Each Redfish request fans out to several + # API calls (a GET System is 6, a boot-device PATCH is 12), so with N BMHs + # in flight the worst request takes roughly 12 * N / qps seconds; the + # defaults saturate at ~15 BMHs. Grep the pod log for "client-side + # throttling"; if it appears under load, raise these (e.g. qps: 100, + # burst: 200). The env vars KUBEVIRT_REDFISH_KUBEVIRT_QPS / + # KUBEVIRT_REDFISH_KUBEVIRT_BURST (deployment.container.env) override both. + # The client is built once at startup: unlike the hot-reloaded parts of the + # config, a change here only takes effect after the pod restarts. + qps: 0 + burst: 0 + # Allow insecure TLS for KubeVirt API operations # Set to false in production environments allow_insecure_tls: true diff --git a/pkg/config/config.go b/pkg/config/config.go index 5188441..75393a0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -50,6 +50,8 @@ // kubevirt: // api_version: "v1" // timeout: 30 +// qps: 5 +// burst: 10 package config import ( @@ -68,6 +70,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/spf13/viper" "golang.org/x/crypto/bcrypt" + "k8s.io/client-go/rest" ) // Config represents the complete application configuration structure. @@ -203,6 +206,12 @@ type KubeVirtConfig struct { APIVersion string `mapstructure:"api_version"` Timeout int `mapstructure:"timeout"` AllowInsecureTLS bool `mapstructure:"allow_insecure_tls"` + // QPS and Burst set the client-go rate limiter for calls to the Kubernetes + // API; zero keeps client-go's defaults. See helm/values.yaml for sizing. + // The client is built once at startup, so unlike the hot-reloaded parts of + // the config a change here only takes effect after a restart. + QPS float32 `mapstructure:"qps"` + Burst int `mapstructure:"burst"` } // CDIConfig holds CDI (Containerized Data Importer) configuration parameters. @@ -318,6 +327,8 @@ func setDefaults() { viper.SetDefault("kubevirt.api_version", "v1") viper.SetDefault("kubevirt.timeout", 30) viper.SetDefault("kubevirt.allow_insecure_tls", false) + viper.SetDefault("kubevirt.qps", rest.DefaultQPS) + viper.SetDefault("kubevirt.burst", rest.DefaultBurst) viper.SetDefault("datavolume.storage_size", "10Gi") viper.SetDefault("datavolume.allow_insecure_tls", false) viper.SetDefault("datavolume.vm_update_timeout", "30s") @@ -513,6 +524,13 @@ func validateKubeVirtConfig(kv *KubeVirtConfig) error { return errors.NewValidationError("Invalid KubeVirt timeout", fmt.Sprintf("kubevirt.timeout must be between 1 and 300 seconds, got %d", kv.Timeout)) } + if kv.QPS < 0 { + return errors.NewValidationError("Invalid KubeVirt QPS", fmt.Sprintf("kubevirt.qps must be greater than 0, got %v", kv.QPS)) + } + if kv.Burst < 0 { + return errors.NewValidationError("Invalid KubeVirt burst", fmt.Sprintf("kubevirt.burst must be greater than 0, got %d", kv.Burst)) + } + return nil } @@ -707,6 +725,13 @@ func (c *Config) GetChassisForUser(username string) ([]*ChassisConfig, error) { return nil, fmt.Errorf("user not found: %s", username) } +// GetKubeClientRateLimits returns the client-go QPS and burst to use for the +// Kubernetes API client. The kubevirt client discovers this method through a +// type assertion, the same way it discovers GetDataVolumeConfig. +func (c *Config) GetKubeClientRateLimits() (qps float32, burst int) { + return c.KubeVirt.QPS, c.KubeVirt.Burst +} + // GetDataVolumeConfig returns the DataVolume configuration settings. // This provides a safe way to access DataVolume settings without reflection. func (c *Config) GetDataVolumeConfig() (storageSize string, allowInsecureTLS bool, storageClass string, vmUpdateTimeout string, isoDownloadTimeout string, helperImage string) { @@ -781,6 +806,8 @@ authentication: kubevirt: api_version: "v1" # KubeVirt API version to use timeout: 30 # Timeout in seconds for KubeVirt operations + qps: 5 # client-go rate limit for Kubernetes API calls (sustained requests/second) + burst: 10 # client-go rate limit burst (max requests above qps in a spike) # DataVolume Configuration # ------------------------ diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index d963937..94ad00b 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -103,6 +103,13 @@ kubevirt: if config.KubeVirt.Timeout != 30 { t.Errorf("Expected timeout 30, got %d", config.KubeVirt.Timeout) } + // Unless configured, the rate limits are client-go's own defaults. + if config.KubeVirt.QPS != 5 { + t.Errorf("Expected default qps 5 (client-go default), got %v", config.KubeVirt.QPS) + } + if config.KubeVirt.Burst != 10 { + t.Errorf("Expected default burst 10 (client-go default), got %d", config.KubeVirt.Burst) + } } // minimalConfig is the smallest config that passes validation, with the @@ -141,6 +148,52 @@ func writeConfig(t *testing.T, content string) string { return configPath } +func TestKubeClientRateLimits(t *testing.T) { + t.Run("from config file", func(t *testing.T) { + config, err := LoadConfig(writeConfig(t, minimalConfig(" qps: 250\n burst: 500\n"))) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + qps, burst := config.GetKubeClientRateLimits() + if qps != 250 || burst != 500 { + t.Errorf("Expected qps=250 burst=500, got qps=%v burst=%d", qps, burst) + } + }) + + t.Run("from environment", func(t *testing.T) { + t.Setenv("KUBEVIRT_REDFISH_KUBEVIRT_QPS", "300") + t.Setenv("KUBEVIRT_REDFISH_KUBEVIRT_BURST", "600") + config, err := LoadConfig(writeConfig(t, minimalConfig(""))) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + qps, burst := config.GetKubeClientRateLimits() + if qps != 300 || burst != 600 { + t.Errorf("Expected qps=300 burst=600 from env, got qps=%v burst=%d", qps, burst) + } + }) + + t.Run("rejects negative values", func(t *testing.T) { + for _, section := range []string{" qps: -1\n", " burst: -1\n"} { + if _, err := LoadConfig(writeConfig(t, minimalConfig(section))); err == nil { + t.Errorf("Expected validation error for %q, got nil", section) + } + } + }) + + // Zero is passed through untouched: the kubevirt client treats it as + // "not set" and applies client-go's defaults. + t.Run("zero is accepted", func(t *testing.T) { + config, err := LoadConfig(writeConfig(t, minimalConfig(" qps: 0\n burst: 0\n"))) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + if qps, burst := config.GetKubeClientRateLimits(); qps != 0 || burst != 0 { + t.Errorf("Expected zero values to be left as they are, got qps=%v burst=%d", qps, burst) + } + }) +} + func TestWorkersConfig(t *testing.T) { t.Run("default", func(t *testing.T) { config, err := LoadConfig(writeConfig(t, minimalConfig(""))) diff --git a/pkg/kubevirt/client.go b/pkg/kubevirt/client.go index 329dcfc..edb25c4 100644 --- a/pkg/kubevirt/client.go +++ b/pkg/kubevirt/client.go @@ -204,6 +204,26 @@ func volumeImportSourceToUnstructured(vis *cdiv1beta1.VolumeImportSource) (*unst return u, nil } +// kubeClientRateLimits returns the client-go QPS and burst for the Kubernetes +// API client, taken from kubevirt.qps / kubevirt.burst in the app config. +// Without a config, or for a value left at zero, client-go's own defaults +// apply. +func kubeClientRateLimits(appConfig interface{}) (float32, int) { + qps, burst := rest.DefaultQPS, rest.DefaultBurst + if rl, ok := appConfig.(interface { + GetKubeClientRateLimits() (float32, int) + }); ok { + cfgQPS, cfgBurst := rl.GetKubeClientRateLimits() + if cfgQPS > 0 { + qps = cfgQPS + } + if cfgBurst > 0 { + burst = cfgBurst + } + } + return qps, burst +} + // NewClient creates a new KubeVirt client with connection pooling func NewClient(configPath string, timeout time.Duration, appConfig interface{}) (*Client, error) { var config *rest.Config @@ -221,6 +241,9 @@ func NewClient(configPath string, timeout time.Duration, appConfig interface{}) return nil, fmt.Errorf("failed to create kubeconfig: %w", err) } + config.QPS, config.Burst = kubeClientRateLimits(appConfig) + logger.Info("Kubernetes client rate limits: qps=%v burst=%d", config.QPS, config.Burst) + // Initialize random seed for unique PVC name generation rand.Seed(time.Now().UnixNano()) diff --git a/pkg/kubevirt/ratelimit_test.go b/pkg/kubevirt/ratelimit_test.go new file mode 100644 index 0000000..2b1ee9c --- /dev/null +++ b/pkg/kubevirt/ratelimit_test.go @@ -0,0 +1,56 @@ +/* + * This file is part of the KubeVirt Redfish project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2025 KubeVirt Redfish project and its authors. + * + */ + +package kubevirt + +import ( + "testing" + + "k8s.io/client-go/rest" +) + +type fakeRateLimitConfig struct { + qps float32 + burst int +} + +func (f fakeRateLimitConfig) GetKubeClientRateLimits() (float32, int) { return f.qps, f.burst } + +func TestKubeClientRateLimits(t *testing.T) { + tests := []struct { + name string + appConfig interface{} + wantQPS float32 + wantBurst int + }{ + {"no config keeps client-go defaults", nil, rest.DefaultQPS, rest.DefaultBurst}, + {"config without rate limits keeps client-go defaults", struct{}{}, rest.DefaultQPS, rest.DefaultBurst}, + {"configured values are used", fakeRateLimitConfig{qps: 100, burst: 200}, 100, 200}, + {"zero falls back per field", fakeRateLimitConfig{qps: 50, burst: 0}, 50, rest.DefaultBurst}, + {"all zero keeps client-go defaults", fakeRateLimitConfig{}, rest.DefaultQPS, rest.DefaultBurst}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + qps, burst := kubeClientRateLimits(tt.appConfig) + if qps != tt.wantQPS || burst != tt.wantBurst { + t.Errorf("got qps=%v burst=%d, want qps=%v burst=%d", qps, burst, tt.wantQPS, tt.wantBurst) + } + }) + } +}