From c8b58f9681c84f72f1e1b03580d6056f8551ecc6 Mon Sep 17 00:00:00 2001 From: itayherzberg Date: Wed, 21 Oct 2026 19:17:58 +0300 Subject: [PATCH 1/2] feat(kubevirt): serve read-only VM/VMI lookups from an informer cache Every Redfish read went to the API server: a GET /Systems/{id} was six Kubernetes API calls (one to resolve the system, then the VM four times and the VMI once), and Ironic polls every host it manages. With many hosts in flight those calls queue behind the client-go rate limiter and reads take 10-35 seconds. Keep one informer per configured namespace for VirtualMachines and one for VirtualMachineInstances, with managedFields stripped, and answer read-only lookups (power state, boot options, CPU, memory, storage, network, virtual media state, the collections, and the per-request system lookup) from the local copy. A warm cache turns a GET /Systems/{id} into zero API calls and a boot-device PATCH from 12 into 6. Three rules keep cached reads correct: - Anything that reads a VM in order to change it still reads live (SetBootOrder, SetBootOnce, virtual media insert/eject, power actions, import labels), so a patch is never computed from a stale copy. GetVM / GetVMI are unchanged; the cached variants are getVMForRead / getVMIForRead. - The cache is either fully trusted or not used: until an informer has synced, reads for that namespace go to the API server; after that its answer is final, including "not found". - After this process writes an object, reads of it skip the cache until the informer has stored that write. A thin wrapper around the dynamic client records the resourceVersion returned by every VM/VMI create, update, patch and delete; the informer's event handler reports each version it stores. The tracker also keeps the last versions the informer stored, because on a real API server the watch event for our own write often arrives before the write call returns. The bypass ends after writeBypassMax (30s) at the latest, with a warning. Writes by anyone else (oc edit, KubeVirt's controllers) reach the cache with their watch event, usually within milliseconds. The cache can be turned off with kubevirt.cache_enabled: false. GetMetrics counts GetVM.cache / GetVM.direct (and the VMI and List equivalents). The informers need list and watch on virtualmachines and virtualmachineinstances; the chart's ClusterRole already grants both. go.mod gains one indirect test-only dependency pulled in by k8s.io/client-go/dynamic/fake. Signed-off-by: itayherzberg --- config/config.yaml | 1 + go.mod | 1 + go.sum | 2 + helm/templates/secret.yaml | 3 + helm/values-example.yaml | 1 + helm/values.yaml | 6 + pkg/config/config.go | 13 + pkg/config/config_test.go | 69 +++++ pkg/kubevirt/cache.go | 601 +++++++++++++++++++++++++++++++++++++ pkg/kubevirt/cache_test.go | 480 +++++++++++++++++++++++++++++ pkg/kubevirt/client.go | 84 +++--- pkg/server/server.go | 4 +- 12 files changed, 1226 insertions(+), 39 deletions(-) create mode 100644 pkg/kubevirt/cache.go create mode 100644 pkg/kubevirt/cache_test.go diff --git a/config/config.yaml b/config/config.yaml index 2cec99f..6fcae5f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -2,6 +2,7 @@ kubevirt: api_version: "v1" # Kubevirt API version (v1 is default (typically unchanged unless testing) timeout: 30 # Timeout for Kubevirt API calls (30 seconds is default (typically unchanged unless testing) + cache_enabled: true # Serve read-only VM/VMI lookups from an informer cache instead of the API (true is default) server: host: "0.0.0.0" # Host for the Redfish server (0.0.0.0 is default (typically unchanged unless testing) diff --git a/go.mod b/go.mod index ee9d8a3..f943f75 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect diff --git a/go.sum b/go.sum index f9896c8..549c176 100644 --- a/go.sum +++ b/go.sum @@ -310,6 +310,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= diff --git a/helm/templates/secret.yaml b/helm/templates/secret.yaml index 2fafc1b..41951b2 100644 --- a/helm/templates/secret.yaml +++ b/helm/templates/secret.yaml @@ -47,6 +47,9 @@ stringData: kubevirt: api_version: "{{ .Values.kubevirt.api_version }}" timeout: {{ .Values.kubevirt.timeout }} + {{- if hasKey .Values.kubevirt "cache_enabled" }} + cache_enabled: {{ .Values.kubevirt.cache_enabled }} + {{- end }} {{- if .Values.kubevirt.allow_insecure_tls }} allow_insecure_tls: {{ .Values.kubevirt.allow_insecure_tls }} {{- end }} diff --git a/helm/values-example.yaml b/helm/values-example.yaml index b93583d..4e94ec7 100644 --- a/helm/values-example.yaml +++ b/helm/values-example.yaml @@ -112,6 +112,7 @@ rbac: kubevirt: api_version: "v1" timeout: 30 + cache_enabled: true allow_insecure_tls: true # ============================================================================= diff --git a/helm/values.yaml b/helm/values.yaml index d4e7a8e..d6c75d9 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -257,6 +257,12 @@ kubevirt: # Timeout for KubeVirt API operations (in seconds) timeout: 30 + # Serve read-only VM/VMI lookups (power state, inventory, boot options, + # virtual media state, collections) from an in-memory informer cache instead + # of the Kubernetes API. Write paths always read live. Set false to fall back + # to one API call per lookup. + cache_enabled: true + # Allow insecure TLS for KubeVirt API operations # Set to false in production environments allow_insecure_tls: true diff --git a/pkg/config/config.go b/pkg/config/config.go index 478bae0..0bdaeed 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -50,6 +50,7 @@ // kubevirt: // api_version: "v1" // timeout: 30 +// cache_enabled: true package config import ( @@ -196,6 +197,9 @@ type KubeVirtConfig struct { APIVersion string `mapstructure:"api_version"` Timeout int `mapstructure:"timeout"` AllowInsecureTLS bool `mapstructure:"allow_insecure_tls"` + // CacheEnabled serves read-only VM/VMI lookups from an informer cache + // instead of the API server. On by default; see pkg/kubevirt/cache.go. + CacheEnabled bool `mapstructure:"cache_enabled"` } // CDIConfig holds CDI (Containerized Data Importer) configuration parameters. @@ -310,6 +314,7 @@ func setDefaults() { viper.SetDefault("kubevirt.api_version", "v1") viper.SetDefault("kubevirt.timeout", 30) viper.SetDefault("kubevirt.allow_insecure_tls", false) + viper.SetDefault("kubevirt.cache_enabled", true) viper.SetDefault("datavolume.storage_size", "10Gi") viper.SetDefault("datavolume.allow_insecure_tls", false) viper.SetDefault("datavolume.vm_update_timeout", "30s") @@ -690,6 +695,13 @@ func (c *Config) GetChassisForUser(username string) ([]*ChassisConfig, error) { return nil, fmt.Errorf("user not found: %s", username) } +// GetVMCacheEnabled reports whether read-only VM/VMI lookups may be served +// from the informer cache. Discovered by the kubevirt client the same way as +// GetDataVolumeConfig. +func (c *Config) GetVMCacheEnabled() bool { + return c.KubeVirt.CacheEnabled +} + // GetDataVolumeConfig returns the DataVolume configuration settings. // This provides a safe way to access DataVolume settings without reflection. func (c *Config) GetDataVolumeConfig() (storageSize string, allowInsecureTLS bool, storageClass string, vmUpdateTimeout string, isoDownloadTimeout string, helperImage string) { @@ -764,6 +776,7 @@ authentication: kubevirt: api_version: "v1" # KubeVirt API version to use timeout: 30 # Timeout in seconds for KubeVirt operations + cache_enabled: true # Serve read-only VM/VMI lookups from an informer cache # DataVolume Configuration # ------------------------ diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index a74626b..875de83 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -104,6 +104,75 @@ kubevirt: } } +// minimalConfig is the smallest config that passes validation, with the +// kubevirt section supplied by the caller. +func minimalConfig(kubevirtSection string) string { + return ` +server: + host: "127.0.0.1" + port: 8443 + tls: + enabled: false + +chassis: + - name: "test-cluster" + namespace: "test-namespace" + service_account: "test-sa" + +authentication: + users: + - username: "testuser" + password: "testpass" + chassis: ["test-cluster"] + +kubevirt: + api_version: "v1" + timeout: 30 +` + kubevirtSection +} + +func writeConfig(t *testing.T, content string) string { + t.Helper() + configPath := filepath.Join(t.TempDir(), "config.yaml") + if err := os.WriteFile(configPath, []byte(content), 0600); err != nil { + t.Fatalf("Failed to write test config: %v", err) + } + return configPath +} + +func TestVMCacheConfig(t *testing.T) { + t.Run("enabled by default", func(t *testing.T) { + config, err := LoadConfig(writeConfig(t, minimalConfig(""))) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + if !config.GetVMCacheEnabled() { + t.Error("Expected VM cache enabled by default") + } + }) + + t.Run("from config file", func(t *testing.T) { + config, err := LoadConfig(writeConfig(t, minimalConfig(" cache_enabled: false\n"))) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + if config.GetVMCacheEnabled() { + t.Error("Expected VM cache disabled by config file") + } + }) + + t.Run("from environment", func(t *testing.T) { + t.Setenv("KUBEVIRT_REDFISH_KUBEVIRT_CACHE_ENABLED", "false") + config, err := LoadConfig(writeConfig(t, minimalConfig(""))) + if err != nil { + t.Fatalf("Failed to load config: %v", err) + } + if config.GetVMCacheEnabled() { + t.Error("Expected VM cache disabled from env") + } + }) +} + func TestLoadConfigWithDefaults(t *testing.T) { // Create a temporary directory and config file tempDir := t.TempDir() diff --git a/pkg/kubevirt/cache.go b/pkg/kubevirt/cache.go new file mode 100644 index 0000000..e90f394 --- /dev/null +++ b/pkg/kubevirt/cache.go @@ -0,0 +1,601 @@ +/* + * This file is part of the KubeVirt Redfish project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2025 KubeVirt Redfish project and its authors. + * + */ + +package kubevirt + +// A read-only cache of VirtualMachine and VirtualMachineInstance objects. +// +// Every Redfish request fans out to several Kubernetes API calls, and nearly +// all of them are reads of the same VM/VMI objects (a GET System is five GETs, +// a boot-device PATCH is eleven). With many BareMetalHosts polling at once, +// those reads are what saturate the client-side rate limiter. The informers +// here keep one watch per namespace per resource open instead, so the +// read-only Redfish paths are served from memory. +// +// Only read-only inventory lookups use the cache (see getVMForRead and its +// callers). Every read-modify-write path still reads from the API server, +// because a patch computed from a stale object would clobber fields written +// moments earlier. Two further rules keep cached reads correct: +// +// - A cache that has not finished its initial sync is not consulted at all, +// and once it has synced, its answer (including "not found") is +// authoritative. Falling back to a GET on every miss would cost a GET on +// every power poll of a stopped VM, which has no VMI, and that is the +// dominant state of a BMH during inspection and provisioning. +// - After this process writes a VM or VMI, reads of that object bypass the +// cache until the informer has received the resourceVersion the write +// returned (or, for a delete, until the cached copy is gone or marked +// for deletion). A read that follows our own write therefore never sees +// the pre-write state, however far the watch lags. writeBypassMax bounds +// this, so a watch that broke and re-listed past our version cannot force +// direct reads forever. +// +// Writes made by anyone else (oc edit, KubeVirt's own controllers) reach the +// cache when their watch event arrives, usually milliseconds later. Until then +// a cached read returns the previous version. That is the price of the cache; +// it is acceptable because no write path reads from it. + +import ( + "context" + "fmt" + "sort" + "sync" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/informers" + toolscache "k8s.io/client-go/tools/cache" + kubevirtv1 "kubevirt.io/api/core/v1" + + "github.com/kubevirt/redfish-controller/pkg/logger" +) + +var ( + vmGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + vmiGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachineinstances"} +) + +// writeBypassMax caps how long a write the cache has not yet reflected keeps +// reads of that object direct. On a healthy API server the informer sees our +// write within milliseconds and the bypass ends then; this only matters when +// the watch broke and the re-list skipped past our version. +const writeBypassMax = 30 * time.Second + +// seenVersionsMax caps how many recently stored versions the write tracker +// remembers per object. +const seenVersionsMax = 32 + +// pendingWrite is a write by this process that the cache may not reflect yet. +type pendingWrite struct { + resourceVersion string // returned by the API server; empty for a delete + deleted bool + at time.Time +} + +type seenVersion struct { + resourceVersion string + at time.Time +} + +// writeTracker remembers the VM and VMI writes by this process that the cache +// has not reflected yet. +// +// It also remembers the versions the informers stored recently. On a real API +// server the watch event for our own write often reaches the informer before +// the write call returns, and KubeVirt's controllers write a newer status +// within milliseconds. Without that history the write would be recorded after +// the cache had already seen it, and the cache would hold only the newer +// version, so reads would bypass it until writeBypassMax. +type writeTracker struct { + mu sync.Mutex + pending map[string]pendingWrite + seen map[string][]seenVersion + lastSweep time.Time + now func() time.Time +} + +func newWriteTracker() *writeTracker { + return &writeTracker{ + pending: make(map[string]pendingWrite), + seen: make(map[string][]seenVersion), + now: time.Now, + } +} + +func writeKey(gvr schema.GroupVersionResource, namespace, name string) string { + return gvr.Resource + "/" + namespace + "/" + name +} + +// sweepLocked drops pending writes and seen versions older than +// writeBypassMax. It scans everything, so it runs at most once per +// writeBypassMax. +func (t *writeTracker) sweepLocked(now time.Time) { + if now.Sub(t.lastSweep) <= writeBypassMax { + return + } + t.lastSweep = now + for key, p := range t.pending { + if now.Sub(p.at) > writeBypassMax { + delete(t.pending, key) + } + } + for key, versions := range t.seen { + if now.Sub(versions[len(versions)-1].at) > writeBypassMax { + delete(t.seen, key) + } + } +} + +func (t *writeTracker) seenLocked(key, resourceVersion string) bool { + for _, v := range t.seen[key] { + if v.resourceVersion == resourceVersion { + return true + } + } + return false +} + +func (t *writeTracker) record(gvr schema.GroupVersionResource, namespace, name string, w pendingWrite) { + if t == nil || (gvr != vmGVR && gvr != vmiGVR) { + return + } + t.mu.Lock() + defer t.mu.Unlock() + w.at = t.now() + t.sweepLocked(w.at) + key := writeKey(gvr, namespace, name) + if !w.deleted && w.resourceVersion != "" && t.seenLocked(key, w.resourceVersion) { + // The cache already stored this write; an older pending write of + // ours is superseded by it. + delete(t.pending, key) + return + } + t.pending[key] = w +} + +// note records a successful create, update or patch and the resourceVersion +// the API server returned for it. Resources other than VM and VMI are ignored. +func (t *writeTracker) note(gvr schema.GroupVersionResource, namespace, name, resourceVersion string) { + t.record(gvr, namespace, name, pendingWrite{resourceVersion: resourceVersion}) +} + +func (t *writeTracker) noteDelete(gvr schema.GroupVersionResource, namespace, name string) { + t.record(gvr, namespace, name, pendingWrite{deleted: true}) +} + +// observed is called by the informers for every object version they store. +// Seeing the version our write produced means the cache has reflected it, +// even if a newer version has replaced it by the time anyone reads. +func (t *writeTracker) observed(gvr schema.GroupVersionResource, namespace, name, resourceVersion string) { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + now := t.now() + t.sweepLocked(now) + key := writeKey(gvr, namespace, name) + if p, ok := t.pending[key]; ok && !p.deleted && p.resourceVersion != "" && p.resourceVersion == resourceVersion { + delete(t.pending, key) + } + + versions := t.seen[key] + for len(versions) > 0 && (len(versions) >= seenVersionsMax || now.Sub(versions[0].at) > writeBypassMax) { + versions = versions[1:] + } + t.seen[key] = append(versions, seenVersion{resourceVersion: resourceVersion, at: now}) +} + +// forget drops the versions remembered for an object the informer deleted, +// so the history does not grow with every VMI ever created. +func (t *writeTracker) forget(gvr schema.GroupVersionResource, namespace, name string) { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + delete(t.seen, writeKey(gvr, namespace, name)) +} + +// cacheReflects reports whether the cached copy (nil when the cache has none) +// may be served, given this process's writes to the object. +func (t *writeTracker) cacheReflects(gvr schema.GroupVersionResource, namespace, name string, cached *unstructured.Unstructured) bool { + if t == nil { + return true + } + t.mu.Lock() + defer t.mu.Unlock() + key := writeKey(gvr, namespace, name) + p, ok := t.pending[key] + if !ok { + return true + } + + var caughtUp bool + if p.deleted { + caughtUp = cached == nil || cached.GetDeletionTimestamp() != nil + } else { + // Equal versions also cover a no-op write, which the API server + // acknowledges with the current version and no watch event. + caughtUp = cached != nil && p.resourceVersion != "" && cached.GetResourceVersion() == p.resourceVersion + } + if caughtUp { + delete(t.pending, key) + return true + } + if t.now().Sub(p.at) > writeBypassMax { + delete(t.pending, key) + logger.Warning("VM cache has not reflected this process's write to %s within %s; serving it from the cache again", key, writeBypassMax) + return true + } + return false +} + +// trackingDynamicClient wraps a dynamic client so that every successful write +// to a VM or VMI is recorded in the writeTracker. Reads pass straight through. +type trackingDynamicClient struct { + dynamic.Interface + tracker *writeTracker +} + +func newTrackingDynamicClient(inner dynamic.Interface, tracker *writeTracker) dynamic.Interface { + if inner == nil { + return nil + } + return &trackingDynamicClient{Interface: inner, tracker: tracker} +} + +func (d *trackingDynamicClient) Resource(gvr schema.GroupVersionResource) dynamic.NamespaceableResourceInterface { + return &trackingNamespaceableResource{ + NamespaceableResourceInterface: d.Interface.Resource(gvr), + gvr: gvr, + tracker: d.tracker, + } +} + +type trackingNamespaceableResource struct { + dynamic.NamespaceableResourceInterface + gvr schema.GroupVersionResource + tracker *writeTracker +} + +func (r *trackingNamespaceableResource) Namespace(namespace string) dynamic.ResourceInterface { + return &trackingResource{ + ResourceInterface: r.NamespaceableResourceInterface.Namespace(namespace), + gvr: r.gvr, + namespace: namespace, + tracker: r.tracker, + } +} + +type trackingResource struct { + dynamic.ResourceInterface + gvr schema.GroupVersionResource + namespace string + tracker *writeTracker +} + +func (r *trackingResource) Create(ctx context.Context, obj *unstructured.Unstructured, options metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) { + out, err := r.ResourceInterface.Create(ctx, obj, options, subresources...) + if err == nil { + r.tracker.note(r.gvr, r.namespace, obj.GetName(), out.GetResourceVersion()) + } + return out, err +} + +func (r *trackingResource) Update(ctx context.Context, obj *unstructured.Unstructured, options metav1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) { + out, err := r.ResourceInterface.Update(ctx, obj, options, subresources...) + if err == nil { + r.tracker.note(r.gvr, r.namespace, obj.GetName(), out.GetResourceVersion()) + } + return out, err +} + +func (r *trackingResource) UpdateStatus(ctx context.Context, obj *unstructured.Unstructured, options metav1.UpdateOptions) (*unstructured.Unstructured, error) { + out, err := r.ResourceInterface.UpdateStatus(ctx, obj, options) + if err == nil { + r.tracker.note(r.gvr, r.namespace, obj.GetName(), out.GetResourceVersion()) + } + return out, err +} + +func (r *trackingResource) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) { + out, err := r.ResourceInterface.Patch(ctx, name, pt, data, options, subresources...) + if err == nil { + r.tracker.note(r.gvr, r.namespace, name, out.GetResourceVersion()) + } + return out, err +} + +func (r *trackingResource) Delete(ctx context.Context, name string, options metav1.DeleteOptions, subresources ...string) error { + err := r.ResourceInterface.Delete(ctx, name, options, subresources...) + if err == nil { + r.tracker.noteDelete(r.gvr, r.namespace, name) + } + return err +} + +// vmCache holds one VM informer and one VMI informer per configured namespace. +type vmCache struct { + stopCh chan struct{} + factories []dynamicinformer.DynamicSharedInformerFactory + vms map[string]informers.GenericInformer + vmis map[string]informers.GenericInformer +} + +// newVMCache starts informers for every distinct, non-empty namespace. It does +// not wait for them to sync; readers check HasSynced per call. Every version +// the informers store is reported to writes. +func newVMCache(client dynamic.Interface, namespaces []string, writes *writeTracker) *vmCache { + vc := &vmCache{ + stopCh: make(chan struct{}), + vms: make(map[string]informers.GenericInformer), + vmis: make(map[string]informers.GenericInformer), + } + seen := make(map[string]bool) + for _, namespace := range namespaces { + if namespace == "" || seen[namespace] { + continue + } + seen[namespace] = true + + factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, namespace, nil) + vmInformer := factory.ForResource(vmGVR) + vmiInformer := factory.ForResource(vmiGVR) + // Transforms and handlers must be set before the factory starts. + for gvr, inf := range map[schema.GroupVersionResource]informers.GenericInformer{vmGVR: vmInformer, vmiGVR: vmiInformer} { + if err := inf.Informer().SetTransform(stripManagedFields); err != nil { + logger.Error("Failed to set informer transform for namespace %s: %v", namespace, err) + } + if _, err := inf.Informer().AddEventHandler(observeVersions(gvr, writes)); err != nil { + logger.Error("Failed to add informer handler for namespace %s: %v", namespace, err) + } + } + vc.vms[namespace] = vmInformer + vc.vmis[namespace] = vmiInformer + vc.factories = append(vc.factories, factory) + factory.Start(vc.stopCh) + } + return vc +} + +// stop closes the stop channel first so the informer goroutines exit, then +// waits for them. The other order deadlocks. +func (vc *vmCache) stop() { + if vc == nil { + return + } + close(vc.stopCh) + for _, factory := range vc.factories { + factory.Shutdown() + } +} + +// informer returns the synced informer for gvr in namespace, or false when +// there is none or it has not finished its initial list. +func (vc *vmCache) informer(gvr schema.GroupVersionResource, namespace string) (informers.GenericInformer, bool) { + if vc == nil { + return nil, false + } + m := vc.vms + if gvr == vmiGVR { + m = vc.vmis + } + inf, ok := m[namespace] + if !ok || !inf.Informer().HasSynced() { + return nil, false + } + return inf, true +} + +// observeVersions reports every version an informer stores to the write +// tracker, and every object it deletes so the tracker can forget it. The +// shared informer updates its store before it notifies handlers, so once a +// version is observed here, the store holds it or a newer one. +func observeVersions(gvr schema.GroupVersionResource, writes *writeTracker) toolscache.ResourceEventHandler { + report := func(obj interface{}) { + if m, err := meta.Accessor(obj); err == nil { + writes.observed(gvr, m.GetNamespace(), m.GetName(), m.GetResourceVersion()) + } + } + return toolscache.ResourceEventHandlerFuncs{ + AddFunc: report, + UpdateFunc: func(_, obj interface{}) { report(obj) }, + DeleteFunc: func(obj interface{}) { + if tombstone, ok := obj.(toolscache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + if m, err := meta.Accessor(obj); err == nil { + writes.forget(gvr, m.GetNamespace(), m.GetName()) + } + }, + } +} + +// stripManagedFields drops the managedFields block before an object enters +// the store; it is the largest part of a VM object and nothing here reads it. +func stripManagedFields(obj interface{}) (interface{}, error) { + if u, ok := obj.(*unstructured.Unstructured); ok { + u.SetManagedFields(nil) + } + return obj, nil +} + +// vmCacheEnabled reads kubevirt.cache_enabled from the app config; the cache +// is on unless the config says otherwise. +func (c *Client) vmCacheEnabled() bool { + if c.appConfig == nil { + return true + } + if cfg, ok := c.appConfig.(interface{ GetVMCacheEnabled() bool }); ok { + return cfg.GetVMCacheEnabled() + } + return true +} + +// restartVMCache replaces the running cache with one covering namespaces. +// Called with the watchers on startup and on config reload. +func (c *Client) restartVMCache(namespaces []string) { + c.cacheMu.Lock() + defer c.cacheMu.Unlock() + + if c.vmCache != nil { + c.vmCache.stop() + c.vmCache = nil + } + if !c.vmCacheEnabled() { + logger.Info("VM cache disabled by configuration; VM/VMI reads go to the API server") + return + } + if c.dynamicClient == nil { + return + } + c.vmCache = newVMCache(c.dynamicClient, namespaces, c.writes) + logger.Info("VM cache started for namespaces %v", namespaces) +} + +func (c *Client) stopVMCache() { + c.cacheMu.Lock() + defer c.cacheMu.Unlock() + if c.vmCache != nil { + c.vmCache.stop() + c.vmCache = nil + } +} + +func (c *Client) currentVMCache() *vmCache { + c.cacheMu.RLock() + defer c.cacheMu.RUnlock() + return c.vmCache +} + +// GetVMForRead returns a VM for a read-only purpose. It is served from the +// informer cache when the cache is synced for the namespace and reflects every +// write this process made to the VM; otherwise it is a GetVM. Callers that go +// on to patch the VM must use GetVM. +func (c *Client) GetVMForRead(namespace, name string) (*kubevirtv1.VirtualMachine, error) { + return c.getVMForRead(namespace, name) +} + +func (c *Client) getVMForRead(namespace, name string) (*kubevirtv1.VirtualMachine, error) { + start := time.Now() + if u, served, err := c.cachedObject(vmGVR, namespace, name); served { + c.trackOperation("GetVM.cache", time.Since(start)) + if err != nil { + return nil, fmt.Errorf("failed to get VM: %w", err) + } + return unstructuredToVM(u) + } + c.trackOperation("GetVM.direct", time.Since(start)) + return c.GetVM(namespace, name) +} + +// getVMIForRead is GetVMI served from the cache under the same rules as +// getVMForRead. A synced cache's "not found" means the VM has no running +// instance, which is what callers use to report power state Off. +func (c *Client) getVMIForRead(namespace, name string) (*kubevirtv1.VirtualMachineInstance, error) { + start := time.Now() + if u, served, err := c.cachedObject(vmiGVR, namespace, name); served { + c.trackOperation("GetVMI.cache", time.Since(start)) + if err != nil { + return nil, fmt.Errorf("failed to get VMI %s: %w", name, err) + } + return unstructuredToVMI(u) + } + c.trackOperation("GetVMI.direct", time.Since(start)) + return c.GetVMI(namespace, name) +} + +// cachedObject looks the object up in the cache. served is false when the +// caller must ask the API server instead: the cache is not synced for the +// namespace, the lister failed, or the cache does not yet reflect a write by +// this process. When served, err is a NotFound error if the object is absent, +// and the returned object is a copy the caller may modify. +func (c *Client) cachedObject(gvr schema.GroupVersionResource, namespace, name string) (u *unstructured.Unstructured, served bool, err error) { + inf, ok := c.currentVMCache().informer(gvr, namespace) + if !ok { + return nil, false, nil + } + obj, err := inf.Lister().ByNamespace(namespace).Get(name) + switch { + case err == nil: + if u, ok = obj.(*unstructured.Unstructured); !ok { + return nil, false, nil + } + case !apierrors.IsNotFound(err): + return nil, false, nil + } + if !c.writes.cacheReflects(gvr, namespace, name, u) { + return nil, false, nil + } + if u == nil { + return nil, true, err + } + return u.DeepCopy(), true, nil +} + +// listVMNamesFromCache returns the names of the VMs in namespace matching +// selector, or false when the cache is not synced for that namespace. +func (c *Client) listVMNamesFromCache(namespace string, selector labels.Selector) ([]string, bool) { + inf, ok := c.currentVMCache().informer(vmGVR, namespace) + if !ok { + return nil, false + } + objs, err := inf.Lister().ByNamespace(namespace).List(selector) + if err != nil { + return nil, false + } + names := make([]string, 0, len(objs)) + for _, obj := range objs { + if u, ok := obj.(*unstructured.Unstructured); ok { + names = append(names, u.GetName()) + } + } + c.trackOperation("ListVMs.cache", 0) + return names, true +} + +// filterVMNames applies the selector's explicit name list, if any, and sorts +// the result so collections render in a stable order regardless of source. +func filterVMNames(names []string, vmSelector *VMSelectorConfig) []string { + var out []string + if vmSelector != nil && len(vmSelector.Names) > 0 { + allowed := make(map[string]bool, len(vmSelector.Names)) + for _, n := range vmSelector.Names { + allowed[n] = true + } + for _, n := range names { + if allowed[n] { + out = append(out, n) + } + } + } else { + out = append(out, names...) + } + sort.Strings(out) + return out +} diff --git a/pkg/kubevirt/cache_test.go b/pkg/kubevirt/cache_test.go new file mode 100644 index 0000000..d9b6b1e --- /dev/null +++ b/pkg/kubevirt/cache_test.go @@ -0,0 +1,480 @@ +/* + * This file is part of the KubeVirt Redfish project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Copyright 2025 KubeVirt Redfish project and its authors. + * + */ + +package kubevirt + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "sync" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" +) + +func testVM(namespace, name string, lbls map[string]string) *unstructured.Unstructured { + u := &unstructured.Unstructured{} + u.SetAPIVersion("kubevirt.io/v1") + u.SetKind("VirtualMachine") + u.SetNamespace(namespace) + u.SetName(name) + u.SetLabels(lbls) + return u +} + +// newCacheTestClient builds a Client over a fake dynamic client, wrapped by +// the write tracker exactly as the constructors do. +func newCacheTestClient(t *testing.T, appConfig interface{}, objs ...runtime.Object) (*Client, *dynamicfake.FakeDynamicClient) { + t.Helper() + fake := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ + vmGVR: "VirtualMachineList", + vmiGVR: "VirtualMachineInstanceList", + }, objs...) + writes := newWriteTracker() + c := &Client{ + dynamicClient: newTrackingDynamicClient(fake, writes), + writes: writes, + timeout: 5 * time.Second, + appConfig: appConfig, + } + c.metrics.operationCounts = make(map[string]int64) + c.metrics.operationTimes = make(map[string]time.Duration) + return c, fake +} + +func waitSynced(t *testing.T, c *Client, namespace string) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + _, vmOK := c.currentVMCache().informer(vmGVR, namespace) + _, vmiOK := c.currentVMCache().informer(vmiGVR, namespace) + if vmOK && vmiOK { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("cache for namespace %s did not sync", namespace) +} + +func failGets(fake *dynamicfake.FakeDynamicClient, msg string) { + fake.PrependReactor("get", "virtualmachines", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New(msg) + }) + fake.PrependReactor("get", "virtualmachineinstances", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New(msg) + }) +} + +func TestVMCacheServesReads(t *testing.T) { + c, fake := newCacheTestClient(t, nil, + testVM("ns", "vm-a", map[string]string{"redfish-enabled": "true"}), + testVM("ns", "vm-b", nil), + ) + // Duplicate and empty namespaces must be tolerated. + c.restartVMCache([]string{"ns", "ns", ""}) + defer c.stopVMCache() + waitSynced(t, c, "ns") + + // From here on any GET that reaches the API server fails, so a + // successful read proves the cache is on the path. + failGets(fake, "unexpected API server GET") + + vm, err := c.getVMForRead("ns", "vm-a") + if err != nil { + t.Fatalf("cached read failed: %v", err) + } + if vm.Name != "vm-a" || vm.Namespace != "ns" { + t.Fatalf("unexpected VM from cache: %s/%s", vm.Namespace, vm.Name) + } + + // A synced cache's not-found is authoritative: no fallback GET. + if _, err := c.getVMForRead("ns", "missing"); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound from cache, got %v", err) + } + + // No VMI exists for vm-a, which callers read as power state Off. + if _, err := c.getVMIForRead("ns", "vm-a"); !apierrors.IsNotFound(err) { + t.Fatalf("expected NotFound VMI from cache, got %v", err) + } + + // The direct path still goes to the API server. + if _, err := c.GetVM("ns", "vm-a"); err == nil { + t.Fatal("expected direct GetVM to reach the API server and fail") + } + + // A namespace the cache does not cover is not served. + if _, err := c.getVMForRead("other", "vm-a"); err == nil || !strings.Contains(err.Error(), "unexpected API server GET") { + t.Fatalf("expected fallback to API server for uncovered namespace, got %v", err) + } + + // List honours the label selector and reports coverage. + names, ok := c.listVMNamesFromCache("ns", labels.SelectorFromSet(labels.Set{"redfish-enabled": "true"})) + if !ok || !reflect.DeepEqual(names, []string{"vm-a"}) { + t.Fatalf("selector list: ok=%v names=%v", ok, names) + } + names, ok = c.listVMNamesFromCache("ns", labels.Everything()) + if !ok || len(names) != 2 { + t.Fatalf("everything list: ok=%v names=%v", ok, names) + } + if _, ok := c.listVMNamesFromCache("other", labels.Everything()); ok { + t.Fatal("uncovered namespace must not be served from cache") + } +} + +// fakeClock is the write tracker's clock in tests. +type fakeClock struct { + mu sync.Mutex + now time.Time +} + +func newFakeClock() *fakeClock { return &fakeClock{now: time.Unix(1_700_000_000, 0)} } + +func (f *fakeClock) Now() time.Time { + f.mu.Lock() + defer f.mu.Unlock() + return f.now +} + +func (f *fakeClock) Advance(d time.Duration) { + f.mu.Lock() + f.now = f.now.Add(d) + f.mu.Unlock() +} + +func withRV(u *unstructured.Unstructured, rv string) *unstructured.Unstructured { + u.SetResourceVersion(rv) + return u +} + +// newWriteTestClient returns a synced client over VMs vm-a and vm-b at +// resourceVersion 1, with a fake clock and every API server GET failing, so +// readsFromCache can tell a cached read from a direct one. +func newWriteTestClient(t *testing.T) (*Client, *dynamicfake.FakeDynamicClient, *fakeClock) { + t.Helper() + c, fake := newCacheTestClient(t, nil, + withRV(testVM("ns", "vm-a", nil), "1"), + withRV(testVM("ns", "vm-b", nil), "1"), + ) + clock := newFakeClock() + c.writes.now = clock.Now + c.restartVMCache([]string{"ns"}) + t.Cleanup(c.stopVMCache) + waitSynced(t, c, "ns") + failGets(fake, "api server GET") + return c, fake, clock +} + +func readsFromCache(c *Client, name string) bool { + _, err := c.getVMForRead("ns", name) + return err == nil || !strings.Contains(err.Error(), "api server GET") +} + +// eventually polls cond, because informer event delivery is asynchronous +// even over the fake client. +func eventually(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("timed out waiting until %s", what) +} + +// acceptUnseen makes the API server acknowledge a verb on VM name, answering +// with resourceVersion rv, without storing anything. The informer never hears +// of it: the state of a cache whose watch lags behind our write. +func acceptUnseen(fake *dynamicfake.FakeDynamicClient, verb, name, rv string) { + fake.PrependReactor(verb, "virtualmachines", func(a k8stesting.Action) (bool, runtime.Object, error) { + var actionName string + switch a := a.(type) { + case k8stesting.PatchAction: + actionName = a.GetName() + case k8stesting.DeleteAction: + actionName = a.GetName() + } + if actionName != name { + return false, nil, nil + } + if verb == "delete" { + return true, nil, nil + } + return true, withRV(testVM("ns", name, nil), rv), nil + }) +} + +func patchVM(t *testing.T, c *Client, name string) { + t.Helper() + if _, err := c.dynamicClient.Resource(vmGVR).Namespace("ns").Patch(context.Background(), name, + types.MergePatchType, []byte(`{"metadata":{"labels":{"x":"y"}}}`), metav1.PatchOptions{}); err != nil { + t.Fatalf("patch %s: %v", name, err) + } +} + +func TestVMCacheBypassUntilOwnWriteIsObserved(t *testing.T) { + c, fake, clock := newWriteTestClient(t) + acceptUnseen(fake, "patch", "vm-a", "2") + patchVM(t, c, "vm-a") + + if readsFromCache(c, "vm-a") { + t.Fatal("a read after our own write must bypass a cache that has not seen it") + } + if !readsFromCache(c, "vm-b") { + t.Fatal("reads of other VMs must still come from the cache") + } + + // No fixed delay ends the bypass: while the cache lags, reads stay direct. + clock.Advance(writeBypassMax - time.Second) + if readsFromCache(c, "vm-a") { + t.Fatal("bypass ended before the cache saw our write") + } + + if err := fake.Tracker().Update(vmGVR, withRV(testVM("ns", "vm-a", nil), "2"), "ns"); err != nil { + t.Fatal(err) + } + eventually(t, "the cache serves vm-a once it holds our version", func() bool { return readsFromCache(c, "vm-a") }) + vm, err := c.getVMForRead("ns", "vm-a") + if err != nil || vm.ResourceVersion != "2" { + t.Fatalf("cached read: rv=%v err=%v", vm, err) + } +} + +func TestVMCacheBypassEndsWhenOwnWriteIsSuperseded(t *testing.T) { + c, fake, _ := newWriteTestClient(t) + acceptUnseen(fake, "patch", "vm-a", "2") + patchVM(t, c, "vm-a") + + // The informer sees our version and then a newer one written by someone + // else. The cache never holds version 2 when we read, but it has seen it. + for _, rv := range []string{"2", "3"} { + if err := fake.Tracker().Update(vmGVR, withRV(testVM("ns", "vm-a", nil), rv), "ns"); err != nil { + t.Fatal(err) + } + } + eventually(t, "the cache serves vm-a after seeing our version", func() bool { return readsFromCache(c, "vm-a") }) + vm, err := c.getVMForRead("ns", "vm-a") + if err != nil || vm.ResourceVersion != "3" { + t.Fatalf("cached read: vm=%v err=%v", vm, err) + } +} + +func TestVMCacheNoBypassWhenOwnWriteWasObservedBeforePatchReturned(t *testing.T) { + // On a real API server the watch event for our write can reach the + // informer before Patch returns, and virt-controller writes a newer status + // right after. The cache has seen our version before we record the write. + c, fake, _ := newWriteTestClient(t) + for _, rv := range []string{"2", "3"} { + if err := fake.Tracker().Update(vmGVR, withRV(testVM("ns", "vm-a", nil), rv), "ns"); err != nil { + t.Fatal(err) + } + } + // Wait on the handler, not the store: the informer updates the store + // before it notifies handlers. + eventually(t, "the informer reported versions 2 and 3", func() bool { + c.writes.mu.Lock() + defer c.writes.mu.Unlock() + key := writeKey(vmGVR, "ns", "vm-a") + return c.writes.seenLocked(key, "2") && c.writes.seenLocked(key, "3") + }) + + acceptUnseen(fake, "patch", "vm-a", "2") + patchVM(t, c, "vm-a") + if !readsFromCache(c, "vm-a") { + t.Fatal("a write the cache saw before Patch returned must not bypass it") + } +} + +func TestWriteTrackerForgetsDeletedObjects(t *testing.T) { + w := newWriteTracker() + w.observed(vmGVR, "ns", "vm-a", "2") + w.observed(vmiGVR, "ns", "vm-a", "5") + w.forget(vmiGVR, "ns", "vm-a") + + w.mu.Lock() + defer w.mu.Unlock() + if _, ok := w.seen[writeKey(vmiGVR, "ns", "vm-a")]; ok { + t.Fatal("versions of a deleted object must be dropped") + } + if _, ok := w.seen[writeKey(vmGVR, "ns", "vm-a")]; !ok { + t.Fatal("versions of other objects must be kept") + } +} + +func TestWriteTrackerSeenHistoryIsBounded(t *testing.T) { + w := newWriteTracker() + clock := newFakeClock() + w.now = clock.Now + for i := 0; i < 3*seenVersionsMax; i++ { + w.observed(vmGVR, "ns", "vm-a", fmt.Sprint(i)) + } + w.mu.Lock() + n := len(w.seen[writeKey(vmGVR, "ns", "vm-a")]) + w.mu.Unlock() + if n > seenVersionsMax { + t.Fatalf("kept %d versions, want at most %d", n, seenVersionsMax) + } + + clock.Advance(writeBypassMax + time.Second) + w.observed(vmGVR, "ns", "vm-b", "1") + w.mu.Lock() + _, stale := w.seen[writeKey(vmGVR, "ns", "vm-a")] + w.mu.Unlock() + if stale { + t.Fatal("versions older than writeBypassMax must be pruned") + } +} + +func TestVMCacheNoOpWriteDoesNotBypass(t *testing.T) { + // The API server does not bump resourceVersion or emit a watch event for + // a write that changes nothing; the cache already holds that version. + c, fake, _ := newWriteTestClient(t) + acceptUnseen(fake, "patch", "vm-a", "1") + patchVM(t, c, "vm-a") + if !readsFromCache(c, "vm-a") { + t.Fatal("a no-op write must not bypass a cache that already holds its version") + } +} + +func TestVMCacheBypassIsBounded(t *testing.T) { + c, fake, clock := newWriteTestClient(t) + acceptUnseen(fake, "patch", "vm-a", "2") + patchVM(t, c, "vm-a") + clock.Advance(writeBypassMax + time.Second) + if !readsFromCache(c, "vm-a") { + t.Fatal("a write the cache never sees must stop forcing direct reads after writeBypassMax") + } + c.writes.mu.Lock() + n := len(c.writes.pending) + c.writes.mu.Unlock() + if n != 0 { + t.Fatalf("expired write still tracked: %d entries", n) + } +} + +func TestVMCacheBypassAfterOwnDelete(t *testing.T) { + c, fake, _ := newWriteTestClient(t) + acceptUnseen(fake, "delete", "vm-a", "") + acceptUnseen(fake, "delete", "vm-b", "") + for _, name := range []string{"vm-a", "vm-b"} { + if err := c.dynamicClient.Resource(vmGVR).Namespace("ns").Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + if readsFromCache(c, name) { + t.Fatalf("a read of %s after our delete must bypass a cache that still holds it", name) + } + } + + // vm-a is removed outright; vm-b is held by a finalizer and only gains a + // deletionTimestamp. Either one shows the cache has seen the delete. + if err := fake.Tracker().Delete(vmGVR, "ns", "vm-a"); err != nil { + t.Fatal(err) + } + terminating := withRV(testVM("ns", "vm-b", nil), "2") + now := metav1.Now() + terminating.SetDeletionTimestamp(&now) + if err := fake.Tracker().Update(vmGVR, terminating, "ns"); err != nil { + t.Fatal(err) + } + eventually(t, "the cache answers NotFound for vm-a", func() bool { + _, err := c.getVMForRead("ns", "vm-a") + return apierrors.IsNotFound(err) + }) + eventually(t, "the cache serves the terminating vm-b", func() bool { return readsFromCache(c, "vm-b") }) +} + +func TestVMCacheFailedWriteDoesNotBypass(t *testing.T) { + c, fake, _ := newWriteTestClient(t) + fake.PrependReactor("patch", "virtualmachines", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("patch rejected") + }) + if _, err := c.dynamicClient.Resource(vmGVR).Namespace("ns").Patch(context.Background(), "vm-a", + types.MergePatchType, []byte(`{}`), metav1.PatchOptions{}); err == nil { + t.Fatal("expected the patch to fail") + } + if !readsFromCache(c, "vm-a") { + t.Fatal("a failed write must not bypass the cache") + } +} + +type cacheOff struct{} + +func (cacheOff) GetVMCacheEnabled() bool { return false } + +func TestVMCacheDisabledByConfig(t *testing.T) { + c, _ := newCacheTestClient(t, cacheOff{}, testVM("ns", "vm-a", nil)) + c.restartVMCache([]string{"ns"}) + defer c.stopVMCache() + if c.currentVMCache() != nil { + t.Fatal("cache must not start when disabled by config") + } + vm, err := c.getVMForRead("ns", "vm-a") + if err != nil || vm.Name != "vm-a" { + t.Fatalf("direct read with cache disabled: vm=%v err=%v", vm, err) + } +} + +func TestVMCacheRestartAndStop(t *testing.T) { + c, _ := newCacheTestClient(t, nil, testVM("ns1", "a", nil), testVM("ns2", "b", nil)) + c.restartVMCache([]string{"ns1"}) + waitSynced(t, c, "ns1") + if _, ok := c.currentVMCache().informer(vmGVR, "ns2"); ok { + t.Fatal("ns2 should not be covered yet") + } + // Config reload: a new namespace set replaces the old cache without deadlocking. + c.restartVMCache([]string{"ns2"}) + waitSynced(t, c, "ns2") + if _, ok := c.currentVMCache().informer(vmGVR, "ns1"); ok { + t.Fatal("ns1 should no longer be covered") + } + c.stopVMCache() + if c.currentVMCache() != nil { + t.Fatal("cache should be nil after stop") + } + // Reads after stop fall back to the API server. + if _, err := c.getVMForRead("ns2", "b"); err != nil { + t.Fatalf("direct read after stop: %v", err) + } +} + +func TestFilterVMNames(t *testing.T) { + in := []string{"c", "a", "b"} + if got := filterVMNames(in, nil); !reflect.DeepEqual(got, []string{"a", "b", "c"}) { + t.Fatalf("nil selector: %v", got) + } + if got := filterVMNames(in, &VMSelectorConfig{Names: []string{"b", "zzz"}}); !reflect.DeepEqual(got, []string{"b"}) { + t.Fatalf("names filter: %v", got) + } + if got := filterVMNames(nil, &VMSelectorConfig{Names: []string{"b"}}); got != nil { + t.Fatalf("empty input should stay nil, got %v", got) + } +} diff --git a/pkg/kubevirt/client.go b/pkg/kubevirt/client.go index 26ad6c2..bd03330 100644 --- a/pkg/kubevirt/client.go +++ b/pkg/kubevirt/client.go @@ -83,6 +83,11 @@ type Client struct { operationTimes map[string]time.Duration } + // Read-only VM/VMI cache and the write tracker that keeps it honest (cache.go) + cacheMu sync.RWMutex + vmCache *vmCache + writes *writeTracker + // Boot-once watcher management watcherCtx context.Context watcherCancel context.CancelFunc @@ -231,13 +236,17 @@ func NewClient(configPath string, timeout time.Duration, appConfig interface{}) }, } + // Every VM/VMI write goes through the tracker so cached reads can skip + // the cache right after this process wrote the object (see cache.go). + writes := newWriteTracker() return &Client{ kubernetesClient: kubernetesClient, - dynamicClient: dynamicClient, + dynamicClient: newTrackingDynamicClient(dynamicClient, writes), config: config, timeout: timeout, appConfig: appConfig, httpClient: httpClient, + writes: writes, }, nil } @@ -249,9 +258,11 @@ func NewClientWithClients( timeout time.Duration, appConfig interface{}, ) *Client { + writes := newWriteTracker() return &Client{ kubernetesClient: kubernetesClient, - dynamicClient: dynamicClient, + dynamicClient: newTrackingDynamicClient(dynamicClient, writes), + writes: writes, timeout: timeout, appConfig: appConfig, config: &rest.Config{}, @@ -310,6 +321,7 @@ func (c *Client) Close() error { c.stopBootOnceWatcher() c.stopImportPodWatcher() c.stopCDIPVCWatcher() + c.stopVMCache() if c.httpClient != nil && c.httpClient.Transport != nil { if transport, ok := c.httpClient.Transport.(*http.Transport); ok { @@ -328,6 +340,7 @@ func (c *Client) RestartWatchers(ctx context.Context, namespaces []string) { c.StartBootOnceWatcher(ctx, namespaces) c.StartImportPodWatcher(ctx, namespaces) c.StartCDIPVCWatcher(ctx, namespaces) + c.restartVMCache(namespaces) } // retryWithBackoff executes a function with exponential backoff retry logic @@ -462,9 +475,9 @@ func (c *Client) ListVMsWithSelector(namespace string, vmSelector *VMSelectorCon } // Build list options with label selector if provided - listOptions := metav1.ListOptions{} + selector := labels.Everything() if vmSelector != nil && len(vmSelector.Labels) > 0 { - selector := labels.NewSelector() + selector = labels.NewSelector() for key, value := range vmSelector.Labels { req, err := labels.NewRequirement(key, selection.Equals, []string{value}) if err != nil { @@ -473,34 +486,31 @@ func (c *Client) ListVMsWithSelector(namespace string, vmSelector *VMSelectorCon } selector = selector.Add(*req) } - listOptions.LabelSelector = selector.String() logger.Debug("Using label selector: %s", selector.String()) } - vms, err := c.dynamicClient.Resource(gvr).Namespace(namespace).List(ctx, listOptions) - if err != nil { - logger.Error("Failed to list VMs in namespace %s: %v", namespace, err) - return nil, fmt.Errorf("failed to list VMs: %w", err) - } - - var vmNames []string - for _, vm := range vms.Items { - vmName := vm.GetName() - - // If explicit names are specified, only include those VMs - if vmSelector != nil && len(vmSelector.Names) > 0 { - for _, allowedName := range vmSelector.Names { - if vmName == allowedName { - vmNames = append(vmNames, vmName) - break - } - } - } else { - // Include all VMs if no explicit names specified - vmNames = append(vmNames, vmName) + // Serve from the informer cache when it is synced for this namespace; + // otherwise list from the API server. + names, ok := c.listVMNamesFromCache(namespace, selector) + if !ok { + listOptions := metav1.ListOptions{} + if !selector.Empty() { + listOptions.LabelSelector = selector.String() + } + vms, err := c.dynamicClient.Resource(gvr).Namespace(namespace).List(ctx, listOptions) + if err != nil { + logger.Error("Failed to list VMs in namespace %s: %v", namespace, err) + return nil, fmt.Errorf("failed to list VMs: %w", err) + } + names = make([]string, 0, len(vms.Items)) + for _, vm := range vms.Items { + names = append(names, vm.GetName()) } } + // If explicit names are specified, only include those VMs + vmNames := filterVMNames(names, vmSelector) + logger.Info("Found %d VMs in namespace %s with selector: %v", len(vmNames), namespace, vmNames) return vmNames, nil } @@ -645,7 +655,7 @@ func (c *Client) GetVMPowerState(namespace, name string) (string, error) { } // Fetch the VM resource using typed API - vm, err := c.GetVM(namespace, name) + vm, err := c.getVMForRead(namespace, name) if err != nil { return "Unknown", fmt.Errorf("failed to get VM %s: %w", name, err) } @@ -685,7 +695,7 @@ func (c *Client) GetVMPowerState(namespace, name string) (string, error) { } // Fallback to VMI phase logic - vmi, err := c.GetVMI(namespace, name) + vmi, err := c.getVMIForRead(namespace, name) if err != nil { // If VMI doesn't exist, VM is stopped return "Off", nil @@ -1337,7 +1347,7 @@ func (c *Client) GetVMNetworkInterfaces(namespace, name string) ([]string, error return nil, fmt.Errorf("dynamic client is not initialized") } - vmi, err := c.GetVMI(namespace, name) + vmi, err := c.getVMIForRead(namespace, name) if err != nil { return nil, fmt.Errorf("failed to get VMI %s: %w", name, err) } @@ -1354,7 +1364,7 @@ func (c *Client) GetVMNetworkInterfaces(namespace, name string) ([]string, error // GetVMStorage gets storage information of a VirtualMachine func (c *Client) GetVMStorage(namespace, name string) ([]string, error) { - vm, err := c.GetVM(namespace, name) + vm, err := c.getVMForRead(namespace, name) if err != nil { return nil, err } @@ -1375,7 +1385,7 @@ func (c *Client) GetVMStorage(namespace, name string) ([]string, error) { // GetVMBootOptions gets boot options of a VirtualMachine func (c *Client) GetVMBootOptions(namespace, name string) (map[string]interface{}, error) { - vm, err := c.GetVM(namespace, name) + vm, err := c.getVMForRead(namespace, name) if err != nil { return nil, err } @@ -1477,7 +1487,7 @@ func (c *Client) RecordVMBootOptionsAsAnnotations(namespace, name string, option // GetVMVirtualMedia gets virtual media information of a VirtualMachine func (c *Client) GetVMVirtualMedia(namespace, name string) ([]string, error) { - vm, err := c.GetVM(namespace, name) + vm, err := c.getVMForRead(namespace, name) if err != nil { return nil, fmt.Errorf("failed to get VM %s: %w", name, err) } @@ -1503,7 +1513,7 @@ func (c *Client) GetVMVirtualMedia(namespace, name string) ([]string, error) { // IsVirtualMediaInserted checks if a specific virtual media device has media inserted and ready func (c *Client) IsVirtualMediaInserted(namespace, name, mediaID string) (bool, error) { - vm, err := c.GetVM(namespace, name) + vm, err := c.getVMForRead(namespace, name) if err != nil { return false, fmt.Errorf("failed to get VM %s: %w", name, err) } @@ -2627,7 +2637,7 @@ func (c *Client) GetNamespaceInfo(namespace string) (*corev1.Namespace, error) { // GetVMMemory gets memory information of a VirtualMachine func (c *Client) GetVMMemory(namespace, name string) (float64, error) { - vm, err := c.GetVM(namespace, name) + vm, err := c.getVMForRead(namespace, name) if err != nil { logger.Warning("Failed to get VM %s/%s for memory lookup: %v", namespace, name, err) return 2.0, nil // Low default fallback @@ -2665,7 +2675,7 @@ func (c *Client) GetVMMemory(namespace, name string) (float64, error) { // GetVMCPU gets CPU information of a VirtualMachine func (c *Client) GetVMCPU(namespace, name string) (int, error) { - vm, err := c.GetVM(namespace, name) + vm, err := c.getVMForRead(namespace, name) if err != nil { logger.Warning("Failed to get VM %s/%s for CPU lookup: %v", namespace, name, err) return 1, nil // Low default fallback @@ -2689,7 +2699,7 @@ func (c *Client) GetVMCPU(namespace, name string) (int, error) { // GetVMStorageDetails gets detailed storage information of a VirtualMachine func (c *Client) GetVMStorageDetails(namespace, name string) (map[string]interface{}, error) { - vm, err := c.GetVM(namespace, name) + vm, err := c.getVMForRead(namespace, name) if err != nil { return nil, err } @@ -2741,7 +2751,7 @@ func (c *Client) GetVMStorageDetails(namespace, name string) (map[string]interfa // GetVMNetworkDetails gets detailed network information of a VirtualMachine func (c *Client) GetVMNetworkDetails(namespace, name string) ([]map[string]interface{}, error) { - vm, err := c.GetVM(namespace, name) + vm, err := c.getVMForRead(namespace, name) if err != nil { return nil, err } diff --git a/pkg/server/server.go b/pkg/server/server.go index df224ce..725741c 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -1899,7 +1899,7 @@ func (s *Server) resolveSystemVMandCheckAccess(w http.ResponseWriter, r *http.Re continue } if cfg.Namespace == namespace { - vm, err := s.kubevirtClient.GetVM(namespace, vmName) + vm, err := s.kubevirtClient.GetVMForRead(namespace, vmName) if err == nil && kubevirt.VMMatchesSelector(vm, cfg.VMSelector) { vmFound = true namespace = cfg.Namespace @@ -1915,7 +1915,7 @@ func (s *Server) resolveSystemVMandCheckAccess(w http.ResponseWriter, r *http.Re continue } - vm, err := s.kubevirtClient.GetVM(cfg.Namespace, vmName) + vm, err := s.kubevirtClient.GetVMForRead(cfg.Namespace, vmName) if err == nil && kubevirt.VMMatchesSelector(vm, cfg.VMSelector) { vmFound = true namespace = cfg.Namespace From 7f478d3480af5417a8c6a73abbeff6afc15ec4b9 Mon Sep 17 00:00:00 2001 From: itayherzberg Date: Thu, 22 Oct 2026 17:09:35 +0300 Subject: [PATCH 2/2] refactor(kubevirt): make the informer cache mandatory and drop the write tracker Review feedback on the first revision: the cache should not be optional, and the write-tracking layer around the informers was too much machinery. The cache_enabled setting and its config, helm and test plumbing are gone. Informers always run for the configured namespaces; the only fallback to a direct GET is while an informer has not finished its initial list. The write tracker existed for one case: the boot-device PATCH handler writes the VM and then reads it back to build its response, and a lister read there could return the pre-write boot options. That read now goes to the API server (GetVMBootOptionsLive). No other write path reads through a cached getter, so the tracker, the wrapping dynamic client and their tests are removed. cache.go is now plain client-go informers: per-namespace dynamic factories for VM and VMI, managedFields stripped on the way in, listers behind HasSynced. Signed-off-by: itayherzberg --- config/config.yaml | 1 - helm/templates/secret.yaml | 3 - helm/values-example.yaml | 1 - helm/values.yaml | 6 - pkg/config/config.go | 13 -- pkg/config/config_test.go | 69 ------- pkg/kubevirt/cache.go | 394 ++++--------------------------------- pkg/kubevirt/cache_test.go | 336 ++++--------------------------- pkg/kubevirt/client.go | 29 +-- pkg/server/server.go | 2 +- 10 files changed, 96 insertions(+), 758 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 6fcae5f..2cec99f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -2,7 +2,6 @@ kubevirt: api_version: "v1" # Kubevirt API version (v1 is default (typically unchanged unless testing) timeout: 30 # Timeout for Kubevirt API calls (30 seconds is default (typically unchanged unless testing) - cache_enabled: true # Serve read-only VM/VMI lookups from an informer cache instead of the API (true is default) server: host: "0.0.0.0" # Host for the Redfish server (0.0.0.0 is default (typically unchanged unless testing) diff --git a/helm/templates/secret.yaml b/helm/templates/secret.yaml index 41951b2..2fafc1b 100644 --- a/helm/templates/secret.yaml +++ b/helm/templates/secret.yaml @@ -47,9 +47,6 @@ stringData: kubevirt: api_version: "{{ .Values.kubevirt.api_version }}" timeout: {{ .Values.kubevirt.timeout }} - {{- if hasKey .Values.kubevirt "cache_enabled" }} - cache_enabled: {{ .Values.kubevirt.cache_enabled }} - {{- end }} {{- if .Values.kubevirt.allow_insecure_tls }} allow_insecure_tls: {{ .Values.kubevirt.allow_insecure_tls }} {{- end }} diff --git a/helm/values-example.yaml b/helm/values-example.yaml index 4e94ec7..b93583d 100644 --- a/helm/values-example.yaml +++ b/helm/values-example.yaml @@ -112,7 +112,6 @@ rbac: kubevirt: api_version: "v1" timeout: 30 - cache_enabled: true allow_insecure_tls: true # ============================================================================= diff --git a/helm/values.yaml b/helm/values.yaml index d6c75d9..d4e7a8e 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -257,12 +257,6 @@ kubevirt: # Timeout for KubeVirt API operations (in seconds) timeout: 30 - # Serve read-only VM/VMI lookups (power state, inventory, boot options, - # virtual media state, collections) from an in-memory informer cache instead - # of the Kubernetes API. Write paths always read live. Set false to fall back - # to one API call per lookup. - cache_enabled: true - # Allow insecure TLS for KubeVirt API operations # Set to false in production environments allow_insecure_tls: true diff --git a/pkg/config/config.go b/pkg/config/config.go index 0bdaeed..478bae0 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -50,7 +50,6 @@ // kubevirt: // api_version: "v1" // timeout: 30 -// cache_enabled: true package config import ( @@ -197,9 +196,6 @@ type KubeVirtConfig struct { APIVersion string `mapstructure:"api_version"` Timeout int `mapstructure:"timeout"` AllowInsecureTLS bool `mapstructure:"allow_insecure_tls"` - // CacheEnabled serves read-only VM/VMI lookups from an informer cache - // instead of the API server. On by default; see pkg/kubevirt/cache.go. - CacheEnabled bool `mapstructure:"cache_enabled"` } // CDIConfig holds CDI (Containerized Data Importer) configuration parameters. @@ -314,7 +310,6 @@ func setDefaults() { viper.SetDefault("kubevirt.api_version", "v1") viper.SetDefault("kubevirt.timeout", 30) viper.SetDefault("kubevirt.allow_insecure_tls", false) - viper.SetDefault("kubevirt.cache_enabled", true) viper.SetDefault("datavolume.storage_size", "10Gi") viper.SetDefault("datavolume.allow_insecure_tls", false) viper.SetDefault("datavolume.vm_update_timeout", "30s") @@ -695,13 +690,6 @@ func (c *Config) GetChassisForUser(username string) ([]*ChassisConfig, error) { return nil, fmt.Errorf("user not found: %s", username) } -// GetVMCacheEnabled reports whether read-only VM/VMI lookups may be served -// from the informer cache. Discovered by the kubevirt client the same way as -// GetDataVolumeConfig. -func (c *Config) GetVMCacheEnabled() bool { - return c.KubeVirt.CacheEnabled -} - // GetDataVolumeConfig returns the DataVolume configuration settings. // This provides a safe way to access DataVolume settings without reflection. func (c *Config) GetDataVolumeConfig() (storageSize string, allowInsecureTLS bool, storageClass string, vmUpdateTimeout string, isoDownloadTimeout string, helperImage string) { @@ -776,7 +764,6 @@ authentication: kubevirt: api_version: "v1" # KubeVirt API version to use timeout: 30 # Timeout in seconds for KubeVirt operations - cache_enabled: true # Serve read-only VM/VMI lookups from an informer cache # DataVolume Configuration # ------------------------ diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 875de83..a74626b 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -104,75 +104,6 @@ kubevirt: } } -// minimalConfig is the smallest config that passes validation, with the -// kubevirt section supplied by the caller. -func minimalConfig(kubevirtSection string) string { - return ` -server: - host: "127.0.0.1" - port: 8443 - tls: - enabled: false - -chassis: - - name: "test-cluster" - namespace: "test-namespace" - service_account: "test-sa" - -authentication: - users: - - username: "testuser" - password: "testpass" - chassis: ["test-cluster"] - -kubevirt: - api_version: "v1" - timeout: 30 -` + kubevirtSection -} - -func writeConfig(t *testing.T, content string) string { - t.Helper() - configPath := filepath.Join(t.TempDir(), "config.yaml") - if err := os.WriteFile(configPath, []byte(content), 0600); err != nil { - t.Fatalf("Failed to write test config: %v", err) - } - return configPath -} - -func TestVMCacheConfig(t *testing.T) { - t.Run("enabled by default", func(t *testing.T) { - config, err := LoadConfig(writeConfig(t, minimalConfig(""))) - if err != nil { - t.Fatalf("Failed to load config: %v", err) - } - if !config.GetVMCacheEnabled() { - t.Error("Expected VM cache enabled by default") - } - }) - - t.Run("from config file", func(t *testing.T) { - config, err := LoadConfig(writeConfig(t, minimalConfig(" cache_enabled: false\n"))) - if err != nil { - t.Fatalf("Failed to load config: %v", err) - } - if config.GetVMCacheEnabled() { - t.Error("Expected VM cache disabled by config file") - } - }) - - t.Run("from environment", func(t *testing.T) { - t.Setenv("KUBEVIRT_REDFISH_KUBEVIRT_CACHE_ENABLED", "false") - config, err := LoadConfig(writeConfig(t, minimalConfig(""))) - if err != nil { - t.Fatalf("Failed to load config: %v", err) - } - if config.GetVMCacheEnabled() { - t.Error("Expected VM cache disabled from env") - } - }) -} - func TestLoadConfigWithDefaults(t *testing.T) { // Create a temporary directory and config file tempDir := t.TempDir() diff --git a/pkg/kubevirt/cache.go b/pkg/kubevirt/cache.go index e90f394..bd1cd7c 100644 --- a/pkg/kubevirt/cache.go +++ b/pkg/kubevirt/cache.go @@ -22,53 +22,43 @@ package kubevirt // A read-only cache of VirtualMachine and VirtualMachineInstance objects. // // Every Redfish request fans out to several Kubernetes API calls, and nearly -// all of them are reads of the same VM/VMI objects (a GET System is five GETs, -// a boot-device PATCH is eleven). With many BareMetalHosts polling at once, -// those reads are what saturate the client-side rate limiter. The informers -// here keep one watch per namespace per resource open instead, so the -// read-only Redfish paths are served from memory. +// all of them are reads of the same VM/VMI objects (a GET System is five GETs). +// With many BareMetalHosts polling at once, those reads are what saturate the +// client-side rate limiter. For every configured namespace the client keeps +// one client-go informer for VMs and one for VMIs, so the read-only Redfish +// paths are served from memory. // -// Only read-only inventory lookups use the cache (see getVMForRead and its -// callers). Every read-modify-write path still reads from the API server, -// because a patch computed from a stale object would clobber fields written -// moments earlier. Two further rules keep cached reads correct: +// Two rules keep cached reads correct: // -// - A cache that has not finished its initial sync is not consulted at all, -// and once it has synced, its answer (including "not found") is -// authoritative. Falling back to a GET on every miss would cost a GET on -// every power poll of a stopped VM, which has no VMI, and that is the -// dominant state of a BMH during inspection and provisioning. -// - After this process writes a VM or VMI, reads of that object bypass the -// cache until the informer has received the resourceVersion the write -// returned (or, for a delete, until the cached copy is gone or marked -// for deletion). A read that follows our own write therefore never sees -// the pre-write state, however far the watch lags. writeBypassMax bounds -// this, so a watch that broke and re-listed past our version cannot force -// direct reads forever. +// - Only read-only inventory lookups use the cache (getVMForRead and its +// callers). Every path that reads an object in order to write it, and the +// handler that reads an object back right after writing it, reads from +// the API server (GetVM, GetVMBootOptionsLive). A write is therefore +// never computed from, or reported from, a copy the watch has not caught +// up with. +// - A cache that has not finished its initial list is not consulted, and +// once it has, its answer is final, including "not found". Falling back +// to a GET on every miss would cost a GET on every power poll of a +// stopped VM, which has no VMI, and that is the dominant state of a BMH +// during inspection and provisioning. // -// Writes made by anyone else (oc edit, KubeVirt's own controllers) reach the -// cache when their watch event arrives, usually milliseconds later. Until then -// a cached read returns the previous version. That is the price of the cache; -// it is acceptable because no write path reads from it. +// A write by anyone, this process included, reaches the cache with its watch +// event, usually within milliseconds. Until then a cached read returns the +// previous version. No write path reads from the cache, so a stale read can +// make a response out of date but can never be written back over a change. import ( - "context" "fmt" "sort" - "sync" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" "k8s.io/client-go/dynamic/dynamicinformer" "k8s.io/client-go/informers" - toolscache "k8s.io/client-go/tools/cache" kubevirtv1 "kubevirt.io/api/core/v1" "github.com/kubevirt/redfish-controller/pkg/logger" @@ -79,266 +69,6 @@ var ( vmiGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachineinstances"} ) -// writeBypassMax caps how long a write the cache has not yet reflected keeps -// reads of that object direct. On a healthy API server the informer sees our -// write within milliseconds and the bypass ends then; this only matters when -// the watch broke and the re-list skipped past our version. -const writeBypassMax = 30 * time.Second - -// seenVersionsMax caps how many recently stored versions the write tracker -// remembers per object. -const seenVersionsMax = 32 - -// pendingWrite is a write by this process that the cache may not reflect yet. -type pendingWrite struct { - resourceVersion string // returned by the API server; empty for a delete - deleted bool - at time.Time -} - -type seenVersion struct { - resourceVersion string - at time.Time -} - -// writeTracker remembers the VM and VMI writes by this process that the cache -// has not reflected yet. -// -// It also remembers the versions the informers stored recently. On a real API -// server the watch event for our own write often reaches the informer before -// the write call returns, and KubeVirt's controllers write a newer status -// within milliseconds. Without that history the write would be recorded after -// the cache had already seen it, and the cache would hold only the newer -// version, so reads would bypass it until writeBypassMax. -type writeTracker struct { - mu sync.Mutex - pending map[string]pendingWrite - seen map[string][]seenVersion - lastSweep time.Time - now func() time.Time -} - -func newWriteTracker() *writeTracker { - return &writeTracker{ - pending: make(map[string]pendingWrite), - seen: make(map[string][]seenVersion), - now: time.Now, - } -} - -func writeKey(gvr schema.GroupVersionResource, namespace, name string) string { - return gvr.Resource + "/" + namespace + "/" + name -} - -// sweepLocked drops pending writes and seen versions older than -// writeBypassMax. It scans everything, so it runs at most once per -// writeBypassMax. -func (t *writeTracker) sweepLocked(now time.Time) { - if now.Sub(t.lastSweep) <= writeBypassMax { - return - } - t.lastSweep = now - for key, p := range t.pending { - if now.Sub(p.at) > writeBypassMax { - delete(t.pending, key) - } - } - for key, versions := range t.seen { - if now.Sub(versions[len(versions)-1].at) > writeBypassMax { - delete(t.seen, key) - } - } -} - -func (t *writeTracker) seenLocked(key, resourceVersion string) bool { - for _, v := range t.seen[key] { - if v.resourceVersion == resourceVersion { - return true - } - } - return false -} - -func (t *writeTracker) record(gvr schema.GroupVersionResource, namespace, name string, w pendingWrite) { - if t == nil || (gvr != vmGVR && gvr != vmiGVR) { - return - } - t.mu.Lock() - defer t.mu.Unlock() - w.at = t.now() - t.sweepLocked(w.at) - key := writeKey(gvr, namespace, name) - if !w.deleted && w.resourceVersion != "" && t.seenLocked(key, w.resourceVersion) { - // The cache already stored this write; an older pending write of - // ours is superseded by it. - delete(t.pending, key) - return - } - t.pending[key] = w -} - -// note records a successful create, update or patch and the resourceVersion -// the API server returned for it. Resources other than VM and VMI are ignored. -func (t *writeTracker) note(gvr schema.GroupVersionResource, namespace, name, resourceVersion string) { - t.record(gvr, namespace, name, pendingWrite{resourceVersion: resourceVersion}) -} - -func (t *writeTracker) noteDelete(gvr schema.GroupVersionResource, namespace, name string) { - t.record(gvr, namespace, name, pendingWrite{deleted: true}) -} - -// observed is called by the informers for every object version they store. -// Seeing the version our write produced means the cache has reflected it, -// even if a newer version has replaced it by the time anyone reads. -func (t *writeTracker) observed(gvr schema.GroupVersionResource, namespace, name, resourceVersion string) { - if t == nil { - return - } - t.mu.Lock() - defer t.mu.Unlock() - now := t.now() - t.sweepLocked(now) - key := writeKey(gvr, namespace, name) - if p, ok := t.pending[key]; ok && !p.deleted && p.resourceVersion != "" && p.resourceVersion == resourceVersion { - delete(t.pending, key) - } - - versions := t.seen[key] - for len(versions) > 0 && (len(versions) >= seenVersionsMax || now.Sub(versions[0].at) > writeBypassMax) { - versions = versions[1:] - } - t.seen[key] = append(versions, seenVersion{resourceVersion: resourceVersion, at: now}) -} - -// forget drops the versions remembered for an object the informer deleted, -// so the history does not grow with every VMI ever created. -func (t *writeTracker) forget(gvr schema.GroupVersionResource, namespace, name string) { - if t == nil { - return - } - t.mu.Lock() - defer t.mu.Unlock() - delete(t.seen, writeKey(gvr, namespace, name)) -} - -// cacheReflects reports whether the cached copy (nil when the cache has none) -// may be served, given this process's writes to the object. -func (t *writeTracker) cacheReflects(gvr schema.GroupVersionResource, namespace, name string, cached *unstructured.Unstructured) bool { - if t == nil { - return true - } - t.mu.Lock() - defer t.mu.Unlock() - key := writeKey(gvr, namespace, name) - p, ok := t.pending[key] - if !ok { - return true - } - - var caughtUp bool - if p.deleted { - caughtUp = cached == nil || cached.GetDeletionTimestamp() != nil - } else { - // Equal versions also cover a no-op write, which the API server - // acknowledges with the current version and no watch event. - caughtUp = cached != nil && p.resourceVersion != "" && cached.GetResourceVersion() == p.resourceVersion - } - if caughtUp { - delete(t.pending, key) - return true - } - if t.now().Sub(p.at) > writeBypassMax { - delete(t.pending, key) - logger.Warning("VM cache has not reflected this process's write to %s within %s; serving it from the cache again", key, writeBypassMax) - return true - } - return false -} - -// trackingDynamicClient wraps a dynamic client so that every successful write -// to a VM or VMI is recorded in the writeTracker. Reads pass straight through. -type trackingDynamicClient struct { - dynamic.Interface - tracker *writeTracker -} - -func newTrackingDynamicClient(inner dynamic.Interface, tracker *writeTracker) dynamic.Interface { - if inner == nil { - return nil - } - return &trackingDynamicClient{Interface: inner, tracker: tracker} -} - -func (d *trackingDynamicClient) Resource(gvr schema.GroupVersionResource) dynamic.NamespaceableResourceInterface { - return &trackingNamespaceableResource{ - NamespaceableResourceInterface: d.Interface.Resource(gvr), - gvr: gvr, - tracker: d.tracker, - } -} - -type trackingNamespaceableResource struct { - dynamic.NamespaceableResourceInterface - gvr schema.GroupVersionResource - tracker *writeTracker -} - -func (r *trackingNamespaceableResource) Namespace(namespace string) dynamic.ResourceInterface { - return &trackingResource{ - ResourceInterface: r.NamespaceableResourceInterface.Namespace(namespace), - gvr: r.gvr, - namespace: namespace, - tracker: r.tracker, - } -} - -type trackingResource struct { - dynamic.ResourceInterface - gvr schema.GroupVersionResource - namespace string - tracker *writeTracker -} - -func (r *trackingResource) Create(ctx context.Context, obj *unstructured.Unstructured, options metav1.CreateOptions, subresources ...string) (*unstructured.Unstructured, error) { - out, err := r.ResourceInterface.Create(ctx, obj, options, subresources...) - if err == nil { - r.tracker.note(r.gvr, r.namespace, obj.GetName(), out.GetResourceVersion()) - } - return out, err -} - -func (r *trackingResource) Update(ctx context.Context, obj *unstructured.Unstructured, options metav1.UpdateOptions, subresources ...string) (*unstructured.Unstructured, error) { - out, err := r.ResourceInterface.Update(ctx, obj, options, subresources...) - if err == nil { - r.tracker.note(r.gvr, r.namespace, obj.GetName(), out.GetResourceVersion()) - } - return out, err -} - -func (r *trackingResource) UpdateStatus(ctx context.Context, obj *unstructured.Unstructured, options metav1.UpdateOptions) (*unstructured.Unstructured, error) { - out, err := r.ResourceInterface.UpdateStatus(ctx, obj, options) - if err == nil { - r.tracker.note(r.gvr, r.namespace, obj.GetName(), out.GetResourceVersion()) - } - return out, err -} - -func (r *trackingResource) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) { - out, err := r.ResourceInterface.Patch(ctx, name, pt, data, options, subresources...) - if err == nil { - r.tracker.note(r.gvr, r.namespace, name, out.GetResourceVersion()) - } - return out, err -} - -func (r *trackingResource) Delete(ctx context.Context, name string, options metav1.DeleteOptions, subresources ...string) error { - err := r.ResourceInterface.Delete(ctx, name, options, subresources...) - if err == nil { - r.tracker.noteDelete(r.gvr, r.namespace, name) - } - return err -} - // vmCache holds one VM informer and one VMI informer per configured namespace. type vmCache struct { stopCh chan struct{} @@ -348,9 +78,8 @@ type vmCache struct { } // newVMCache starts informers for every distinct, non-empty namespace. It does -// not wait for them to sync; readers check HasSynced per call. Every version -// the informers store is reported to writes. -func newVMCache(client dynamic.Interface, namespaces []string, writes *writeTracker) *vmCache { +// not wait for them to sync; readers check HasSynced per call. +func newVMCache(client dynamic.Interface, namespaces []string) *vmCache { vc := &vmCache{ stopCh: make(chan struct{}), vms: make(map[string]informers.GenericInformer), @@ -366,14 +95,11 @@ func newVMCache(client dynamic.Interface, namespaces []string, writes *writeTrac factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(client, 0, namespace, nil) vmInformer := factory.ForResource(vmGVR) vmiInformer := factory.ForResource(vmiGVR) - // Transforms and handlers must be set before the factory starts. - for gvr, inf := range map[schema.GroupVersionResource]informers.GenericInformer{vmGVR: vmInformer, vmiGVR: vmiInformer} { + // Transforms must be set before the factory starts. + for _, inf := range []informers.GenericInformer{vmInformer, vmiInformer} { if err := inf.Informer().SetTransform(stripManagedFields); err != nil { logger.Error("Failed to set informer transform for namespace %s: %v", namespace, err) } - if _, err := inf.Informer().AddEventHandler(observeVersions(gvr, writes)); err != nil { - logger.Error("Failed to add informer handler for namespace %s: %v", namespace, err) - } } vc.vms[namespace] = vmInformer vc.vmis[namespace] = vmiInformer @@ -412,30 +138,6 @@ func (vc *vmCache) informer(gvr schema.GroupVersionResource, namespace string) ( return inf, true } -// observeVersions reports every version an informer stores to the write -// tracker, and every object it deletes so the tracker can forget it. The -// shared informer updates its store before it notifies handlers, so once a -// version is observed here, the store holds it or a newer one. -func observeVersions(gvr schema.GroupVersionResource, writes *writeTracker) toolscache.ResourceEventHandler { - report := func(obj interface{}) { - if m, err := meta.Accessor(obj); err == nil { - writes.observed(gvr, m.GetNamespace(), m.GetName(), m.GetResourceVersion()) - } - } - return toolscache.ResourceEventHandlerFuncs{ - AddFunc: report, - UpdateFunc: func(_, obj interface{}) { report(obj) }, - DeleteFunc: func(obj interface{}) { - if tombstone, ok := obj.(toolscache.DeletedFinalStateUnknown); ok { - obj = tombstone.Obj - } - if m, err := meta.Accessor(obj); err == nil { - writes.forget(gvr, m.GetNamespace(), m.GetName()) - } - }, - } -} - // stripManagedFields drops the managedFields block before an object enters // the store; it is the largest part of a VM object and nothing here reads it. func stripManagedFields(obj interface{}) (interface{}, error) { @@ -445,18 +147,6 @@ func stripManagedFields(obj interface{}) (interface{}, error) { return obj, nil } -// vmCacheEnabled reads kubevirt.cache_enabled from the app config; the cache -// is on unless the config says otherwise. -func (c *Client) vmCacheEnabled() bool { - if c.appConfig == nil { - return true - } - if cfg, ok := c.appConfig.(interface{ GetVMCacheEnabled() bool }); ok { - return cfg.GetVMCacheEnabled() - } - return true -} - // restartVMCache replaces the running cache with one covering namespaces. // Called with the watchers on startup and on config reload. func (c *Client) restartVMCache(namespaces []string) { @@ -467,14 +157,10 @@ func (c *Client) restartVMCache(namespaces []string) { c.vmCache.stop() c.vmCache = nil } - if !c.vmCacheEnabled() { - logger.Info("VM cache disabled by configuration; VM/VMI reads go to the API server") - return - } if c.dynamicClient == nil { return } - c.vmCache = newVMCache(c.dynamicClient, namespaces, c.writes) + c.vmCache = newVMCache(c.dynamicClient, namespaces) logger.Info("VM cache started for namespaces %v", namespaces) } @@ -494,9 +180,8 @@ func (c *Client) currentVMCache() *vmCache { } // GetVMForRead returns a VM for a read-only purpose. It is served from the -// informer cache when the cache is synced for the namespace and reflects every -// write this process made to the VM; otherwise it is a GetVM. Callers that go -// on to patch the VM must use GetVM. +// informer cache when the cache is synced for the namespace; otherwise it is +// a GetVM. Callers that go on to patch the VM must use GetVM. func (c *Client) GetVMForRead(namespace, name string) (*kubevirtv1.VirtualMachine, error) { return c.getVMForRead(namespace, name) } @@ -532,28 +217,23 @@ func (c *Client) getVMIForRead(namespace, name string) (*kubevirtv1.VirtualMachi // cachedObject looks the object up in the cache. served is false when the // caller must ask the API server instead: the cache is not synced for the -// namespace, the lister failed, or the cache does not yet reflect a write by -// this process. When served, err is a NotFound error if the object is absent, -// and the returned object is a copy the caller may modify. -func (c *Client) cachedObject(gvr schema.GroupVersionResource, namespace, name string) (u *unstructured.Unstructured, served bool, err error) { +// namespace or the lister failed. When served, err is a NotFound error if the +// object is absent, and the returned object is a copy the caller may modify. +func (c *Client) cachedObject(gvr schema.GroupVersionResource, namespace, name string) (*unstructured.Unstructured, bool, error) { inf, ok := c.currentVMCache().informer(gvr, namespace) if !ok { return nil, false, nil } obj, err := inf.Lister().ByNamespace(namespace).Get(name) - switch { - case err == nil: - if u, ok = obj.(*unstructured.Unstructured); !ok { - return nil, false, nil - } - case !apierrors.IsNotFound(err): - return nil, false, nil + if apierrors.IsNotFound(err) { + return nil, true, err } - if !c.writes.cacheReflects(gvr, namespace, name, u) { + if err != nil { return nil, false, nil } - if u == nil { - return nil, true, err + u, ok := obj.(*unstructured.Unstructured) + if !ok { + return nil, false, nil } return u.DeepCopy(), true, nil } diff --git a/pkg/kubevirt/cache_test.go b/pkg/kubevirt/cache_test.go index d9b6b1e..dbfe685 100644 --- a/pkg/kubevirt/cache_test.go +++ b/pkg/kubevirt/cache_test.go @@ -20,22 +20,17 @@ package kubevirt import ( - "context" "errors" - "fmt" "reflect" "strings" - "sync" "testing" "time" apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" dynamicfake "k8s.io/client-go/dynamic/fake" k8stesting "k8s.io/client-go/testing" ) @@ -50,20 +45,20 @@ func testVM(namespace, name string, lbls map[string]string) *unstructured.Unstru return u } -// newCacheTestClient builds a Client over a fake dynamic client, wrapped by -// the write tracker exactly as the constructors do. -func newCacheTestClient(t *testing.T, appConfig interface{}, objs ...runtime.Object) (*Client, *dynamicfake.FakeDynamicClient) { +func withBootTarget(u *unstructured.Unstructured, target string) *unstructured.Unstructured { + u.SetAnnotations(map[string]string{"redfish.boot.source.override.target": target}) + return u +} + +func newCacheTestClient(t *testing.T, objs ...runtime.Object) (*Client, *dynamicfake.FakeDynamicClient) { t.Helper() fake := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ vmGVR: "VirtualMachineList", vmiGVR: "VirtualMachineInstanceList", }, objs...) - writes := newWriteTracker() c := &Client{ - dynamicClient: newTrackingDynamicClient(fake, writes), - writes: writes, + dynamicClient: fake, timeout: 5 * time.Second, - appConfig: appConfig, } c.metrics.operationCounts = make(map[string]int64) c.metrics.operationTimes = make(map[string]time.Duration) @@ -94,7 +89,7 @@ func failGets(fake *dynamicfake.FakeDynamicClient, msg string) { } func TestVMCacheServesReads(t *testing.T) { - c, fake := newCacheTestClient(t, nil, + c, fake := newCacheTestClient(t, testVM("ns", "vm-a", map[string]string{"redfish-enabled": "true"}), testVM("ns", "vm-b", nil), ) @@ -149,302 +144,40 @@ func TestVMCacheServesReads(t *testing.T) { } } -// fakeClock is the write tracker's clock in tests. -type fakeClock struct { - mu sync.Mutex - now time.Time -} - -func newFakeClock() *fakeClock { return &fakeClock{now: time.Unix(1_700_000_000, 0)} } - -func (f *fakeClock) Now() time.Time { - f.mu.Lock() - defer f.mu.Unlock() - return f.now -} - -func (f *fakeClock) Advance(d time.Duration) { - f.mu.Lock() - f.now = f.now.Add(d) - f.mu.Unlock() -} - -func withRV(u *unstructured.Unstructured, rv string) *unstructured.Unstructured { - u.SetResourceVersion(rv) - return u -} - -// newWriteTestClient returns a synced client over VMs vm-a and vm-b at -// resourceVersion 1, with a fake clock and every API server GET failing, so -// readsFromCache can tell a cached read from a direct one. -func newWriteTestClient(t *testing.T) (*Client, *dynamicfake.FakeDynamicClient, *fakeClock) { - t.Helper() - c, fake := newCacheTestClient(t, nil, - withRV(testVM("ns", "vm-a", nil), "1"), - withRV(testVM("ns", "vm-b", nil), "1"), - ) - clock := newFakeClock() - c.writes.now = clock.Now +// The boot-device PATCH handler writes the VM and then reads it back to build +// its response. That read-back must come from the API server, because the +// watch event for the write may not have reached the cache yet. +func TestGetVMBootOptionsLiveReadsAPIServer(t *testing.T) { + c, fake := newCacheTestClient(t, withBootTarget(testVM("ns", "vm-a", nil), "Hdd")) c.restartVMCache([]string{"ns"}) - t.Cleanup(c.stopVMCache) + defer c.stopVMCache() waitSynced(t, c, "ns") - failGets(fake, "api server GET") - return c, fake, clock -} -func readsFromCache(c *Client, name string) bool { - _, err := c.getVMForRead("ns", name) - return err == nil || !strings.Contains(err.Error(), "api server GET") -} - -// eventually polls cond, because informer event delivery is asynchronous -// even over the fake client. -func eventually(t *testing.T, what string, cond func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - if cond() { - return - } - time.Sleep(10 * time.Millisecond) - } - t.Fatalf("timed out waiting until %s", what) -} - -// acceptUnseen makes the API server acknowledge a verb on VM name, answering -// with resourceVersion rv, without storing anything. The informer never hears -// of it: the state of a cache whose watch lags behind our write. -func acceptUnseen(fake *dynamicfake.FakeDynamicClient, verb, name, rv string) { - fake.PrependReactor(verb, "virtualmachines", func(a k8stesting.Action) (bool, runtime.Object, error) { - var actionName string - switch a := a.(type) { - case k8stesting.PatchAction: - actionName = a.GetName() - case k8stesting.DeleteAction: - actionName = a.GetName() - } - if actionName != name { - return false, nil, nil - } - if verb == "delete" { - return true, nil, nil - } - return true, withRV(testVM("ns", name, nil), rv), nil - }) -} - -func patchVM(t *testing.T, c *Client, name string) { - t.Helper() - if _, err := c.dynamicClient.Resource(vmGVR).Namespace("ns").Patch(context.Background(), name, - types.MergePatchType, []byte(`{"metadata":{"labels":{"x":"y"}}}`), metav1.PatchOptions{}); err != nil { - t.Fatalf("patch %s: %v", name, err) - } -} - -func TestVMCacheBypassUntilOwnWriteIsObserved(t *testing.T) { - c, fake, clock := newWriteTestClient(t) - acceptUnseen(fake, "patch", "vm-a", "2") - patchVM(t, c, "vm-a") - - if readsFromCache(c, "vm-a") { - t.Fatal("a read after our own write must bypass a cache that has not seen it") - } - if !readsFromCache(c, "vm-b") { - t.Fatal("reads of other VMs must still come from the cache") - } - - // No fixed delay ends the bypass: while the cache lags, reads stay direct. - clock.Advance(writeBypassMax - time.Second) - if readsFromCache(c, "vm-a") { - t.Fatal("bypass ended before the cache saw our write") - } - - if err := fake.Tracker().Update(vmGVR, withRV(testVM("ns", "vm-a", nil), "2"), "ns"); err != nil { - t.Fatal(err) - } - eventually(t, "the cache serves vm-a once it holds our version", func() bool { return readsFromCache(c, "vm-a") }) - vm, err := c.getVMForRead("ns", "vm-a") - if err != nil || vm.ResourceVersion != "2" { - t.Fatalf("cached read: rv=%v err=%v", vm, err) - } -} - -func TestVMCacheBypassEndsWhenOwnWriteIsSuperseded(t *testing.T) { - c, fake, _ := newWriteTestClient(t) - acceptUnseen(fake, "patch", "vm-a", "2") - patchVM(t, c, "vm-a") - - // The informer sees our version and then a newer one written by someone - // else. The cache never holds version 2 when we read, but it has seen it. - for _, rv := range []string{"2", "3"} { - if err := fake.Tracker().Update(vmGVR, withRV(testVM("ns", "vm-a", nil), rv), "ns"); err != nil { - t.Fatal(err) - } - } - eventually(t, "the cache serves vm-a after seeing our version", func() bool { return readsFromCache(c, "vm-a") }) - vm, err := c.getVMForRead("ns", "vm-a") - if err != nil || vm.ResourceVersion != "3" { - t.Fatalf("cached read: vm=%v err=%v", vm, err) - } -} - -func TestVMCacheNoBypassWhenOwnWriteWasObservedBeforePatchReturned(t *testing.T) { - // On a real API server the watch event for our write can reach the - // informer before Patch returns, and virt-controller writes a newer status - // right after. The cache has seen our version before we record the write. - c, fake, _ := newWriteTestClient(t) - for _, rv := range []string{"2", "3"} { - if err := fake.Tracker().Update(vmGVR, withRV(testVM("ns", "vm-a", nil), rv), "ns"); err != nil { - t.Fatal(err) - } - } - // Wait on the handler, not the store: the informer updates the store - // before it notifies handlers. - eventually(t, "the informer reported versions 2 and 3", func() bool { - c.writes.mu.Lock() - defer c.writes.mu.Unlock() - key := writeKey(vmGVR, "ns", "vm-a") - return c.writes.seenLocked(key, "2") && c.writes.seenLocked(key, "3") - }) - - acceptUnseen(fake, "patch", "vm-a", "2") - patchVM(t, c, "vm-a") - if !readsFromCache(c, "vm-a") { - t.Fatal("a write the cache saw before Patch returned must not bypass it") - } -} - -func TestWriteTrackerForgetsDeletedObjects(t *testing.T) { - w := newWriteTracker() - w.observed(vmGVR, "ns", "vm-a", "2") - w.observed(vmiGVR, "ns", "vm-a", "5") - w.forget(vmiGVR, "ns", "vm-a") - - w.mu.Lock() - defer w.mu.Unlock() - if _, ok := w.seen[writeKey(vmiGVR, "ns", "vm-a")]; ok { - t.Fatal("versions of a deleted object must be dropped") - } - if _, ok := w.seen[writeKey(vmGVR, "ns", "vm-a")]; !ok { - t.Fatal("versions of other objects must be kept") - } -} - -func TestWriteTrackerSeenHistoryIsBounded(t *testing.T) { - w := newWriteTracker() - clock := newFakeClock() - w.now = clock.Now - for i := 0; i < 3*seenVersionsMax; i++ { - w.observed(vmGVR, "ns", "vm-a", fmt.Sprint(i)) - } - w.mu.Lock() - n := len(w.seen[writeKey(vmGVR, "ns", "vm-a")]) - w.mu.Unlock() - if n > seenVersionsMax { - t.Fatalf("kept %d versions, want at most %d", n, seenVersionsMax) - } - - clock.Advance(writeBypassMax + time.Second) - w.observed(vmGVR, "ns", "vm-b", "1") - w.mu.Lock() - _, stale := w.seen[writeKey(vmGVR, "ns", "vm-a")] - w.mu.Unlock() - if stale { - t.Fatal("versions older than writeBypassMax must be pruned") - } -} - -func TestVMCacheNoOpWriteDoesNotBypass(t *testing.T) { - // The API server does not bump resourceVersion or emit a watch event for - // a write that changes nothing; the cache already holds that version. - c, fake, _ := newWriteTestClient(t) - acceptUnseen(fake, "patch", "vm-a", "1") - patchVM(t, c, "vm-a") - if !readsFromCache(c, "vm-a") { - t.Fatal("a no-op write must not bypass a cache that already holds its version") - } -} - -func TestVMCacheBypassIsBounded(t *testing.T) { - c, fake, clock := newWriteTestClient(t) - acceptUnseen(fake, "patch", "vm-a", "2") - patchVM(t, c, "vm-a") - clock.Advance(writeBypassMax + time.Second) - if !readsFromCache(c, "vm-a") { - t.Fatal("a write the cache never sees must stop forcing direct reads after writeBypassMax") - } - c.writes.mu.Lock() - n := len(c.writes.pending) - c.writes.mu.Unlock() - if n != 0 { - t.Fatalf("expired write still tracked: %d entries", n) - } -} - -func TestVMCacheBypassAfterOwnDelete(t *testing.T) { - c, fake, _ := newWriteTestClient(t) - acceptUnseen(fake, "delete", "vm-a", "") - acceptUnseen(fake, "delete", "vm-b", "") - for _, name := range []string{"vm-a", "vm-b"} { - if err := c.dynamicClient.Resource(vmGVR).Namespace("ns").Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil { - t.Fatal(err) - } - if readsFromCache(c, name) { - t.Fatalf("a read of %s after our delete must bypass a cache that still holds it", name) - } - } - - // vm-a is removed outright; vm-b is held by a finalizer and only gains a - // deletionTimestamp. Either one shows the cache has seen the delete. - if err := fake.Tracker().Delete(vmGVR, "ns", "vm-a"); err != nil { - t.Fatal(err) - } - terminating := withRV(testVM("ns", "vm-b", nil), "2") - now := metav1.Now() - terminating.SetDeletionTimestamp(&now) - if err := fake.Tracker().Update(vmGVR, terminating, "ns"); err != nil { - t.Fatal(err) - } - eventually(t, "the cache answers NotFound for vm-a", func() bool { - _, err := c.getVMForRead("ns", "vm-a") - return apierrors.IsNotFound(err) + // The API server holds a newer version the watch has not delivered. + live := withBootTarget(testVM("ns", "vm-a", nil), "Cd") + fake.PrependReactor("get", "virtualmachines", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, live, nil }) - eventually(t, "the cache serves the terminating vm-b", func() bool { return readsFromCache(c, "vm-b") }) -} -func TestVMCacheFailedWriteDoesNotBypass(t *testing.T) { - c, fake, _ := newWriteTestClient(t) - fake.PrependReactor("patch", "virtualmachines", func(k8stesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("patch rejected") - }) - if _, err := c.dynamicClient.Resource(vmGVR).Namespace("ns").Patch(context.Background(), "vm-a", - types.MergePatchType, []byte(`{}`), metav1.PatchOptions{}); err == nil { - t.Fatal("expected the patch to fail") + cached, err := c.GetVMBootOptions("ns", "vm-a") + if err != nil { + t.Fatalf("cached boot options: %v", err) } - if !readsFromCache(c, "vm-a") { - t.Fatal("a failed write must not bypass the cache") + if cached["bootSourceOverrideTarget"] != "Hdd" { + t.Fatalf("cached read should still hold the old target, got %v", cached["bootSourceOverrideTarget"]) } -} -type cacheOff struct{} - -func (cacheOff) GetVMCacheEnabled() bool { return false } - -func TestVMCacheDisabledByConfig(t *testing.T) { - c, _ := newCacheTestClient(t, cacheOff{}, testVM("ns", "vm-a", nil)) - c.restartVMCache([]string{"ns"}) - defer c.stopVMCache() - if c.currentVMCache() != nil { - t.Fatal("cache must not start when disabled by config") + fresh, err := c.GetVMBootOptionsLive("ns", "vm-a") + if err != nil { + t.Fatalf("live boot options: %v", err) } - vm, err := c.getVMForRead("ns", "vm-a") - if err != nil || vm.Name != "vm-a" { - t.Fatalf("direct read with cache disabled: vm=%v err=%v", vm, err) + if fresh["bootSourceOverrideTarget"] != "Cd" { + t.Fatalf("live read must come from the API server, got %v", fresh["bootSourceOverrideTarget"]) } } func TestVMCacheRestartAndStop(t *testing.T) { - c, _ := newCacheTestClient(t, nil, testVM("ns1", "a", nil), testVM("ns2", "b", nil)) + c, _ := newCacheTestClient(t, testVM("ns1", "a", nil), testVM("ns2", "b", nil)) c.restartVMCache([]string{"ns1"}) waitSynced(t, c, "ns1") if _, ok := c.currentVMCache().informer(vmGVR, "ns2"); ok { @@ -466,6 +199,17 @@ func TestVMCacheRestartAndStop(t *testing.T) { } } +// A client built without a dynamic client, as the server tests do, must not +// start a cache or panic. +func TestVMCacheWithoutDynamicClient(t *testing.T) { + c := &Client{} + c.restartVMCache([]string{"ns"}) + if c.currentVMCache() != nil { + t.Fatal("cache must not start without a dynamic client") + } + c.stopVMCache() +} + func TestFilterVMNames(t *testing.T) { in := []string{"c", "a", "b"} if got := filterVMNames(in, nil); !reflect.DeepEqual(got, []string{"a", "b", "c"}) { diff --git a/pkg/kubevirt/client.go b/pkg/kubevirt/client.go index bd03330..4bb6f1c 100644 --- a/pkg/kubevirt/client.go +++ b/pkg/kubevirt/client.go @@ -83,10 +83,9 @@ type Client struct { operationTimes map[string]time.Duration } - // Read-only VM/VMI cache and the write tracker that keeps it honest (cache.go) + // Read-only VM/VMI informer cache (cache.go) cacheMu sync.RWMutex vmCache *vmCache - writes *writeTracker // Boot-once watcher management watcherCtx context.Context @@ -236,17 +235,13 @@ func NewClient(configPath string, timeout time.Duration, appConfig interface{}) }, } - // Every VM/VMI write goes through the tracker so cached reads can skip - // the cache right after this process wrote the object (see cache.go). - writes := newWriteTracker() return &Client{ kubernetesClient: kubernetesClient, - dynamicClient: newTrackingDynamicClient(dynamicClient, writes), + dynamicClient: dynamicClient, config: config, timeout: timeout, appConfig: appConfig, httpClient: httpClient, - writes: writes, }, nil } @@ -258,11 +253,9 @@ func NewClientWithClients( timeout time.Duration, appConfig interface{}, ) *Client { - writes := newWriteTracker() return &Client{ kubernetesClient: kubernetesClient, - dynamicClient: newTrackingDynamicClient(dynamicClient, writes), - writes: writes, + dynamicClient: dynamicClient, timeout: timeout, appConfig: appConfig, config: &rest.Config{}, @@ -1389,7 +1382,21 @@ func (c *Client) GetVMBootOptions(namespace, name string) (map[string]interface{ if err != nil { return nil, err } + return bootOptionsFromVM(vm), nil +} + +// GetVMBootOptionsLive is GetVMBootOptions read from the API server instead of +// the cache. The boot-device PATCH handler uses it to build its response from +// the VM it has just written, which the cache may not reflect yet. +func (c *Client) GetVMBootOptionsLive(namespace, name string) (map[string]interface{}, error) { + vm, err := c.GetVM(namespace, name) + if err != nil { + return nil, err + } + return bootOptionsFromVM(vm), nil +} +func bootOptionsFromVM(vm *kubevirtv1.VirtualMachine) map[string]interface{} { bootOptions := map[string]interface{}{ "bootSourceOverrideEnabled": "Disabled", "bootSourceOverrideTarget": "None", @@ -1418,7 +1425,7 @@ func (c *Client) GetVMBootOptions(namespace, name string) (map[string]interface{ } } - return bootOptions, nil + return bootOptions } // RecordVMBootOptionsAsAnnotations stores the boot options of a VirtualMachine diff --git a/pkg/server/server.go b/pkg/server/server.go index 725741c..1564964 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -1203,7 +1203,7 @@ func (s *Server) handleBootUpdate(w http.ResponseWriter, r *http.Request, system } // Get updated boot options to return in response - bootOptions, err := s.kubevirtClient.GetVMBootOptions(namespace, vmName) + bootOptions, err := s.kubevirtClient.GetVMBootOptionsLive(namespace, vmName) if err != nil { logger.Error("Failed to get updated boot options for VM %s: %v", vmName, err) bootOptions = map[string]interface{}{