From c74205f994857c65e20cb4e5f133f620d36cb30e Mon Sep 17 00:00:00 2001 From: itayherzberg Date: Wed, 21 Oct 2026 20:57:41 +0300 Subject: [PATCH] fix(kubevirt): stop concurrent VM writers from clobbering each other's spec changes Every change to a VirtualMachine was a read, an edit of a copy and a JSON merge patch of the difference. The patch carried no resourceVersion, so the API server could not reject one computed from a stale read, and a merge patch replaces the disks and volumes arrays wholesale. The insert of virtual media (task worker) and the boot-once override (HTTP handler) run within milliseconds of each other on the same VM; when the boot-once patch landed last it carried the disks array from before the insert, deleting the cdrom0 disk entry while its volume survived. KubeVirt then attached the volume as a writable hard disk, QEMU failed to open the read-only ISO and the VM crash-looped. Every read-modify-write now goes through one helper, updateVM, which serializes writers of the same VM inside the process with a per-VM lock, seeds the patch with the resourceVersion it was computed from so the API server answers a stale write with 409 Conflict, and re-runs the mutation on a fresh read when that happens. The lock covers only the spec change; PVC creation and ISO downloads stay outside it. Blind single-key label patches and the runStrategy JSON patch are unchanged: they cannot clobber anything. An insert that finds a boot-once-to-CD already recorded places the new CD-ROM first in the boot order and adds it to the saved original order, so both request orders end with the same VM. The VM watcher re-adds a missing disk entry for any inserted device whose volume survived, repairing VMs corrupted before this fix on the next event or reconcile tick. The mock dynamic client now applies patches atomically, bumps resourceVersion on every write and rejects a stale resourceVersion with a Conflict, so the tests reproduce the interleaving that corrupted the production VMs and show it cannot happen any more. Signed-off-by: itayherzberg --- pkg/kubevirt/client.go | 519 +++++++++-------------- pkg/kubevirt/mock_dynamic_client.go | 154 ++++++- pkg/kubevirt/vm_update.go | 304 ++++++++++++++ pkg/kubevirt/vm_update_test.go | 619 ++++++++++++++++++++++++++++ pkg/kubevirt/vmlock.go | 51 +++ pkg/kubevirt/vmlock_test.go | 91 ++++ 6 files changed, 1397 insertions(+), 341 deletions(-) create mode 100644 pkg/kubevirt/vm_update.go create mode 100644 pkg/kubevirt/vm_update_test.go create mode 100644 pkg/kubevirt/vmlock.go create mode 100644 pkg/kubevirt/vmlock_test.go diff --git a/pkg/kubevirt/client.go b/pkg/kubevirt/client.go index 26ad6c2..5f0be23 100644 --- a/pkg/kubevirt/client.go +++ b/pkg/kubevirt/client.go @@ -97,6 +97,11 @@ type Client struct { pvcWatcherCtx context.Context pvcWatcherCancel context.CancelFunc pvcWatcherWg sync.WaitGroup + + // Per-VM serialization of read-modify-write updates (see updateVM) and + // the backoff sleep between conflict retries, replaceable in tests. + vmLocks *keyedMutex + sleep func(time.Duration) } // init registers KubeVirt and CDI types with the runtime scheme for conversion @@ -143,6 +148,12 @@ func vmToUnstructured(vm *kubevirtv1.VirtualMachine) (*unstructured.Unstructured // computeVMPatch compares original and modified VMs and returns a JSON Merge Patch. // This preserves unknown fields that may exist in the cluster object but are not // represented in the typed VirtualMachine struct. +// +// The patch carries the resourceVersion of original. A merge patch replaces +// arrays such as disks and volumes wholesale, so one computed from a stale +// read would silently drop entries another writer just added; with the +// resourceVersion in it, the API server rejects such a patch with a Conflict +// and the caller (updateVM) recomputes it from a fresh read. func computeVMPatch(original, modified *kubevirtv1.VirtualMachine) ([]byte, error) { originalJSON, err := json.Marshal(original) if err != nil { @@ -157,7 +168,10 @@ func computeVMPatch(original, modified *kubevirtv1.VirtualMachine) ([]byte, erro if err != nil { return nil, fmt.Errorf("failed to create merge patch: %w", err) } - return patch, nil + if original.ResourceVersion == "" || isEmptyMergePatch(patch) { + return patch, nil + } + return withResourceVersion(patch, original.ResourceVersion) } // vmiToUnstructured converts a typed VirtualMachineInstance to an unstructured object @@ -238,6 +252,8 @@ func NewClient(configPath string, timeout time.Duration, appConfig interface{}) timeout: timeout, appConfig: appConfig, httpClient: httpClient, + vmLocks: newKeyedMutex(), + sleep: time.Sleep, }, nil } @@ -255,6 +271,8 @@ func NewClientWithClients( timeout: timeout, appConfig: appConfig, config: &rest.Config{}, + vmLocks: newKeyedMutex(), + sleep: time.Sleep, httpClient: &http.Client{ Timeout: timeout, Transport: &http.Transport{ @@ -1419,54 +1437,32 @@ func (c *Client) RecordVMBootOptionsAsAnnotations(namespace, name string, option logger.Info("Setting boot options for VM %s/%s: %v", namespace, name, options) - // Get the current VM - vm, err := c.GetVM(namespace, name) - if err != nil { - return fmt.Errorf("failed to get VM %s: %w", name, err) - } - - // Update VM with boot options - vmCopy := vm.DeepCopy() - // Store boot options in VM annotations for persistence - annotations := vmCopy.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + annotations := vmCopy.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } - // Update annotations with boot options - if bootSourceOverrideEnabled, found := options["bootSourceOverrideEnabled"]; found { - if enabled, ok := bootSourceOverrideEnabled.(string); ok { - annotations["redfish.boot.source.override.enabled"] = enabled + if bootSourceOverrideEnabled, found := options["bootSourceOverrideEnabled"]; found { + if enabled, ok := bootSourceOverrideEnabled.(string); ok { + annotations["redfish.boot.source.override.enabled"] = enabled + } } - } - if bootSourceOverrideTarget, found := options["bootSourceOverrideTarget"]; found { - if target, ok := bootSourceOverrideTarget.(string); ok { - annotations["redfish.boot.source.override.target"] = target + if bootSourceOverrideTarget, found := options["bootSourceOverrideTarget"]; found { + if target, ok := bootSourceOverrideTarget.(string); ok { + annotations["redfish.boot.source.override.target"] = target + } } - } - if bootSourceOverrideMode, found := options["bootSourceOverrideMode"]; found { - if mode, ok := bootSourceOverrideMode.(string); ok { - annotations["redfish.boot.source.override.mode"] = mode + if bootSourceOverrideMode, found := options["bootSourceOverrideMode"]; found { + if mode, ok := bootSourceOverrideMode.(string); ok { + annotations["redfish.boot.source.override.mode"] = mode + } } - } - - vmCopy.SetAnnotations(annotations) - - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - return fmt.Errorf("failed to compute VM patch: %w", err) - } - - // Apply patch - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + vmCopy.SetAnnotations(annotations) + return nil + }) if err != nil { return fmt.Errorf("failed to update VM boot options: %w", err) } @@ -1738,43 +1734,13 @@ func (c *Client) insertVirtualMediaAsync(namespace, name, mediaID, imageURL stri logger.Info("Creating CD-ROM device %s in VM spec first", deviceName) logger.Debug("Creating CD-ROM device %s in VM spec", deviceName) + // Only the spec change runs under the per-VM lock; PVC creation and the + // ISO download below are slow and must not block other writers. + // dataVolumeName was generated once above so a retried update keeps the + // same PVC name. var oldPVCName string - - maxRetries := 3 - for attempt := 1; attempt <= maxRetries; attempt++ { - logger.Debug("VM update attempt %d/%d", attempt, maxRetries) - - vm, err := c.GetVM(namespace, name) - if err != nil { - logger.Error("Failed to get VM %s/%s: %v", namespace, name, err) - return fmt.Errorf("failed to get VM %s: %w", name, err) - } - logger.Debug("Successfully retrieved VM %s/%s", namespace, name) - - vmCopy := vm.DeepCopy() - - if vmCopy.Spec.Template == nil { - vmCopy.Spec.Template = &kubevirtv1.VirtualMachineInstanceTemplateSpec{} - } - - diskExists := false - for _, disk := range vmCopy.Spec.Template.Spec.Domain.Devices.Disks { - if disk.Name == deviceName { - diskExists = true - break - } - } - - if !diskExists { - newDisk := kubevirtv1.Disk{ - Name: deviceName, - DiskDevice: kubevirtv1.DiskDevice{ - CDRom: &kubevirtv1.CDRomTarget{ - Bus: kubevirtv1.DiskBusSATA, - }, - }, - } - vmCopy.Spec.Template.Spec.Domain.Devices.Disks = append(vmCopy.Spec.Template.Spec.Domain.Devices.Disks, newDisk) + _, err = c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + if addCDROMDisk(vmCopy, deviceName) { logger.Info("Added new CD-ROM device %s", deviceName) } @@ -1823,34 +1789,19 @@ func (c *Client) insertVirtualMediaAsync(namespace, name, mediaID, imageURL stri annotations[VirtualMediaURLAnnotationPrefix+deviceName] = imageURL vmCopy.SetAnnotations(annotations) - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - logger.Error("Failed to compute VM patch: %v", err) - return fmt.Errorf("failed to compute VM patch: %w", err) - } - - gvrVM := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } - - logger.Debug("Updating VM %s/%s with new CD-ROM device", namespace, name) - _, err = c.dynamicClient.Resource(gvrVM).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) - if err != nil { - if strings.Contains(err.Error(), "the object has been modified") && attempt < maxRetries { - logger.Info("Concurrent modification detected, retrying VM update (attempt %d/%d)", attempt, maxRetries) - logger.Debug("Concurrent modification detected, retrying VM update (attempt %d/%d)", attempt, maxRetries) - time.Sleep(time.Duration(attempt) * time.Second) - continue - } - logger.Error("Failed to update VM %s/%s: %v", namespace, name, err) - return fmt.Errorf("failed to update VM: %w", err) + // Ironic sends the boot override right after the insert request. When + // the override reached the VM first it could only order the disks that + // existed then, so the new CD-ROM is placed first now. The result is + // the same whichever request reached the VM first. + if bootOverrideWantsCD(vmCopy) { + logger.Info("Boot override to CD is pending for VM %s/%s, placing %s first in the boot order", namespace, name, deviceName) + return c.applyPendingCDBootOverride(vmCopy, deviceName) } - - logger.Debug("Successfully updated VM %s/%s on attempt %d", namespace, name, attempt) - break + return nil + }) + if err != nil { + logger.Error("Failed to update VM %s/%s: %v", namespace, name, err) + return fmt.Errorf("failed to update VM: %w", err) } logger.Info("Successfully created CD-ROM device %s in VM spec", deviceName) @@ -2221,41 +2172,19 @@ func (c *Client) SetBootOrder(namespace, name, bootTarget string) error { logger.Info("Setting boot order to %s for VM %s/%s", bootTarget, namespace, name) - // Get the current VM - vm, err := c.GetVM(namespace, name) - if err != nil { - return fmt.Errorf("failed to get VM %s: %w", name, err) - } - - // Update VM with boot order configuration - vmCopy := vm.DeepCopy() - - // Collect current volumes and detect their type - // This is needed, because the device list does not provide enough - // information about disk type and there are disks that are not bootable - // like cloud init. - err = c.modifyVmBootOrder(vmCopy, bootTarget) - if err != nil { - return err - } - - // A permanent boot order change supersedes any pending boot-once state. - c.clearBootOnceState(vmCopy) - - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - return fmt.Errorf("failed to compute VM patch: %w", err) - } - - // Apply patch - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + // Collect current volumes and detect their type + // This is needed, because the device list does not provide enough + // information about disk type and there are disks that are not bootable + // like cloud init. + if err := c.modifyVmBootOrder(vmCopy, bootTarget); err != nil { + return err + } - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + // A permanent boot order change supersedes any pending boot-once state. + c.clearBootOnceState(vmCopy) + return nil + }) if err != nil { return fmt.Errorf("failed to update VM boot order: %w", err) } @@ -2339,106 +2268,84 @@ func (c *Client) SetBootOnce(namespace, name, bootTarget string) error { logger.Info("Setting boot once to %s for VM %s/%s", bootTarget, namespace, name) - // Get the current VM - vm, err := c.GetVM(namespace, name) - if err != nil { - return fmt.Errorf("failed to get VM %s: %w", name, err) - } + var currentVMIUID string + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + annotations := vmCopy.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + vmLabels := vmCopy.GetLabels() + if vmLabels == nil { + vmLabels = make(map[string]string) + } - // Update VM with boot once configuration - vmCopy := vm.DeepCopy() + // Check if boot-once is already configured + existingOriginalConfig := annotations[BootOnceOriginalConfigAnnotation] + existingVMIUID := annotations[BootOnceVMIUIDAnnotation] + hasExistingBootOnce := existingOriginalConfig != "" - // Get current annotations and labels - annotations := vmCopy.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - vmLabels := vmCopy.GetLabels() - if vmLabels == nil { - vmLabels = make(map[string]string) - } - - // Check if boot-once is already configured - existingOriginalConfig := annotations[BootOnceOriginalConfigAnnotation] - existingVMIUID := annotations[BootOnceVMIUIDAnnotation] - hasExistingBootOnce := existingOriginalConfig != "" + currentVMIUID = c.getVMIUID(namespace, name) - // Get current VMI UID - currentVMIUID := c.getVMIUID(namespace, name) + if hasExistingBootOnce { + // Boot-once already configured - check if the recorded VMI is still running + if existingVMIUID != "" && existingVMIUID == currentVMIUID { + // Same VMI is still running - keep original config, just update boot order + logger.Info("Boot-once already configured for VM %s/%s with same VMI, updating boot target only", namespace, name) + } else { + // VMI has changed or was empty - clear old state and capture new original config + logger.Info("Boot-once state is stale for VM %s/%s (VMI changed), capturing new original config", namespace, name) - if hasExistingBootOnce { - // Boot-once already configured - check if the recorded VMI is still running - if existingVMIUID != "" && existingVMIUID == currentVMIUID { - // Same VMI is still running - keep original config, just update boot order - logger.Info("Boot-once already configured for VM %s/%s with same VMI, updating boot target only", namespace, name) - } else { - // VMI has changed or was empty - clear old state and capture new original config - logger.Info("Boot-once state is stale for VM %s/%s (VMI changed), capturing new original config", namespace, name) + // Restore the original boot order so we capture the real original state below + if err := c.restoreBootOrder(vmCopy, existingOriginalConfig); err != nil { + logger.Warning("Failed to restore original boot order before recapture: %v", err) + } - // Restore the original boot order so we capture the real original state below - if err := c.restoreBootOrder(vmCopy, existingOriginalConfig); err != nil { - logger.Warning("Failed to restore original boot order before recapture: %v", err) + // Capture the (now restored) boot order as the new original config + originalConfig, err := c.captureCurrentBootOrder(vmCopy) + if err != nil { + return fmt.Errorf("failed to capture boot order: %w", err) + } + annotations[BootOnceOriginalConfigAnnotation] = originalConfig + annotations[BootOnceVMIUIDAnnotation] = currentVMIUID + // Keep existing BootOnceOriginalRebootPolicyAnnotation — it + // already holds the real original value from the first SetBootOnce call. } - - // Capture the (now restored) boot order as the new original config + } else { + // No existing boot-once — capture current boot order and rebootPolicy originalConfig, err := c.captureCurrentBootOrder(vmCopy) if err != nil { return fmt.Errorf("failed to capture boot order: %w", err) } annotations[BootOnceOriginalConfigAnnotation] = originalConfig annotations[BootOnceVMIUIDAnnotation] = currentVMIUID - // Keep existing BootOnceOriginalRebootPolicyAnnotation — it - // already holds the real original value from the first SetBootOnce call. - } - } else { - // No existing boot-once — capture current boot order and rebootPolicy - originalConfig, err := c.captureCurrentBootOrder(vmCopy) - if err != nil { - return fmt.Errorf("failed to capture boot order: %w", err) - } - annotations[BootOnceOriginalConfigAnnotation] = originalConfig - annotations[BootOnceVMIUIDAnnotation] = currentVMIUID - if vmCopy.Spec.Template != nil && vmCopy.Spec.Template.Spec.Domain.RebootPolicy != nil { - annotations[BootOnceOriginalRebootPolicyAnnotation] = string(*vmCopy.Spec.Template.Spec.Domain.RebootPolicy) + if vmCopy.Spec.Template != nil && vmCopy.Spec.Template.Spec.Domain.RebootPolicy != nil { + annotations[BootOnceOriginalRebootPolicyAnnotation] = string(*vmCopy.Spec.Template.Spec.Domain.RebootPolicy) + } } - } - // Add boot-once label for watch selector - vmLabels[BootOnceLabel] = "enabled" - vmCopy.SetLabels(vmLabels) - - // Store Redfish boot override annotations - annotations["redfish.boot.source.override.enabled"] = "Once" - annotations["redfish.boot.source.override.target"] = bootTarget - annotations["redfish.boot.source.override.mode"] = "UEFI" - vmCopy.SetAnnotations(annotations) - - // Modify boot order to boot from target - if err := c.modifyVmBootOrder(vmCopy, bootTarget); err != nil { - return fmt.Errorf("failed to modify boot order: %w", err) - } - - // Set rebootPolicy to Terminate so that KubeVirt destroys the VMI on - // guest-initiated reboot and we can detect the VMI UID change. - if vmCopy.Spec.Template != nil { - terminate := kubevirtv1.RebootPolicyTerminate - vmCopy.Spec.Template.Spec.Domain.RebootPolicy = &terminate - } + // Add boot-once label for watch selector + vmLabels[BootOnceLabel] = "enabled" + vmCopy.SetLabels(vmLabels) - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - return fmt.Errorf("failed to compute VM patch: %w", err) - } + // Store Redfish boot override annotations + annotations["redfish.boot.source.override.enabled"] = "Once" + annotations["redfish.boot.source.override.target"] = bootTarget + annotations["redfish.boot.source.override.mode"] = "UEFI" + vmCopy.SetAnnotations(annotations) - // Apply patch - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } + // Modify boot order to boot from target + if err := c.modifyVmBootOrder(vmCopy, bootTarget); err != nil { + return fmt.Errorf("failed to modify boot order: %w", err) + } - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + // Set rebootPolicy to Terminate so that KubeVirt destroys the VMI on + // guest-initiated reboot and we can detect the VMI UID change. + if vmCopy.Spec.Template != nil { + terminate := kubevirtv1.RebootPolicyTerminate + vmCopy.Spec.Template.Spec.Domain.RebootPolicy = &terminate + } + return nil + }) if err != nil { return fmt.Errorf("failed to update VM boot once configuration: %w", err) } @@ -2454,89 +2361,60 @@ func (c *Client) EjectVirtualMedia(namespace, name, mediaID string) error { logger.Info("Ejecting virtual media %s for VM %s/%s", mediaID, namespace, name) - // Get the current VM - vm, err := c.GetVM(namespace, name) - if err != nil { - logger.Error("Failed to get VM %s/%s for ejection: %v", namespace, name, err) - return fmt.Errorf("failed to get VM %s: %w", name, err) - } + // The PVC name and the helper pod name are taken from the copy that is + // actually written, because PVC names carry unique suffixes and a retried + // update may see a different one. + var actualPVCName, helperPodName string + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + if vmCopy.Spec.Template == nil { + logger.Warning("No template found in VM %s/%s when trying to eject media %s", namespace, name, mediaID) + return fmt.Errorf("no template found in VM") + } - logger.Debug("Current VM spec before ejection") + actualPVCName = "" + helperPodName = vmCopy.GetLabels()[ImportingLabelPrefix+mediaID] - // Get the actual PVC name from the volume BEFORE removing it from the VM - // This is necessary because PVC names have unique identifiers appended - var actualPVCName string - if vm.Spec.Template != nil { - for _, volume := range vm.Spec.Template.Spec.Volumes { - if volume.Name == mediaID && volume.PersistentVolumeClaim != nil { - actualPVCName = volume.PersistentVolumeClaim.ClaimName - logger.Info("Found actual PVC name %s for media %s", actualPVCName, mediaID) - break + // Remove the specified disk from disks list + var newDisks []kubevirtv1.Disk + foundDisk := false + for _, disk := range vmCopy.Spec.Template.Spec.Domain.Devices.Disks { + if disk.Name != mediaID { + newDisks = append(newDisks, disk) + } else { + logger.Info("Removing CD-ROM device %s from VM spec", mediaID) + foundDisk = true } } - } - - // Update VM to remove CD-ROM disk and volume reference - vmCopy := vm.DeepCopy() - - if vmCopy.Spec.Template == nil { - logger.Warning("No template found in VM %s/%s when trying to eject media %s", namespace, name, mediaID) - return fmt.Errorf("no template found in VM") - } - - // Remove the specified disk from disks list - var newDisks []kubevirtv1.Disk - foundDisk := false - for _, disk := range vmCopy.Spec.Template.Spec.Domain.Devices.Disks { - if disk.Name != mediaID { - newDisks = append(newDisks, disk) - } else { - logger.Info("Removing CD-ROM device %s from VM spec", mediaID) - foundDisk = true + if !foundDisk { + logger.Warning("CD-ROM device %s not found in VM %s/%s disks list during ejection", mediaID, namespace, name) } - } - if !foundDisk { - logger.Warning("CD-ROM device %s not found in VM %s/%s disks list during ejection", mediaID, namespace, name) - } - vmCopy.Spec.Template.Spec.Domain.Devices.Disks = newDisks + vmCopy.Spec.Template.Spec.Domain.Devices.Disks = newDisks - // Remove the volume reference - var newVolumes []kubevirtv1.Volume - foundVolume := false - for _, volume := range vmCopy.Spec.Template.Spec.Volumes { - if volume.Name != mediaID { - newVolumes = append(newVolumes, volume) - } else { + // Remove the volume reference + var newVolumes []kubevirtv1.Volume + foundVolume := false + for _, volume := range vmCopy.Spec.Template.Spec.Volumes { + if volume.Name != mediaID { + newVolumes = append(newVolumes, volume) + continue + } + if volume.PersistentVolumeClaim != nil { + actualPVCName = volume.PersistentVolumeClaim.ClaimName + logger.Info("Found actual PVC name %s for media %s", actualPVCName, mediaID) + } logger.Info("Removing volume reference %s from VM spec", mediaID) foundVolume = true } - } - if !foundVolume { - logger.Warning("Volume reference %s not found in VM %s/%s during ejection", mediaID, namespace, name) - } - vmCopy.Spec.Template.Spec.Volumes = newVolumes - - annotations := vmCopy.GetAnnotations() - delete(annotations, VirtualMediaURLAnnotationPrefix+mediaID) - vmCopy.SetAnnotations(annotations) - - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - logger.Error("Failed to compute VM patch for %s/%s: %v", namespace, name, err) - return fmt.Errorf("failed to compute VM patch: %w", err) - } - - logger.Debug("VM spec after removing CD-ROM and volume") - - // Apply patch - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } + if !foundVolume { + logger.Warning("Volume reference %s not found in VM %s/%s during ejection", mediaID, namespace, name) + } + vmCopy.Spec.Template.Spec.Volumes = newVolumes - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + annotations := vmCopy.GetAnnotations() + delete(annotations, VirtualMediaURLAnnotationPrefix+mediaID) + vmCopy.SetAnnotations(annotations) + return nil + }) if err != nil { logger.Error("Failed to update VM %s/%s after removing CD-ROM and volume: %v", namespace, name, err) return fmt.Errorf("failed to update VM: %w", err) @@ -2546,8 +2424,6 @@ func (c *Client) EjectVirtualMedia(namespace, name, mediaID string) error { // Kill any running helper pod for this device and remove the importing label // so a stale pod cannot trigger a deferred power-on after the media is gone. - labelKey := ImportingLabelPrefix + mediaID - helperPodName := vm.GetLabels()[labelKey] if helperPodName != "" { logger.Info("Cleaning up active import for device %s: deleting helper pod %s and removing importing label", mediaID, helperPodName) if delErr := c.kubernetesClient.CoreV1().Pods(namespace).Delete(ctx, helperPodName, metav1.DeleteOptions{}); delErr != nil { @@ -3270,6 +3146,10 @@ func (c *Client) handleVMUpdate(vm *kubevirtv1.VirtualMachine) { return } + // A lost write can leave an inserted device with its volume but no disk + // entry; put the disk back before anything else looks at the VM. + c.repairVirtualMediaDevices(currentVM) + vmLabels := currentVM.GetLabels() // Handle power-after-import if applicable @@ -3314,35 +3194,28 @@ func (c *Client) handleVMUpdate(vm *kubevirtv1.VirtualMachine) { logger.Info("VMI UID changed for VM %s/%s (recorded: %s, current: %s), restoring boot order", namespace, name, recordedVMIUID, currentVMIUID) - vmCopy := currentVM.DeepCopy() - - // Restore the original boot order - if err := c.restoreBootOrder(vmCopy, originalConfig); err != nil { - logger.Error("Failed to restore boot order for VM %s/%s: %v", namespace, name, err) - return - } - - // Remove boot-once state (label, annotations) and restore original rebootPolicy - c.clearBootOnceState(vmCopy) - - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(currentVM, vmCopy) - if err != nil { - logger.Error("Failed to compute VM patch: %v", err) - return - } - - // Apply patch ctx, cancel := context.WithTimeout(context.Background(), c.timeout) defer cancel() - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + // Decide again on the fresh copy: another writer, or an earlier + // pass of this watcher, may already have restored the VM. + if vmCopy.GetLabels()[BootOnceLabel] != "enabled" { + return nil + } + originalConfig := vmCopy.GetAnnotations()[BootOnceOriginalConfigAnnotation] + if originalConfig == "" { + return nil + } + + if err := c.restoreBootOrder(vmCopy, originalConfig); err != nil { + return fmt.Errorf("failed to restore boot order: %w", err) + } - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + // Remove boot-once state (label, annotations) and restore original rebootPolicy + c.clearBootOnceState(vmCopy) + return nil + }) if err != nil { logger.Error("Failed to update VM %s/%s with restored boot order: %v", namespace, name, err) return diff --git a/pkg/kubevirt/mock_dynamic_client.go b/pkg/kubevirt/mock_dynamic_client.go index 3eec7ee..3cbdb94 100644 --- a/pkg/kubevirt/mock_dynamic_client.go +++ b/pkg/kubevirt/mock_dynamic_client.go @@ -23,9 +23,11 @@ import ( "context" "encoding/json" "fmt" + "strconv" "strings" "sync" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -54,10 +56,28 @@ func deepCopyUnstructured(obj *unstructured.Unstructured) (*unstructured.Unstruc return &unstructured.Unstructured{Object: newObj}, nil } -// MockDynamicClient implements dynamic.Interface for testing +// MockDynamicClient implements dynamic.Interface for testing. +// +// Writes follow the API server's rules that matter for read-modify-write +// code: every stored object has a resourceVersion that is bumped on each +// write, a patch is applied atomically, and a merge patch that carries a +// metadata.resourceVersion different from the stored one is rejected with a +// Conflict error. type MockDynamicClient struct { mu sync.RWMutex resources map[string]map[string]*unstructured.Unstructured // "gvr/namespace" -> name -> object + + // IgnoreResourceVersion accepts patches whose resourceVersion is stale, + // which is how the API server treats a patch that carries none. Tests use + // it to show what happens without the resourceVersion precondition. + IgnoreResourceVersion bool + + // BeforePatch, when set, runs before a patch is applied and outside the + // store lock. Tests use it to let another writer change the object between + // a caller's read and its write. It must change the store through + // MutateVM or the dynamic interface, never through a *Client, which would + // re-enter the client's per-VM lock. + BeforePatch func(gvr schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte) } // NewMockDynamicClient creates a new mock dynamic client @@ -93,6 +113,14 @@ func (m *MockDynamicClient) addResource(gvr schema.GroupVersionResource, namespa m.mu.Lock() defer m.mu.Unlock() + if obj.GetResourceVersion() == "" { + obj.SetResourceVersion("1") + } + m.storeLocked(gvr, namespace, name, obj) +} + +// storeLocked stores obj; the caller holds m.mu. +func (m *MockDynamicClient) storeLocked(gvr schema.GroupVersionResource, namespace, name string, obj *unstructured.Unstructured) { key := resourceKey(gvr, namespace) if m.resources[key] == nil { m.resources[key] = make(map[string]*unstructured.Unstructured) @@ -100,6 +128,67 @@ func (m *MockDynamicClient) addResource(gvr schema.GroupVersionResource, namespa m.resources[key][name] = obj } +// getLocked returns the stored object; the caller holds m.mu. +func (m *MockDynamicClient) getLocked(gvr schema.GroupVersionResource, namespace, name string) (*unstructured.Unstructured, bool) { + key := resourceKey(gvr, namespace) + if m.resources[key] == nil { + return nil, false + } + obj, found := m.resources[key][name] + return obj, found +} + +// bumpResourceVersion gives obj the next resourceVersion, as the API server +// does on every successful write. +func bumpResourceVersion(obj *unstructured.Unstructured) { + n, err := strconv.Atoi(obj.GetResourceVersion()) + if err != nil { + n = 0 + } + obj.SetResourceVersion(strconv.Itoa(n + 1)) +} + +func notFoundError(gvr schema.GroupVersionResource, name string) error { + return apierrors.NewNotFound(schema.GroupResource{Group: gvr.Group, Resource: gvr.Resource}, name) +} + +func conflictError(gvr schema.GroupVersionResource, name string) error { + return apierrors.NewConflict(schema.GroupResource{Group: gvr.Group, Resource: gvr.Resource}, name, + fmt.Errorf("the object has been modified; please apply your changes to the latest version and try again")) +} + +// MutateVM changes a stored VM in place, bumping its resourceVersion, the way +// a write by another client would. It is safe to call from BeforePatch. +func (m *MockDynamicClient) MutateVM(namespace, name string, mutate func(vm *kubevirtv1.VirtualMachine)) error { + gvr := schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + + m.mu.Lock() + defer m.mu.Unlock() + + obj, found := m.getLocked(gvr, namespace, name) + if !found { + return notFoundError(gvr, name) + } + vm, err := unstructuredToVM(obj) + if err != nil { + return err + } + if vm.Labels == nil { + vm.Labels = map[string]string{} + } + if vm.Annotations == nil { + vm.Annotations = map[string]string{} + } + mutate(vm) + u, err := vmToUnstructured(vm) + if err != nil { + return err + } + bumpResourceVersion(u) + m.storeLocked(gvr, namespace, name, u) + return nil +} + // getResource gets a resource from the mock client func (m *MockDynamicClient) getResource(gvr schema.GroupVersionResource, namespace, name string) (*unstructured.Unstructured, bool) { m.mu.RLock() @@ -307,8 +396,18 @@ func (r *mockNamespaceableResource) Update(ctx context.Context, obj *unstructure if err != nil { return nil, err } - r.client.addResource(r.gvr, namespace, name, copied) - return deepCopyUnstructured(obj) + + r.client.mu.Lock() + defer r.client.mu.Unlock() + + if existing, found := r.client.getLocked(r.gvr, namespace, name); found && !r.client.IgnoreResourceVersion { + if rv := copied.GetResourceVersion(); rv != "" && rv != existing.GetResourceVersion() { + return nil, conflictError(r.gvr, name) + } + } + bumpResourceVersion(copied) + r.client.storeLocked(r.gvr, namespace, name, copied) + return deepCopyUnstructured(copied) } // UpdateStatus implements dynamic.ResourceInterface @@ -321,7 +420,7 @@ func (r *mockNamespaceableResource) Delete(ctx context.Context, name string, opt if r.client.deleteResource(r.gvr, r.namespace, name) { return nil } - return fmt.Errorf("resource %s/%s not found", r.namespace, name) + return notFoundError(r.gvr, name) } // DeleteCollection implements dynamic.ResourceInterface @@ -333,7 +432,7 @@ func (r *mockNamespaceableResource) DeleteCollection(ctx context.Context, option func (r *mockNamespaceableResource) Get(ctx context.Context, name string, options metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { obj, found := r.client.getResource(r.gvr, r.namespace, name) if !found { - return nil, fmt.Errorf("%s \"%s\" not found", r.gvr.Resource, name) + return nil, notFoundError(r.gvr, name) } return deepCopyUnstructured(obj) } @@ -365,11 +464,19 @@ func (r *mockNamespaceableResource) Watch(ctx context.Context, opts metav1.ListO return watch.NewFake(), nil } -// Patch implements dynamic.ResourceInterface +// Patch implements dynamic.ResourceInterface. The patch is applied under the +// store lock so two concurrent patches cannot both start from the same copy. func (r *mockNamespaceableResource) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) { - obj, found := r.client.getResource(r.gvr, r.namespace, name) + if r.client.BeforePatch != nil { + r.client.BeforePatch(r.gvr, r.namespace, name, pt, data) + } + + r.client.mu.Lock() + defer r.client.mu.Unlock() + + obj, found := r.client.getLocked(r.gvr, r.namespace, name) if !found { - return nil, fmt.Errorf("%s \"%s\" not found", r.gvr.Resource, name) + return nil, notFoundError(r.gvr, name) } // Make a copy to modify @@ -378,8 +485,8 @@ func (r *mockNamespaceableResource) Patch(ctx context.Context, name string, pt t return nil, err } - // Apply JSON patch if it's a JSONPatchType - if pt == types.JSONPatchType { + switch pt { + case types.JSONPatchType: var patches []jsonPatchOp if err := json.Unmarshal(data, &patches); err != nil { return nil, fmt.Errorf("failed to unmarshal JSON patch: %w", err) @@ -391,27 +498,38 @@ func (r *mockNamespaceableResource) Patch(ctx context.Context, name string, pt t } } - // Store the updated object - r.client.addResource(r.gvr, r.namespace, name, copied) - } - - // Apply Merge patch if it's a MergePatchType (RFC 7396) - if pt == types.MergePatchType { + case types.MergePatchType: // RFC 7396 var patchMap map[string]interface{} if err := json.Unmarshal(data, &patchMap); err != nil { return nil, fmt.Errorf("failed to unmarshal merge patch: %w", err) } + if !r.client.IgnoreResourceVersion { + if rv := mergePatchResourceVersion(patchMap); rv != "" && rv != obj.GetResourceVersion() { + return nil, conflictError(r.gvr, name) + } + } + // Deep merge patchMap into copied.Object mergeMaps(copied.Object, patchMap) - // Store the updated object - r.client.addResource(r.gvr, r.namespace, name, copied) + default: + return deepCopyUnstructured(copied) } + bumpResourceVersion(copied) + r.client.storeLocked(r.gvr, r.namespace, name, copied) return deepCopyUnstructured(copied) } +// mergePatchResourceVersion returns metadata.resourceVersion from a decoded +// merge patch, or "" when the patch does not carry one. +func mergePatchResourceVersion(patchMap map[string]interface{}) string { + metadata, _ := patchMap["metadata"].(map[string]interface{}) + rv, _ := metadata["resourceVersion"].(string) + return rv +} + // mergeMaps recursively merges src into dst according to RFC 7396 JSON Merge Patch. // - If src has a key with value null, the key is deleted from dst // - If src has a key with a non-null value, it replaces the value in dst diff --git a/pkg/kubevirt/vm_update.go b/pkg/kubevirt/vm_update.go new file mode 100644 index 0000000..e226c45 --- /dev/null +++ b/pkg/kubevirt/vm_update.go @@ -0,0 +1,304 @@ +package kubevirt + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + kubevirtv1 "kubevirt.io/api/core/v1" + + "github.com/kubevirt/redfish-controller/pkg/logger" +) + +const ( + // vmUpdateMaxAttempts bounds how often updateVM re-reads the VM and + // re-applies a mutation after the API server rejected the write as stale. + vmUpdateMaxAttempts = 5 + vmUpdateBaseBackoff = 50 * time.Millisecond + vmUpdateMaxBackoff = 500 * time.Millisecond +) + +// updateVM is the only path for changing a VirtualMachine by reading it, +// editing a copy and patching the difference back. +// +// Two things make such a read-modify-write safe against another writer: +// +// - A per-VM lock serializes the writers inside this process. An HTTP +// handler (boot override), a task worker (virtual media insert) and the +// boot-once watcher can all touch the same VM within milliseconds of each +// other. The lock covers only the read-modify-write; slow work such as +// creating PVCs or downloading ISOs must stay outside it. +// - The patch carries the resourceVersion of the object it was computed +// from (see computeVMPatch), so the API server rejects it with a Conflict +// when anyone changed the VM in between, instead of letting a stale copy +// of the disks or volumes array silently replace a newer one. The +// mutation is then re-run on a fresh read. This also covers writers +// outside this process. +// +// Because mutate can run more than once it must not have side effects beyond +// the copy it is given, and it must take every decision from that copy. +// Nothing inside mutate may call updateVM again: the lock is not re-entrant. +// The read is always a live GET; an object served from a cache would both +// manufacture spurious conflicts and be re-read unchanged on retry. +func (c *Client) updateVM(ctx context.Context, namespace, name string, mutate func(vm *kubevirtv1.VirtualMachine) error) (*kubevirtv1.VirtualMachine, error) { + unlock := c.lockVM(namespace, name) + defer unlock() + + gvr := schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + + for attempt := 1; ; attempt++ { + vm, err := c.GetVM(namespace, name) + if err != nil { + return nil, fmt.Errorf("failed to get VM %s: %w", name, err) + } + + vmCopy := vm.DeepCopy() + if err := mutate(vmCopy); err != nil { + return nil, err + } + + patch, err := computeVMPatch(vm, vmCopy) + if err != nil { + return nil, fmt.Errorf("failed to compute VM patch: %w", err) + } + if isEmptyMergePatch(patch) { + return vm, nil + } + + updated, err := c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err == nil { + result, convErr := unstructuredToVM(updated) + if convErr != nil { + return vmCopy, nil + } + return result, nil + } + if !apierrors.IsConflict(err) || attempt >= vmUpdateMaxAttempts { + return nil, fmt.Errorf("failed to update VM %s/%s: %w", namespace, name, err) + } + + logger.Info("VM %s/%s changed while an update was being prepared, retrying on a fresh copy (attempt %d/%d)", namespace, name, attempt, vmUpdateMaxAttempts) + c.sleepFor(vmUpdateBackoff(attempt)) + } +} + +func (c *Client) lockVM(namespace, name string) func() { + if c.vmLocks == nil { + return func() {} + } + return c.vmLocks.Lock(namespace + "/" + name) +} + +func (c *Client) sleepFor(d time.Duration) { + if c.sleep != nil { + c.sleep(d) + return + } + time.Sleep(d) +} + +func vmUpdateBackoff(attempt int) time.Duration { + d := vmUpdateBaseBackoff * time.Duration(attempt) + if d > vmUpdateMaxBackoff { + return vmUpdateMaxBackoff + } + return d +} + +func isEmptyMergePatch(patch []byte) bool { + var fields map[string]json.RawMessage + if err := json.Unmarshal(patch, &fields); err != nil { + return false + } + return len(fields) == 0 +} + +// withResourceVersion sets metadata.resourceVersion in a JSON merge patch. +func withResourceVersion(patch []byte, resourceVersion string) ([]byte, error) { + var fields map[string]interface{} + if err := json.Unmarshal(patch, &fields); err != nil { + return nil, fmt.Errorf("failed to decode merge patch: %w", err) + } + metadata, _ := fields["metadata"].(map[string]interface{}) + if metadata == nil { + metadata = map[string]interface{}{} + fields["metadata"] = metadata + } + metadata["resourceVersion"] = resourceVersion + return json.Marshal(fields) +} + +// addCDROMDisk appends a SATA CD-ROM disk entry named deviceName unless one +// already exists. It returns whether the entry was added. +func addCDROMDisk(vm *kubevirtv1.VirtualMachine, deviceName string) bool { + if vm.Spec.Template == nil { + vm.Spec.Template = &kubevirtv1.VirtualMachineInstanceTemplateSpec{} + } + for _, disk := range vm.Spec.Template.Spec.Domain.Devices.Disks { + if disk.Name == deviceName { + return false + } + } + vm.Spec.Template.Spec.Domain.Devices.Disks = append(vm.Spec.Template.Spec.Domain.Devices.Disks, kubevirtv1.Disk{ + Name: deviceName, + DiskDevice: kubevirtv1.DiskDevice{ + CDRom: &kubevirtv1.CDRomTarget{Bus: kubevirtv1.DiskBusSATA}, + }, + }) + return true +} + +// bootOverrideWantsCD reports whether a one-time boot override to CD was +// recorded on the VM before the CD-ROM device existed, so a device added now +// must be placed first in the boot order. +// +// It requires the boot-once state to be fully recorded (label and saved +// original boot order), not just the override annotations: handleBootUpdate +// writes the annotations first and the boot-once state a moment later, and +// reordering in between would make SetBootOnce capture the reordered disks +// as the "original" boot order. A permanent boot order change (SetBootOrder) +// clears the override annotations, so it never reaches here. +func bootOverrideWantsCD(vm *kubevirtv1.VirtualMachine) bool { + annotations := vm.GetAnnotations() + return annotations["redfish.boot.source.override.enabled"] == "Once" && + annotations["redfish.boot.source.override.target"] == "Cd" && + vm.GetLabels()[BootOnceLabel] == "enabled" && + annotations[BootOnceOriginalConfigAnnotation] != "" +} + +// applyPendingCDBootOverride re-stamps the boot order so CD-ROM devices come +// first and records the new device in the saved original boot order with no +// boot order of its own. The restore after the one-time boot then clears the +// device's bootOrder instead of leaving two disks at the same position. +func (c *Client) applyPendingCDBootOverride(vm *kubevirtv1.VirtualMachine, deviceName string) error { + if err := c.modifyVmBootOrder(vm, "Cd"); err != nil { + return err + } + return addDiskToOriginalBootConfig(vm, deviceName) +} + +func addDiskToOriginalBootConfig(vm *kubevirtv1.VirtualMachine, diskName string) error { + annotations := vm.GetAnnotations() + configJSON := annotations[BootOnceOriginalConfigAnnotation] + if configJSON == "" { + return nil + } + var bootOrders []BootOrderConfig + if err := json.Unmarshal([]byte(configJSON), &bootOrders); err != nil { + return fmt.Errorf("failed to unmarshal boot order config: %w", err) + } + for _, entry := range bootOrders { + if entry.DiskName == diskName { + return nil + } + } + bootOrders = append(bootOrders, BootOrderConfig{DiskName: diskName}) + updated, err := json.Marshal(bootOrders) + if err != nil { + return fmt.Errorf("failed to marshal boot order config: %w", err) + } + annotations[BootOnceOriginalConfigAnnotation] = string(updated) + vm.SetAnnotations(annotations) + return nil +} + +// virtualMediaDevices lists the devices the bridge has inserted media into, +// taken from the virtual-media.redfish/ annotations. +func virtualMediaDevices(vm *kubevirtv1.VirtualMachine) []string { + var devices []string + for key := range vm.GetAnnotations() { + if strings.HasPrefix(key, VirtualMediaURLAnnotationPrefix) { + devices = append(devices, strings.TrimPrefix(key, VirtualMediaURLAnnotationPrefix)) + } + } + return devices +} + +func hasDisk(vm *kubevirtv1.VirtualMachine, name string) bool { + if vm.Spec.Template == nil { + return false + } + for _, disk := range vm.Spec.Template.Spec.Domain.Devices.Disks { + if disk.Name == name { + return true + } + } + return false +} + +func hasVolume(vm *kubevirtv1.VirtualMachine, name string) bool { + if vm.Spec.Template == nil { + return false + } + for _, volume := range vm.Spec.Template.Spec.Volumes { + if volume.Name == name { + return true + } + } + return false +} + +// virtualMediaDevicesMissingDisk returns the inserted devices whose volume is +// present but whose disk entry is not. A volume without a disk entry is the +// half-state a lost write leaves behind; KubeVirt attaches it as a writable +// hard disk, which fails on a read-only ISO and leaves the VM crash-looping. +func virtualMediaDevicesMissingDisk(vm *kubevirtv1.VirtualMachine) []string { + var missing []string + for _, device := range virtualMediaDevices(vm) { + if hasVolume(vm, device) && !hasDisk(vm, device) { + missing = append(missing, device) + } + } + return missing +} + +// repairVirtualMediaDevices restores the disk entry of every inserted device +// that lost it, and places it first in the boot order when a boot override to +// CD is pending. It is run from the VM watcher, so the repair happens on the +// next event or reconcile tick without any long-lived goroutine, and the +// decision is re-taken on a fresh copy inside the locked update. +func (c *Client) repairVirtualMediaDevices(vm *kubevirtv1.VirtualMachine) { + namespace, name := vm.GetNamespace(), vm.GetName() + + // The opposite half-state is only logged: it is not the state a lost + // write produces, and the next insert or eject rewrites both halves. + // The watcher sees a VM only when it changes, so this does not repeat. + for _, device := range virtualMediaDevices(vm) { + if hasDisk(vm, device) && !hasVolume(vm, device) { + logger.Warning("VM %s/%s has a disk entry for virtual media device %s but no volume", namespace, name, device) + } + } + if len(virtualMediaDevicesMissingDisk(vm)) == 0 { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + + var repaired []string + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + repaired = virtualMediaDevicesMissingDisk(vmCopy) + for _, device := range repaired { + addCDROMDisk(vmCopy, device) + if bootOverrideWantsCD(vmCopy) { + if err := c.applyPendingCDBootOverride(vmCopy, device); err != nil { + return err + } + } + } + return nil + }) + if err != nil { + logger.Error("Failed to repair virtual media disk entries for VM %s/%s: %v", namespace, name, err) + return + } + if len(repaired) > 0 { + logger.Warning("Repaired VM %s/%s: re-added the missing CD-ROM disk entry for virtual media device(s) %v", namespace, name, repaired) + } +} diff --git a/pkg/kubevirt/vm_update_test.go b/pkg/kubevirt/vm_update_test.go new file mode 100644 index 0000000..dbeeab1 --- /dev/null +++ b/pkg/kubevirt/vm_update_test.go @@ -0,0 +1,619 @@ +package kubevirt + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + kubevirtv1 "kubevirt.io/api/core/v1" +) + +const ( + testISOURL = "https://example.com/boot.iso" + cdromDevice = "cdrom0" + rootDiskName = "rootdisk" +) + +// newRootDiskVM mirrors the VMs Ironic drives: one virtio root disk that is +// the only boot device, and no CD-ROM yet. +func newRootDiskVM(namespace, name string) *kubevirtv1.VirtualMachine { + rootBootOrder := uint(1) + return &kubevirtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + Spec: kubevirtv1.VirtualMachineSpec{ + Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Domain: kubevirtv1.DomainSpec{ + Devices: kubevirtv1.Devices{ + Disks: []kubevirtv1.Disk{{ + Name: rootDiskName, + BootOrder: &rootBootOrder, + DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: kubevirtv1.DiskBusVirtio}}, + }}, + }, + }, + Volumes: []kubevirtv1.Volume{{ + Name: rootDiskName, + VolumeSource: kubevirtv1.VolumeSource{DataVolume: &kubevirtv1.DataVolumeSource{Name: name + "-root"}}, + }}, + }, + }, + }, + } +} + +// withRootBootOrder sets the root disk's boot order. A previous enroll cycle +// leaves it at 2 (the CD-ROM was 1) after the eject removed the CD-ROM, which +// is the state the corrupted production VMs were in: the next boot-once then +// changes the disks array (2 -> 1), so its patch carries the whole array. +func withRootBootOrder(vm *kubevirtv1.VirtualMachine, order uint) *kubevirtv1.VirtualMachine { + diskByName(vm, rootDiskName).BootOrder = &order + return vm +} + +// withBootOnceToCD puts the VM in the state SetBootOnce leaves behind when +// it ran before any CD-ROM existed: boot-once label, saved original boot +// order with only the root disk, override annotations, root disk still first. +func withBootOnceToCD(vm *kubevirtv1.VirtualMachine) *kubevirtv1.VirtualMachine { + vm.Labels[BootOnceLabel] = "enabled" + vm.Annotations[BootOnceOriginalConfigAnnotation] = `[{"diskName":"rootdisk","bootOrder":1}]` + vm.Annotations[BootOnceVMIUIDAnnotation] = "" + vm.Annotations["redfish.boot.source.override.enabled"] = "Once" + vm.Annotations["redfish.boot.source.override.target"] = "Cd" + vm.Annotations["redfish.boot.source.override.mode"] = "UEFI" + return vm +} + +func withCDROM(vm *kubevirtv1.VirtualMachine, claimName string) *kubevirtv1.VirtualMachine { + if vm.Annotations == nil { + vm.Annotations = map[string]string{} + } + addCDROMDisk(vm, cdromDevice) + vm.Spec.Template.Spec.Volumes = append(vm.Spec.Template.Spec.Volumes, kubevirtv1.Volume{ + Name: cdromDevice, + VolumeSource: kubevirtv1.VolumeSource{PersistentVolumeClaim: &kubevirtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ClaimName: claimName}, + }}, + }) + vm.Annotations[VirtualMediaURLAnnotationPrefix+cdromDevice] = testISOURL + return vm +} + +func newVMWriterTestClient(t *testing.T, vms ...*kubevirtv1.VirtualMachine) (*Client, *MockDynamicClient, *fake.Clientset) { + t.Helper() + mock := NewMockDynamicClient() + for _, vm := range vms { + if err := mock.AddVM(vm); err != nil { + t.Fatalf("failed to add VM %s: %v", vm.Name, err) + } + } + cfg := &MockConfig{} + cfg.dataVolumeConfig.storageSize = "3Gi" + cfg.dataVolumeConfig.vmUpdateTimeout = "30s" + cfg.dataVolumeConfig.isoDownloadTimeout = "30m" + cfg.dataVolumeConfig.helperImage = "alpine:latest" + + k8s := fake.NewSimpleClientset() + client := NewClientWithClients(k8s, mock, 30*time.Second, cfg) + client.sleep = func(time.Duration) {} + return client, mock, k8s +} + +func diskByName(vm *kubevirtv1.VirtualMachine, name string) *kubevirtv1.Disk { + for i := range vm.Spec.Template.Spec.Domain.Devices.Disks { + if vm.Spec.Template.Spec.Domain.Devices.Disks[i].Name == name { + return &vm.Spec.Template.Spec.Domain.Devices.Disks[i] + } + } + return nil +} + +func volumeByName(vm *kubevirtv1.VirtualMachine, name string) *kubevirtv1.Volume { + for i := range vm.Spec.Template.Spec.Volumes { + if vm.Spec.Template.Spec.Volumes[i].Name == name { + return &vm.Spec.Template.Spec.Volumes[i] + } + } + return nil +} + +func bootOrderOf(disk *kubevirtv1.Disk) string { + if disk == nil { + return "" + } + if disk.BootOrder == nil { + return "none" + } + return fmt.Sprint(*disk.BootOrder) +} + +func assertBootOrder(t *testing.T, vm *kubevirtv1.VirtualMachine, diskName, want string) { + t.Helper() + if got := bootOrderOf(diskByName(vm, diskName)); got != want { + t.Errorf("disk %s: boot order %s, want %s", diskName, got, want) + } +} + +// assertVolumeDiskPairs checks the invariant the bug violated: an inserted +// virtual media device has both its volume and its disk entry, or neither. +func assertVolumeDiskPairs(t *testing.T, vm *kubevirtv1.VirtualMachine) { + t.Helper() + for _, device := range virtualMediaDevices(vm) { + if hasVolume(vm, device) != hasDisk(vm, device) { + t.Errorf("VM %s: device %s has volume=%v disk=%v, must have both or neither", vm.Name, device, hasVolume(vm, device), hasDisk(vm, device)) + } + } + for _, volume := range vm.Spec.Template.Spec.Volumes { + if strings.HasPrefix(volume.Name, "cdrom") && !hasDisk(vm, volume.Name) { + t.Errorf("VM %s: volume %s has no disk entry; KubeVirt would attach it as a writable hard disk", vm.Name, volume.Name) + } + } +} + +// landInsertDirectly simulates the insert worker's patch reaching the API +// server while another writer sits between its read and its write. +func landInsertDirectly(t *testing.T, mock *MockDynamicClient, namespace, name string) { + t.Helper() + if err := mock.MutateVM(namespace, name, func(vm *kubevirtv1.VirtualMachine) { withCDROM(vm, name+"-bootiso-1") }); err != nil { + t.Fatalf("failed to inject the insert: %v", err) + } +} + +func TestComputeVMPatch_CarriesResourceVersion(t *testing.T) { + vm := newRootDiskVM("ns", "vm") + vm.ResourceVersion = "42" + modified := vm.DeepCopy() + modified.Annotations["a"] = "b" + + patch, err := computeVMPatch(vm, modified) + if err != nil { + t.Fatal(err) + } + var fields map[string]interface{} + if err := json.Unmarshal(patch, &fields); err != nil { + t.Fatal(err) + } + metadata, _ := fields["metadata"].(map[string]interface{}) + if metadata["resourceVersion"] != "42" { + t.Errorf("patch %s does not carry resourceVersion 42", patch) + } + if annotations, _ := metadata["annotations"].(map[string]interface{}); annotations["a"] != "b" { + t.Errorf("patch %s lost the annotation change", patch) + } +} + +func TestComputeVMPatch_NoChangeStaysEmpty(t *testing.T) { + vm := newRootDiskVM("ns", "vm") + vm.ResourceVersion = "42" + + patch, err := computeVMPatch(vm, vm.DeepCopy()) + if err != nil { + t.Fatal(err) + } + if !isEmptyMergePatch(patch) { + t.Errorf("expected an empty patch for an unchanged VM, got %s", patch) + } +} + +func TestMockPatch_StaleResourceVersionIsRejected(t *testing.T) { + _, mock, _ := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + gvr := schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + ctx := context.Background() + + stale := []byte(`{"metadata":{"resourceVersion":"999","annotations":{"x":"y"}}}`) + if _, err := mock.Resource(gvr).Namespace("ns").Patch(ctx, "vm", types.MergePatchType, stale, metav1.PatchOptions{}); !apierrors.IsConflict(err) { + t.Fatalf("expected a Conflict for a stale resourceVersion, got %v", err) + } + + current, err := mock.GetVM("ns", "vm") + if err != nil { + t.Fatal(err) + } + fresh := []byte(fmt.Sprintf(`{"metadata":{"resourceVersion":%q,"annotations":{"x":"y"}}}`, current.ResourceVersion)) + if _, err := mock.Resource(gvr).Namespace("ns").Patch(ctx, "vm", types.MergePatchType, fresh, metav1.PatchOptions{}); err != nil { + t.Fatalf("patch with the current resourceVersion failed: %v", err) + } + + after, _ := mock.GetVM("ns", "vm") + if after.ResourceVersion == current.ResourceVersion { + t.Errorf("resourceVersion was not bumped by a successful write") + } + if after.Annotations["x"] != "y" { + t.Errorf("patch was not applied") + } + + // A patch without a resourceVersion is applied blindly, like the API server does. + blind := []byte(`{"metadata":{"labels":{"l":"v"}}}`) + if _, err := mock.Resource(gvr).Namespace("ns").Patch(ctx, "vm", types.MergePatchType, blind, metav1.PatchOptions{}); err != nil { + t.Fatalf("patch without resourceVersion failed: %v", err) + } +} + +// TestSetBootOnce_StaleRead_WithoutResourceVersionDropsCDROMDisk reproduces +// the production corruption: SetBootOnce reads the VM, the insert lands, then +// the boot-once patch carrying the stale disks array lands on top. Without +// the resourceVersion precondition (the behaviour before this fix) the CD-ROM +// disk entry is deleted while its volume survives. +func TestSetBootOnce_StaleRead_WithoutResourceVersionDropsCDROMDisk(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, withRootBootOrder(newRootDiskVM("ns", "vm"), 2)) + mock.IgnoreResourceVersion = true + + var injected bool + mock.BeforePatch = func(_ schema.GroupVersionResource, namespace, name string, _ types.PatchType, _ []byte) { + if !injected { + injected = true + landInsertDirectly(t, mock, namespace, name) + } + } + + if err := client.SetBootOnce("ns", "vm", "Cd"); err != nil { + t.Fatalf("SetBootOnce failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + if diskByName(got, cdromDevice) != nil { + t.Fatalf("expected the stale boot-once patch to delete the %s disk entry (the bug), but it is still there", cdromDevice) + } + if volumeByName(got, cdromDevice) == nil { + t.Fatalf("expected the %s volume to survive the stale patch", cdromDevice) + } +} + +// TestSetBootOnce_StaleRead_RetriesOnFreshCopy is the same interleaving with +// the fix: the stale patch is rejected, SetBootOnce re-reads, sees the +// CD-ROM and orders it first. +func TestSetBootOnce_StaleRead_RetriesOnFreshCopy(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, withRootBootOrder(newRootDiskVM("ns", "vm"), 2)) + + var patches int + mock.BeforePatch = func(_ schema.GroupVersionResource, namespace, name string, _ types.PatchType, _ []byte) { + patches++ + if patches == 1 { + landInsertDirectly(t, mock, namespace, name) + } + } + + if err := client.SetBootOnce("ns", "vm", "Cd"); err != nil { + t.Fatalf("SetBootOnce failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + if patches != 2 { + t.Errorf("expected the rejected patch and one retry, got %d patch attempts", patches) + } + if cfg := got.Annotations[BootOnceOriginalConfigAnnotation]; !strings.Contains(cfg, `"diskName":"cdrom0"`) { + t.Errorf("original boot order captured on the retry should include the CD-ROM, got %s", cfg) + } +} + +func TestUpdateVM_GivesUpAfterRepeatedConflicts(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + + var patches int + mock.BeforePatch = func(_ schema.GroupVersionResource, namespace, name string, _ types.PatchType, _ []byte) { + patches++ + _ = mock.MutateVM(namespace, name, func(vm *kubevirtv1.VirtualMachine) { vm.Annotations["touched"] = fmt.Sprint(patches) }) + } + + _, err := client.updateVM(context.Background(), "ns", "vm", func(vm *kubevirtv1.VirtualMachine) error { + vm.SetAnnotations(map[string]string{"mine": "x"}) + return nil + }) + if err == nil || !apierrors.IsConflict(err) { + t.Fatalf("expected a Conflict after exhausting retries, got %v", err) + } + if patches != vmUpdateMaxAttempts { + t.Errorf("expected %d attempts, got %d", vmUpdateMaxAttempts, patches) + } +} + +func TestUpdateVM_NoWriteWhenNothingChanged(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + + var patches int + mock.BeforePatch = func(schema.GroupVersionResource, string, string, types.PatchType, []byte) { patches++ } + + if _, err := client.updateVM(context.Background(), "ns", "vm", func(*kubevirtv1.VirtualMachine) error { return nil }); err != nil { + t.Fatal(err) + } + if patches != 0 { + t.Errorf("a mutation that changes nothing must not write, got %d patches", patches) + } +} + +func TestInsertVirtualMedia_ConflictRetryKeepsPVCNameAndOtherWriters(t *testing.T) { + client, mock, k8s := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + + var injected bool + mock.BeforePatch = func(_ schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte) { + if !injected && strings.Contains(string(data), "cdrom0") { + injected = true + _ = mock.MutateVM(namespace, name, func(vm *kubevirtv1.VirtualMachine) { vm.Annotations["someone-else"] = "was-here" }) + } + } + + if err := client.insertVirtualMediaAsync("ns", "vm", cdromDevice, testISOURL); err != nil { + t.Fatalf("insert failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + if got.Annotations["someone-else"] != "was-here" { + t.Errorf("the concurrent writer's change was clobbered") + } + assertBootOrder(t, got, rootDiskName, "1") + assertBootOrder(t, got, cdromDevice, "none") + + claim := volumeByName(got, cdromDevice).PersistentVolumeClaim.ClaimName + pvcs, err := k8s.CoreV1().PersistentVolumeClaims("ns").List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(pvcs.Items) != 1 || pvcs.Items[0].Name != claim { + var names []string + for _, pvc := range pvcs.Items { + names = append(names, pvc.Name) + } + t.Errorf("the retried insert must keep one PVC named like the volume (%s), got PVCs %v", claim, names) + } +} + +func TestInsertVirtualMedia_AfterBootOnceToCD_PutsCDFirst(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, withBootOnceToCD(newRootDiskVM("ns", "vm"))) + + if err := client.insertVirtualMediaAsync("ns", "vm", cdromDevice, testISOURL); err != nil { + t.Fatalf("insert failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + + // After the one-time boot the watcher restores the saved order; the CD-ROM + // must then end up with no boot order, not a duplicate of the root disk's. + restored := got.DeepCopy() + if err := client.restoreBootOrder(restored, got.Annotations[BootOnceOriginalConfigAnnotation]); err != nil { + t.Fatal(err) + } + assertBootOrder(t, restored, rootDiskName, "1") + assertBootOrder(t, restored, cdromDevice, "none") +} + +func TestInsertVirtualMedia_WithoutBootOverride_LeavesBootOrderAlone(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + + if err := client.insertVirtualMediaAsync("ns", "vm", cdromDevice, testISOURL); err != nil { + t.Fatalf("insert failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, rootDiskName, "1") + assertBootOrder(t, got, cdromDevice, "none") +} + +// The boot PATCH handler records the override annotations first and the +// boot-once state a moment later. An insert that lands in between must not +// reorder, or SetBootOnce would save the reordered disks as the original. +func TestInsertVirtualMedia_BetweenAnnotationsAndBootOnce_DoesNotReorder(t *testing.T) { + vm := newRootDiskVM("ns", "vm") + vm.Annotations["redfish.boot.source.override.enabled"] = "Once" + vm.Annotations["redfish.boot.source.override.target"] = "Cd" + client, mock, _ := newVMWriterTestClient(t, vm) + + if err := client.insertVirtualMediaAsync("ns", "vm", cdromDevice, testISOURL); err != nil { + t.Fatalf("insert failed: %v", err) + } + got, _ := mock.GetVM("ns", "vm") + assertBootOrder(t, got, rootDiskName, "1") + assertBootOrder(t, got, cdromDevice, "none") + + if err := client.SetBootOnce("ns", "vm", "Cd"); err != nil { + t.Fatalf("SetBootOnce failed: %v", err) + } + got, _ = mock.GetVM("ns", "vm") + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + if cfg := got.Annotations[BootOnceOriginalConfigAnnotation]; !strings.Contains(cfg, `{"diskName":"rootdisk","bootOrder":1}`) || !strings.Contains(cfg, `{"diskName":"cdrom0"}`) { + t.Errorf("original boot order must record root first and the CD-ROM without an order, got %s", cfg) + } +} + +// TestConcurrentVMWriters_ConvergeRegardlessOfOrder runs Ironic's request +// mix (insert, boot override annotations, boot-once; then eject with another +// boot-once) concurrently on many VMs, through the real client code and the +// mock's API-server semantics. Whatever the interleaving, every VM must end +// with its CD-ROM volume and disk paired and the CD-ROM first in the boot +// order; different VMs must not block each other. +func TestConcurrentVMWriters_ConvergeRegardlessOfOrder(t *testing.T) { + const vmCount = 24 + var vms []*kubevirtv1.VirtualMachine + initialRootOrder := map[string]uint{} + for i := 0; i < vmCount; i++ { + vm := newRootDiskVM("ns", fmt.Sprintf("vm-%d", i)) + // Half the VMs look like a first enroll (root first), half like the + // next cycle after an eject (root second), where a stale boot-once + // patch rewrites the disks array. + initialRootOrder[vm.Name] = uint(1 + i%2) + vms = append(vms, withRootBootOrder(vm, initialRootOrder[vm.Name])) + } + client, mock, _ := newVMWriterTestClient(t, vms...) + + // Widen the window between every writer's read and its write so the + // goroutines really interleave instead of running one after another. + var patches int64 + mock.BeforePatch = func(schema.GroupVersionResource, string, string, types.PatchType, []byte) { + if atomic.AddInt64(&patches, 1)%3 == 0 { + time.Sleep(time.Millisecond) + } + } + + bootOptions := map[string]interface{}{"bootSourceOverrideEnabled": "Once", "bootSourceOverrideTarget": "Cd", "bootSourceOverrideMode": "UEFI"} + var wg sync.WaitGroup + fail := func(format string, args ...interface{}) { t.Errorf(format, args...) } + + for _, vm := range vms { + name := vm.Name + wg.Add(3) + go func() { + defer wg.Done() + if err := client.insertVirtualMediaAsync("ns", name, cdromDevice, testISOURL); err != nil { + fail("%s: insert failed: %v", name, err) + } + }() + go func() { + defer wg.Done() + if err := client.RecordVMBootOptionsAsAnnotations("ns", name, bootOptions); err != nil { + fail("%s: record boot options failed: %v", name, err) + } + }() + go func() { + defer wg.Done() + if err := client.SetBootOnce("ns", name, "Cd"); err != nil { + fail("%s: SetBootOnce failed: %v", name, err) + } + }() + } + wg.Wait() + + for _, vm := range vms { + got, err := mock.GetVM("ns", vm.Name) + if err != nil { + t.Fatal(err) + } + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + if got.Labels[BootOnceLabel] != "enabled" { + t.Errorf("%s: boot-once label missing", vm.Name) + } + wantRoot := fmt.Sprintf(`{"diskName":"rootdisk","bootOrder":%d}`, initialRootOrder[vm.Name]) + if cfg := got.Annotations[BootOnceOriginalConfigAnnotation]; !strings.Contains(cfg, wantRoot) || !strings.Contains(cfg, `{"diskName":"cdrom0"}`) { + t.Errorf("%s: original boot order must keep the root disk at %d and the CD-ROM unordered, got %s", vm.Name, initialRootOrder[vm.Name], cfg) + } + } + + // Ironic's next cycle: eject while a new boot-once arrives. + for _, vm := range vms { + name := vm.Name + wg.Add(2) + go func() { + defer wg.Done() + if err := client.EjectVirtualMedia("ns", name, cdromDevice); err != nil { + fail("%s: eject failed: %v", name, err) + } + }() + go func() { + defer wg.Done() + if err := client.SetBootOnce("ns", name, "Cd"); err != nil { + fail("%s: second SetBootOnce failed: %v", name, err) + } + }() + } + wg.Wait() + + for _, vm := range vms { + got, _ := mock.GetVM("ns", vm.Name) + assertVolumeDiskPairs(t, got) + if hasVolume(got, cdromDevice) || hasDisk(got, cdromDevice) { + t.Errorf("%s: CD-ROM still present after eject", vm.Name) + } + if _, ok := got.Annotations[VirtualMediaURLAnnotationPrefix+cdromDevice]; ok { + t.Errorf("%s: virtual media annotation still present after eject", vm.Name) + } + } + + if client.vmLocks.size() != 0 { + t.Errorf("per-VM locks leaked: %d entries still alive", client.vmLocks.size()) + } +} + +func TestRepairVirtualMediaDevices_RestoresMissingDiskAndBootOrder(t *testing.T) { + // The exact state found on the corrupted production VMs: volume and + // annotation present, disk entry gone, boot-once to CD pending. + vm := withBootOnceToCD(withCDROM(newRootDiskVM("ns", "vm"), "vm-bootiso-1")) + vm.Spec.Template.Spec.Domain.Devices.Disks = vm.Spec.Template.Spec.Domain.Devices.Disks[:1] + client, mock, _ := newVMWriterTestClient(t, vm) + + client.repairVirtualMediaDevices(vm) + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + disk := diskByName(got, cdromDevice) + if disk == nil || disk.CDRom == nil || disk.CDRom.Bus != kubevirtv1.DiskBusSATA { + t.Fatalf("expected a SATA CD-ROM disk entry to be restored, got %+v", disk) + } + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + if cfg := got.Annotations[BootOnceOriginalConfigAnnotation]; !strings.Contains(cfg, `{"diskName":"cdrom0"}`) { + t.Errorf("saved original boot order must now list the CD-ROM without an order, got %s", cfg) + } +} + +func TestRepairVirtualMediaDevices_NoWriteWhenHealthyOrUnmanaged(t *testing.T) { + healthy := withCDROM(newRootDiskVM("ns", "healthy"), "claim") + + // A cdrom volume without the bridge's annotation is not ours to touch. + unmanaged := newRootDiskVM("ns", "unmanaged") + unmanaged.Spec.Template.Spec.Volumes = append(unmanaged.Spec.Template.Spec.Volumes, kubevirtv1.Volume{ + Name: "cdrom1", + VolumeSource: kubevirtv1.VolumeSource{ContainerDisk: &kubevirtv1.ContainerDiskSource{Image: "x"}}, + }) + + client, mock, _ := newVMWriterTestClient(t, healthy, unmanaged) + var patches int + mock.BeforePatch = func(schema.GroupVersionResource, string, string, types.PatchType, []byte) { patches++ } + + client.repairVirtualMediaDevices(healthy) + client.repairVirtualMediaDevices(unmanaged) + + if patches != 0 { + t.Errorf("expected no writes, got %d", patches) + } +} + +// TestHandleVMUpdate_RepairsThenRestoresAfterReboot drives the watcher entry +// point on a corrupted VM whose one-time boot has completed (VMI UID changed): +// the disk entry comes back and the saved boot order is restored in the same pass. +func TestHandleVMUpdate_RepairsThenRestoresAfterReboot(t *testing.T) { + vm := withBootOnceToCD(withCDROM(newRootDiskVM("ns", "vm"), "vm-bootiso-1")) + vm.Spec.Template.Spec.Domain.Devices.Disks = vm.Spec.Template.Spec.Domain.Devices.Disks[:1] + vm.Annotations[BootOnceVMIUIDAnnotation] = "old-vmi" + client, mock, _ := newVMWriterTestClient(t, vm) + if err := mock.AddVMI(&kubevirtv1.VirtualMachineInstance{ObjectMeta: metav1.ObjectMeta{Name: "vm", Namespace: "ns", UID: "new-vmi"}}); err != nil { + t.Fatal(err) + } + + client.handleVMUpdate(vm) + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, rootDiskName, "1") + assertBootOrder(t, got, cdromDevice, "none") + if got.Labels[BootOnceLabel] != "" { + t.Errorf("boot-once label should be cleared after restore") + } + if _, ok := got.Annotations[BootOnceOriginalConfigAnnotation]; ok { + t.Errorf("boot-once annotations should be cleared after restore") + } +} diff --git a/pkg/kubevirt/vmlock.go b/pkg/kubevirt/vmlock.go new file mode 100644 index 0000000..2678382 --- /dev/null +++ b/pkg/kubevirt/vmlock.go @@ -0,0 +1,51 @@ +package kubevirt + +import "sync" + +// keyedMutex hands out one mutex per key so that work on different keys runs +// in parallel while work on the same key is serialized. Entries are reference +// counted and dropped as soon as nobody holds or waits for them, so the map +// does not grow with the number of VMs seen over the lifetime of the process. +type keyedMutex struct { + mu sync.Mutex + entries map[string]*keyedMutexEntry +} + +type keyedMutexEntry struct { + sync.Mutex + refs int +} + +func newKeyedMutex() *keyedMutex { + return &keyedMutex{entries: make(map[string]*keyedMutexEntry)} +} + +// Lock blocks until the caller owns key and returns the function that releases it. +func (k *keyedMutex) Lock(key string) func() { + k.mu.Lock() + entry, ok := k.entries[key] + if !ok { + entry = &keyedMutexEntry{} + k.entries[key] = entry + } + entry.refs++ + k.mu.Unlock() + + entry.Lock() + return func() { + entry.Unlock() + k.mu.Lock() + entry.refs-- + if entry.refs == 0 { + delete(k.entries, key) + } + k.mu.Unlock() + } +} + +// size reports how many keys are currently held or waited for. +func (k *keyedMutex) size() int { + k.mu.Lock() + defer k.mu.Unlock() + return len(k.entries) +} diff --git a/pkg/kubevirt/vmlock_test.go b/pkg/kubevirt/vmlock_test.go new file mode 100644 index 0000000..304c10b --- /dev/null +++ b/pkg/kubevirt/vmlock_test.go @@ -0,0 +1,91 @@ +package kubevirt + +import ( + "testing" + "time" +) + +func TestKeyedMutex_SameKeySerializes(t *testing.T) { + km := newKeyedMutex() + + unlockA := km.Lock("ns/vm") + + acquired := make(chan struct{}) + go func() { + unlockB := km.Lock("ns/vm") + close(acquired) + unlockB() + }() + + select { + case <-acquired: + t.Fatal("second lock on the same key was acquired while the first was still held") + case <-time.After(50 * time.Millisecond): + } + + unlockA() + + select { + case <-acquired: + case <-time.After(time.Second): + t.Fatal("second lock on the same key was not acquired after the first was released") + } +} + +func TestKeyedMutex_DifferentKeysDoNotBlock(t *testing.T) { + km := newKeyedMutex() + + unlockA := km.Lock("ns/vm-a") + defer unlockA() + + acquired := make(chan struct{}) + go func() { + unlockB := km.Lock("ns/vm-b") + close(acquired) + unlockB() + }() + + select { + case <-acquired: + case <-time.After(time.Second): + t.Fatal("lock on a different key was blocked by an unrelated holder") + } +} + +func TestKeyedMutex_EntriesAreDroppedWhenUnused(t *testing.T) { + km := newKeyedMutex() + + unlockA := km.Lock("ns/vm-a") + unlockB := km.Lock("ns/vm-b") + if got := km.size(); got != 2 { + t.Fatalf("expected 2 live entries while both keys are held, got %d", got) + } + + unlockA() + if got := km.size(); got != 1 { + t.Fatalf("expected 1 live entry after releasing one key, got %d", got) + } + + unlockB() + if got := km.size(); got != 0 { + t.Fatalf("expected no live entries after releasing every key, got %d", got) + } + + // A key that is held and waited for at the same time stays alive until the last waiter is done. + unlockA = km.Lock("ns/vm-a") + done := make(chan struct{}) + go func() { + unlock := km.Lock("ns/vm-a") + unlock() + close(done) + }() + time.Sleep(20 * time.Millisecond) + if got := km.size(); got != 1 { + t.Fatalf("expected the entry to stay alive while a waiter is queued, got %d entries", got) + } + unlockA() + <-done + if got := km.size(); got != 0 { + t.Fatalf("expected the entry to be dropped after the waiter finished, got %d entries", got) + } +}