diff --git a/pkg/kubevirt/client.go b/pkg/kubevirt/client.go index 26ad6c2..5f0be23 100644 --- a/pkg/kubevirt/client.go +++ b/pkg/kubevirt/client.go @@ -97,6 +97,11 @@ type Client struct { pvcWatcherCtx context.Context pvcWatcherCancel context.CancelFunc pvcWatcherWg sync.WaitGroup + + // Per-VM serialization of read-modify-write updates (see updateVM) and + // the backoff sleep between conflict retries, replaceable in tests. + vmLocks *keyedMutex + sleep func(time.Duration) } // init registers KubeVirt and CDI types with the runtime scheme for conversion @@ -143,6 +148,12 @@ func vmToUnstructured(vm *kubevirtv1.VirtualMachine) (*unstructured.Unstructured // computeVMPatch compares original and modified VMs and returns a JSON Merge Patch. // This preserves unknown fields that may exist in the cluster object but are not // represented in the typed VirtualMachine struct. +// +// The patch carries the resourceVersion of original. A merge patch replaces +// arrays such as disks and volumes wholesale, so one computed from a stale +// read would silently drop entries another writer just added; with the +// resourceVersion in it, the API server rejects such a patch with a Conflict +// and the caller (updateVM) recomputes it from a fresh read. func computeVMPatch(original, modified *kubevirtv1.VirtualMachine) ([]byte, error) { originalJSON, err := json.Marshal(original) if err != nil { @@ -157,7 +168,10 @@ func computeVMPatch(original, modified *kubevirtv1.VirtualMachine) ([]byte, erro if err != nil { return nil, fmt.Errorf("failed to create merge patch: %w", err) } - return patch, nil + if original.ResourceVersion == "" || isEmptyMergePatch(patch) { + return patch, nil + } + return withResourceVersion(patch, original.ResourceVersion) } // vmiToUnstructured converts a typed VirtualMachineInstance to an unstructured object @@ -238,6 +252,8 @@ func NewClient(configPath string, timeout time.Duration, appConfig interface{}) timeout: timeout, appConfig: appConfig, httpClient: httpClient, + vmLocks: newKeyedMutex(), + sleep: time.Sleep, }, nil } @@ -255,6 +271,8 @@ func NewClientWithClients( timeout: timeout, appConfig: appConfig, config: &rest.Config{}, + vmLocks: newKeyedMutex(), + sleep: time.Sleep, httpClient: &http.Client{ Timeout: timeout, Transport: &http.Transport{ @@ -1419,54 +1437,32 @@ func (c *Client) RecordVMBootOptionsAsAnnotations(namespace, name string, option logger.Info("Setting boot options for VM %s/%s: %v", namespace, name, options) - // Get the current VM - vm, err := c.GetVM(namespace, name) - if err != nil { - return fmt.Errorf("failed to get VM %s: %w", name, err) - } - - // Update VM with boot options - vmCopy := vm.DeepCopy() - // Store boot options in VM annotations for persistence - annotations := vmCopy.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + annotations := vmCopy.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } - // Update annotations with boot options - if bootSourceOverrideEnabled, found := options["bootSourceOverrideEnabled"]; found { - if enabled, ok := bootSourceOverrideEnabled.(string); ok { - annotations["redfish.boot.source.override.enabled"] = enabled + if bootSourceOverrideEnabled, found := options["bootSourceOverrideEnabled"]; found { + if enabled, ok := bootSourceOverrideEnabled.(string); ok { + annotations["redfish.boot.source.override.enabled"] = enabled + } } - } - if bootSourceOverrideTarget, found := options["bootSourceOverrideTarget"]; found { - if target, ok := bootSourceOverrideTarget.(string); ok { - annotations["redfish.boot.source.override.target"] = target + if bootSourceOverrideTarget, found := options["bootSourceOverrideTarget"]; found { + if target, ok := bootSourceOverrideTarget.(string); ok { + annotations["redfish.boot.source.override.target"] = target + } } - } - if bootSourceOverrideMode, found := options["bootSourceOverrideMode"]; found { - if mode, ok := bootSourceOverrideMode.(string); ok { - annotations["redfish.boot.source.override.mode"] = mode + if bootSourceOverrideMode, found := options["bootSourceOverrideMode"]; found { + if mode, ok := bootSourceOverrideMode.(string); ok { + annotations["redfish.boot.source.override.mode"] = mode + } } - } - - vmCopy.SetAnnotations(annotations) - - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - return fmt.Errorf("failed to compute VM patch: %w", err) - } - - // Apply patch - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + vmCopy.SetAnnotations(annotations) + return nil + }) if err != nil { return fmt.Errorf("failed to update VM boot options: %w", err) } @@ -1738,43 +1734,13 @@ func (c *Client) insertVirtualMediaAsync(namespace, name, mediaID, imageURL stri logger.Info("Creating CD-ROM device %s in VM spec first", deviceName) logger.Debug("Creating CD-ROM device %s in VM spec", deviceName) + // Only the spec change runs under the per-VM lock; PVC creation and the + // ISO download below are slow and must not block other writers. + // dataVolumeName was generated once above so a retried update keeps the + // same PVC name. var oldPVCName string - - maxRetries := 3 - for attempt := 1; attempt <= maxRetries; attempt++ { - logger.Debug("VM update attempt %d/%d", attempt, maxRetries) - - vm, err := c.GetVM(namespace, name) - if err != nil { - logger.Error("Failed to get VM %s/%s: %v", namespace, name, err) - return fmt.Errorf("failed to get VM %s: %w", name, err) - } - logger.Debug("Successfully retrieved VM %s/%s", namespace, name) - - vmCopy := vm.DeepCopy() - - if vmCopy.Spec.Template == nil { - vmCopy.Spec.Template = &kubevirtv1.VirtualMachineInstanceTemplateSpec{} - } - - diskExists := false - for _, disk := range vmCopy.Spec.Template.Spec.Domain.Devices.Disks { - if disk.Name == deviceName { - diskExists = true - break - } - } - - if !diskExists { - newDisk := kubevirtv1.Disk{ - Name: deviceName, - DiskDevice: kubevirtv1.DiskDevice{ - CDRom: &kubevirtv1.CDRomTarget{ - Bus: kubevirtv1.DiskBusSATA, - }, - }, - } - vmCopy.Spec.Template.Spec.Domain.Devices.Disks = append(vmCopy.Spec.Template.Spec.Domain.Devices.Disks, newDisk) + _, err = c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + if addCDROMDisk(vmCopy, deviceName) { logger.Info("Added new CD-ROM device %s", deviceName) } @@ -1823,34 +1789,19 @@ func (c *Client) insertVirtualMediaAsync(namespace, name, mediaID, imageURL stri annotations[VirtualMediaURLAnnotationPrefix+deviceName] = imageURL vmCopy.SetAnnotations(annotations) - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - logger.Error("Failed to compute VM patch: %v", err) - return fmt.Errorf("failed to compute VM patch: %w", err) - } - - gvrVM := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } - - logger.Debug("Updating VM %s/%s with new CD-ROM device", namespace, name) - _, err = c.dynamicClient.Resource(gvrVM).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) - if err != nil { - if strings.Contains(err.Error(), "the object has been modified") && attempt < maxRetries { - logger.Info("Concurrent modification detected, retrying VM update (attempt %d/%d)", attempt, maxRetries) - logger.Debug("Concurrent modification detected, retrying VM update (attempt %d/%d)", attempt, maxRetries) - time.Sleep(time.Duration(attempt) * time.Second) - continue - } - logger.Error("Failed to update VM %s/%s: %v", namespace, name, err) - return fmt.Errorf("failed to update VM: %w", err) + // Ironic sends the boot override right after the insert request. When + // the override reached the VM first it could only order the disks that + // existed then, so the new CD-ROM is placed first now. The result is + // the same whichever request reached the VM first. + if bootOverrideWantsCD(vmCopy) { + logger.Info("Boot override to CD is pending for VM %s/%s, placing %s first in the boot order", namespace, name, deviceName) + return c.applyPendingCDBootOverride(vmCopy, deviceName) } - - logger.Debug("Successfully updated VM %s/%s on attempt %d", namespace, name, attempt) - break + return nil + }) + if err != nil { + logger.Error("Failed to update VM %s/%s: %v", namespace, name, err) + return fmt.Errorf("failed to update VM: %w", err) } logger.Info("Successfully created CD-ROM device %s in VM spec", deviceName) @@ -2221,41 +2172,19 @@ func (c *Client) SetBootOrder(namespace, name, bootTarget string) error { logger.Info("Setting boot order to %s for VM %s/%s", bootTarget, namespace, name) - // Get the current VM - vm, err := c.GetVM(namespace, name) - if err != nil { - return fmt.Errorf("failed to get VM %s: %w", name, err) - } - - // Update VM with boot order configuration - vmCopy := vm.DeepCopy() - - // Collect current volumes and detect their type - // This is needed, because the device list does not provide enough - // information about disk type and there are disks that are not bootable - // like cloud init. - err = c.modifyVmBootOrder(vmCopy, bootTarget) - if err != nil { - return err - } - - // A permanent boot order change supersedes any pending boot-once state. - c.clearBootOnceState(vmCopy) - - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - return fmt.Errorf("failed to compute VM patch: %w", err) - } - - // Apply patch - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + // Collect current volumes and detect their type + // This is needed, because the device list does not provide enough + // information about disk type and there are disks that are not bootable + // like cloud init. + if err := c.modifyVmBootOrder(vmCopy, bootTarget); err != nil { + return err + } - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + // A permanent boot order change supersedes any pending boot-once state. + c.clearBootOnceState(vmCopy) + return nil + }) if err != nil { return fmt.Errorf("failed to update VM boot order: %w", err) } @@ -2339,106 +2268,84 @@ func (c *Client) SetBootOnce(namespace, name, bootTarget string) error { logger.Info("Setting boot once to %s for VM %s/%s", bootTarget, namespace, name) - // Get the current VM - vm, err := c.GetVM(namespace, name) - if err != nil { - return fmt.Errorf("failed to get VM %s: %w", name, err) - } + var currentVMIUID string + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + annotations := vmCopy.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + vmLabels := vmCopy.GetLabels() + if vmLabels == nil { + vmLabels = make(map[string]string) + } - // Update VM with boot once configuration - vmCopy := vm.DeepCopy() + // Check if boot-once is already configured + existingOriginalConfig := annotations[BootOnceOriginalConfigAnnotation] + existingVMIUID := annotations[BootOnceVMIUIDAnnotation] + hasExistingBootOnce := existingOriginalConfig != "" - // Get current annotations and labels - annotations := vmCopy.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - vmLabels := vmCopy.GetLabels() - if vmLabels == nil { - vmLabels = make(map[string]string) - } - - // Check if boot-once is already configured - existingOriginalConfig := annotations[BootOnceOriginalConfigAnnotation] - existingVMIUID := annotations[BootOnceVMIUIDAnnotation] - hasExistingBootOnce := existingOriginalConfig != "" + currentVMIUID = c.getVMIUID(namespace, name) - // Get current VMI UID - currentVMIUID := c.getVMIUID(namespace, name) + if hasExistingBootOnce { + // Boot-once already configured - check if the recorded VMI is still running + if existingVMIUID != "" && existingVMIUID == currentVMIUID { + // Same VMI is still running - keep original config, just update boot order + logger.Info("Boot-once already configured for VM %s/%s with same VMI, updating boot target only", namespace, name) + } else { + // VMI has changed or was empty - clear old state and capture new original config + logger.Info("Boot-once state is stale for VM %s/%s (VMI changed), capturing new original config", namespace, name) - if hasExistingBootOnce { - // Boot-once already configured - check if the recorded VMI is still running - if existingVMIUID != "" && existingVMIUID == currentVMIUID { - // Same VMI is still running - keep original config, just update boot order - logger.Info("Boot-once already configured for VM %s/%s with same VMI, updating boot target only", namespace, name) - } else { - // VMI has changed or was empty - clear old state and capture new original config - logger.Info("Boot-once state is stale for VM %s/%s (VMI changed), capturing new original config", namespace, name) + // Restore the original boot order so we capture the real original state below + if err := c.restoreBootOrder(vmCopy, existingOriginalConfig); err != nil { + logger.Warning("Failed to restore original boot order before recapture: %v", err) + } - // Restore the original boot order so we capture the real original state below - if err := c.restoreBootOrder(vmCopy, existingOriginalConfig); err != nil { - logger.Warning("Failed to restore original boot order before recapture: %v", err) + // Capture the (now restored) boot order as the new original config + originalConfig, err := c.captureCurrentBootOrder(vmCopy) + if err != nil { + return fmt.Errorf("failed to capture boot order: %w", err) + } + annotations[BootOnceOriginalConfigAnnotation] = originalConfig + annotations[BootOnceVMIUIDAnnotation] = currentVMIUID + // Keep existing BootOnceOriginalRebootPolicyAnnotation — it + // already holds the real original value from the first SetBootOnce call. } - - // Capture the (now restored) boot order as the new original config + } else { + // No existing boot-once — capture current boot order and rebootPolicy originalConfig, err := c.captureCurrentBootOrder(vmCopy) if err != nil { return fmt.Errorf("failed to capture boot order: %w", err) } annotations[BootOnceOriginalConfigAnnotation] = originalConfig annotations[BootOnceVMIUIDAnnotation] = currentVMIUID - // Keep existing BootOnceOriginalRebootPolicyAnnotation — it - // already holds the real original value from the first SetBootOnce call. - } - } else { - // No existing boot-once — capture current boot order and rebootPolicy - originalConfig, err := c.captureCurrentBootOrder(vmCopy) - if err != nil { - return fmt.Errorf("failed to capture boot order: %w", err) - } - annotations[BootOnceOriginalConfigAnnotation] = originalConfig - annotations[BootOnceVMIUIDAnnotation] = currentVMIUID - if vmCopy.Spec.Template != nil && vmCopy.Spec.Template.Spec.Domain.RebootPolicy != nil { - annotations[BootOnceOriginalRebootPolicyAnnotation] = string(*vmCopy.Spec.Template.Spec.Domain.RebootPolicy) + if vmCopy.Spec.Template != nil && vmCopy.Spec.Template.Spec.Domain.RebootPolicy != nil { + annotations[BootOnceOriginalRebootPolicyAnnotation] = string(*vmCopy.Spec.Template.Spec.Domain.RebootPolicy) + } } - } - // Add boot-once label for watch selector - vmLabels[BootOnceLabel] = "enabled" - vmCopy.SetLabels(vmLabels) - - // Store Redfish boot override annotations - annotations["redfish.boot.source.override.enabled"] = "Once" - annotations["redfish.boot.source.override.target"] = bootTarget - annotations["redfish.boot.source.override.mode"] = "UEFI" - vmCopy.SetAnnotations(annotations) - - // Modify boot order to boot from target - if err := c.modifyVmBootOrder(vmCopy, bootTarget); err != nil { - return fmt.Errorf("failed to modify boot order: %w", err) - } - - // Set rebootPolicy to Terminate so that KubeVirt destroys the VMI on - // guest-initiated reboot and we can detect the VMI UID change. - if vmCopy.Spec.Template != nil { - terminate := kubevirtv1.RebootPolicyTerminate - vmCopy.Spec.Template.Spec.Domain.RebootPolicy = &terminate - } + // Add boot-once label for watch selector + vmLabels[BootOnceLabel] = "enabled" + vmCopy.SetLabels(vmLabels) - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - return fmt.Errorf("failed to compute VM patch: %w", err) - } + // Store Redfish boot override annotations + annotations["redfish.boot.source.override.enabled"] = "Once" + annotations["redfish.boot.source.override.target"] = bootTarget + annotations["redfish.boot.source.override.mode"] = "UEFI" + vmCopy.SetAnnotations(annotations) - // Apply patch - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } + // Modify boot order to boot from target + if err := c.modifyVmBootOrder(vmCopy, bootTarget); err != nil { + return fmt.Errorf("failed to modify boot order: %w", err) + } - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + // Set rebootPolicy to Terminate so that KubeVirt destroys the VMI on + // guest-initiated reboot and we can detect the VMI UID change. + if vmCopy.Spec.Template != nil { + terminate := kubevirtv1.RebootPolicyTerminate + vmCopy.Spec.Template.Spec.Domain.RebootPolicy = &terminate + } + return nil + }) if err != nil { return fmt.Errorf("failed to update VM boot once configuration: %w", err) } @@ -2454,89 +2361,60 @@ func (c *Client) EjectVirtualMedia(namespace, name, mediaID string) error { logger.Info("Ejecting virtual media %s for VM %s/%s", mediaID, namespace, name) - // Get the current VM - vm, err := c.GetVM(namespace, name) - if err != nil { - logger.Error("Failed to get VM %s/%s for ejection: %v", namespace, name, err) - return fmt.Errorf("failed to get VM %s: %w", name, err) - } + // The PVC name and the helper pod name are taken from the copy that is + // actually written, because PVC names carry unique suffixes and a retried + // update may see a different one. + var actualPVCName, helperPodName string + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + if vmCopy.Spec.Template == nil { + logger.Warning("No template found in VM %s/%s when trying to eject media %s", namespace, name, mediaID) + return fmt.Errorf("no template found in VM") + } - logger.Debug("Current VM spec before ejection") + actualPVCName = "" + helperPodName = vmCopy.GetLabels()[ImportingLabelPrefix+mediaID] - // Get the actual PVC name from the volume BEFORE removing it from the VM - // This is necessary because PVC names have unique identifiers appended - var actualPVCName string - if vm.Spec.Template != nil { - for _, volume := range vm.Spec.Template.Spec.Volumes { - if volume.Name == mediaID && volume.PersistentVolumeClaim != nil { - actualPVCName = volume.PersistentVolumeClaim.ClaimName - logger.Info("Found actual PVC name %s for media %s", actualPVCName, mediaID) - break + // Remove the specified disk from disks list + var newDisks []kubevirtv1.Disk + foundDisk := false + for _, disk := range vmCopy.Spec.Template.Spec.Domain.Devices.Disks { + if disk.Name != mediaID { + newDisks = append(newDisks, disk) + } else { + logger.Info("Removing CD-ROM device %s from VM spec", mediaID) + foundDisk = true } } - } - - // Update VM to remove CD-ROM disk and volume reference - vmCopy := vm.DeepCopy() - - if vmCopy.Spec.Template == nil { - logger.Warning("No template found in VM %s/%s when trying to eject media %s", namespace, name, mediaID) - return fmt.Errorf("no template found in VM") - } - - // Remove the specified disk from disks list - var newDisks []kubevirtv1.Disk - foundDisk := false - for _, disk := range vmCopy.Spec.Template.Spec.Domain.Devices.Disks { - if disk.Name != mediaID { - newDisks = append(newDisks, disk) - } else { - logger.Info("Removing CD-ROM device %s from VM spec", mediaID) - foundDisk = true + if !foundDisk { + logger.Warning("CD-ROM device %s not found in VM %s/%s disks list during ejection", mediaID, namespace, name) } - } - if !foundDisk { - logger.Warning("CD-ROM device %s not found in VM %s/%s disks list during ejection", mediaID, namespace, name) - } - vmCopy.Spec.Template.Spec.Domain.Devices.Disks = newDisks + vmCopy.Spec.Template.Spec.Domain.Devices.Disks = newDisks - // Remove the volume reference - var newVolumes []kubevirtv1.Volume - foundVolume := false - for _, volume := range vmCopy.Spec.Template.Spec.Volumes { - if volume.Name != mediaID { - newVolumes = append(newVolumes, volume) - } else { + // Remove the volume reference + var newVolumes []kubevirtv1.Volume + foundVolume := false + for _, volume := range vmCopy.Spec.Template.Spec.Volumes { + if volume.Name != mediaID { + newVolumes = append(newVolumes, volume) + continue + } + if volume.PersistentVolumeClaim != nil { + actualPVCName = volume.PersistentVolumeClaim.ClaimName + logger.Info("Found actual PVC name %s for media %s", actualPVCName, mediaID) + } logger.Info("Removing volume reference %s from VM spec", mediaID) foundVolume = true } - } - if !foundVolume { - logger.Warning("Volume reference %s not found in VM %s/%s during ejection", mediaID, namespace, name) - } - vmCopy.Spec.Template.Spec.Volumes = newVolumes - - annotations := vmCopy.GetAnnotations() - delete(annotations, VirtualMediaURLAnnotationPrefix+mediaID) - vmCopy.SetAnnotations(annotations) - - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(vm, vmCopy) - if err != nil { - logger.Error("Failed to compute VM patch for %s/%s: %v", namespace, name, err) - return fmt.Errorf("failed to compute VM patch: %w", err) - } - - logger.Debug("VM spec after removing CD-ROM and volume") - - // Apply patch - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } + if !foundVolume { + logger.Warning("Volume reference %s not found in VM %s/%s during ejection", mediaID, namespace, name) + } + vmCopy.Spec.Template.Spec.Volumes = newVolumes - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + annotations := vmCopy.GetAnnotations() + delete(annotations, VirtualMediaURLAnnotationPrefix+mediaID) + vmCopy.SetAnnotations(annotations) + return nil + }) if err != nil { logger.Error("Failed to update VM %s/%s after removing CD-ROM and volume: %v", namespace, name, err) return fmt.Errorf("failed to update VM: %w", err) @@ -2546,8 +2424,6 @@ func (c *Client) EjectVirtualMedia(namespace, name, mediaID string) error { // Kill any running helper pod for this device and remove the importing label // so a stale pod cannot trigger a deferred power-on after the media is gone. - labelKey := ImportingLabelPrefix + mediaID - helperPodName := vm.GetLabels()[labelKey] if helperPodName != "" { logger.Info("Cleaning up active import for device %s: deleting helper pod %s and removing importing label", mediaID, helperPodName) if delErr := c.kubernetesClient.CoreV1().Pods(namespace).Delete(ctx, helperPodName, metav1.DeleteOptions{}); delErr != nil { @@ -3270,6 +3146,10 @@ func (c *Client) handleVMUpdate(vm *kubevirtv1.VirtualMachine) { return } + // A lost write can leave an inserted device with its volume but no disk + // entry; put the disk back before anything else looks at the VM. + c.repairVirtualMediaDevices(currentVM) + vmLabels := currentVM.GetLabels() // Handle power-after-import if applicable @@ -3314,35 +3194,28 @@ func (c *Client) handleVMUpdate(vm *kubevirtv1.VirtualMachine) { logger.Info("VMI UID changed for VM %s/%s (recorded: %s, current: %s), restoring boot order", namespace, name, recordedVMIUID, currentVMIUID) - vmCopy := currentVM.DeepCopy() - - // Restore the original boot order - if err := c.restoreBootOrder(vmCopy, originalConfig); err != nil { - logger.Error("Failed to restore boot order for VM %s/%s: %v", namespace, name, err) - return - } - - // Remove boot-once state (label, annotations) and restore original rebootPolicy - c.clearBootOnceState(vmCopy) - - // Compute merge patch from changes (preserves unknown fields) - patch, err := computeVMPatch(currentVM, vmCopy) - if err != nil { - logger.Error("Failed to compute VM patch: %v", err) - return - } - - // Apply patch ctx, cancel := context.WithTimeout(context.Background(), c.timeout) defer cancel() - gvr := schema.GroupVersionResource{ - Group: "kubevirt.io", - Version: "v1", - Resource: "virtualmachines", - } + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + // Decide again on the fresh copy: another writer, or an earlier + // pass of this watcher, may already have restored the VM. + if vmCopy.GetLabels()[BootOnceLabel] != "enabled" { + return nil + } + originalConfig := vmCopy.GetAnnotations()[BootOnceOriginalConfigAnnotation] + if originalConfig == "" { + return nil + } + + if err := c.restoreBootOrder(vmCopy, originalConfig); err != nil { + return fmt.Errorf("failed to restore boot order: %w", err) + } - _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + // Remove boot-once state (label, annotations) and restore original rebootPolicy + c.clearBootOnceState(vmCopy) + return nil + }) if err != nil { logger.Error("Failed to update VM %s/%s with restored boot order: %v", namespace, name, err) return diff --git a/pkg/kubevirt/mock_dynamic_client.go b/pkg/kubevirt/mock_dynamic_client.go index 3eec7ee..3cbdb94 100644 --- a/pkg/kubevirt/mock_dynamic_client.go +++ b/pkg/kubevirt/mock_dynamic_client.go @@ -23,9 +23,11 @@ import ( "context" "encoding/json" "fmt" + "strconv" "strings" "sync" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -54,10 +56,28 @@ func deepCopyUnstructured(obj *unstructured.Unstructured) (*unstructured.Unstruc return &unstructured.Unstructured{Object: newObj}, nil } -// MockDynamicClient implements dynamic.Interface for testing +// MockDynamicClient implements dynamic.Interface for testing. +// +// Writes follow the API server's rules that matter for read-modify-write +// code: every stored object has a resourceVersion that is bumped on each +// write, a patch is applied atomically, and a merge patch that carries a +// metadata.resourceVersion different from the stored one is rejected with a +// Conflict error. type MockDynamicClient struct { mu sync.RWMutex resources map[string]map[string]*unstructured.Unstructured // "gvr/namespace" -> name -> object + + // IgnoreResourceVersion accepts patches whose resourceVersion is stale, + // which is how the API server treats a patch that carries none. Tests use + // it to show what happens without the resourceVersion precondition. + IgnoreResourceVersion bool + + // BeforePatch, when set, runs before a patch is applied and outside the + // store lock. Tests use it to let another writer change the object between + // a caller's read and its write. It must change the store through + // MutateVM or the dynamic interface, never through a *Client, which would + // re-enter the client's per-VM lock. + BeforePatch func(gvr schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte) } // NewMockDynamicClient creates a new mock dynamic client @@ -93,6 +113,14 @@ func (m *MockDynamicClient) addResource(gvr schema.GroupVersionResource, namespa m.mu.Lock() defer m.mu.Unlock() + if obj.GetResourceVersion() == "" { + obj.SetResourceVersion("1") + } + m.storeLocked(gvr, namespace, name, obj) +} + +// storeLocked stores obj; the caller holds m.mu. +func (m *MockDynamicClient) storeLocked(gvr schema.GroupVersionResource, namespace, name string, obj *unstructured.Unstructured) { key := resourceKey(gvr, namespace) if m.resources[key] == nil { m.resources[key] = make(map[string]*unstructured.Unstructured) @@ -100,6 +128,67 @@ func (m *MockDynamicClient) addResource(gvr schema.GroupVersionResource, namespa m.resources[key][name] = obj } +// getLocked returns the stored object; the caller holds m.mu. +func (m *MockDynamicClient) getLocked(gvr schema.GroupVersionResource, namespace, name string) (*unstructured.Unstructured, bool) { + key := resourceKey(gvr, namespace) + if m.resources[key] == nil { + return nil, false + } + obj, found := m.resources[key][name] + return obj, found +} + +// bumpResourceVersion gives obj the next resourceVersion, as the API server +// does on every successful write. +func bumpResourceVersion(obj *unstructured.Unstructured) { + n, err := strconv.Atoi(obj.GetResourceVersion()) + if err != nil { + n = 0 + } + obj.SetResourceVersion(strconv.Itoa(n + 1)) +} + +func notFoundError(gvr schema.GroupVersionResource, name string) error { + return apierrors.NewNotFound(schema.GroupResource{Group: gvr.Group, Resource: gvr.Resource}, name) +} + +func conflictError(gvr schema.GroupVersionResource, name string) error { + return apierrors.NewConflict(schema.GroupResource{Group: gvr.Group, Resource: gvr.Resource}, name, + fmt.Errorf("the object has been modified; please apply your changes to the latest version and try again")) +} + +// MutateVM changes a stored VM in place, bumping its resourceVersion, the way +// a write by another client would. It is safe to call from BeforePatch. +func (m *MockDynamicClient) MutateVM(namespace, name string, mutate func(vm *kubevirtv1.VirtualMachine)) error { + gvr := schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + + m.mu.Lock() + defer m.mu.Unlock() + + obj, found := m.getLocked(gvr, namespace, name) + if !found { + return notFoundError(gvr, name) + } + vm, err := unstructuredToVM(obj) + if err != nil { + return err + } + if vm.Labels == nil { + vm.Labels = map[string]string{} + } + if vm.Annotations == nil { + vm.Annotations = map[string]string{} + } + mutate(vm) + u, err := vmToUnstructured(vm) + if err != nil { + return err + } + bumpResourceVersion(u) + m.storeLocked(gvr, namespace, name, u) + return nil +} + // getResource gets a resource from the mock client func (m *MockDynamicClient) getResource(gvr schema.GroupVersionResource, namespace, name string) (*unstructured.Unstructured, bool) { m.mu.RLock() @@ -307,8 +396,18 @@ func (r *mockNamespaceableResource) Update(ctx context.Context, obj *unstructure if err != nil { return nil, err } - r.client.addResource(r.gvr, namespace, name, copied) - return deepCopyUnstructured(obj) + + r.client.mu.Lock() + defer r.client.mu.Unlock() + + if existing, found := r.client.getLocked(r.gvr, namespace, name); found && !r.client.IgnoreResourceVersion { + if rv := copied.GetResourceVersion(); rv != "" && rv != existing.GetResourceVersion() { + return nil, conflictError(r.gvr, name) + } + } + bumpResourceVersion(copied) + r.client.storeLocked(r.gvr, namespace, name, copied) + return deepCopyUnstructured(copied) } // UpdateStatus implements dynamic.ResourceInterface @@ -321,7 +420,7 @@ func (r *mockNamespaceableResource) Delete(ctx context.Context, name string, opt if r.client.deleteResource(r.gvr, r.namespace, name) { return nil } - return fmt.Errorf("resource %s/%s not found", r.namespace, name) + return notFoundError(r.gvr, name) } // DeleteCollection implements dynamic.ResourceInterface @@ -333,7 +432,7 @@ func (r *mockNamespaceableResource) DeleteCollection(ctx context.Context, option func (r *mockNamespaceableResource) Get(ctx context.Context, name string, options metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { obj, found := r.client.getResource(r.gvr, r.namespace, name) if !found { - return nil, fmt.Errorf("%s \"%s\" not found", r.gvr.Resource, name) + return nil, notFoundError(r.gvr, name) } return deepCopyUnstructured(obj) } @@ -365,11 +464,19 @@ func (r *mockNamespaceableResource) Watch(ctx context.Context, opts metav1.ListO return watch.NewFake(), nil } -// Patch implements dynamic.ResourceInterface +// Patch implements dynamic.ResourceInterface. The patch is applied under the +// store lock so two concurrent patches cannot both start from the same copy. func (r *mockNamespaceableResource) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, options metav1.PatchOptions, subresources ...string) (*unstructured.Unstructured, error) { - obj, found := r.client.getResource(r.gvr, r.namespace, name) + if r.client.BeforePatch != nil { + r.client.BeforePatch(r.gvr, r.namespace, name, pt, data) + } + + r.client.mu.Lock() + defer r.client.mu.Unlock() + + obj, found := r.client.getLocked(r.gvr, r.namespace, name) if !found { - return nil, fmt.Errorf("%s \"%s\" not found", r.gvr.Resource, name) + return nil, notFoundError(r.gvr, name) } // Make a copy to modify @@ -378,8 +485,8 @@ func (r *mockNamespaceableResource) Patch(ctx context.Context, name string, pt t return nil, err } - // Apply JSON patch if it's a JSONPatchType - if pt == types.JSONPatchType { + switch pt { + case types.JSONPatchType: var patches []jsonPatchOp if err := json.Unmarshal(data, &patches); err != nil { return nil, fmt.Errorf("failed to unmarshal JSON patch: %w", err) @@ -391,27 +498,38 @@ func (r *mockNamespaceableResource) Patch(ctx context.Context, name string, pt t } } - // Store the updated object - r.client.addResource(r.gvr, r.namespace, name, copied) - } - - // Apply Merge patch if it's a MergePatchType (RFC 7396) - if pt == types.MergePatchType { + case types.MergePatchType: // RFC 7396 var patchMap map[string]interface{} if err := json.Unmarshal(data, &patchMap); err != nil { return nil, fmt.Errorf("failed to unmarshal merge patch: %w", err) } + if !r.client.IgnoreResourceVersion { + if rv := mergePatchResourceVersion(patchMap); rv != "" && rv != obj.GetResourceVersion() { + return nil, conflictError(r.gvr, name) + } + } + // Deep merge patchMap into copied.Object mergeMaps(copied.Object, patchMap) - // Store the updated object - r.client.addResource(r.gvr, r.namespace, name, copied) + default: + return deepCopyUnstructured(copied) } + bumpResourceVersion(copied) + r.client.storeLocked(r.gvr, r.namespace, name, copied) return deepCopyUnstructured(copied) } +// mergePatchResourceVersion returns metadata.resourceVersion from a decoded +// merge patch, or "" when the patch does not carry one. +func mergePatchResourceVersion(patchMap map[string]interface{}) string { + metadata, _ := patchMap["metadata"].(map[string]interface{}) + rv, _ := metadata["resourceVersion"].(string) + return rv +} + // mergeMaps recursively merges src into dst according to RFC 7396 JSON Merge Patch. // - If src has a key with value null, the key is deleted from dst // - If src has a key with a non-null value, it replaces the value in dst diff --git a/pkg/kubevirt/vm_update.go b/pkg/kubevirt/vm_update.go new file mode 100644 index 0000000..e226c45 --- /dev/null +++ b/pkg/kubevirt/vm_update.go @@ -0,0 +1,304 @@ +package kubevirt + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + kubevirtv1 "kubevirt.io/api/core/v1" + + "github.com/kubevirt/redfish-controller/pkg/logger" +) + +const ( + // vmUpdateMaxAttempts bounds how often updateVM re-reads the VM and + // re-applies a mutation after the API server rejected the write as stale. + vmUpdateMaxAttempts = 5 + vmUpdateBaseBackoff = 50 * time.Millisecond + vmUpdateMaxBackoff = 500 * time.Millisecond +) + +// updateVM is the only path for changing a VirtualMachine by reading it, +// editing a copy and patching the difference back. +// +// Two things make such a read-modify-write safe against another writer: +// +// - A per-VM lock serializes the writers inside this process. An HTTP +// handler (boot override), a task worker (virtual media insert) and the +// boot-once watcher can all touch the same VM within milliseconds of each +// other. The lock covers only the read-modify-write; slow work such as +// creating PVCs or downloading ISOs must stay outside it. +// - The patch carries the resourceVersion of the object it was computed +// from (see computeVMPatch), so the API server rejects it with a Conflict +// when anyone changed the VM in between, instead of letting a stale copy +// of the disks or volumes array silently replace a newer one. The +// mutation is then re-run on a fresh read. This also covers writers +// outside this process. +// +// Because mutate can run more than once it must not have side effects beyond +// the copy it is given, and it must take every decision from that copy. +// Nothing inside mutate may call updateVM again: the lock is not re-entrant. +// The read is always a live GET; an object served from a cache would both +// manufacture spurious conflicts and be re-read unchanged on retry. +func (c *Client) updateVM(ctx context.Context, namespace, name string, mutate func(vm *kubevirtv1.VirtualMachine) error) (*kubevirtv1.VirtualMachine, error) { + unlock := c.lockVM(namespace, name) + defer unlock() + + gvr := schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + + for attempt := 1; ; attempt++ { + vm, err := c.GetVM(namespace, name) + if err != nil { + return nil, fmt.Errorf("failed to get VM %s: %w", name, err) + } + + vmCopy := vm.DeepCopy() + if err := mutate(vmCopy); err != nil { + return nil, err + } + + patch, err := computeVMPatch(vm, vmCopy) + if err != nil { + return nil, fmt.Errorf("failed to compute VM patch: %w", err) + } + if isEmptyMergePatch(patch) { + return vm, nil + } + + updated, err := c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err == nil { + result, convErr := unstructuredToVM(updated) + if convErr != nil { + return vmCopy, nil + } + return result, nil + } + if !apierrors.IsConflict(err) || attempt >= vmUpdateMaxAttempts { + return nil, fmt.Errorf("failed to update VM %s/%s: %w", namespace, name, err) + } + + logger.Info("VM %s/%s changed while an update was being prepared, retrying on a fresh copy (attempt %d/%d)", namespace, name, attempt, vmUpdateMaxAttempts) + c.sleepFor(vmUpdateBackoff(attempt)) + } +} + +func (c *Client) lockVM(namespace, name string) func() { + if c.vmLocks == nil { + return func() {} + } + return c.vmLocks.Lock(namespace + "/" + name) +} + +func (c *Client) sleepFor(d time.Duration) { + if c.sleep != nil { + c.sleep(d) + return + } + time.Sleep(d) +} + +func vmUpdateBackoff(attempt int) time.Duration { + d := vmUpdateBaseBackoff * time.Duration(attempt) + if d > vmUpdateMaxBackoff { + return vmUpdateMaxBackoff + } + return d +} + +func isEmptyMergePatch(patch []byte) bool { + var fields map[string]json.RawMessage + if err := json.Unmarshal(patch, &fields); err != nil { + return false + } + return len(fields) == 0 +} + +// withResourceVersion sets metadata.resourceVersion in a JSON merge patch. +func withResourceVersion(patch []byte, resourceVersion string) ([]byte, error) { + var fields map[string]interface{} + if err := json.Unmarshal(patch, &fields); err != nil { + return nil, fmt.Errorf("failed to decode merge patch: %w", err) + } + metadata, _ := fields["metadata"].(map[string]interface{}) + if metadata == nil { + metadata = map[string]interface{}{} + fields["metadata"] = metadata + } + metadata["resourceVersion"] = resourceVersion + return json.Marshal(fields) +} + +// addCDROMDisk appends a SATA CD-ROM disk entry named deviceName unless one +// already exists. It returns whether the entry was added. +func addCDROMDisk(vm *kubevirtv1.VirtualMachine, deviceName string) bool { + if vm.Spec.Template == nil { + vm.Spec.Template = &kubevirtv1.VirtualMachineInstanceTemplateSpec{} + } + for _, disk := range vm.Spec.Template.Spec.Domain.Devices.Disks { + if disk.Name == deviceName { + return false + } + } + vm.Spec.Template.Spec.Domain.Devices.Disks = append(vm.Spec.Template.Spec.Domain.Devices.Disks, kubevirtv1.Disk{ + Name: deviceName, + DiskDevice: kubevirtv1.DiskDevice{ + CDRom: &kubevirtv1.CDRomTarget{Bus: kubevirtv1.DiskBusSATA}, + }, + }) + return true +} + +// bootOverrideWantsCD reports whether a one-time boot override to CD was +// recorded on the VM before the CD-ROM device existed, so a device added now +// must be placed first in the boot order. +// +// It requires the boot-once state to be fully recorded (label and saved +// original boot order), not just the override annotations: handleBootUpdate +// writes the annotations first and the boot-once state a moment later, and +// reordering in between would make SetBootOnce capture the reordered disks +// as the "original" boot order. A permanent boot order change (SetBootOrder) +// clears the override annotations, so it never reaches here. +func bootOverrideWantsCD(vm *kubevirtv1.VirtualMachine) bool { + annotations := vm.GetAnnotations() + return annotations["redfish.boot.source.override.enabled"] == "Once" && + annotations["redfish.boot.source.override.target"] == "Cd" && + vm.GetLabels()[BootOnceLabel] == "enabled" && + annotations[BootOnceOriginalConfigAnnotation] != "" +} + +// applyPendingCDBootOverride re-stamps the boot order so CD-ROM devices come +// first and records the new device in the saved original boot order with no +// boot order of its own. The restore after the one-time boot then clears the +// device's bootOrder instead of leaving two disks at the same position. +func (c *Client) applyPendingCDBootOverride(vm *kubevirtv1.VirtualMachine, deviceName string) error { + if err := c.modifyVmBootOrder(vm, "Cd"); err != nil { + return err + } + return addDiskToOriginalBootConfig(vm, deviceName) +} + +func addDiskToOriginalBootConfig(vm *kubevirtv1.VirtualMachine, diskName string) error { + annotations := vm.GetAnnotations() + configJSON := annotations[BootOnceOriginalConfigAnnotation] + if configJSON == "" { + return nil + } + var bootOrders []BootOrderConfig + if err := json.Unmarshal([]byte(configJSON), &bootOrders); err != nil { + return fmt.Errorf("failed to unmarshal boot order config: %w", err) + } + for _, entry := range bootOrders { + if entry.DiskName == diskName { + return nil + } + } + bootOrders = append(bootOrders, BootOrderConfig{DiskName: diskName}) + updated, err := json.Marshal(bootOrders) + if err != nil { + return fmt.Errorf("failed to marshal boot order config: %w", err) + } + annotations[BootOnceOriginalConfigAnnotation] = string(updated) + vm.SetAnnotations(annotations) + return nil +} + +// virtualMediaDevices lists the devices the bridge has inserted media into, +// taken from the virtual-media.redfish/ annotations. +func virtualMediaDevices(vm *kubevirtv1.VirtualMachine) []string { + var devices []string + for key := range vm.GetAnnotations() { + if strings.HasPrefix(key, VirtualMediaURLAnnotationPrefix) { + devices = append(devices, strings.TrimPrefix(key, VirtualMediaURLAnnotationPrefix)) + } + } + return devices +} + +func hasDisk(vm *kubevirtv1.VirtualMachine, name string) bool { + if vm.Spec.Template == nil { + return false + } + for _, disk := range vm.Spec.Template.Spec.Domain.Devices.Disks { + if disk.Name == name { + return true + } + } + return false +} + +func hasVolume(vm *kubevirtv1.VirtualMachine, name string) bool { + if vm.Spec.Template == nil { + return false + } + for _, volume := range vm.Spec.Template.Spec.Volumes { + if volume.Name == name { + return true + } + } + return false +} + +// virtualMediaDevicesMissingDisk returns the inserted devices whose volume is +// present but whose disk entry is not. A volume without a disk entry is the +// half-state a lost write leaves behind; KubeVirt attaches it as a writable +// hard disk, which fails on a read-only ISO and leaves the VM crash-looping. +func virtualMediaDevicesMissingDisk(vm *kubevirtv1.VirtualMachine) []string { + var missing []string + for _, device := range virtualMediaDevices(vm) { + if hasVolume(vm, device) && !hasDisk(vm, device) { + missing = append(missing, device) + } + } + return missing +} + +// repairVirtualMediaDevices restores the disk entry of every inserted device +// that lost it, and places it first in the boot order when a boot override to +// CD is pending. It is run from the VM watcher, so the repair happens on the +// next event or reconcile tick without any long-lived goroutine, and the +// decision is re-taken on a fresh copy inside the locked update. +func (c *Client) repairVirtualMediaDevices(vm *kubevirtv1.VirtualMachine) { + namespace, name := vm.GetNamespace(), vm.GetName() + + // The opposite half-state is only logged: it is not the state a lost + // write produces, and the next insert or eject rewrites both halves. + // The watcher sees a VM only when it changes, so this does not repeat. + for _, device := range virtualMediaDevices(vm) { + if hasDisk(vm, device) && !hasVolume(vm, device) { + logger.Warning("VM %s/%s has a disk entry for virtual media device %s but no volume", namespace, name, device) + } + } + if len(virtualMediaDevicesMissingDisk(vm)) == 0 { + return + } + + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + + var repaired []string + _, err := c.updateVM(ctx, namespace, name, func(vmCopy *kubevirtv1.VirtualMachine) error { + repaired = virtualMediaDevicesMissingDisk(vmCopy) + for _, device := range repaired { + addCDROMDisk(vmCopy, device) + if bootOverrideWantsCD(vmCopy) { + if err := c.applyPendingCDBootOverride(vmCopy, device); err != nil { + return err + } + } + } + return nil + }) + if err != nil { + logger.Error("Failed to repair virtual media disk entries for VM %s/%s: %v", namespace, name, err) + return + } + if len(repaired) > 0 { + logger.Warning("Repaired VM %s/%s: re-added the missing CD-ROM disk entry for virtual media device(s) %v", namespace, name, repaired) + } +} diff --git a/pkg/kubevirt/vm_update_test.go b/pkg/kubevirt/vm_update_test.go new file mode 100644 index 0000000..dbeeab1 --- /dev/null +++ b/pkg/kubevirt/vm_update_test.go @@ -0,0 +1,619 @@ +package kubevirt + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + kubevirtv1 "kubevirt.io/api/core/v1" +) + +const ( + testISOURL = "https://example.com/boot.iso" + cdromDevice = "cdrom0" + rootDiskName = "rootdisk" +) + +// newRootDiskVM mirrors the VMs Ironic drives: one virtio root disk that is +// the only boot device, and no CD-ROM yet. +func newRootDiskVM(namespace, name string) *kubevirtv1.VirtualMachine { + rootBootOrder := uint(1) + return &kubevirtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + Spec: kubevirtv1.VirtualMachineSpec{ + Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Domain: kubevirtv1.DomainSpec{ + Devices: kubevirtv1.Devices{ + Disks: []kubevirtv1.Disk{{ + Name: rootDiskName, + BootOrder: &rootBootOrder, + DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: kubevirtv1.DiskBusVirtio}}, + }}, + }, + }, + Volumes: []kubevirtv1.Volume{{ + Name: rootDiskName, + VolumeSource: kubevirtv1.VolumeSource{DataVolume: &kubevirtv1.DataVolumeSource{Name: name + "-root"}}, + }}, + }, + }, + }, + } +} + +// withRootBootOrder sets the root disk's boot order. A previous enroll cycle +// leaves it at 2 (the CD-ROM was 1) after the eject removed the CD-ROM, which +// is the state the corrupted production VMs were in: the next boot-once then +// changes the disks array (2 -> 1), so its patch carries the whole array. +func withRootBootOrder(vm *kubevirtv1.VirtualMachine, order uint) *kubevirtv1.VirtualMachine { + diskByName(vm, rootDiskName).BootOrder = &order + return vm +} + +// withBootOnceToCD puts the VM in the state SetBootOnce leaves behind when +// it ran before any CD-ROM existed: boot-once label, saved original boot +// order with only the root disk, override annotations, root disk still first. +func withBootOnceToCD(vm *kubevirtv1.VirtualMachine) *kubevirtv1.VirtualMachine { + vm.Labels[BootOnceLabel] = "enabled" + vm.Annotations[BootOnceOriginalConfigAnnotation] = `[{"diskName":"rootdisk","bootOrder":1}]` + vm.Annotations[BootOnceVMIUIDAnnotation] = "" + vm.Annotations["redfish.boot.source.override.enabled"] = "Once" + vm.Annotations["redfish.boot.source.override.target"] = "Cd" + vm.Annotations["redfish.boot.source.override.mode"] = "UEFI" + return vm +} + +func withCDROM(vm *kubevirtv1.VirtualMachine, claimName string) *kubevirtv1.VirtualMachine { + if vm.Annotations == nil { + vm.Annotations = map[string]string{} + } + addCDROMDisk(vm, cdromDevice) + vm.Spec.Template.Spec.Volumes = append(vm.Spec.Template.Spec.Volumes, kubevirtv1.Volume{ + Name: cdromDevice, + VolumeSource: kubevirtv1.VolumeSource{PersistentVolumeClaim: &kubevirtv1.PersistentVolumeClaimVolumeSource{ + PersistentVolumeClaimVolumeSource: corev1.PersistentVolumeClaimVolumeSource{ClaimName: claimName}, + }}, + }) + vm.Annotations[VirtualMediaURLAnnotationPrefix+cdromDevice] = testISOURL + return vm +} + +func newVMWriterTestClient(t *testing.T, vms ...*kubevirtv1.VirtualMachine) (*Client, *MockDynamicClient, *fake.Clientset) { + t.Helper() + mock := NewMockDynamicClient() + for _, vm := range vms { + if err := mock.AddVM(vm); err != nil { + t.Fatalf("failed to add VM %s: %v", vm.Name, err) + } + } + cfg := &MockConfig{} + cfg.dataVolumeConfig.storageSize = "3Gi" + cfg.dataVolumeConfig.vmUpdateTimeout = "30s" + cfg.dataVolumeConfig.isoDownloadTimeout = "30m" + cfg.dataVolumeConfig.helperImage = "alpine:latest" + + k8s := fake.NewSimpleClientset() + client := NewClientWithClients(k8s, mock, 30*time.Second, cfg) + client.sleep = func(time.Duration) {} + return client, mock, k8s +} + +func diskByName(vm *kubevirtv1.VirtualMachine, name string) *kubevirtv1.Disk { + for i := range vm.Spec.Template.Spec.Domain.Devices.Disks { + if vm.Spec.Template.Spec.Domain.Devices.Disks[i].Name == name { + return &vm.Spec.Template.Spec.Domain.Devices.Disks[i] + } + } + return nil +} + +func volumeByName(vm *kubevirtv1.VirtualMachine, name string) *kubevirtv1.Volume { + for i := range vm.Spec.Template.Spec.Volumes { + if vm.Spec.Template.Spec.Volumes[i].Name == name { + return &vm.Spec.Template.Spec.Volumes[i] + } + } + return nil +} + +func bootOrderOf(disk *kubevirtv1.Disk) string { + if disk == nil { + return "" + } + if disk.BootOrder == nil { + return "none" + } + return fmt.Sprint(*disk.BootOrder) +} + +func assertBootOrder(t *testing.T, vm *kubevirtv1.VirtualMachine, diskName, want string) { + t.Helper() + if got := bootOrderOf(diskByName(vm, diskName)); got != want { + t.Errorf("disk %s: boot order %s, want %s", diskName, got, want) + } +} + +// assertVolumeDiskPairs checks the invariant the bug violated: an inserted +// virtual media device has both its volume and its disk entry, or neither. +func assertVolumeDiskPairs(t *testing.T, vm *kubevirtv1.VirtualMachine) { + t.Helper() + for _, device := range virtualMediaDevices(vm) { + if hasVolume(vm, device) != hasDisk(vm, device) { + t.Errorf("VM %s: device %s has volume=%v disk=%v, must have both or neither", vm.Name, device, hasVolume(vm, device), hasDisk(vm, device)) + } + } + for _, volume := range vm.Spec.Template.Spec.Volumes { + if strings.HasPrefix(volume.Name, "cdrom") && !hasDisk(vm, volume.Name) { + t.Errorf("VM %s: volume %s has no disk entry; KubeVirt would attach it as a writable hard disk", vm.Name, volume.Name) + } + } +} + +// landInsertDirectly simulates the insert worker's patch reaching the API +// server while another writer sits between its read and its write. +func landInsertDirectly(t *testing.T, mock *MockDynamicClient, namespace, name string) { + t.Helper() + if err := mock.MutateVM(namespace, name, func(vm *kubevirtv1.VirtualMachine) { withCDROM(vm, name+"-bootiso-1") }); err != nil { + t.Fatalf("failed to inject the insert: %v", err) + } +} + +func TestComputeVMPatch_CarriesResourceVersion(t *testing.T) { + vm := newRootDiskVM("ns", "vm") + vm.ResourceVersion = "42" + modified := vm.DeepCopy() + modified.Annotations["a"] = "b" + + patch, err := computeVMPatch(vm, modified) + if err != nil { + t.Fatal(err) + } + var fields map[string]interface{} + if err := json.Unmarshal(patch, &fields); err != nil { + t.Fatal(err) + } + metadata, _ := fields["metadata"].(map[string]interface{}) + if metadata["resourceVersion"] != "42" { + t.Errorf("patch %s does not carry resourceVersion 42", patch) + } + if annotations, _ := metadata["annotations"].(map[string]interface{}); annotations["a"] != "b" { + t.Errorf("patch %s lost the annotation change", patch) + } +} + +func TestComputeVMPatch_NoChangeStaysEmpty(t *testing.T) { + vm := newRootDiskVM("ns", "vm") + vm.ResourceVersion = "42" + + patch, err := computeVMPatch(vm, vm.DeepCopy()) + if err != nil { + t.Fatal(err) + } + if !isEmptyMergePatch(patch) { + t.Errorf("expected an empty patch for an unchanged VM, got %s", patch) + } +} + +func TestMockPatch_StaleResourceVersionIsRejected(t *testing.T) { + _, mock, _ := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + gvr := schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + ctx := context.Background() + + stale := []byte(`{"metadata":{"resourceVersion":"999","annotations":{"x":"y"}}}`) + if _, err := mock.Resource(gvr).Namespace("ns").Patch(ctx, "vm", types.MergePatchType, stale, metav1.PatchOptions{}); !apierrors.IsConflict(err) { + t.Fatalf("expected a Conflict for a stale resourceVersion, got %v", err) + } + + current, err := mock.GetVM("ns", "vm") + if err != nil { + t.Fatal(err) + } + fresh := []byte(fmt.Sprintf(`{"metadata":{"resourceVersion":%q,"annotations":{"x":"y"}}}`, current.ResourceVersion)) + if _, err := mock.Resource(gvr).Namespace("ns").Patch(ctx, "vm", types.MergePatchType, fresh, metav1.PatchOptions{}); err != nil { + t.Fatalf("patch with the current resourceVersion failed: %v", err) + } + + after, _ := mock.GetVM("ns", "vm") + if after.ResourceVersion == current.ResourceVersion { + t.Errorf("resourceVersion was not bumped by a successful write") + } + if after.Annotations["x"] != "y" { + t.Errorf("patch was not applied") + } + + // A patch without a resourceVersion is applied blindly, like the API server does. + blind := []byte(`{"metadata":{"labels":{"l":"v"}}}`) + if _, err := mock.Resource(gvr).Namespace("ns").Patch(ctx, "vm", types.MergePatchType, blind, metav1.PatchOptions{}); err != nil { + t.Fatalf("patch without resourceVersion failed: %v", err) + } +} + +// TestSetBootOnce_StaleRead_WithoutResourceVersionDropsCDROMDisk reproduces +// the production corruption: SetBootOnce reads the VM, the insert lands, then +// the boot-once patch carrying the stale disks array lands on top. Without +// the resourceVersion precondition (the behaviour before this fix) the CD-ROM +// disk entry is deleted while its volume survives. +func TestSetBootOnce_StaleRead_WithoutResourceVersionDropsCDROMDisk(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, withRootBootOrder(newRootDiskVM("ns", "vm"), 2)) + mock.IgnoreResourceVersion = true + + var injected bool + mock.BeforePatch = func(_ schema.GroupVersionResource, namespace, name string, _ types.PatchType, _ []byte) { + if !injected { + injected = true + landInsertDirectly(t, mock, namespace, name) + } + } + + if err := client.SetBootOnce("ns", "vm", "Cd"); err != nil { + t.Fatalf("SetBootOnce failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + if diskByName(got, cdromDevice) != nil { + t.Fatalf("expected the stale boot-once patch to delete the %s disk entry (the bug), but it is still there", cdromDevice) + } + if volumeByName(got, cdromDevice) == nil { + t.Fatalf("expected the %s volume to survive the stale patch", cdromDevice) + } +} + +// TestSetBootOnce_StaleRead_RetriesOnFreshCopy is the same interleaving with +// the fix: the stale patch is rejected, SetBootOnce re-reads, sees the +// CD-ROM and orders it first. +func TestSetBootOnce_StaleRead_RetriesOnFreshCopy(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, withRootBootOrder(newRootDiskVM("ns", "vm"), 2)) + + var patches int + mock.BeforePatch = func(_ schema.GroupVersionResource, namespace, name string, _ types.PatchType, _ []byte) { + patches++ + if patches == 1 { + landInsertDirectly(t, mock, namespace, name) + } + } + + if err := client.SetBootOnce("ns", "vm", "Cd"); err != nil { + t.Fatalf("SetBootOnce failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + if patches != 2 { + t.Errorf("expected the rejected patch and one retry, got %d patch attempts", patches) + } + if cfg := got.Annotations[BootOnceOriginalConfigAnnotation]; !strings.Contains(cfg, `"diskName":"cdrom0"`) { + t.Errorf("original boot order captured on the retry should include the CD-ROM, got %s", cfg) + } +} + +func TestUpdateVM_GivesUpAfterRepeatedConflicts(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + + var patches int + mock.BeforePatch = func(_ schema.GroupVersionResource, namespace, name string, _ types.PatchType, _ []byte) { + patches++ + _ = mock.MutateVM(namespace, name, func(vm *kubevirtv1.VirtualMachine) { vm.Annotations["touched"] = fmt.Sprint(patches) }) + } + + _, err := client.updateVM(context.Background(), "ns", "vm", func(vm *kubevirtv1.VirtualMachine) error { + vm.SetAnnotations(map[string]string{"mine": "x"}) + return nil + }) + if err == nil || !apierrors.IsConflict(err) { + t.Fatalf("expected a Conflict after exhausting retries, got %v", err) + } + if patches != vmUpdateMaxAttempts { + t.Errorf("expected %d attempts, got %d", vmUpdateMaxAttempts, patches) + } +} + +func TestUpdateVM_NoWriteWhenNothingChanged(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + + var patches int + mock.BeforePatch = func(schema.GroupVersionResource, string, string, types.PatchType, []byte) { patches++ } + + if _, err := client.updateVM(context.Background(), "ns", "vm", func(*kubevirtv1.VirtualMachine) error { return nil }); err != nil { + t.Fatal(err) + } + if patches != 0 { + t.Errorf("a mutation that changes nothing must not write, got %d patches", patches) + } +} + +func TestInsertVirtualMedia_ConflictRetryKeepsPVCNameAndOtherWriters(t *testing.T) { + client, mock, k8s := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + + var injected bool + mock.BeforePatch = func(_ schema.GroupVersionResource, namespace, name string, pt types.PatchType, data []byte) { + if !injected && strings.Contains(string(data), "cdrom0") { + injected = true + _ = mock.MutateVM(namespace, name, func(vm *kubevirtv1.VirtualMachine) { vm.Annotations["someone-else"] = "was-here" }) + } + } + + if err := client.insertVirtualMediaAsync("ns", "vm", cdromDevice, testISOURL); err != nil { + t.Fatalf("insert failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + if got.Annotations["someone-else"] != "was-here" { + t.Errorf("the concurrent writer's change was clobbered") + } + assertBootOrder(t, got, rootDiskName, "1") + assertBootOrder(t, got, cdromDevice, "none") + + claim := volumeByName(got, cdromDevice).PersistentVolumeClaim.ClaimName + pvcs, err := k8s.CoreV1().PersistentVolumeClaims("ns").List(context.Background(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(pvcs.Items) != 1 || pvcs.Items[0].Name != claim { + var names []string + for _, pvc := range pvcs.Items { + names = append(names, pvc.Name) + } + t.Errorf("the retried insert must keep one PVC named like the volume (%s), got PVCs %v", claim, names) + } +} + +func TestInsertVirtualMedia_AfterBootOnceToCD_PutsCDFirst(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, withBootOnceToCD(newRootDiskVM("ns", "vm"))) + + if err := client.insertVirtualMediaAsync("ns", "vm", cdromDevice, testISOURL); err != nil { + t.Fatalf("insert failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + + // After the one-time boot the watcher restores the saved order; the CD-ROM + // must then end up with no boot order, not a duplicate of the root disk's. + restored := got.DeepCopy() + if err := client.restoreBootOrder(restored, got.Annotations[BootOnceOriginalConfigAnnotation]); err != nil { + t.Fatal(err) + } + assertBootOrder(t, restored, rootDiskName, "1") + assertBootOrder(t, restored, cdromDevice, "none") +} + +func TestInsertVirtualMedia_WithoutBootOverride_LeavesBootOrderAlone(t *testing.T) { + client, mock, _ := newVMWriterTestClient(t, newRootDiskVM("ns", "vm")) + + if err := client.insertVirtualMediaAsync("ns", "vm", cdromDevice, testISOURL); err != nil { + t.Fatalf("insert failed: %v", err) + } + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, rootDiskName, "1") + assertBootOrder(t, got, cdromDevice, "none") +} + +// The boot PATCH handler records the override annotations first and the +// boot-once state a moment later. An insert that lands in between must not +// reorder, or SetBootOnce would save the reordered disks as the original. +func TestInsertVirtualMedia_BetweenAnnotationsAndBootOnce_DoesNotReorder(t *testing.T) { + vm := newRootDiskVM("ns", "vm") + vm.Annotations["redfish.boot.source.override.enabled"] = "Once" + vm.Annotations["redfish.boot.source.override.target"] = "Cd" + client, mock, _ := newVMWriterTestClient(t, vm) + + if err := client.insertVirtualMediaAsync("ns", "vm", cdromDevice, testISOURL); err != nil { + t.Fatalf("insert failed: %v", err) + } + got, _ := mock.GetVM("ns", "vm") + assertBootOrder(t, got, rootDiskName, "1") + assertBootOrder(t, got, cdromDevice, "none") + + if err := client.SetBootOnce("ns", "vm", "Cd"); err != nil { + t.Fatalf("SetBootOnce failed: %v", err) + } + got, _ = mock.GetVM("ns", "vm") + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + if cfg := got.Annotations[BootOnceOriginalConfigAnnotation]; !strings.Contains(cfg, `{"diskName":"rootdisk","bootOrder":1}`) || !strings.Contains(cfg, `{"diskName":"cdrom0"}`) { + t.Errorf("original boot order must record root first and the CD-ROM without an order, got %s", cfg) + } +} + +// TestConcurrentVMWriters_ConvergeRegardlessOfOrder runs Ironic's request +// mix (insert, boot override annotations, boot-once; then eject with another +// boot-once) concurrently on many VMs, through the real client code and the +// mock's API-server semantics. Whatever the interleaving, every VM must end +// with its CD-ROM volume and disk paired and the CD-ROM first in the boot +// order; different VMs must not block each other. +func TestConcurrentVMWriters_ConvergeRegardlessOfOrder(t *testing.T) { + const vmCount = 24 + var vms []*kubevirtv1.VirtualMachine + initialRootOrder := map[string]uint{} + for i := 0; i < vmCount; i++ { + vm := newRootDiskVM("ns", fmt.Sprintf("vm-%d", i)) + // Half the VMs look like a first enroll (root first), half like the + // next cycle after an eject (root second), where a stale boot-once + // patch rewrites the disks array. + initialRootOrder[vm.Name] = uint(1 + i%2) + vms = append(vms, withRootBootOrder(vm, initialRootOrder[vm.Name])) + } + client, mock, _ := newVMWriterTestClient(t, vms...) + + // Widen the window between every writer's read and its write so the + // goroutines really interleave instead of running one after another. + var patches int64 + mock.BeforePatch = func(schema.GroupVersionResource, string, string, types.PatchType, []byte) { + if atomic.AddInt64(&patches, 1)%3 == 0 { + time.Sleep(time.Millisecond) + } + } + + bootOptions := map[string]interface{}{"bootSourceOverrideEnabled": "Once", "bootSourceOverrideTarget": "Cd", "bootSourceOverrideMode": "UEFI"} + var wg sync.WaitGroup + fail := func(format string, args ...interface{}) { t.Errorf(format, args...) } + + for _, vm := range vms { + name := vm.Name + wg.Add(3) + go func() { + defer wg.Done() + if err := client.insertVirtualMediaAsync("ns", name, cdromDevice, testISOURL); err != nil { + fail("%s: insert failed: %v", name, err) + } + }() + go func() { + defer wg.Done() + if err := client.RecordVMBootOptionsAsAnnotations("ns", name, bootOptions); err != nil { + fail("%s: record boot options failed: %v", name, err) + } + }() + go func() { + defer wg.Done() + if err := client.SetBootOnce("ns", name, "Cd"); err != nil { + fail("%s: SetBootOnce failed: %v", name, err) + } + }() + } + wg.Wait() + + for _, vm := range vms { + got, err := mock.GetVM("ns", vm.Name) + if err != nil { + t.Fatal(err) + } + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + if got.Labels[BootOnceLabel] != "enabled" { + t.Errorf("%s: boot-once label missing", vm.Name) + } + wantRoot := fmt.Sprintf(`{"diskName":"rootdisk","bootOrder":%d}`, initialRootOrder[vm.Name]) + if cfg := got.Annotations[BootOnceOriginalConfigAnnotation]; !strings.Contains(cfg, wantRoot) || !strings.Contains(cfg, `{"diskName":"cdrom0"}`) { + t.Errorf("%s: original boot order must keep the root disk at %d and the CD-ROM unordered, got %s", vm.Name, initialRootOrder[vm.Name], cfg) + } + } + + // Ironic's next cycle: eject while a new boot-once arrives. + for _, vm := range vms { + name := vm.Name + wg.Add(2) + go func() { + defer wg.Done() + if err := client.EjectVirtualMedia("ns", name, cdromDevice); err != nil { + fail("%s: eject failed: %v", name, err) + } + }() + go func() { + defer wg.Done() + if err := client.SetBootOnce("ns", name, "Cd"); err != nil { + fail("%s: second SetBootOnce failed: %v", name, err) + } + }() + } + wg.Wait() + + for _, vm := range vms { + got, _ := mock.GetVM("ns", vm.Name) + assertVolumeDiskPairs(t, got) + if hasVolume(got, cdromDevice) || hasDisk(got, cdromDevice) { + t.Errorf("%s: CD-ROM still present after eject", vm.Name) + } + if _, ok := got.Annotations[VirtualMediaURLAnnotationPrefix+cdromDevice]; ok { + t.Errorf("%s: virtual media annotation still present after eject", vm.Name) + } + } + + if client.vmLocks.size() != 0 { + t.Errorf("per-VM locks leaked: %d entries still alive", client.vmLocks.size()) + } +} + +func TestRepairVirtualMediaDevices_RestoresMissingDiskAndBootOrder(t *testing.T) { + // The exact state found on the corrupted production VMs: volume and + // annotation present, disk entry gone, boot-once to CD pending. + vm := withBootOnceToCD(withCDROM(newRootDiskVM("ns", "vm"), "vm-bootiso-1")) + vm.Spec.Template.Spec.Domain.Devices.Disks = vm.Spec.Template.Spec.Domain.Devices.Disks[:1] + client, mock, _ := newVMWriterTestClient(t, vm) + + client.repairVirtualMediaDevices(vm) + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + disk := diskByName(got, cdromDevice) + if disk == nil || disk.CDRom == nil || disk.CDRom.Bus != kubevirtv1.DiskBusSATA { + t.Fatalf("expected a SATA CD-ROM disk entry to be restored, got %+v", disk) + } + assertBootOrder(t, got, cdromDevice, "1") + assertBootOrder(t, got, rootDiskName, "2") + if cfg := got.Annotations[BootOnceOriginalConfigAnnotation]; !strings.Contains(cfg, `{"diskName":"cdrom0"}`) { + t.Errorf("saved original boot order must now list the CD-ROM without an order, got %s", cfg) + } +} + +func TestRepairVirtualMediaDevices_NoWriteWhenHealthyOrUnmanaged(t *testing.T) { + healthy := withCDROM(newRootDiskVM("ns", "healthy"), "claim") + + // A cdrom volume without the bridge's annotation is not ours to touch. + unmanaged := newRootDiskVM("ns", "unmanaged") + unmanaged.Spec.Template.Spec.Volumes = append(unmanaged.Spec.Template.Spec.Volumes, kubevirtv1.Volume{ + Name: "cdrom1", + VolumeSource: kubevirtv1.VolumeSource{ContainerDisk: &kubevirtv1.ContainerDiskSource{Image: "x"}}, + }) + + client, mock, _ := newVMWriterTestClient(t, healthy, unmanaged) + var patches int + mock.BeforePatch = func(schema.GroupVersionResource, string, string, types.PatchType, []byte) { patches++ } + + client.repairVirtualMediaDevices(healthy) + client.repairVirtualMediaDevices(unmanaged) + + if patches != 0 { + t.Errorf("expected no writes, got %d", patches) + } +} + +// TestHandleVMUpdate_RepairsThenRestoresAfterReboot drives the watcher entry +// point on a corrupted VM whose one-time boot has completed (VMI UID changed): +// the disk entry comes back and the saved boot order is restored in the same pass. +func TestHandleVMUpdate_RepairsThenRestoresAfterReboot(t *testing.T) { + vm := withBootOnceToCD(withCDROM(newRootDiskVM("ns", "vm"), "vm-bootiso-1")) + vm.Spec.Template.Spec.Domain.Devices.Disks = vm.Spec.Template.Spec.Domain.Devices.Disks[:1] + vm.Annotations[BootOnceVMIUIDAnnotation] = "old-vmi" + client, mock, _ := newVMWriterTestClient(t, vm) + if err := mock.AddVMI(&kubevirtv1.VirtualMachineInstance{ObjectMeta: metav1.ObjectMeta{Name: "vm", Namespace: "ns", UID: "new-vmi"}}); err != nil { + t.Fatal(err) + } + + client.handleVMUpdate(vm) + + got, _ := mock.GetVM("ns", "vm") + assertVolumeDiskPairs(t, got) + assertBootOrder(t, got, rootDiskName, "1") + assertBootOrder(t, got, cdromDevice, "none") + if got.Labels[BootOnceLabel] != "" { + t.Errorf("boot-once label should be cleared after restore") + } + if _, ok := got.Annotations[BootOnceOriginalConfigAnnotation]; ok { + t.Errorf("boot-once annotations should be cleared after restore") + } +} diff --git a/pkg/kubevirt/vmlock.go b/pkg/kubevirt/vmlock.go new file mode 100644 index 0000000..2678382 --- /dev/null +++ b/pkg/kubevirt/vmlock.go @@ -0,0 +1,51 @@ +package kubevirt + +import "sync" + +// keyedMutex hands out one mutex per key so that work on different keys runs +// in parallel while work on the same key is serialized. Entries are reference +// counted and dropped as soon as nobody holds or waits for them, so the map +// does not grow with the number of VMs seen over the lifetime of the process. +type keyedMutex struct { + mu sync.Mutex + entries map[string]*keyedMutexEntry +} + +type keyedMutexEntry struct { + sync.Mutex + refs int +} + +func newKeyedMutex() *keyedMutex { + return &keyedMutex{entries: make(map[string]*keyedMutexEntry)} +} + +// Lock blocks until the caller owns key and returns the function that releases it. +func (k *keyedMutex) Lock(key string) func() { + k.mu.Lock() + entry, ok := k.entries[key] + if !ok { + entry = &keyedMutexEntry{} + k.entries[key] = entry + } + entry.refs++ + k.mu.Unlock() + + entry.Lock() + return func() { + entry.Unlock() + k.mu.Lock() + entry.refs-- + if entry.refs == 0 { + delete(k.entries, key) + } + k.mu.Unlock() + } +} + +// size reports how many keys are currently held or waited for. +func (k *keyedMutex) size() int { + k.mu.Lock() + defer k.mu.Unlock() + return len(k.entries) +} diff --git a/pkg/kubevirt/vmlock_test.go b/pkg/kubevirt/vmlock_test.go new file mode 100644 index 0000000..304c10b --- /dev/null +++ b/pkg/kubevirt/vmlock_test.go @@ -0,0 +1,91 @@ +package kubevirt + +import ( + "testing" + "time" +) + +func TestKeyedMutex_SameKeySerializes(t *testing.T) { + km := newKeyedMutex() + + unlockA := km.Lock("ns/vm") + + acquired := make(chan struct{}) + go func() { + unlockB := km.Lock("ns/vm") + close(acquired) + unlockB() + }() + + select { + case <-acquired: + t.Fatal("second lock on the same key was acquired while the first was still held") + case <-time.After(50 * time.Millisecond): + } + + unlockA() + + select { + case <-acquired: + case <-time.After(time.Second): + t.Fatal("second lock on the same key was not acquired after the first was released") + } +} + +func TestKeyedMutex_DifferentKeysDoNotBlock(t *testing.T) { + km := newKeyedMutex() + + unlockA := km.Lock("ns/vm-a") + defer unlockA() + + acquired := make(chan struct{}) + go func() { + unlockB := km.Lock("ns/vm-b") + close(acquired) + unlockB() + }() + + select { + case <-acquired: + case <-time.After(time.Second): + t.Fatal("lock on a different key was blocked by an unrelated holder") + } +} + +func TestKeyedMutex_EntriesAreDroppedWhenUnused(t *testing.T) { + km := newKeyedMutex() + + unlockA := km.Lock("ns/vm-a") + unlockB := km.Lock("ns/vm-b") + if got := km.size(); got != 2 { + t.Fatalf("expected 2 live entries while both keys are held, got %d", got) + } + + unlockA() + if got := km.size(); got != 1 { + t.Fatalf("expected 1 live entry after releasing one key, got %d", got) + } + + unlockB() + if got := km.size(); got != 0 { + t.Fatalf("expected no live entries after releasing every key, got %d", got) + } + + // A key that is held and waited for at the same time stays alive until the last waiter is done. + unlockA = km.Lock("ns/vm-a") + done := make(chan struct{}) + go func() { + unlock := km.Lock("ns/vm-a") + unlock() + close(done) + }() + time.Sleep(20 * time.Millisecond) + if got := km.size(); got != 1 { + t.Fatalf("expected the entry to stay alive while a waiter is queued, got %d entries", got) + } + unlockA() + <-done + if got := km.size(); got != 0 { + t.Fatalf("expected the entry to be dropped after the waiter finished, got %d entries", got) + } +}