From 1d59df4afc07f1ead3c07e83610a010358c8093c Mon Sep 17 00:00:00 2001 From: itayherzberg Date: Sun, 25 Oct 2026 19:57:55 +0200 Subject: [PATCH] fix(server): run a power-off at once while an ISO import is pending Every power action was deferred while the importing label was on the VM, including ForceOff and GracefulShutdown. When an import could not finish (the VM held the ISO volume, or the helper pod never ran) the label stayed and Ironic's power-off was deferred as well, so the host could not be recovered through Redfish. Only an action that starts the VM (On, ForceRestart, GracefulRestart) now waits for the import. A power-off runs immediately and withdraws a deferred power-on: the host was told to be off, so it must not start when the import completes. Signed-off-by: itayherzberg --- pkg/kubevirt/client.go | 31 ++++ pkg/kubevirt/power_after_import_test.go | 50 ++++++ pkg/server/power_off_while_importing_test.go | 165 +++++++++++++++++++ pkg/server/server.go | 22 +++ 4 files changed, 268 insertions(+) create mode 100644 pkg/kubevirt/power_after_import_test.go create mode 100644 pkg/server/power_off_while_importing_test.go diff --git a/pkg/kubevirt/client.go b/pkg/kubevirt/client.go index 26ad6c2..c885e26 100644 --- a/pkg/kubevirt/client.go +++ b/pkg/kubevirt/client.go @@ -2201,6 +2201,37 @@ func (c *Client) SetPowerAfterImportLabel(namespace, name, powerState string) er return nil } +// ClearPowerAfterImportLabel withdraws a deferred power command. A VM without +// the label is left as it is; a missing VM is an error. +func (c *Client) ClearPowerAfterImportLabel(namespace, name string) error { + vm, err := c.GetVM(namespace, name) + if err != nil { + return fmt.Errorf("failed to get VM %s/%s: %w", namespace, name, err) + } + powerCommand := vm.GetLabels()[PowerAfterImportLabel] + if powerCommand == "" { + return nil + } + + patchJSON := fmt.Sprintf(`{"metadata":{"labels":{%q:null}}}`, PowerAfterImportLabel) + + gvr := schema.GroupVersionResource{ + Group: "kubevirt.io", + Version: "v1", + Resource: "virtualmachines", + } + + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + + _, err = c.dynamicClient.Resource(gvr).Namespace(namespace).Patch(ctx, name, types.MergePatchType, []byte(patchJSON), metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("failed to remove power-after-import label from VM %s/%s: %w", namespace, name, err) + } + logger.Info("Withdrew deferred power command %s from VM %s/%s", powerCommand, namespace, name) + return nil +} + // resourceMustParse is a helper for resource.Quantity func resourceMustParse(s string) resource.Quantity { q, _ := resource.ParseQuantity(s) diff --git a/pkg/kubevirt/power_after_import_test.go b/pkg/kubevirt/power_after_import_test.go new file mode 100644 index 0000000..2f0840b --- /dev/null +++ b/pkg/kubevirt/power_after_import_test.go @@ -0,0 +1,50 @@ +package kubevirt + +import ( + "testing" + "time" + + "k8s.io/client-go/kubernetes/fake" +) + +func TestClearPowerAfterImportLabel(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", map[string]string{ + PowerAfterImportLabel: "On", + ImportingLabelPrefix + "cdrom0": "copy-iso-test-vm-1", + }, nil)) + before, _ := mockDynamic.GetVM("test-ns", "test-vm") + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + if err := client.ClearPowerAfterImportLabel("test-ns", "test-vm"); err != nil { + t.Fatalf("ClearPowerAfterImportLabel failed: %v", err) + } + + after, _ := mockDynamic.GetVM("test-ns", "test-vm") + if got, found := after.GetLabels()[PowerAfterImportLabel]; found { + t.Errorf("label %s = %q, want it removed", PowerAfterImportLabel, got) + } + if got := after.GetLabels()[ImportingLabelPrefix+"cdrom0"]; got != "copy-iso-test-vm-1" { + t.Errorf("importing label was modified: %q", got) + } + if after.GetLabels()["existing-label"] != "original-value" { + t.Errorf("unrelated label was modified: %v", after.GetLabels()) + } + assertVMUnchangedExceptLabels(t, before, after) +} + +func TestClearPowerAfterImportLabel_NoLabelIsNoop(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", nil, nil)) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + if err := client.ClearPowerAfterImportLabel("test-ns", "test-vm"); err != nil { + t.Fatalf("ClearPowerAfterImportLabel failed: %v", err) + } +} + +func TestClearPowerAfterImportLabel_VMNotFound(t *testing.T) { + client := NewClientWithClients(fake.NewSimpleClientset(), NewMockDynamicClient(), 30*time.Second, nil) + if err := client.ClearPowerAfterImportLabel("test-ns", "missing"); err == nil { + t.Fatal("expected an error for a missing VM") + } +} diff --git a/pkg/server/power_off_while_importing_test.go b/pkg/server/power_off_while_importing_test.go new file mode 100644 index 0000000..1286627 --- /dev/null +++ b/pkg/server/power_off_while_importing_test.go @@ -0,0 +1,165 @@ +package server + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/kubevirt/redfish-controller/pkg/auth" + "github.com/kubevirt/redfish-controller/pkg/config" + "github.com/kubevirt/redfish-controller/pkg/kubevirt" + "github.com/kubevirt/redfish-controller/pkg/logger" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + kubevirtv1 "kubevirt.io/api/core/v1" +) + +const ( + poffTestNS = "ns-a" + poffTestVM = "vm-a" +) + +// poffTestServer builds a server over a halted VM whose ISO import is still +// in progress (importing label set), the state in which Ironic may decide to +// power the host off. +func poffTestServer(t *testing.T) (*Server, *kubevirt.MockDynamicClient) { + t.Helper() + mockDynamic := kubevirt.NewMockDynamicClient() + halted := kubevirtv1.RunStrategyHalted + if err := mockDynamic.AddVM(&kubevirtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: poffTestVM, + Namespace: poffTestNS, + Labels: map[string]string{ + "redfish-enabled": "true", + kubevirt.ImportingLabelPrefix + "cdrom0": "copy-iso-vm-a-bootiso-1", + }, + }, + Spec: kubevirtv1.VirtualMachineSpec{ + RunStrategy: &halted, + Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Domain: kubevirtv1.DomainSpec{Devices: kubevirtv1.Devices{Disks: []kubevirtv1.Disk{{Name: "rootdisk"}}}}, + Volumes: []kubevirtv1.Volume{{Name: "rootdisk"}}, + }, + }, + }, + }); err != nil { + t.Fatalf("AddVM failed: %v", err) + } + client := kubevirt.NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + cfg := &config.Config{ + Server: config.ServerConfig{Host: "localhost", Port: 8080, TestMode: true}, + Auth: config.AuthConfig{ + Users: []config.UserConfig{ + {Username: "user", Password: config.PasswordConfig{Plain: "pass"}, Chassis: []string{"chassis-a"}}, + }, + }, + Chassis: []config.ChassisConfig{ + { + Name: "chassis-a", + Namespace: poffTestNS, + VMSelector: &kubevirt.VMSelectorConfig{Labels: map[string]string{"redfish-enabled": "true"}}, + }, + }, + } + return NewServer(cfg, client), mockDynamic +} + +func poffReset(server *Server, resetType string) *httptest.ResponseRecorder { + req := httptest.NewRequest("POST", "/redfish/v1/Systems/"+poffTestVM+"/Actions/ComputerSystem.Reset", + strings.NewReader(`{"ResetType":"`+resetType+`"}`)) + req.Header.Set("Content-Type", "application/json") + authCtx := &auth.AuthContext{User: &auth.User{Username: "user", Chassis: []string{"chassis-a"}}} + req = req.WithContext(logger.WithAuth(req.Context(), authCtx)) + w := httptest.NewRecorder() + server.handlePowerAction(w, req, poffTestVM) + return w +} + +func poffVM(t *testing.T, mockDynamic *kubevirt.MockDynamicClient) *kubevirtv1.VirtualMachine { + t.Helper() + vm, err := mockDynamic.GetVM(poffTestNS, poffTestVM) + if err != nil { + t.Fatalf("GetVM failed: %v", err) + } + return vm +} + +// Actions that would start the VM wait for the import; the rest run now. +func TestHandlePowerAction_OnlyStartingActionsAreDeferredWhileImporting(t *testing.T) { + tests := []struct { + resetType string + deferred bool + }{ + {"On", true}, + {"ForceRestart", true}, + {"GracefulRestart", true}, + {"ForceOff", false}, + {"GracefulShutdown", false}, + } + for _, tt := range tests { + t.Run(tt.resetType, func(t *testing.T) { + server, mockDynamic := poffTestServer(t) + + w := poffReset(server, tt.resetType) + + vm := poffVM(t, mockDynamic) + deferredTo := vm.GetLabels()[kubevirt.PowerAfterImportLabel] + if tt.deferred { + if w.Code != http.StatusAccepted || !strings.Contains(w.Body.String(), "deferred") { + t.Fatalf("%s: status %d, body %s; want 202 deferred", tt.resetType, w.Code, w.Body.String()) + } + if deferredTo != tt.resetType { + t.Errorf("%s: label %s = %q, want %q", tt.resetType, kubevirt.PowerAfterImportLabel, deferredTo, tt.resetType) + } + return + } + if w.Code != http.StatusOK { + t.Fatalf("%s: status %d, body %s; want 200", tt.resetType, w.Code, w.Body.String()) + } + if deferredTo != "" { + t.Errorf("%s: label %s = %q, want none", tt.resetType, kubevirt.PowerAfterImportLabel, deferredTo) + } + if got := vm.Spec.RunStrategy; got == nil || *got != kubevirtv1.RunStrategyHalted { + t.Errorf("%s: runStrategy = %v, want Halted", tt.resetType, got) + } + }) + } +} + +// Ironic powers a host off when it gives up waiting for it. A power-off that +// arrives while a power-on is deferred must withdraw the power-on, otherwise +// the VM starts when the import finishes although it was told to stop. +func TestHandlePowerAction_PowerOffWithdrawsDeferredPowerOn(t *testing.T) { + for _, off := range []string{"ForceOff", "GracefulShutdown"} { + t.Run(off, func(t *testing.T) { + server, mockDynamic := poffTestServer(t) + + if w := poffReset(server, "On"); w.Code != http.StatusAccepted { + t.Fatalf("On: status %d, want 202", w.Code) + } + if got := poffVM(t, mockDynamic).GetLabels()[kubevirt.PowerAfterImportLabel]; got != "On" { + t.Fatalf("label %s = %q after On, want On", kubevirt.PowerAfterImportLabel, got) + } + + if w := poffReset(server, off); w.Code != http.StatusOK { + t.Fatalf("%s: status %d, body %s; want 200", off, w.Code, w.Body.String()) + } + + vm := poffVM(t, mockDynamic) + if got, found := vm.GetLabels()[kubevirt.PowerAfterImportLabel]; found { + t.Errorf("label %s = %q is still set after %s", kubevirt.PowerAfterImportLabel, got, off) + } + if got := vm.GetLabels()[kubevirt.ImportingLabelPrefix+"cdrom0"]; got != "copy-iso-vm-a-bootiso-1" { + t.Errorf("importing label was changed by the power-off: %q", got) + } + if got := vm.Spec.RunStrategy; got == nil || *got != kubevirtv1.RunStrategyHalted { + t.Errorf("runStrategy = %v, want Halted", got) + } + }) + } +} diff --git a/pkg/server/server.go b/pkg/server/server.go index df224ce..ecf511d 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -1477,6 +1477,16 @@ func (s *Server) handleTask(w http.ResponseWriter, r *http.Request) { // - w: HTTP response writer // - r: HTTP request // - systemName: Name of the system to perform the action on +// powerStateStartsVM reports whether a reset type leaves the VM running, which +// is what has to wait for a pending ISO import. +func powerStateStartsVM(state string) bool { + switch state { + case "On", "ForceRestart", "GracefulRestart": + return true + } + return false +} + func (s *Server) handlePowerAction(w http.ResponseWriter, r *http.Request, systemName string) { // Parse the request body var resetRequest redfish.ResetRequest @@ -1521,6 +1531,18 @@ func (s *Server) handlePowerAction(w http.ResponseWriter, r *http.Request, syste return } + // Only an action that would start the VM waits for the import. A power-off + // runs at once and withdraws a deferred power-on: the host was told to be + // off, so it must not start when the import finishes. + if importing && !powerStateStartsVM(powerState) { + if clearErr := s.kubevirtClient.ClearPowerAfterImportLabel(namespace, vmName); clearErr != nil { + logger.Error("Failed to withdraw deferred power command for VM %s/%s: %v", namespace, vmName, clearErr) + s.sendInternalError(w, fmt.Sprintf("Failed to withdraw deferred power action: %v", clearErr)) + return + } + importing = false + } + if importing { // Defer the power command until import completes if setErr := s.kubevirtClient.SetPowerAfterImportLabel(namespace, vmName, powerState); setErr != nil {