From d80d1e3cb03652e98e8a8b9a20c3e6253d9d5e84 Mon Sep 17 00:00:00 2001 From: itayherzberg Date: Sun, 25 Oct 2026 19:21:08 +0200 Subject: [PATCH] fix(server): defer Power On for an InsertMedia whose job has not run yet InsertMedia is answered with 202 before its job runs, and the job only puts the importing label on the VM after creating the import pod. Power On is deferred only while that label is present, and Ironic sends it right after InsertMedia, so it landed in the gap: the VM started with an empty CD-ROM and took the RWO ISO volume away from the import pod, which then hung on Multi-Attach (or failed with Permission denied on the same node) and left the label on the VM, deferring every later power action. The handler now writes a dated placeholder into the importing label and the image URL annotation before answering. The job replaces it with the pod name as before. A repeated InsertMedia while the placeholder is set is answered as in progress instead of queueing a second job. The job's terminal failure clears the placeholder, and the periodic import-pod reconcile removes placeholders older than five minutes, whose job was lost with the in-memory queue. Signed-off-by: itayherzberg --- pkg/kubevirt/client.go | 16 +- pkg/kubevirt/import_pending.go | 131 +++++++++++++++ pkg/kubevirt/import_pending_test.go | 245 ++++++++++++++++++++++++++++ pkg/server/insert_pending_test.go | 212 ++++++++++++++++++++++++ pkg/server/server.go | 18 ++ pkg/server/task_manager.go | 17 ++ 6 files changed, 634 insertions(+), 5 deletions(-) create mode 100644 pkg/kubevirt/import_pending.go create mode 100644 pkg/kubevirt/import_pending_test.go create mode 100644 pkg/server/insert_pending_test.go diff --git a/pkg/kubevirt/client.go b/pkg/kubevirt/client.go index 26ad6c2..8055b17 100644 --- a/pkg/kubevirt/client.go +++ b/pkg/kubevirt/client.go @@ -1658,11 +1658,15 @@ func (c *Client) CheckInsertedMedia(namespace, name, mediaID, imageURL string) ( } } } - if !hasPVC { + // An accepted insert whose job has not run yet has the importing label + // (a placeholder) but no PVC; it counts as inserted so a retried request + // does not queue a second job. + importing := vm.GetLabels()[ImportingLabelPrefix+mediaID] != "" + if !hasPVC && !importing { return MediaStateNone, nil } - // A PVC is attached. Compare the stored URL. + // Media is attached or on its way. Compare the stored URL. annotationKey := VirtualMediaURLAnnotationPrefix + mediaID storedURL := vm.GetAnnotations()[annotationKey] @@ -1670,9 +1674,7 @@ func (c *Client) CheckInsertedMedia(namespace, name, mediaID, imageURL string) ( return MediaStateConflict, nil } - // Same URL — check whether import is still in progress. - labelKey := ImportingLabelPrefix + mediaID - if vm.GetLabels()[labelKey] != "" { + if importing { return MediaStateImporting, nil } @@ -3680,6 +3682,10 @@ func (c *Client) removeImportingLabel(namespace, vmName, deviceName string) erro func (c *Client) reconcileExistingImportPods(namespace string) error { logger.Info("Reconciling existing import pods in namespace %s", namespace) + if err := c.reconcileStalePendingImports(namespace, time.Now()); err != nil { + logger.Error("Failed to reconcile pending imports in namespace %s: %v", namespace, err) + } + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) defer cancel() diff --git a/pkg/kubevirt/import_pending.go b/pkg/kubevirt/import_pending.go new file mode 100644 index 0000000..6ac7757 --- /dev/null +++ b/pkg/kubevirt/import_pending.go @@ -0,0 +1,131 @@ +package kubevirt + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/kubevirt/redfish-controller/pkg/logger" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +// InsertMedia is answered with 202 before its job runs, and the job labels the +// VM with the import pod only after creating it. Ironic sends Power On right +// after InsertMedia, and power-on is deferred only while the importing label +// is on the VM. Without a marker written before the 202, Power On lands in +// that gap, the VM starts with an empty CD-ROM and takes the RWO ISO volume +// away from the import pod. The handler therefore writes a placeholder value +// into the importing label up front; the job later replaces it with the pod +// name, so the pod watcher's ownership check is unchanged. +// +// The placeholder carries its creation time. The job queue is in memory, so a +// restart loses queued jobs, and a job can also fail before it ever creates a +// pod: a placeholder nobody replaces would defer power actions forever. The +// job's terminal failure clears it, and the periodic reconcile sweeps +// placeholders older than stalePendingImportAge. +const ImportPendingPrefix = "pending-" + +const stalePendingImportAge = 5 * time.Minute + +func importPendingValue(now time.Time) string { + return ImportPendingPrefix + strconv.FormatInt(now.Unix(), 10) +} + +func isPendingImport(value string) bool { + return strings.HasPrefix(value, ImportPendingPrefix) +} + +// pendingImportIsStale reports whether a placeholder is old enough that its job +// cannot still be queued. A placeholder that cannot be parsed counts as stale. +func pendingImportIsStale(value string, now time.Time) bool { + if !isPendingImport(value) { + return false + } + created, err := strconv.ParseInt(strings.TrimPrefix(value, ImportPendingPrefix), 10, 64) + if err != nil { + return true + } + return now.Sub(time.Unix(created, 0)) > stalePendingImportAge +} + +// MarkImportPending records on the VM that an import of imageURL into mediaID +// has been accepted but not started. It returns false, without touching the +// VM, when an import is already recorded for the device. +func (c *Client) MarkImportPending(namespace, name, mediaID, imageURL string) (bool, error) { + vm, err := c.GetVM(namespace, name) + if err != nil { + return false, fmt.Errorf("failed to get VM %s/%s: %w", namespace, name, err) + } + labelKey := ImportingLabelPrefix + mediaID + if vm.GetLabels()[labelKey] != "" { + return false, nil + } + + patchJSON := fmt.Sprintf(`{"metadata":{"labels":{%q:%q},"annotations":{%q:%q}}}`, + labelKey, importPendingValue(time.Now()), VirtualMediaURLAnnotationPrefix+mediaID, imageURL) + if err := c.patchVMMetadata(namespace, name, patchJSON); err != nil { + return false, fmt.Errorf("failed to mark import pending on VM %s/%s: %w", namespace, name, err) + } + logger.Info("Marked import of %s as pending on VM %s/%s", mediaID, namespace, name) + return true, nil +} + +// ClearPendingImport removes the placeholder for mediaID. A label that already +// names an import pod or a CDI import is owned by the watchers and is left alone. +func (c *Client) ClearPendingImport(namespace, name, mediaID string) error { + vm, err := c.GetVM(namespace, name) + if err != nil { + return fmt.Errorf("failed to get VM %s/%s: %w", namespace, name, err) + } + labelKey := ImportingLabelPrefix + mediaID + if !isPendingImport(vm.GetLabels()[labelKey]) { + return nil + } + if err := c.removeImportingLabel(namespace, name, mediaID); err != nil { + return err + } + logger.Info("Cleared pending import of %s from VM %s/%s", mediaID, namespace, name) + return nil +} + +// reconcileStalePendingImports removes placeholders whose job can no longer be +// waiting in the queue (see ImportPendingPrefix). +func (c *Client) reconcileStalePendingImports(namespace string, now time.Time) error { + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + + vmList, err := c.dynamicClient.Resource(virtualMachinesGVR).Namespace(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list VMs in namespace %s: %w", namespace, err) + } + + for _, item := range vmList.Items { + for key, value := range item.GetLabels() { + if !strings.HasPrefix(key, ImportingLabelPrefix) || !pendingImportIsStale(value, now) { + continue + } + mediaID := strings.TrimPrefix(key, ImportingLabelPrefix) + logger.Warning("Import of %s on VM %s/%s was accepted but never started (label %s=%s), clearing it", mediaID, namespace, item.GetName(), key, value) + if err := c.removeImportingLabel(namespace, item.GetName(), mediaID); err != nil { + logger.Error("Failed to clear stale pending import on VM %s/%s: %v", namespace, item.GetName(), err) + } + } + } + return nil +} + +var virtualMachinesGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + +// patchVMMetadata applies a merge patch that only touches labels or +// annotations. Such a patch is atomic on the API server and cannot clobber a +// concurrent spec write, so it needs no resourceVersion. +func (c *Client) patchVMMetadata(namespace, name, patchJSON string) error { + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + _, err := c.dynamicClient.Resource(virtualMachinesGVR).Namespace(namespace).Patch(ctx, name, types.MergePatchType, []byte(patchJSON), metav1.PatchOptions{}) + return err +} diff --git a/pkg/kubevirt/import_pending_test.go b/pkg/kubevirt/import_pending_test.go new file mode 100644 index 0000000..892e06b --- /dev/null +++ b/pkg/kubevirt/import_pending_test.go @@ -0,0 +1,245 @@ +package kubevirt + +import ( + "strings" + "testing" + "time" + + "k8s.io/client-go/kubernetes/fake" +) + +const ( + pendingTestURL = "http://example.com/boot.iso" + pendingTestLabel = ImportingLabelPrefix + "cdrom0" + pendingTestURLAn = VirtualMediaURLAnnotationPrefix + "cdrom0" +) + +func TestMarkImportPending_SetsPlaceholderLabelAndURL(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", nil, nil)) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + marked, err := client.MarkImportPending("test-ns", "test-vm", "cdrom0", pendingTestURL) + if err != nil { + t.Fatalf("MarkImportPending failed: %v", err) + } + if !marked { + t.Fatal("expected the VM to be marked") + } + + vm, _ := mockDynamic.GetVM("test-ns", "test-vm") + if got := vm.GetLabels()[pendingTestLabel]; !isPendingImport(got) { + t.Errorf("label %s = %q, want a %q placeholder", pendingTestLabel, got, ImportPendingPrefix) + } + if got := vm.GetAnnotations()[pendingTestURLAn]; got != pendingTestURL { + t.Errorf("annotation %s = %q, want %q", pendingTestURLAn, got, pendingTestURL) + } + if vm.GetLabels()["existing-label"] != "original-value" || vm.GetAnnotations()["existing-annotation"] != "original-value" { + t.Errorf("unrelated metadata was modified: labels=%v annotations=%v", vm.GetLabels(), vm.GetAnnotations()) + } +} + +func TestMarkImportPending_DoesNotOverwriteRunningImport(t *testing.T) { + for _, existing := range []string{"copy-iso-test-vm-bootiso-1", importPendingValue(time.Now())} { + t.Run(existing, func(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", + map[string]string{pendingTestLabel: existing}, + map[string]string{pendingTestURLAn: pendingTestURL})) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + marked, err := client.MarkImportPending("test-ns", "test-vm", "cdrom0", "http://example.com/other.iso") + if err != nil { + t.Fatalf("MarkImportPending failed: %v", err) + } + if marked { + t.Error("expected the existing import to be left alone") + } + vm, _ := mockDynamic.GetVM("test-ns", "test-vm") + if got := vm.GetLabels()[pendingTestLabel]; got != existing { + t.Errorf("label changed from %q to %q", existing, got) + } + if got := vm.GetAnnotations()[pendingTestURLAn]; got != pendingTestURL { + t.Errorf("annotation changed to %q", got) + } + }) + } +} + +func TestMarkImportPending_VMNotFound(t *testing.T) { + client := NewClientWithClients(fake.NewSimpleClientset(), NewMockDynamicClient(), 30*time.Second, nil) + if _, err := client.MarkImportPending("test-ns", "missing", "cdrom0", pendingTestURL); err == nil { + t.Fatal("expected an error for a missing VM") + } +} + +func TestClearPendingImport_RemovesOnlyThePlaceholder(t *testing.T) { + tests := []struct { + name string + value string + wantLabel string + }{ + {"placeholder is removed", importPendingValue(time.Now()), ""}, + {"helper pod name is kept", "copy-iso-test-vm-bootiso-1", "copy-iso-test-vm-bootiso-1"}, + {"CDI import is kept", CDIImportPrefix + "test-vm-bootiso-1", CDIImportPrefix + "test-vm-bootiso-1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", map[string]string{pendingTestLabel: tt.value}, nil)) + before, _ := mockDynamic.GetVM("test-ns", "test-vm") + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + if err := client.ClearPendingImport("test-ns", "test-vm", "cdrom0"); err != nil { + t.Fatalf("ClearPendingImport failed: %v", err) + } + + after, _ := mockDynamic.GetVM("test-ns", "test-vm") + if got := after.GetLabels()[pendingTestLabel]; got != tt.wantLabel { + t.Errorf("label = %q, want %q", got, tt.wantLabel) + } + if after.GetLabels()["existing-label"] != "original-value" { + t.Errorf("unrelated label was modified: %v", after.GetLabels()) + } + assertVMUnchangedExceptLabels(t, before, after) + }) + } +} + +func TestClearPendingImport_NoLabelIsNoop(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", nil, nil)) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + if err := client.ClearPendingImport("test-ns", "test-vm", "cdrom0"); err != nil { + t.Fatalf("ClearPendingImport failed: %v", err) + } +} + +func TestPendingImportValue(t *testing.T) { + now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) + value := importPendingValue(now) + if !strings.HasPrefix(value, ImportPendingPrefix) { + t.Fatalf("value %q lacks the %q prefix", value, ImportPendingPrefix) + } + if !isPendingImport(value) || isPendingImport("copy-iso-vm-1") || isPendingImport("") { + t.Error("isPendingImport misclassifies values") + } + + if pendingImportIsStale(value, now.Add(stalePendingImportAge-time.Second)) { + t.Error("a placeholder younger than the limit must not be stale") + } + if !pendingImportIsStale(value, now.Add(stalePendingImportAge+time.Second)) { + t.Error("a placeholder older than the limit must be stale") + } + if !pendingImportIsStale(ImportPendingPrefix+"garbage", now) { + t.Error("an unparsable placeholder must be treated as stale") + } + if pendingImportIsStale("copy-iso-vm-1", now.Add(24*time.Hour)) { + t.Error("a helper pod name is never a stale placeholder") + } +} + +// A placeholder whose job was lost (the queue lives in memory, so a restart +// drops it) would defer power actions for that VM forever. The periodic sweep +// removes placeholders older than stalePendingImportAge and nothing else. +func TestReconcileStalePendingImports(t *testing.T) { + now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) + mockDynamic := NewMockDynamicClient() + fresh := importPendingValue(now.Add(-time.Minute)) + stale := importPendingValue(now.Add(-stalePendingImportAge - time.Minute)) + mockDynamic.AddVM(newTestVM("test-ns", "fresh", map[string]string{pendingTestLabel: fresh}, nil)) + mockDynamic.AddVM(newTestVM("test-ns", "stale", map[string]string{pendingTestLabel: stale}, nil)) + mockDynamic.AddVM(newTestVM("test-ns", "importing", map[string]string{pendingTestLabel: "copy-iso-importing-1"}, nil)) + mockDynamic.AddVM(newTestVM("test-ns", "two-devices", map[string]string{ + pendingTestLabel: stale, + ImportingLabelPrefix + "cdrom1": "copy-iso-two-devices-1", + }, nil)) + mockDynamic.AddVM(newTestVM("other-ns", "stale", map[string]string{pendingTestLabel: stale}, nil)) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + if err := client.reconcileStalePendingImports("test-ns", now); err != nil { + t.Fatalf("reconcileStalePendingImports failed: %v", err) + } + + want := map[string]map[string]string{ + "test-ns/fresh": {pendingTestLabel: fresh}, + "test-ns/stale": {pendingTestLabel: ""}, + "test-ns/importing": {pendingTestLabel: "copy-iso-importing-1"}, + "test-ns/two-devices": {pendingTestLabel: "", ImportingLabelPrefix + "cdrom1": "copy-iso-two-devices-1"}, + "other-ns/stale": {pendingTestLabel: stale}, + } + for key, labels := range want { + ns, name, _ := strings.Cut(key, "/") + vm, err := mockDynamic.GetVM(ns, name) + if err != nil { + t.Fatalf("VM %s: %v", key, err) + } + for k, v := range labels { + if got := vm.GetLabels()[k]; got != v { + t.Errorf("VM %s: label %s = %q, want %q", key, k, got, v) + } + } + } +} + +// Ironic retries InsertMedia if the first answer is lost. While the placeholder +// is on the VM there is no PVC yet, and the retry must not queue a second job. +func TestCheckInsertedMedia_PendingWithoutPVC(t *testing.T) { + tests := []struct { + name string + url string + want MediaState + }{ + {"same URL is importing", pendingTestURL, MediaStateImporting}, + {"different URL conflicts", "http://example.com/other.iso", MediaStateConflict}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", + map[string]string{pendingTestLabel: importPendingValue(time.Now())}, + map[string]string{pendingTestURLAn: pendingTestURL})) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + state, err := client.CheckInsertedMedia("test-ns", "test-vm", "cdrom0", tt.url) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != tt.want { + t.Errorf("state = %d, want %d", state, tt.want) + } + }) + } +} + +// A cleared placeholder leaves the URL annotation behind; with no PVC and no +// label that must read as "nothing inserted" so a new insert is accepted. +func TestCheckInsertedMedia_StaleURLAnnotationAlone(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", nil, map[string]string{pendingTestURLAn: pendingTestURL})) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + state, err := client.CheckInsertedMedia("test-ns", "test-vm", "cdrom0", "http://example.com/other.iso") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != MediaStateNone { + t.Errorf("state = %d, want MediaStateNone", state) + } +} + +// The insert job replaces the placeholder with the helper pod name, so the +// pod watcher's "is this pod still the tracked import" check keeps working. +func TestSetImportingLabel_ReplacesPlaceholder(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", map[string]string{pendingTestLabel: importPendingValue(time.Now())}, nil)) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + if err := client.setImportingLabel("test-ns", "test-vm", "cdrom0", "copy-iso-test-vm-bootiso-1"); err != nil { + t.Fatalf("setImportingLabel failed: %v", err) + } + vm, _ := mockDynamic.GetVM("test-ns", "test-vm") + if got := vm.GetLabels()[pendingTestLabel]; got != "copy-iso-test-vm-bootiso-1" { + t.Errorf("label = %q, want the pod name", got) + } +} diff --git a/pkg/server/insert_pending_test.go b/pkg/server/insert_pending_test.go new file mode 100644 index 0000000..1815b46 --- /dev/null +++ b/pkg/server/insert_pending_test.go @@ -0,0 +1,212 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/kubevirt/redfish-controller/pkg/auth" + "github.com/kubevirt/redfish-controller/pkg/config" + "github.com/kubevirt/redfish-controller/pkg/kubevirt" + "github.com/kubevirt/redfish-controller/pkg/logger" + "github.com/kubevirt/redfish-controller/pkg/redfish" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + kubevirtv1 "kubevirt.io/api/core/v1" +) + +const ( + pendingTestNS = "ns-a" + pendingTestVM = "vm-a" + pendingTestURL = "http://example.com/boot.iso" + pendingTestLabel = kubevirt.ImportingLabelPrefix + "cdrom0" +) + +// pendingTestServer builds a server whose task manager has no workers: queued +// jobs never run, which is the state Ironic's Power On meets in production +// when it arrives right after InsertMedia. +func pendingTestServer(t *testing.T) (*Server, *kubevirt.MockDynamicClient) { + t.Helper() + mockDynamic := kubevirt.NewMockDynamicClient() + halted := kubevirtv1.RunStrategyHalted + if err := mockDynamic.AddVM(&kubevirtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: pendingTestVM, + Namespace: pendingTestNS, + Labels: map[string]string{"redfish-enabled": "true"}, + }, + Spec: kubevirtv1.VirtualMachineSpec{ + RunStrategy: &halted, + Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Domain: kubevirtv1.DomainSpec{ + Devices: kubevirtv1.Devices{ + Disks: []kubevirtv1.Disk{{Name: "rootdisk"}}, + }, + }, + Volumes: []kubevirtv1.Volume{{Name: "rootdisk"}}, + }, + }, + }, + }); err != nil { + t.Fatalf("AddVM failed: %v", err) + } + client := kubevirt.NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + cfg := &config.Config{ + Server: config.ServerConfig{Host: "localhost", Port: 8080, TestMode: true}, + Auth: config.AuthConfig{ + Users: []config.UserConfig{ + {Username: "user", Password: config.PasswordConfig{Plain: "pass"}, Chassis: []string{"chassis-a"}}, + }, + }, + Chassis: []config.ChassisConfig{ + { + Name: "chassis-a", + Namespace: pendingTestNS, + VMSelector: &kubevirt.VMSelectorConfig{Labels: map[string]string{"redfish-enabled": "true"}}, + }, + }, + } + server := NewServer(cfg, client) + server.taskManager = NewTaskManager(0, client) + t.Cleanup(server.taskManager.Stop) + return server, mockDynamic +} + +func pendingTestRequest(method, path, body string) *http.Request { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + authCtx := &auth.AuthContext{User: &auth.User{Username: "user", Chassis: []string{"chassis-a"}}} + return req.WithContext(logger.WithAuth(req.Context(), authCtx)) +} + +func insertMedia(server *Server, url string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + req := pendingTestRequest("POST", "/redfish/v1/Systems/"+pendingTestVM+"/VirtualMedia/Cd/Actions/VirtualMedia.InsertMedia", + `{"Image":"`+url+`","Inserted":true,"WriteProtected":true}`) + server.handleInsertVirtualMedia(w, req, pendingTestVM, "Cd") + return w +} + +func powerAction(server *Server, resetType string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + req := pendingTestRequest("POST", "/redfish/v1/Systems/"+pendingTestVM+"/Actions/ComputerSystem.Reset", + `{"ResetType":"`+resetType+`"}`) + server.handlePowerAction(w, req, pendingTestVM) + return w +} + +func taskCount(server *Server) int { + server.taskManager.taskMutex.RLock() + defer server.taskManager.taskMutex.RUnlock() + return len(server.taskManager.tasks) +} + +// The insert job is what creates the import pod and labels the VM, and it runs +// some time after the 202 is sent. Power On arriving in between must already +// find the VM marked, otherwise it starts the VM on an empty CD-ROM. +func TestHandleInsertVirtualMedia_MarksVMBeforeAnswering(t *testing.T) { + server, mockDynamic := pendingTestServer(t) + + w := insertMedia(server, pendingTestURL) + if w.Code != http.StatusAccepted { + t.Fatalf("InsertMedia status = %d, body %s", w.Code, w.Body.String()) + } + if n := taskCount(server); n != 1 { + t.Fatalf("tasks = %d, want 1", n) + } + + vm, _ := mockDynamic.GetVM(pendingTestNS, pendingTestVM) + if got := vm.GetLabels()[pendingTestLabel]; !strings.HasPrefix(got, kubevirt.ImportPendingPrefix) { + t.Fatalf("label %s = %q after InsertMedia, want a %q placeholder", pendingTestLabel, got, kubevirt.ImportPendingPrefix) + } + if got := vm.GetAnnotations()[kubevirt.VirtualMediaURLAnnotationPrefix+"cdrom0"]; got != pendingTestURL { + t.Errorf("URL annotation = %q, want %q", got, pendingTestURL) + } + + w = powerAction(server, "On") + if w.Code != http.StatusAccepted { + t.Fatalf("Power On status = %d, want 202 (deferred); body %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "deferred") { + t.Errorf("Power On body does not say deferred: %s", w.Body.String()) + } + vm, _ = mockDynamic.GetVM(pendingTestNS, pendingTestVM) + if got := vm.GetLabels()[kubevirt.PowerAfterImportLabel]; got != "On" { + t.Errorf("label %s = %q, want On", kubevirt.PowerAfterImportLabel, got) + } + if got := vm.Spec.RunStrategy; got == nil || *got != kubevirtv1.RunStrategyHalted { + t.Errorf("VM was started while its import was pending: runStrategy = %v", got) + } +} + +// Ironic retries a lost InsertMedia. While the placeholder is on the VM the +// retry is answered as "in progress" and no second job is queued; a different +// image is a conflict. +func TestHandleInsertVirtualMedia_RepeatWhilePending(t *testing.T) { + server, _ := pendingTestServer(t) + if w := insertMedia(server, pendingTestURL); w.Code != http.StatusAccepted { + t.Fatalf("first InsertMedia status = %d", w.Code) + } + + w := insertMedia(server, pendingTestURL) + if w.Code != http.StatusAccepted { + t.Errorf("repeated InsertMedia status = %d, want 202", w.Code) + } + var body map[string]string + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || !strings.Contains(body["Message"], "in progress") { + t.Errorf("repeated InsertMedia body = %s, want an in-progress message", w.Body.String()) + } + if n := taskCount(server); n != 1 { + t.Errorf("tasks = %d after the repeat, want 1", n) + } + + w = insertMedia(server, "http://example.com/other.iso") + if w.Code != http.StatusConflict { + t.Errorf("InsertMedia with another image status = %d, want 409", w.Code) + } + if n := taskCount(server); n != 1 { + t.Errorf("tasks = %d after the conflict, want 1", n) + } +} + +// When the job gives up, the placeholder must go, or every later power action +// on that VM would be deferred forever. +func TestWorker_FailedInsertJobClearsPlaceholder(t *testing.T) { + server, mockDynamic := pendingTestServer(t) + tm := server.taskManager + if _, err := server.kubevirtClient.MarkImportPending(pendingTestNS, pendingTestVM, "cdrom0", "://not-a-url"); err != nil { + t.Fatalf("MarkImportPending failed: %v", err) + } + + // The task is created the way the handler does it; its job is then run + // directly, with no retries, on a worker that was never started. The URL + // cannot be parsed, so the job fails before it changes the VM or creates + // a pod. + taskID := tm.CreateTask("insert", pendingTestNS, pendingTestVM, "cdrom0", "://not-a-url") + job := &Job{ + ID: "job-1", + TaskID: taskID, + Type: TaskTypeVirtualMediaInsert, + Payload: map[string]string{ + "namespace": pendingTestNS, + "vmName": pendingTestVM, + "mediaID": "cdrom0", + "imageURL": "://not-a-url", + }, + } + NewWorker(1, tm).processJob(job) + + task, ok := tm.GetTask(taskID) + if !ok || task.TaskState != redfish.TaskStateException { + t.Fatalf("task state = %v, want Exception", task) + } + vm, _ := mockDynamic.GetVM(pendingTestNS, pendingTestVM) + if got, ok := vm.GetLabels()[pendingTestLabel]; ok { + t.Errorf("label %s = %q is still on the VM after the job failed", pendingTestLabel, got) + } +} diff --git a/pkg/server/server.go b/pkg/server/server.go index df224ce..959d059 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -1026,6 +1026,24 @@ func (s *Server) handleInsertVirtualMedia(w http.ResponseWriter, r *http.Request )) return } + + // Ironic sends Power On right after this request is answered, and the + // queued job only labels the VM once it has created the import pod. + // Mark the VM now so that power-on is deferred until the import is done. + marked, err := s.kubevirtClient.MarkImportPending(namespace, vmName, internalMediaID, insertRequest.Image) + if err != nil { + logger.Error("Failed to mark pending import for VM %s/%s: %v", namespace, vmName, err) + s.sendInternalError(w, "Failed to record virtual media insertion") + return + } + if !marked { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + s.encodeJSONResponse(w, map[string]string{ + "Message": "Virtual media import is still in progress", + }) + return + } } // Create a task for this operation diff --git a/pkg/server/task_manager.go b/pkg/server/task_manager.go index 026248c..e619ba5 100644 --- a/pkg/server/task_manager.go +++ b/pkg/server/task_manager.go @@ -277,6 +277,7 @@ func (w *Worker) processJob(job *Job) { if taskErr := w.taskMgr.FailTask(job.TaskID, fmt.Sprintf("Job failed after %d retries: %v", job.MaxRetries, err)); taskErr != nil { logger.Error("Failed to mark task %s as failed: %v", job.TaskID, taskErr) } + w.clearPendingImport(job) } } else { logger.Info("Worker %d completed job %s successfully in %v", w.ID, job.ID, duration) @@ -287,6 +288,22 @@ func (w *Worker) processJob(job *Job) { } } +// clearPendingImport drops the placeholder the InsertMedia handler put on the +// VM when the insert job gives up before creating an import pod; the pod +// watcher owns the label from that point on and the placeholder is gone anyway. +func (w *Worker) clearPendingImport(job *Job) { + if job.Type != TaskTypeVirtualMediaInsert || w.taskMgr.kubevirtClient == nil { + return + } + payload, ok := job.Payload.(map[string]string) + if !ok { + return + } + if err := w.taskMgr.kubevirtClient.ClearPendingImport(payload["namespace"], payload["vmName"], payload["mediaID"]); err != nil { + logger.Error("Failed to clear pending import for VM %s/%s after job %s failed: %v", payload["namespace"], payload["vmName"], job.ID, err) + } +} + // processVirtualMediaInsert processes a virtual media insertion job func (w *Worker) processVirtualMediaInsert(job *Job) error { logger.Debug("Worker %d starting virtual media insert job %s (task %s)", w.ID, job.ID, job.TaskID)