diff --git a/pkg/kubevirt/client.go b/pkg/kubevirt/client.go index 26ad6c2..8055b17 100644 --- a/pkg/kubevirt/client.go +++ b/pkg/kubevirt/client.go @@ -1658,11 +1658,15 @@ func (c *Client) CheckInsertedMedia(namespace, name, mediaID, imageURL string) ( } } } - if !hasPVC { + // An accepted insert whose job has not run yet has the importing label + // (a placeholder) but no PVC; it counts as inserted so a retried request + // does not queue a second job. + importing := vm.GetLabels()[ImportingLabelPrefix+mediaID] != "" + if !hasPVC && !importing { return MediaStateNone, nil } - // A PVC is attached. Compare the stored URL. + // Media is attached or on its way. Compare the stored URL. annotationKey := VirtualMediaURLAnnotationPrefix + mediaID storedURL := vm.GetAnnotations()[annotationKey] @@ -1670,9 +1674,7 @@ func (c *Client) CheckInsertedMedia(namespace, name, mediaID, imageURL string) ( return MediaStateConflict, nil } - // Same URL — check whether import is still in progress. - labelKey := ImportingLabelPrefix + mediaID - if vm.GetLabels()[labelKey] != "" { + if importing { return MediaStateImporting, nil } @@ -3680,6 +3682,10 @@ func (c *Client) removeImportingLabel(namespace, vmName, deviceName string) erro func (c *Client) reconcileExistingImportPods(namespace string) error { logger.Info("Reconciling existing import pods in namespace %s", namespace) + if err := c.reconcileStalePendingImports(namespace, time.Now()); err != nil { + logger.Error("Failed to reconcile pending imports in namespace %s: %v", namespace, err) + } + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) defer cancel() diff --git a/pkg/kubevirt/import_pending.go b/pkg/kubevirt/import_pending.go new file mode 100644 index 0000000..6ac7757 --- /dev/null +++ b/pkg/kubevirt/import_pending.go @@ -0,0 +1,131 @@ +package kubevirt + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/kubevirt/redfish-controller/pkg/logger" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" +) + +// InsertMedia is answered with 202 before its job runs, and the job labels the +// VM with the import pod only after creating it. Ironic sends Power On right +// after InsertMedia, and power-on is deferred only while the importing label +// is on the VM. Without a marker written before the 202, Power On lands in +// that gap, the VM starts with an empty CD-ROM and takes the RWO ISO volume +// away from the import pod. The handler therefore writes a placeholder value +// into the importing label up front; the job later replaces it with the pod +// name, so the pod watcher's ownership check is unchanged. +// +// The placeholder carries its creation time. The job queue is in memory, so a +// restart loses queued jobs, and a job can also fail before it ever creates a +// pod: a placeholder nobody replaces would defer power actions forever. The +// job's terminal failure clears it, and the periodic reconcile sweeps +// placeholders older than stalePendingImportAge. +const ImportPendingPrefix = "pending-" + +const stalePendingImportAge = 5 * time.Minute + +func importPendingValue(now time.Time) string { + return ImportPendingPrefix + strconv.FormatInt(now.Unix(), 10) +} + +func isPendingImport(value string) bool { + return strings.HasPrefix(value, ImportPendingPrefix) +} + +// pendingImportIsStale reports whether a placeholder is old enough that its job +// cannot still be queued. A placeholder that cannot be parsed counts as stale. +func pendingImportIsStale(value string, now time.Time) bool { + if !isPendingImport(value) { + return false + } + created, err := strconv.ParseInt(strings.TrimPrefix(value, ImportPendingPrefix), 10, 64) + if err != nil { + return true + } + return now.Sub(time.Unix(created, 0)) > stalePendingImportAge +} + +// MarkImportPending records on the VM that an import of imageURL into mediaID +// has been accepted but not started. It returns false, without touching the +// VM, when an import is already recorded for the device. +func (c *Client) MarkImportPending(namespace, name, mediaID, imageURL string) (bool, error) { + vm, err := c.GetVM(namespace, name) + if err != nil { + return false, fmt.Errorf("failed to get VM %s/%s: %w", namespace, name, err) + } + labelKey := ImportingLabelPrefix + mediaID + if vm.GetLabels()[labelKey] != "" { + return false, nil + } + + patchJSON := fmt.Sprintf(`{"metadata":{"labels":{%q:%q},"annotations":{%q:%q}}}`, + labelKey, importPendingValue(time.Now()), VirtualMediaURLAnnotationPrefix+mediaID, imageURL) + if err := c.patchVMMetadata(namespace, name, patchJSON); err != nil { + return false, fmt.Errorf("failed to mark import pending on VM %s/%s: %w", namespace, name, err) + } + logger.Info("Marked import of %s as pending on VM %s/%s", mediaID, namespace, name) + return true, nil +} + +// ClearPendingImport removes the placeholder for mediaID. A label that already +// names an import pod or a CDI import is owned by the watchers and is left alone. +func (c *Client) ClearPendingImport(namespace, name, mediaID string) error { + vm, err := c.GetVM(namespace, name) + if err != nil { + return fmt.Errorf("failed to get VM %s/%s: %w", namespace, name, err) + } + labelKey := ImportingLabelPrefix + mediaID + if !isPendingImport(vm.GetLabels()[labelKey]) { + return nil + } + if err := c.removeImportingLabel(namespace, name, mediaID); err != nil { + return err + } + logger.Info("Cleared pending import of %s from VM %s/%s", mediaID, namespace, name) + return nil +} + +// reconcileStalePendingImports removes placeholders whose job can no longer be +// waiting in the queue (see ImportPendingPrefix). +func (c *Client) reconcileStalePendingImports(namespace string, now time.Time) error { + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + + vmList, err := c.dynamicClient.Resource(virtualMachinesGVR).Namespace(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("failed to list VMs in namespace %s: %w", namespace, err) + } + + for _, item := range vmList.Items { + for key, value := range item.GetLabels() { + if !strings.HasPrefix(key, ImportingLabelPrefix) || !pendingImportIsStale(value, now) { + continue + } + mediaID := strings.TrimPrefix(key, ImportingLabelPrefix) + logger.Warning("Import of %s on VM %s/%s was accepted but never started (label %s=%s), clearing it", mediaID, namespace, item.GetName(), key, value) + if err := c.removeImportingLabel(namespace, item.GetName(), mediaID); err != nil { + logger.Error("Failed to clear stale pending import on VM %s/%s: %v", namespace, item.GetName(), err) + } + } + } + return nil +} + +var virtualMachinesGVR = schema.GroupVersionResource{Group: "kubevirt.io", Version: "v1", Resource: "virtualmachines"} + +// patchVMMetadata applies a merge patch that only touches labels or +// annotations. Such a patch is atomic on the API server and cannot clobber a +// concurrent spec write, so it needs no resourceVersion. +func (c *Client) patchVMMetadata(namespace, name, patchJSON string) error { + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + _, err := c.dynamicClient.Resource(virtualMachinesGVR).Namespace(namespace).Patch(ctx, name, types.MergePatchType, []byte(patchJSON), metav1.PatchOptions{}) + return err +} diff --git a/pkg/kubevirt/import_pending_test.go b/pkg/kubevirt/import_pending_test.go new file mode 100644 index 0000000..892e06b --- /dev/null +++ b/pkg/kubevirt/import_pending_test.go @@ -0,0 +1,245 @@ +package kubevirt + +import ( + "strings" + "testing" + "time" + + "k8s.io/client-go/kubernetes/fake" +) + +const ( + pendingTestURL = "http://example.com/boot.iso" + pendingTestLabel = ImportingLabelPrefix + "cdrom0" + pendingTestURLAn = VirtualMediaURLAnnotationPrefix + "cdrom0" +) + +func TestMarkImportPending_SetsPlaceholderLabelAndURL(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", nil, nil)) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + marked, err := client.MarkImportPending("test-ns", "test-vm", "cdrom0", pendingTestURL) + if err != nil { + t.Fatalf("MarkImportPending failed: %v", err) + } + if !marked { + t.Fatal("expected the VM to be marked") + } + + vm, _ := mockDynamic.GetVM("test-ns", "test-vm") + if got := vm.GetLabels()[pendingTestLabel]; !isPendingImport(got) { + t.Errorf("label %s = %q, want a %q placeholder", pendingTestLabel, got, ImportPendingPrefix) + } + if got := vm.GetAnnotations()[pendingTestURLAn]; got != pendingTestURL { + t.Errorf("annotation %s = %q, want %q", pendingTestURLAn, got, pendingTestURL) + } + if vm.GetLabels()["existing-label"] != "original-value" || vm.GetAnnotations()["existing-annotation"] != "original-value" { + t.Errorf("unrelated metadata was modified: labels=%v annotations=%v", vm.GetLabels(), vm.GetAnnotations()) + } +} + +func TestMarkImportPending_DoesNotOverwriteRunningImport(t *testing.T) { + for _, existing := range []string{"copy-iso-test-vm-bootiso-1", importPendingValue(time.Now())} { + t.Run(existing, func(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", + map[string]string{pendingTestLabel: existing}, + map[string]string{pendingTestURLAn: pendingTestURL})) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + marked, err := client.MarkImportPending("test-ns", "test-vm", "cdrom0", "http://example.com/other.iso") + if err != nil { + t.Fatalf("MarkImportPending failed: %v", err) + } + if marked { + t.Error("expected the existing import to be left alone") + } + vm, _ := mockDynamic.GetVM("test-ns", "test-vm") + if got := vm.GetLabels()[pendingTestLabel]; got != existing { + t.Errorf("label changed from %q to %q", existing, got) + } + if got := vm.GetAnnotations()[pendingTestURLAn]; got != pendingTestURL { + t.Errorf("annotation changed to %q", got) + } + }) + } +} + +func TestMarkImportPending_VMNotFound(t *testing.T) { + client := NewClientWithClients(fake.NewSimpleClientset(), NewMockDynamicClient(), 30*time.Second, nil) + if _, err := client.MarkImportPending("test-ns", "missing", "cdrom0", pendingTestURL); err == nil { + t.Fatal("expected an error for a missing VM") + } +} + +func TestClearPendingImport_RemovesOnlyThePlaceholder(t *testing.T) { + tests := []struct { + name string + value string + wantLabel string + }{ + {"placeholder is removed", importPendingValue(time.Now()), ""}, + {"helper pod name is kept", "copy-iso-test-vm-bootiso-1", "copy-iso-test-vm-bootiso-1"}, + {"CDI import is kept", CDIImportPrefix + "test-vm-bootiso-1", CDIImportPrefix + "test-vm-bootiso-1"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", map[string]string{pendingTestLabel: tt.value}, nil)) + before, _ := mockDynamic.GetVM("test-ns", "test-vm") + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + if err := client.ClearPendingImport("test-ns", "test-vm", "cdrom0"); err != nil { + t.Fatalf("ClearPendingImport failed: %v", err) + } + + after, _ := mockDynamic.GetVM("test-ns", "test-vm") + if got := after.GetLabels()[pendingTestLabel]; got != tt.wantLabel { + t.Errorf("label = %q, want %q", got, tt.wantLabel) + } + if after.GetLabels()["existing-label"] != "original-value" { + t.Errorf("unrelated label was modified: %v", after.GetLabels()) + } + assertVMUnchangedExceptLabels(t, before, after) + }) + } +} + +func TestClearPendingImport_NoLabelIsNoop(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", nil, nil)) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + if err := client.ClearPendingImport("test-ns", "test-vm", "cdrom0"); err != nil { + t.Fatalf("ClearPendingImport failed: %v", err) + } +} + +func TestPendingImportValue(t *testing.T) { + now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) + value := importPendingValue(now) + if !strings.HasPrefix(value, ImportPendingPrefix) { + t.Fatalf("value %q lacks the %q prefix", value, ImportPendingPrefix) + } + if !isPendingImport(value) || isPendingImport("copy-iso-vm-1") || isPendingImport("") { + t.Error("isPendingImport misclassifies values") + } + + if pendingImportIsStale(value, now.Add(stalePendingImportAge-time.Second)) { + t.Error("a placeholder younger than the limit must not be stale") + } + if !pendingImportIsStale(value, now.Add(stalePendingImportAge+time.Second)) { + t.Error("a placeholder older than the limit must be stale") + } + if !pendingImportIsStale(ImportPendingPrefix+"garbage", now) { + t.Error("an unparsable placeholder must be treated as stale") + } + if pendingImportIsStale("copy-iso-vm-1", now.Add(24*time.Hour)) { + t.Error("a helper pod name is never a stale placeholder") + } +} + +// A placeholder whose job was lost (the queue lives in memory, so a restart +// drops it) would defer power actions for that VM forever. The periodic sweep +// removes placeholders older than stalePendingImportAge and nothing else. +func TestReconcileStalePendingImports(t *testing.T) { + now := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC) + mockDynamic := NewMockDynamicClient() + fresh := importPendingValue(now.Add(-time.Minute)) + stale := importPendingValue(now.Add(-stalePendingImportAge - time.Minute)) + mockDynamic.AddVM(newTestVM("test-ns", "fresh", map[string]string{pendingTestLabel: fresh}, nil)) + mockDynamic.AddVM(newTestVM("test-ns", "stale", map[string]string{pendingTestLabel: stale}, nil)) + mockDynamic.AddVM(newTestVM("test-ns", "importing", map[string]string{pendingTestLabel: "copy-iso-importing-1"}, nil)) + mockDynamic.AddVM(newTestVM("test-ns", "two-devices", map[string]string{ + pendingTestLabel: stale, + ImportingLabelPrefix + "cdrom1": "copy-iso-two-devices-1", + }, nil)) + mockDynamic.AddVM(newTestVM("other-ns", "stale", map[string]string{pendingTestLabel: stale}, nil)) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + if err := client.reconcileStalePendingImports("test-ns", now); err != nil { + t.Fatalf("reconcileStalePendingImports failed: %v", err) + } + + want := map[string]map[string]string{ + "test-ns/fresh": {pendingTestLabel: fresh}, + "test-ns/stale": {pendingTestLabel: ""}, + "test-ns/importing": {pendingTestLabel: "copy-iso-importing-1"}, + "test-ns/two-devices": {pendingTestLabel: "", ImportingLabelPrefix + "cdrom1": "copy-iso-two-devices-1"}, + "other-ns/stale": {pendingTestLabel: stale}, + } + for key, labels := range want { + ns, name, _ := strings.Cut(key, "/") + vm, err := mockDynamic.GetVM(ns, name) + if err != nil { + t.Fatalf("VM %s: %v", key, err) + } + for k, v := range labels { + if got := vm.GetLabels()[k]; got != v { + t.Errorf("VM %s: label %s = %q, want %q", key, k, got, v) + } + } + } +} + +// Ironic retries InsertMedia if the first answer is lost. While the placeholder +// is on the VM there is no PVC yet, and the retry must not queue a second job. +func TestCheckInsertedMedia_PendingWithoutPVC(t *testing.T) { + tests := []struct { + name string + url string + want MediaState + }{ + {"same URL is importing", pendingTestURL, MediaStateImporting}, + {"different URL conflicts", "http://example.com/other.iso", MediaStateConflict}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", + map[string]string{pendingTestLabel: importPendingValue(time.Now())}, + map[string]string{pendingTestURLAn: pendingTestURL})) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + state, err := client.CheckInsertedMedia("test-ns", "test-vm", "cdrom0", tt.url) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != tt.want { + t.Errorf("state = %d, want %d", state, tt.want) + } + }) + } +} + +// A cleared placeholder leaves the URL annotation behind; with no PVC and no +// label that must read as "nothing inserted" so a new insert is accepted. +func TestCheckInsertedMedia_StaleURLAnnotationAlone(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", nil, map[string]string{pendingTestURLAn: pendingTestURL})) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + state, err := client.CheckInsertedMedia("test-ns", "test-vm", "cdrom0", "http://example.com/other.iso") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if state != MediaStateNone { + t.Errorf("state = %d, want MediaStateNone", state) + } +} + +// The insert job replaces the placeholder with the helper pod name, so the +// pod watcher's "is this pod still the tracked import" check keeps working. +func TestSetImportingLabel_ReplacesPlaceholder(t *testing.T) { + mockDynamic := NewMockDynamicClient() + mockDynamic.AddVM(newTestVM("test-ns", "test-vm", map[string]string{pendingTestLabel: importPendingValue(time.Now())}, nil)) + client := NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + if err := client.setImportingLabel("test-ns", "test-vm", "cdrom0", "copy-iso-test-vm-bootiso-1"); err != nil { + t.Fatalf("setImportingLabel failed: %v", err) + } + vm, _ := mockDynamic.GetVM("test-ns", "test-vm") + if got := vm.GetLabels()[pendingTestLabel]; got != "copy-iso-test-vm-bootiso-1" { + t.Errorf("label = %q, want the pod name", got) + } +} diff --git a/pkg/server/insert_pending_test.go b/pkg/server/insert_pending_test.go new file mode 100644 index 0000000..1815b46 --- /dev/null +++ b/pkg/server/insert_pending_test.go @@ -0,0 +1,212 @@ +package server + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/kubevirt/redfish-controller/pkg/auth" + "github.com/kubevirt/redfish-controller/pkg/config" + "github.com/kubevirt/redfish-controller/pkg/kubevirt" + "github.com/kubevirt/redfish-controller/pkg/logger" + "github.com/kubevirt/redfish-controller/pkg/redfish" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + kubevirtv1 "kubevirt.io/api/core/v1" +) + +const ( + pendingTestNS = "ns-a" + pendingTestVM = "vm-a" + pendingTestURL = "http://example.com/boot.iso" + pendingTestLabel = kubevirt.ImportingLabelPrefix + "cdrom0" +) + +// pendingTestServer builds a server whose task manager has no workers: queued +// jobs never run, which is the state Ironic's Power On meets in production +// when it arrives right after InsertMedia. +func pendingTestServer(t *testing.T) (*Server, *kubevirt.MockDynamicClient) { + t.Helper() + mockDynamic := kubevirt.NewMockDynamicClient() + halted := kubevirtv1.RunStrategyHalted + if err := mockDynamic.AddVM(&kubevirtv1.VirtualMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: pendingTestVM, + Namespace: pendingTestNS, + Labels: map[string]string{"redfish-enabled": "true"}, + }, + Spec: kubevirtv1.VirtualMachineSpec{ + RunStrategy: &halted, + Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ + Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Domain: kubevirtv1.DomainSpec{ + Devices: kubevirtv1.Devices{ + Disks: []kubevirtv1.Disk{{Name: "rootdisk"}}, + }, + }, + Volumes: []kubevirtv1.Volume{{Name: "rootdisk"}}, + }, + }, + }, + }); err != nil { + t.Fatalf("AddVM failed: %v", err) + } + client := kubevirt.NewClientWithClients(fake.NewSimpleClientset(), mockDynamic, 30*time.Second, nil) + + cfg := &config.Config{ + Server: config.ServerConfig{Host: "localhost", Port: 8080, TestMode: true}, + Auth: config.AuthConfig{ + Users: []config.UserConfig{ + {Username: "user", Password: config.PasswordConfig{Plain: "pass"}, Chassis: []string{"chassis-a"}}, + }, + }, + Chassis: []config.ChassisConfig{ + { + Name: "chassis-a", + Namespace: pendingTestNS, + VMSelector: &kubevirt.VMSelectorConfig{Labels: map[string]string{"redfish-enabled": "true"}}, + }, + }, + } + server := NewServer(cfg, client) + server.taskManager = NewTaskManager(0, client) + t.Cleanup(server.taskManager.Stop) + return server, mockDynamic +} + +func pendingTestRequest(method, path, body string) *http.Request { + req := httptest.NewRequest(method, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + authCtx := &auth.AuthContext{User: &auth.User{Username: "user", Chassis: []string{"chassis-a"}}} + return req.WithContext(logger.WithAuth(req.Context(), authCtx)) +} + +func insertMedia(server *Server, url string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + req := pendingTestRequest("POST", "/redfish/v1/Systems/"+pendingTestVM+"/VirtualMedia/Cd/Actions/VirtualMedia.InsertMedia", + `{"Image":"`+url+`","Inserted":true,"WriteProtected":true}`) + server.handleInsertVirtualMedia(w, req, pendingTestVM, "Cd") + return w +} + +func powerAction(server *Server, resetType string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + req := pendingTestRequest("POST", "/redfish/v1/Systems/"+pendingTestVM+"/Actions/ComputerSystem.Reset", + `{"ResetType":"`+resetType+`"}`) + server.handlePowerAction(w, req, pendingTestVM) + return w +} + +func taskCount(server *Server) int { + server.taskManager.taskMutex.RLock() + defer server.taskManager.taskMutex.RUnlock() + return len(server.taskManager.tasks) +} + +// The insert job is what creates the import pod and labels the VM, and it runs +// some time after the 202 is sent. Power On arriving in between must already +// find the VM marked, otherwise it starts the VM on an empty CD-ROM. +func TestHandleInsertVirtualMedia_MarksVMBeforeAnswering(t *testing.T) { + server, mockDynamic := pendingTestServer(t) + + w := insertMedia(server, pendingTestURL) + if w.Code != http.StatusAccepted { + t.Fatalf("InsertMedia status = %d, body %s", w.Code, w.Body.String()) + } + if n := taskCount(server); n != 1 { + t.Fatalf("tasks = %d, want 1", n) + } + + vm, _ := mockDynamic.GetVM(pendingTestNS, pendingTestVM) + if got := vm.GetLabels()[pendingTestLabel]; !strings.HasPrefix(got, kubevirt.ImportPendingPrefix) { + t.Fatalf("label %s = %q after InsertMedia, want a %q placeholder", pendingTestLabel, got, kubevirt.ImportPendingPrefix) + } + if got := vm.GetAnnotations()[kubevirt.VirtualMediaURLAnnotationPrefix+"cdrom0"]; got != pendingTestURL { + t.Errorf("URL annotation = %q, want %q", got, pendingTestURL) + } + + w = powerAction(server, "On") + if w.Code != http.StatusAccepted { + t.Fatalf("Power On status = %d, want 202 (deferred); body %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "deferred") { + t.Errorf("Power On body does not say deferred: %s", w.Body.String()) + } + vm, _ = mockDynamic.GetVM(pendingTestNS, pendingTestVM) + if got := vm.GetLabels()[kubevirt.PowerAfterImportLabel]; got != "On" { + t.Errorf("label %s = %q, want On", kubevirt.PowerAfterImportLabel, got) + } + if got := vm.Spec.RunStrategy; got == nil || *got != kubevirtv1.RunStrategyHalted { + t.Errorf("VM was started while its import was pending: runStrategy = %v", got) + } +} + +// Ironic retries a lost InsertMedia. While the placeholder is on the VM the +// retry is answered as "in progress" and no second job is queued; a different +// image is a conflict. +func TestHandleInsertVirtualMedia_RepeatWhilePending(t *testing.T) { + server, _ := pendingTestServer(t) + if w := insertMedia(server, pendingTestURL); w.Code != http.StatusAccepted { + t.Fatalf("first InsertMedia status = %d", w.Code) + } + + w := insertMedia(server, pendingTestURL) + if w.Code != http.StatusAccepted { + t.Errorf("repeated InsertMedia status = %d, want 202", w.Code) + } + var body map[string]string + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil || !strings.Contains(body["Message"], "in progress") { + t.Errorf("repeated InsertMedia body = %s, want an in-progress message", w.Body.String()) + } + if n := taskCount(server); n != 1 { + t.Errorf("tasks = %d after the repeat, want 1", n) + } + + w = insertMedia(server, "http://example.com/other.iso") + if w.Code != http.StatusConflict { + t.Errorf("InsertMedia with another image status = %d, want 409", w.Code) + } + if n := taskCount(server); n != 1 { + t.Errorf("tasks = %d after the conflict, want 1", n) + } +} + +// When the job gives up, the placeholder must go, or every later power action +// on that VM would be deferred forever. +func TestWorker_FailedInsertJobClearsPlaceholder(t *testing.T) { + server, mockDynamic := pendingTestServer(t) + tm := server.taskManager + if _, err := server.kubevirtClient.MarkImportPending(pendingTestNS, pendingTestVM, "cdrom0", "://not-a-url"); err != nil { + t.Fatalf("MarkImportPending failed: %v", err) + } + + // The task is created the way the handler does it; its job is then run + // directly, with no retries, on a worker that was never started. The URL + // cannot be parsed, so the job fails before it changes the VM or creates + // a pod. + taskID := tm.CreateTask("insert", pendingTestNS, pendingTestVM, "cdrom0", "://not-a-url") + job := &Job{ + ID: "job-1", + TaskID: taskID, + Type: TaskTypeVirtualMediaInsert, + Payload: map[string]string{ + "namespace": pendingTestNS, + "vmName": pendingTestVM, + "mediaID": "cdrom0", + "imageURL": "://not-a-url", + }, + } + NewWorker(1, tm).processJob(job) + + task, ok := tm.GetTask(taskID) + if !ok || task.TaskState != redfish.TaskStateException { + t.Fatalf("task state = %v, want Exception", task) + } + vm, _ := mockDynamic.GetVM(pendingTestNS, pendingTestVM) + if got, ok := vm.GetLabels()[pendingTestLabel]; ok { + t.Errorf("label %s = %q is still on the VM after the job failed", pendingTestLabel, got) + } +} diff --git a/pkg/server/server.go b/pkg/server/server.go index df224ce..959d059 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -1026,6 +1026,24 @@ func (s *Server) handleInsertVirtualMedia(w http.ResponseWriter, r *http.Request )) return } + + // Ironic sends Power On right after this request is answered, and the + // queued job only labels the VM once it has created the import pod. + // Mark the VM now so that power-on is deferred until the import is done. + marked, err := s.kubevirtClient.MarkImportPending(namespace, vmName, internalMediaID, insertRequest.Image) + if err != nil { + logger.Error("Failed to mark pending import for VM %s/%s: %v", namespace, vmName, err) + s.sendInternalError(w, "Failed to record virtual media insertion") + return + } + if !marked { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + s.encodeJSONResponse(w, map[string]string{ + "Message": "Virtual media import is still in progress", + }) + return + } } // Create a task for this operation diff --git a/pkg/server/task_manager.go b/pkg/server/task_manager.go index 026248c..e619ba5 100644 --- a/pkg/server/task_manager.go +++ b/pkg/server/task_manager.go @@ -277,6 +277,7 @@ func (w *Worker) processJob(job *Job) { if taskErr := w.taskMgr.FailTask(job.TaskID, fmt.Sprintf("Job failed after %d retries: %v", job.MaxRetries, err)); taskErr != nil { logger.Error("Failed to mark task %s as failed: %v", job.TaskID, taskErr) } + w.clearPendingImport(job) } } else { logger.Info("Worker %d completed job %s successfully in %v", w.ID, job.ID, duration) @@ -287,6 +288,22 @@ func (w *Worker) processJob(job *Job) { } } +// clearPendingImport drops the placeholder the InsertMedia handler put on the +// VM when the insert job gives up before creating an import pod; the pod +// watcher owns the label from that point on and the placeholder is gone anyway. +func (w *Worker) clearPendingImport(job *Job) { + if job.Type != TaskTypeVirtualMediaInsert || w.taskMgr.kubevirtClient == nil { + return + } + payload, ok := job.Payload.(map[string]string) + if !ok { + return + } + if err := w.taskMgr.kubevirtClient.ClearPendingImport(payload["namespace"], payload["vmName"], payload["mediaID"]); err != nil { + logger.Error("Failed to clear pending import for VM %s/%s after job %s failed: %v", payload["namespace"], payload["vmName"], job.ID, err) + } +} + // processVirtualMediaInsert processes a virtual media insertion job func (w *Worker) processVirtualMediaInsert(job *Job) error { logger.Debug("Worker %d starting virtual media insert job %s (task %s)", w.ID, job.ID, job.TaskID)