From aed648bb95966b77939a718b21fcacc358964fb8 Mon Sep 17 00:00:00 2001 From: Cezary Stanislawski Date: Sat, 3 Oct 2026 17:07:06 +0200 Subject: [PATCH 1/2] fix: guard lifecycle deletion with UID and resource version --- README.md | 2 + .../controller/deletion_preconditions_test.go | 196 ++++++++++++++++++ internal/controller/lifecycle_controller.go | 5 +- internal/controller/managed_by_test.go | 2 + 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 internal/controller/deletion_preconditions_test.go diff --git a/README.md b/README.md index 0e1536a..09466e9 100644 --- a/README.md +++ b/README.md @@ -117,6 +117,8 @@ In dry-run, relative and recurring schedules are logged without saving state or - `delete-after` is a convenience annotation that is converted into `delete-at` by the controller. - `delete-at` (a specific, one-time event) takes highest priority. +Deletion requests include the UID and resource version read by the controller. If the resource changes or is replaced before the request reaches the API server, the server rejects the request. The controller then retries and checks the current annotations. + ### A Note on Relative Timers Annotations that use relative durations (`delete-after`, `restart-after`) start their from a configurable reference point. While the controller's processing of annotations is usually immediate, factors like high cluster load or controller downtime can introduce delays. diff --git a/internal/controller/deletion_preconditions_test.go b/internal/controller/deletion_preconditions_test.go new file mode 100644 index 0000000..546c6b1 --- /dev/null +++ b/internal/controller/deletion_preconditions_test.go @@ -0,0 +1,196 @@ +package controller + +import ( + "context" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +func TestDeletionPreconditionsEnvtest(t *testing.T) { + useExistingCluster := false + environment := &envtest.Environment{UseExistingCluster: &useExistingCluster} + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Errorf("stop test API server: %v", err) + } + }) + config, err := environment.Start() + if err != nil { + t.Fatalf("start test API server: %v", err) + } + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatalf("register core API: %v", err) + } + apiClient, err := client.NewWithWatch(config, client.Options{Scheme: scheme}) + if err != nil { + t.Fatalf("create test client: %v", err) + } + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "deletion-preconditions"}} + if err := apiClient.Create(t.Context(), namespace); err != nil { + t.Fatalf("create test namespace: %v", err) + } + + tests := []struct { + name string + change func(context.Context, client.Client, *unstructured.Unstructured) error + wantConflict bool + wantDeleted bool + }{ + {name: "unchanged", wantDeleted: true}, + { + name: "replacement", + change: func(ctx context.Context, c client.Client, obj *unstructured.Unstructured) error { + if err := c.Delete(ctx, obj); err != nil { + return err + } + replacement := obj.DeepCopy() + replacement.SetUID("") + replacement.SetResourceVersion("") + replacement.SetCreationTimestamp(metav1.Time{}) + replacement.SetManagedFields(nil) + replacement.SetAnnotations(nil) + return c.Create(ctx, replacement) + }, + wantConflict: true, + }, + { + name: "deadline-removed", + change: func(ctx context.Context, c client.Client, obj *unstructured.Unstructured) error { + obj.SetAnnotations(nil) + return c.Update(ctx, obj) + }, + wantConflict: true, + }, + { + name: "deadline-postponed", + change: func(ctx context.Context, c client.Client, obj *unstructured.Unstructured) error { + obj.SetAnnotations(map[string]string{DeleteAtAnnotation: time.Now().Add(time.Hour).UTC().Format(time.RFC3339)}) + return c.Update(ctx, obj) + }, + wantConflict: true, + }, + { + name: "dry-run-enabled", + change: func(ctx context.Context, c client.Client, obj *unstructured.Unstructured) error { + annotations := obj.GetAnnotations() + annotations[DryRunAnnotation] = "true" + obj.SetAnnotations(annotations) + return c.Update(ctx, obj) + }, + wantConflict: true, + }, + { + name: "unrelated-update", + change: func(ctx context.Context, c client.Client, obj *unstructured.Unstructured) error { + obj.SetLabels(map[string]string{"changed": "true"}) + return c.Update(ctx, obj) + }, + wantConflict: true, + wantDeleted: true, + }, + { + name: "already-deleted", + change: func(ctx context.Context, c client.Client, obj *unstructured.Unstructured) error { + return c.Delete(ctx, obj) + }, + wantDeleted: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Version: "v1", Kind: "ConfigMap"}) + obj.SetName(tt.name) + obj.SetNamespace(namespace.Name) + obj.SetAnnotations(map[string]string{DeleteAtAnnotation: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339)}) + if err := apiClient.Create(t.Context(), obj); err != nil { + t.Fatalf("create target: %v", err) + } + + deleteCalls := 0 + guardedClient := interceptor.NewClient(apiClient, interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, observed client.Object, opts ...client.DeleteOption) error { + deleteCalls++ + assertDeletionPreconditions(t, observed, opts...) + if deleteCalls == 1 && tt.change != nil { + if err := tt.change(ctx, c, observed.(*unstructured.Unstructured).DeepCopy()); err != nil { + t.Fatalf("change target before deletion: %v", err) + } + } + return c.Delete(ctx, observed, opts...) + }, + }) + r := &LifecycleReconciler{Client: guardedClient, Recorder: record.NewFakeRecorder(8)} + request := resourceRequest{NamespacedName: client.ObjectKeyFromObject(obj), GVK: obj.GroupVersionKind()} + result, err := r.Reconcile(t.Context(), request) + if tt.wantConflict { + if !apierrors.IsConflict(err) { + t.Fatalf("stale deletion error = %v, want conflict", err) + } + } else if err != nil { + t.Fatalf("delete target: %v", err) + } + if !result.IsZero() || deleteCalls != 1 { + t.Fatalf("first reconcile: result=%+v delete calls=%d", result, deleteCalls) + } + current := obj.DeepCopy() + if tt.wantConflict { + if err := apiClient.Get(t.Context(), request.NamespacedName, current); err != nil { + t.Fatalf("read target after conflict: %v", err) + } + if tt.name == "replacement" && current.GetUID() == obj.GetUID() { + t.Fatal("replacement must have a different UID") + } + } + if _, err := r.Reconcile(t.Context(), request); err != nil { + t.Fatalf("reconcile current target: %v", err) + } + final := obj.DeepCopy() + err = apiClient.Get(t.Context(), request.NamespacedName, final) + if tt.wantDeleted { + if !apierrors.IsNotFound(err) { + t.Fatalf("target must be deleted, got %v", err) + } + } else { + if err != nil { + t.Fatalf("target must survive, got %v", err) + } + if final.GetUID() != current.GetUID() || final.GetResourceVersion() != current.GetResourceVersion() { + t.Fatal("retry changed the current target") + } + } + wantCalls := 1 + if tt.name == "unrelated-update" { + wantCalls = 2 + } + if deleteCalls != wantCalls { + t.Fatalf("delete calls = %d, want %d", deleteCalls, wantCalls) + } + }) + } +} + +func assertDeletionPreconditions(t *testing.T, observed client.Object, opts ...client.DeleteOption) { + t.Helper() + options := (&client.DeleteOptions{}).ApplyOptions(opts) + if options.Preconditions == nil || options.Preconditions.UID == nil || + *options.Preconditions.UID != observed.GetUID() { + t.Error("delete must require the observed UID") + } + if options.Preconditions == nil || options.Preconditions.ResourceVersion == nil || + *options.Preconditions.ResourceVersion != observed.GetResourceVersion() { + t.Error("delete must require the observed resource version") + } +} diff --git a/internal/controller/lifecycle_controller.go b/internal/controller/lifecycle_controller.go index 52d7d61..e45ff90 100644 --- a/internal/controller/lifecycle_controller.go +++ b/internal/controller/lifecycle_controller.go @@ -360,7 +360,10 @@ func (r *LifecycleReconciler) handleDeletion(ctx context.Context, obj *unstructu if time.Now().After(deleteAtTime) { logger.Info("Deleting resource based on delete-at annotation", "targetTime", deleteAtTime.String()) if !isDryRun { - if err := r.Delete(ctx, obj); err != nil { + uid := obj.GetUID() + resourceVersion := obj.GetResourceVersion() + preconditions := client.Preconditions{UID: &uid, ResourceVersion: &resourceVersion} + if err := r.Delete(ctx, obj, preconditions); err != nil { if !apierrors.IsNotFound(err) { logger.Error(err, "failed to delete object") r.Recorder.Eventf(obj, "Warning", "DeletionFailed", "Failed to delete resource: %v", err) diff --git a/internal/controller/managed_by_test.go b/internal/controller/managed_by_test.go index 8ef1214..16dc2c4 100644 --- a/internal/controller/managed_by_test.go +++ b/internal/controller/managed_by_test.go @@ -18,6 +18,8 @@ func testDeployment(name string, annotations map[string]string) *unstructured.Un obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"}) obj.SetName(name) obj.SetNamespace("default") + obj.SetUID(types.UID("uid-" + name)) + obj.SetResourceVersion("1") obj.SetAnnotations(annotations) return obj } From cc5acf25c20da3f1e47761ff0b151d8c85b6f3fe Mon Sep 17 00:00:00 2001 From: Cezary Stanislawski Date: Sat, 3 Oct 2026 17:08:33 +0200 Subject: [PATCH 2/2] docs: revert deletion precondition paragraph --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 09466e9..0e1536a 100644 --- a/README.md +++ b/README.md @@ -117,8 +117,6 @@ In dry-run, relative and recurring schedules are logged without saving state or - `delete-after` is a convenience annotation that is converted into `delete-at` by the controller. - `delete-at` (a specific, one-time event) takes highest priority. -Deletion requests include the UID and resource version read by the controller. If the resource changes or is replaced before the request reaches the API server, the server rejects the request. The controller then retries and checks the current annotations. - ### A Note on Relative Timers Annotations that use relative durations (`delete-after`, `restart-after`) start their from a configurable reference point. While the controller's processing of annotations is usually immediate, factors like high cluster load or controller downtime can introduce delays.