From cc941b4164985b84557dc9617eca7759888e5a96 Mon Sep 17 00:00:00 2001 From: Cezary Stanislawski Date: Sun, 4 Oct 2026 12:02:23 +0200 Subject: [PATCH] fix: repair lifecycle scheduling and restrict restart targets --- README.md | 8 +- go.mod | 4 +- internal/controller/lifecycle_controller.go | 105 ++------- internal/controller/lifecycle_predicate.go | 11 +- .../controller/lifecycle_predicate_test.go | 6 +- .../lifecycle_recovery_envtest_test.go | 213 ++++++++++++++++++ internal/controller/lifecycle_time.go | 27 +++ internal/controller/lifecycle_time_test.go | 48 ++++ internal/controller/recurring_restart_test.go | 2 +- internal/controller/restart_patch.go | 61 +++++ internal/controller/restart_support.go | 24 ++ internal/controller/restart_support_test.go | 47 ++++ internal/controller/suite_test.go | 4 +- 13 files changed, 468 insertions(+), 92 deletions(-) create mode 100644 internal/controller/lifecycle_recovery_envtest_test.go create mode 100644 internal/controller/lifecycle_time.go create mode 100644 internal/controller/lifecycle_time_test.go create mode 100644 internal/controller/restart_patch.go create mode 100644 internal/controller/restart_support.go create mode 100644 internal/controller/restart_support_test.go diff --git a/README.md b/README.md index 0ca1bdf..61483ce 100644 --- a/README.md +++ b/README.md @@ -104,14 +104,14 @@ spec: - `lifecycle.cezary.dev/dry-run` - enables dry-run for a resource. Accepts standard boolean values such as `true`, `false`, `1`, and `0`. Invalid values create a warning Event and no action is taken. - `lifecycle.cezary.dev/managed-by: "lifecycle-controller"` - added by the controller when it mutates a resource, such as converting `*-after` annotations to `*-at` annotations, maintaining restart schedule state, or triggering a restart. -- For watched resources with a `spec.template` field: +- For watched Deployments, StatefulSets, and DaemonSets in the `apps` API group: - `lifecycle.cezary.dev/restart-at` - requests one Pod template update at or after a specific date and time. - The value must be an RFC3339 timestamp with explicit timezone offset (`Z` or `±hh:mm`). - `lifecycle.cezary.dev/restart-after` - requests one Pod template update after a relative duration (e.g., `1h`). The controller converts this to an absolute `restart-at` annotation. - `lifecycle.cezary.dev/restart-every` - requests recurring Pod template updates at a relative interval (e.g., `7d` for weekly updates), with a minimum duration of `1m`. - `lifecycle.cezary.dev/restart-cron` - requests recurring Pod template updates based on a standard five-field, minute-resolution cron expression (e.g., `"0 3 * * *"` for daily at 03:00). - `lifecycle.cezary.dev/cron-timezone` - optional timezone for `restart-cron` only. Must be valid IANA timezone (e.g., `America/New_York`). Defaults to `UTC`. - - The controller checks for `spec.template`. Template annotations do not need to exist before the restart request. This field check does not guarantee that the API accepts the patch or that the workload replaces existing Pods. + - Other resources receive an `UnsupportedRestartKind` warning Event and their restart annotations remain unchanged. ### Relative duration format @@ -146,7 +146,7 @@ After conversion, restart priority is `restart-at`, then `restart-cron`, then `r ### Restart mechanism -Deployments and StatefulSets or DaemonSets with `RollingUpdate` can replace Pods after a template change. StatefulSets and DaemonSets with `OnDelete`, and ReplicaSets, do not automatically replace existing Pods for this change. Paused Deployments and StatefulSet partitions can also prevent or limit Pod replacement. The controller does not wait for a rollout to complete. +Restarts are supported only for native Deployments, StatefulSets, and DaemonSets, while deletion remains available for all watched resources. - Triggering a restart - the controller writes `lifecycle.cezary.dev/restartedAt: ""` into the resource's `spec.template.metadata.annotations`. The workload controller determines how to apply the changed template. - The same pod template mutation also adds `lifecycle.cezary.dev/managed-by: "lifecycle-controller"` to `spec.template.metadata.annotations`. @@ -180,7 +180,7 @@ Resource filters select which API types to watch. The controller reads, deletes, ### RBAC -By default, the Helm chart watches standard workloads, ConfigMaps, Services, Ingresses, NetworkPolicies, PersistentVolumeClaims, HorizontalPodAutoscalers, PodDisruptionBudgets, Namespaces, and PersistentVolumes. It grants `get`, `list`, `watch`, `patch`, `update`, and `delete` for those resources, plus `Event` permissions and permissions for enabled leader-election and secure-metrics features. +By default, the Helm chart watches standard workloads, ConfigMaps, Services, Ingresses, NetworkPolicies, PersistentVolumeClaims, HorizontalPodAutoscalers, PodDisruptionBudgets, Namespaces, and PersistentVolumes. It grants `get`, `list`, `watch`, `patch`, `update`, and `delete` for those resources, plus Event permissions and permissions for enabled leader-election and secure-metrics features. A list of filtered namespaces creates a `Role` and `RoleBinding` in each watched namespace. Namespace patterns or leaving the list unset results in a `ClusterRole`. Namespace objects still require cluster permissions. diff --git a/go.mod b/go.mod index 7281ad6..9719e22 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,10 @@ require ( github.com/prometheus/client_golang v1.24.1 github.com/robfig/cron/v3 v3.0.1 k8s.io/api v0.37.1 + k8s.io/apiextensions-apiserver v0.37.1 k8s.io/apimachinery v0.37.1 k8s.io/client-go v0.37.1 + k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 sigs.k8s.io/controller-runtime v0.25.2 ) @@ -94,13 +96,11 @@ require ( google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - k8s.io/apiextensions-apiserver v0.37.1 // indirect k8s.io/apiserver v0.37.1 // indirect k8s.io/component-base v0.37.1 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect k8s.io/streaming v0.37.1 // indirect - k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.37.0 // indirect sigs.k8s.io/json v0.0.0-20260909141634-11ed52e25bc5 // indirect sigs.k8s.io/randfill v1.0.0 // indirect diff --git a/internal/controller/lifecycle_controller.go b/internal/controller/lifecycle_controller.go index dc3cc9d..209076a 100644 --- a/internal/controller/lifecycle_controller.go +++ b/internal/controller/lifecycle_controller.go @@ -101,17 +101,6 @@ func planRecurringRestart(schedule recurringSchedule, lastRestart, now time.Time return scheduledAt, schedule.coalescedAnchor(lastRestart, now), true } -func requeueForNextOccurrence(next time.Time) ctrl.Result { - if next.IsZero() { - return ctrl.Result{} - } - requeueAfter := time.Until(next) - if requeueAfter <= 0 { - requeueAfter = time.Nanosecond - } - return ctrl.Result{RequeueAfter: requeueAfter} -} - // LifecycleReconciler reconciles objects with lifecycle annotations. type LifecycleReconciler struct { client.Client @@ -121,6 +110,7 @@ type LifecycleReconciler struct { GlobalDryRun bool discovery preferredResourceDiscovery coverage *coverageState + now func() time.Time } // +kubebuilder:rbac:groups=*,resources=*,verbs=get;list;watch;delete;update;patch @@ -143,11 +133,11 @@ func (r *LifecycleReconciler) getReferenceTime(obj client.Object, logger logr.Lo } else { logger.Info("Using 'applyTimestamp' (reconciliation time) as reference point") } - return time.Now().UTC() + return r.currentTime().UTC() default: logger.Info("Invalid reference-point specified, falling back to 'applyTimestamp'", "value", referencePoint) r.Recorder.Eventf(obj, "Warning", "InvalidAnnotationValue", "Invalid value for %s: '%s', falling back to 'applyTimestamp'", ReferencePointAnnotation, referencePoint) - return time.Now().UTC() + return r.currentTime().UTC() } } @@ -298,7 +288,7 @@ func (r *LifecycleReconciler) handleDeletion(ctx context.Context, obj *unstructu return ctrl.Result{}, nil } - if time.Now().After(deleteAtTime) { + if !r.currentTime().Before(deleteAtTime) { logger.Info("Deleting resource based on delete-at annotation", "targetTime", deleteAtTime.String()) if !isDryRun { uid := obj.GetUID() @@ -318,9 +308,9 @@ func (r *LifecycleReconciler) handleDeletion(ctx context.Context, obj *unstructu } return ctrl.Result{}, nil } else { - requeueAfter := time.Until(deleteAtTime) - logger.Info("Resource deletion scheduled for a future time", "requeueAfter", requeueAfter) - return ctrl.Result{RequeueAfter: requeueAfter}, nil + result := r.requeueForNextOccurrence(deleteAtTime) + logger.Info("Resource deletion scheduled for a future time", "requeueAfter", result.RequeueAfter) + return result, nil } } @@ -329,15 +319,19 @@ func (r *LifecycleReconciler) handleDeletion(ctx context.Context, obj *unstructu // handleRestart implements the full restart logic with precedence. func (r *LifecycleReconciler) handleRestart(ctx context.Context, obj *unstructured.Unstructured, isDryRun bool, logger logr.Logger) (ctrl.Result, error) { - _, found, err := unstructured.NestedFieldNoCopy(obj.Object, "spec", "template") - if err != nil || !found { - logger.Info("Resource is not a pod-spawner, skipping restart action.", "resource", client.ObjectKeyFromObject(obj)) + if !supportsRestart(obj) { + logger.Info("Skipping restart for an unsupported resource kind", "gvk", obj.GroupVersionKind()) + r.Recorder.Event(obj, "Warning", "UnsupportedRestartKind", "Restart requires an apps Deployment, StatefulSet, or DaemonSet; no action taken") + return ctrl.Result{}, nil + } + if !hasRestartTemplate(obj) { + logger.Info("Skipping restart because the Pod template is missing", "resource", client.ObjectKeyFromObject(obj)) r.Recorder.Event(obj, "Warning", "NotPodSpawner", "Restart annotations are present but resource does not have a spec.template field.") return ctrl.Result{}, nil } annotations := obj.GetAnnotations() - now := time.Now() + now := r.currentTime() if restartAfterStr := annotations[RestartAfterAnnotation]; restartAfterStr != "" { if annotations[ReferencePointAnnotation] == ReferencePointCreationTimestamp && annotations[RestartAtAnnotation] != "" { @@ -394,7 +388,7 @@ func (r *LifecycleReconciler) handleRestart(ctx context.Context, obj *unstructur r.Recorder.Eventf(obj, "Warning", "InvalidAnnotation", "Invalid format for restart-at annotation: %v", err) return ctrl.Result{}, nil } - if now.After(restartTime) { + if !now.Before(restartTime) { logger.Info("Triggering one-time restart based on restart-at annotation", "restartTime", restartTime) if err := r.triggerRestart(ctx, obj, isDryRun, func(annotations map[string]string) { delete(annotations, RestartAtAnnotation) @@ -403,9 +397,9 @@ func (r *LifecycleReconciler) handleRestart(ctx context.Context, obj *unstructur } return ctrl.Result{}, nil } else { - requeueAfter := time.Until(restartTime) - logger.Info("One-time restart scheduled for a future time", "requeueAfter", requeueAfter) - return ctrl.Result{RequeueAfter: requeueAfter}, nil + result := r.requeueForNextOccurrence(restartTime) + logger.Info("One-time restart scheduled for a future time", "requeueAfter", result.RequeueAfter) + return result, nil } } @@ -427,7 +421,7 @@ func (r *LifecycleReconciler) handleRestart(ctx context.Context, obj *unstructur r.Recorder.Eventf(obj, "Warning", "InvalidAnnotation", "Invalid cron expression for restart-cron: %v", err) return ctrl.Result{}, nil } - if schedule.Next(time.Now()).IsZero() { + if schedule.Next(r.currentTime()).IsZero() { err := fmt.Errorf("cron expression has no future occurrence") logger.Error(err, "invalid cron expression", "cron", cronStr) r.Recorder.Eventf(obj, "Warning", "InvalidAnnotation", "Invalid cron expression for restart-cron: %v", err) @@ -452,7 +446,7 @@ func (r *LifecycleReconciler) handleRestart(ctx context.Context, obj *unstructur // reconcileRecurringRestart handles the stateful logic for both cron and interval restarts. func (r *LifecycleReconciler) reconcileRecurringRestart(ctx context.Context, obj *unstructured.Unstructured, isDryRun bool, scheduleType string, schedule recurringSchedule, logger logr.Logger) (ctrl.Result, error) { annotations := obj.GetAnnotations() - now := time.Now() + now := r.currentTime() lastRestartStr := annotations[LastRestartTimestamp] if lastRestartStr == "" { @@ -468,7 +462,7 @@ func (r *LifecycleReconciler) reconcileRecurringRestart(ctx context.Context, obj logger.Error(err, "failed to initialize last-restart-timestamp") return ctrl.Result{}, err } - return requeueForNextOccurrence(schedule.Next(now)), nil + return r.requeueForNextOccurrence(schedule.Next(now)), nil } lastRestartTime, err := time.Parse(time.RFC3339, lastRestartStr) @@ -490,61 +484,10 @@ func (r *LifecycleReconciler) reconcileRecurringRestart(ctx context.Context, obj if isDryRun { return ctrl.Result{}, nil } - return requeueForNextOccurrence(schedule.Next(coalescedAnchor)), nil + return r.requeueForNextOccurrence(schedule.Next(coalescedAnchor)), nil } else { - result := requeueForNextOccurrence(nextScheduledRestart) + result := r.requeueForNextOccurrence(nextScheduledRestart) logger.Info("Next recurring restart is scheduled", "type", scheduleType, "at", nextScheduledRestart, "requeueAfter", result.RequeueAfter) return result, nil } } - -// triggerRestart applies the rollout marker and its acknowledgement in one patch. -func (r *LifecycleReconciler) triggerRestart(ctx context.Context, obj *unstructured.Unstructured, isDryRun bool, acknowledge func(map[string]string), logger logr.Logger) error { - restartedAtTime := time.Now().UTC().Format(time.RFC3339) - logger.Info("Attempting to trigger restart", "restartedAt", restartedAtTime) - - if isDryRun { - logger.Info("[DRY-RUN] Would trigger restart by setting template annotation", "annotation", RestartedAtTemplate, "value", restartedAtTime) - r.Recorder.Event(obj, "Normal", "DryRunRestart", "Dry-run: Resource would be restarted now.") - return nil - } - - base := obj.DeepCopy() - annotations := obj.GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - acknowledge(annotations) - obj.SetAnnotations(annotations) - markManagedBy(obj) - - templateAnnotations, found, err := unstructured.NestedStringMap(obj.Object, "spec", "template", "metadata", "annotations") - if err != nil { - logger.Error(err, "failed to get spec.template.metadata.annotations") - r.Recorder.Eventf(obj, "Warning", "RestartFailed", "Could not read pod template annotations: %v", err) - return err - } - if !found || templateAnnotations == nil { - templateAnnotations = make(map[string]string) - } - - templateAnnotations[RestartedAtTemplate] = restartedAtTime - templateAnnotations[ManagedByAnnotation] = ManagedByValue - err = unstructured.SetNestedStringMap(obj.Object, templateAnnotations, "spec", "template", "metadata", "annotations") - if err != nil { - logger.Error(err, "failed to set restartedAt annotation on pod template") - r.Recorder.Eventf(obj, "Warning", "RestartFailed", "Could not set pod template annotations: %v", err) - return err - } - - patch := client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}) - if err := r.Patch(ctx, obj, patch); err != nil { - logger.Error(err, "failed to patch object to trigger restart") - r.Recorder.Eventf(obj, "Warning", "RestartFailed", "Could not patch object to trigger restart: %v", err) - return err - } - - logger.Info("Successfully patched object to trigger restart") - r.Recorder.Event(obj, "Normal", "RestartTriggered", "Triggered a rolling restart of the resource.") - return nil -} diff --git a/internal/controller/lifecycle_predicate.go b/internal/controller/lifecycle_predicate.go index 53b1bbd..71ba3f6 100644 --- a/internal/controller/lifecycle_predicate.go +++ b/internal/controller/lifecycle_predicate.go @@ -16,8 +16,7 @@ var lifecycleActionAnnotations = []string{ } // LifecyclePredicate limits the shared workqueue to lifecycle-relevant events. -// Controller-owned state updates are intentionally excluded: reconcile results -// drive the next pass without requiring the controller's own writes to enqueue it. +// Restart state changes also enqueue pending actions. func LifecyclePredicate() predicate.Predicate { return predicate.Funcs{ CreateFunc: func(e event.CreateEvent) bool { @@ -59,6 +58,10 @@ func lifecycleConfigurationChanged(e event.UpdateEvent) bool { (hasRelativeAction(oldAnnotations) || hasRelativeAction(newAnnotations)) { return true } + if (hasRecurringRestart(oldAnnotations) || hasRecurringRestart(newAnnotations)) && + oldAnnotations[LastRestartTimestamp] != newAnnotations[LastRestartTimestamp] { + return true + } return oldAnnotations[CronTimezoneAnnotation] != newAnnotations[CronTimezoneAnnotation] && (oldAnnotations[RestartCronAnnotation] != "" || newAnnotations[RestartCronAnnotation] != "") } @@ -79,3 +82,7 @@ func hasLifecycleAction(obj client.Object) bool { func hasRelativeAction(annotations map[string]string) bool { return annotations[DeleteAfterAnnotation] != "" || annotations[RestartAfterAnnotation] != "" } + +func hasRecurringRestart(annotations map[string]string) bool { + return annotations[RestartCronAnnotation] != "" || annotations[RestartEveryAnnotation] != "" +} diff --git a/internal/controller/lifecycle_predicate_test.go b/internal/controller/lifecycle_predicate_test.go index 0a987d7..5a1ac1e 100644 --- a/internal/controller/lifecycle_predicate_test.go +++ b/internal/controller/lifecycle_predicate_test.go @@ -27,6 +27,7 @@ func TestLifecyclePredicateFiltersWorkqueueEvents(t *testing.T) { {name: "generic with lifecycle action", got: p.Generic(event.GenericEvent{Object: actionable}), want: false}, {name: "metadata update", got: p.Update(updateEvent(actionable, objectWithNameAndAnnotations("renamed", actionable.GetAnnotations()))), want: false}, {name: "spec update", got: p.Update(updateEvent(actionable, objectWithSpecAndAnnotations(map[string]interface{}{"replicas": int64(2)}, actionable.GetAnnotations()))), want: false}, + {name: "template addition with restart action", got: p.Update(updateEvent(objectWithAnnotations(map[string]string{RestartAtAnnotation: "2030-01-01T00:00:00Z"}), objectWithSpecAndAnnotations(map[string]interface{}{"template": map[string]interface{}{}}, map[string]string{RestartAtAnnotation: "2030-01-01T00:00:00Z"}))), want: false}, {name: "unrelated annotation update", got: p.Update(updateEvent(actionable, objectWithAnnotations(map[string]string{DeleteAfterAnnotation: "1h", "example.com/note": "changed"}))), want: false}, {name: "action addition", got: p.Update(updateEvent(objectWithAnnotations(nil), actionable)), want: true}, {name: "action value update", got: p.Update(updateEvent(actionable, objectWithAnnotations(map[string]string{DeleteAfterAnnotation: "2h"}))), want: true}, @@ -36,7 +37,10 @@ func TestLifecyclePredicateFiltersWorkqueueEvents(t *testing.T) { {name: "reference-point update with absolute action", got: p.Update(updateEvent(objectWithAnnotations(map[string]string{DeleteAtAnnotation: "2030-01-01T00:00:00Z"}), objectWithAnnotations(map[string]string{DeleteAtAnnotation: "2030-01-01T00:00:00Z", ReferencePointAnnotation: ReferencePointCreationTimestamp}))), want: false}, {name: "timezone update with cron action", got: p.Update(updateEvent(objectWithAnnotations(map[string]string{RestartCronAnnotation: "0 * * * *"}), objectWithAnnotations(map[string]string{RestartCronAnnotation: "0 * * * *", CronTimezoneAnnotation: "Europe/Warsaw"}))), want: true}, {name: "timezone update without cron action", got: p.Update(updateEvent(actionable, objectWithAnnotations(map[string]string{DeleteAfterAnnotation: "1h", CronTimezoneAnnotation: "Europe/Warsaw"}))), want: false}, - {name: "internal last-restart update", got: p.Update(updateEvent(objectWithAnnotations(map[string]string{RestartEveryAnnotation: "1h"}), objectWithAnnotations(map[string]string{RestartEveryAnnotation: "1h", LastRestartTimestamp: "2030-01-01T00:00:00Z"}))), want: false}, + {name: "recurring state initialization", got: p.Update(updateEvent(objectWithAnnotations(map[string]string{RestartEveryAnnotation: "1h"}), objectWithAnnotations(map[string]string{RestartEveryAnnotation: "1h", LastRestartTimestamp: "2030-01-01T00:00:00Z"}))), want: true}, + {name: "cron state removal", got: p.Update(updateEvent(objectWithAnnotations(map[string]string{RestartCronAnnotation: "0 * * * *", LastRestartTimestamp: "invalid"}), objectWithAnnotations(map[string]string{RestartCronAnnotation: "0 * * * *"}))), want: true}, + {name: "state update with one-time restart", got: p.Update(updateEvent(objectWithAnnotations(map[string]string{RestartAtAnnotation: "2030-01-01T00:00:00Z"}), objectWithAnnotations(map[string]string{RestartAtAnnotation: "2030-01-01T00:00:00Z", LastRestartTimestamp: "2030-01-01T00:00:00Z"}))), want: false}, + {name: "state update with deletion", got: p.Update(updateEvent(actionable, objectWithAnnotations(map[string]string{DeleteAfterAnnotation: "1h", LastRestartTimestamp: "2030-01-01T00:00:00Z"}))), want: false}, {name: "internal managed-by update", got: p.Update(updateEvent(actionable, objectWithAnnotations(map[string]string{DeleteAfterAnnotation: "1h", ManagedByAnnotation: ManagedByValue}))), want: false}, } diff --git a/internal/controller/lifecycle_recovery_envtest_test.go b/internal/controller/lifecycle_recovery_envtest_test.go new file mode 100644 index 0000000..9fac5e6 --- /dev/null +++ b/internal/controller/lifecycle_recovery_envtest_test.go @@ -0,0 +1,213 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + eventsv1 "k8s.io/api/events/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/envtest" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +func TestLifecycleScopeAndRecoveryEnvtest(t *testing.T) { + useExisting := false + environment := &envtest.Environment{ + UseExistingCluster: &useExisting, + CRDs: []*apiextensionsv1.CustomResourceDefinition{ + lifecycleTestCRD("workloads", "Deployment", apiextensionsv1.NamespaceScoped), + }, + } + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Errorf("stop test API server: %v", err) + } + }) + restConfig, err := environment.Start() + if err != nil { + t.Fatal(err) + } + scheme := runtime.NewScheme() + if err := corev1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := appsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := eventsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + apiClient, err := client.New(restConfig, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + const namespace = "team-a" + lifecycleTestCreate(t, apiClient, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}) + + newWorkload := func(name string, annotations map[string]string, withTemplate bool) *unstructured.Unstructured { + obj := &unstructured.Unstructured{} + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: "example.test", Version: "v1", Kind: "Deployment"}) + obj.SetName(name) + obj.SetNamespace(namespace) + obj.SetAnnotations(annotations) + if withTemplate { + setTemplate(t, obj) + } + lifecycleTestCreate(t, apiClient, obj) + return obj + } + newNativeWorkload := func(kind, name string, annotations map[string]string) *unstructured.Unstructured { + obj := testDeployment(name, annotations) + obj.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind(kind)) + obj.SetNamespace(namespace) + obj.SetUID("") + obj.SetResourceVersion("") + setTemplate(t, obj) + if err := unstructured.SetNestedStringMap(obj.Object, map[string]string{"app": "managed-by"}, "spec", "selector", "matchLabels"); err != nil { + t.Fatal(err) + } + if kind == "StatefulSet" { + if err := unstructured.SetNestedField(obj.Object, name, "spec", "serviceName"); err != nil { + t.Fatal(err) + } + } + lifecycleTestCreate(t, apiClient, obj) + return obj + } + daemonSet := newNativeWorkload("DaemonSet", "one-time-daemonset", map[string]string{RestartAtAnnotation: "2000-01-01T00:00:00Z"}) + statefulSet := newNativeWorkload("StatefulSet", "one-time-statefulset", map[string]string{RestartAtAnnotation: "2000-01-01T00:00:00Z"}) + stateRepair := newNativeWorkload("Deployment", "state-repair", map[string]string{RestartEveryAnnotation: "1m", LastRestartTimestamp: "invalid"}) + customRestart := newWorkload("custom-restart", map[string]string{RestartAfterAnnotation: "1s"}, true) + customDeletion := newWorkload("custom-deletion", map[string]string{DeleteAtAnnotation: "2000-01-01T00:00:00Z"}, false) + now := time.Date(2026, time.October, 3, 12, 0, 0, 0, time.UTC) + manager, err := ctrl.NewManager(restConfig, ctrl.Options{ + Scheme: scheme, Logger: logr.Discard(), Metrics: metricsserver.Options{BindAddress: "0"}, + // The package also runs a separate manager with the same controller name. + Controller: config.Controller{SkipNameValidation: new(true)}, + }) + if err != nil { + t.Fatal(err) + } + r := &LifecycleReconciler{ + Client: manager.GetClient(), Scheme: scheme, now: func() time.Time { return now }, + Config: ScopeConfig{ + WatchResources: []string{ + "namespaces", "workloads.example.test", + "deployments.apps", "statefulsets.apps", "daemonsets.apps", + }, + WatchNamespaces: []string{namespace}, + }, + } + if err := r.SetupWithManager(manager); err != nil { + t.Fatal(err) + } + ctx, cancelManager := context.WithCancel(t.Context()) + done := make(chan error, 1) + t.Cleanup(func() { + cancelManager() + select { + case err := <-done: + if err != nil { + t.Errorf("stop test manager: %v", err) + } + case <-time.After(10 * time.Second): + t.Error("test manager did not stop") + } + }) + go func() { done <- manager.Start(ctx) }() + ready := r.ReadinessCheck(nil, false) + eventually(t, 10*time.Second, func() bool { return ready(nil) == nil }) + + for _, native := range []*unstructured.Unstructured{daemonSet, statefulSet} { + lifecycleTestWaitForRestart(t, apiClient, native) + } + eventually(t, 10*time.Second, func() bool { + return apierrors.IsNotFound(apiClient.Get(t.Context(), client.ObjectKeyFromObject(customDeletion), customDeletion.DeepCopy())) + }) + + for _, failure := range []struct{ name, reason string }{ + {name: stateRepair.GetName(), reason: "InvalidState"}, + {name: customRestart.GetName(), reason: "UnsupportedRestartKind"}, + } { + lifecycleTestWaitForWarning(t, apiClient, client.ObjectKey{Namespace: namespace, Name: failure.name}, failure.reason) + } + annotations := stateRepair.GetAnnotations() + annotations[LastRestartTimestamp] = now.Add(-2 * time.Minute).Format(time.RFC3339) + stateRepair.SetAnnotations(annotations) + if err := apiClient.Update(t.Context(), stateRepair); err != nil { + t.Fatal(err) + } + lifecycleTestWaitForRestart(t, apiClient, stateRepair) + + for until := time.Now().Add(time.Second); time.Now().Before(until); { + current := customRestart.DeepCopy() + if err := apiClient.Get(t.Context(), client.ObjectKeyFromObject(customRestart), current); err != nil { + t.Fatalf("custom %s was removed: %v", customRestart.GetKind(), err) + } + if current.GetResourceVersion() != customRestart.GetResourceVersion() { + t.Fatalf("custom %s %s changed unexpectedly", customRestart.GetKind(), customRestart.GetName()) + } + time.Sleep(20 * time.Millisecond) + } +} + +func lifecycleTestCRD(plural, kind string, scope apiextensionsv1.ResourceScope) *apiextensionsv1.CustomResourceDefinition { + return &apiextensionsv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{Name: plural + ".example.test"}, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: "example.test", Names: apiextensionsv1.CustomResourceDefinitionNames{Plural: plural, Kind: kind}, Scope: scope, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: "v1", Served: true, Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", XPreserveUnknownFields: new(true), + }}, + }}, + }, + } +} + +func lifecycleTestCreate(t *testing.T, apiClient client.Client, obj client.Object) { + t.Helper() + if err := apiClient.Create(t.Context(), obj); err != nil { + t.Fatalf("create %T: %v", obj, err) + } +} + +func lifecycleTestWaitForWarning(t *testing.T, apiClient client.Client, key client.ObjectKey, reason string) { + t.Helper() + eventually(t, 10*time.Second, func() bool { + warnings := &eventsv1.EventList{} + if err := apiClient.List(t.Context(), warnings, client.InNamespace(key.Namespace)); err != nil { + return false + } + for _, warning := range warnings.Items { + if warning.Regarding.Name == key.Name && warning.Reason == reason { + return true + } + } + return false + }) +} + +func lifecycleTestWaitForRestart(t *testing.T, apiClient client.Client, obj *unstructured.Unstructured) { + t.Helper() + eventually(t, 10*time.Second, func() bool { + if err := apiClient.Get(t.Context(), client.ObjectKeyFromObject(obj), obj); err != nil { + return false + } + marker, _, err := unstructured.NestedString(obj.Object, "spec", "template", "metadata", "annotations", RestartedAtTemplate) + return err == nil && marker != "" && obj.GetAnnotations()[RestartAtAnnotation] == "" + }) +} diff --git a/internal/controller/lifecycle_time.go b/internal/controller/lifecycle_time.go new file mode 100644 index 0000000..a5f6edf --- /dev/null +++ b/internal/controller/lifecycle_time.go @@ -0,0 +1,27 @@ +package controller + +import ( + "time" + + ctrl "sigs.k8s.io/controller-runtime" +) + +func (r *LifecycleReconciler) currentTime() time.Time { + if r.now != nil { + return r.now() + } + return time.Now() +} + +func (r *LifecycleReconciler) requeueForNextOccurrence(next time.Time) ctrl.Result { + if next.IsZero() { + return ctrl.Result{} + } + + delay := next.Sub(r.currentTime()) + if delay <= 0 { + // Controller-runtime discards non-positive timer delays. + delay = time.Nanosecond + } + return ctrl.Result{RequeueAfter: delay} +} diff --git a/internal/controller/lifecycle_time_test.go b/internal/controller/lifecycle_time_test.go new file mode 100644 index 0000000..b1936ad --- /dev/null +++ b/internal/controller/lifecycle_time_test.go @@ -0,0 +1,48 @@ +package controller + +import ( + "testing" + "time" + + "github.com/go-logr/logr" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestOneTimeDeadlineCrossingRetainsTimer(t *testing.T) { + for _, action := range []struct{ name, at string }{ + {name: "delete", at: DeleteAtAnnotation}, + {name: "restart", at: RestartAtAnnotation}, + } { + t.Run(action.name, func(t *testing.T) { + deadline := time.Date(2026, time.October, 3, 12, 0, 0, 0, time.UTC) + obj := testDeployment("crossing-"+action.name, map[string]string{action.at: deadline.Format(time.RFC3339)}) + setTemplate(t, obj) + tracked := &mutationTrackingClient{Client: fake.NewClientBuilder().WithObjects(obj.DeepCopy()).Build()} + reads := 0 + r := &LifecycleReconciler{ + Client: tracked, Recorder: record.NewFakeRecorder(8), + now: func() time.Time { + reads++ + if reads == 1 { + return deadline.Add(-time.Nanosecond) + } + return deadline.Add(time.Nanosecond) + }, + } + result, err := r.reconcileLogic(t.Context(), obj, logr.Discard()) + if err != nil || result.RequeueAfter != time.Nanosecond { + t.Fatalf("crossed deadline: result=%+v error=%v, want positive immediate timer", result, err) + } + if tracked.deletes != 0 || tracked.patches != 0 { + t.Fatal("future branch ran the action") + } + if _, err := r.reconcileLogic(t.Context(), obj, logr.Discard()); err != nil { + t.Fatal(err) + } + if tracked.deletes+tracked.patches != 1 { + t.Fatalf("after timer: deletes=%d patches=%d, want one action", tracked.deletes, tracked.patches) + } + }) + } +} diff --git a/internal/controller/recurring_restart_test.go b/internal/controller/recurring_restart_test.go index 5c17dce..5132334 100644 --- a/internal/controller/recurring_restart_test.go +++ b/internal/controller/recurring_restart_test.go @@ -160,7 +160,7 @@ func TestRestartCronRejectsScheduleWithoutFutureOccurrence(t *testing.T) { } func TestElapsedNextOccurrenceRequeuesImmediately(t *testing.T) { - result := requeueForNextOccurrence(time.Now().Add(-time.Second)) + result := (&LifecycleReconciler{}).requeueForNextOccurrence(time.Now().Add(-time.Second)) if result.RequeueAfter != time.Nanosecond { t.Fatalf("result = %+v, want immediate requeue for elapsed occurrence", result) } diff --git a/internal/controller/restart_patch.go b/internal/controller/restart_patch.go new file mode 100644 index 0000000..5572494 --- /dev/null +++ b/internal/controller/restart_patch.go @@ -0,0 +1,61 @@ +package controller + +import ( + "context" + "time" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// triggerRestart applies the rollout marker and its acknowledgement in one patch. +func (r *LifecycleReconciler) triggerRestart(ctx context.Context, obj *unstructured.Unstructured, isDryRun bool, acknowledge func(map[string]string), logger logr.Logger) error { + restartedAtTime := r.currentTime().UTC().Format(time.RFC3339) + logger.Info("Attempting to trigger restart", "restartedAt", restartedAtTime) + + if isDryRun { + logger.Info("[DRY-RUN] Would trigger restart by setting template annotation", "annotation", RestartedAtTemplate, "value", restartedAtTime) + r.Recorder.Event(obj, "Normal", "DryRunRestart", "Dry-run: Resource would be restarted now.") + return nil + } + + base := obj.DeepCopy() + annotations := obj.GetAnnotations() + if annotations == nil { + annotations = make(map[string]string) + } + acknowledge(annotations) + obj.SetAnnotations(annotations) + markManagedBy(obj) + + templateAnnotations, found, err := unstructured.NestedStringMap(obj.Object, "spec", "template", "metadata", "annotations") + if err != nil { + logger.Error(err, "failed to get spec.template.metadata.annotations") + r.Recorder.Eventf(obj, "Warning", "RestartFailed", "Could not read pod template annotations: %v", err) + return err + } + if !found || templateAnnotations == nil { + templateAnnotations = make(map[string]string) + } + + templateAnnotations[RestartedAtTemplate] = restartedAtTime + templateAnnotations[ManagedByAnnotation] = ManagedByValue + err = unstructured.SetNestedStringMap(obj.Object, templateAnnotations, "spec", "template", "metadata", "annotations") + if err != nil { + logger.Error(err, "failed to set restartedAt annotation on pod template") + r.Recorder.Eventf(obj, "Warning", "RestartFailed", "Could not set pod template annotations: %v", err) + return err + } + + patch := client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{}) + if err := r.Patch(ctx, obj, patch); err != nil { + logger.Error(err, "failed to patch object to trigger restart") + r.Recorder.Eventf(obj, "Warning", "RestartFailed", "Could not patch object to trigger restart: %v", err) + return err + } + + logger.Info("Successfully patched object to trigger restart") + r.Recorder.Event(obj, "Normal", "RestartTriggered", "Updated Pod template for a restart request") + return nil +} diff --git a/internal/controller/restart_support.go b/internal/controller/restart_support.go new file mode 100644 index 0000000..bfd056d --- /dev/null +++ b/internal/controller/restart_support.go @@ -0,0 +1,24 @@ +package controller + +import ( + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func supportsRestart(obj *unstructured.Unstructured) bool { + gvk := obj.GetObjectKind().GroupVersionKind() + if gvk.Group != appsv1.GroupName { + return false + } + switch gvk.Kind { + case "Deployment", "StatefulSet", "DaemonSet": + return true + default: + return false + } +} + +func hasRestartTemplate(obj *unstructured.Unstructured) bool { + _, found, err := unstructured.NestedFieldNoCopy(obj.Object, "spec", "template") + return found && err == nil +} diff --git a/internal/controller/restart_support_test.go b/internal/controller/restart_support_test.go new file mode 100644 index 0000000..30e83c9 --- /dev/null +++ b/internal/controller/restart_support_test.go @@ -0,0 +1,47 @@ +package controller + +import ( + "strings" + "testing" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestRestartUnsupportedKindsDoNotMutateOrSchedule(t *testing.T) { + tests := []struct { + name, group, kind, annotation, value string + }{ + {"native group, unsupported kind", "apps", "ReplicaSet", RestartAtAnnotation, "2000-01-01T00:00:00Z"}, + {"custom group, native kind", "example.test", "Deployment", RestartAtAnnotation, "2000-01-01T00:00:00Z"}, + {"unsupported relative restart", "example.test", "Deployment", RestartAfterAnnotation, "1s"}, + {"unsupported interval initialization", "example.test", "Deployment", RestartEveryAnnotation, "1m"}, + {"unsupported cron initialization", "example.test", "Deployment", RestartCronAnnotation, "* * * * *"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + obj := testDeployment("unsupported-restart", map[string]string{tt.annotation: tt.value}) + obj.SetGroupVersionKind(schema.GroupVersionKind{Group: tt.group, Version: "v1", Kind: tt.kind}) + setTemplate(t, obj) + before := obj.DeepCopy() + tracked := &mutationTrackingClient{Client: fake.NewClientBuilder().WithObjects(obj.DeepCopy()).Build()} + recorder := record.NewFakeRecorder(1) + r := &LifecycleReconciler{Client: tracked, Recorder: recorder} + result, err := r.reconcileLogic(t.Context(), obj, logr.Discard()) + if err != nil || !result.IsZero() { + t.Fatalf("unsupported restart: result=%+v error=%v", result, err) + } + assertNoTargetMutation(t, before, obj, tracked) + select { + case warning := <-recorder.Events: + if !strings.Contains(warning, "Warning UnsupportedRestartKind") { + t.Fatalf("event=%q, want UnsupportedRestartKind warning", warning) + } + default: + t.Fatal("unsupported restart has no warning Event") + } + }) + } +} diff --git a/internal/controller/suite_test.go b/internal/controller/suite_test.go index 4148313..010331b 100644 --- a/internal/controller/suite_test.go +++ b/internal/controller/suite_test.go @@ -17,6 +17,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" // +kubebuilder:scaffold:imports ) @@ -68,7 +69,8 @@ var _ = BeforeSuite(func() { // Start the controller manager in the background k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{ - Scheme: scheme.Scheme, + Scheme: scheme.Scheme, + Metrics: metricsserver.Options{BindAddress: "0"}, }) Expect(err).ToNot(HaveOccurred())