feat(controller): detect slow restart loops via termination reason

README.md

@@ -64,7 +64,9 @@ Short name: `clp` (`kubectl get clp`).
 
 | Field | Default | Description |
 |---|---|---|
-| `watchReasons` | `[CrashLoopBackOff, ImagePullBackOff, ErrImagePull, CreateContainerConfigError, InvalidImageName, RunContainerError]` | Container waiting reasons to watch |
+| `watchReasons` | `[CrashLoopBackOff, ImagePullBackOff, ErrImagePull, CreateContainerConfigError, InvalidImageName, RunContainerError]` | Container **waiting** reasons to watch. Termination reasons such as `OOMKilled` never appear here; use `watchTerminationReasons` for those |
+| `watchTerminationReasons` | `[]` | Container **termination** reasons to act on, for containers that restart repeatedly without settling into a watched waiting state. Off by default |
+| `restartWindow` | `1h` | How recently the last termination must have happened for `watchTerminationReasons` to match |
 | `restartThreshold` | `10` | Number of container restarts before action |
 | `durationThreshold` | `30m` | How long a pod must have been continuously not ready before action. Go duration format, rejected by the API server if malformed |
 | `allReplicasFailing` | `true` | Require all replicas to be failing |
@@ -226,6 +228,41 @@ Two alerts worth having:
 Note that `crashloop_scaled_down_total` counts dry-run actions too. Filter with
 `dry_run="false"` when alerting on real ones.
 
+### Slow restart loops
+
+`watchReasons` only sees containers that are **waiting**. Kubelet resets its
+restart backoff once a container has stayed up longer than roughly twice the
+maximum backoff, so a container that survives beyond that between deaths
+restarts immediately every time and never enters `CrashLoopBackOff`. A memory
+leak has exactly this shape: run, grow, get OOM-killed, restart at once,
+repeat. Such a workload can reach hundreds of restarts unnoticed.
+
+`watchTerminationReasons` covers that case by matching on why the container
+last exited rather than on what it is waiting for:
+
+```yaml
+spec:
+  watchTerminationReasons:
+    - OOMKilled
+  restartThreshold: 10
+  restartWindow: 1h
+```
+
+The workload is acted on when a container has reached `restartThreshold`
+restarts **and** its most recent exit carries a listed reason **and** that exit
+happened within `restartWindow`. The window matters: the restart count is
+cumulative for the pod's whole life and never decays, so without it a workload
+that misbehaved last month would still be scaled down.
+
+`OOMKilled` is the safe value to start with. `Error` also works but is broad,
+covering ordinary crashes, liveness kills and SIGKILL after the grace period
+alike.
+
+Not counted: pods being deleted, pods that have completed, containers still
+inside their startup probe, and classic init containers, which run once and
+cannot loop. Init containers declared with `restartPolicy: Always` are sidecars
+and do count.
+
 ## Troubleshooting
 
 ### The policy exists but nothing is scaled down
@@ -260,7 +297,9 @@ work through the conditions the operator applies, in the order it applies them:
   default rather than adding to it.
 - **`namespaceSelector` or `excludeWorkloadSelector` filters it out.**
 - **The failure reason is not watched.** `watchReasons` matches the container's
-  waiting reason exactly. Check it with
+  waiting reason exactly. Note that termination reasons such as `OOMKilled`
+  never appear as a waiting reason, so putting one in `watchReasons` matches
+  nothing; see [Slow restart loops](#slow-restart-loops). Check it with
   `kubectl get pod <pod> -o jsonpath='{.status.containerStatuses[*].state.waiting.reason}'`.
 
 ### Ready is False

api/v1alpha1/crashlooppolicy_types.go

@@ -19,6 +19,31 @@ type CrashLoopPolicySpec struct {
 	// +kubebuilder:default={"CrashLoopBackOff","ImagePullBackOff","ErrImagePull","CreateContainerConfigError","InvalidImageName","RunContainerError"}
 	WatchReasons []string `json:"watchReasons,omitempty"`
 
+	// WatchTerminationReasons lists container termination reasons to act on,
+	// for containers that restart repeatedly without ever settling into a
+	// watched waiting state. Kubelet forgets its restart backoff once a
+	// container has stayed up long enough, so a container that dies every
+	// fifteen minutes restarts immediately every time and never enters
+	// CrashLoopBackOff, which makes it invisible to WatchReasons.
+	//
+	// Empty by default, which leaves behaviour unchanged. "OOMKilled" is the
+	// safe value to start with. "Error" also works but is broad: it covers
+	// ordinary crashes, liveness kills and SIGKILL after the grace period
+	// alike.
+	//
+	// A match additionally requires RestartThreshold to be reached and the
+	// most recent termination to fall inside RestartWindow.
+	// +optional
+	WatchTerminationReasons []string `json:"watchTerminationReasons,omitempty"`
+
+	// RestartWindow bounds how recently the last termination must have
+	// happened for WatchTerminationReasons to match. Without it the check
+	// would act on a lifetime restart counter that never decays, so a
+	// workload that misbehaved last month would still be scaled down.
+	// +kubebuilder:default="1h"
+	// +kubebuilder:validation:Pattern=`^([0-9]+(\.[0-9]+)?(ns|us|ms|s|m|h))+$`
+	RestartWindow string `json:"restartWindow,omitempty"`
+
 	// RestartThreshold is the number of container restarts before action.
 	// +kubebuilder:default=10
 	// +kubebuilder:validation:Minimum=1

charts/crashloop-operator/crds/crashloop-operator.lauger.de_crashlooppolicies.yaml

@@ -196,6 +196,15 @@ spec:
                 format: int32
                 minimum: 1
                 type: integer
+              restartWindow:
+                default: 1h
+                description: |-
+                  RestartWindow bounds how recently the last termination must have
+                  happened for WatchTerminationReasons to match. Without it the check
+                  would act on a lifetime restart counter that never decays, so a
+                  workload that misbehaved last month would still be scaled down.
+                pattern: ^([0-9]+(\.[0-9]+)?(ns|us|ms|s|m|h))+$
+                type: string
               targets:
                 default:
                 - Deployment
@@ -221,6 +230,25 @@ spec:
                 items:
                   type: string
                 type: array
+              watchTerminationReasons:
+                description: |-
+                  WatchTerminationReasons lists container termination reasons to act on,
+                  for containers that restart repeatedly without ever settling into a
+                  watched waiting state. Kubelet forgets its restart backoff once a
+                  container has stayed up long enough, so a container that dies every
+                  fifteen minutes restarts immediately every time and never enters
+                  CrashLoopBackOff, which makes it invisible to WatchReasons.
+
+                  Empty by default, which leaves behaviour unchanged. "OOMKilled" is the
+                  safe value to start with. "Error" also works but is broad: it covers
+                  ordinary crashes, liveness kills and SIGKILL after the grace period
+                  alike.
+
+                  A match additionally requires RestartThreshold to be reached and the
+                  most recent termination to fall inside RestartWindow.
+                items:
+                  type: string
+                type: array
             type: object
           status:
             description: CrashLoopPolicyStatus defines the observed state of CrashLoopPolicy.

config/crd/bases/crashloop-operator.lauger.de_crashlooppolicies.yaml

@@ -196,6 +196,15 @@ spec:
                 format: int32
                 minimum: 1
                 type: integer
+              restartWindow:
+                default: 1h
+                description: |-
+                  RestartWindow bounds how recently the last termination must have
+                  happened for WatchTerminationReasons to match. Without it the check
+                  would act on a lifetime restart counter that never decays, so a
+                  workload that misbehaved last month would still be scaled down.
+                pattern: ^([0-9]+(\.[0-9]+)?(ns|us|ms|s|m|h))+$
+                type: string
               targets:
                 default:
                 - Deployment
@@ -221,6 +230,25 @@ spec:
                 items:
                   type: string
                 type: array
+              watchTerminationReasons:
+                description: |-
+                  WatchTerminationReasons lists container termination reasons to act on,
+                  for containers that restart repeatedly without ever settling into a
+                  watched waiting state. Kubelet forgets its restart backoff once a
+                  container has stayed up long enough, so a container that dies every
+                  fifteen minutes restarts immediately every time and never enters
+                  CrashLoopBackOff, which makes it invisible to WatchReasons.
+
+                  Empty by default, which leaves behaviour unchanged. "OOMKilled" is the
+                  safe value to start with. "Error" also works but is broad: it covers
+                  ordinary crashes, liveness kills and SIGKILL after the grace period
+                  alike.
+
+                  A match additionally requires RestartThreshold to be reached and the
+                  most recent termination to fall inside RestartWindow.
+                items:
+                  type: string
+                type: array
             type: object
           status:
             description: CrashLoopPolicyStatus defines the observed state of CrashLoopPolicy.

internal/controller/constants.go

@@ -27,6 +27,9 @@ var DefaultWatchReasons = []string{
 	"RunContainerError",
 }
 
+// DefaultRestartWindow mirrors the kubebuilder default on spec.restartWindow.
+const DefaultRestartWindow = time.Hour
+
 // DefaultTargets mirrors the kubebuilder default on spec.targets.
 var DefaultTargets = []string{"Deployment", "StatefulSet", "CronJob"}
 

internal/controller/crashlooppolicy_controller.go

@@ -84,6 +84,7 @@ func (r *CrashLoopPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	targets := effectiveTargets(policy)
 	requireAllReplicasFailing := effectiveAllReplicasFailing(policy)
 	dryRun := effectiveDryRun(policy)
+	restartWindow := effectiveRestartWindow(policy)
 
 	// Every policy sees every pod, so overlapping policies have to agree on who
 	// acts. Load the full set once and let the most restrictive matching policy
@@ -105,7 +106,7 @@ func (r *CrashLoopPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 	// Ask the cache only for pods waiting on one of the reasons this policy
 	// watches. Listing every pod in the cluster and discarding the healthy
 	// ones does not scale with cluster size.
-	pods, err := listPodsByWaitingReasons(ctx, r.Client, watchReasons)
+	pods, err := listCandidatePods(ctx, r.Client, watchReasons, effectiveTerminationReasons(policy))
 	if err != nil {
 		logger.Error(err, "failed to list failing pods")
 		return ctrl.Result{}, err
@@ -134,16 +135,19 @@ func (r *CrashLoopPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 			continue
 		}
 
-		// Check if pod has a matching failure reason
-		reason, failing := podHasFailureReason(pod, watchReasons)
-		if !failing {
+		// Check if pod has a matching failure reason. A restart loop already
+		// carries its own restart and recency thresholds, so only the waiting
+		// path is gated on restartThreshold and durationThreshold here.
+		reason, waitingFailing := podHasFailureReason(pod, watchReasons)
+		loopReason, looping := podIsRestartLooping(pod,
+			effectiveTerminationReasons(policy), restartThreshold, restartWindow)
+		if !waitingFailing && !looping {
 			continue
 		}
-
-		// Check thresholds: restart count OR duration
-		restartExceeded := podExceedsRestartThreshold(pod, restartThreshold)
-		durationExceeded := podExceedsDurationThreshold(pod, durationThreshold)
-		if !restartExceeded && !durationExceeded {
+		if !waitingFailing {
+			reason = loopReason
+		} else if !podExceedsRestartThreshold(pod, restartThreshold) &&
+			!podExceedsDurationThreshold(pod, durationThreshold) && !looping {
 			continue
 		}
 
@@ -186,7 +190,7 @@ func (r *CrashLoopPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Requ
 
 		// Check if all replicas are failing (if configured)
 		if requireAllReplicasFailing {
-			allFailing, err := allReplicasFailing(ctx, r.Client, owner, watchReasons)
+			allFailing, err := allReplicasFailing(ctx, r.Client, owner, policy)
 			if err != nil {
 				logger.Error(err, "failed to check all replicas", "workload", key)
 				loopErrors++
@@ -328,6 +332,11 @@ func (r *CrashLoopPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	); err != nil {
 		return err
 	}
+	if err := mgr.GetFieldIndexer().IndexField(
+		context.Background(), &corev1.Pod{}, IndexPodTerminationReason, podTerminationReasons,
+	); err != nil {
+		return err
+	}
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&crashloopv1alpha1.CrashLoopPolicy{}).
@@ -336,15 +345,17 @@ func (r *CrashLoopPolicyReconciler) SetupWithManager(mgr ctrl.Manager) error {
 			handler.EnqueueRequestsFromMapFunc(r.mapPodToPolicy),
 			// Healthy pods vastly outnumber stuck ones and cannot trigger an
 			// action, so filtering them out here keeps the queue quiet.
-			builder.WithPredicates(predicate.NewPredicateFuncs(podHasWaitingContainer)),
+			builder.WithPredicates(predicate.NewPredicateFuncs(podIsInteresting)),
 		).
 		Complete(r)
 }
 
-// podHasWaitingContainer reports whether any container is waiting with a
-// reason, which is the precondition for a pod being interesting to any policy.
-func podHasWaitingContainer(obj client.Object) bool {
-	return len(podWaitingReasons(obj)) > 0
+// podIsInteresting reports whether a pod could matter to any policy: it is
+// either waiting with a reason, or it has restarted with a recorded
+// termination reason. A pod in a slow restart loop is running when observed,
+// so the waiting check alone would drop its events.
+func podIsInteresting(obj client.Object) bool {
+	return len(podWaitingReasons(obj)) > 0 || len(podTerminationReasons(obj)) > 0
 }
 
 // mapPodToPolicy maps a pod event to the CrashLoopPolicy objects that should be reconciled.