From 08972e2dd087c50394db3fe385c85e722e352555 Mon Sep 17 00:00:00 2001 From: Gautam Verma Date: Thu, 6 Aug 2026 09:36:26 -0700 Subject: [PATCH] Fix nsx-ncp liveness probe in OpenShift/RHEL/Ubuntu manifests NCP's liveness probe mechanism changed from an exec-based external script to an in-process httpGet probe served on :8086/healthz. The old exec probe command is no longer supported and now always fails, causing kubelet to restart-loop the nsx-ncp pod on every platform this operator manages. Switch the exec probe to httpGet on 127.0.0.1:8086/healthz (all three pods run hostNetwork: true, so kubelet and NCP share the host network namespace and loopback is reachable), keeping the same initialDelaySeconds/timeoutSeconds/periodSeconds/failureThreshold as before. Deliberately not declaring a containerPort for 8086: on a hostNetwork: true pod, declaring it makes the scheduler track that port as an exclusive per-node reservation. A rolling update needs two new-template replicas to be schedulable before any old replica is removed, and two pods can never simultaneously hold the same exclusive hostPort on one node -- when available nodes drop to or below the replica count, this deadlocks the rollout forever. E2E CI caught this directly: a single-node Kind cluster scaled to 2 replicas got stuck with both new-template pods Pending indefinitely, and the NcpInstall CRD never reported Available. The declaration isn't needed for the probe to work either -- hostNetwork means the process binds :8086 regardless, and kubelet's httpGet probe reaches it via host/port directly, not via the pod's ports field. Co-Authored-By: Claude Sonnet 5 --- manifest/kubernetes/rhel/ncp-rhel.yaml | 13 ++++++++----- manifest/kubernetes/ubuntu/ncp-ubuntu.yaml | 13 ++++++++----- manifest/openshift4/coreos/ncp-openshift4.yaml | 13 ++++++++----- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/manifest/kubernetes/rhel/ncp-rhel.yaml b/manifest/kubernetes/rhel/ncp-rhel.yaml index 9b5fee57..bcd82874 100644 --- a/manifest/kubernetes/rhel/ncp-rhel.yaml +++ b/manifest/kubernetes/rhel/ncp-rhel.yaml @@ -649,12 +649,15 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # liveness.DEFAULT_LIVENESS_TIMEOUT bounds the in-process deep check + # below timeoutSeconds, so a wedged check surfaces as a probe + # failure rather than a kubelet client timeout. livenessProbe: - exec: - command: - - /bin/sh - - -c - - check_pod_liveness nsx-ncp 30 + httpGet: + path: /healthz + port: 8086 + host: 127.0.0.1 + scheme: HTTP initialDelaySeconds: 5 timeoutSeconds: 30 periodSeconds: 10 diff --git a/manifest/kubernetes/ubuntu/ncp-ubuntu.yaml b/manifest/kubernetes/ubuntu/ncp-ubuntu.yaml index 30dd4664..3e14f009 100644 --- a/manifest/kubernetes/ubuntu/ncp-ubuntu.yaml +++ b/manifest/kubernetes/ubuntu/ncp-ubuntu.yaml @@ -649,12 +649,15 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # liveness.DEFAULT_LIVENESS_TIMEOUT bounds the in-process deep check + # below timeoutSeconds, so a wedged check surfaces as a probe + # failure rather than a kubelet client timeout. livenessProbe: - exec: - command: - - /bin/sh - - -c - - check_pod_liveness nsx-ncp 30 + httpGet: + path: /healthz + port: 8086 + host: 127.0.0.1 + scheme: HTTP initialDelaySeconds: 5 timeoutSeconds: 30 periodSeconds: 10 diff --git a/manifest/openshift4/coreos/ncp-openshift4.yaml b/manifest/openshift4/coreos/ncp-openshift4.yaml index 99611a1d..7d190f40 100644 --- a/manifest/openshift4/coreos/ncp-openshift4.yaml +++ b/manifest/openshift4/coreos/ncp-openshift4.yaml @@ -587,12 +587,15 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + # liveness.DEFAULT_LIVENESS_TIMEOUT bounds the in-process deep check + # below timeoutSeconds, so a wedged check surfaces as a probe + # failure rather than a kubelet client timeout. livenessProbe: - exec: - command: - - /bin/sh - - -c - - check_pod_liveness nsx-ncp 30 + httpGet: + path: /healthz + port: 8086 + host: 127.0.0.1 + scheme: HTTP initialDelaySeconds: 5 timeoutSeconds: 30 periodSeconds: 10