From 6701516a1ce11f9d1a9252ab0251834ed32b79f5 Mon Sep 17 00:00:00 2001 From: shobham Date: Wed, 19 Aug 2026 17:04:05 +0530 Subject: [PATCH] feat(self-managed): add highAvailability Helmfile schema and value mapping Expose an opt-in highAvailability profile and map it through global.yaml.gotmpl so Tier-1 replicas/affinity/PDB and Tier-2 sizing can be enabled without editing per-chart values. #985 #986 --- .../self-managed/environments/base.yaml | 70 ++++++++ deploy/stacks/self-managed/global.yaml.gotmpl | 152 +++++++++++++++++- .../self-managed/tests/ha-value-wiring.sh | 125 ++++++++++++++ 3 files changed, 345 insertions(+), 2 deletions(-) create mode 100755 deploy/stacks/self-managed/tests/ha-value-wiring.sh diff --git a/deploy/stacks/self-managed/environments/base.yaml b/deploy/stacks/self-managed/environments/base.yaml index 4b97eb8f9..fc289c8c8 100644 --- a/deploy/stacks/self-managed/environments/base.yaml +++ b/deploy/stacks/self-managed/environments/base.yaml @@ -466,3 +466,73 @@ grpcproxy: # on a control-plane-local API service address. nvcfGrpcServiceURL: "" workerConnectBaseURL: "" + +# ============================================================================= +# Control-plane high availability +# ============================================================================= +# Resilience settings are exposed as a single highAvailability: block in +# self-managed Helmfile environment values. +# deploy/stacks/self-managed/global.yaml.gotmpl maps highAvailability.* onto +# chart values (replicaCount, affinity, topology spread, PDB, update strategy). +# +# deploy/stacks/self-managed/ +# ├── environments/.yaml # highAvailability: configuration +# └── global.yaml.gotmpl # value mapping to charts +# +# Profile Behavior +# production Full P0 defaults; requires ≥3 schedulable nodes +# staging Same shape; soft anti-affinity for smaller pools +# single-node Single replicas; hard anti-affinity disabled (local / CI / BDD) +# +# In-scope charts MUST expose the required value hooks. The local BDD fixture +# SHALL remain on single-node. +# ============================================================================= +highAvailability: + enabled: false + profile: single-node # production | staging | single-node + + tier1: + replicaCount: 2 + podAntiAffinity: + enabled: true + type: preferred + topologySpread: + enabled: true + maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + podDisruptionBudget: + enabled: true + minAvailable: 1 + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + + nats: + replicas: 3 + jetstream: + replicaFactor: 2 + podDisruptionBudget: + enabled: true + merge: + spec: + minAvailable: 2 + + openbao: + ha: + enabled: true + replicas: 3 + injector: + replicas: 2 + + cassandra: + replicaCount: 3 + # Multi-AZ (production two-site): NetworkTopologyStrategy with RF per DC; + # clients use LOCAL_QUORUM so reads/writes do not wait on the remote AZ. + # Single-DC: SimpleStrategy with RF=3 and QUORUM is fine. + replicationFactor: 3 + replicationStrategy: NetworkTopologyStrategy + consistencyLevel: LOCAL_QUORUM + podDisruptionBudget: + enabled: true + minAvailable: 2 diff --git a/deploy/stacks/self-managed/global.yaml.gotmpl b/deploy/stacks/self-managed/global.yaml.gotmpl index d5cfa926d..2b6d5d12a 100644 --- a/deploy/stacks/self-managed/global.yaml.gotmpl +++ b/deploy/stacks/self-managed/global.yaml.gotmpl @@ -24,6 +24,77 @@ tolerations: {{- end -}} {{- end -}} +{{/* +highAvailability helpers. + +global.yaml.gotmpl maps highAvailability.* onto chart values (replicaCount, +affinity, topology spread, PDB, update strategy). + +When highAvailability.enabled is false, callers keep existing env/chart defaults. +When enabled, profile selects effective sizing: + + production — Full P0 defaults; requires ≥3 schedulable nodes + staging — Same shape; soft anti-affinity for smaller pools + single-node — Single replicas; hard anti-affinity disabled (local / CI / BDD) + +In-scope charts MUST expose the required value hooks. The local BDD fixture +SHALL remain on single-node. +*/}} +{{- define "nvcf.ha.isMultiNode" -}} +{{- $enabled := dig "highAvailability" "enabled" false . -}} +{{- $profile := dig "highAvailability" "profile" "single-node" . -}} +{{- if and $enabled (has $profile (list "production" "staging")) -}}true{{- else -}}false{{- end -}} +{{- end -}} + +{{- define "nvcf.ha.tier1ReplicaCount" -}} +{{- if eq (include "nvcf.ha.isMultiNode" .) "true" -}} +{{- dig "highAvailability" "tier1" "replicaCount" 2 . -}} +{{- else -}} +1 +{{- end -}} +{{- end -}} + +{{/* Soft/hard pod anti-affinity on hostname for a Helm release instance name. + Context: dict "Values" $.Values "instance" "" */}} +{{- define "nvcf.ha.tier1Affinity" -}} +{{- if eq (include "nvcf.ha.isMultiNode" .Values) "true" -}} +{{- $paa := dig "highAvailability" "tier1" "podAntiAffinity" dict .Values -}} +{{- if dig "enabled" true $paa -}} +{{- $type := dig "type" "preferred" $paa -}} +affinity: + podAntiAffinity: +{{- if eq $type "required" }} + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/instance + operator: In + values: + - {{ .instance | quote }} + topologyKey: kubernetes.io/hostname +{{- else }} + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/instance + operator: In + values: + - {{ .instance | quote }} + topologyKey: kubernetes.io/hostname +{{- end }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- $haEnabled := dig "highAvailability" "enabled" false .Values }} +{{- $haProfile := dig "highAvailability" "profile" "single-node" .Values }} +{{- if and $haEnabled (not (has $haProfile (list "production" "staging" "single-node"))) }} +{{- fail (printf "highAvailability.profile must be production, staging, or single-node, got %q" $haProfile) }} +{{- end }} +{{- $haMultiNode := eq (include "nvcf.ha.isMultiNode" .Values) "true" }} + cassandra: global: {{- if .Values.global.imagePullSecrets }} @@ -36,13 +107,20 @@ cassandra: defaultStorageClass: {{ .Values.global.storageClass }} {{- end }} - replicaCount: {{ dig "cassandra" "replicaCount" 3 .Values }} + replicaCount: {{ if $haEnabled }}{{ if $haMultiNode }}{{ dig "highAvailability" "cassandra" "replicaCount" 3 .Values }}{{ else }}1{{ end }}{{ else }}{{ dig "cassandra" "replicaCount" 3 .Values }}{{ end }} resourcesPreset: {{ dig "cassandra" "resourcesPreset" "xlarge" .Values }} + {{- if and $haEnabled $haMultiNode }} + {{- with dig "highAvailability" "cassandra" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "cassandra" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} {{- with include "nvcf.nodeSelector" (dict "type" "cassandra" "selectors" .Values.global.nodeSelectors) }} {{- . | nindent 2 }} @@ -140,7 +218,7 @@ openbao: {{- with include "nvcf.tolerations" (dict "type" "vault" "tolerations" .Values.global.tolerations) }} {{- . | nindent 4 }} {{- end }} - replicas: {{ .Values.openbao.injector.replicas }} + replicas: {{ if $haEnabled }}{{ if $haMultiNode }}{{ dig "highAvailability" "openbao" "injector" "replicas" 2 .Values }}{{ else }}1{{ end }}{{ else }}{{ .Values.openbao.injector.replicas }}{{ end }} {{- $nvcfUiEnabled := dig "addons" "nvcfUi" "enabled" false .Values }} {{- with dig "openbao" "injector" "webhook" dict .Values }} webhook: @@ -178,6 +256,10 @@ openbao: size: {{ .Values.global.storageSize | default "10Gi" }} ha: + {{- if $haEnabled }} + enabled: {{ dig "highAvailability" "openbao" "ha" "enabled" true .Values }} + replicas: {{ if $haMultiNode }}{{ dig "highAvailability" "openbao" "ha" "replicas" 3 .Values }}{{ else }}1{{ end }} + {{- end }} {{- with dig "openbao" "server" "ha" "disruptionBudget" dict .Values }} disruptionBudget: {{- toYaml . | nindent 8 }} @@ -228,16 +310,33 @@ nats: {{- if .Values.global.storageClass }} config: + {{- if $haEnabled }} + cluster: + enabled: true + replicas: {{ if $haMultiNode }}{{ dig "highAvailability" "nats" "replicas" 3 .Values }}{{ else }}1{{ end }} + {{- end }} jetstream: fileStore: pvc: storageClassName: {{ .Values.global.storageClass }} + {{- else if $haEnabled }} + config: + cluster: + enabled: true + replicas: {{ if $haMultiNode }}{{ dig "highAvailability" "nats" "replicas" 3 .Values }}{{ else }}1{{ end }} {{- end }} + {{- if and $haEnabled $haMultiNode }} + {{- with dig "highAvailability" "nats" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "nats" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} apikeys: fullnameOverride: api-keys @@ -342,6 +441,9 @@ natsAuthCalloutService: {{- $nvctApiGrpcRouteHostnames := dig "ingress" "gatewayApi" "routes" "nvctApi" "grpc" "hostnames" (list) .Values | default (list (printf "tasks-grpc.%s" .Values.global.domain)) }} api: fullnameOverride: nvcf-api + {{- if $haEnabled }} + replicaCount: {{ include "nvcf.ha.tier1ReplicaCount" .Values }} + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -360,6 +462,9 @@ api: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with include "nvcf.ha.tier1Affinity" (dict "Values" .Values "instance" "api") }} + {{- . | nindent 2 }} + {{- end }} {{- if $llmEnabled }} remoteConfig: configData: @@ -436,6 +541,9 @@ api: invocation: fullnameOverride: invocation-service + {{- if $haEnabled }} + replicaCount: {{ include "nvcf.ha.tier1ReplicaCount" .Values }} + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -449,6 +557,9 @@ invocation: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with include "nvcf.ha.tier1Affinity" (dict "Values" .Values "instance" "invocation-service") }} + {{- . | nindent 2 }} + {{- end }} env: # Observability @@ -465,10 +576,17 @@ invocation: {{- with $invocationWorkerBaseURL }} WORKER_STREAM_PROPERTIES__SELF_ADDRESS: {{ . | quote }} {{- end }} + {{- if and $haEnabled $haMultiNode }} + {{- with dig "highAvailability" "tier1" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "invocation" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} nvctApi: fullnameOverride: nvct-api @@ -510,6 +628,9 @@ nvctApi: grpcproxy: fullnameOverride: grpc-proxy + {{- if $haEnabled }} + replicaCount: {{ include "nvcf.ha.tier1ReplicaCount" .Values }} + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -523,6 +644,13 @@ grpcproxy: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- if $haMultiNode }} + {{- $grpcAffinity := include "nvcf.ha.tier1Affinity" (dict "Values" .Values "instance" "grpc-proxy") }} + {{- if $grpcAffinity }} + deployment: + {{- $grpcAffinity | nindent 4 }} + {{- end }} + {{- end }} {{- with $grpcProxyWorkerConnectBaseURL }} workerConnectBaseURL: {{ . | quote }} {{- end }} @@ -544,10 +672,17 @@ grpcproxy: RATE_LIMIT_ENABLED: "true" RATE_LIMIT_ADDR: "http://ratelimiter.nvcf.svc.cluster.local:7777" {{- end }} + {{- if and $haEnabled $haMultiNode }} + {{- with dig "highAvailability" "tier1" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "grpcproxy" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} rateLimiter: {{- if .Values.global.imagePullSecrets }} @@ -662,6 +797,9 @@ sis: adminIssuerProxy: fullnameOverride: admin-token-issuer-proxy + {{- if $haEnabled }} + replicaCount: {{ include "nvcf.ha.tier1ReplicaCount" .Values }} + {{- end }} {{- if .Values.global.imagePullSecrets }} imagePullSecrets: {{- toYaml .Values.global.imagePullSecrets | nindent 4 }} @@ -675,6 +813,9 @@ adminIssuerProxy: {{- with include "nvcf.tolerations" (dict "type" "controlplane" "tolerations" .Values.global.tolerations) }} {{- . | nindent 2 }} {{- end }} + {{- with include "nvcf.ha.tier1Affinity" (dict "Values" .Values "instance" "admin-token-issuer-proxy") }} + {{- . | nindent 2 }} + {{- end }} gateway: enabled: true namespace: {{ .Values.ingress.gatewayApi.gateways.shared.namespace }} @@ -682,10 +823,17 @@ adminIssuerProxy: name: {{ required "ingress.gatewayApi.gateways.shared.name is required" .Values.ingress.gatewayApi.gateways.shared.name }} hostname: "api-keys.{{ .Values.global.domain }}" path: "/v1/admin/keys" + {{- if and $haEnabled $haMultiNode }} + {{- with dig "highAvailability" "tier1" "podDisruptionBudget" dict .Values }} + podDisruptionBudget: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- else }} {{- with dig "adminIssuerProxy" "podDisruptionBudget" dict .Values }} podDisruptionBudget: {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} stateMetrics: enabled: false diff --git a/deploy/stacks/self-managed/tests/ha-value-wiring.sh b/deploy/stacks/self-managed/tests/ha-value-wiring.sh new file mode 100755 index 000000000..4092fdf18 --- /dev/null +++ b/deploy/stacks/self-managed/tests/ha-value-wiring.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash +# Test that highAvailability values thread from environment files through +# global.yaml.gotmpl into chart values for Tier-1 / Tier-2 releases. +set -euo pipefail + +stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +work_dir="$(mktemp -d)" +test_stack_dir="$work_dir/self-managed" +environment_name="ha-value-wiring-test" +environment_file="$test_stack_dir/environments/$environment_name.yaml" +secrets_file="$test_stack_dir/secrets/$environment_name-secrets.yaml" +trap 'rm -rf "$work_dir"' EXIT + +fail() { + echo "ha-value-wiring: $*" >&2 + exit 1 +} + +mkdir -p "$test_stack_dir" +cp -R "$stack_dir"/. "$test_stack_dir" +printf '{}\n' >"$secrets_file" + +render_chart_values() { + local release="$1" + local output_file="$2" + local helmfile_file="$3" + shift 3 + + # global.yaml.gotmpl evaluates adminIssuerProxy gateway refs for every release. + HELMFILE_ENV="$environment_name" \ + HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \ + helmfile \ + --file "$helmfile_file" \ + --environment default \ + --state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \ + --state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system \ + --state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \ + --state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \ + --selector "name=$release" \ + "$@" \ + write-values \ + --output-file-template "$output_file" +} + +write_env() { + cat >"$environment_file" +} + +deps="$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" +core="$test_stack_dir/helmfile.d/02-core.yaml.gotmpl" + +echo "== highAvailability disabled: chart defaults / base values unchanged ==" +write_env <<'EOF' +highAvailability: + enabled: false +EOF + +render_chart_values api "$work_dir/api-off.yaml" "$core" || fail "render api (ha off)" +# HA must not inject replicaCount into the api values when disabled. +if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "replicaCount:"; then + fail "api: HA replicaCount leaked while highAvailability.enabled=false" +fi +if awk '/^api:/{p=1;next} /^[a-zA-Z]/{p=0} p' "$work_dir/api-off.yaml" | grep -q "podAntiAffinity:"; then + fail "api: HA affinity leaked while highAvailability.enabled=false" +fi + +echo "== highAvailability production: Tier-1 / Tier-2 sizing ==" +write_env <<'EOF' +highAvailability: + enabled: true + profile: production +EOF + +render_chart_values api "$work_dir/api-on.yaml" "$core" || fail "render api (ha on)" +grep -E "replicaCount:[[:space:]]*2" "$work_dir/api-on.yaml" >/dev/null || + fail "api: expected replicaCount 2 for production HA" +grep -q "podAntiAffinity:" "$work_dir/api-on.yaml" || + fail "api: expected podAntiAffinity for production HA" + +render_chart_values cassandra "$work_dir/cassandra-on.yaml" "$deps" || fail "render cassandra (ha on)" +grep -E "replicaCount:[[:space:]]*3" "$work_dir/cassandra-on.yaml" >/dev/null || + fail "cassandra: expected replicaCount 3 for production HA" +grep -A2 "podDisruptionBudget:" "$work_dir/cassandra-on.yaml" | grep -q "enabled: true" || + fail "cassandra: expected HA PDB enabled" + +render_chart_values openbao-server "$work_dir/openbao-on.yaml" "$deps" || fail "render openbao (ha on)" +grep -A5 "^[[:space:]]*ha:" "$work_dir/openbao-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null || + fail "openbao: expected server.ha.replicas 3 for production HA" + +render_chart_values nats "$work_dir/nats-on.yaml" "$deps" || fail "render nats (ha on)" +grep -A5 "cluster:" "$work_dir/nats-on.yaml" | grep -E "replicas:[[:space:]]*3" >/dev/null || + fail "nats: expected config.cluster.replicas 3 for production HA" + +echo "== highAvailability single-node: force replicas=1 ==" +write_env <<'EOF' +highAvailability: + enabled: true + profile: single-node +EOF + +render_chart_values api "$work_dir/api-single.yaml" "$core" || fail "render api (single-node)" +grep -E "replicaCount:[[:space:]]*1" "$work_dir/api-single.yaml" >/dev/null || + fail "api: expected replicaCount 1 for single-node HA profile" +if grep -q "podAntiAffinity:" "$work_dir/api-single.yaml"; then + fail "api: single-node must not set podAntiAffinity" +fi + +render_chart_values cassandra "$work_dir/cassandra-single.yaml" "$deps" || fail "render cassandra (single-node)" +grep -E "replicaCount:[[:space:]]*1" "$work_dir/cassandra-single.yaml" >/dev/null || + fail "cassandra: expected replicaCount 1 for single-node HA profile" + +echo "== invalid profile fails render ==" +write_env <<'EOF' +highAvailability: + enabled: true + profile: nope +EOF +if render_chart_values api "$work_dir/api-bad.yaml" "$core" 2>"$work_dir/api-bad.err"; then + fail "api: invalid highAvailability.profile should fail" +fi +grep -q "highAvailability.profile" "$work_dir/api-bad.err" || + fail "api: expected profile validation error message" + +echo "ha-value-wiring: ok"