Switch regional and management clusters from EKS Auto Mode to OSS Karpenter

Makefile

@@ -121,7 +121,7 @@ terraform-validate: terraform-init ## Check formatting and validate all Terrafor
 
 # Global values (aws_region, environment, cluster_type) are injected by the
 # ApplicationSet at deploy time, so we supply stubs here for linting.
-HELM_LINT_SET := --set global.aws_region=us-east-1 --set global.environment=lint --set global.cluster_type=lint
+HELM_LINT_SET := --set global.aws_region=us-east-1 --set global.environment=lint --set global.cluster_type=lint --set global.cluster_name=lint
 helm-lint: ## Lint all Helm charts
 	@echo "🔍 Linting Helm charts..."
 	@failed=false; \

argocd/config/management-cluster/eks-nodepool/templates/00-nodeclass.yaml

@@ -1,15 +1,18 @@
-apiVersion: eks.amazonaws.com/v1
-kind: NodeClass
+{{- $clusterName := required "global.cluster_name must be set via ApplicationSet valuesObject" .Values.global.cluster_name -}}
+apiVersion: karpenter.k8s.aws/v1
+kind: EC2NodeClass
 metadata:
   name: fips
 spec:
-  role: "{{ .Values.global.cluster_name }}-auto-node-role"
+  amiSelectorTerms:
+    - alias: bottlerocket@latest
+  role: {{ $clusterName }}-karpenter-node-role
   subnetSelectorTerms:
     - tags:
-        "kubernetes.io/cluster/{{ .Values.global.cluster_name }}": owned
+        kubernetes.io/cluster/{{ $clusterName }}: owned
   securityGroupSelectorTerms:
     - tags:
-        aws:eks:cluster-name: "{{ .Values.global.cluster_name }}"
-  advancedSecurity:
-    fips: true
-    kernelLockdown: Integrity
+        aws:eks:cluster-name: {{ $clusterName }}
+  metadataOptions:
+    httpTokens: required
+    httpPutResponseHopLimit: 2

argocd/config/management-cluster/eks-nodepool/templates/10-nodepool.yaml

@@ -6,8 +6,8 @@ spec:
   template:
     spec:
       nodeClassRef:
-        group: eks.amazonaws.com
-        kind: NodeClass
+        group: karpenter.k8s.aws
+        kind: EC2NodeClass
         name: fips
       requirements:
         - key: karpenter.sh/capacity-type

argocd/config/management-cluster/eks-nodepool/values.yaml

@@ -10,3 +10,6 @@ eksNodePool:
   disruption:
     consolidationPolicy: WhenEmpty
     consolidateAfter: 60s
+
+global:
+  cluster_name: ""

argocd/config/management-cluster/hypershift/templates/05-job.yaml

@@ -46,6 +46,7 @@ spec:
         - /bin/sh
         - -c
         - |
+          set -euo pipefail
           mkdir -p /tmp/aws
 
           # Private platform creds — Pod Identity provides MC-account credentials
@@ -62,6 +63,13 @@ spec:
             cp /tmp/aws/private-creds /tmp/aws/oidc-creds
           fi
 
+          # Prometheus Operator CRDs must exist before hypershift install applies
+          # ServiceMonitor/PrometheusRule. The monitoring chart may still be syncing.
+          until kubectl get crd servicemonitors.monitoring.coreos.com prometheusrules.monitoring.coreos.com &>/dev/null 2>&1; do
+            echo "Waiting for Prometheus Operator CRDs..."
+            sleep 10
+          done
+
           hypershift install \
             --namespace hypershift \
             --enable-conversion-webhook=false \
@@ -73,11 +81,13 @@ spec:
             --oidc-storage-provider-s3-bucket-name $(OIDC_BUCKET_NAME) \
             --oidc-storage-provider-s3-region $(OIDC_BUCKET_REGION) \
             --oidc-storage-provider-s3-credentials /tmp/aws/oidc-creds \
+            {{- if .Values.hypershift.externalDns.domain }}
             --external-dns-provider aws \
             --external-dns-domain-filter {{ .Values.hypershift.externalDns.domain }} \
             --external-dns-secret external-dns \
             --external-dns-image {{ .Values.hypershift.externalDns.image }} \
-
+            {{- end }}
+          {{- if .Values.hypershift.externalDns.domain }}
           # TODO(hypershift): Upstream --aws-assume-role + Pod Identity support
           # to hypershift install CLI, then replace this post-install patch
           # with native flags.
@@ -110,5 +120,6 @@ spec:
               ]}' \
               -o /dev/null -w "  HTTP %{http_code}\n"
           fi
+          {{- end }}
       restartPolicy: Never
       serviceAccountName: hypershift-installer

argocd/config/regional-cluster/aws-load-balancer-controller/Chart.yaml

@@ -0,0 +1,10 @@
+apiVersion: v2
+name: aws-load-balancer-controller
+description: AWS Load Balancer Controller — provides TargetGroupBinding CRDs for OSS Karpenter clusters
+type: application
+version: 0.1.0
+
+dependencies:
+  - name: aws-load-balancer-controller
+    version: 1.17.1
+    repository: https://aws.github.io/eks-charts

argocd/config/regional-cluster/aws-load-balancer-controller/values.yaml

@@ -0,0 +1,17 @@
+aws-load-balancer-controller:
+  # clusterName injected via ApplicationSet valuesObject (aws-load-balancer-controller.clusterName)
+  serviceAccount:
+    create: true
+    name: aws-load-balancer-controller
+
+  podSecurityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
+    fsGroup: 65534
+
+  securityContext:
+    allowPrivilegeEscalation: false
+    readOnlyRootFilesystem: true
+    capabilities:
+      drop:
+        - ALL

argocd/config/regional-cluster/eks-nodepool/templates/00-nodeclass.yaml

@@ -1,15 +1,18 @@
-apiVersion: eks.amazonaws.com/v1
-kind: NodeClass
+{{- $clusterName := required "global.cluster_name must be set via ApplicationSet valuesObject" .Values.global.cluster_name -}}
+apiVersion: karpenter.k8s.aws/v1
+kind: EC2NodeClass
 metadata:
   name: fips
 spec:
-  role: "{{ .Values.global.cluster_name }}-auto-node-role"
+  amiSelectorTerms:
+    - alias: bottlerocket@latest
+  role: {{ $clusterName }}-karpenter-node-role
   subnetSelectorTerms:
     - tags:
-        "kubernetes.io/cluster/{{ .Values.global.cluster_name }}": owned
+        kubernetes.io/cluster/{{ $clusterName }}: owned
   securityGroupSelectorTerms:
     - tags:
-        aws:eks:cluster-name: "{{ .Values.global.cluster_name }}"
-  advancedSecurity:
-    fips: true
-    kernelLockdown: Integrity
+        aws:eks:cluster-name: {{ $clusterName }}
+  metadataOptions:
+    httpTokens: required
+    httpPutResponseHopLimit: 2

argocd/config/regional-cluster/eks-nodepool/templates/10-nodepool.yaml

@@ -7,8 +7,8 @@ spec:
   template:
     spec:
       nodeClassRef:
-        group: eks.amazonaws.com
-        kind: NodeClass
+        group: karpenter.k8s.aws
+        kind: EC2NodeClass
         name: fips
       requirements:
         - key: karpenter.sh/capacity-type

argocd/config/regional-cluster/eks-nodepool/values.yaml

@@ -10,3 +10,6 @@ eksNodePool:
   disruption:
     consolidationPolicy: WhenEmpty
     consolidateAfter: 60s
+
+global:
+  cluster_name: ""

argocd/config/regional-cluster/loki/templates/targetgroupbinding.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.platform.distributorTargetGroup.arn }}
-apiVersion: eks.amazonaws.com/v1
+apiVersion: elbv2.k8s.aws/v1beta1
 kind: TargetGroupBinding
 metadata:
   name: loki-distributor
@@ -17,7 +17,7 @@ spec:
 {{- end }}
 ---
 {{- if .Values.platform.queryFrontendTargetGroup.arn }}
-apiVersion: eks.amazonaws.com/v1
+apiVersion: elbv2.k8s.aws/v1beta1
 kind: TargetGroupBinding
 metadata:
   name: loki-query-frontend

argocd/config/regional-cluster/platform-api/templates/targetgroupbinding.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: eks.amazonaws.com/v1
+apiVersion: elbv2.k8s.aws/v1beta1
 kind: TargetGroupBinding
 metadata:
   name: {{ .Values.platformApi.app.name }}

argocd/config/regional-cluster/thanos/templates/targetgroupbinding.yaml

@@ -1,5 +1,5 @@
 {{- if .Values.thanos.targetGroup.arn }}
-apiVersion: eks.amazonaws.com/v1
+apiVersion: elbv2.k8s.aws/v1beta1
 kind: TargetGroupBinding
 metadata:
   name: thanos-receive
@@ -18,7 +18,7 @@ spec:
 {{- end }}
 ---
 {{- if .Values.thanos.queryTargetGroup.arn }}
-apiVersion: eks.amazonaws.com/v1
+apiVersion: elbv2.k8s.aws/v1beta1
 kind: TargetGroupBinding
 metadata:
   name: thanos-query-frontend

argocd/config/shared/argocd/values.yaml

@@ -5,8 +5,21 @@ argo-cd:
   ha:
     enabled: true
 
+  # CriticalAddonsOnly toleration for all ArgoCD components so they can
+  # schedule on the karpenter-bootstrap node group before Karpenter has
+  # provisioned any worker nodes. Harmless on regular nodes.
+  global:
+    tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+        effect: NoSchedule
+
   # Server configuration
   server:
+    tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+        effect: NoSchedule
     autoscaling:
       enabled: true
       minReplicas: 2
@@ -40,6 +53,10 @@ argo-cd:
 
   # Controller configuration
   controller:
+    tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+        effect: NoSchedule
     replicas: 2
     pdb:
       enabled: true
@@ -71,6 +88,10 @@ argo-cd:
 
   # Repo Server configuration
   repoServer:
+    tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+        effect: NoSchedule
     autoscaling:
       enabled: true
       minReplicas: 2
@@ -104,6 +125,10 @@ argo-cd:
 
   # ApplicationSet controller
   applicationSet:
+    tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+        effect: NoSchedule
     resources:
       limits:
         cpu: 250m
@@ -118,6 +143,10 @@ argo-cd:
 
   # Dex configuration
   dex:
+    tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+        effect: NoSchedule
     resources:
       limits:
         cpu: 250m
@@ -128,6 +157,10 @@ argo-cd:
 
   # Notifications controller
   notifications:
+    tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+        effect: NoSchedule
     resources:
       limits:
         cpu: 250m
@@ -136,7 +169,20 @@ argo-cd:
         cpu: 50m
         memory: 64Mi
 
-  # Redis configuration
+  # Redis HA configuration (enabled via ha.enabled=true above).
+  # redis-ha is a subchart and does not inherit global.tolerations.
+  redis-ha:
+    tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+        effect: NoSchedule
+    haproxy:
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+          effect: NoSchedule
+
+  # redis key kept for non-HA mode reference (unused when ha.enabled=true)
   redis:
     resources:
       limits:

argocd/config/shared/storageclass/templates/gp3.yaml

@@ -4,12 +4,7 @@ metadata:
   name: gp3
   annotations:
     storageclass.kubernetes.io/is-default-class: "true"
-allowedTopologies:
-- matchLabelExpressions:
-  - key: eks.amazonaws.com/compute-type
-    values:
-    - auto
-provisioner: ebs.csi.eks.amazonaws.com
+provisioner: ebs.csi.aws.com
 volumeBindingMode: WaitForFirstConsumer
 parameters:
   type: gp3

config/templates/argocd-bootstrap/applicationset.yaml.j2

@@ -53,6 +53,28 @@ spec:
         syncOptions:
         - ServerSideApply=true
         - CreateNamespace=true
+        - RespectIgnoreDifferences=true
+      # Fields managed by controllers post-deployment that should not cause OutOfSync:
+      #   LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls.
+      #   LBC populates TargetGroupBinding status after reconciliation.
+      ignoreDifferences:
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        jqPathExpressions:
+        - .webhooks[].clientConfig.caBundle
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        jqPathExpressions:
+        - .webhooks[].clientConfig.caBundle
+      - group: ""
+        kind: Secret
+        name: aws-load-balancer-tls
+        jqPathExpressions:
+        - .data
+      - group: elbv2.k8s.aws
+        kind: TargetGroupBinding
+        jqPathExpressions:
+        - .status
       sources:
       - helm:
           valueFiles:
@@ -106,6 +128,8 @@ spec:
                   bucketNames:
                     chunks: '{{ '{{ .metadata.labels.cluster_name }}' }}-loki-logs-{{ '{{ .metadata.annotations.aws_account_id }}' }}'
                     ruler: '{{ '{{ .metadata.labels.cluster_name }}' }}-loki-logs-{{ '{{ .metadata.annotations.aws_account_id }}' }}'
+            aws-load-balancer-controller:
+              clusterName: '{{ '{{ .metadata.labels.cluster_name }}' }}'
             hyperfleetApi:
               aws:
                 region: '{{ '{{ .metadata.labels.aws_region }}' }}'

deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml

@@ -52,6 +52,28 @@ spec:
         syncOptions:
         - ServerSideApply=true
         - CreateNamespace=true
+        - RespectIgnoreDifferences=true
+      # Fields managed by controllers post-deployment that should not cause OutOfSync:
+      #   LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls.
+      #   LBC populates TargetGroupBinding status after reconciliation.
+      ignoreDifferences:
+      - group: admissionregistration.k8s.io
+        kind: MutatingWebhookConfiguration
+        jqPathExpressions:
+        - .webhooks[].clientConfig.caBundle
+      - group: admissionregistration.k8s.io
+        kind: ValidatingWebhookConfiguration
+        jqPathExpressions:
+        - .webhooks[].clientConfig.caBundle
+      - group: ""
+        kind: Secret
+        name: aws-load-balancer-tls
+        jqPathExpressions:
+        - .data
+      - group: elbv2.k8s.aws
+        kind: TargetGroupBinding
+        jqPathExpressions:
+        - .status
       sources:
       - helm:
           valueFiles:
@@ -105,6 +127,8 @@ spec:
                   bucketNames:
                     chunks: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}'
                     ruler: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}'
+            aws-load-balancer-controller:
+              clusterName: '{{ .metadata.labels.cluster_name }}'
             hyperfleetApi:
               aws:
                 region: '{{ .metadata.labels.aws_region }}'