From 3de95ed1422281f545574614347aec398c1429f1 Mon Sep 17 00:00:00 2001 From: Brian Smith Date: Mon, 13 Jul 2026 15:14:34 -0400 Subject: [PATCH 1/5] Migrate from EKS Auto Mode to OSS Karpenter Co-Authored-By: Claude Sonnet 4.6 --- Makefile | 2 +- .../eks-nodepool/templates/00-nodeclass.yaml | 19 +- .../eks-nodepool/templates/10-nodepool.yaml | 4 +- .../eks-nodepool/values.yaml | 3 + .../hypershift/templates/05-job.yaml | 13 +- .../aws-load-balancer-controller/Chart.yaml | 10 + .../aws-load-balancer-controller/values.yaml | 17 + .../eks-nodepool/templates/00-nodeclass.yaml | 19 +- .../eks-nodepool/templates/10-nodepool.yaml | 4 +- .../regional-cluster/eks-nodepool/values.yaml | 3 + .../loki/templates/targetgroupbinding.yaml | 4 +- .../templates/targetgroupbinding.yaml | 2 +- .../thanos/templates/targetgroupbinding.yaml | 4 +- argocd/config/shared/argocd/values.yaml | 48 +- .../shared/storageclass/templates/gp3.yaml | 7 +- .../argocd-bootstrap/applicationset.yaml.j2 | 24 + .../applicationset.yaml | 24 + .../applicationset.yaml | 24 + .../applicationset.yaml | 24 + .../applicationset.yaml | 24 + docs/README.md | 2 + docs/design/fips-eks-compute.md | 169 +++---- docs/design/fully-private-eks-bootstrap.md | 2 +- docs/design/gitops-cluster-configuration.md | 8 +- docs/design/karpenter-node-provisioning.md | 98 ++++ docs/design/logging-platform.md | 2 +- docs/design/rate-limiting-architecture.md | 2 +- docs/design/thanos-metrics-infrastructure.md | 3 +- docs/design/zoa-trusted-actions.md | 2 +- scripts/buildspec/register.sh | 10 +- scripts/validate-mc-aws.sh | 347 ++++++++++++++ scripts/validate-mc-k8s.sh | 311 ++++++++++++ scripts/validate-rc-aws.sh | 307 ++++++++++++ scripts/validate-rc-k8s.sh | 351 ++++++++++++++ terraform/config/management-cluster/main.tf | 3 + terraform/config/regional-cluster/imports.sh | 41 +- terraform/config/regional-cluster/main.tf | 15 + .../aws-load-balancer-controller/README.md | 57 +++ .../aws-load-balancer-controller/iam.tf | 333 +++++++++++++ .../aws-load-balancer-controller/main.tf | 9 + .../aws-load-balancer-controller/outputs.tf | 14 + .../aws-load-balancer-controller/variables.tf | 22 + .../aws-load-balancer-controller/versions.tf | 10 + terraform/modules/ecs-bootstrap/README.md | 67 +-- terraform/modules/ecs-bootstrap/main.tf | 253 +++++++--- terraform/modules/ecs-bootstrap/variables.tf | 18 + terraform/modules/eks-cluster/README.md | 88 ++-- terraform/modules/eks-cluster/data.tf | 7 + terraform/modules/eks-cluster/iam.tf | 446 +++++++++++++++++- terraform/modules/eks-cluster/locals.tf | 4 + terraform/modules/eks-cluster/main.tf | 145 +++++- terraform/modules/eks-cluster/outputs.tf | 21 +- terraform/modules/eks-cluster/variables.tf | 16 + terraform/modules/eks-cluster/versions.tf | 4 + .../hyperfleet-infrastructure/amazonmq.tf | 2 +- 55 files changed, 3154 insertions(+), 314 deletions(-) create mode 100644 argocd/config/regional-cluster/aws-load-balancer-controller/Chart.yaml create mode 100644 argocd/config/regional-cluster/aws-load-balancer-controller/values.yaml create mode 100644 docs/design/karpenter-node-provisioning.md create mode 100755 scripts/validate-mc-aws.sh create mode 100755 scripts/validate-mc-k8s.sh create mode 100755 scripts/validate-rc-aws.sh create mode 100755 scripts/validate-rc-k8s.sh create mode 100644 terraform/modules/aws-load-balancer-controller/README.md create mode 100644 terraform/modules/aws-load-balancer-controller/iam.tf create mode 100644 terraform/modules/aws-load-balancer-controller/main.tf create mode 100644 terraform/modules/aws-load-balancer-controller/outputs.tf create mode 100644 terraform/modules/aws-load-balancer-controller/variables.tf create mode 100644 terraform/modules/aws-load-balancer-controller/versions.tf diff --git a/Makefile b/Makefile index 997b70121..798907191 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ terraform-validate: terraform-init ## Check formatting and validate all Terrafor # Global values (aws_region, environment, cluster_type) are injected by the # ApplicationSet at deploy time, so we supply stubs here for linting. -HELM_LINT_SET := --set global.aws_region=us-east-1 --set global.environment=lint --set global.cluster_type=lint +HELM_LINT_SET := --set global.aws_region=us-east-1 --set global.environment=lint --set global.cluster_type=lint --set global.cluster_name=lint helm-lint: ## Lint all Helm charts @echo "🔍 Linting Helm charts..." @failed=false; \ diff --git a/argocd/config/management-cluster/eks-nodepool/templates/00-nodeclass.yaml b/argocd/config/management-cluster/eks-nodepool/templates/00-nodeclass.yaml index c6401bb9e..7d452c796 100644 --- a/argocd/config/management-cluster/eks-nodepool/templates/00-nodeclass.yaml +++ b/argocd/config/management-cluster/eks-nodepool/templates/00-nodeclass.yaml @@ -1,15 +1,18 @@ -apiVersion: eks.amazonaws.com/v1 -kind: NodeClass +{{- $clusterName := required "global.cluster_name must be set via ApplicationSet valuesObject" .Values.global.cluster_name -}} +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass metadata: name: fips spec: - role: "{{ .Values.global.cluster_name }}-auto-node-role" + amiSelectorTerms: + - alias: bottlerocket@latest + role: {{ $clusterName }}-karpenter-node-role subnetSelectorTerms: - tags: - "kubernetes.io/cluster/{{ .Values.global.cluster_name }}": owned + kubernetes.io/cluster/{{ $clusterName }}: owned securityGroupSelectorTerms: - tags: - aws:eks:cluster-name: "{{ .Values.global.cluster_name }}" - advancedSecurity: - fips: true - kernelLockdown: Integrity + aws:eks:cluster-name: {{ $clusterName }} + metadataOptions: + httpTokens: required + httpPutResponseHopLimit: 2 diff --git a/argocd/config/management-cluster/eks-nodepool/templates/10-nodepool.yaml b/argocd/config/management-cluster/eks-nodepool/templates/10-nodepool.yaml index a1425dee6..50e5938c1 100644 --- a/argocd/config/management-cluster/eks-nodepool/templates/10-nodepool.yaml +++ b/argocd/config/management-cluster/eks-nodepool/templates/10-nodepool.yaml @@ -6,8 +6,8 @@ spec: template: spec: nodeClassRef: - group: eks.amazonaws.com - kind: NodeClass + group: karpenter.k8s.aws + kind: EC2NodeClass name: fips requirements: - key: karpenter.sh/capacity-type diff --git a/argocd/config/management-cluster/eks-nodepool/values.yaml b/argocd/config/management-cluster/eks-nodepool/values.yaml index 654d5a9ed..6acb99b3c 100644 --- a/argocd/config/management-cluster/eks-nodepool/values.yaml +++ b/argocd/config/management-cluster/eks-nodepool/values.yaml @@ -10,3 +10,6 @@ eksNodePool: disruption: consolidationPolicy: WhenEmpty consolidateAfter: 60s + +global: + cluster_name: "" diff --git a/argocd/config/management-cluster/hypershift/templates/05-job.yaml b/argocd/config/management-cluster/hypershift/templates/05-job.yaml index cc1f77b10..19d151979 100644 --- a/argocd/config/management-cluster/hypershift/templates/05-job.yaml +++ b/argocd/config/management-cluster/hypershift/templates/05-job.yaml @@ -46,6 +46,7 @@ spec: - /bin/sh - -c - | + set -euo pipefail mkdir -p /tmp/aws # Private platform creds — Pod Identity provides MC-account credentials @@ -62,6 +63,13 @@ spec: cp /tmp/aws/private-creds /tmp/aws/oidc-creds fi + # Prometheus Operator CRDs must exist before hypershift install applies + # ServiceMonitor/PrometheusRule. The monitoring chart may still be syncing. + until kubectl get crd servicemonitors.monitoring.coreos.com prometheusrules.monitoring.coreos.com &>/dev/null 2>&1; do + echo "Waiting for Prometheus Operator CRDs..." + sleep 10 + done + hypershift install \ --namespace hypershift \ --enable-conversion-webhook=false \ @@ -73,11 +81,13 @@ spec: --oidc-storage-provider-s3-bucket-name $(OIDC_BUCKET_NAME) \ --oidc-storage-provider-s3-region $(OIDC_BUCKET_REGION) \ --oidc-storage-provider-s3-credentials /tmp/aws/oidc-creds \ + {{- if .Values.hypershift.externalDns.domain }} --external-dns-provider aws \ --external-dns-domain-filter {{ .Values.hypershift.externalDns.domain }} \ --external-dns-secret external-dns \ --external-dns-image {{ .Values.hypershift.externalDns.image }} \ - + {{- end }} + {{- if .Values.hypershift.externalDns.domain }} # TODO(hypershift): Upstream --aws-assume-role + Pod Identity support # to hypershift install CLI, then replace this post-install patch # with native flags. @@ -110,5 +120,6 @@ spec: ]}' \ -o /dev/null -w " HTTP %{http_code}\n" fi + {{- end }} restartPolicy: Never serviceAccountName: hypershift-installer diff --git a/argocd/config/regional-cluster/aws-load-balancer-controller/Chart.yaml b/argocd/config/regional-cluster/aws-load-balancer-controller/Chart.yaml new file mode 100644 index 000000000..6b0902088 --- /dev/null +++ b/argocd/config/regional-cluster/aws-load-balancer-controller/Chart.yaml @@ -0,0 +1,10 @@ +apiVersion: v2 +name: aws-load-balancer-controller +description: AWS Load Balancer Controller — provides TargetGroupBinding CRDs for OSS Karpenter clusters +type: application +version: 0.1.0 + +dependencies: + - name: aws-load-balancer-controller + version: 1.17.1 + repository: https://aws.github.io/eks-charts diff --git a/argocd/config/regional-cluster/aws-load-balancer-controller/values.yaml b/argocd/config/regional-cluster/aws-load-balancer-controller/values.yaml new file mode 100644 index 000000000..2ca2a4455 --- /dev/null +++ b/argocd/config/regional-cluster/aws-load-balancer-controller/values.yaml @@ -0,0 +1,17 @@ +aws-load-balancer-controller: + # clusterName injected via ApplicationSet valuesObject (aws-load-balancer-controller.clusterName) + serviceAccount: + create: true + name: aws-load-balancer-controller + + podSecurityContext: + runAsNonRoot: true + runAsUser: 65534 + fsGroup: 65534 + + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL diff --git a/argocd/config/regional-cluster/eks-nodepool/templates/00-nodeclass.yaml b/argocd/config/regional-cluster/eks-nodepool/templates/00-nodeclass.yaml index c6401bb9e..7d452c796 100644 --- a/argocd/config/regional-cluster/eks-nodepool/templates/00-nodeclass.yaml +++ b/argocd/config/regional-cluster/eks-nodepool/templates/00-nodeclass.yaml @@ -1,15 +1,18 @@ -apiVersion: eks.amazonaws.com/v1 -kind: NodeClass +{{- $clusterName := required "global.cluster_name must be set via ApplicationSet valuesObject" .Values.global.cluster_name -}} +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass metadata: name: fips spec: - role: "{{ .Values.global.cluster_name }}-auto-node-role" + amiSelectorTerms: + - alias: bottlerocket@latest + role: {{ $clusterName }}-karpenter-node-role subnetSelectorTerms: - tags: - "kubernetes.io/cluster/{{ .Values.global.cluster_name }}": owned + kubernetes.io/cluster/{{ $clusterName }}: owned securityGroupSelectorTerms: - tags: - aws:eks:cluster-name: "{{ .Values.global.cluster_name }}" - advancedSecurity: - fips: true - kernelLockdown: Integrity + aws:eks:cluster-name: {{ $clusterName }} + metadataOptions: + httpTokens: required + httpPutResponseHopLimit: 2 diff --git a/argocd/config/regional-cluster/eks-nodepool/templates/10-nodepool.yaml b/argocd/config/regional-cluster/eks-nodepool/templates/10-nodepool.yaml index e04546d09..fd386d4b1 100644 --- a/argocd/config/regional-cluster/eks-nodepool/templates/10-nodepool.yaml +++ b/argocd/config/regional-cluster/eks-nodepool/templates/10-nodepool.yaml @@ -7,8 +7,8 @@ spec: template: spec: nodeClassRef: - group: eks.amazonaws.com - kind: NodeClass + group: karpenter.k8s.aws + kind: EC2NodeClass name: fips requirements: - key: karpenter.sh/capacity-type diff --git a/argocd/config/regional-cluster/eks-nodepool/values.yaml b/argocd/config/regional-cluster/eks-nodepool/values.yaml index 654d5a9ed..6acb99b3c 100644 --- a/argocd/config/regional-cluster/eks-nodepool/values.yaml +++ b/argocd/config/regional-cluster/eks-nodepool/values.yaml @@ -10,3 +10,6 @@ eksNodePool: disruption: consolidationPolicy: WhenEmpty consolidateAfter: 60s + +global: + cluster_name: "" diff --git a/argocd/config/regional-cluster/loki/templates/targetgroupbinding.yaml b/argocd/config/regional-cluster/loki/templates/targetgroupbinding.yaml index 331a5a534..9f8102275 100644 --- a/argocd/config/regional-cluster/loki/templates/targetgroupbinding.yaml +++ b/argocd/config/regional-cluster/loki/templates/targetgroupbinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.platform.distributorTargetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1beta1 kind: TargetGroupBinding metadata: name: loki-distributor @@ -17,7 +17,7 @@ spec: {{- end }} --- {{- if .Values.platform.queryFrontendTargetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1beta1 kind: TargetGroupBinding metadata: name: loki-query-frontend diff --git a/argocd/config/regional-cluster/platform-api/templates/targetgroupbinding.yaml b/argocd/config/regional-cluster/platform-api/templates/targetgroupbinding.yaml index 1b76cbf6a..e0f790d1c 100644 --- a/argocd/config/regional-cluster/platform-api/templates/targetgroupbinding.yaml +++ b/argocd/config/regional-cluster/platform-api/templates/targetgroupbinding.yaml @@ -1,5 +1,5 @@ --- -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1beta1 kind: TargetGroupBinding metadata: name: {{ .Values.platformApi.app.name }} diff --git a/argocd/config/regional-cluster/thanos/templates/targetgroupbinding.yaml b/argocd/config/regional-cluster/thanos/templates/targetgroupbinding.yaml index 9d817d162..16c25e5f5 100644 --- a/argocd/config/regional-cluster/thanos/templates/targetgroupbinding.yaml +++ b/argocd/config/regional-cluster/thanos/templates/targetgroupbinding.yaml @@ -1,5 +1,5 @@ {{- if .Values.thanos.targetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1beta1 kind: TargetGroupBinding metadata: name: thanos-receive @@ -18,7 +18,7 @@ spec: {{- end }} --- {{- if .Values.thanos.queryTargetGroup.arn }} -apiVersion: eks.amazonaws.com/v1 +apiVersion: elbv2.k8s.aws/v1beta1 kind: TargetGroupBinding metadata: name: thanos-query-frontend diff --git a/argocd/config/shared/argocd/values.yaml b/argocd/config/shared/argocd/values.yaml index 2d93dd57e..998b4f459 100644 --- a/argocd/config/shared/argocd/values.yaml +++ b/argocd/config/shared/argocd/values.yaml @@ -5,8 +5,21 @@ argo-cd: ha: enabled: true + # CriticalAddonsOnly toleration for all ArgoCD components so they can + # schedule on the karpenter-bootstrap node group before Karpenter has + # provisioned any worker nodes. Harmless on regular nodes. + global: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule + # Server configuration server: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule autoscaling: enabled: true minReplicas: 2 @@ -40,6 +53,10 @@ argo-cd: # Controller configuration controller: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule replicas: 2 pdb: enabled: true @@ -71,6 +88,10 @@ argo-cd: # Repo Server configuration repoServer: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule autoscaling: enabled: true minReplicas: 2 @@ -104,6 +125,10 @@ argo-cd: # ApplicationSet controller applicationSet: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule resources: limits: cpu: 250m @@ -118,6 +143,10 @@ argo-cd: # Dex configuration dex: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule resources: limits: cpu: 250m @@ -128,6 +157,10 @@ argo-cd: # Notifications controller notifications: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule resources: limits: cpu: 250m @@ -136,7 +169,20 @@ argo-cd: cpu: 50m memory: 64Mi - # Redis configuration + # Redis HA configuration (enabled via ha.enabled=true above). + # redis-ha is a subchart and does not inherit global.tolerations. + redis-ha: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule + haproxy: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule + + # redis key kept for non-HA mode reference (unused when ha.enabled=true) redis: resources: limits: diff --git a/argocd/config/shared/storageclass/templates/gp3.yaml b/argocd/config/shared/storageclass/templates/gp3.yaml index bfcdf67aa..492a7fe4a 100644 --- a/argocd/config/shared/storageclass/templates/gp3.yaml +++ b/argocd/config/shared/storageclass/templates/gp3.yaml @@ -4,12 +4,7 @@ metadata: name: gp3 annotations: storageclass.kubernetes.io/is-default-class: "true" -allowedTopologies: -- matchLabelExpressions: - - key: eks.amazonaws.com/compute-type - values: - - auto -provisioner: ebs.csi.eks.amazonaws.com +provisioner: ebs.csi.aws.com volumeBindingMode: WaitForFirstConsumer parameters: type: gp3 diff --git a/config/templates/argocd-bootstrap/applicationset.yaml.j2 b/config/templates/argocd-bootstrap/applicationset.yaml.j2 index a33d735aa..34736d416 100644 --- a/config/templates/argocd-bootstrap/applicationset.yaml.j2 +++ b/config/templates/argocd-bootstrap/applicationset.yaml.j2 @@ -53,6 +53,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -106,6 +128,8 @@ spec: bucketNames: chunks: '{{ '{{ .metadata.labels.cluster_name }}' }}-loki-logs-{{ '{{ .metadata.annotations.aws_account_id }}' }}' ruler: '{{ '{{ .metadata.labels.cluster_name }}' }}-loki-logs-{{ '{{ .metadata.annotations.aws_account_id }}' }}' + aws-load-balancer-controller: + clusterName: '{{ '{{ .metadata.labels.cluster_name }}' }}' hyperfleetApi: aws: region: '{{ '{{ .metadata.labels.aws_region }}' }}' diff --git a/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml b/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml index 0899673e8..22c537897 100644 --- a/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml +++ b/deploy/ephemeral/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml @@ -52,6 +52,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -105,6 +127,8 @@ spec: bucketNames: chunks: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}' ruler: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}' + aws-load-balancer-controller: + clusterName: '{{ .metadata.labels.cluster_name }}' hyperfleetApi: aws: region: '{{ .metadata.labels.aws_region }}' diff --git a/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml b/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml index 8ab03a7be..5c0bd71c0 100644 --- a/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml +++ b/deploy/ephemeral/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml @@ -52,6 +52,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -105,6 +127,8 @@ spec: bucketNames: chunks: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}' ruler: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}' + aws-load-balancer-controller: + clusterName: '{{ .metadata.labels.cluster_name }}' hyperfleetApi: aws: region: '{{ .metadata.labels.aws_region }}' diff --git a/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml b/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml index c4d7c16a5..79a100404 100644 --- a/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml +++ b/deploy/integration/us-east-1/argocd-bootstrap-management-cluster/applicationset.yaml @@ -52,6 +52,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -105,6 +127,8 @@ spec: bucketNames: chunks: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}' ruler: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}' + aws-load-balancer-controller: + clusterName: '{{ .metadata.labels.cluster_name }}' hyperfleetApi: aws: region: '{{ .metadata.labels.aws_region }}' diff --git a/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml b/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml index 50b2adec9..271022621 100644 --- a/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml +++ b/deploy/integration/us-east-1/argocd-bootstrap-regional-cluster/applicationset.yaml @@ -52,6 +52,28 @@ spec: syncOptions: - ServerSideApply=true - CreateNamespace=true + - RespectIgnoreDifferences=true + # Fields managed by controllers post-deployment that should not cause OutOfSync: + # LBC rotates its own webhook TLS cert into caBundle and aws-load-balancer-tls. + # LBC populates TargetGroupBinding status after reconciliation. + ignoreDifferences: + - group: admissionregistration.k8s.io + kind: MutatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: admissionregistration.k8s.io + kind: ValidatingWebhookConfiguration + jqPathExpressions: + - .webhooks[].clientConfig.caBundle + - group: "" + kind: Secret + name: aws-load-balancer-tls + jqPathExpressions: + - .data + - group: elbv2.k8s.aws + kind: TargetGroupBinding + jqPathExpressions: + - .status sources: - helm: valueFiles: @@ -105,6 +127,8 @@ spec: bucketNames: chunks: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}' ruler: '{{ .metadata.labels.cluster_name }}-loki-logs-{{ .metadata.annotations.aws_account_id }}' + aws-load-balancer-controller: + clusterName: '{{ .metadata.labels.cluster_name }}' hyperfleetApi: aws: region: '{{ .metadata.labels.aws_region }}' diff --git a/docs/README.md b/docs/README.md index 3accaa9cd..ecb2483c6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,6 +25,7 @@ Detailed architecture and rationale for key technical decisions: | [DNS Architecture](design/dns-architecture.md) | Hierarchical DNS with zone shards, `deployment_name`, DNSSEC chain | | [ECS Fargate Bootstrap](design/fully-private-eks-bootstrap.md) | How fully private EKS clusters are bootstrapped via ECS | | [FIPS-Only EKS Compute](design/fips-eks-compute.md) | FIPS NodeClass/NodePool strategy for FedRAMP workload nodes | +| [Karpenter Node Provisioning](design/karpenter-node-provisioning.md) | OSS Karpenter IAM roles, IRSA rationale, SQS interruption handling | | [GitOps Cluster Configuration](design/gitops-cluster-configuration.md) | ApplicationSet pattern, progressive deployment, config modes | | [Infrastructure Logging](design/infrastructure-logging.md) | AWS CloudWatch log groups, KMS encryption, Grafana access | | [Logging Platform](design/logging-platform.md) | Application-level log collection (Vector + Loki) | @@ -72,6 +73,7 @@ Each module has its own README with usage, inputs, outputs, and architecture: - [`maestro-agent`](../terraform/modules/maestro-agent/README.md) - IAM and Pod Identity for Maestro Agent - [`grafana-cloudwatch-logs`](../terraform/modules/grafana-cloudwatch-logs/) - IAM + Pod Identity for Grafana CloudWatch Logs datasources (RC primary + MC reader) - [`hyperfleet-infrastructure`](../terraform/modules/hyperfleet-infrastructure/README.md) - RDS, Amazon MQ, IAM for HyperFleet (CLM) +- [`aws-load-balancer-controller`](../terraform/modules/aws-load-balancer-controller/README.md) - IAM role and Pod Identity association for AWS Load Balancer Controller ### ArgoCD Helm Chart Documentation diff --git a/docs/design/fips-eks-compute.md b/docs/design/fips-eks-compute.md index c56622208..3de6d9bf7 100644 --- a/docs/design/fips-eks-compute.md +++ b/docs/design/fips-eks-compute.md @@ -1,14 +1,14 @@ -# FIPS-Only Compute for EKS Auto Mode +# FIPS-Only Compute for EKS Clusters -**Last Updated Date**: 2026-05-15 +**Last Updated Date**: 2026-07-08 ## Summary -All EKS clusters in the ROSA HyperFleet use a custom Karpenter NodePool referencing a -FIPS-validated NodeClass for platform and application workloads. The built-in EKS Auto Mode -`system` node pool is retained to provide nodes for CoreDNS and metrics-server. The built-in -`general-purpose` pool is disabled so that non-system workloads land on the FIPS NodePool rather -than on non-FIPS nodes. +All EKS clusters in the ROSA HyperFleet use OSS Karpenter v1 with a FIPS-validated +`EC2NodeClass` (`fips`) and a cluster-type-specific `NodePool` for platform and application +workloads. A dedicated `karpenter-bootstrap` managed node group (t3.medium, 2 nodes, tainted +`CriticalAddonsOnly`) provides stable capacity for Karpenter itself, CoreDNS, and metrics-server. +All other workloads land on Karpenter-provisioned nodes using FIPS Bottlerocket AMIs. ## Context @@ -17,133 +17,118 @@ FIPS 140-3 validated modules. On EKS, this means workload compute must run a FIP operating system — specifically Bottlerocket with FIPS mode enabled. - **Problem Statement**: EKS Auto Mode's built-in node pools (`system` and `general-purpose`) - provision standard (non-FIPS) Bottlerocket AMIs. Disabling both pools with `node_pools = []` - creates a bootstrap deadlock: EKS Auto Mode's embedded Karpenter is reactive — it only creates - nodes in response to unschedulable pods. With no built-in pools, CoreDNS and metrics-server - remain Pending until a FIPS NodePool exists, but the FIPS NodeClass's `InstanceProfileReady` - condition is evaluated only at creation time and depends on `node_role_arn` being set in the - cluster's `compute_config`. Any mismatch between cluster configuration and NodeClass creation - order results in `UnauthorizedNodeRole`, permanently blocking node provisioning. + provision standard (non-FIPS) Bottlerocket AMIs and cannot be patched to use a custom + `EC2NodeClass`. AWS auto-reverts any modifications to built-in pools within minutes. The + bootstrap deadlock caused by disabling all pools (`node_pools = []`) and repeated + `UnauthorizedNodeRole` failures with the embedded Karpenter made Auto Mode operationally + fragile for FIPS requirements. - **Constraints**: - - EKS Auto Mode's built-in node pools cannot be patched to reference a custom NodeClass. AWS - auto-reverts any modifications. - - EKS Auto Mode manages CoreDNS and metrics-server scheduling constraints; they are not - user-configurable. + - EKS nodes providing FIPS-validated compute must run Bottlerocket with `advancedSecurity.fips: true` - The cluster bootstrap runs inside an ECS Fargate task in a private subnet with no public cluster API access. See [ECS Fargate Bootstrap for Fully Private EKS Clusters](./fully-private-eks-bootstrap.md). -- **Assumptions**: EKS Auto Mode is retained for operational simplicity (managed control plane, - embedded Karpenter, automatic node lifecycle management). Self-managed Karpenter is not a - preferred alternative. + - Karpenter controller must run on stable, pre-provisioned nodes — it cannot schedule itself +- **Assumptions**: All clusters run OSS Karpenter (`enable_karpenter = true`, the default). EKS + Auto Mode is disabled. ## Alternatives Considered -1. **Keep both built-in pools enabled (`system` + `general-purpose`)**: CoreDNS and metrics-server - provision naturally; all workloads land on non-FIPS nodes. Violates the FedRAMP cryptographic - module requirement for platform workloads. Rejected. +1. **EKS Auto Mode with `system` pool + custom FIPS NodePool**: Retains the built-in `system` pool + for CoreDNS and metrics-server (non-FIPS, AWS-managed), adds a custom FIPS NodePool for + workloads. Partially FIPS-compliant but requires the embedded Karpenter's bootstrap ordering + constraints. Replaced because Auto Mode's embedded Karpenter cannot be independently upgraded + and the `node_role_arn` / `InstanceProfileReady` sequencing caused repeated bootstrap failures. -2. **Disable all built-in pools (`node_pools = []`)**: All nodes come from custom FIPS NodePools. - Creates a bootstrap deadlock: the FIPS NodeClass `InstanceProfileReady` condition is only - evaluated at NodeClass creation time. If `node_role_arn` is absent at cluster creation (even - temporarily), the NodeClass is permanently stuck with `UnauthorizedNodeRole` and no nodes can - be provisioned. Operationally fragile. Rejected. +2. **Disable all Auto Mode pools (`node_pools = []`)**: All nodes from custom FIPS NodePools. + Creates a bootstrap deadlock: the FIPS `EC2NodeClass` `InstanceProfileReady` condition is + evaluated only at creation time. If `node_role_arn` is absent at cluster creation, the + NodeClass is permanently stuck with `UnauthorizedNodeRole`. Operationally fragile. Rejected. -3. **Patch built-in node pools to use a FIPS NodeClass**: AWS Auto Mode auto-reverts user - modifications to built-in pools within minutes. Not durable. Rejected. +3. **Keep EKS Auto Mode, patch built-in pools**: AWS auto-reverts user modifications to built-in + pools. Not durable. Rejected. -4. **Replace EKS Auto Mode with self-managed Karpenter**: Achieves FIPS compliance but loses Auto - Mode's managed node lifecycle and unified support. Significantly increases operational - complexity. Rejected. - -5. **Retain `system` pool, disable `general-purpose`, add FIPS workloads NodePool**: The built-in - `system` pool provides nodes for CoreDNS and metrics-server immediately at cluster creation, - avoiding the bootstrap deadlock. The `general-purpose` pool is disabled so workloads cannot - land on non-FIPS nodes. A custom FIPS `*-workloads` NodePool handles all platform and - application workloads. **Chosen.** +4. **OSS Karpenter with dedicated bootstrap node group**: Provides a stable, pre-provisioned node + group (tainted `CriticalAddonsOnly`) for Karpenter controller, CoreDNS, and metrics-server. + Karpenter provisions all other nodes on demand using FIPS `EC2NodeClass`. Fully + FIPS-compliant for customer-bearing workloads. **Chosen.** ## Design Rationale -- **Justification**: Retaining the built-in `system` pool eliminates the bootstrap deadlock - entirely — CoreDNS and metrics-server are Active before the ECS bootstrap task runs. Disabling - `general-purpose` ensures that pods without explicit FIPS node selectors are not silently - scheduled on non-FIPS nodes. All platform and application workloads land on the FIPS NodePool. - -- **Evidence**: The `InstanceProfileReady` condition on a custom NodeClass is evaluated only at - NodeClass creation time, not when cluster configuration changes. This makes the `node_pools = []` - pattern operationally fragile: it requires that `node_role_arn` be set in `compute_config` before - the NodeClass is first applied, and there is no mechanism to re-trigger evaluation on an existing - NodeClass. Retaining the `system` pool removes this dependency entirely. +- **Justification**: The `karpenter-bootstrap` managed node group (t3.medium, 2 nodes, + `CriticalAddonsOnly` taint) provides stable, pre-provisioned capacity for Karpenter itself and + EKS system addons. This eliminates the bootstrap chicken-and-egg problem: Karpenter is installed + first (on the bootstrap node group), then the FIPS `EC2NodeClass` and `NodePool` are applied, + then a prewarm pod validates EC2 provisioning before ArgoCD is installed. -- **Tradeoff**: Nodes provisioned by the built-in `system` pool run non-FIPS Bottlerocket AMIs. - These nodes host only EKS-managed system addons (CoreDNS, metrics-server). Platform and - application workloads run exclusively on the FIPS NodePool. This is an accepted scope boundary: - EKS-managed system infrastructure vs. customer-bearing workloads. +- **Evidence**: The prewarm step in the ECS bootstrap task explicitly provisions one Karpenter node + and waits up to 8 minutes for it to become Ready before proceeding to ArgoCD installation. This + surfaces EC2 API rate limiting and IAM role sequencing issues as early ECS task failures + (retried automatically by ECS) rather than silent cascades post-ArgoCD. -- **Comparison**: Alternative 2 (`node_pools = []`) is the theoretically cleanest approach but - introduces bootstrap fragility that caused repeated failures in practice. Alternative 5 (chosen) - achieves the same workload-level FIPS compliance with a simpler, more robust bootstrap. +- **Tradeoff**: The `karpenter-bootstrap` node group runs non-FIPS standard Amazon Linux 2 nodes. + These nodes host only Karpenter controller, CoreDNS, and metrics-server — EKS system + infrastructure, not customer-bearing workloads. Platform and application workloads run + exclusively on FIPS Karpenter-provisioned nodes. This scope boundary is an accepted tradeoff + for operational reliability. ## Consequences ### Positive -- Platform and application workloads run on Bottlerocket with FIPS-validated cryptographic modules, - satisfying FedRAMP High/Moderate cryptographic requirements for customer-bearing compute. -- Bootstrap is reliable: the built-in `system` pool provisions nodes and brings up CoreDNS and - metrics-server automatically. The bootstrap task applies the FIPS NodeClass and workloads - NodePool, waits for both addons to become Active, then installs ArgoCD — no custom node-wait - loop or addon creation required. -- The FIPS NodeClass and workloads NodePool are applied by the ECS bootstrap task and subsequently +- Platform and application workloads run on Bottlerocket with FIPS-validated cryptographic + modules, satisfying FedRAMP High/Moderate cryptographic requirements for customer-bearing + compute. +- Bootstrap is reliable: the bootstrap node group provisions nodes immediately, Karpenter installs + cleanly, and the prewarm pod validates EC2 provisioning before ArgoCD is involved. +- OSS Karpenter can be independently upgraded via Helm without waiting for AWS EKS Auto Mode + support cycles. +- The FIPS `EC2NodeClass` and `NodePool` are applied by the ECS bootstrap task and subsequently adopted by ArgoCD on first sync, making them GitOps-managed. -- Disabling `general-purpose` prevents accidental scheduling of platform workloads on non-FIPS - nodes. ### Negative -- EKS-managed system addons (CoreDNS, metrics-server) run on non-FIPS `system` pool nodes. These - nodes are AWS-managed infrastructure, not customer-bearing workloads, but they are not - FIPS-validated. -- EKS Auto Mode enforces a mandatory 21-day maximum node lifetime. Stateful workloads (Thanos, - Grafana) must have `PodDisruptionBudgets` to prevent data loss during node rotation. -- Adding a new cluster type requires updating the bootstrap script's `NODEPOOL_NAME` selection - logic. +- Karpenter controller, CoreDNS, and metrics-server run on non-FIPS t3.medium nodes. These are + AWS-managed system addons, not customer-bearing workloads, but they are not FIPS-validated. +- Two IAM roles are required: `karpenter-node-role` (for Karpenter-provisioned nodes) and a + lightweight role for the `karpenter-bootstrap` node group (not used directly by workloads). ## Cross-Cutting Concerns ### Reliability -- **Scalability**: The FIPS `*-workloads` NodePool is large (64 CPU / 256 GiB) and handles all - platform and application workloads. The built-in `system` pool is AWS-managed and scales - automatically for CoreDNS and metrics-server. +- **Scalability**: The FIPS `NodePool` handles all platform and application workloads. Karpenter + scales reactively on pending pods using EC2 instance provisioning. - **Observability**: Karpenter NodeClaims are visible via `kubectl get nodeclaims`. CloudWatch - logs for the ECS bootstrap task provide a full audit trail. -- **Resiliency**: The 21-day mandatory rotation means PodDisruptionBudgets are load-bearing for - stateful workloads — their absence is a reliability risk, not just a compliance gap. + logs for the ECS bootstrap task provide a full audit trail including the prewarm validation. +- **Resiliency**: The `karpenter-bootstrap` node group is a fixed-size managed node group (2 + nodes); AWS manages availability. Karpenter nodes are ephemeral and replaced automatically. ### Security - Platform and application workload nodes run with `advancedSecurity.fips: true` and `kernelLockdown: Integrity`, satisfying FIPS 140-2/140-3 requirements for SC-13. -- The FIPS NodeClass selects subnets and security groups via cluster-owned tags, ensuring nodes - land in the correct private subnets with correct network policies. -- Node IAM role (`${cluster_id}-auto-node-role`) is referenced directly in the NodeClass, scoping - node permissions to a cluster-specific role. +- The FIPS `EC2NodeClass` selects subnets and security groups via cluster-owned tags, ensuring + nodes land in the correct private subnets with correct network policies. +- Karpenter controller IAM role uses IRSA (ServiceAccount annotation on `kube-system/karpenter`) + with least-privilege SQS, EC2, and IAM instance profile permissions. +- Karpenter node IAM role (`${cluster_id}-karpenter-node-role`) is referenced directly in the + `EC2NodeClass`, scoping node permissions to a cluster-specific role. ### Performance - FIPS-mode Bottlerocket has negligible performance overhead for general-purpose workloads. -- `consolidateAfter: 60s` on the workloads NodePool enables rapid scale-down of idle capacity. +- `consolidateAfter: 60s` on the workloads `NodePool` enables rapid scale-down of idle capacity. ### Cost -- One custom NodePool adds no direct AWS cost. Node costs are identical: on-demand EC2 instances - running Bottlerocket. -- `WhenEmpty` consolidation reclaims idle capacity promptly, reducing EC2 spend. +- The `karpenter-bootstrap` node group (2x t3.medium) runs continuously. Karpenter workload nodes + are on-demand EC2 instances provisioned reactively. +- `WhenEmpty` consolidation reclaims idle Karpenter capacity promptly, reducing EC2 spend. ### Operability -- The FIPS NodeClass and workloads NodePool are created by the ECS bootstrap task on first run and +- The FIPS `EC2NodeClass` and `NodePool` are created by the ECS bootstrap task on first run and subsequently managed by ArgoCD. Day-2 changes are made via GitOps — no manual `kubectl apply`. -- The 21-day mandatory rotation is fully automatic. Operators need only ensure PodDisruptionBudgets - exist for stateful workloads. -- Cluster type-specific NodePool naming (`management-workloads` vs `regional-workloads`) is - resolved at bootstrap time via the `CLUSTER_TYPE` environment variable. +- Adding a new cluster type requires adding an `eks-nodepool` Helm chart entry for the new + `cluster_type`, which the bootstrap task selects via the `CLUSTER_TYPE` environment variable. +- EC2 interruption events (spot reclamation, instance retirement) are handled by Karpenter via + an SQS queue wired to EventBridge rules provisioned by the `eks-cluster` module. diff --git a/docs/design/fully-private-eks-bootstrap.md b/docs/design/fully-private-eks-bootstrap.md index 2f9dfbf3d..5d4201dbf 100644 --- a/docs/design/fully-private-eks-bootstrap.md +++ b/docs/design/fully-private-eks-bootstrap.md @@ -98,7 +98,7 @@ Creates fully private EKS cluster with: - KMS encryption at rest for Kubernetes secrets using customer-managed keys - Dedicated security groups for VPC endpoints (port 443 from VPC CIDR only) - Restricted cluster egress limited to HTTPS for container registries and VPC internal traffic -- EKS Auto Mode authentication configured for API_AND_CONFIG_MAP mode +- EKS authentication configured for API_AND_CONFIG_MAP mode **High Availability Network Architecture**: diff --git a/docs/design/gitops-cluster-configuration.md b/docs/design/gitops-cluster-configuration.md index 21ad9734d..9440b01ce 100644 --- a/docs/design/gitops-cluster-configuration.md +++ b/docs/design/gitops-cluster-configuration.md @@ -455,7 +455,7 @@ metadata: cluster_name: mc01 annotations: git_repo: https://github.com/openshift-online/rosa-platform - git_revision: HEAD # Allows changes for dev + git_revision: HEAD # Allows changes for dev type: Opaque # Application created by ECS task @@ -483,12 +483,12 @@ spec: - git: directories: - path: argocd/config/management-cluster/* - repoURL: '{{ .metadata.annotations.git_repo }}' - revision: '{{ .metadata.annotations.git_revision }}' + repoURL: "{{ .metadata.annotations.git_repo }}" + revision: "{{ .metadata.annotations.git_revision }}" template: spec: source: - path: '{{ .path.path }}' + path: "{{ .path.path }}" helm: valueFiles: - values.yaml diff --git a/docs/design/karpenter-node-provisioning.md b/docs/design/karpenter-node-provisioning.md new file mode 100644 index 000000000..c887f81eb --- /dev/null +++ b/docs/design/karpenter-node-provisioning.md @@ -0,0 +1,98 @@ +# Karpenter Node Provisioning + +**Last Updated Date**: 2026-07-08 + +## Summary + +All EKS clusters use OSS Karpenter v1 (1.13.0) for node provisioning. A dedicated +`karpenter-bootstrap` managed node group provides stable capacity for the Karpenter controller. +The Karpenter controller IAM role uses IRSA (IAM Roles for Service Accounts) rather than EKS Pod +Identity because Karpenter predates EKS Pod Identity GA support. + +## Context + +When clusters migrated from EKS Auto Mode to OSS Karpenter, two IAM authentication mechanisms +were available for the Karpenter controller ServiceAccount: + +- **IRSA (IAM Roles for Service Accounts)**: ServiceAccount carries an annotation + (`eks.amazonaws.com/role-arn`); the OIDC provider validates the JWT and assumes the annotated + role. Requires an OIDC provider resource (`aws_iam_openid_connect_provider`) per cluster. +- **EKS Pod Identity**: Newer mechanism; IAM role is bound to a ServiceAccount via an API + association (no annotation needed). Simpler Terraform — no OIDC provider resource required. + +## Decision: IRSA for Karpenter Controller + +**Chosen**: IRSA for the Karpenter controller; EKS Pod Identity for all other workloads. + +**Rationale**: Karpenter v1 (1.13.0) ships with built-in IRSA support (ServiceAccount annotation +set during `helm install` via `serviceAccount.annotations`). EKS Pod Identity support in Karpenter +requires a separate admission webhook and additional configuration that the upstream chart does not +handle automatically. Using IRSA for Karpenter matches the upstream recommended installation +pattern, minimizes bootstrap complexity, and avoids a separate admission controller dependency +during the ECS bootstrap task. + +All other platform workloads (Thanos, Loki, Maestro Agent, AWS Load Balancer Controller, ZOA +jobs) use EKS Pod Identity exclusively. + +## Architecture + +```mermaid +graph LR + KC["Karpenter Controller\n(kube-system/karpenter)"] -->|"IRSA (JWT → OIDC)"| KCR["karpenter-controller\nIAM Role"] + KCR -->|"SQS: interruption events"| SQS["${cluster_id}-karpenter\nSQS Queue"] + KCR -->|"EC2: RunInstances, TerminateInstances"| EC2["EC2 API"] + KCR -->|"iam:PassRole → instance profile"| KNR["karpenter-node-role\nInstance Profile"] + KNR -->|"assumed by"| KN["Karpenter-provisioned\nnodes"] + BNG["karpenter-bootstrap\nManaged Node Group\n(2x t3.medium)"] -->|"tolerates CriticalAddonsOnly"| KC + EB["EventBridge Rules\n(EC2 lifecycle events)"] --> SQS +``` + +## IAM Resources + +### Karpenter Controller Role (IRSA) + +- **Name**: `${cluster_id}-karpenter-controller` +- **Trust**: OIDC provider for the cluster; constrained to `system:serviceaccount:kube-system:karpenter` +- **Permissions**: EC2 fleet operations (describe, run, terminate instances), IAM PassRole to the + node instance profile, SQS receive/delete on the interruption queue + +### Karpenter Node Role + +- **Name**: `${cluster_id}-karpenter-node-role` +- **Managed policies**: `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, ECR pull-only +- **Optional inline policy**: `kms:Decrypt` and `kms:CreateGrant` on the FIPS AMI KMS key when + `ami_kms_key_arn` is set +- **Referenced in**: `EC2NodeClass.spec.role` (instance profile name) + +### SQS Queue and EventBridge Rules + +The `eks-cluster` module provisions: + +- SQS queue (`${cluster_id}-karpenter`) with SQS-managed SSE, allowing `events.amazonaws.com` + and `sqs.amazonaws.com` to send messages +- Four EventBridge rules forwarding EC2 events to the queue: + - `scheduled-change` (AWS Health events) + - `spot-interruption` (EC2 Spot Instance interruption) + - `rebalance` (EC2 Instance Rebalance Recommendation) + - `instance-state-change` (EC2 Instance State-change Notification) + +## Consequences + +### Positive + +- IRSA is the upstream-recommended Karpenter auth mechanism; no additional admission controllers required +- Karpenter controller role trust policy is scoped to a single ServiceAccount — no broader cluster-level access +- SQS interruption handling enables graceful draining before spot reclamation or instance retirement +- OSS Karpenter can be upgraded independently via Helm without AWS EKS Auto Mode release cycles + +### Negative + +- One OIDC provider resource (`aws_iam_openid_connect_provider`) is required per cluster when `enable_karpenter = true` +- IRSA and Pod Identity coexist; operators must know which mechanism applies to which workload (Karpenter = IRSA, everything else = Pod Identity) + +## Related + +- [FIPS-Only EKS Compute](./fips-eks-compute.md) — EC2NodeClass and NodePool design for FIPS workloads +- [ECS Fargate Bootstrap](./fully-private-eks-bootstrap.md) — How Karpenter is installed during cluster bootstrap +- [Karpenter documentation](https://karpenter.sh/docs/) +- [IRSA documentation](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) diff --git a/docs/design/logging-platform.md b/docs/design/logging-platform.md index 3cddfaca0..c727e563e 100644 --- a/docs/design/logging-platform.md +++ b/docs/design/logging-platform.md @@ -29,7 +29,7 @@ We evaluated deploying the same stack that RHOBS uses internally: **Constraints**: -- EKS Auto Mode (no OpenShift operators available) +- EKS with OSS Karpenter (no OpenShift operators available) - EKS Pod Identity for IAM auth — no static credentials - KMS encryption at rest (S3 bucket-level default, transparent to Loki) - Minimize locally-maintained operator code diff --git a/docs/design/rate-limiting-architecture.md b/docs/design/rate-limiting-architecture.md index e3be9ddff..f3187c357 100644 --- a/docs/design/rate-limiting-architecture.md +++ b/docs/design/rate-limiting-architecture.md @@ -239,7 +239,7 @@ A Lambda function runs before the backend, extracts caller identity from SigV4 c **Limitations:** - **Cold start latency**: 50-500ms on first invocation (varies by runtime) -- **Provisioned Concurrency** eliminates cold starts but adds cost. Note: one provisioned instance handles one **concurrent** request, not one per second — if each rate check takes ~5ms, a single instance can serve ~200 req/s. At current traffic levels (~200 req/s), 2-3 provisioned instances (~$30-45/month) would suffice. +- **Provisioned Concurrency** eliminates cold starts but adds cost. Note: one provisioned instance handles one **concurrent** request, not one per second — if each rate check takes ~5ms, a single instance can serve ~200 req/s. At current traffic levels (~~200 req/s), 2-3 provisioned instances (~~$30-45/month) would suffice. - Authorizer response caching (TTL) conflicts with rate limiting — cached responses skip the Lambda, so counters aren't checked - Custom code to write and maintain diff --git a/docs/design/thanos-metrics-infrastructure.md b/docs/design/thanos-metrics-infrastructure.md index 20034f040..ec7d212ac 100644 --- a/docs/design/thanos-metrics-infrastructure.md +++ b/docs/design/thanos-metrics-infrastructure.md @@ -26,8 +26,7 @@ causing ArgoCD ServerSideApply failures when field names changed. - UBI9 base images with automated security scanning (Clair, ClamAV, Snyk) - Minimize locally-maintained operator code -**Assumptions**: Management clusters send metrics via Prometheus `remote_write`. EKS Auto Mode remains -the compute strategy. Raw retention is 90d; downsampled retention is 180d (5m) and 365d (1h). +**Assumptions**: Management clusters send metrics via Prometheus `remote_write`. OSS Karpenter is the compute strategy. Raw retention is 90d; downsampled retention is 180d (5m) and 365d (1h). ## Decision diff --git a/docs/design/zoa-trusted-actions.md b/docs/design/zoa-trusted-actions.md index 2ebb707cc..908c6908c 100644 --- a/docs/design/zoa-trusted-actions.md +++ b/docs/design/zoa-trusted-actions.md @@ -822,7 +822,7 @@ Platform API Reconciler (5s loop): 2. **Single shared ServiceAccount**: One SA (`zoa-job-runner`) for all TAs. Rejected because Kubernetes audit logs only show SA identity — all TAs would be indistinguishable at the K8s audit level. Additionally, a shared SA bound to N possible Roles means parallel executions share permissions — any running TA would have access to RBAC granted for a different concurrent TA. -3. **IRSA (IAM Roles for Service Accounts)**: Allows per-SA roles via annotations. Rejected because IRSA is not fully supported in EKS Auto Mode and is being deprecated in favor of Pod Identity. +3. **IRSA (IAM Roles for Service Accounts)**: Allows per-SA roles via annotations. Rejected because IRSA is being deprecated in favor of EKS Pod Identity, which does not require OIDC provider management per cluster and is the platform-standard auth mechanism for workload SAs. 4. **Sidecar container for S3 upload**: A separate container watches `/artifacts` and uploads. Rejected because sidecars add complexity around container ordering and completion detection. Additionally, containers in the same Pod share the same ServiceAccount — the runner would inherit S3 write permissions, breaking the isolation between operational actions and output transport. diff --git a/scripts/buildspec/register.sh b/scripts/buildspec/register.sh index 3f110d132..1313e9f76 100755 --- a/scripts/buildspec/register.sh +++ b/scripts/buildspec/register.sh @@ -68,9 +68,13 @@ fi CLOUDFRONT_URL="https://${CLOUDFRONT_DOMAIN}" -# Wait for API Gateway /live endpoint +# Wait for API Gateway /live endpoint. +# RC and MC pipelines run in parallel, so RC outputs become available as soon +# as terraform apply finishes — before the ECS bootstrap (ArgoCD install, +# ~15 min) and initial ArgoCD sync (~10 min) have completed. Allow 40 minutes +# so the Platform API has time to be deployed and reach a healthy state. set +e -MAX_RETRIES=10 +MAX_RETRIES=80 RETRY_DELAY=30 RETRY_COUNT=0 LIVE_OK=false @@ -101,7 +105,7 @@ done set -e if [ "$LIVE_OK" != "true" ]; then - echo "ERROR: /live did not return 200 after $MAX_RETRIES attempts" >&2 + echo "ERROR: /live did not return 200 after $((MAX_RETRIES * RETRY_DELAY / 60)) minutes" >&2 exit 1 fi diff --git a/scripts/validate-mc-aws.sh b/scripts/validate-mc-aws.sh new file mode 100755 index 000000000..7f2a23f8f --- /dev/null +++ b/scripts/validate-mc-aws.sh @@ -0,0 +1,347 @@ +#!/usr/bin/env bash +# Validate AWS-level configuration and resources for a Management Cluster (MC). +# +# Usage: +# ./scripts/validate-mc-aws.sh # auto-derives CLUSTER_ID and AWS_REGION from kubectl context +# CLUSTER_ID= AWS_REGION= ./scripts/validate-mc-aws.sh # override if needed +# +# Prerequisites: aws CLI configured with appropriate credentials for the MC account. + +set -euo pipefail + +# Auto-derive CLUSTER_ID and AWS_REGION from the active kubectl context when not set explicitly. +# aws eks update-kubeconfig names contexts: arn:aws:eks:::cluster/ +_ctx=$(kubectl config current-context 2>/dev/null || true) +if [[ -z "${CLUSTER_ID:-}" && "$_ctx" =~ :cluster/(.+)$ ]]; then + CLUSTER_ID="${BASH_REMATCH[1]}" +fi +if [[ -z "${AWS_REGION:-}" && "$_ctx" =~ arn:aws:eks:([^:]+): ]]; then + AWS_REGION="${BASH_REMATCH[1]}" +fi +CLUSTER_ID="${CLUSTER_ID:?Cannot derive CLUSTER_ID — set it manually or ensure the active kubectl context points at an EKS cluster}" +AWS_REGION="${AWS_REGION:?Cannot derive AWS_REGION — set it manually or ensure the active kubectl context points at an EKS cluster}" + +export AWS_DEFAULT_REGION="$AWS_REGION" + +# --------------------------------------------------------------------------- +# Output helpers +# --------------------------------------------------------------------------- + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RESET='\033[0m' + +PASS=0 +FAIL=0 +WARN=0 + +pass() { echo -e "${GREEN}[PASS]${RESET} $*"; ((++PASS)); } +fail() { echo -e "${RED}[FAIL]${RESET} $*"; ((++FAIL)); } +warn() { echo -e "${YELLOW}[WARN]${RESET} $*"; ((++WARN)); } + +section() { echo; echo "=== $* ==="; } + +# --------------------------------------------------------------------------- +# 1. EKS cluster +# --------------------------------------------------------------------------- + +section "EKS cluster" + +cluster_status=$(aws eks describe-cluster \ + --name "$CLUSTER_ID" \ + --query "cluster.status" \ + --output text 2>/dev/null || echo "NOT_FOUND") + +if [[ "$cluster_status" == "ACTIVE" ]]; then + pass "EKS cluster '${CLUSTER_ID}' ACTIVE" +else + fail "EKS cluster '${CLUSTER_ID}' status: ${cluster_status}" +fi + +cluster_version=$(aws eks describe-cluster \ + --name "$CLUSTER_ID" \ + --query "cluster.version" \ + --output text 2>/dev/null || echo "unknown") +pass "EKS cluster version: ${cluster_version}" + +auth_mode=$(aws eks describe-cluster \ + --name "$CLUSTER_ID" \ + --query "cluster.accessConfig.authenticationMode" \ + --output text 2>/dev/null || echo "UNKNOWN") +if [[ "$auth_mode" == "API_AND_CONFIG_MAP" ]]; then + pass "EKS auth mode: API_AND_CONFIG_MAP" +else + fail "EKS auth mode: ${auth_mode} (expected API_AND_CONFIG_MAP)" +fi + +# Private endpoint required — no public access +public_access=$(aws eks describe-cluster \ + --name "$CLUSTER_ID" \ + --query "cluster.resourcesVpcConfig.endpointPublicAccess" \ + --output text 2>/dev/null || echo "unknown") +if [[ "$public_access" == "False" ]]; then + pass "EKS public endpoint access: disabled" +else + fail "EKS public endpoint access: ${public_access} (must be False)" +fi + +# --------------------------------------------------------------------------- +# 2. EKS managed add-ons +# --------------------------------------------------------------------------- + +section "EKS managed add-ons" + +EXPECTED_ADDONS=( + "coredns" + "vpc-cni" + "kube-proxy" + "eks-pod-identity-agent" + "aws-ebs-csi-driver" +) + +addon_json=$(aws eks list-addons \ + --cluster-name "$CLUSTER_ID" \ + --output json 2>/dev/null | jq -r '.addons[]') + +for addon in "${EXPECTED_ADDONS[@]}"; do + if echo "$addon_json" | grep -q "^${addon}$"; then + status=$(aws eks describe-addon \ + --cluster-name "$CLUSTER_ID" \ + --addon-name "$addon" \ + --query "addon.status" \ + --output text 2>/dev/null || echo "UNKNOWN") + if [[ "$status" == "ACTIVE" ]]; then + pass "Add-on ${addon}: ACTIVE" + else + fail "Add-on ${addon}: ${status}" + fi + else + warn "Add-on ${addon}: not installed" + fi +done + +# --------------------------------------------------------------------------- +# 3. Karpenter bootstrap node group +# --------------------------------------------------------------------------- + +section "Karpenter bootstrap node group" + +ng_name="${CLUSTER_ID}-karpenter-bootstrap" + +ng_status=$(aws eks describe-nodegroup \ + --cluster-name "$CLUSTER_ID" \ + --nodegroup-name "$ng_name" \ + --query "nodegroup.status" \ + --output text 2>/dev/null || echo "NOT_FOUND") + +if [[ "$ng_status" == "ACTIVE" ]]; then + ng_desired=$(aws eks describe-nodegroup \ + --cluster-name "$CLUSTER_ID" \ + --nodegroup-name "$ng_name" \ + --query "nodegroup.scalingConfig.desiredSize" \ + --output text 2>/dev/null || echo "?") + pass "Node group '${ng_name}': ACTIVE (desired: ${ng_desired})" +else + fail "Node group '${ng_name}': ${ng_status}" +fi + +# --------------------------------------------------------------------------- +# 4. EC2 instances (Karpenter-provisioned) +# --------------------------------------------------------------------------- + +section "EC2 instances" + +kp_instance_count=$(aws ec2 describe-instances \ + --filters \ + "Name=tag:karpenter.sh/discovery,Values=${CLUSTER_ID}" \ + "Name=instance-state-name,Values=running" \ + --query "length(Reservations[*].Instances[])" \ + --output text 2>/dev/null || echo 0) + +if [[ "$kp_instance_count" -ge 1 ]]; then + pass "Karpenter-provisioned EC2 instances running: ${kp_instance_count}" +else + warn "No running EC2 instances tagged karpenter.sh/discovery=${CLUSTER_ID} (expected once workloads are scheduled)" +fi + +# Confirm all running cluster instances are in the right VPC +cluster_vpc=$(aws eks describe-cluster \ + --name "$CLUSTER_ID" \ + --query "cluster.resourcesVpcConfig.vpcId" \ + --output text 2>/dev/null || echo "") + +if [[ -n "$cluster_vpc" ]]; then + wrong_vpc=$(aws ec2 describe-instances \ + --filters \ + "Name=tag:kubernetes.io/cluster/${CLUSTER_ID},Values=owned" \ + "Name=instance-state-name,Values=running" \ + --query "Reservations[*].Instances[?VpcId!='${cluster_vpc}'] | length(@)" \ + --output text 2>/dev/null | paste -sd+ | bc 2>/dev/null || echo 0) + if [[ "$wrong_vpc" -eq 0 ]]; then + pass "All cluster EC2 instances in correct VPC (${cluster_vpc})" + else + fail "${wrong_vpc} cluster EC2 instance(s) in unexpected VPC" + fi +fi + +# --------------------------------------------------------------------------- +# 5. IAM roles +# --------------------------------------------------------------------------- + +section "IAM roles" + +declare -A IAM_ROLES=( + ["karpenter-controller"]="${CLUSTER_ID}-karpenter-controller" + ["karpenter-node"]="${CLUSTER_ID}-karpenter-node-role" + ["eks-cluster"]="${CLUSTER_ID}-cluster-role" + ["ebs-csi"]="${CLUSTER_ID}-ebs-csi-role" +) + +for label in "${!IAM_ROLES[@]}"; do + role_name="${IAM_ROLES[$label]}" + if aws iam get-role --role-name "$role_name" &>/dev/null; then + pass "IAM role exists: ${role_name}" + else + fail "IAM role missing: ${role_name}" + fi +done + +# HyperShift installs a service account that needs a role — check it exists if HC is running +hs_role="${CLUSTER_ID}-hypershift-operator" +if aws iam get-role --role-name "$hs_role" &>/dev/null; then + pass "IAM role exists: ${hs_role}" +else + warn "IAM role '${hs_role}' not found (expected if HyperShift installed via IRSA)" +fi + +# --------------------------------------------------------------------------- +# 6. SQS queue (Karpenter interruption handling) +# --------------------------------------------------------------------------- + +section "SQS queue" + +queue_name="${CLUSTER_ID}-karpenter" + +if aws sqs get-queue-url --queue-name "$queue_name" &>/dev/null; then + pass "SQS queue '${queue_name}' exists" +else + fail "SQS queue '${queue_name}' not found" +fi + +# --------------------------------------------------------------------------- +# 7. ECS bootstrap cluster +# --------------------------------------------------------------------------- + +section "ECS bootstrap cluster" + +ecs_cluster_name="${CLUSTER_ID}-bootstrap" + +ecs_status=$(aws ecs describe-clusters \ + --clusters "$ecs_cluster_name" \ + --query "clusters[0].status" \ + --output text 2>/dev/null || echo "NOT_FOUND") + +if [[ "$ecs_status" == "ACTIVE" ]]; then + pass "ECS cluster '${ecs_cluster_name}' ACTIVE" +else + fail "ECS cluster '${ecs_cluster_name}' status: ${ecs_status}" +fi + +# --------------------------------------------------------------------------- +# 8. CloudWatch log group +# --------------------------------------------------------------------------- + +section "CloudWatch log group" + +log_group="/aws/eks/${CLUSTER_ID}/cluster" + +if aws logs describe-log-groups \ + --log-group-name-prefix "$log_group" \ + --query "logGroups[?logGroupName=='${log_group}'] | length(@)" \ + --output text 2>/dev/null | grep -q "^[1-9]"; then + pass "CloudWatch log group '${log_group}' exists" +else + fail "CloudWatch log group '${log_group}' not found" +fi + +# --------------------------------------------------------------------------- +# 9. VPC and subnet availability +# --------------------------------------------------------------------------- + +section "VPC and subnets" + +vpc_id=$(aws eks describe-cluster \ + --name "$CLUSTER_ID" \ + --query "cluster.resourcesVpcConfig.vpcId" \ + --output text 2>/dev/null || echo "") + +if [[ -z "$vpc_id" || "$vpc_id" == "None" ]]; then + fail "Could not retrieve VPC ID for cluster '${CLUSTER_ID}'" +else + vpc_state=$(aws ec2 describe-vpcs \ + --vpc-ids "$vpc_id" \ + --query "Vpcs[0].State" \ + --output text 2>/dev/null || echo "not-found") + if [[ "$vpc_state" == "available" ]]; then + pass "VPC ${vpc_id} state: available" + else + fail "VPC ${vpc_id} state: ${vpc_state}" + fi + + # Each private subnet should have available IPs + subnet_ids=$(aws eks describe-cluster \ + --name "$CLUSTER_ID" \ + --query "cluster.resourcesVpcConfig.subnetIds[]" \ + --output text 2>/dev/null || echo "") + + no_ips=0 + total_subnets=0 + for subnet in $subnet_ids; do + ((total_subnets++)) + available_ips=$(aws ec2 describe-subnets \ + --subnet-ids "$subnet" \ + --query "Subnets[0].AvailableIpAddressCount" \ + --output text 2>/dev/null || echo 0) + if [[ "$available_ips" -lt 5 ]]; then + ((no_ips++)) + warn "Subnet ${subnet}: only ${available_ips} available IPs" + fi + done + if [[ "$no_ips" -eq 0 ]]; then + pass "All ${total_subnets} subnets have adequate available IPs" + else + fail "${no_ips}/${total_subnets} subnet(s) with fewer than 5 available IPs" + fi +fi + +# --------------------------------------------------------------------------- +# 10. KMS key aliases +# --------------------------------------------------------------------------- + +section "KMS key aliases" + +declare -A KMS_ALIASES=( + ["cloudwatch-logs"]="alias/${CLUSTER_ID}-cloudwatch-logs" + ["eks-secrets"]="alias/${CLUSTER_ID}-eks-secrets" +) + +for label in "${!KMS_ALIASES[@]}"; do + alias_name="${KMS_ALIASES[$label]}" + if aws kms list-aliases \ + --query "Aliases[?AliasName=='${alias_name}'] | length(@)" \ + --output text 2>/dev/null | grep -q "^[1-9]"; then + pass "KMS alias '${alias_name}' (${label}) exists" + else + fail "KMS alias '${alias_name}' (${label}) not found" + fi +done + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + +section "Summary" +echo -e " ${GREEN}PASS${RESET}: ${PASS} ${RED}FAIL${RESET}: ${FAIL} ${YELLOW}WARN${RESET}: ${WARN}" + +[[ "$FAIL" -eq 0 ]] diff --git a/scripts/validate-mc-k8s.sh b/scripts/validate-mc-k8s.sh new file mode 100755 index 000000000..8c20b0b56 --- /dev/null +++ b/scripts/validate-mc-k8s.sh @@ -0,0 +1,311 @@ +#!/usr/bin/env bash +# Validate Kubernetes-level processes on a Management Cluster (MC). +# +# Usage: +# ./scripts/validate-mc-k8s.sh # auto-derives CLUSTER_ID from kubectl context +# CLUSTER_ID= ./scripts/validate-mc-k8s.sh # override if needed +# +# Prerequisites: active kubectl context pointing at the target MC, kubectl/jq on PATH. + +set -euo pipefail + +# Auto-derive CLUSTER_ID from the active kubectl context when not set explicitly. +# aws eks update-kubeconfig names contexts: arn:aws:eks:::cluster/ +_ctx=$(kubectl config current-context 2>/dev/null || true) +if [[ -z "${CLUSTER_ID:-}" && "$_ctx" =~ :cluster/(.+)$ ]]; then + CLUSTER_ID="${BASH_REMATCH[1]}" +fi +CLUSTER_ID="${CLUSTER_ID:?Cannot derive CLUSTER_ID — set it manually or ensure the active kubectl context points at an EKS cluster}" + +# --------------------------------------------------------------------------- +# Output helpers +# --------------------------------------------------------------------------- + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RESET='\033[0m' + +PASS=0 +FAIL=0 +WARN=0 + +pass() { echo -e "${GREEN}[PASS]${RESET} $*"; ((++PASS)); } +fail() { echo -e "${RED}[FAIL]${RESET} $*"; ((++FAIL)); } +warn() { echo -e "${YELLOW}[WARN]${RESET} $*"; ((++WARN)); } + +section() { echo; echo "=== $* ==="; } + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +pods_running() { + local ns="$1" selector="${2:-}" + local args=(-n "$ns" --no-headers) + [[ -n "$selector" ]] && args+=(-l "$selector") + + if ! kubectl get pods "${args[@]}" 2>/dev/null | grep -q .; then + return 2 + fi + + local not_running + not_running=$(kubectl get pods "${args[@]}" 2>/dev/null \ + | awk '$3 ~ /^(Running|Completed)$/ { if ($3 == "Running") { split($2, r, "/"); if (r[1]+0 < r[2]+0) print }; next } { print }' \ + || true) + [[ -z "$not_running" ]] +} + +pod_count() { + local ns="$1" selector="${2:-}" + local args=(-n "$ns" --no-headers) + [[ -n "$selector" ]] && args+=(-l "$selector") + kubectl get pods "${args[@]}" 2>/dev/null | grep -c . || echo 0 +} + +# --------------------------------------------------------------------------- +# 1. Nodes +# --------------------------------------------------------------------------- + +section "Nodes" + +if ! _nodes_raw=$(kubectl get nodes --no-headers 2>/dev/null); then + fail "Cannot list nodes — check kubeconfig and RBAC" +else + not_ready=$(echo "$_nodes_raw" | awk '{print $2}' | grep -v "^Ready$" || true) + if [[ -z "$not_ready" ]]; then + node_count=$(echo "$_nodes_raw" | wc -l | tr -d ' ') + pass "All ${node_count} nodes are Ready" + else + bad=$(echo "$not_ready" | wc -l | tr -d ' ') + fail "${bad} node(s) not Ready" + fi +fi + +# Karpenter-provisioned nodes carry karpenter.sh/nodepool label +kp_nodes=$(kubectl get nodes -l "karpenter.sh/nodepool" --no-headers 2>/dev/null | wc -l | tr -d ' ') +if [[ "$kp_nodes" -ge 1 ]]; then + pass "Karpenter-provisioned nodes present (${kp_nodes})" +else + warn "No Karpenter-provisioned nodes found (may be expected if no workload scheduled yet)" +fi + +# --------------------------------------------------------------------------- +# 2. HyperShift operator +# --------------------------------------------------------------------------- + +section "HyperShift operator" + +if kubectl get namespace hypershift &>/dev/null; then + rc=0 + pods_running hypershift "app=operator" || rc=$? + if [[ $rc -eq 0 ]]; then + count=$(pod_count hypershift "app=operator") + pass "HyperShift operator Running (${count} pod(s))" + elif [[ $rc -eq 2 ]]; then + fail "HyperShift operator: namespace exists but no operator pods found" + echo " [diag] hypershift-install Job:" + kubectl get job hypershift-install -n hypershift-install --no-headers 2>/dev/null \ + | sed 's/^/ /' || echo " job not found in namespace hypershift-install" + echo " [diag] Installer pod logs (last 40 lines):" + kubectl logs -n hypershift-install -l "job-name=hypershift-install" \ + --tail=40 2>/dev/null | sed 's/^/ /' \ + || echo " no logs — pod may have been evicted or namespace missing" + echo " [diag] Resources in hypershift namespace:" + kubectl get all -n hypershift 2>/dev/null | sed 's/^/ /' || true + echo " [diag] Events in hypershift namespace:" + kubectl get events -n hypershift --sort-by='.lastTimestamp' 2>/dev/null \ + | tail -10 | sed 's/^/ /' || true + else + fail "HyperShift operator pods not all Running" + kubectl get pods -n hypershift -l "app=operator" --no-headers 2>/dev/null \ + | sed 's/^/ /' || true + echo " [diag] Events:" + kubectl get events -n hypershift --sort-by='.lastTimestamp' 2>/dev/null \ + | tail -10 | sed 's/^/ /' || true + fi +else + fail "Namespace 'hypershift' does not exist — HyperShift not installed" + echo " [diag] Installer job:" + kubectl get job hypershift-install -n hypershift-install 2>/dev/null \ + | sed 's/^/ /' || echo " namespace hypershift-install not found" +fi + +# --------------------------------------------------------------------------- +# 3. HostedClusters and NodePools +# --------------------------------------------------------------------------- + +section "HostedClusters" + +if ! kubectl api-resources --api-group=hypershift.openshift.io 2>/dev/null | grep -q HostedCluster; then + warn "HyperShift CRDs not registered — skipping HostedCluster checks" +else + hc_total=$(kubectl get hostedclusters -A --no-headers 2>/dev/null | wc -l | tr -d ' ') + if [[ "$hc_total" -eq 0 ]]; then + warn "No HostedClusters found" + else + pass "HostedClusters found: ${hc_total}" + + # Check each HC is Available + while IFS= read -r line; do + hc_ns=$(echo "$line" | awk '{print $1}') + hc_name=$(echo "$line" | awk '{print $2}') + available=$(kubectl get hostedcluster "$hc_name" -n "$hc_ns" \ + -o jsonpath='{.status.conditions[?(@.type=="Available")].status}' 2>/dev/null || true) + if [[ "$available" == "True" ]]; then + pass "HostedCluster ${hc_ns}/${hc_name} Available=True" + else + reason=$(kubectl get hostedcluster "$hc_name" -n "$hc_ns" \ + -o jsonpath='{.status.conditions[?(@.type=="Available")].message}' 2>/dev/null || true) + fail "HostedCluster ${hc_ns}/${hc_name} Available=${available:-Unknown} — ${reason:-no detail}" + fi + done < <(kubectl get hostedclusters -A --no-headers 2>/dev/null) + fi + + np_total=$(kubectl get nodepools -A --no-headers 2>/dev/null | wc -l | tr -d ' ') + if [[ "$np_total" -eq 0 ]]; then + warn "No NodePools found" + else + while IFS= read -r line; do + np_ns=$(echo "$line" | awk '{print $1}') + np_name=$(echo "$line" | awk '{print $2}') + desired=$(kubectl get nodepool "$np_name" -n "$np_ns" \ + -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "?") + ready=$(kubectl get nodepool "$np_name" -n "$np_ns" \ + -o jsonpath='{.status.replicas}' 2>/dev/null || echo "0") + ready="${ready:-0}" + if [[ "$ready" -ge 1 ]]; then + pass "NodePool ${np_ns}/${np_name}: ${ready}/${desired} replicas Ready" + else + fail "NodePool ${np_ns}/${np_name}: ${ready}/${desired} replicas Ready" + fi + done < <(kubectl get nodepools -A --no-headers 2>/dev/null) + fi +fi + +# --------------------------------------------------------------------------- +# 4. Control plane pods per HostedCluster +# --------------------------------------------------------------------------- + +section "HostedCluster control plane pods" + +if kubectl api-resources --api-group=hypershift.openshift.io 2>/dev/null | grep -q HostedCluster; then + while IFS= read -r line; do + hc_ns=$(echo "$line" | awk '{print $1}') + hc_name=$(echo "$line" | awk '{print $2}') + cp_ns="clusters-${hc_name}" + rc=0 + pods_running "$cp_ns" || rc=$? + if [[ $rc -eq 0 ]]; then + count=$(pod_count "$cp_ns") + pass "Control plane pods for ${hc_name} (${cp_ns}): ${count} Running" + elif [[ $rc -eq 2 ]]; then + warn "No control plane pods in ${cp_ns} yet" + else + not_running=$(kubectl get pods -n "$cp_ns" --no-headers 2>/dev/null \ + | awk '{print $1, $3}' | grep -v "Running\|Completed" || true) + fail "Control plane pods not all Running in ${cp_ns}:" + echo "$not_running" | sed 's/^/ /' + fi + done < <(kubectl get hostedclusters -A --no-headers 2>/dev/null) +fi + +# --------------------------------------------------------------------------- +# 5. Core add-ons +# --------------------------------------------------------------------------- + +section "Core add-ons (kube-system)" + +declare -A CORE_SELECTORS=( + ["CoreDNS"]="k8s-app=kube-dns" + ["vpc-cni (aws-node)"]="k8s-app=aws-node" + ["kube-proxy"]="k8s-app=kube-proxy" +) + +for label in "${!CORE_SELECTORS[@]}"; do + selector="${CORE_SELECTORS[$label]}" + rc=0 + pods_running kube-system "$selector" || rc=$? + if [[ $rc -eq 0 ]]; then + pass "${label} Running" + elif [[ $rc -eq 2 ]]; then + warn "${label}: no pods found" + else + fail "${label}: pods not all Running" + fi +done + +# --------------------------------------------------------------------------- +# 6. Maestro agent +# --------------------------------------------------------------------------- + +section "Maestro agent" + +rc=0 +pods_running maestro-agent || rc=$? +if [[ $rc -eq 0 ]]; then + count=$(pod_count maestro-agent) + pass "Maestro agent: ${count} pod(s) Running" +elif [[ $rc -eq 2 ]]; then + warn "Maestro agent: no pods in namespace 'maestro-agent'" +else + fail "Maestro agent: pods not all Running" +fi + +# --------------------------------------------------------------------------- +# 7. ArgoCD (optional on MC) +# --------------------------------------------------------------------------- + +section "ArgoCD (if present)" + +if kubectl get namespace argocd &>/dev/null; then + if ! _apps_raw=$(kubectl get applications -n argocd --no-headers 2>/dev/null); then + fail "ArgoCD: cannot list applications — check RBAC and ArgoCD CRD availability" + else + not_synced=$(echo "$_apps_raw" \ + | awk '{print $1, $2, $3}' | grep -v "Synced.*Healthy" | grep -v "Synced.*Progressing" || true) + if [[ -z "$not_synced" ]]; then + total=$(echo "$_apps_raw" | wc -l | tr -d ' ') + pass "All ${total} ArgoCD applications Synced" + else + count=$(echo "$not_synced" | wc -l | tr -d ' ') + fail "${count} ArgoCD application(s) not Synced/Healthy" + echo "$not_synced" | sed 's/^/ /' + while IFS= read -r _line; do + _app=$(echo "$_line" | awk '{print $1}') + _sync=$(echo "$_line" | awk '{print $2}') + _health=$(echo "$_line" | awk '{print $3}') + echo " [diag] ${_app} (${_sync}/${_health}):" + if [[ "$_sync" == "OutOfSync" ]]; then + kubectl get application "$_app" -n argocd \ + -o jsonpath='{.status.resources}' 2>/dev/null \ + | jq -r '.[] | select(.status != "Synced") | " \(.kind)/\(.name): \(.status) \(.health.status // "")"' \ + 2>/dev/null | head -10 || true + fi + if [[ "$_health" == "Degraded" ]]; then + _app_health_msg=$(kubectl get application "$_app" -n argocd \ + -o jsonpath='{.status.health.message}' 2>/dev/null || true) + [[ -n "$_app_health_msg" ]] && echo " health: ${_app_health_msg}" + kubectl get application "$_app" -n argocd \ + -o jsonpath='{.status.conditions}' 2>/dev/null \ + | jq -r '.[]? | " condition: \(.type): \(.message // "-")"' 2>/dev/null || true + kubectl get application "$_app" -n argocd \ + -o jsonpath='{.status.resources}' 2>/dev/null \ + | jq -r '.[]? | select(.health.status == "Degraded" or .health.status == "Missing" or .health.status == "Unknown") | " \(.kind)/\(.name): \(.health.status) — \(.health.message // "-")"' \ + 2>/dev/null | head -10 || true + fi + done <<< "$not_synced" + fi + fi +else + warn "ArgoCD not installed on this MC (namespace 'argocd' absent)" +fi + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + +section "Summary" +echo -e " ${GREEN}PASS${RESET}: ${PASS} ${RED}FAIL${RESET}: ${FAIL} ${YELLOW}WARN${RESET}: ${WARN}" + +[[ "$FAIL" -eq 0 ]] diff --git a/scripts/validate-rc-aws.sh b/scripts/validate-rc-aws.sh new file mode 100755 index 000000000..417dc0202 --- /dev/null +++ b/scripts/validate-rc-aws.sh @@ -0,0 +1,307 @@ +#!/usr/bin/env bash +# Validate AWS-level configuration and resources for the Regional Cluster (RC). +# +# Usage: +# ./scripts/validate-rc-aws.sh # auto-derives CLUSTER_ID and AWS_REGION from kubectl context +# CLUSTER_ID= AWS_REGION= ./scripts/validate-rc-aws.sh # override if needed +# +# Optional: +# PLATFORM_API_TG_ARN= — ALB target group ARN for the platform-api service. +# If unset, the target-health check is skipped. +# +# Prerequisites: aws CLI configured with appropriate credentials for the RC account. + +set -euo pipefail + +# Auto-derive CLUSTER_ID and AWS_REGION from the active kubectl context when not set explicitly. +# aws eks update-kubeconfig names contexts: arn:aws:eks:::cluster/ +_ctx=$(kubectl config current-context 2>/dev/null || true) +if [[ -z "${CLUSTER_ID:-}" && "$_ctx" =~ :cluster/(.+)$ ]]; then + CLUSTER_ID="${BASH_REMATCH[1]}" +fi +if [[ -z "${AWS_REGION:-}" && "$_ctx" =~ arn:aws:eks:([^:]+): ]]; then + AWS_REGION="${BASH_REMATCH[1]}" +fi +CLUSTER_ID="${CLUSTER_ID:?Cannot derive CLUSTER_ID — set it manually or ensure the active kubectl context points at an EKS cluster}" +AWS_REGION="${AWS_REGION:?Cannot derive AWS_REGION — set it manually or ensure the active kubectl context points at an EKS cluster}" +PLATFORM_API_TG_ARN="${PLATFORM_API_TG_ARN:-}" + +export AWS_DEFAULT_REGION="$AWS_REGION" + +# --------------------------------------------------------------------------- +# Output helpers +# --------------------------------------------------------------------------- + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RESET='\033[0m' + +PASS=0 +FAIL=0 +WARN=0 + +pass() { echo -e "${GREEN}[PASS]${RESET} $*"; ((++PASS)); } +fail() { echo -e "${RED}[FAIL]${RESET} $*"; ((++FAIL)); } +warn() { echo -e "${YELLOW}[WARN]${RESET} $*"; ((++WARN)); } + +section() { echo; echo "=== $* ==="; } + +# --------------------------------------------------------------------------- +# 1. EKS cluster +# --------------------------------------------------------------------------- + +section "EKS cluster" + +cluster_status=$(aws eks describe-cluster \ + --name "$CLUSTER_ID" \ + --query "cluster.status" \ + --output text 2>/dev/null || echo "NOT_FOUND") + +if [[ "$cluster_status" == "ACTIVE" ]]; then + pass "EKS cluster '${CLUSTER_ID}' ACTIVE" +else + fail "EKS cluster '${CLUSTER_ID}' status: ${cluster_status}" +fi + +# Verify auth mode is API_AND_CONFIG_MAP (required for Karpenter node access entries) +auth_mode=$(aws eks describe-cluster \ + --name "$CLUSTER_ID" \ + --query "cluster.accessConfig.authenticationMode" \ + --output text 2>/dev/null || echo "UNKNOWN") +if [[ "$auth_mode" == "API_AND_CONFIG_MAP" ]]; then + pass "EKS auth mode: API_AND_CONFIG_MAP" +else + fail "EKS auth mode: ${auth_mode} (expected API_AND_CONFIG_MAP)" +fi + +# --------------------------------------------------------------------------- +# 2. EKS managed add-ons +# --------------------------------------------------------------------------- + +section "EKS managed add-ons" + +EXPECTED_ADDONS=( + "coredns" + "metrics-server" + "eks-pod-identity-agent" + "vpc-cni" + "kube-proxy" + "aws-ebs-csi-driver" + "aws-secrets-store-csi-driver-provider" +) + +addon_json=$(aws eks list-addons \ + --cluster-name "$CLUSTER_ID" \ + --output json 2>/dev/null | jq -r '.addons[]') + +for addon in "${EXPECTED_ADDONS[@]}"; do + if echo "$addon_json" | grep -q "^${addon}$"; then + status=$(aws eks describe-addon \ + --cluster-name "$CLUSTER_ID" \ + --addon-name "$addon" \ + --query "addon.status" \ + --output text 2>/dev/null || echo "UNKNOWN") + if [[ "$status" == "ACTIVE" ]]; then + pass "Add-on ${addon}: ACTIVE" + else + fail "Add-on ${addon}: ${status}" + fi + else + warn "Add-on ${addon}: not installed" + fi +done + +# --------------------------------------------------------------------------- +# 3. Karpenter bootstrap node group +# --------------------------------------------------------------------------- + +section "Karpenter bootstrap node group" + +ng_name="${CLUSTER_ID}-karpenter-bootstrap" + +ng_status=$(aws eks describe-nodegroup \ + --cluster-name "$CLUSTER_ID" \ + --nodegroup-name "$ng_name" \ + --query "nodegroup.status" \ + --output text 2>/dev/null || echo "NOT_FOUND") + +if [[ "$ng_status" == "ACTIVE" ]]; then + pass "Node group '${ng_name}': ACTIVE" +else + fail "Node group '${ng_name}': ${ng_status}" +fi + +ng_desired=$(aws eks describe-nodegroup \ + --cluster-name "$CLUSTER_ID" \ + --nodegroup-name "$ng_name" \ + --query "nodegroup.scalingConfig.desiredSize" \ + --output text 2>/dev/null || echo "0") + +ng_ready=$(aws eks describe-nodegroup \ + --cluster-name "$CLUSTER_ID" \ + --nodegroup-name "$ng_name" \ + --query "nodegroup.health.issues" \ + --output json 2>/dev/null | jq 'length') + +if [[ "$ng_ready" -eq 0 ]]; then + pass "Node group '${ng_name}': ${ng_desired} nodes, no health issues" +else + fail "Node group '${ng_name}': ${ng_ready} health issue(s)" +fi + +# --------------------------------------------------------------------------- +# 4. IAM roles +# --------------------------------------------------------------------------- + +section "IAM roles" + +declare -A IAM_ROLES=( + ["karpenter-controller"]="${CLUSTER_ID}-karpenter-controller" + ["karpenter-node"]="${CLUSTER_ID}-karpenter-node-role" + ["aws-load-balancer-controller"]="${CLUSTER_ID}-aws-load-balancer-controller" + ["eks-cluster"]="${CLUSTER_ID}-cluster-role" + ["ebs-csi"]="${CLUSTER_ID}-ebs-csi-role" +) + +for label in "${!IAM_ROLES[@]}"; do + role_name="${IAM_ROLES[$label]}" + if aws iam get-role --role-name "$role_name" &>/dev/null; then + pass "IAM role exists: ${role_name}" + else + fail "IAM role missing: ${role_name}" + fi +done + +# --------------------------------------------------------------------------- +# 5. SQS queue (Karpenter interruption handling) +# --------------------------------------------------------------------------- + +section "SQS queue" + +queue_name="${CLUSTER_ID}-karpenter" + +if aws sqs get-queue-url --queue-name "$queue_name" &>/dev/null; then + pass "SQS queue '${queue_name}' exists" +else + fail "SQS queue '${queue_name}' not found" +fi + +# --------------------------------------------------------------------------- +# 6. Karpenter-tagged EC2 instances +# --------------------------------------------------------------------------- + +section "Karpenter EC2 instances" + +kp_instances=$(aws ec2 describe-instances \ + --filters \ + "Name=tag:karpenter.sh/discovery,Values=${CLUSTER_ID}" \ + "Name=instance-state-name,Values=running" \ + --query "Reservations[*].Instances[*].InstanceId" \ + --output json 2>/dev/null | jq -r '.[][] | length' | paste -sd+ | bc 2>/dev/null || echo 0) + +# Simpler count approach +kp_instance_count=$(aws ec2 describe-instances \ + --filters \ + "Name=tag:karpenter.sh/discovery,Values=${CLUSTER_ID}" \ + "Name=instance-state-name,Values=running" \ + --query "length(Reservations[*].Instances[])" \ + --output text 2>/dev/null || echo 0) + +if [[ "$kp_instance_count" -ge 1 ]]; then + pass "Karpenter-provisioned EC2 instances running: ${kp_instance_count}" +else + warn "No running EC2 instances tagged karpenter.sh/discovery=${CLUSTER_ID}" +fi + +# --------------------------------------------------------------------------- +# 7. ALB target health (platform-api) +# --------------------------------------------------------------------------- + +section "ALB target health" + +if [[ -n "$PLATFORM_API_TG_ARN" ]]; then + healthy=$(aws elbv2 describe-target-health \ + --target-group-arn "$PLATFORM_API_TG_ARN" \ + --query "TargetHealthDescriptions[?TargetHealth.State=='healthy'] | length(@)" \ + --output text 2>/dev/null || echo 0) + unhealthy=$(aws elbv2 describe-target-health \ + --target-group-arn "$PLATFORM_API_TG_ARN" \ + --query "TargetHealthDescriptions[?TargetHealth.State!='healthy'] | length(@)" \ + --output text 2>/dev/null || echo 0) + if [[ "$healthy" -ge 1 ]]; then + pass "Platform API target group: ${healthy} healthy target(s), ${unhealthy} unhealthy" + else + fail "Platform API target group: 0 healthy targets (${unhealthy} unhealthy)" + fi +else + warn "PLATFORM_API_TG_ARN not set — skipping target health check" + warn " Set it to: kubectl get svc -n platform-api -o jsonpath='{.items[0].metadata.annotations.service\.beta\.kubernetes\.io/aws-load-balancer-arn}'" +fi + +# --------------------------------------------------------------------------- +# 8. ECS bootstrap cluster +# --------------------------------------------------------------------------- + +section "ECS bootstrap cluster" + +ecs_cluster_name="${CLUSTER_ID}-bootstrap" + +ecs_status=$(aws ecs describe-clusters \ + --clusters "$ecs_cluster_name" \ + --query "clusters[0].status" \ + --output text 2>/dev/null || echo "NOT_FOUND") + +if [[ "$ecs_status" == "ACTIVE" ]]; then + pass "ECS cluster '${ecs_cluster_name}' ACTIVE" +else + fail "ECS cluster '${ecs_cluster_name}' status: ${ecs_status}" +fi + +# --------------------------------------------------------------------------- +# 9. CloudWatch log group +# --------------------------------------------------------------------------- + +section "CloudWatch log group" + +log_group="/aws/eks/${CLUSTER_ID}/cluster" + +if aws logs describe-log-groups \ + --log-group-name-prefix "$log_group" \ + --query "logGroups[?logGroupName=='${log_group}'] | length(@)" \ + --output text 2>/dev/null | grep -q "^[1-9]"; then + pass "CloudWatch log group '${log_group}' exists" +else + fail "CloudWatch log group '${log_group}' not found" +fi + +# --------------------------------------------------------------------------- +# 10. KMS key aliases +# --------------------------------------------------------------------------- + +section "KMS key aliases" + +declare -A KMS_ALIASES=( + ["cloudwatch-logs"]="alias/${CLUSTER_ID}-cloudwatch-logs" + ["eks-secrets"]="alias/${CLUSTER_ID}-eks-secrets" +) + +for label in "${!KMS_ALIASES[@]}"; do + alias_name="${KMS_ALIASES[$label]}" + if aws kms list-aliases \ + --query "Aliases[?AliasName=='${alias_name}'] | length(@)" \ + --output text 2>/dev/null | grep -q "^[1-9]"; then + pass "KMS alias '${alias_name}' (${label}) exists" + else + fail "KMS alias '${alias_name}' (${label}) not found" + fi +done + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + +section "Summary" +echo -e " ${GREEN}PASS${RESET}: ${PASS} ${RED}FAIL${RESET}: ${FAIL} ${YELLOW}WARN${RESET}: ${WARN}" + +[[ "$FAIL" -eq 0 ]] diff --git a/scripts/validate-rc-k8s.sh b/scripts/validate-rc-k8s.sh new file mode 100755 index 000000000..75306085e --- /dev/null +++ b/scripts/validate-rc-k8s.sh @@ -0,0 +1,351 @@ +#!/usr/bin/env bash +# Validate Kubernetes-level processes on the Regional Cluster (RC). +# +# Usage: +# ./scripts/validate-rc-k8s.sh # auto-derives CLUSTER_ID from kubectl context +# CLUSTER_ID= ./scripts/validate-rc-k8s.sh # override if needed +# +# Prerequisites: active kubectl context pointing at the RC, kubectl/jq on PATH. + +set -euo pipefail + +# Auto-derive CLUSTER_ID from the active kubectl context when not set explicitly. +# aws eks update-kubeconfig names contexts: arn:aws:eks:::cluster/ +_ctx=$(kubectl config current-context 2>/dev/null || true) +if [[ -z "${CLUSTER_ID:-}" && "$_ctx" =~ :cluster/(.+)$ ]]; then + CLUSTER_ID="${BASH_REMATCH[1]}" +fi +CLUSTER_ID="${CLUSTER_ID:?Cannot derive CLUSTER_ID — set it manually or ensure the active kubectl context points at an EKS cluster}" + +# --------------------------------------------------------------------------- +# Output helpers +# --------------------------------------------------------------------------- + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +RESET='\033[0m' + +PASS=0 +FAIL=0 +WARN=0 + +pass() { echo -e "${GREEN}[PASS]${RESET} $*"; ((++PASS)); } +fail() { echo -e "${RED}[FAIL]${RESET} $*"; ((++FAIL)); } +warn() { echo -e "${YELLOW}[WARN]${RESET} $*"; ((++WARN)); } + +section() { echo; echo "=== $* ==="; } + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# Returns 0 if all pods in a namespace with an optional label selector are Running. +# $1=namespace $2=optional label selector (e.g. app=foo) +pods_running() { + local ns="$1" selector="${2:-}" + local args=(-n "$ns" --no-headers) + [[ -n "$selector" ]] && args+=(-l "$selector") + + if ! kubectl get pods "${args[@]}" 2>/dev/null | grep -q .; then + return 2 # no pods found + fi + + local not_running + not_running=$(kubectl get pods "${args[@]}" 2>/dev/null \ + | awk '$3 ~ /^(Running|Completed)$/ { if ($3 == "Running") { split($2, r, "/"); if (r[1]+0 < r[2]+0) print }; next } { print }' \ + || true) + [[ -z "$not_running" ]] +} + +# Returns pod count in a namespace with optional label selector. +pod_count() { + local ns="$1" selector="${2:-}" + local args=(-n "$ns" --no-headers) + [[ -n "$selector" ]] && args+=(-l "$selector") + kubectl get pods "${args[@]}" 2>/dev/null | grep -c . || echo 0 +} + +# --------------------------------------------------------------------------- +# 1. Nodes +# --------------------------------------------------------------------------- + +section "Nodes" + +if ! _nodes_raw=$(kubectl get nodes --no-headers 2>/dev/null); then + fail "Cannot list nodes — check kubeconfig and RBAC" +else + not_ready=$(echo "$_nodes_raw" | awk '{print $2}' | grep -v "^Ready$" || true) + if [[ -z "$not_ready" ]]; then + node_count=$(echo "$_nodes_raw" | wc -l | tr -d ' ') + pass "All ${node_count} nodes are Ready" + else + fail "Nodes not Ready: $(echo "$not_ready" | wc -l | tr -d ' ') node(s)" + fi +fi + +bootstrap_nodes=$(kubectl get nodes -l "eks.amazonaws.com/nodegroup=${CLUSTER_ID}-karpenter-bootstrap" \ + --no-headers 2>/dev/null | wc -l | tr -d ' ') +if [[ "$bootstrap_nodes" -ge 2 ]]; then + pass "Karpenter bootstrap node group: ${bootstrap_nodes} node(s) present" +else + fail "Karpenter bootstrap node group: expected ≥2 nodes, found ${bootstrap_nodes}" +fi + +taint_count=$(kubectl get nodes -l "eks.amazonaws.com/nodegroup=${CLUSTER_ID}-karpenter-bootstrap" \ + -o json 2>/dev/null \ + | jq '[.items[] | select((.spec.taints // []) | any(.key == "CriticalAddonsOnly"))] | length') +if [[ "$taint_count" -eq "$bootstrap_nodes" && "$bootstrap_nodes" -ge 1 ]]; then + pass "Bootstrap nodes have CriticalAddonsOnly taint (${taint_count}/${bootstrap_nodes})" +else + warn "CriticalAddonsOnly taint missing on some bootstrap nodes (${taint_count}/${bootstrap_nodes} tainted)" +fi + +# --------------------------------------------------------------------------- +# 2. Karpenter +# --------------------------------------------------------------------------- + +section "Karpenter" + +if pods_running kube-system "app.kubernetes.io/name=karpenter"; then + kp_count=$(pod_count kube-system "app.kubernetes.io/name=karpenter") + pass "Karpenter pods Running (${kp_count})" +else + fail "Karpenter pods not all Running in kube-system" +fi + +# Verify Karpenter controller runs on bootstrap nodes (not on nodes it would provision) +kp_nodes=$(kubectl get pods -n kube-system -l "app.kubernetes.io/name=karpenter" \ + -o jsonpath='{.items[*].spec.nodeName}' 2>/dev/null || true) +if [[ -z "$kp_nodes" ]]; then + warn "Karpenter pods have no nodeName assigned yet — still scheduling?" +else + off_bootstrap=0 + for node in $kp_nodes; do + ng=$(kubectl get node "$node" \ + -o jsonpath='{.metadata.labels.eks\.amazonaws\.com/nodegroup}' 2>/dev/null || true) + if [[ "$ng" != "${CLUSTER_ID}-karpenter-bootstrap" ]]; then + ((off_bootstrap++)) + fi + done + if [[ "$off_bootstrap" -eq 0 ]]; then + pass "Karpenter pods scheduled on bootstrap node group" + else + warn "${off_bootstrap} Karpenter pod(s) NOT on bootstrap node group" + fi +fi + +ec2nc_ready=$(kubectl get ec2nodeclass fips \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) +if [[ "$ec2nc_ready" == "True" ]]; then + pass "EC2NodeClass 'fips' Ready=True" +else + # Distinguish transient vs hard failure + val_reason=$(kubectl get ec2nodeclass fips \ + -o jsonpath='{.status.conditions[?(@.type=="ValidationSucceeded")].message}' 2>/dev/null || true) + fail "EC2NodeClass 'fips' Ready=${ec2nc_ready:-Unknown} — ${val_reason:-no detail}" +fi + +# The RC NodePool is named 'regional-workloads'; check all NodePools so this +# doesn't break if the name changes. +_np_names=$(kubectl get nodepool --no-headers 2>/dev/null | awk '{print $1}' || true) +if [[ -z "$_np_names" ]]; then + fail "No NodePools found" +else + while IFS= read -r _np; do + np_ready=$(kubectl get nodepool "$_np" \ + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) + if [[ "$np_ready" == "True" ]]; then + pass "NodePool '${_np}' Ready=True" + else + fail "NodePool '${_np}' Ready=${np_ready:-Unknown}" + echo " [diag] NodePool conditions:" + kubectl get nodepool "$_np" -o jsonpath='{.status.conditions}' 2>/dev/null \ + | jq -r '.[] | " \(.type)=\(.status): \(.message // "-")"' 2>/dev/null || true + echo " [diag] nodeClassRef: $(kubectl get nodepool "$_np" \ + -o jsonpath='{.spec.template.spec.nodeClassRef.name}' 2>/dev/null || echo 'unknown')" + echo " [diag] Recent Karpenter logs (errors):" + kubectl logs -n kube-system -l "app.kubernetes.io/name=karpenter" --tail=50 2>/dev/null \ + | grep -iE "nodepool|error|failed" | tail -10 | sed 's/^/ /' || true + fi + done <<< "$_np_names" +fi + +nc_count=$(kubectl get nodeclaims --no-headers 2>/dev/null | wc -l | tr -d ' ') +if [[ "$nc_count" -ge 1 ]]; then + pass "NodeClaims present (${nc_count}) — Karpenter has provisioned nodes" +else + warn "No NodeClaims found — Karpenter has not yet provisioned any nodes" +fi + +# --------------------------------------------------------------------------- +# 3. AWS Load Balancer Controller +# --------------------------------------------------------------------------- + +section "AWS Load Balancer Controller" + +if pods_running aws-load-balancer-controller "app.kubernetes.io/name=aws-load-balancer-controller"; then + lbc_count=$(pod_count aws-load-balancer-controller "app.kubernetes.io/name=aws-load-balancer-controller") + pass "LBC pods Running (${lbc_count})" +else + fail "LBC pods not all Running in aws-load-balancer-controller" +fi + +if kubectl get crd targetgroupbindings.elbv2.k8s.aws &>/dev/null; then + pass "TargetGroupBinding CRD (elbv2.k8s.aws) registered" +else + fail "TargetGroupBinding CRD missing — LBC may not have started cleanly" +fi + +# --------------------------------------------------------------------------- +# 4. Core add-on daemonsets / deployments (kube-system) +# --------------------------------------------------------------------------- + +section "Core add-ons (kube-system)" + +declare -A CORE_SELECTORS=( + ["CoreDNS"]="k8s-app=kube-dns" + ["metrics-server"]="app.kubernetes.io/name=metrics-server" + ["vpc-cni (aws-node)"]="k8s-app=aws-node" + ["kube-proxy"]="k8s-app=kube-proxy" + ["ebs-csi-node"]="app=ebs-csi-node" + ["ebs-csi-controller"]="app=ebs-csi-controller" + ["secrets-store-csi"]="app=secrets-store-csi-driver" +) + +for label in "${!CORE_SELECTORS[@]}"; do + selector="${CORE_SELECTORS[$label]}" + rc=0 + pods_running kube-system "$selector" || rc=$? + if [[ $rc -eq 0 ]]; then + pass "${label} Running" + elif [[ $rc -eq 2 ]]; then + warn "${label}: no pods found (may not be installed)" + else + fail "${label}: pods not all Running" + fi +done + +# Secrets Store CSI also deploys as provider in kube-system +if pods_running kube-system "app=csi-secrets-store-provider-aws"; then + pass "AWS Secrets Store CSI provider Running" +else + warn "AWS Secrets Store CSI provider: not found" +fi + +# pod-identity-agent is installed as an EKS addon (Terraform-managed). The addon +# DaemonSet may not carry the standard app label, so check by DaemonSet name first. +_pia_rc=0 +pods_running kube-system "app.kubernetes.io/name=eks-pod-identity-agent" || _pia_rc=$? +if [[ $_pia_rc -eq 0 ]]; then + pass "pod-identity-agent Running" +elif kubectl get daemonset eks-pod-identity-agent -n kube-system &>/dev/null; then + _pia_desired=$(kubectl get daemonset eks-pod-identity-agent -n kube-system \ + -o jsonpath='{.status.desiredNumberScheduled}' 2>/dev/null || echo 0) + _pia_ready=$(kubectl get daemonset eks-pod-identity-agent -n kube-system \ + -o jsonpath='{.status.numberReady}' 2>/dev/null || echo 0) + if [[ "$_pia_ready" -ge 1 ]]; then + pass "pod-identity-agent Running (${_pia_ready}/${_pia_desired} ready, EKS addon)" + else + fail "pod-identity-agent DaemonSet exists but ${_pia_ready}/${_pia_desired} pods ready" + kubectl get pods -n kube-system -l "app.kubernetes.io/name=eks-pod-identity-agent" \ + --no-headers 2>/dev/null | sed 's/^/ /' || true + kubectl get events -n kube-system \ + --field-selector "involvedObject.name=eks-pod-identity-agent" \ + --sort-by='.lastTimestamp' 2>/dev/null | tail -5 | sed 's/^/ /' || true + fi +else + warn "pod-identity-agent: DaemonSet not found — EKS addon may not be installed" +fi + +# --------------------------------------------------------------------------- +# 5. Platform services +# --------------------------------------------------------------------------- + +section "Platform services" + +declare -A PLATFORM_NS=( + ["platform-api"]="platform-api" + ["maestro-server"]="maestro-server" +) + +for svc in "${!PLATFORM_NS[@]}"; do + ns="${PLATFORM_NS[$svc]}" + rc=0 + pods_running "$ns" || rc=$? + if [[ $rc -eq 0 ]]; then + count=$(pod_count "$ns") + pass "${svc}: ${count} pod(s) Running" + elif [[ $rc -eq 2 ]]; then + warn "${svc}: namespace '${ns}' has no pods yet" + else + fail "${svc}: pods not all Running in ${ns}" + fi +done + +# --------------------------------------------------------------------------- +# 6. ArgoCD +# --------------------------------------------------------------------------- + +section "ArgoCD" + +if pods_running argocd "app.kubernetes.io/name=argocd-server"; then + pass "ArgoCD server Running" +else + fail "ArgoCD server not Running" +fi + +if ! _apps_raw=$(kubectl get applications -n argocd --no-headers 2>/dev/null); then + fail "ArgoCD: cannot list applications — check RBAC and ArgoCD CRD availability" +else + not_synced=$(echo "$_apps_raw" \ + | awk '{print $1, $2, $3}' | grep -v "Synced.*Healthy" | grep -v "Synced.*Progressing" || true) + if [[ -z "$not_synced" ]]; then + total=$(echo "$_apps_raw" | wc -l | tr -d ' ') + pass "All ${total} ArgoCD applications Synced" + else + count=$(echo "$not_synced" | wc -l | tr -d ' ') + fail "${count} ArgoCD application(s) not Synced/Healthy:" + echo "$not_synced" | sed 's/^/ /' + while IFS= read -r _line; do + _app=$(echo "$_line" | awk '{print $1}') + _sync=$(echo "$_line" | awk '{print $2}') + _health=$(echo "$_line" | awk '{print $3}') + echo " [diag] ${_app} (${_sync}/${_health}):" + if [[ "$_sync" == "OutOfSync" ]]; then + kubectl get application "$_app" -n argocd \ + -o jsonpath='{.status.resources}' 2>/dev/null \ + | jq -r '.[] | select(.status != "Synced") | " \(.kind)/\(.name): \(.status) \(.health.status // "")"' \ + 2>/dev/null | head -10 || true + fi + if [[ "$_health" == "Degraded" ]]; then + _app_health_msg=$(kubectl get application "$_app" -n argocd \ + -o jsonpath='{.status.health.message}' 2>/dev/null || true) + [[ -n "$_app_health_msg" ]] && echo " health: ${_app_health_msg}" + kubectl get application "$_app" -n argocd \ + -o jsonpath='{.status.conditions}' 2>/dev/null \ + | jq -r '.[]? | " condition: \(.type): \(.message // "-")"' 2>/dev/null || true + kubectl get application "$_app" -n argocd \ + -o jsonpath='{.status.resources}' 2>/dev/null \ + | jq -r '.[]? | select(.health.status == "Degraded" or .health.status == "Missing" or .health.status == "Unknown") | " \(.kind)/\(.name): \(.health.status) — \(.health.message // "-")"' \ + 2>/dev/null | head -10 || true + fi + done <<< "$not_synced" + fi + + progressing=$(echo "$_apps_raw" \ + | awk '{print $1, $2, $3}' | grep "Progressing" || true) + if [[ -n "$progressing" ]]; then + warn "Applications still progressing:" + echo "$progressing" | sed 's/^/ /' + fi +fi + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- + +section "Summary" +echo -e " ${GREEN}PASS${RESET}: ${PASS} ${RED}FAIL${RESET}: ${FAIL} ${YELLOW}WARN${RESET}: ${WARN}" + +[[ "$FAIL" -eq 0 ]] diff --git a/terraform/config/management-cluster/main.tf b/terraform/config/management-cluster/main.tf index 5911d9bc2..e59234393 100755 --- a/terraform/config/management-cluster/main.tf +++ b/terraform/config/management-cluster/main.tf @@ -67,6 +67,9 @@ module "ecs_bootstrap" { repository_url = var.repository_url repository_branch = var.repository_branch + + karpenter_controller_role_arn = module.management_cluster.karpenter_controller_role_arn != null ? module.management_cluster.karpenter_controller_role_arn : "" + karpenter_queue_url = module.management_cluster.karpenter_queue_url != null ? module.management_cluster.karpenter_queue_url : "" } # ============================================================================= diff --git a/terraform/config/regional-cluster/imports.sh b/terraform/config/regional-cluster/imports.sh index 66ba48f77..4d6172b5b 100644 --- a/terraform/config/regional-cluster/imports.sh +++ b/terraform/config/regional-cluster/imports.sh @@ -52,16 +52,45 @@ BROKER_ID=$(tf_state_value \ 'module.hyperfleet_infrastructure.aws_mq_broker.hyperfleet' '.values.id') echo " [debug] BROKER_ID=${BROKER_ID:-}" if [ -n "$BROKER_ID" ]; then - import_if_needed \ - 'module.hyperfleet_infrastructure.aws_cloudwatch_log_group.mq_general' \ - "/aws/amazonmq/broker/${BROKER_ID}/general" - import_if_needed \ - 'module.hyperfleet_infrastructure.aws_cloudwatch_log_group.mq_connection' \ - "/aws/amazonmq/broker/${BROKER_ID}/connection" + # The MQ log group names embed the broker UUID, so they can only be created + # after the broker exists. On the first apply after broker creation the log + # groups may not exist in AWS yet — in that case skip the import and let + # Terraform create them. Use an explicit AWS CLI check rather than relying + # on Terraform's error-message pattern matching for this resource type. + _MQ_GENERAL="/aws/amazonmq/broker/${BROKER_ID}/general" + _MQ_CONNECTION="/aws/amazonmq/broker/${BROKER_ID}/connection" + + _GENERAL_EXISTS=$(aws logs describe-log-groups \ + --log-group-name-prefix "$_MQ_GENERAL" \ + --query "logGroups[?logGroupName=='${_MQ_GENERAL}'] | length(@)" \ + --output text 2>/dev/null || echo 0) + if [ "${_GENERAL_EXISTS:-0}" -ge 1 ]; then + import_if_needed \ + 'module.hyperfleet_infrastructure.aws_cloudwatch_log_group.mq_general' \ + "$_MQ_GENERAL" + else + echo " [skip] AmazonMQ general log group — not yet in AWS" + fi + + _CONNECTION_EXISTS=$(aws logs describe-log-groups \ + --log-group-name-prefix "$_MQ_CONNECTION" \ + --query "logGroups[?logGroupName=='${_MQ_CONNECTION}'] | length(@)" \ + --output text 2>/dev/null || echo 0) + if [ "${_CONNECTION_EXISTS:-0}" -ge 1 ]; then + import_if_needed \ + 'module.hyperfleet_infrastructure.aws_cloudwatch_log_group.mq_connection' \ + "$_MQ_CONNECTION" + else + echo " [skip] AmazonMQ connection log group — not yet in AWS" + fi else echo " [skip] AmazonMQ log groups — broker not yet provisioned" fi +import_if_needed \ + 'module.rhobs_api_gateway.aws_cloudwatch_log_group.api_gateway_access' \ + "/aws/api-gateway/${TF_VAR_regional_id}-rhobs/${TF_VAR_stage_name:-prod}/access" + API_ID=$(tf_state_value \ 'module.api_gateway.aws_api_gateway_rest_api.main' '.values.id') STAGE_NAME=$(tf_state_value \ diff --git a/terraform/config/regional-cluster/main.tf b/terraform/config/regional-cluster/main.tf index 71a05a5bf..4fede5022 100755 --- a/terraform/config/regional-cluster/main.tf +++ b/terraform/config/regional-cluster/main.tf @@ -190,6 +190,9 @@ module "ecs_bootstrap" { loki_kms_key_arn = module.loki_infrastructure.kms_key_arn management_clusters = var.management_clusters + + karpenter_controller_role_arn = module.regional_cluster.karpenter_controller_role_arn != null ? module.regional_cluster.karpenter_controller_role_arn : "" + karpenter_queue_url = module.regional_cluster.karpenter_queue_url != null ? module.regional_cluster.karpenter_queue_url : "" } # ============================================================================= @@ -452,6 +455,18 @@ module "cloudwatch_exporter" { cluster_name = module.regional_cluster.cluster_name } +# ============================================================================= +# AWS Load Balancer Controller (Pod Identity for OSS Karpenter clusters) +# +# EKS Auto Mode includes LBC built-in. OSS Karpenter clusters must install it +# explicitly to provide the TargetGroupBinding CRD used by platform-api. +# ============================================================================= + +module "aws_load_balancer_controller" { + source = "../../modules/aws-load-balancer-controller" + cluster_name = module.regional_cluster.cluster_name +} + # ============================================================================= # Regional OIDC Module # diff --git a/terraform/modules/aws-load-balancer-controller/README.md b/terraform/modules/aws-load-balancer-controller/README.md new file mode 100644 index 000000000..b8176707b --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/README.md @@ -0,0 +1,57 @@ +# AWS Load Balancer Controller Module + +Creates the IAM role and EKS Pod Identity association required to run the [AWS Load Balancer Controller](https://kubernetes-sigs.github.io/aws-load-balancer-controller/) on a private EKS cluster. + +## Overview + +The AWS Load Balancer Controller (LBC) provides the `TargetGroupBinding` CRD used by platform +services (Thanos, Loki, RHOBS API Gateway) to wire Kubernetes services to ALB target groups. +LBC replaced the load balancing functionality previously bundled with EKS Auto Mode when clusters +migrated to OSS Karpenter. + +This module provisions: + +- **IAM role** (`-aws-load-balancer-controller`): IAM policy derived from the + [upstream recommended policy](https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.13.3/docs/install/iam_policy.json) +- **EKS Pod Identity association**: Binds the IAM role to the LBC Kubernetes ServiceAccount + +The LBC Helm chart itself is deployed via ArgoCD from +`argocd/config/regional-cluster/aws-load-balancer-controller/`. + +## Usage + +```hcl +module "aws_load_balancer_controller" { + source = "./terraform/modules/aws-load-balancer-controller" + + cluster_name = module.eks_cluster.cluster_name + + tags = { + Environment = var.environment + } +} +``` + +## Variables + +| Name | Description | Type | Default | Required | +| ----------------- | ---------------------------------------------- | ------------- | -------------------------------- | -------- | +| `cluster_name` | Name of the EKS cluster | `string` | n/a | yes | +| `namespace` | Kubernetes namespace where the LBC is deployed | `string` | `"aws-load-balancer-controller"` | no | +| `service_account` | Kubernetes service account name for the LBC | `string` | `"aws-load-balancer-controller"` | no | +| `tags` | Additional tags to apply to resources | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +| ----------------------------- | ------------------------------- | +| `role_name` | IAM role name for the LBC | +| `role_arn` | IAM role ARN for the LBC | +| `pod_identity_association_id` | EKS Pod Identity association ID | + +## Requirements + +| Name | Version | +| --------- | --------- | +| terraform | >= 1.14.3 | +| aws | >= 6.0.0 | diff --git a/terraform/modules/aws-load-balancer-controller/iam.tf b/terraform/modules/aws-load-balancer-controller/iam.tf new file mode 100644 index 000000000..ff73b6859 --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/iam.tf @@ -0,0 +1,333 @@ +# ============================================================================= +# AWS Load Balancer Controller IAM Role and Policies +# +# IAM policy derived from the upstream recommended policy: +# https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/v2.13.3/docs/install/iam_policy.json +# ============================================================================= + +resource "aws_iam_role" "aws_lbc" { + name = "${var.cluster_name}-aws-load-balancer-controller" + description = "IAM role for AWS Load Balancer Controller (provides TargetGroupBinding CRDs)" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { + Service = "pods.eks.amazonaws.com" + } + Action = [ + "sts:AssumeRole", + "sts:TagSession" + ] + }] + }) + + tags = merge( + local.common_tags, + { + Name = "${var.cluster_name}-aws-load-balancer-controller" + } + ) +} + +resource "aws_iam_role_policy" "aws_lbc" { + #checkov:skip=CKV_AWS_355: Describe-only APIs (EC2, ELB, ACM, etc.) do not support resource-level ARN restrictions; Resource="*" matches the upstream AWS Load Balancer Controller recommended policy. + #checkov:skip=CKV_AWS_290: iam:CreateServiceLinkedRole requires Resource="*" per AWS; the statement is already constrained by a Condition on iam:AWSServiceName=elasticloadbalancing.amazonaws.com. + name = "${var.cluster_name}-aws-load-balancer-controller" + role = aws_iam_role.aws_lbc.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "CreateServiceLinkedRole" + Effect = "Allow" + Action = ["iam:CreateServiceLinkedRole"] + Resource = "*" + Condition = { + StringEquals = { + "iam:AWSServiceName" = "elasticloadbalancing.amazonaws.com" + } + } + }, + { + Sid = "EC2Read" + Effect = "Allow" + Action = [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "ec2:GetSecurityGroupsForVpc", + "ec2:DescribeIpamPools", + "ec2:DescribeRouteTables", + ] + Resource = "*" + }, + { + Sid = "ELBRead" + Effect = "Allow" + Action = [ + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTrustStores", + "elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeCapacityReservation", + ] + Resource = "*" + }, + { + Sid = "CognitoRead" + Effect = "Allow" + Action = ["cognito-idp:DescribeUserPoolClient"] + Resource = "*" + }, + { + Sid = "ACMRead" + Effect = "Allow" + Action = [ + "acm:ListCertificates", + "acm:DescribeCertificate", + ] + Resource = "*" + }, + { + Sid = "IAMRead" + Effect = "Allow" + Action = [ + "iam:ListServerCertificates", + "iam:GetServerCertificate", + ] + Resource = "*" + }, + { + Sid = "WAFRead" + Effect = "Allow" + Action = [ + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + ] + Resource = "*" + }, + { + Sid = "WAFv2" + Effect = "Allow" + Action = [ + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + ] + Resource = "*" + }, + { + Sid = "ShieldRead" + Effect = "Allow" + Action = [ + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection", + ] + Resource = "*" + }, + { + Sid = "EC2Mutate" + Effect = "Allow" + Action = [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + ] + Resource = "*" + }, + { + Sid = "EC2CreateSecurityGroup" + Effect = "Allow" + Action = ["ec2:CreateSecurityGroup"] + Resource = "*" + }, + { + Sid = "EC2CreateTags" + Effect = "Allow" + Action = ["ec2:CreateTags"] + Resource = "arn:aws:ec2:*:*:security-group/*" + Condition = { + StringEquals = { + "ec2:CreateAction" = "CreateSecurityGroup" + } + Null = { + "aws:RequestTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "EC2MutateTags" + Effect = "Allow" + Action = [ + "ec2:CreateTags", + "ec2:DeleteTags", + ] + Resource = "arn:aws:ec2:*:*:security-group/*" + Condition = { + Null = { + "aws:RequestTag/elbv2.k8s.aws/cluster" = "true" + "aws:ResourceTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "EC2DeleteSecurityGroup" + Effect = "Allow" + Action = ["ec2:DeleteSecurityGroup"] + Resource = "*" + Condition = { + Null = { + "aws:ResourceTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "ELBCreateTagged" + Effect = "Allow" + Action = [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup", + ] + Resource = "*" + Condition = { + Null = { + "aws:RequestTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "ELBCreateListenerAndRule" + Effect = "Allow" + Action = [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule", + ] + Resource = "*" + }, + { + Sid = "ELBMutateTags" + Effect = "Allow" + Action = [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags", + ] + Resource = [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*", + ] + Condition = { + Null = { + "aws:RequestTag/elbv2.k8s.aws/cluster" = "true" + "aws:ResourceTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "ELBMutateListenerRuleTags" + Effect = "Allow" + Action = [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags", + ] + Resource = [ + "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*", + ] + }, + { + Sid = "ELBMutateTagged" + Effect = "Allow" + Action = [ + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:ModifyListenerAttributes", + "elasticloadbalancing:ModifyCapacityReservation", + ] + Resource = "*" + Condition = { + Null = { + "aws:ResourceTag/elbv2.k8s.aws/cluster" = "false" + } + } + }, + { + Sid = "ELBAddListenerCert" + Effect = "Allow" + Action = [ + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule", + ] + Resource = "*" + }, + { + Sid = "ELBTargets" + Effect = "Allow" + Action = [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets", + ] + Resource = "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + Sid = "ELBMutateAttributes" + Effect = "Allow" + Action = [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + ] + Resource = "*" + }, + ] + }) +} + +resource "aws_eks_pod_identity_association" "aws_lbc" { + cluster_name = var.cluster_name + namespace = var.namespace + service_account = var.service_account + role_arn = aws_iam_role.aws_lbc.arn + + tags = merge( + local.common_tags, + { + Name = "${var.cluster_name}-aws-load-balancer-controller-pod-identity" + } + ) +} diff --git a/terraform/modules/aws-load-balancer-controller/main.tf b/terraform/modules/aws-load-balancer-controller/main.tf new file mode 100644 index 000000000..46741d9f1 --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/main.tf @@ -0,0 +1,9 @@ +locals { + common_tags = merge( + var.tags, + { + Component = "aws-load-balancer-controller" + ManagedBy = "terraform" + } + ) +} diff --git a/terraform/modules/aws-load-balancer-controller/outputs.tf b/terraform/modules/aws-load-balancer-controller/outputs.tf new file mode 100644 index 000000000..9470a5cef --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/outputs.tf @@ -0,0 +1,14 @@ +output "role_name" { + description = "IAM role name for the AWS Load Balancer Controller" + value = aws_iam_role.aws_lbc.name +} + +output "role_arn" { + description = "IAM role ARN for the AWS Load Balancer Controller" + value = aws_iam_role.aws_lbc.arn +} + +output "pod_identity_association_id" { + description = "EKS Pod Identity association ID" + value = aws_eks_pod_identity_association.aws_lbc.association_id +} diff --git a/terraform/modules/aws-load-balancer-controller/variables.tf b/terraform/modules/aws-load-balancer-controller/variables.tf new file mode 100644 index 000000000..23cc4d2a7 --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/variables.tf @@ -0,0 +1,22 @@ +variable "cluster_name" { + description = "Name of the EKS cluster" + type = string +} + +variable "namespace" { + description = "Kubernetes namespace where the LBC is deployed" + type = string + default = "aws-load-balancer-controller" +} + +variable "service_account" { + description = "Kubernetes service account name for the LBC" + type = string + default = "aws-load-balancer-controller" +} + +variable "tags" { + description = "Additional tags to apply to resources" + type = map(string) + default = {} +} diff --git a/terraform/modules/aws-load-balancer-controller/versions.tf b/terraform/modules/aws-load-balancer-controller/versions.tf new file mode 100644 index 000000000..3dccf26c7 --- /dev/null +++ b/terraform/modules/aws-load-balancer-controller/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.14.3" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 6.0" + } + } +} diff --git a/terraform/modules/ecs-bootstrap/README.md b/terraform/modules/ecs-bootstrap/README.md index a2717a282..d41c65099 100644 --- a/terraform/modules/ecs-bootstrap/README.md +++ b/terraform/modules/ecs-bootstrap/README.md @@ -1,6 +1,6 @@ # ECS Bootstrap Module -This Terraform module creates an ECS Fargate infrastructure for external ArgoCD bootstrap execution. It provides acess to secure, auditable tasks to run against the regional/management AWS accounts and EKS cluster. +This Terraform module creates an ECS Fargate infrastructure for external ArgoCD bootstrap execution. It provides access to secure, auditable tasks to run against the regional/management AWS accounts and EKS cluster. ## Overview @@ -24,9 +24,26 @@ module "ecs_bootstrap" { eks_cluster_name = module.eks_cluster.cluster_name eks_cluster_security_group_id = module.eks_cluster.cluster_security_group_id cluster_id = var.regional_id # or var.management_id + + # Karpenter inputs (from eks-cluster module outputs) + karpenter_controller_role_arn = module.eks_cluster.karpenter_controller_role_arn + karpenter_queue_url = module.eks_cluster.karpenter_queue_url + karpenter_version = "1.13.0" } ``` +## Bootstrap Sequence + +The ECS task executes the following steps in order: + +1. **Clone repository**: Checks out the configured git branch +2. **Configure kubectl**: Updates kubeconfig for the private EKS cluster +3. **Wait for addons**: Polls until CoreDNS and metrics-server are Active on the `karpenter-bootstrap` node group +4. **Install Karpenter** (when `karpenter_controller_role_arn` is set): Installs Karpenter via Helm from ECR public; skipped if already deployed +5. **Apply EC2NodeClass and NodePool**: Applies the FIPS `EC2NodeClass` and cluster-type-specific workloads `NodePool` from the `eks-nodepool` chart; always applied (idempotent) so any stale spec is corrected +6. **Prewarm validation**: Schedules a lightweight pod, waits up to 8 minutes for Karpenter to provision an EC2 node and bring it Ready. Failure prints diagnostic output (NodeClass, NodePool, NodeClaims, Karpenter logs) and exits — ECS retries the task automatically +7. **Install ArgoCD**: Installs ArgoCD via Helm and creates the Application of Applications for GitOps self-management + ## Security Features ### Network Security @@ -36,36 +53,39 @@ module "ecs_bootstrap" { ### IAM Security -- **EKS Access Entries**: Uses EKS access entry mechanism for Kubernetes RBAC - which can later receive further fine grained permissions -- **Minimal Permissions**: Task role has only required EKS and SSM permissions +- **EKS Access Entries**: Uses EKS access entry mechanism for Kubernetes RBAC +- **Minimal Permissions**: Task role has only required EKS, SSM, and Helm/kubectl permissions ### Audit Trail -- **CloudWatch Logs**: Complete logging of all bootstrap operations +- **CloudWatch Logs**: Complete logging of all bootstrap operations including Karpenter prewarm diagnostics - **ECS Task Tracking**: Task execution history and status - **Infrastructure as Code**: All permissions and configuration defined in Terraform ## Inputs -| Name | Description | Type | Default | Required | -| ----------------------------- | ----------------------------------------------------------------- | -------------- | ------- | :------: | -| cluster_id | Cluster identifier for resource naming (e.g., `regional`, `mc01`) | `string` | n/a | yes | -| vpc_id | VPC ID for ECS task execution | `string` | n/a | yes | -| private_subnets | Private subnet IDs for task execution | `list(string)` | n/a | yes | -| eks_cluster_arn | EKS cluster ARN for bootstrap configuration | `string` | n/a | yes | -| eks_cluster_name | EKS cluster name for bootstrap configuration | `string` | n/a | yes | -| eks_cluster_security_group_id | EKS cluster security group ID | `string` | n/a | yes | -| environment | Environment name for tagging | `string` | `"dev"` | no | +| Name | Description | Type | Default | Required | +| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------- | :------: | +| `cluster_id` | Cluster identifier for resource naming (e.g., `regional`, `mc01`) | `string` | n/a | yes | +| `vpc_id` | VPC ID for ECS task execution | `string` | n/a | yes | +| `private_subnets` | Private subnet IDs for task execution | `list(string)` | n/a | yes | +| `eks_cluster_arn` | EKS cluster ARN for bootstrap configuration | `string` | n/a | yes | +| `eks_cluster_name` | EKS cluster name for bootstrap configuration | `string` | n/a | yes | +| `eks_cluster_security_group_id` | EKS cluster security group ID | `string` | n/a | yes | +| `karpenter_controller_role_arn` | IAM role ARN for Karpenter controller (IRSA). Set from `eks_cluster.karpenter_controller_role_arn`. When non-empty, Karpenter is installed before ArgoCD. | `string` | `""` | no | +| `karpenter_queue_url` | SQS queue URL for Karpenter EC2 interruption handling | `string` | `""` | no | +| `karpenter_version` | Karpenter Helm chart version to install (e.g., `"1.13.0"`) | `string` | `""` | no | +| `environment` | Environment name for tagging | `string` | `"dev"` | no | ## Outputs -| Name | Description | -| --------------------------- | ------------------------------------------------------ | -| ecs_cluster_arn | ARN of the ECS cluster for bootstrap tasks | -| task_definition_arn | ARN of the ECS task definition for bootstrap execution | -| log_group_name | CloudWatch log group name for bootstrap operations | -| bootstrap_security_group_id | Security group ID for bootstrap ECS tasks | -| private_subnets | Private subnet IDs where bootstrap tasks run | +| Name | Description | +| ----------------------------- | ------------------------------------------------------ | +| `ecs_cluster_arn` | ARN of the ECS cluster for bootstrap tasks | +| `task_definition_arn` | ARN of the ECS task definition for bootstrap execution | +| `log_group_name` | CloudWatch log group name for bootstrap operations | +| `bootstrap_security_group_id` | Security group ID for bootstrap ECS tasks | +| `private_subnets` | Private subnet IDs where bootstrap tasks run | ## Requirements @@ -73,10 +93,3 @@ module "ecs_bootstrap" { | --------- | --------- | | terraform | >= 1.14.3 | | aws | >= 5.0 | - -## Future Enhancements - -This ECS infrastructure is designed to support future SRE operations beyond bootstrap: - -- **Operational Tasks**: Cluster maintenance, backup operations, monitoring setup -- **Pre-built Containers**: In the future ad-hoc script pulling will be replaced by versioned containers built through konflux diff --git a/terraform/modules/ecs-bootstrap/main.tf b/terraform/modules/ecs-bootstrap/main.tf index 4c12ebcb1..fcd51cd7e 100644 --- a/terraform/modules/ecs-bootstrap/main.tf +++ b/terraform/modules/ecs-bootstrap/main.tf @@ -103,15 +103,51 @@ resource "aws_ecs_task_definition" "bootstrap" { # Configure kubectl for EKS aws eks update-kubeconfig --name $CLUSTER_NAME - # Seed the FIPS NodePool only on first bootstrap. On subsequent - # runs (resync), ArgoCD owns this resource via the eks-nodepool - # chart — we must not re-apply it to avoid Server-Side Apply - # ownership conflicts. When creating, we pass the environment - # values file so the initial NodePool matches what ArgoCD will - # enforce, avoiding Karpenter provisioning nodes with default - # instance types before ArgoCD syncs. - if ! kubectl get nodepool workloads 2>/dev/null; then - echo "Applying FIPS NodeClass and workloads NodePool from chart..." + # Wait for coredns and metrics-server (on the bootstrap node group) + # before installing Karpenter and ArgoCD. + for ADDON in coredns metrics-server; do + echo "Waiting for $ADDON to be active..." + aws eks wait addon-active \ + --cluster-name "$CLUSTER_NAME" \ + --addon-name "$ADDON" \ + --region "$AWS_REGION" + echo "✓ $ADDON active" + done + + if [ -n "$${KARPENTER_CONTROLLER_ROLE_ARN:-}" ]; then + # Install Karpenter before seeding the NodePool: the NodePool and + # EC2NodeClass CRDs (karpenter.sh/v1, karpenter.k8s.aws/v1) don't + # exist until Karpenter is installed. ArgoCD adopts this release + # via its self-managed Karpenter Application after bootstrap. + _KARPENTER_READY=$(kubectl get deployment karpenter -n kube-system \ + -o jsonpath='{.status.readyReplicas}' 2>/dev/null || true) + if [ -z "$_KARPENTER_READY" ] || [ "$_KARPENTER_READY" -lt 1 ]; then + echo "Installing Karpenter $KARPENTER_VERSION..." + _KARPENTER_QUEUE_NAME=$(basename "$KARPENTER_QUEUE_URL") + helm upgrade --install karpenter \ + oci://public.ecr.aws/karpenter/karpenter \ + --version "$KARPENTER_VERSION" \ + --namespace kube-system \ + --set "settings.clusterName=$CLUSTER_NAME" \ + --set "settings.interruptionQueue=$_KARPENTER_QUEUE_NAME" \ + --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=$KARPENTER_CONTROLLER_ROLE_ARN" \ + --set 'tolerations[0].key=CriticalAddonsOnly' \ + --set 'tolerations[0].operator=Exists' \ + --set 'tolerations[0].effect=NoSchedule' \ + --wait --timeout=5m + echo "✓ Karpenter installed" + else + echo "✓ Karpenter ready (readyReplicas=$_KARPENTER_READY), skipping" + fi + + # Always apply the EC2NodeClass and NodePool from the current chart. + # kubectl apply --server-side is idempotent — it patches in-place. + # The original skip-if-exists guard caused a bootstrap bug: the first + # run seeded the EC2NodeClass with the wrong IAM role name, and all + # subsequent runs silently kept the broken spec, so Karpenter could + # never provision nodes. ArgoCD eventually owns these resources, but + # we must ensure the correct spec is present before ArgoCD is up. + echo "Applying FIPS EC2NodeClass and workloads NodePool from chart..." _NODEPOOL_VALUES="$REPO_DIR/deploy/$ENVIRONMENT/$REGION_DEPLOYMENT/argocd-values-$CLUSTER_TYPE.yaml" _VALUES_FLAG="" [ -f "$_NODEPOOL_VALUES" ] && _VALUES_FLAG="-f $_NODEPOOL_VALUES" @@ -119,61 +155,146 @@ resource "aws_ecs_task_definition" "bootstrap" { --set global.cluster_name="$CLUSTER_NAME" \ $_VALUES_FLAG \ | kubectl apply --server-side -f - - echo "✓ FIPS NodePool applied" - else - echo "✓ FIPS NodePool already exists, skipping (managed by ArgoCD)" + echo "✓ FIPS EC2NodeClass and NodePool applied" + + # Pre-warm: provision one node now so EC2 API rate limiting from + # Terraform surfaces as an ECS task failure (with automatic retry) + # rather than a silent cascade after ArgoCD is installed. + echo "Pre-warming Karpenter: provisioning one node before ArgoCD install..." + kubectl delete pod karpenter-prewarm -n kube-system --ignore-not-found=true + kubectl apply -f - <<-PREWARM_EOF + apiVersion: v1 + kind: Pod + metadata: + name: karpenter-prewarm + namespace: kube-system + labels: + app: karpenter-prewarm + spec: + containers: + - name: pause + image: public.ecr.aws/eks-distro/kubernetes/pause:3.9 + resources: + requests: + cpu: 100m + memory: 128Mi + terminationGracePeriodSeconds: 0 + PREWARM_EOF + if ! kubectl wait pod karpenter-prewarm -n kube-system --for=condition=Ready --timeout=8m; then + echo "=== PREWARM TIMEOUT — diagnostic dump ===" + echo "--- EC2NodeClass fips ---" + kubectl get ec2nodeclass fips -o yaml 2>/dev/null || true + echo "--- NodePool management-workloads ---" + kubectl get nodepool management-workloads -o yaml 2>/dev/null || true + echo "--- NodeClaims ---" + kubectl get nodeclaims -o yaml 2>/dev/null || true + echo "--- Karpenter controller logs (last 200 lines) ---" + kubectl logs -n kube-system -l app.kubernetes.io/name=karpenter --tail=200 --since=15m 2>/dev/null || true + echo "--- Prewarm pod events ---" + kubectl describe pod karpenter-prewarm -n kube-system 2>/dev/null || true + echo "--- All nodes ---" + kubectl get nodes -o wide 2>/dev/null || true + exit 1 + fi + kubectl delete pod karpenter-prewarm -n kube-system --wait=false + echo "✓ Karpenter node provisioned, proceeding with ArgoCD install" fi - # Wait for coredns and metrics-server (managed by the built-in system pool) - # to be active before installing ArgoCD. - for ADDON in coredns metrics-server; do - echo "Waiting for $ADDON to be active..." - aws eks wait addon-active \ - --cluster-name "$CLUSTER_NAME" \ - --addon-name "$ADDON" \ - --region "$AWS_REGION" - echo "✓ $ADDON active" + # If a previous bootstrap run failed mid-install, the Helm release is + # left in 'failed' state. Running helm upgrade on a failed HA ArgoCD + # install causes a StatefulSet rolling-update deadlock: redis-ha uses + # OrderedReady policy, so pod-0 must be Ready before pod-1 is created, + # but pod-0's Sentinel readiness probe requires quorum from pods 1 & 2. + # Fix: uninstall the broken release so the next helm upgrade --install + # does a clean initial install with all pods created from scratch. + if helm status argocd -n argocd 2>/dev/null | grep -q "^STATUS: failed\|^STATUS: pending"; then + echo "ArgoCD Helm release is in a broken state, uninstalling for clean reinstall..." + helm uninstall argocd -n argocd 2>/dev/null || true + kubectl wait --for=delete pod --all -n argocd --timeout=120s 2>/dev/null || true + fi + + echo "Installing/upgrading ArgoCD from repo chart..." + + # Create argocd namespace + kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - + + # Re-stamp Helm release ownership annotations before upgrade. + # ArgoCD's default client-side apply strips meta.helm.sh/* annotations + # because they are not part of chart templates: the 3-way merge removes + # keys present in the last-applied-configuration but absent from the new + # desired state. Without these annotations helm upgrade refuses to manage + # the resource ("cannot be imported into the current release"). + # This is a no-op on fresh clusters where no resources exist yet. + echo "Re-stamping Helm release ownership annotations on existing argocd resources..." + for _RT in \ + deployments statefulsets services configmaps serviceaccounts \ + roles rolebindings secrets \ + poddisruptionbudgets horizontalpodautoscalers networkpolicies \ + servicemonitors prometheusrules podmonitors; do + kubectl get "$_RT" -n argocd -o name 2>/dev/null | while read -r _RES; do + kubectl annotate -n argocd "$_RES" \ + "meta.helm.sh/release-name=argocd" \ + "meta.helm.sh/release-namespace=argocd" \ + --overwrite 2>/dev/null || true + done || true done - # Check if ArgoCD already exists - if ! kubectl get deployment argocd-server -n argocd 2>/dev/null; then - echo "Installing ArgoCD from repo chart..." - - # Create argocd namespace - kubectl create namespace argocd --dry-run=client -o yaml | kubectl apply -f - - - # Fetch chart dependencies (charts/ is gitignored) - helm repo add argo https://argoproj.github.io/argo-helm - helm dependency build "$REPO_DIR/argocd/config/shared/argocd" - - # Install using the same chart that the self-managed ArgoCD app - # uses (argocd/config/shared/argocd/), with tracking-id annotations - # so the self-managed ArgoCD app can adopt these resources. - # redisSecretInit is enabled here to create the Redis auth secret; - # the self-managed ArgoCD app has it disabled and prunes the - # completed Job on adoption. - helm upgrade --install argocd "$REPO_DIR/argocd/config/shared/argocd" \ - --namespace argocd \ - --set argo-cd.redisSecretInit.enabled=true \ - --set 'argo-cd.redisSecretInit.tolerations[0].key=CriticalAddonsOnly' \ - --set 'argo-cd.redisSecretInit.tolerations[0].operator=Exists' \ - --set 'argo-cd.redisSecretInit.tolerations[0].effect=NoSchedule' \ - --set-string 'argo-cd.controller.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ - --set-string 'argo-cd.server.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ - --set-string 'argo-cd.repoServer.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ - --wait --timeout=5m - - echo "✓ ArgoCD installation complete" - - # Wait for ArgoCD to be ready - kubectl wait --for=condition=available --timeout=600s deployment/argocd-server -n argocd - kubectl wait --for=condition=available --timeout=600s deployment/argocd-repo-server -n argocd - kubectl wait --for=condition=available --timeout=600s deployment/argocd-applicationset-controller -n argocd - - echo "✓ ArgoCD is running and ready" - else - echo "✓ ArgoCD is already installed and running, skipping installation" - fi + # Fetch chart dependencies (charts/ is gitignored) + helm repo add argo https://argoproj.github.io/argo-helm + helm dependency build "$REPO_DIR/argocd/config/shared/argocd" + + # Install using the same chart that the self-managed ArgoCD app + # uses (argocd/config/shared/argocd/), with tracking-id annotations + # so the self-managed ArgoCD app can adopt these resources. + # redisSecretInit is enabled here to create the Redis auth secret; + # the self-managed ArgoCD app has it disabled and prunes the + # completed Job on adoption. + # + # CriticalAddonsOnly tolerations are set both here (via --set, for + # any git branch) and in values.yaml (for ArgoCD self-management). + helm upgrade --install argocd "$REPO_DIR/argocd/config/shared/argocd" \ + --namespace argocd \ + --set argo-cd.redisSecretInit.enabled=true \ + --set 'argo-cd.redisSecretInit.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.redisSecretInit.tolerations[0].operator=Exists' \ + --set 'argo-cd.redisSecretInit.tolerations[0].effect=NoSchedule' \ + --set 'argo-cd.server.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.server.tolerations[0].operator=Exists' \ + --set 'argo-cd.server.tolerations[0].effect=NoSchedule' \ + --set 'argo-cd.controller.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.controller.tolerations[0].operator=Exists' \ + --set 'argo-cd.controller.tolerations[0].effect=NoSchedule' \ + --set 'argo-cd.repoServer.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.repoServer.tolerations[0].operator=Exists' \ + --set 'argo-cd.repoServer.tolerations[0].effect=NoSchedule' \ + --set 'argo-cd.applicationSet.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.applicationSet.tolerations[0].operator=Exists' \ + --set 'argo-cd.applicationSet.tolerations[0].effect=NoSchedule' \ + --set 'argo-cd.dex.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.dex.tolerations[0].operator=Exists' \ + --set 'argo-cd.dex.tolerations[0].effect=NoSchedule' \ + --set 'argo-cd.notifications.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.notifications.tolerations[0].operator=Exists' \ + --set 'argo-cd.notifications.tolerations[0].effect=NoSchedule' \ + --set 'argo-cd.redis-ha.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.redis-ha.tolerations[0].operator=Exists' \ + --set 'argo-cd.redis-ha.tolerations[0].effect=NoSchedule' \ + --set 'argo-cd.redis-ha.haproxy.tolerations[0].key=CriticalAddonsOnly' \ + --set 'argo-cd.redis-ha.haproxy.tolerations[0].operator=Exists' \ + --set 'argo-cd.redis-ha.haproxy.tolerations[0].effect=NoSchedule' \ + --set-string 'argo-cd.controller.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ + --set-string 'argo-cd.server.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ + --set-string 'argo-cd.repoServer.annotations.argocd\.argoproj\.io/tracking-id=argocd:argoproj.io/Application:argocd/argocd' \ + --wait --timeout=10m + + echo "✓ ArgoCD installation complete" + + # Wait for ArgoCD to be ready + kubectl wait --for=condition=available --timeout=600s deployment/argocd-server -n argocd + kubectl wait --for=condition=available --timeout=600s deployment/argocd-repo-server -n argocd + kubectl wait --for=condition=available --timeout=600s deployment/argocd-applicationset-controller -n argocd + + echo "✓ ArgoCD is running and ready" echo "Creating/updating cluster identity secret with values:" echo " ENVIRONMENT: $ENVIRONMENT" @@ -281,6 +402,18 @@ resource "aws_ecs_task_definition" "bootstrap" { { name = "MANAGEMENT_CLUSTERS" value = var.management_clusters + }, + { + name = "KARPENTER_CONTROLLER_ROLE_ARN" + value = var.karpenter_controller_role_arn + }, + { + name = "KARPENTER_QUEUE_URL" + value = var.karpenter_queue_url + }, + { + name = "KARPENTER_VERSION" + value = var.karpenter_version } ] diff --git a/terraform/modules/ecs-bootstrap/variables.tf b/terraform/modules/ecs-bootstrap/variables.tf index 9e143bb72..6c6577774 100644 --- a/terraform/modules/ecs-bootstrap/variables.tf +++ b/terraform/modules/ecs-bootstrap/variables.tf @@ -65,3 +65,21 @@ variable "management_clusters" { default = "" } +variable "karpenter_controller_role_arn" { + description = "IAM role ARN for the Karpenter controller (IRSA). Required when the EKS cluster uses OSS Karpenter." + type = string + default = "" +} + +variable "karpenter_queue_url" { + description = "SQS queue URL for Karpenter interruption handling." + type = string + default = "" +} + +variable "karpenter_version" { + description = "Karpenter Helm chart version to install during bootstrap." + type = string + default = "1.13.0" +} + diff --git a/terraform/modules/eks-cluster/README.md b/terraform/modules/eks-cluster/README.md index bd3fb0450..7fdfc1a54 100644 --- a/terraform/modules/eks-cluster/README.md +++ b/terraform/modules/eks-cluster/README.md @@ -10,6 +10,7 @@ Creates private EKS clusters with security-first configuration and standardized - **GitOps Bootstrap**: Automated ArgoCD installation via ECS Fargate task for self-management - **Security Hardening**: KMS encryption, IMDSv2 enforcement, and network segmentation - **High Availability**: Multi-AZ NAT Gateways for fault-tolerant egress connectivity +- **OSS Karpenter**: Node provisioning via Karpenter v1 with FIPS-validated EC2NodeClass ## Security & Scalability Enhancements @@ -18,7 +19,7 @@ Creates private EKS clusters with security-first configuration and standardized - **KMS Encryption**: Kubernetes secrets encrypted at rest using customer-managed keys - **Dedicated Security Groups**: VPC endpoints use isolated security groups (port 443 from VPC CIDR only) - **Restricted Egress**: Cluster egress limited to HTTPS for container registries and VPC internal traffic -- **Auto Mode Authentication**: EKS authentication configured for API_AND_CONFIG_MAP mode +- **EKS Authentication**: Configured for API_AND_CONFIG_MAP mode ### High Availability Network Architecture @@ -87,59 +88,64 @@ module "regional_cluster" { ## Variables -| Name | Description | Type | Default | Required | -| ------------------------------- | ------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------- | -------- | -| `cluster_id` | Deterministic cluster identifier for resource naming (e.g., `regional`, `mc01`) | `string` | n/a | yes | -| `cluster_type` | Type of cluster: `regional-cluster` or `management-cluster` | `string` | n/a | yes | -| `cluster_version` | Kubernetes version | `string` | `"1.34"` | no | -| `vpc_cidr` | VPC CIDR block | `string` | `"10.0.0.0/16"` | no | -| `availability_zones` | List of availability zones (auto-detected if empty) | `list(string)` | `[]` | no | -| `private_subnet_cidrs` | CIDR blocks for private subnets | `list(string)` | `["10.0.0.0/18", "10.0.64.0/18", "10.0.128.0/18"]` | no | -| `public_subnet_cidrs` | CIDR blocks for public subnets | `list(string)` | `["10.0.192.0/22", "10.0.196.0/22", "10.0.200.0/22"]` | no | -| `enable_pod_security_standards` | Enable Pod Security Standards | `bool` | `true` | no | -| `bootstrap_enabled` | Enable ArgoCD bootstrap for GitOps management | `bool` | `true` | no | -| `argocd_namespace` | Kubernetes namespace for ArgoCD installation | `string` | `"argocd"` | no | -| `argocd_chart_version` | ArgoCD Helm chart version | `string` | `"9.3.0"` | no | -| `bootstrap_repository_url` | Git repository URL for ArgoCD configuration | `string` | `"https://github.com/openshift-online/rosa-hyperfleet"` | no | -| `bootstrap_repository_branch` | Git branch to track | `string` | `"main"` | no | +| Name | Description | Type | Default | Required | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | ------------------------------------------------------- | -------- | +| `cluster_id` | Deterministic cluster identifier for resource naming (e.g., `regional`, `mc01`) | `string` | n/a | yes | +| `cluster_type` | Type of cluster: `regional-cluster` or `management-cluster` | `string` | n/a | yes | +| `cluster_version` | Kubernetes version | `string` | `"1.34"` | no | +| `vpc_cidr` | VPC CIDR block | `string` | `"10.0.0.0/16"` | no | +| `availability_zones` | List of availability zones (auto-detected if empty) | `list(string)` | `[]` | no | +| `private_subnet_cidrs` | CIDR blocks for private subnets | `list(string)` | `["10.0.0.0/18", "10.0.64.0/18", "10.0.128.0/18"]` | no | +| `public_subnet_cidrs` | CIDR blocks for public subnets | `list(string)` | `["10.0.192.0/22", "10.0.196.0/22", "10.0.200.0/22"]` | no | +| `enable_pod_security_standards` | Enable Pod Security Standards | `bool` | `true` | no | +| `enable_karpenter` | Enable OSS Karpenter instead of EKS Auto Mode. Disables Auto Mode compute, storage, and load balancing. Mutually exclusive with Auto Mode. See `karpenter-node-provisioning.md`. | `bool` | `true` | no | +| `ami_kms_key_arn` | ARN of the Red Hat KMS key encrypting FIPS AMI EBS snapshots. When set, adds `kms:Decrypt` and `kms:CreateGrant` to Karpenter node and controller roles. | `string` | `""` | no | +| `bootstrap_enabled` | Enable ArgoCD bootstrap for GitOps management | `bool` | `true` | no | +| `argocd_namespace` | Kubernetes namespace for ArgoCD installation | `string` | `"argocd"` | no | +| `argocd_chart_version` | ArgoCD Helm chart version | `string` | `"9.3.0"` | no | +| `bootstrap_repository_url` | Git repository URL for ArgoCD configuration | `string` | `"https://github.com/openshift-online/rosa-hyperfleet"` | no | +| `bootstrap_repository_branch` | Git branch to track | `string` | `"main"` | no | ## Outputs -| Name | Description | -| ------------------------------------ | -------------------------------------------------- | -| `cluster_name` | EKS cluster name (same as `cluster_id`) | -| `cluster_endpoint` | EKS cluster API endpoint | -| `cluster_certificate_authority_data` | Base64 encoded certificate data | -| `vpc_id` | VPC ID where cluster is deployed | -| `private_subnets` | Private subnet IDs where worker nodes are deployed | -| `cluster_security_group_id` | EKS cluster security group ID | -| `bootstrap_report` | Bootstrap process information and status | +| Name | Description | +| -------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| `cluster_name` | EKS cluster name (same as `cluster_id`) | +| `cluster_endpoint` | EKS cluster API endpoint | +| `cluster_certificate_authority_data` | Base64 encoded certificate data | +| `vpc_id` | VPC ID where cluster is deployed | +| `private_subnets` | Private subnet IDs where worker nodes are deployed | +| `cluster_security_group_id` | EKS cluster security group ID | +| `karpenter_controller_role_arn` | IAM role ARN for Karpenter controller (IRSA). `null` when `enable_karpenter = false`. | +| `karpenter_queue_url` | SQS queue URL for Karpenter EC2 interruption handling. `null` when `enable_karpenter = false`. | +| `karpenter_node_instance_profile_name` | Instance profile name for Karpenter-provisioned nodes (matches `EC2NodeClass.spec.role`). `null` when `enable_karpenter = false`. | +| `bootstrap_report` | Bootstrap process information and status | ## Bootstrap Functionality -When `bootstrap_enabled` is `true`, the module automatically installs ArgoCD for GitOps management: +When `bootstrap_enabled` is `true`, the module automatically installs Karpenter and ArgoCD via an ECS Fargate task: 1. **ECS Fargate Task**: Executes within cluster VPC for secure bootstrap operations 2. **Tool Installation**: Downloads kubectl, helm, and AWS CLI at runtime -3. **FIPS Node Setup**: Applies FIPS NodeClass and cluster-type-specific workloads NodePool -4. **Addon Wait**: Waits for CoreDNS and metrics-server addons to become Active -5. **ArgoCD Installation**: Installs ArgoCD via Helm with cluster-only access -6. **GitOps Configuration**: Creates Application of Applications for self-management -7. **Synchronous Execution**: Bootstrap completes during `terraform apply` with visible logs +3. **Addon Wait**: Waits for CoreDNS and metrics-server to become Active on the `karpenter-bootstrap` node group +4. **Karpenter Install**: Installs Karpenter via Helm from ECR public (`oci://public.ecr.aws/karpenter/karpenter`) +5. **FIPS Node Setup**: Applies FIPS `EC2NodeClass` (`fips`) and cluster-type-specific workloads `NodePool` +6. **Prewarm Validation**: Provisions one Karpenter node and waits for it to be Ready before continuing +7. **ArgoCD Installation**: Installs ArgoCD via Helm with cluster-only access +8. **GitOps Configuration**: Creates Application of Applications for self-management +9. **Synchronous Execution**: Bootstrap completes during `terraform apply` with visible logs -### Bootstrap Process +### Karpenter Infrastructure -The ECS bootstrap task: +When `enable_karpenter = true` (default), the module provisions: -- Runs in the cluster's private subnets for network access -- Updates kubeconfig using EKS access entries and Pod Identity -- Applies a FIPS-validated Bottlerocket NodeClass (`fips`) and a workloads NodePool -- Waits for CoreDNS and metrics-server to be Active (scheduled on the built-in `system` pool) -- Installs ArgoCD using Helm from the official repository -- Creates bootstrap application pointing to your repository -- Enables ArgoCD to take over cluster management +- **`karpenter-bootstrap` managed node group**: 2x t3.medium nodes tainted `CriticalAddonsOnly=true:NoSchedule`. Hosts Karpenter controller, CoreDNS, and metrics-server. +- **Karpenter controller IAM role**: IRSA-backed, scoped to `kube-system/karpenter` ServiceAccount with SQS, EC2, and IAM instance profile permissions. +- **Karpenter node IAM role**: Full `AmazonEKSWorkerNodePolicy`, VPC CNI, ECR pull-only, and optional KMS decrypt for FIPS AMI snapshots. +- **SQS queue**: Receives EC2 interruption events (spot reclamation, instance health, rebalance) for graceful node draining. +- **EventBridge rules**: Four rules forward EC2 lifecycle events to the SQS queue. -For the FIPS node strategy, including why the built-in `system` pool is retained and `general-purpose` is disabled, see [FIPS-Only EKS Compute](../../../docs/design/fips-eks-compute.md). +For the FIPS node strategy, including why Auto Mode was replaced with OSS Karpenter, see [FIPS-Only EKS Compute](../../../docs/design/fips-eks-compute.md). For Karpenter IAM role design, see [Karpenter Node Provisioning](../../../docs/design/karpenter-node-provisioning.md). ## Requirements diff --git a/terraform/modules/eks-cluster/data.tf b/terraform/modules/eks-cluster/data.tf index 92cf26442..980fc3f15 100644 --- a/terraform/modules/eks-cluster/data.tf +++ b/terraform/modules/eks-cluster/data.tf @@ -8,3 +8,10 @@ data "aws_partition" "current" {} # Current AWS region data "aws_region" "current" {} + +# TLS certificate for the EKS OIDC issuer endpoint — provides the thumbprint required +# by aws_iam_openid_connect_provider. Only fetched when Karpenter IRSA is needed. +data "tls_certificate" "eks_oidc" { + count = var.enable_karpenter ? 1 : 0 + url = aws_eks_cluster.main.identity[0].oidc[0].issuer +} diff --git a/terraform/modules/eks-cluster/iam.tf b/terraform/modules/eks-cluster/iam.tf index 4ccaf18c8..890e41813 100644 --- a/terraform/modules/eks-cluster/iam.tf +++ b/terraform/modules/eks-cluster/iam.tf @@ -1,23 +1,28 @@ # ============================================================================= # IAM Roles and Policies for EKS Cluster # -# Creates IAM roles required for EKS Auto Mode operation: -# - Cluster service role with required permissions -# - Node group role for Auto Mode managed nodes +# Supports two compute modes, selected by var.enable_karpenter: +# +# false — EKS Auto Mode +# - eks_cluster role: AmazonEKSClusterPolicy + all four Auto Mode policies +# - eks_auto_mode_node role: AmazonEKSWorkerNodeMinimalPolicy + ECR pull-only +# +# true (default) — OSS Karpenter +# - eks_cluster role: AmazonEKSClusterPolicy only (Auto Mode policies removed) +# - eks_auto_mode_node role: exists but has no policy attachments (unused) +# - karpenter_node role: full AmazonEKSWorkerNodePolicy + CNI + ECR + SSM +# - karpenter_controller role: IRSA-backed, scoped to kube-system/karpenter SA +# - ebs_csi role: Pod Identity-backed, scoped to kube-system/ebs-csi-controller-sa +# - SQS interruption queue + four EventBridge rules # ============================================================================= # ----------------------------------------------------------------------------- # EKS Cluster Service Role -# -# Role assumed by EKS control plane. Auto Mode requires additional permissions -# including sts:TagSession for resource tagging. -# See: https://docs.aws.amazon.com/eks/latest/userguide/automode-get-started-cli.html#auto-mode-create-roles # ----------------------------------------------------------------------------- resource "aws_iam_role" "eks_cluster" { name = "${local.cluster_id}-cluster-role" - # Auto Mode REQUIRES sts:TagSession to propagate tags to managed infra assume_role_policy = jsonencode({ Version = "2012-10-17" Statement = [{ @@ -29,13 +34,15 @@ resource "aws_iam_role" "eks_cluster" { } resource "aws_iam_role_policy_attachment" "eks_cluster_managed" { - for_each = toset([ - "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", - "arn:aws:iam::aws:policy/AmazonEKSComputePolicy", - "arn:aws:iam::aws:policy/AmazonEKSBlockStoragePolicy", - "arn:aws:iam::aws:policy/AmazonEKSLoadBalancingPolicy", - "arn:aws:iam::aws:policy/AmazonEKSNetworkingPolicy" - ]) + for_each = toset(concat( + ["arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"], + var.enable_karpenter ? [] : [ + "arn:aws:iam::aws:policy/AmazonEKSComputePolicy", + "arn:aws:iam::aws:policy/AmazonEKSBlockStoragePolicy", + "arn:aws:iam::aws:policy/AmazonEKSLoadBalancingPolicy", + "arn:aws:iam::aws:policy/AmazonEKSNetworkingPolicy", + ] + )) policy_arn = each.value role = aws_iam_role.eks_cluster.name } @@ -43,10 +50,12 @@ resource "aws_iam_role_policy_attachment" "eks_cluster_managed" { # ----------------------------------------------------------------------------- # EKS Auto Mode Node Role # -# Role assumed by Auto Mode managed nodes. Includes all required policies -# for node operation, networking, storage, and load balancing. -# See: https://docs.aws.amazon.com/eks/latest/userguide/automode-get-started-cli.html#auto-mode-create-roles +# Always created so that existing Auto Mode clusters can reference it in +# compute_config without requiring a count-indexed reference. When +# enable_karpenter = true the policy attachments are empty and the cluster's +# compute_config dynamic block is absent, so this role is unused but harmless. # ----------------------------------------------------------------------------- + resource "aws_iam_role" "eks_auto_mode_node" { name = "${local.cluster_id}-auto-node-role" @@ -63,10 +72,405 @@ resource "aws_iam_role" "eks_auto_mode_node" { } resource "aws_iam_role_policy_attachment" "auto_node_managed" { - for_each = toset([ + for_each = var.enable_karpenter ? toset([]) : toset([ "arn:aws:iam::aws:policy/AmazonEKSWorkerNodeMinimalPolicy", - "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly" + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly", ]) policy_arn = each.value role = aws_iam_role.eks_auto_mode_node.name -} \ No newline at end of file +} + +# ============================================================================= +# Karpenter — all resources below are gated on var.enable_karpenter +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Karpenter Node Role + Instance Profile +# +# Used by both Karpenter-provisioned RHEL FIPS nodes (via EC2NodeClass.spec.role) +# and the AL2023 bootstrap managed node group. +# ----------------------------------------------------------------------------- + +resource "aws_iam_role" "karpenter_node" { + count = var.enable_karpenter ? 1 : 0 + name = "${local.cluster_id}-karpenter-node-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Action = ["sts:AssumeRole", "sts:TagSession"] + Effect = "Allow" + Principal = { Service = "ec2.amazonaws.com" } + }] + }) +} + +resource "aws_iam_role_policy_attachment" "karpenter_node_managed" { + for_each = var.enable_karpenter ? toset([ + "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPullOnly", + "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore", + ]) : toset([]) + policy_arn = each.value + role = aws_iam_role.karpenter_node[0].name +} + +# Inline KMS policy for RHEL FIPS AMI EBS snapshot decryption. +# EC2 calls kms:CreateGrant on the Red Hat key on behalf of the node role when +# launching from an encrypted AMI. +resource "aws_iam_role_policy" "karpenter_node_kms" { + count = var.enable_karpenter && var.ami_kms_key_arn != "" ? 1 : 0 + name = "rhel-ami-kms-decrypt" + role = aws_iam_role.karpenter_node[0].id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "RhelAmiKmsDecrypt" + Effect = "Allow" + Action = ["kms:Decrypt", "kms:DescribeKey", "kms:CreateGrant", "kms:GenerateDataKey*", "kms:ReEncrypt*"] + Resource = var.ami_kms_key_arn + }] + }) +} + +# Instance profile wrapping the node role. Karpenter looks up (or creates) a +# profile with the same name as EC2NodeClass.spec.role. Pre-creating it here +# avoids race conditions during bootstrap. +resource "aws_iam_instance_profile" "karpenter_node" { + count = var.enable_karpenter ? 1 : 0 + name = "${local.cluster_id}-karpenter-node-role" + role = aws_iam_role.karpenter_node[0].name +} + +# ----------------------------------------------------------------------------- +# OIDC Provider — required for Karpenter controller IRSA +# ----------------------------------------------------------------------------- + +resource "aws_iam_openid_connect_provider" "eks" { + count = var.enable_karpenter ? 1 : 0 + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = [data.tls_certificate.eks_oidc[0].certificates[0].sha1_fingerprint] + url = aws_eks_cluster.main.identity[0].oidc[0].issuer +} + +# ----------------------------------------------------------------------------- +# Karpenter Controller Role (IRSA) +# +# Karpenter predates EKS Pod Identity support; IRSA is the supported auth +# mechanism. See ADR docs/design/karpenter-node-provisioning.md. +# ----------------------------------------------------------------------------- + +resource "aws_iam_role" "karpenter_controller" { + count = var.enable_karpenter ? 1 : 0 + name = "${local.cluster_id}-karpenter-controller" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { + Federated = aws_iam_openid_connect_provider.eks[0].arn + } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "${local.oidc_issuer}:sub" = "system:serviceaccount:kube-system:karpenter" + "${local.oidc_issuer}:aud" = "sts.amazonaws.com" + } + } + }] + }) +} + +resource "aws_iam_role_policy" "karpenter_controller" { + count = var.enable_karpenter ? 1 : 0 + name = "karpenter-controller" + role = aws_iam_role.karpenter_controller[0].id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "EC2FleetDescribe" + Effect = "Allow" + Action = [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets", + ] + Resource = "*" + }, + { + Sid = "EC2FleetCreate" + Effect = "Allow" + Action = [ + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", + "ec2:CreateTags", + "ec2:RunInstances", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:RequestTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + # The nodeclaim.tagging controller calls CreateTags on already-running instances + # to apply karpenter.sh/* labels post-creation. aws:RequestTag only applies to + # tags set during resource creation, so a separate statement scoped by + # aws:ResourceTag is required for post-creation tagging. + Sid = "EC2NodeClaimTagging" + Effect = "Allow" + Action = ["ec2:CreateTags"] + Resource = [ + "arn:${data.aws_partition.current.partition}:ec2:*:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:volume/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:network-interface/*", + ] + Condition = { + StringEquals = { + "aws:ResourceTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + # RunInstances on pre-existing resources (AMI, security-group, subnet) must be + # unconditional: these resources don't receive aws:RequestTag during RunInstances, + # so the RequestTag condition in EC2FleetCreate always denies them. This is the + # same split used in the official Karpenter IAM policy. + Sid = "EC2RunInstancesValidation" + Effect = "Allow" + Action = ["ec2:RunInstances"] + Resource = [ + "arn:${data.aws_partition.current.partition}:ec2:*::image/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:fleet/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:instance/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:launch-template/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:network-interface/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:security-group/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:spot-instances-request/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:subnet/*", + "arn:${data.aws_partition.current.partition}:ec2:*:*:volume/*", + ] + }, + { + Sid = "EC2FleetDelete" + Effect = "Allow" + Action = [ + "ec2:DeleteLaunchTemplate", + "ec2:TerminateInstances", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:ResourceTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + Sid = "IAMInstanceProfileCreate" + Effect = "Allow" + Action = [ + "iam:CreateInstanceProfile", + "iam:TagInstanceProfile", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:RequestTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + Sid = "IAMInstanceProfileModify" + Effect = "Allow" + Action = [ + "iam:AddRoleToInstanceProfile", + "iam:DeleteInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + ] + Resource = "*" + Condition = { + StringEquals = { + "aws:ResourceTag/kubernetes.io/cluster/${local.cluster_id}" = "owned" + } + } + }, + { + # GetInstanceProfile and ListInstanceProfiles are read-only and must be + # unconditional: Karpenter calls GetInstanceProfile before creating (and + # tagging) a profile, so a ResourceTag condition always denies it. + # ListInstanceProfiles is required by the instance-profile GC controller. + Sid = "IAMInstanceProfileRead" + Effect = "Allow" + Action = [ + "iam:GetInstanceProfile", + "iam:ListInstanceProfiles", + ] + Resource = "*" + }, + { + Sid = "IAMPassRole" + Effect = "Allow" + Action = "iam:PassRole" + Resource = aws_iam_role.karpenter_node[0].arn + }, + { + Sid = "SQS" + Effect = "Allow" + Action = [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + ] + Resource = aws_sqs_queue.karpenter_interruption[0].arn + }, + { + Sid = "EKS" + Effect = "Allow" + Action = "eks:DescribeCluster" + Resource = aws_eks_cluster.main.arn + }, + { + Sid = "SSM" + Effect = "Allow" + Action = "ssm:GetParameter" + Resource = "arn:${data.aws_partition.current.partition}:ssm:*:*:parameter/aws/service/*" + }, + { + Sid = "Pricing" + Effect = "Allow" + Action = "pricing:GetProducts" + Resource = "*" + }, + ] + }) +} + +resource "aws_iam_role_policy" "karpenter_controller_kms" { + count = var.enable_karpenter && var.ami_kms_key_arn != "" ? 1 : 0 + name = "rhel-ami-kms-describe" + role = aws_iam_role.karpenter_controller[0].id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "RhelAmiKmsGrant" + Effect = "Allow" + Action = ["kms:CreateGrant", "kms:DescribeKey"] + Resource = var.ami_kms_key_arn + }] + }) +} + +# ----------------------------------------------------------------------------- +# SQS Interruption Queue + EventBridge Rules +# +# Receives EC2 Spot, rebalance, state-change, and AWS Health events so Karpenter +# can drain nodes before the 2-minute Spot termination window expires. +# ----------------------------------------------------------------------------- + +resource "aws_sqs_queue" "karpenter_interruption" { + count = var.enable_karpenter ? 1 : 0 + name = "${local.cluster_id}-karpenter" + + message_retention_seconds = 300 + sqs_managed_sse_enabled = true +} + +resource "aws_sqs_queue_policy" "karpenter_interruption" { + count = var.enable_karpenter ? 1 : 0 + queue_url = aws_sqs_queue.karpenter_interruption[0].id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "AllowEventBridge" + Effect = "Allow" + Principal = { Service = "events.amazonaws.com" } + Action = "sqs:SendMessage" + Resource = aws_sqs_queue.karpenter_interruption[0].arn + }] + }) +} + +locals { + karpenter_event_rules = var.enable_karpenter ? { + spot-interruption = { + description = "Karpenter: EC2 Spot Instance Interruption Warning" + event_pattern = jsonencode({ source = ["aws.ec2"], "detail-type" = ["EC2 Spot Instance Interruption Warning"] }) + } + instance-terminated = { + description = "Karpenter: EC2 Instance Terminated" + event_pattern = jsonencode({ source = ["aws.ec2"], "detail-type" = ["EC2 Instance State-change Notification"], detail = { state = ["terminated"] } }) + } + rebalance-recommendation = { + description = "Karpenter: EC2 Instance Rebalance Recommendation" + event_pattern = jsonencode({ source = ["aws.ec2"], "detail-type" = ["EC2 Instance Rebalance Recommendation"] }) + } + health-scheduled-change = { + description = "Karpenter: AWS Health EC2 Scheduled Change" + event_pattern = jsonencode({ source = ["aws.health"], "detail-type" = ["AWS Health Event"], detail = { service = ["EC2"], eventTypeCategory = ["scheduledChange"] } }) + } + } : {} +} + +resource "aws_cloudwatch_event_rule" "karpenter" { + for_each = local.karpenter_event_rules + name = "${local.cluster_id}-karpenter-${each.key}" + description = each.value.description + event_pattern = each.value.event_pattern +} + +resource "aws_cloudwatch_event_target" "karpenter" { + for_each = local.karpenter_event_rules + rule = aws_cloudwatch_event_rule.karpenter[each.key].name + arn = aws_sqs_queue.karpenter_interruption[0].arn +} + +# ----------------------------------------------------------------------------- +# EBS CSI Driver Role (Pod Identity) +# +# Pod Identity is the platform-standard auth mechanism for addons. The controller +# service account (ebs-csi-controller-sa in kube-system) is bound via +# aws_eks_pod_identity_association — no service_account_role_arn annotation needed. +# ----------------------------------------------------------------------------- + +resource "aws_iam_role" "ebs_csi" { + count = var.enable_karpenter ? 1 : 0 + name = "${local.cluster_id}-ebs-csi-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "pods.eks.amazonaws.com" } + Action = ["sts:AssumeRole", "sts:TagSession"] + }] + }) +} + +resource "aws_iam_role_policy_attachment" "ebs_csi" { + count = var.enable_karpenter ? 1 : 0 + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" + role = aws_iam_role.ebs_csi[0].name +} + +resource "aws_eks_pod_identity_association" "ebs_csi" { + count = var.enable_karpenter ? 1 : 0 + cluster_name = aws_eks_cluster.main.name + namespace = "kube-system" + service_account = "ebs-csi-controller-sa" + role_arn = aws_iam_role.ebs_csi[0].arn +} diff --git a/terraform/modules/eks-cluster/locals.tf b/terraform/modules/eks-cluster/locals.tf index c2c553cc6..3f9e854e6 100644 --- a/terraform/modules/eks-cluster/locals.tf +++ b/terraform/modules/eks-cluster/locals.tf @@ -6,4 +6,8 @@ locals { cluster_id = var.cluster_id log_retention_days = 365 + + # OIDC issuer URL without https:// prefix — used as the condition key in IRSA trust policies. + # Empty string when Karpenter is disabled to avoid a forward reference on the cluster resource. + oidc_issuer = var.enable_karpenter ? trimprefix(aws_eks_cluster.main.identity[0].oidc[0].issuer, "https://") : "" } diff --git a/terraform/modules/eks-cluster/main.tf b/terraform/modules/eks-cluster/main.tf index 33fe541b1..fe3603e70 100644 --- a/terraform/modules/eks-cluster/main.tf +++ b/terraform/modules/eks-cluster/main.tf @@ -111,27 +111,30 @@ resource "aws_eks_cluster" "main" { security_group_ids = [var.cluster_security_group_id] } - compute_config { - enabled = true - node_pools = ["system"] - node_role_arn = aws_iam_role.eks_auto_mode_node.arn - - # TODO: Enable IMDSv2 enforcement for security compliance - # node_pool_defaults configuration for launch template metadata_options - # is not yet supported in AWS provider 6.x for EKS Auto Mode. - # Will be implemented when provider support becomes available. - # See https://github.com/hashicorp/terraform-provider-aws/issues/40486 + dynamic "compute_config" { + for_each = var.enable_karpenter ? [] : [1] + content { + enabled = true + node_pools = ["system"] + node_role_arn = aws_iam_role.eks_auto_mode_node.arn + } } - kubernetes_network_config { - elastic_load_balancing { - enabled = true + dynamic "kubernetes_network_config" { + for_each = var.enable_karpenter ? [] : [1] + content { + elastic_load_balancing { + enabled = true + } } } - storage_config { - block_storage { - enabled = true + dynamic "storage_config" { + for_each = var.enable_karpenter ? [] : [1] + content { + block_storage { + enabled = true + } } } @@ -142,6 +145,36 @@ resource "aws_eks_cluster" "main" { aws_cloudwatch_log_group.eks_cluster, aws_kms_key.eks_secrets ] + + # Terminate Karpenter-provisioned EC2 instances before the cluster is deleted. + # Karpenter nodes are not in Terraform state, so they survive EKS deletion and + # block VPC/subnet teardown with DependencyViolation due to lingering ENIs. + # on_failure = continue so a missing AWS CLI or zero instances doesn't abort destroy. + provisioner "local-exec" { + when = destroy + on_failure = continue + command = <<-EOT + CLUSTER_NAME="${self.name}" + REGION=$(echo "${self.arn}" | cut -d: -f4) + echo "Terminating Karpenter EC2 instances for cluster: $CLUSTER_NAME" + INSTANCE_IDS=$(aws ec2 describe-instances \ + --region "$REGION" \ + --filters \ + "Name=tag:aws:eks:cluster-name,Values=$CLUSTER_NAME" \ + "Name=tag-key,Values=karpenter.sh/nodeclaim" \ + "Name=instance-state-name,Values=pending,running,stopping,stopped" \ + --query 'Reservations[].Instances[].InstanceId' \ + --output text) + if [ -z "$INSTANCE_IDS" ]; then + echo "No Karpenter-managed instances found." + exit 0 + fi + echo "Terminating: $INSTANCE_IDS" + aws ec2 terminate-instances --region "$REGION" --instance-ids $INSTANCE_IDS + aws ec2 wait instance-terminated --region "$REGION" --instance-ids $INSTANCE_IDS + echo "Done." + EOT + } } # ----------------------------------------------------------------------------- @@ -163,16 +196,94 @@ resource "aws_eks_cluster" "main" { resource "aws_eks_addon" "coredns" { cluster_name = aws_eks_cluster.main.name addon_name = "coredns" + depends_on = [aws_eks_node_group.karpenter_bootstrap] } resource "aws_eks_addon" "metrics_server" { cluster_name = aws_eks_cluster.main.name addon_name = "metrics-server" + depends_on = [aws_eks_node_group.karpenter_bootstrap] } resource "aws_eks_addon" "pod_identity" { cluster_name = aws_eks_cluster.main.name addon_name = "eks-pod-identity-agent" + depends_on = [aws_eks_node_group.karpenter_bootstrap] +} + +# ----------------------------------------------------------------------------- +# Karpenter Bootstrap Node Group +# +# AL2023 managed node group (t3.medium × 2, CriticalAddonsOnly:NoSchedule) that +# provides fixed capacity for the Karpenter controller and VPC CNI daemonset +# before any Karpenter-provisioned nodes exist. This breaks the bootstrap +# deadlock: Karpenter cannot provision nodes for itself. +# +# No custom launch template: EKS managed node groups set IMDSv2 hop limit to 2 +# by default for AL2023, and managed node group auth is handled automatically +# by EKS regardless of node name format. +# ----------------------------------------------------------------------------- + +resource "aws_eks_node_group" "karpenter_bootstrap" { + count = var.enable_karpenter ? 1 : 0 + cluster_name = aws_eks_cluster.main.name + node_group_name = "${local.cluster_id}-karpenter-bootstrap" + node_role_arn = aws_iam_role.karpenter_node[0].arn + subnet_ids = var.private_subnet_ids + + ami_type = "AL2023_x86_64_STANDARD" + instance_types = ["t3.medium"] + + scaling_config { + desired_size = 2 + min_size = 2 + max_size = 2 + } + + taint { + key = "CriticalAddonsOnly" + value = "true" + effect = "NO_SCHEDULE" + } + + tags = { + "karpenter.sh/discovery" = aws_eks_cluster.main.name + } + + depends_on = [ + aws_iam_role_policy_attachment.karpenter_node_managed, + aws_eks_addon.vpc_cni, + ] +} + +# ----------------------------------------------------------------------------- +# Explicit Core Addons (Karpenter mode only) +# +# bootstrap_self_managed_addons = false prevents EKS from auto-installing these. +# Auto Mode clusters receive VPC CNI and kube-proxy from the managed control +# plane; Karpenter clusters must declare them explicitly. +# ----------------------------------------------------------------------------- + +resource "aws_eks_addon" "vpc_cni" { + count = var.enable_karpenter ? 1 : 0 + cluster_name = aws_eks_cluster.main.name + addon_name = "vpc-cni" +} + +resource "aws_eks_addon" "kube_proxy" { + count = var.enable_karpenter ? 1 : 0 + cluster_name = aws_eks_cluster.main.name + addon_name = "kube-proxy" + + depends_on = [aws_eks_node_group.karpenter_bootstrap] +} + +resource "aws_eks_addon" "ebs_csi" { + count = var.enable_karpenter ? 1 : 0 + cluster_name = aws_eks_cluster.main.name + addon_name = "aws-ebs-csi-driver" + + depends_on = [aws_eks_node_group.karpenter_bootstrap, aws_eks_pod_identity_association.ebs_csi] } # AWS Secrets Store CSI Driver Provider (e.g. for Maestro agent secret mounting) @@ -187,4 +298,6 @@ resource "aws_eks_addon" "aws_secrets_store_csi_driver_provider" { } } }) + + depends_on = [aws_eks_node_group.karpenter_bootstrap] } diff --git a/terraform/modules/eks-cluster/outputs.tf b/terraform/modules/eks-cluster/outputs.tf index 042e84ffe..980663c49 100644 --- a/terraform/modules/eks-cluster/outputs.tf +++ b/terraform/modules/eks-cluster/outputs.tf @@ -43,7 +43,7 @@ output "vpc_endpoints_security_group_id" { } output "node_security_group_id" { - description = "EKS node security group ID (Auto Mode primary SG - only available after EKS creation)" + description = "EKS cluster security group ID (primary node SG, available after cluster creation)" value = aws_eks_cluster.main.vpc_config[0].cluster_security_group_id } @@ -87,6 +87,21 @@ output "cluster_iam_role_arn" { } output "node_iam_role_arn" { - description = "IAM role ARN of the EKS Auto Mode nodes" - value = aws_iam_role.eks_auto_mode_node.arn + description = "IAM role ARN for cluster nodes (Auto Mode node role or Karpenter node role, depending on enable_karpenter)" + value = var.enable_karpenter ? aws_iam_role.karpenter_node[0].arn : aws_iam_role.eks_auto_mode_node.arn +} + +output "karpenter_controller_role_arn" { + description = "IAM role ARN for the Karpenter controller (IRSA). Null when enable_karpenter = false." + value = var.enable_karpenter ? aws_iam_role.karpenter_controller[0].arn : null +} + +output "karpenter_queue_url" { + description = "SQS queue URL for Karpenter interruption handling. Null when enable_karpenter = false." + value = var.enable_karpenter ? aws_sqs_queue.karpenter_interruption[0].url : null +} + +output "karpenter_node_instance_profile_name" { + description = "Instance profile name for Karpenter-provisioned nodes (matches EC2NodeClass.spec.role). Null when enable_karpenter = false." + value = var.enable_karpenter ? aws_iam_instance_profile.karpenter_node[0].name : null } diff --git a/terraform/modules/eks-cluster/variables.tf b/terraform/modules/eks-cluster/variables.tf index 52be8b723..f063085b9 100644 --- a/terraform/modules/eks-cluster/variables.tf +++ b/terraform/modules/eks-cluster/variables.tf @@ -71,3 +71,19 @@ variable "enable_pod_security_standards" { default = true } +# ============================================================================= +# Karpenter configuration +# ============================================================================= + +variable "enable_karpenter" { + description = "Enable OSS Karpenter instead of EKS Auto Mode. Disables Auto Mode compute, storage, and load balancing blocks. Mutually exclusive with Auto Mode." + type = bool + default = true +} + +variable "ami_kms_key_arn" { + description = "ARN of the Red Hat KMS key used to encrypt RHEL FIPS AMI EBS snapshots. When set, IAM policies granting kms:Decrypt and kms:CreateGrant on this key are added to the Karpenter node and controller roles. Leave empty to skip KMS policy creation." + type = string + default = "" +} + diff --git a/terraform/modules/eks-cluster/versions.tf b/terraform/modules/eks-cluster/versions.tf index c7e4c3e70..1187b1bf6 100644 --- a/terraform/modules/eks-cluster/versions.tf +++ b/terraform/modules/eks-cluster/versions.tf @@ -10,5 +10,9 @@ terraform { source = "hashicorp/aws" version = ">= 6.0" } + tls = { + source = "hashicorp/tls" + version = ">= 4.0" + } } } \ No newline at end of file diff --git a/terraform/modules/hyperfleet-infrastructure/amazonmq.tf b/terraform/modules/hyperfleet-infrastructure/amazonmq.tf index 9c38f8ee8..2d3979e94 100644 --- a/terraform/modules/hyperfleet-infrastructure/amazonmq.tf +++ b/terraform/modules/hyperfleet-infrastructure/amazonmq.tf @@ -142,7 +142,7 @@ resource "aws_security_group_rule" "hyperfleet_mq_eks_cluster" { # It does NOT block broker creation - only defines who can connect. resource "aws_security_group_rule" "hyperfleet_mq_eks_primary" { type = "ingress" - description = "AMQPS from EKS cluster primary security group (Auto Mode)" + description = "AMQPS from EKS cluster primary security group (Karpenter nodes)" from_port = 5671 to_port = 5671 protocol = "tcp" From d2c320d8e1be40493a7a9c24e3d13f3ad10c7457 Mon Sep 17 00:00:00 2001 From: Brian Smith Date: Tue, 14 Jul 2026 10:04:14 -0400 Subject: [PATCH 2/5] Allow prometheus-operator to schedule on bootstrap nodes --- argocd/config/management-cluster/monitoring/values.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/argocd/config/management-cluster/monitoring/values.yaml b/argocd/config/management-cluster/monitoring/values.yaml index d2a7682d6..4b00fc6de 100644 --- a/argocd/config/management-cluster/monitoring/values.yaml +++ b/argocd/config/management-cluster/monitoring/values.yaml @@ -16,6 +16,10 @@ kube-prometheus-stack: enabled: false prometheusOperator: + tolerations: + - key: CriticalAddonsOnly + operator: Exists + effect: NoSchedule admissionWebhooks: # During bootstrap, ArgoCD syncs itself and restarts its controllers # while other app syncs are in flight. If a PreSync hook Job completes From 6a8a00e1cc6ab8a75ac998b92f10c1c062a2a2f0 Mon Sep 17 00:00:00 2001 From: Brian Smith Date: Tue, 14 Jul 2026 13:58:22 -0400 Subject: [PATCH 3/5] Wait for hypershift app Synced+Healthy before bootstrap exits --- terraform/modules/ecs-bootstrap/main.tf | 29 +++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/terraform/modules/ecs-bootstrap/main.tf b/terraform/modules/ecs-bootstrap/main.tf index fcd51cd7e..af0031277 100644 --- a/terraform/modules/ecs-bootstrap/main.tf +++ b/terraform/modules/ecs-bootstrap/main.tf @@ -376,6 +376,35 @@ resource "aws_ecs_task_definition" "bootstrap" { - CreateNamespace=true APP_EOF + if [ "$CLUSTER_TYPE" = "management-cluster" ]; then + echo "Waiting for hypershift ArgoCD application to become Synced+Healthy..." + _HS_TIMEOUT=1800 + _HS_START=$(date +%s) + until kubectl get application hypershift -n argocd &>/dev/null; do + _ELAPSED=$(( $(date +%s) - _HS_START )) + if [ "$_ELAPSED" -ge "$_HS_TIMEOUT" ]; then + echo "ERROR: hypershift application did not appear after $((_ELAPSED/60))m" >&2 + exit 1 + fi + echo "[$((_ELAPSED/60))m] waiting for hypershift application to appear..." + sleep 15 + done + until [ "$(kubectl get application hypershift -n argocd -o jsonpath='{.status.sync.status}' 2>/dev/null)" = "Synced" ] && \ + [ "$(kubectl get application hypershift -n argocd -o jsonpath='{.status.health.status}' 2>/dev/null)" = "Healthy" ]; do + _ELAPSED=$(( $(date +%s) - _HS_START )) + if [ "$_ELAPSED" -ge "$_HS_TIMEOUT" ]; then + echo "ERROR: hypershift application did not become Synced+Healthy after $((_ELAPSED/60))m" >&2 + kubectl get application hypershift -n argocd -o jsonpath='{.status}' 2>/dev/null >&2 || true + exit 1 + fi + _SYNC=$(kubectl get application hypershift -n argocd -o jsonpath='{.status.sync.status}' 2>/dev/null || echo "Unknown") + _HEALTH=$(kubectl get application hypershift -n argocd -o jsonpath='{.status.health.status}' 2>/dev/null || echo "Unknown") + echo "[$((_ELAPSED/60))m] hypershift sync=$_SYNC health=$_HEALTH" + sleep 15 + done + echo "hypershift application is Synced+Healthy" + fi + echo "=== Bootstrap completed successfully ===" EOF ] From 84417c5e24e56f2d20f3ba53990e4b9900df61d3 Mon Sep 17 00:00:00 2001 From: Brian Smith Date: Tue, 14 Jul 2026 16:15:01 -0400 Subject: [PATCH 4/5] Increase hypershift bootstrap and job timeouts to 60m --- .../config/management-cluster/hypershift/templates/05-job.yaml | 2 +- terraform/modules/ecs-bootstrap/main.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/argocd/config/management-cluster/hypershift/templates/05-job.yaml b/argocd/config/management-cluster/hypershift/templates/05-job.yaml index 19d151979..ab89113d5 100644 --- a/argocd/config/management-cluster/hypershift/templates/05-job.yaml +++ b/argocd/config/management-cluster/hypershift/templates/05-job.yaml @@ -6,7 +6,7 @@ metadata: annotations: argocd.argoproj.io/sync-options: Replace=true,Force=true spec: - activeDeadlineSeconds: 1800 + activeDeadlineSeconds: 3600 backoffLimit: 1 template: spec: diff --git a/terraform/modules/ecs-bootstrap/main.tf b/terraform/modules/ecs-bootstrap/main.tf index af0031277..9670d4f5d 100644 --- a/terraform/modules/ecs-bootstrap/main.tf +++ b/terraform/modules/ecs-bootstrap/main.tf @@ -378,7 +378,7 @@ resource "aws_ecs_task_definition" "bootstrap" { if [ "$CLUSTER_TYPE" = "management-cluster" ]; then echo "Waiting for hypershift ArgoCD application to become Synced+Healthy..." - _HS_TIMEOUT=1800 + _HS_TIMEOUT=3600 _HS_START=$(date +%s) until kubectl get application hypershift -n argocd &>/dev/null; do _ELAPSED=$(( $(date +%s) - _HS_START )) From c66211f2d587e4fa23cf1e39d9a4bb0283a74563 Mon Sep 17 00:00:00 2001 From: Brian Smith Date: Tue, 14 Jul 2026 16:26:06 -0400 Subject: [PATCH 5/5] Apply code review fixes to register.sh and karpenter docs - register.sh: replace MAX_RETRIES counter with SECONDS-based 40m deadline for /live endpoint wait; accounts for curl execution time in the budget - karpenter-node-provisioning.md: remove stale "(instance profile name)" parenthetical from EC2NodeClass.spec.role reference Co-Authored-By: Claude Sonnet 4.6 --- docs/design/karpenter-node-provisioning.md | 2 +- scripts/buildspec/register.sh | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/docs/design/karpenter-node-provisioning.md b/docs/design/karpenter-node-provisioning.md index c887f81eb..5a3375080 100644 --- a/docs/design/karpenter-node-provisioning.md +++ b/docs/design/karpenter-node-provisioning.md @@ -62,7 +62,7 @@ graph LR - **Managed policies**: `AmazonEKSWorkerNodePolicy`, `AmazonEKS_CNI_Policy`, ECR pull-only - **Optional inline policy**: `kms:Decrypt` and `kms:CreateGrant` on the FIPS AMI KMS key when `ami_kms_key_arn` is set -- **Referenced in**: `EC2NodeClass.spec.role` (instance profile name) +- **Referenced in**: `EC2NodeClass.spec.role` ### SQS Queue and EventBridge Rules diff --git a/scripts/buildspec/register.sh b/scripts/buildspec/register.sh index 1313e9f76..5a83d0fc7 100755 --- a/scripts/buildspec/register.sh +++ b/scripts/buildspec/register.sh @@ -74,14 +74,12 @@ CLOUDFRONT_URL="https://${CLOUDFRONT_DOMAIN}" # ~15 min) and initial ArgoCD sync (~10 min) have completed. Allow 40 minutes # so the Platform API has time to be deployed and reach a healthy state. set +e -MAX_RETRIES=80 +LIVE_DEADLINE=2400 +LIVE_START=$SECONDS RETRY_DELAY=30 -RETRY_COUNT=0 LIVE_OK=false -while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do - RETRY_COUNT=$((RETRY_COUNT + 1)) - +while [ $((SECONDS - LIVE_START)) -lt $LIVE_DEADLINE ]; do SECURITY_TOKEN_HEADER=() if [ -n "${AWS_SESSION_TOKEN:-}" ]; then SECURITY_TOKEN_HEADER=(-H "x-amz-security-token: ${AWS_SESSION_TOKEN}") @@ -99,13 +97,13 @@ while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do LIVE_OK=true break fi - echo "/live returned $HTTP_CODE (attempt $RETRY_COUNT/$MAX_RETRIES), retrying in ${RETRY_DELAY}s..." + echo "/live returned $HTTP_CODE ($((SECONDS - LIVE_START))s elapsed), retrying in ${RETRY_DELAY}s..." sleep $RETRY_DELAY done set -e if [ "$LIVE_OK" != "true" ]; then - echo "ERROR: /live did not return 200 after $((MAX_RETRIES * RETRY_DELAY / 60)) minutes" >&2 + echo "ERROR: /live did not return 200 after $((LIVE_DEADLINE / 60)) minutes" >&2 exit 1 fi