From 67496050db342e351bb2fe7e099caccea348c1f0 Mon Sep 17 00:00:00 2001 From: Wouter Devriendt Date: Tue, 3 Mar 2026 22:52:07 -0800 Subject: [PATCH] feat: use Karpenter for CPU node autoscaling (alternative to custom Lambda approach) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace fixed 60 CPU nodes with Karpenter-managed dynamic scaling (0-30 nodes per type). Karpenter provisions nodes on-demand when pods are pending (~60s) and consolidates when idle (60s). This is the industry-standard approach for EKS autoscaling. Key changes: - main.tf: CPU types set to karpenter_managed=true, instance_count=0 - eks.tf: Filter Karpenter types from ASG creation - karpenter.tf: Full Karpenter setup (IAM, SQS, Helm, NodePools, EC2NodeClasses) - availability_updater Lambda: Handle Karpenter types (query K8s directly, not ASG) - CLI: Show "0 / 90" scalable capacity and "~1min (scaling up)" wait time Architecture: - Karpenter controller runs on management CPU nodes (c5.4xlarge ASG, 2 nodes) - NodePool per architecture (cpu-x86, cpu-arm) with 30-node CPU limits - EC2NodeClass defines AL2023, 500GB gp3, security groups, subnets - SQS queue for spot/interruption handling (EventBridge → SQS → Karpenter) Benefits vs custom Lambda approach: - Faster scale-up (~60s vs ~3min ASG polling) - Event-driven (reacts to pending pods immediately) - Built-in consolidation with pod disruption budgets - Less custom code to maintain (~100 lines removed) - Industry standard for EKS Co-Authored-By: Claude Sonnet 4.5 --- cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py | 27 +- .../gpu-dev-cli/gpu_dev_cli/reservations.py | 1 + terraform-gpu-devservers/eks.tf | 1 + terraform-gpu-devservers/karpenter.tf | 459 ++++++++++++++++++ .../lambda/availability_updater/index.py | 65 ++- terraform-gpu-devservers/main.tf | 16 +- 6 files changed, 537 insertions(+), 32 deletions(-) create mode 100644 terraform-gpu-devservers/karpenter.tf diff --git a/cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py b/cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py index 1fe2a4fe..a951b321 100644 --- a/cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py +++ b/cli-tools/gpu-dev-cli/gpu_dev_cli/cli.py @@ -2441,14 +2441,17 @@ def _show_availability() -> None: available = info.get("available", 0) max_reservable = info.get("max_reservable", 0) total = info.get("total", 0) + scalable_total = info.get("scalable_total", 0) full_nodes_available = info.get("full_nodes_available", 0) gpus_per_instance = info.get("gpus_per_instance", 0) queue_length = info.get("queue_length", 0) est_wait = info.get("estimated_wait_minutes", 0) - # Format wait time + # Format wait time — for Karpenter types, show scale-up estimate if available > 0: wait_display = "Available now" + elif scalable_total > 0 and available == 0: + wait_display = "~1min (scaling up)" elif est_wait == 0: wait_display = "Unknown" elif est_wait < 60: @@ -2461,10 +2464,13 @@ def _show_availability() -> None: else: wait_display = f"{hours}h {minutes}min" + # Show total as "current / scalable" for Karpenter types + if scalable_total > 0: + total_display = f"{total} / {scalable_total}" + else: + total_display = str(total) + # Color code availability based on full nodes available - # Red: 0 GPUs available - # Yellow: Some GPUs available but no full node - # Green: At least one full node available if available == 0: available_display = f"[red]{available}[/red]" elif full_nodes_available > 0: @@ -2476,7 +2482,7 @@ def _show_availability() -> None: gpu_type.upper(), available_display, str(max_reservable), - str(total), + total_display, str(queue_length), arch, wait_display, @@ -2583,12 +2589,15 @@ def _show_availability_watch(interval: int) -> None: last_arch = arch available = info.get("available", 0) total = info.get("total", 0) + scalable_total = info.get("scalable_total", 0) queue_length = info.get("queue_length", 0) est_wait = info.get("estimated_wait_minutes", 0) # Format wait time if available > 0: wait_display = "Available now" + elif scalable_total > 0 and available == 0: + wait_display = "~1min (scaling up)" elif est_wait == 0: wait_display = "Unknown" elif est_wait < 60: @@ -2601,6 +2610,12 @@ def _show_availability_watch(interval: int) -> None: else: wait_display = f"{hours}h {minutes}min" + # Show total as "current / scalable" for Karpenter types + if scalable_total > 0: + total_display = f"{total} / {scalable_total}" + else: + total_display = str(total) + # Color code availability if available > 0: available_display = f"[green]{available}[/green]" @@ -2610,7 +2625,7 @@ def _show_availability_watch(interval: int) -> None: table.add_row( gpu_type.upper(), available_display, - str(total), + total_display, str(queue_length), arch, wait_display, diff --git a/cli-tools/gpu-dev-cli/gpu_dev_cli/reservations.py b/cli-tools/gpu-dev-cli/gpu_dev_cli/reservations.py index f2d4866b..00bcd1d4 100644 --- a/cli-tools/gpu-dev-cli/gpu_dev_cli/reservations.py +++ b/cli-tools/gpu-dev-cli/gpu_dev_cli/reservations.py @@ -959,6 +959,7 @@ def get_gpu_availability_by_type(self) -> Optional[Dict[str, Dict[str, Any]]]: availability_info[gpu_type] = { "available": int(item.get("available_gpus", 0)), "total": int(item.get("total_gpus", 0)), + "scalable_total": int(item.get("scalable_total", 0)), "max_reservable": int(item.get("max_reservable", 0)), "full_nodes_available": int(item.get("full_nodes_available", 0)), "gpus_per_instance": int(item.get("gpus_per_instance", 0)), diff --git a/terraform-gpu-devservers/eks.tf b/terraform-gpu-devservers/eks.tf index 5808cfeb..f6c9039e 100644 --- a/terraform-gpu-devservers/eks.tf +++ b/terraform-gpu-devservers/eks.tf @@ -191,6 +191,7 @@ locals { instance_count = cr_config != null ? cr_config.instance_count : gpu_config.instance_count } ] + if !try(gpu_config.karpenter_managed, false) # Skip types managed by Karpenter ]) # Convert to map for for_each diff --git a/terraform-gpu-devservers/karpenter.tf b/terraform-gpu-devservers/karpenter.tf new file mode 100644 index 00000000..baa97363 --- /dev/null +++ b/terraform-gpu-devservers/karpenter.tf @@ -0,0 +1,459 @@ +# Karpenter - Node autoscaler for CPU dev nodes +# Karpenter provisions nodes on-demand when pods are pending, and consolidates when idle. + +locals { + karpenter_namespace = "kube-system" + + # Extract Karpenter-managed GPU types + karpenter_managed_types = { + for gpu_type, config in local.current_config.supported_gpu_types : gpu_type => config + if try(config.karpenter_managed, false) + } +} + +# --- IAM Role for Karpenter Controller (IRSA) --- + +resource "aws_iam_role" "karpenter_controller" { + name = "${local.workspace_prefix}-karpenter-controller" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Federated = aws_iam_openid_connect_provider.eks.arn + } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "${replace(aws_eks_cluster.gpu_dev_cluster.identity[0].oidc[0].issuer, "https://", "")}:aud" = "sts.amazonaws.com" + "${replace(aws_eks_cluster.gpu_dev_cluster.identity[0].oidc[0].issuer, "https://", "")}:sub" = "system:serviceaccount:${local.karpenter_namespace}:karpenter" + } + } + } + ] + }) + + tags = { + Name = "${var.prefix}-karpenter-controller" + Environment = local.current_config.environment + } +} + +resource "aws_iam_role_policy" "karpenter_controller" { + name = "${local.workspace_prefix}-karpenter-controller-policy" + role = aws_iam_role.karpenter_controller.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate", + "ec2:CreateTags", + "ec2:DeleteLaunchTemplate", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets", + "ec2:RunInstances", + "ec2:TerminateInstances", + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = "iam:PassRole" + Resource = aws_iam_role.eks_node_role.arn + }, + { + Effect = "Allow" + Action = [ + "eks:DescribeCluster", + ] + Resource = aws_eks_cluster.gpu_dev_cluster.arn + }, + { + Effect = "Allow" + Action = [ + "ssm:GetParameter", + ] + Resource = "arn:aws:ssm:${local.current_config.aws_region}::parameter/aws/service/eks/optimized-ami/*" + }, + { + Effect = "Allow" + Action = [ + "pricing:GetProducts", + ] + Resource = "*" + }, + { + Effect = "Allow" + Action = [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + ] + Resource = aws_sqs_queue.karpenter_interruption.arn + }, + ] + }) +} + +# --- SQS Queue for node interruption/spot events --- + +resource "aws_sqs_queue" "karpenter_interruption" { + name = "${var.prefix}-karpenter-interruption" + message_retention_seconds = 300 + sqs_managed_sse_enabled = true + + tags = { + Name = "${var.prefix}-karpenter-interruption" + Environment = local.current_config.environment + } +} + +resource "aws_sqs_queue_policy" "karpenter_interruption" { + queue_url = aws_sqs_queue.karpenter_interruption.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { Service = ["events.amazonaws.com", "sqs.amazonaws.com"] } + Action = "sqs:SendMessage" + Resource = aws_sqs_queue.karpenter_interruption.arn + } + ] + }) +} + +# EventBridge rules to forward EC2 events to Karpenter's SQS queue +resource "aws_cloudwatch_event_rule" "karpenter_instance_state_change" { + name = "${var.prefix}-karpenter-instance-state" + event_pattern = jsonencode({ + source = ["aws.ec2"] + detail-type = ["EC2 Instance State-change Notification"] + }) +} + +resource "aws_cloudwatch_event_target" "karpenter_instance_state_change" { + rule = aws_cloudwatch_event_rule.karpenter_instance_state_change.name + arn = aws_sqs_queue.karpenter_interruption.arn +} + +resource "aws_cloudwatch_event_rule" "karpenter_spot_interruption" { + name = "${var.prefix}-karpenter-spot-interruption" + event_pattern = jsonencode({ + source = ["aws.ec2"] + detail-type = ["EC2 Spot Instance Interruption Warning"] + }) +} + +resource "aws_cloudwatch_event_target" "karpenter_spot_interruption" { + rule = aws_cloudwatch_event_rule.karpenter_spot_interruption.name + arn = aws_sqs_queue.karpenter_interruption.arn +} + +# --- Helm Release --- + +resource "helm_release" "karpenter" { + name = "karpenter" + repository = "oci://public.ecr.aws/karpenter" + chart = "karpenter" + version = "1.1.1" + namespace = local.karpenter_namespace + + wait = true + timeout = 600 + + set { + name = "settings.clusterName" + value = aws_eks_cluster.gpu_dev_cluster.name + } + + set { + name = "settings.clusterEndpoint" + value = aws_eks_cluster.gpu_dev_cluster.endpoint + } + + set { + name = "settings.interruptionQueue" + value = aws_sqs_queue.karpenter_interruption.name + } + + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = aws_iam_role.karpenter_controller.arn + } + + # Run Karpenter controller on the management CPU nodes (not Karpenter-managed nodes) + set { + name = "nodeSelector.NodeType" + value = "cpu-management" + } + + set { + name = "tolerations[0].operator" + value = "Exists" + } + + depends_on = [ + aws_eks_cluster.gpu_dev_cluster, + aws_iam_role_policy.karpenter_controller, + aws_autoscaling_group.cpu_nodes, # Management CPU nodes must exist first + ] +} + +# --- EC2NodeClass per architecture --- + +resource "kubernetes_manifest" "karpenter_node_class_cpu_x86" { + manifest = { + apiVersion = "karpenter.k8s.aws/v1" + kind = "EC2NodeClass" + metadata = { + name = "cpu-x86" + } + spec = { + role = aws_iam_role.eks_node_role.name + + amiSelectorTerms = [ + { alias = "al2023@latest" } + ] + + subnetSelectorTerms = [ + { + tags = { + "kubernetes.io/cluster/${aws_eks_cluster.gpu_dev_cluster.name}" = "shared" + } + } + ] + + securityGroupSelectorTerms = [ + { + tags = { + Name = "${var.prefix}-gpu-dev-sg" + } + } + ] + + blockDeviceMappings = [ + { + deviceName = "/dev/xvda" + ebs = { + volumeSize = "500Gi" + volumeType = "gp3" + deleteOnTermination = true + encrypted = true + } + } + ] + + # Extra user data (runs before nodeadm init — Karpenter handles cluster join) + userData = <<-EOT + #!/bin/bash + yum install -y htop wget + cat >/etc/sysctl.d/99-net.conf <<'SYSCTL' + net.core.rmem_default=262144000 + net.core.rmem_max=262144000 + net.core.wmem_default=262144000 + net.core.wmem_max=262144000 + SYSCTL + sysctl --system + EOT + } + } + + depends_on = [helm_release.karpenter] +} + +resource "kubernetes_manifest" "karpenter_node_class_cpu_arm" { + manifest = { + apiVersion = "karpenter.k8s.aws/v1" + kind = "EC2NodeClass" + metadata = { + name = "cpu-arm" + } + spec = { + role = aws_iam_role.eks_node_role.name + + amiSelectorTerms = [ + { alias = "al2023@latest" } + ] + + subnetSelectorTerms = [ + { + tags = { + "kubernetes.io/cluster/${aws_eks_cluster.gpu_dev_cluster.name}" = "shared" + } + } + ] + + securityGroupSelectorTerms = [ + { + tags = { + Name = "${var.prefix}-gpu-dev-sg" + } + } + ] + + blockDeviceMappings = [ + { + deviceName = "/dev/xvda" + ebs = { + volumeSize = "500Gi" + volumeType = "gp3" + deleteOnTermination = true + encrypted = true + } + } + ] + + userData = <<-EOT + #!/bin/bash + yum install -y htop wget + cat >/etc/sysctl.d/99-net.conf <<'SYSCTL' + net.core.rmem_default=262144000 + net.core.rmem_max=262144000 + net.core.wmem_default=262144000 + net.core.wmem_max=262144000 + SYSCTL + sysctl --system + EOT + } + } + + depends_on = [helm_release.karpenter] +} + +# --- NodePool per CPU type --- + +resource "kubernetes_manifest" "karpenter_node_pool_cpu_x86" { + manifest = { + apiVersion = "karpenter.sh/v1" + kind = "NodePool" + metadata = { + name = "cpu-x86" + } + spec = { + template = { + metadata = { + labels = { + NodeType = "gpu" + GpuType = "cpu-x86" + } + } + spec = { + nodeClassRef = { + group = "karpenter.k8s.aws" + kind = "EC2NodeClass" + name = "cpu-x86" + } + requirements = [ + { + key = "kubernetes.io/arch" + operator = "In" + values = ["amd64"] + }, + { + key = "karpenter.sh/capacity-type" + operator = "In" + values = ["on-demand"] + }, + { + key = "node.kubernetes.io/instance-type" + operator = "In" + values = [local.current_config.supported_gpu_types["cpu-x86"].instance_type] + }, + ] + + # Consolidate idle nodes after 30 seconds (fast scale-down) + expireAfter = "Never" + } + } + + limits = { + # Max 30 nodes worth of CPU (matches previous ASG max) + cpu = tostring(30 * (local.current_config.environment == "prod" ? 32 : 16)) + } + + disruption = { + consolidationPolicy = "WhenEmptyOrUnderutilized" + consolidateAfter = "60s" + } + } + } + + depends_on = [ + kubernetes_manifest.karpenter_node_class_cpu_x86, + ] +} + +resource "kubernetes_manifest" "karpenter_node_pool_cpu_arm" { + manifest = { + apiVersion = "karpenter.sh/v1" + kind = "NodePool" + metadata = { + name = "cpu-arm" + } + spec = { + template = { + metadata = { + labels = { + NodeType = "gpu" + GpuType = "cpu-arm" + } + } + spec = { + nodeClassRef = { + group = "karpenter.k8s.aws" + kind = "EC2NodeClass" + name = "cpu-arm" + } + requirements = [ + { + key = "kubernetes.io/arch" + operator = "In" + values = ["arm64"] + }, + { + key = "karpenter.sh/capacity-type" + operator = "In" + values = ["on-demand"] + }, + { + key = "node.kubernetes.io/instance-type" + operator = "In" + values = [local.current_config.supported_gpu_types["cpu-arm"].instance_type] + }, + ] + + expireAfter = "Never" + } + } + + limits = { + cpu = tostring(30 * (local.current_config.environment == "prod" ? 32 : 16)) + } + + disruption = { + consolidationPolicy = "WhenEmptyOrUnderutilized" + consolidateAfter = "60s" + } + } + } + + depends_on = [ + kubernetes_manifest.karpenter_node_class_cpu_arm, + ] +} diff --git a/terraform-gpu-devservers/lambda/availability_updater/index.py b/terraform-gpu-devservers/lambda/availability_updater/index.py index 2b4605ae..3d1a9235 100644 --- a/terraform-gpu-devservers/lambda/availability_updater/index.py +++ b/terraform-gpu-devservers/lambda/availability_updater/index.py @@ -94,25 +94,31 @@ def update_gpu_availability(gpu_type: str, k8s_client=None) -> None: if asg["AutoScalingGroupName"].startswith(asg_name_prefix) ] - if not matching_asgs: - logger.warning(f"No ASGs found matching pattern: {asg_name_prefix}*") - return - - asg_names = [asg["AutoScalingGroupName"] for asg in matching_asgs] - logger.info(f"Found {len(matching_asgs)} ASGs: {asg_names}") - - # Calculate total availability metrics across all matching ASGs - desired_capacity = sum(asg["DesiredCapacity"] for asg in matching_asgs) - running_instances = sum( - len([ - instance for instance in asg["Instances"] - if instance["LifecycleState"] == "InService" - ]) for asg in matching_asgs - ) - # Get GPU configuration for this type gpu_config = SUPPORTED_GPU_TYPES.get(gpu_type, {}) gpus_per_instance = gpu_config.get("gpus_per_instance", 8) + is_karpenter_managed = gpu_config.get("karpenter_managed", False) + + if not matching_asgs and not is_karpenter_managed: + logger.warning(f"No ASGs found matching pattern: {asg_name_prefix}* and not Karpenter-managed") + return + + if is_karpenter_managed: + logger.info(f"{gpu_type} is Karpenter-managed, skipping ASG checks") + desired_capacity = 0 + running_instances = 0 + else: + asg_names = [asg["AutoScalingGroupName"] for asg in matching_asgs] + logger.info(f"Found {len(matching_asgs)} ASGs: {asg_names}") + + # Calculate total availability metrics across all matching ASGs + desired_capacity = sum(asg["DesiredCapacity"] for asg in matching_asgs) + running_instances = sum( + len([ + instance for instance in asg["Instances"] + if instance["LifecycleState"] == "InService" + ]) for asg in matching_asgs + ) # Handle CPU-only nodes differently (they don't have GPUs) is_cpu_type = gpus_per_instance == 0 @@ -120,15 +126,24 @@ def update_gpu_availability(gpu_type: str, k8s_client=None) -> None: if is_cpu_type: # For CPU nodes, report instance slots (assuming 3 users per node) max_users_per_node = 3 + + # For Karpenter-managed nodes, get actual running count from K8s (not ASG) + if is_karpenter_managed and k8s_client is not None: + from kubernetes import client + v1 = client.CoreV1Api(k8s_client) + nodes = v1.list_node(label_selector=f"GpuType={gpu_type}") + running_instances = len([n for n in nodes.items if is_node_ready_and_schedulable(n)]) + logger.info(f"Karpenter-managed {gpu_type}: {running_instances} nodes from K8s API") + total_gpus = running_instances * max_users_per_node logger.info( - f"CPU ASG calculation: {running_instances} instances * {max_users_per_node} slots = {total_gpus} total slots") + f"CPU calculation: {running_instances} instances * {max_users_per_node} slots = {total_gpus} total slots") # Check actual pod usage on CPU nodes if k8s_client is not None: try: logger.info(f"Checking CPU node availability for {gpu_type}") - # Count available slots by checking pod count on each node + from kubernetes import client v1 = client.CoreV1Api(k8s_client) nodes = v1.list_node(label_selector=f"GpuType={gpu_type}") @@ -223,6 +238,14 @@ def update_gpu_availability(gpu_type: str, k8s_client=None) -> None: full_nodes_available = available_gpus # Each "GPU" represents one CPU node slot max_reservable = 1 if available_gpus > 0 else 0 # Max 1 CPU node per reservation + # For Karpenter-managed CPU types, report scalable total based on max node limit + scalable_total = 0 + if is_cpu_type and is_karpenter_managed: + karpenter_max = gpu_config.get("karpenter_max_nodes", 0) + if karpenter_max > 0: + scalable_total = karpenter_max * max_users_per_node + logger.info(f"Karpenter-managed {gpu_type}: scalable_total = {karpenter_max} nodes * {max_users_per_node} = {scalable_total}") + # Update DynamoDB table table = dynamodb.Table(AVAILABILITY_TABLE) @@ -231,15 +254,13 @@ def update_gpu_availability(gpu_type: str, k8s_client=None) -> None: "gpu_type": gpu_type, "total_gpus": total_gpus, "available_gpus": available_gpus, + "scalable_total": scalable_total, "max_reservable": max_reservable, "full_nodes_available": full_nodes_available, "running_instances": running_instances, "desired_capacity": desired_capacity, "gpus_per_instance": gpus_per_instance, - "last_updated": context.aws_request_id - if "context" in locals() - else "unknown", - "last_updated_timestamp": int(time.time()) if "time" in dir() else 0, + "last_updated_timestamp": int(time.time()), } ) diff --git a/terraform-gpu-devservers/main.tf b/terraform-gpu-devservers/main.tf index 635a9d07..c08e131c 100644 --- a/terraform-gpu-devservers/main.tf +++ b/terraform-gpu-devservers/main.tf @@ -95,18 +95,22 @@ locals { "cpu-arm" = { instance_type = "c7g.4xlarge" instance_types = null - instance_count = 30 + instance_count = 0 gpus_per_instance = 0 use_placement_group = false architecture = "arm64" + karpenter_managed = true + karpenter_max_nodes = 30 } "cpu-x86" = { instance_type = "c7i.4xlarge" instance_types = null - instance_count = 30 + instance_count = 0 gpus_per_instance = 0 use_placement_group = false architecture = "x86_64" + karpenter_managed = true + karpenter_max_nodes = 30 } "t4" = { instance_type = "g4dn.12xlarge" @@ -221,18 +225,22 @@ locals { "cpu-arm" = { instance_type = "c7g.8xlarge" instance_types = null - instance_count = 30 + instance_count = 0 gpus_per_instance = 0 use_placement_group = false architecture = "arm64" + karpenter_managed = true + karpenter_max_nodes = 30 } "cpu-x86" = { instance_type = "c7i.8xlarge" instance_types = null - instance_count = 30 + instance_count = 0 gpus_per_instance = 0 use_placement_group = false architecture = "x86_64" + karpenter_managed = true + karpenter_max_nodes = 30 } } }