diff --git a/.github/scripts/check-catalog-requests.sh b/.github/scripts/check-catalog-requests.sh index 60e1cfec..a81cd7aa 100755 --- a/.github/scripts/check-catalog-requests.sh +++ b/.github/scripts/check-catalog-requests.sh @@ -118,7 +118,8 @@ pull() { # : three attempts, helm's error kept fi rm -rf "${dest}" echo "helm pull $* (attempt ${n}/3): ${err}" >&2 - [ "${n}" = 3 ] || sleep $((n * 5)) + # 20 s, then 40 s: a throttling registry (ECR Public, 429) needs longer than a blip. + [ "${n}" = 3 ] || sleep $((n * 20)) done echo "::error::cannot pull $*: ${err}" exit 1 diff --git a/.github/scripts/upstream-monitoring.sh b/.github/scripts/upstream-monitoring.sh index 2c7965fd..765ddfda 100755 --- a/.github/scripts/upstream-monitoring.sh +++ b/.github/scripts/upstream-monitoring.sh @@ -27,6 +27,8 @@ argocd|dashboard|dashboard-argocd|https://raw.githubusercontent.com/argoproj/arg argocd-image-updater|dashboard|dashboard-argocd-image-updater|https://raw.githubusercontent.com/argoproj-labs/argocd-image-updater/v1.3.0/config/example-grafana-dashboard.json crossplane|dashboard|dashboard-crossplane|https://grafana.com/api/dashboards/24549/revisions/3/download external-dns|dashboard|dashboard-external-dns|https://grafana.com/api/dashboards/23969/revisions/1/download +karpenter|dashboard|dashboard-karpenter-capacity|https://raw.githubusercontent.com/aws/karpenter-provider-aws/v1.15.0/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +karpenter|dashboard|dashboard-karpenter-performance|https://raw.githubusercontent.com/aws/karpenter-provider-aws/v1.15.0/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json keda|dashboard|dashboard-keda|https://raw.githubusercontent.com/kedacore/keda/v2.21.0/config/grafana/keda-dashboard.json velero|dashboard|dashboard-velero|https://grafana.com/api/dashboards/16829/revisions/5/download EOF diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index dd3f2809..3b1f19d5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -232,7 +232,10 @@ jobs: name: ${{ matrix.module }} (${{ matrix.cloud }}) runs-on: ubuntu-24.04 needs: discover - timeout-minutes: 20 + # velero's job ran 17.8 to 20.3 min once Crossplane installs six AWS + # providers (#53), and karpenter's is of the same length: 20 left no + # room for the destroy. + timeout-minutes: 25 permissions: contents: read checks: write diff --git a/.github/workflows/pr-static.yaml b/.github/workflows/pr-static.yaml index 302e71b6..82fd2c99 100644 --- a/.github/workflows/pr-static.yaml +++ b/.github/workflows/pr-static.yaml @@ -383,8 +383,13 @@ jobs: with: path: /tmp/charts # The pinned chart versions live in the modules' ResourceSets and - # in the clouds' patches. + # in the clouds' patches. A chart is cached under its URL, name and + # version, which never change content, so the last cache restores + # whatever charts it has when the key moves: a registry that + # throttles the runners (ECR Public answers 429 to anonymous pulls, + # the karpenter chart's) is then asked only for a new version. key: catalog-charts-${{ hashFiles('oci/catalog/*/resourceset.yaml', 'oci/clusters/*/*/patch.yaml') }} + restore-keys: catalog-charts- - name: Every container the catalog runs states its requests if: steps.detect.outputs.found == 'true' run: | diff --git a/README.md b/README.md index 6e843786..2fa168fb 100644 --- a/README.md +++ b/README.md @@ -156,13 +156,14 @@ The stack's design: [Observability](docs/architecture/observability.md). | Module | What it does | Version | Default | Clouds | Notes | | --- | --- | --- | :---: | --- | --- | -| [`crossplane`](docs/catalog/crossplane.md) | Lets each module declare its own cloud IAM | 2.4.2 | off | All | Needed by `velero`, and on AWS and GCP by `external_dns`, `keda` (`services`), `external_secrets` and `argocd_image_updater` (`repositories`) for their own access; on GCP no Google service account | +| [`crossplane`](docs/catalog/crossplane.md) | Lets each module declare its own cloud IAM | 2.4.2 | off | All | Needed by `velero` and `karpenter`, and on AWS and GCP by `external_dns`, `keda` (`services`), `external_secrets` and `argocd_image_updater` (`repositories`) for their own access; on GCP no Google service account | ### Autoscaling | Module | What it does | Version | Default | Clouds | Notes | | --- | --- | --- | :---: | --- | --- | | [`keda`](docs/catalog/keda.md) | Event-driven autoscaling, down to zero | 2.21.0 | off | All | | +| [`karpenter`](docs/catalog/karpenter.md) | Nodes for your workloads, launched for what is Pending, Spot and Graviton first, removed when empty; a private pool, and a public one for traffic-heavy workloads (no NAT Gateway) | 1.15.0 | off | AWS | Needs `crossplane`; GKE Autopilot provisions its own nodes | ### Backup @@ -191,7 +192,8 @@ the minor. - The foundations modules exist for all four clouds. The single-apply root exists for AWS and GCP, so far. - On GCP every catalog module that runs on AWS runs too, metrics-server - aside (GKE serves its own), each cloud-backed one with its own access + and karpenter aside (GKE serves its own metrics API, and Autopilot + provisions its own nodes), each cloud-backed one with its own access through Crossplane and no Google service account. Its proof is a GKE Autopilot sandbox, by hand: floci-gcp has no Cloud DNS and no Workload Identity, so CI renders and plans GCP but runs no GKE e2e. One socle diff --git a/docs/architecture/cilium-before-flux.md b/docs/architecture/cilium-before-flux.md index eaad95b8..ad524be9 100644 --- a/docs/architecture/cilium-before-flux.md +++ b/docs/architecture/cilium-before-flux.md @@ -60,10 +60,15 @@ follows `cilium.gateway_api`, the class being the catalog's.
Under the hood -**AWS.** `eni.enabled: true`: ENI IPAM, native routing, masquerade off; pods -carry VPC addresses and leave through the NAT Gateway. `kubeProxyReplacement: -true`, with `k8sServiceHost`/`Port` from the foundations' `cluster_endpoint`. -The operator creates ENIs from the bootstrap nodes' role. +**AWS.** `eni.enabled: true`: ENI IPAM, native routing; pods carry VPC +addresses. What leaves the VPC is masqueraded to the node's primary address +(`ipv4NativeRoutingCIDR` the foundations' `vpc_cidr`, through `ens+`): a +pod's secondary address never crosses an Internet Gateway, which a node of +Karpenter's public pool leaves through +([AWS-23](../decisions/aws.md#aws-23-public-nodes-their-own-security-group-and-cilium-masquerading-out-of-the-vpc)). +`kubeProxyReplacement: true`, with `k8sServiceHost`/`Port` from the +foundations' `cluster_endpoint`. The operator creates ENIs from the bootstrap +nodes' role. **Azure.** `aksbyocni.enabled: true`: cluster-pool IPAM over VXLAN, with masquerade. The pool is the foundations' `pod_cidr` (`10.244.0.0/16`), since diff --git a/docs/architecture/module-iam.md b/docs/architecture/module-iam.md index f0d0ede6..ead2649c 100644 --- a/docs/architecture/module-iam.md +++ b/docs/architecture/module-iam.md @@ -6,7 +6,8 @@ sidebar: --- Some modules act on the cloud: external-dns writes DNS records, KEDA reads -queues, External Secrets reads a secret manager, Velero writes a bucket. Each +queues, External Secrets reads a secret manager, Velero writes a bucket, +Karpenter launches nodes. Each needs a role on each cloud. ## The rule @@ -30,7 +31,14 @@ foundations grant it once, the same for every client, bounded by shape default) and always denies `iam`, `sts`, `organizations`, `account`, `sso` and `identitystore`; - Crossplane cannot attach managed policies, edit the boundary, or touch a - role outside its path. + role outside its path; +- with `ec2` in the list, and only then, a role may pass the bootstrap + nodes' role to EC2 and read instance profiles: Karpenter's nodes run as + that role, through the instance profile the foundations give it, and no + other IAM action is allowed + ([AWS-21](../decisions/aws.md#aws-21-the-boundarys-one-iam-exception-the-node-role-passed-to-ec2)); +- beside roles, Crossplane manages a module's buckets, queues and + EventBridge rules under `-` only. A module on a new service needs one word in `allowed_services`: a reviewed change. The root wires the boundary into `kube.crossplane.permissions_boundary`. diff --git a/docs/catalog/crossplane.md b/docs/catalog/crossplane.md index bf37224b..2bcb80fc 100644 --- a/docs/catalog/crossplane.md +++ b/docs/catalog/crossplane.md @@ -8,8 +8,8 @@ requires: [] Crossplane is the tooling a module that needs a cloud service uses to declare its own role, beside its workload. It grants nothing by itself. **Off by default**; turn it on when a module you want needs cloud access (external-dns, -external-secrets, keda with `services`, velero, argocd-image-updater with -`repositories`). Providers exist on aws and gcp. +external-secrets, keda with `services`, velero, karpenter, +argocd-image-updater with `repositories`). Providers exist on aws and gcp. ## Getting started @@ -29,7 +29,7 @@ kube = { ``` Then `kubectl get providers.pkg.crossplane.io` shows the providers `Healthy`: -four on aws, three on gcp. +six on aws, three on gcp. ## Settings @@ -88,7 +88,7 @@ kube = { or an `extraEnvVars*` entry named like a password, token or key. Use `values_secret`. - **Upgrades**: the chart moves with `socle_version`, and its CRDs with it - (Crossplane applies them at start). The four providers move together, on + (Crossplane applies them at start). The providers move together, on one tag on each cloud. - **Coming from an alpha before 0.1.0?** Once 0.1.0 is applied, strip the owners the old providers left on their ServiceAccount, once, or the garbage @@ -100,7 +100,8 @@ kube = { **Installed**: chart `crossplane` 2.4.2 from `https://charts.crossplane.io/stable`, in `crossplane-system`. On aws, the providers `provider-family-aws`, -`provider-aws-iam`, `provider-aws-eks` and `provider-aws-s3` (v2.8.1), then +`provider-aws-iam`, `provider-aws-eks`, `provider-aws-s3`, `provider-aws-sqs` +and `provider-aws-cloudwatchevents` (v2.8.1), then `ClusterProviderConfig/default` on EKS Pod Identity, once they are `Healthy`. On gcp, `provider-family-gcp`, `provider-gcp-cloudplatform` and `provider-gcp-storage` (v3.0.0), then `ClusterProviderConfig/default` on @@ -122,7 +123,11 @@ write a billed line of the EKS audit log **Cloud access**: on aws, the foundations' role for `crossplane-system/provider-aws` creates roles only under `/socle//` and only with the boundary, which allows `allowed_services` and always denies -`iam`, `sts` and the account services. On gcp there is no Google service +`iam`, `sts` and the account services — but for one exception with `ec2` +allowed: passing the bootstrap nodes' role to EC2, what Karpenter launches +its nodes with ([AWS-21](../decisions/aws.md#aws-21-the-boundarys-one-iam-exception-the-node-role-passed-to-ec2)). +Beside roles, it manages buckets, SQS queues and EventBridge rules under +`-` only, and never deletes a bucket. On gcp there is no Google service account: the providers' own ServiceAccount is the principal, `principal://iam.googleapis.com/projects//locations/global/workloadIdentityPools/.svc.id.goog/subject/ns/crossplane-system/sa/provider-gcp`, and `InjectedIdentity` makes the provider take its token from the GKE metadata diff --git a/docs/catalog/index.mdx b/docs/catalog/index.mdx index b07993a4..9ae9725b 100644 --- a/docs/catalog/index.mdx +++ b/docs/catalog/index.mdx @@ -1,6 +1,6 @@ --- title: Catalog -description: The sixteen catalog modules, what each gives you, whether it is on by default, the clouds it runs on, and how kube turns them on. +description: The seventeen catalog modules, what each gives you, whether it is on by default, the clouds it runs on, and how kube turns them on. sidebar: label: Overview order: 0 diff --git a/docs/catalog/karpenter.md b/docs/catalog/karpenter.md new file mode 100644 index 00000000..468b5e2b --- /dev/null +++ b/docs/catalog/karpenter.md @@ -0,0 +1,254 @@ +--- +title: karpenter +description: Nodes for your workloads on aws — launched for what is Pending, Spot and Graviton first, removed when empty or under-used. +category: autoscaling +requires: + - module: crossplane + clouds: [aws] + why: "the controller's IAM role and its interruption queue are Crossplane managed resources" +--- + +Karpenter launches the EC2 nodes your pods need and removes those they no +longer use, in two pools: **private**, where a pod goes by default, its +egress through the NAT Gateway; **public**, for workloads costly in network +traffic that ask for it, their egress straight through the Internet Gateway, +with no NAT Gateway to pay. Without the module, a socle cluster on aws has +only its two bootstrap nodes, sized for the socle, not for your workloads. +**Off by default**; **aws only**: on gcp, GKE Autopilot already provisions a +node for every pod. + +## Getting started + +Allow its services in the foundations, then turn it on with Crossplane: + +```hcl +aws = { + crossplane = { allowed_services = ["ec2", "ssm", "pricing", "sqs", "eks"] } +} +``` + +```hcl kube-start="karpenter" +kube = { + crossplane = { enabled = true } + karpenter = { + enabled = true + private_pool = { cpu = 64, memory = "256Gi" } + } +} +``` + +Then `kubectl get nodepools` shows `private` and `public` `Ready`, and a pod +that fits on no node gets one within a minute: `kubectl get nodeclaims` +shows it. A workload asks for the public pool with both of these in its pod +template: + +```yaml +nodeSelector: + socle.do-now.io/node-pool: public +tolerations: + - key: socle.do-now.io/node-pool + value: public + effect: NoSchedule +``` + +## Settings + +| Attribute | Default | | +| --- | --- | --- | +| `enabled` | `false` | Turns the module on or off. | +| `private_pool` | see below | The private NodePool, where a pod goes by default. | +| `public_pool` | see below | The public NodePool, for the pods that tolerate it. | +| `.capacity_types` | `["spot", "on-demand"]` | What the pool may launch. Karpenter takes the cheapest that fits: Spot first. | +| `.architectures` | `["arm64", "amd64"]` | Graviton and x86; Graviton is cheaper, so it comes first. | +| `.cpu` | `32` | vCPU of all its nodes together: the ceiling. | +| `.memory` | `"128Gi"` | Memory of all its nodes together. | +| `.consolidate_after` | `"1m"` | How long a node stays empty or under-used before it goes. | +| `.ami` | `"al2023@latest"` | The AL2023 AMI: `latest` follows each release, `al2023@v` pins one. | +| `values` | `{}` | Any [`karpenter` chart](https://gallery.ecr.aws/karpenter/karpenter) value; yours win. | +| `values_secret` | `""` | A Secret in `karpenter` with a `values.yaml` key, for what must stay out of the OpenTofu state. | + +`tofu plan` refuses the module without Crossplane or a region, a capacity +type or architecture Karpenter does not know, a limit that is not a +quantity, an AMI outside AL2023, and a `controller.env` entry named like a +credential. + +### Every setting + +```hcl kube-full="karpenter" +kube = { + karpenter = { + enabled = false # off by default; needs crossplane + + # The private NodePool, where a pod goes by default: private subnets, + # egress through the NAT Gateway. + private_pool = { + capacity_types = ["spot", "on-demand"] # "spot", "on-demand" + architectures = ["arm64", "amd64"] # kubernetes.io/arch values + cpu = 32 # vCPU, all nodes together + memory = "128Gi" # all nodes together + consolidate_after = "1m" # empty or under-used this long, the node goes + ami = "al2023@latest" # or al2023@v20261001 + } + + # The public NodePool, tainted: public subnets, a public IP per node, + # egress through the Internet Gateway. The same keys. + public_pool = { + capacity_types = ["spot", "on-demand"] + architectures = ["arm64", "amd64"] + cpu = 32 + memory = "128Gi" + consolidate_after = "1m" + ami = "al2023@latest" + } + + # Any value of the karpenter chart 1.15.0; yours win over the socle's. + values = { + logLevel = "debug" + settings = { featureGates = { spotToSpotConsolidation = true } } + } + + # A Secret you create in karpenter, whose values.yaml key holds chart + # values that must not reach the OpenTofu state, such as a + # controller.env proxy URL with a password in it. + values_secret = "karpenter-values" + } +} +``` + +## Good to know + +- **Five services in `aws.crossplane.allowed_services`**: `ec2`, `ssm`, + `pricing`, `sqs`, `eks`. The plan cannot see them from the bootstrap; the + aws root warns when one is missing, and the controller is otherwise refused + by IAM, the reason in its logs. +- **An x86-only image needs a `nodeSelector`**: `kubernetes.io/arch: amd64`. + Without one, Karpenter may place the pod on Graviton, cheaper, where it + fails with `exec format error`. The socle's own images run on both. +- **Your nodes are the bootstrap nodes' peers**: same role, AL2023, IMDSv2 + on one hop, an encrypted gp3 root volume of 40 GiB. Cilium's agent starts + first: no pod lands before the node has a network. Private nodes share the + bootstrap nodes' security group and subnets. +- **The public pool is opt-in, and takes nothing from the internet.** Its + nodes have a public IP for their egress only: their own security group + admits the VPC and nothing else, and no load balancer targets them — + a Gateway's traffic reaches your pods there through the private nodes. + Put on it what moves a lot of data out (crawlers, media, exports) or in; + keep the rest private. A pod there leaves with its node's public IP: + allow-list the addresses of the public nodes, not the NAT Gateway's, at + any third party that filters by source. +- **Nodes live 30 days at most** (`expireAfter`), then are replaced, on the + latest AMI with `al2023@latest`; Karpenter replaces a node whose AMI is + superseded the same way (drift). At most one node in ten is disrupted at a + time. Guard a pod that must not move with `karpenter.sh/do-not-disrupt: + "true"`, and give your Deployments a PodDisruptionBudget. +- **Spot reclaims are drained ahead**: EC2's two-minute warning, rebalance + recommendations and AWS Health maintenance reach Karpenter through its + queue, and it moves the pods before the node goes. +- **More NodePools are yours to write**: a GPU pool, a tainted pool for one + team. They reference the socle's `EC2NodeClass` `private` or `public`, or + one of yours. The socle's pools' limits are yours to raise through + `private_pool` and `public_pool`. +- **Turning it off removes its nodes first**: the NodePool goes, Karpenter + drains and terminates its nodes, then the controller goes. Your pods on + those nodes go Pending. The CRDs, and the NodePools you wrote, stay. +- **Before `tofu destroy`**, turn the module off and wait for + `kubectl get nodeclaims` to be empty: a destroy that removes the socle + while Karpenter is draining can leave instances behind, whose network + interfaces then stop the VPC's deletion. If it did, terminate the + instances tagged `karpenter.sh/nodepool` and `kubernetes.io/cluster/` + in EC2, then destroy again. +- **Upgrades**: the chart moves with `socle_version`, and Karpenter's CRDs + with it (created and replaced at install and upgrade). + +
+Under the hood + +**Installed**: chart `karpenter` 1.15.0 from `oci://public.ecr.aws/karpenter`, +in `karpenter` (Pod Security `restricted`): the controller, two replicas, +never on a node Karpenter manages, preferably one per bootstrap node and zone; +its CRDs (`NodePool`, `NodeClaim`, `EC2NodeClass`, `NodeOverlay`, +`CapacityBuffer`). Then `EC2NodeClass` and `NodePool` `private` and `public`, +from the `karpenter-nodes` Kustomization. On aws, `Role/karpenter`, +`PodIdentityAssociation/karpenter`, `Queue/karpenter` and five `Rule`s with their +`Target`s. + +**What the socle sets**: the cluster's name, its endpoint and service range +from `DescribeCluster` (AL2023 nodes need the range), the interruption queue, +`AWS_REGION` (a pod cannot reach instance metadata on these nodes), requests +with no limit, the scrape annotations. Your `values` are merged over these; +a `controller.env` of yours replaces the socle's, so keep `AWS_REGION` in it. + +**The nodes**: both EC2NodeClasses name the instance profile +`-bootstrap-node`, which the foundations create for the bootstrap +nodes' role, so Karpenter creates no profile and the node joins through the +access entry EKS made for that role. `private` selects the subnets tagged +`kubernetes.io/cluster/: owned` and `kubernetes.io/role/internal-elb` +and the security groups tagged for the cluster — the cluster security group. +`public` selects the subnets tagged `kubernetes.io/role/elb`, the security +group `-public-nodes` the foundations create, asks for a public IP, +and has its kubelet register the node labelled +`node.kubernetes.io/exclude-from-external-load-balancers` (an AL2023 +`NodeConfig` beside Karpenter's): EKS's load balancer controller opens a +Service's NodePort to `0.0.0.0/0` in the security group of every node it +targets, and does so in the seconds before Karpenter could label the node — +measured on the sandbox. Both NodePools allow families `c`, `m`, `r` of +generation 6 and later, the startup taint `node.cilium.io/agent-not-ready`, +consolidation `WhenEmptyOrUnderutilized`; `public` is tainted +`socle.do-now.io/node-pool=public:NoSchedule`, and each carries the label +`socle.do-now.io/node-pool`. A pod's address is an ENI's secondary one, +which the Internet Gateway never translates: Cilium masquerades what leaves +the VPC to the node's primary address +([AWS-23](../decisions/aws.md#aws-23-public-nodes-their-own-security-group-and-cilium-masquerading-out-of-the-vpc)). +Flux applies a kind named `*Class` before the rest and waits for it: a +change to a pool reaches its NodePool only while its EC2NodeClass is Ready. + +**Cloud access**: the role `-karpenter`, under `/socle//` +with the foundations' boundary, holds AWS's own controller policy for +Karpenter 1.15.0, minus the instance-profile writes and the zonal-shift read: +EC2 launches, tags and terminations only on what carries this cluster's tag +and a NodePool's; `iam:PassRole` on `-bootstrap-node`, to EC2 only; +reads of instance profiles, AMIs, prices, subnets; its own queue. The +boundary allows that one IAM action only when `ec2` is in +`allowed_services`, and only on that role +([AWS-21](../decisions/aws.md#aws-21-the-boundarys-one-iam-exception-the-node-role-passed-to-ec2)). +The queue `-karpenter` keeps a message five minutes, encrypted, +HTTPS only, written by EventBridge only; the rules `-karpenter-*` +send it Spot interruption warnings, rebalance recommendations, instance state +changes, capacity-reservation interruptions and AWS Health events. + +**Ordering**: the role, association and queue are a step of their own, +`cloud`, which aws adds to the module +([`oci/clusters/aws/karpenter/patch.yaml`](../../oci/clusters/aws/karpenter/patch.yaml), +[SOCLE-32](../decisions/socle.md#socle-32-what-only-one-cloud-has-that-cloud-adds-to-the-module-as-steps)): +the module waits for the `crossplane` ResourceSet, the controller starts once +they are Ready (`karpenter-access`), and the NodePool once the controller is. +Removal runs the other way: the NodePool and the EC2NodeClass sit in a Flux +Kustomization, which flux-operator deletes first and waits for, the controller +in the child ResourceSet `karpenter-controller`, deleted after +([KARPENTER-04](../decisions/karpenter.md#karpenter-04-the-nodes-go-before-the-controller)). +flux-operator waits one minute for the first stage: a drain longer than that +is why the module goes off before a destroy. + +**Measured** on floci 2.1.0, 2026-10-16: the controller Ready with static +keys, the EC2NodeClasses reconciled against floci's EC2 (no subnet found, +floci's VPC carries no cluster tag). On the aws sandbox (eu-west-3, +2026-10-17, `karpenter-module-eks`): a private node launched in 46 s, Ready +with Cilium 8 s later, consolidated 2 min after its pods went; a pod on a +public node reached the internet with its node's IP, another on a private +node with the NAT Gateway's; a public node never became a load balancer +target, and its NodePort stayed closed from the internet. With the module +on: the controller's access Ready and its Pod Identity check Complete in +14 s; a private node Ready with Cilium in 55 s, a public node and its pod's +egress in 54 s, both consolidated 2 min 21 s after their pods went. Turned +off with a node running: the NodePools went first, the instance was +terminated within a minute, then the controller went; nothing left. + +
+ +## Per cloud + +| Cloud | | +| --- | --- | +| aws | the module, as above | +| gcp | not offered: GKE Autopilot provisions, sizes and removes a node for every pod itself — the socle's GKE has no node pool to scale ([KARPENTER-05](../decisions/karpenter.md#karpenter-05-aws-only-gke-autopilot-is-its-own-node-autoscaler)) | +| azure, scaleway | not offered: AKS has Node Auto Provisioning, Scaleway its cluster autoscaler, both decided with their foundations | diff --git a/docs/clouds/aws/foundations.md b/docs/clouds/aws/foundations.md index e3d0bcfb..a4eb7e8f 100644 --- a/docs/clouds/aws/foundations.md +++ b/docs/clouds/aws/foundations.md @@ -31,7 +31,9 @@ module together with the bootstrap. | Control-plane logs off: CloudWatch bills them by the gigabyte. `cluster_log_types = ["audit", "authenticator"]` when your audit asks for the API server's record | default | | Two Spot Graviton bootstrap nodes, untainted, not autoscaled | default | | Bootstrap nodes on AL2023, IMDSv2 hop limit 1, encrypted gp3, private subnets | enforced | -| Crossplane's role and permissions boundary, only when `crossplane` is set | default | +| An instance profile for the bootstrap nodes' role, the identity Karpenter's nodes run as | enforced | +| A security group for Karpenter's public nodes: the VPC in, nothing from the internet | enforced | +| Crossplane's role and permissions boundary, only when `crossplane` is set; with `ec2` allowed, the boundary lets a role pass the nodes' role to EC2, nothing else of IAM | default |
Under the hood @@ -70,8 +72,11 @@ unavailable at a time), `crossplane = null`. - **Workload identities other than Crossplane's**: each catalog module declares its own; the CSI drivers' roles are the bootstrap's. -- **Fargate and Karpenter**: Karpenter is to be a catalog module - ([#53](https://github.com/do-now-io/socle/issues/53)). +- **Fargate and Karpenter**: no Fargate; Karpenter is the + [karpenter](../../catalog/karpenter.md) catalog module. The foundations + give the bootstrap nodes' role an instance profile, which its nodes run + as, and nothing else of it + ([AWS-21](../../decisions/aws.md#aws-21-the-boundarys-one-iam-exception-the-node-role-passed-to-ec2)). - **GuardDuty**. - **A DynamoDB endpoint, a region variable, any credential as input.** diff --git a/docs/clouds/aws/index.md b/docs/clouds/aws/index.md index ff43c95f..951d4a4c 100644 --- a/docs/clouds/aws/index.md +++ b/docs/clouds/aws/index.md @@ -62,5 +62,5 @@ September 2026, 730 hours a month. every module checked by Chainsaw, then uninstalled and destroyed ([`e2e.yaml`](../../../.github/workflows/e2e.yaml)). Not proven: Cilium on real nodes, Pod Identity, IAM enforcement, ACM, Route 53, the NLBs. -- **Not built**: Karpenter ([#53](https://github.com/do-now-io/socle/issues/53)), - the Upgrade Insights pre-check, cloud-resource observability. +- **Not built**: the Upgrade Insights pre-check, cloud-resource observability. + Karpenter is the [karpenter](../../catalog/karpenter.md) catalog module. diff --git a/docs/clouds/aws/limits.md b/docs/clouds/aws/limits.md index d37ac223..a1d88084 100644 --- a/docs/clouds/aws/limits.md +++ b/docs/clouds/aws/limits.md @@ -23,9 +23,10 @@ sidebar: - **An existing VPC** (`create_vpc = false`) gets no VPC endpoints and no flow logs. - **The NLBs' TLS policy is AWS's default**; the socle sets none. - **Elastic IPs**: one per AZ per cluster, 5 per region by default ([Prerequisites](prerequisites.md#quotas)). +- **Capacity for workloads comes from [karpenter](../../catalog/karpenter.md)**, off by default: without it everything runs on the two bootstrap Spot nodes. Its launch of a node, Cilium on it and consolidation are proven on the sandbox only: floci launches no EC2 node. +- **What leaves the VPC leaves with a node's address**: Cilium masquerades it, so the NAT Gateway and the flow logs see nodes, not pods; pods on the public pool leave with their node's public IP ([AWS-23](../../decisions/aws.md#aws-23-public-nodes-their-own-security-group-and-cilium-masquerading-out-of-the-vpc)). ## What the socle does not offer here yet -- **Capacity for workloads**: no Karpenter ([#53](https://github.com/do-now-io/socle/issues/53)); everything runs on the two bootstrap Spot nodes. - **Backups outside the cluster's account and region** ([velero](../../catalog/velero.md)). - **The Upgrade Insights pre-check** and **cloud-resource observability**: proposed, not built. diff --git a/docs/clouds/gcp/index.md b/docs/clouds/gcp/index.md index bdd3165b..f2c5783c 100644 --- a/docs/clouds/gcp/index.md +++ b/docs/clouds/gcp/index.md @@ -52,7 +52,8 @@ Autopilot bills Pod requests, not nodes. US dollars, us-central1 list prices, - **Built**: the foundations module and the [`opentofu/clusters/gcp`](../../../opentofu/clusters/gcp/README.md) root; on gcp the catalog offers every module but `metrics_server` (GKE serves - `metrics.k8s.io` itself), with Crossplane's Cloud Platform and Storage + `metrics.k8s.io` itself) and `karpenter` (Autopilot provisions its own + nodes, [KARPENTER-05](../../decisions/karpenter.md#karpenter-05-aws-only-gke-autopilot-is-its-own-node-autoscaler)), with Crossplane's Cloud Platform and Storage providers, the shared Gateways, Velero and `kube.keda.services = ["pubsub"]`. - **Proven**: in CI, a plan against floci-gcp, which runs no Compute Engine API; on a real GKE Autopilot sandbox project (2026-10-07, PR #84), the diff --git a/docs/clouds/gcp/sizing.md b/docs/clouds/gcp/sizing.md index 4fff539f..7f6fee07 100644 --- a/docs/clouds/gcp/sizing.md +++ b/docs/clouds/gcp/sizing.md @@ -76,7 +76,7 @@ So ask for the `SSD_TOTAL_GB` increase ([prerequisites](prerequisites.md#quotas) ## On AWS -The same requests apply. With metrics-server, the node exporter, Velero's node-agent (512Mi of disk per node, no limit: Kopia's cache grows with what it uploads) and four Crossplane providers, the catalog requests about 2.1 vCPU, 5.7 GiB of memory and 6.2 GiB of ephemeral storage. On EKS a request only schedules; nothing turns it into a limit, but under DiskPressure the kubelet evicts first the pods furthest over their ephemeral-storage request. +The same requests apply. With metrics-server, the node exporter, Velero's node-agent (512Mi of disk per node, no limit: Kopia's cache grows with what it uploads), six Crossplane providers and Karpenter's two replicas, the catalog requests about 2.4 vCPU, 6.8 GiB of memory and 6.8 GiB of ephemeral storage. On EKS a request only schedules; nothing turns it into a limit, but under DiskPressure the kubelet evicts first the pods furthest over their ephemeral-storage request. ## The guard diff --git a/docs/decisions/aws.md b/docs/decisions/aws.md index c6e783f9..a7da7d51 100644 --- a/docs/decisions/aws.md +++ b/docs/decisions/aws.md @@ -371,3 +371,101 @@ Delivering to S3 instead was set aside: the flow logs would leave CloudWatch Logs Insights, and the control plane has no S3 destination. **Sources.** [VPC Flow Logs troubleshooting](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-troubleshooting.html#flow-logs-troubleshooting-no-log-group) · [IAM role for flow logs](https://docs.aws.amazon.com/vpc/latest/userguide/flow-logs-iam-role.html) · [Delete a flow log](https://docs.aws.amazon.com/vpc/latest/userguide/working-with-flow-logs.html#delete-flow-log) · [EKS control plane logs](https://docs.aws.amazon.com/eks/latest/userguide/control-plane-logs.html) · [AmazonEKSServiceRolePolicy](https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonEKSServiceRolePolicy.html) · #98. + +## AWS-21: The boundary's one IAM exception: the node role, passed to EC2 + +**accepted** · 2026-10-16 · [`opentofu/aws/iam.tf`](../../opentofu/aws/iam.tf), [`opentofu/aws/nodes.tf`](../../opentofu/aws/nodes.tf) + +**Decision.** The bootstrap nodes' role gets an instance profile of the same +name, `-bootstrap-node`, unconditionally: the identity of every node +of the cluster, the bootstrap group's and Karpenter's. When +`crossplane.allowed_services` names `ec2`, and only then, the boundary allows +`iam:PassRole` on that role with `iam:PassedToService = ec2.amazonaws.com`, +and `iam:GetInstanceProfile` and `iam:ListInstanceProfiles`; `iam:*` stays +denied on every other resource. + +**Context.** Karpenter's controller launches instances with an instance +profile, which EC2 refuses unless the caller may pass its role; its +controller is a Crossplane-made role under the boundary +([SOCLE-04](socle.md#socle-04-each-module-owns-its-cloud-access-the-foundations-never-change)), +and the boundary denied all of `iam` ([AWS-18](#aws-18-crossplanes-identity-and-its-permissions-boundary-in-the-foundations)). +AWS's own setup gives Karpenter a node role and lets it create and tag +instance profiles itself. Weighed: a scoped exception in the boundary; a +second node role made by the foundations for Karpenter; the node role, +profile and access entry made by Crossplane, which would need +`CreateInstanceProfile`, PassRole to EC2 of any role it makes and +`eks:CreateAccessEntry`, and a boundary allowing what nodes do to every +module. Karpenter's nodes need exactly the bootstrap nodes' rights — kubelet, +ECR pull, Cilium's ENI calls — and EKS already made the `EC2_LINUX` access +entry for that role. With `spec.instanceProfile` set, Karpenter creates no +profile and only reads it (`GetInstanceProfile` to validate, +`ListInstanceProfiles` for its garbage collection). + +**Consequences.** A module role allowed `ec2` may launch instances as the +nodes' role and nothing more: no other role to pass, no profile to create or +change. No role is made by the foundations for a module, and nothing in +them names Karpenter: the exception follows `ec2`, a reviewed word in the +client's tfvars. Karpenter's nodes and the bootstrap nodes share one +identity, so Cilium's operator works wherever it lands. Without `ec2`, the +boundary is the one AWS-18 describes. + +**Sources.** [Karpenter 1.15.0 getting started, CloudFormation](https://github.com/aws/karpenter-provider-aws/blob/v1.15.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml) · [Karpenter `EC2NodeClass.spec.instanceProfile`](https://karpenter.sh/docs/concepts/nodeclasses/#specinstanceprofile) · [IAM PassRole](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_passrole.html) · [EKS access entries for nodes](https://docs.aws.amazon.com/eks/latest/userguide/creating-access-entries.html) · [karpenter module](../catalog/karpenter.md). + +## AWS-22: Crossplane manages queues and EventBridge rules under the cluster's prefix + +**accepted** · 2026-10-16 · [`opentofu/aws/iam.tf`](../../opentofu/aws/iam.tf) + +**Decision.** With `crossplane` set, Crossplane's role may create, configure, +tag and delete SQS queues `-*` and EventBridge rules +`-*` on the default bus, with their targets, in the cluster's +region; never read or send a message. + +**Context.** A module's resources beside its role are Crossplane's to make, as +Velero's bucket is ([AWS-18](#aws-18-crossplanes-identity-and-its-permissions-boundary-in-the-foundations)): +Karpenter's interruption queue and the rules that fill it. + +**Consequences.** Unlike a bucket, a queue may be deleted: it holds notices +minutes old. The providers `provider-aws-sqs` and +`provider-aws-cloudwatchevents` run on every aws cluster with Crossplane on. + +**Sources.** [Karpenter interruption handling](https://karpenter.sh/docs/concepts/disruption/#interruption) · [SQS actions](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonsqs.html) · [EventBridge actions](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazoneventbridge.html). + +## AWS-23: Public nodes: their own security group, and Cilium masquerading out of the VPC + +**accepted** · 2026-10-17 · [`opentofu/aws/nodes.tf`](../../opentofu/aws/nodes.tf), [`opentofu/bootstrap/cilium.tf`](../../opentofu/bootstrap/cilium.tf), [`oci/catalog/karpenter/nodes/ec2nodeclass-public.yaml`](../../oci/catalog/karpenter/nodes/ec2nodeclass-public.yaml) + +**Decision.** The foundations create the security group +`-public-nodes`: in from the VPC's CIDR — the control plane's +ENIs and every node — out to anywhere; the cluster security group admits it +back (on an emulated EKS that reports no cluster security group, floci, that +rule lands on the group itself). Not +tagged for the cluster. Karpenter's public nodes run in it alone, and their +kubelet registers them labelled +`node.kubernetes.io/exclude-from-external-load-balancers`. On aws Cilium +masquerades what leaves the VPC to the node's primary address +(`enableIPv4Masquerade`, `ipv4NativeRoutingCIDR` the VPC's CIDR, +`egressMasqueradeInterfaces: ens+`), on every node. + +**Context.** Measured on the sandbox, 2026-10-17, with a public NodePool: +a pod's address is an ENI's secondary one, which the Internet Gateway never +translates, so a pod on a public node reached nothing outside the VPC until +Cilium masqueraded it; Cilium refuses masquerading in ENI mode without +`egressMasqueradeInterfaces` (the agents crashed). EKS's in-tree load +balancer controller opened a Service's NodePort to `0.0.0.0/0` in the +security group of every node it targets: on the cluster security group, the +public node's NodePort answered from the internet. With their own group and +the exclusion label set by Karpenter, the controller still targeted a new +node in the seconds before the label arrived, opened the NodePort in that +group, and never closed it; Karpenter keeps `kubernetes.io` labels off the +kubelet's flags, so the label goes in an AL2023 `NodeConfig`, which the +kubelet may set (`node.kubernetes.io/` is a kubelet label domain). With +both, a public node was never a target and its NodePort stayed closed. + +**Consequences.** A Gateway's traffic reaches pods on public nodes through +the private nodes. Pods on private nodes leave with their node's address too, +so the NAT Gateway and the flow logs show nodes, not pods, for what leaves +the VPC; inside it nothing changes. With `create_vpc = false`, `vpc_cidr` +must be the VPC's primary CIDR. The primary interface is `ens5` on AL2023 on +Nitro; another AMI family would need its name. + +**Sources.** [Cilium ENI mode, masquerading](https://docs.cilium.io/en/stable/network/concepts/ipam/eni/) · [cloud-provider-aws, `findSecurityGroupForInstance`](https://github.com/kubernetes/cloud-provider-aws/blob/master/pkg/providers/v1/aws.go) · [Well-known label `node.kubernetes.io/exclude-from-external-load-balancers`](https://kubernetes.io/docs/reference/labels-annotations-taints/#node-kubernetes-io-exclude-from-external-load-balancers) · [AL2023 NodeConfig](https://awslabs.github.io/amazon-eks-ami/nodeadm/) · #53. diff --git a/docs/decisions/index.md b/docs/decisions/index.md index 5627e287..c388969d 100644 --- a/docs/decisions/index.md +++ b/docs/decisions/index.md @@ -37,7 +37,7 @@ section gives its **status**, **date** and implementing code, then the [argocd](argocd.md) · [crossplane](crossplane.md) · [external-dns](external-dns.md) · [external-secrets](external-secrets.md) · -[gateway-api](gateway-api.md) · [keda](keda.md) · +[gateway-api](gateway-api.md) · [karpenter](karpenter.md) · [keda](keda.md) · [kyverno](kyverno.md) · [kyverno-policies](kyverno-policies.md) · [metrics-server](metrics-server.md) · [monitoring](monitoring.md) · [otel-agent](otel-agent.md) · [otel-gateway](otel-gateway.md) · diff --git a/docs/decisions/karpenter.md b/docs/decisions/karpenter.md new file mode 100644 index 00000000..db8c3515 --- /dev/null +++ b/docs/decisions/karpenter.md @@ -0,0 +1,155 @@ +--- +title: karpenter decisions +description: The decisions behind the karpenter module, one per section, each with its status. +sidebar: + order: 10 +--- + +Karpenter is the socle's node autoscaler on AWS, implementing +[AWS-01](aws.md#aws-01-eks-standard-not-auto-mode): EKS Standard with a +self-hosted Karpenter. The decisions that shape the rest are the nodes' +identity ([KARPENTER-02](#karpenter-02-the-nodes-are-the-bootstrap-nodes-peers)) +and the order in which it is removed +([KARPENTER-04](#karpenter-04-the-nodes-go-before-the-controller)). + +## KARPENTER-01: AWS's controller policy, without instance-profile writes + +**accepted** · 2026-10-16 · [`oci/clusters/aws/karpenter/patch.yaml`](../../oci/clusters/aws/karpenter/patch.yaml) + +**Decision.** The controller's role, `-karpenter`, is a Crossplane +`Role` under `/socle//` with the foundations' boundary, bound by Pod +Identity to `karpenter/karpenter`. Its inline policy is AWS's own controller +policy for Karpenter 1.15.0, statement for statement, minus +`iam:CreateInstanceProfile`, `TagInstanceProfile`, `AddRoleToInstanceProfile`, +`RemoveRoleFromInstanceProfile`, `DeleteInstanceProfile` and the zonal-shift +read; `iam:PassRole` names the bootstrap nodes' role only. + +**Context.** The issue's blocking question: the boundary denies `iam:*` +([AWS-18](aws.md#aws-18-crossplanes-identity-and-its-permissions-boundary-in-the-foundations)), +and Karpenter needs `iam:PassRole` and, when it manages profiles, profile +writes. OpenTofu never creates a role for a module +([SOCLE-04](socle.md#socle-04-each-module-owns-its-cloud-access-the-foundations-never-change)). +Every EC2 write in AWS's policy is already conditioned on the cluster's tag +and a NodePool's. + +**Consequences.** The boundary must allow `ec2`, `ssm`, `pricing`, `sqs` and +`eks`; the aws root warns when one is missing. Its one IAM exception is +[AWS-21](aws.md#aws-21-the-boundarys-one-iam-exception-the-node-role-passed-to-ec2). +`eks:DescribeCluster` stays: AL2023's nodeadm needs the service range, which +only it gives Karpenter. Zonal shift stays off. + +**Sources.** [Karpenter 1.15.0 getting started, CloudFormation](https://github.com/aws/karpenter-provider-aws/blob/v1.15.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml) · `pkg/providers/instanceprofile` and `pkg/controllers/nodeclass` at v1.15.0 · #53. + +## KARPENTER-02: The nodes are the bootstrap nodes' peers + +**accepted** · 2026-10-16 · [`oci/catalog/karpenter/nodes/ec2nodeclass.yaml`](../../oci/catalog/karpenter/nodes/ec2nodeclass.yaml), [`opentofu/aws/nodes.tf`](../../opentofu/aws/nodes.tf) + +**Decision.** Both EC2NodeClasses name the instance profile +`-bootstrap-node` (`spec.instanceProfile`) and run AL2023 with +IMDSv2 on one hop and an encrypted gp3 root volume. `private` selects the +subnets tagged `kubernetes.io/cluster/: owned` and +`kubernetes.io/role/internal-elb: "1"` and the security groups tagged +`kubernetes.io/cluster/: owned`; `public` the subnets tagged +`kubernetes.io/role/elb: "1"` and the security group `-public-nodes` +([KARPENTER-03](#karpenter-03-two-nodepools-private-and-public-their-shape-typed-per-pool)). + +**Context.** The issue asked for no new foundations tag. The foundations tag +the private subnets for internal load balancers and the cluster; EKS tags the +cluster security group, the bootstrap nodes' one, for the cluster, and no +other security group in the VPC carries that tag. The bootstrap nodes' role +already holds what a node needs, Cilium's ENI calls included, and EKS made +its access entry. + +**Consequences.** No second identity to keep in step: Cilium's operator may +run on any node. A client attaching an existing VPC tags its private subnets +the same way, as the Gateways' load balancers already require. Pods do not +reach instance metadata, so the controller is given `AWS_REGION`. Flux +applies a kind named `*Class` first and waits for it to be Ready, whatever +the Kustomization's `wait` (fluxcd/pkg `ssa`, `ApplyAllStaged`): a change to +the NodePool waits on a Ready EC2NodeClass, which floci's never is. + +**Sources.** [Karpenter NodeClasses](https://karpenter.sh/docs/concepts/nodeclasses/) · [EKS cluster security group](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) · [AWS-17](aws.md#aws-17-a-bootstrap-node-group-of-two-spot-nodes). + +## KARPENTER-03: Two NodePools, private and public, their shape typed per pool + +**accepted** · 2026-10-17 · [`oci/catalog/karpenter/nodes/`](../../oci/catalog/karpenter/nodes), [`opentofu/bootstrap/catalog.tf`](../../opentofu/bootstrap/catalog.tf), [`opentofu/bootstrap/variables.tf`](../../opentofu/bootstrap/variables.tf) + +**Decision.** The socle renders two NodePools, always. `private`, where a +pod goes by default: the private subnets, egress through the NAT Gateway. +`public`, tainted `socle.do-now.io/node-pool=public:NoSchedule`: the public +subnets, a public IP per node, egress through the Internet Gateway, for the +workloads that tolerate it and select the label `socle.do-now.io/node-pool`. +Each: Spot and on demand, arm64 and amd64, families `c`, `m`, `r` of +generation 6 and later, 32 vCPU and 128Gi in all, `WhenEmptyOrUnderutilized` +after one minute, nodes expiring after 30 days, Cilium's startup taint +`node.cilium.io/agent-not-ready`. `kube.karpenter.private_pool` and +`public_pool` type each one's capacity types, architectures, limits, +consolidation delay and AMI alias, validated at plan; any other NodePool is +the client's to write. + +**Context.** The issue asked for Spot and Graviton first, like the bootstrap +group, limits, a consolidation policy, and whether `values` or a typed +attribute carries it; the NodePool is not a chart value, `values` cannot +reach it. Karpenter launches the cheapest offering that fits, so allowing +both capacity types and both architectures is Spot and Graviton first. +Hugo asked (2026-10-17) for a public pool beside the private one: a NAT +Gateway bills every gigabyte it processes, so a workload heavy in network +traffic is cheaper on a node that leaves through the Internet Gateway, and +the rest stays private, where nothing reaches it from the internet. Tainted, +so that nothing lands public without asking. + +**Consequences.** An x86-only image needs a `nodeSelector` on +`kubernetes.io/arch`. A limit reached leaves pods Pending, with an alert +(`KarpenterNodePoolNearLimit`). `al2023@latest` moves nodes onto each new +AMI by drift; a client pins one with `al2023@v`. A public node takes +nothing from the internet ([AWS-23](aws.md#aws-23-public-nodes-their-own-security-group-and-cilium-masquerading-out-of-the-vpc)), +and its pods leave with its public IP, not the NAT Gateway's: a third party +that allow-lists the NAT Gateway does not see them. + +**Sources.** [Karpenter NodePools](https://karpenter.sh/docs/concepts/nodepools/) · [Cilium, Karpenter's startup taint](https://docs.cilium.io/en/stable/installation/taints/) · [NAT Gateway pricing](https://aws.amazon.com/vpc/pricing/) · #53. + +## KARPENTER-04: The nodes go before the controller + +**accepted** · 2026-10-16 · [`oci/catalog/karpenter/resourceset.yaml`](../../oci/catalog/karpenter/resourceset.yaml) + +**Decision.** The NodePool and the EC2NodeClass are applied by a Flux +Kustomization, `karpenter-nodes`, with `deletionPolicy: WaitForTermination`; +the controller's HelmRelease is the one object of a child ResourceSet, +`karpenter-controller`. + +**Context.** The nodes Karpenter launched are in no OpenTofu state, and only +Karpenter terminates them, when their NodePool goes, while it runs. A +`ResourceSet` deletes what it rendered in two stages: Flux objects first, +waiting up to a minute for them, then the rest (flux-operator 0.60.0, +`deleteAllStaged`). A NodePool beside the controller's HelmRelease would go +after it, and its nodes, and their network interfaces, would outlive the +cluster and stop the VPC's deletion. + +**Consequences.** Turning the module off and destroying the socle remove the +nodes first. A drain that takes more than flux-operator's minute (a +PodDisruptionBudget holding) lets the controller go before the last node: +hence the rule to turn the module off and wait for `kubectl get nodeclaims` to +be empty before `tofu destroy`, and the EC2 clean-up in the module's page. + +**Sources.** flux-operator v0.60.0 `internal/controller/resourceset_controller.go` · [Karpenter, uninstall](https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/#delete-the-cluster) · #53. + +## KARPENTER-05: aws only; GKE Autopilot is its own node autoscaler + +**accepted** · 2026-10-16 · [`opentofu/bootstrap/catalog.tf`](../../opentofu/bootstrap/catalog.tf) (`catalog_clouds`) + +**Decision.** No karpenter module on gcp, and none in its place: the socle's +GKE is Autopilot, which provisions, sizes and removes nodes for the pods it +schedules. + +**Context.** Karpenter has providers for AWS and Azure only. On Autopilot +there is no node pool to scale and no node auto-provisioning to turn on: +Google operates both +([GCP-01](gcp.md#gcp-01-autopilot-only-no-cluster-mode-option)). A pod's requests are what Autopilot bills and sizes, and +Spot is a `nodeSelector` (`cloud.google.com/gke-spot`) or a ComputeClass the +workload asks for. + +**Consequences.** On gcp a client sizes requests and chooses Spot per +workload; nothing in the catalog sets a node shape. A ComputeClass catalog +module would be a decision of its own, not this one. + +**Sources.** [GKE Autopilot overview](https://cloud.google.com/kubernetes-engine/docs/concepts/autopilot-overview) · [Autopilot Spot Pods](https://cloud.google.com/kubernetes-engine/docs/how-to/autopilot-spot-pods) · [Karpenter providers](https://karpenter.sh/docs/). diff --git a/docs/guides/uninstall.md b/docs/guides/uninstall.md index 781458ac..e2c3c5a1 100644 --- a/docs/guides/uninstall.md +++ b/docs/guides/uninstall.md @@ -48,6 +48,11 @@ suite). ## Everything +With `karpenter` on, turn it off first and wait for its nodes to go +(`kubectl get nodeclaims` empty): they are in no OpenTofu state, and an +instance left behind stops the VPC's deletion +([karpenter](../catalog/karpenter.md)). + On GCP, the cluster carries deletion protection: set `gcp.deletion_protection = false` and apply first. Then, on both: @@ -81,6 +86,8 @@ below. | Module roles (AWS), when `crossplane` went first or the bootstrap left the state | nothing is left to delete them | delete the roles under `/socle//` in IAM | | Module bindings (GCP), when `crossplane` went first or the bootstrap left the state | nothing is left to delete them; a socle created in the project next inherits them | remove the `principal://…` members of the project's IAM policy | | Velero's bucket (S3 on AWS, GCS on GCP) | Crossplane may not delete a bucket of data | in the cloud, once its backups are not needed | +| Karpenter's CRDs, and the NodePools you wrote | Helm never deletes a chart's CRDs | by hand once unused, or with the cluster | +| An instance Karpenter launched, when the socle went while it drained | only Karpenter terminates its nodes | terminate the instances tagged `karpenter.sh/nodepool` and `kubernetes.io/cluster/` in EC2 | | A pending Velero `Restore` | its finalizer holds the namespace | delete the `Restore` ([Restore a backup](restore-a-backup.md)) | | `ImageUpdater` objects in `argocd`, once `argocd_image_updater` is off | their finalizer holds them, and the `argocd` namespace, with no controller to release it | delete them before turning the module off, or release them: `kubectl -n argocd patch imageupdater --type merge -p '{"metadata":{"finalizers":[]}}'` | | A mirror's pull secret | `flux-system` outlives the operator's release | with the cluster | diff --git a/docs/reference/alert-rules.md b/docs/reference/alert-rules.md index d94414e3..f4954f75 100644 --- a/docs/reference/alert-rules.md +++ b/docs/reference/alert-rules.md @@ -78,6 +78,7 @@ Each in the module's `monitoring/` folder, present only while the module and | `crossplane` | 2 | the socle: managed resources not Ready, not Synced | | `external_dns` | 2 | the socle: no successful sync for an hour, source or registry errors | | `external_secrets` | 2 | the socle: an ExternalSecret not Ready, a store not Ready | +| `karpenter` | 5 | the socle: an EC2NodeClass or a NodePool not Ready, a NodePool near its limits, EC2 calls failing, pods no NodePool can place | | `keda` | 3 | the chart's example (scaler errors), and the socle: ScaledObject and ScaledJob errors | | `kyverno` | 3 | the chart's examples: admission latency, policy execution latency | | `reloader` | 1 | the socle: a reload that failed | diff --git a/docs/reference/catalog-module-standard.md b/docs/reference/catalog-module-standard.md index 80077beb..549a4982 100644 --- a/docs/reference/catalog-module-standard.md +++ b/docs/reference/catalog-module-standard.md @@ -65,6 +65,7 @@ root is applied once, in `root ()`. | `cloud: any` / `cloud: ` | every cloud, or one | a per-cloud assertion is its own `Test` | | `platform: any` / `platform: floci` | everywhere, or floci only | what only floci can state | | `platform: gke` | the GKE sandbox, run by hand; the floci CI never selects it | what only a real GKE can state; the Test carries its own `spec.timeouts`, at least apply 30s, assert 180s, error 60s, delete 60s, cleanup 120s and exec 30s: Chainsaw's defaults are too short for calls through the DNS endpoint and Autopilot's restarts (measured 2026-10-16) | +| `platform: eks` | the aws sandbox, run by hand; the floci CI never selects it | what only a real EKS can state — an EC2 node launched and joined, Cilium on it; the Test carries its own `spec.timeouts`, as for gke | The `module` phase proves at least: diff --git a/docs/reference/compatibility.md b/docs/reference/compatibility.md index 9f1dcc67..d293963b 100644 --- a/docs/reference/compatibility.md +++ b/docs/reference/compatibility.md @@ -58,7 +58,7 @@ From each `oci/catalog//resourceset.yaml`. Offered is not proven: CI app | --- | --- | --- | --- | --- | | `hello` | `podinfo`, `oci://ghcr.io/stefanprodan/charts` | `6.15.0` | | all | | `gateway_api` | no chart | | | all | -| `crossplane` | `crossplane`, `https://charts.crossplane.io/stable` | `2.4.2` | v2.4.2; AWS providers v2.8.1; GCP providers v3.0.0 (`provider-family-gcp`, `provider-gcp-cloudplatform`, `provider-gcp-storage`) | all (providers on aws and gcp) | +| `crossplane` | `crossplane`, `https://charts.crossplane.io/stable` | `2.4.2` | v2.4.2; AWS providers v2.8.1 (`provider-family-aws`, `provider-aws-iam`, `provider-aws-eks`, `provider-aws-s3`, `provider-aws-sqs`, `provider-aws-cloudwatchevents`); GCP providers v3.0.0 (`provider-family-gcp`, `provider-gcp-cloudplatform`, `provider-gcp-storage`) | all (providers on aws and gcp) | | `external_dns` | `external-dns`, `https://kubernetes-sigs.github.io/external-dns/` | `1.22.0` | 0.22.0 | all | | `argocd` | `argo-cd`, `oci://ghcr.io/argoproj/argo-helm` | `10.9.2` | v3.5.3 | all | | `argocd_image_updater` | `argocd-image-updater`, `oci://ghcr.io/argoproj/argo-helm` | `1.3.1` | v1.3.0 | all (`repositories` on aws and gcp) | @@ -67,6 +67,7 @@ From each `oci/catalog//resourceset.yaml`. Offered is not proven: CI app | `otel_gateway` | `opentelemetry-collector`, same | `0.173.1` | 0.160.0 | all | | `keda` | `keda`, `https://kedacore.github.io/charts` | `2.21.0` | 2.21.0 | all (`services` on aws and gcp) | | `metrics_server` | `metrics-server`, `https://kubernetes-sigs.github.io/metrics-server/` | `3.14.0` | 0.9.0 | aws | +| `karpenter` | `karpenter`, `oci://public.ecr.aws/karpenter` | `1.15.0` | 1.15.0 | aws | | `kyverno` | `kyverno`, `oci://ghcr.io/kyverno/charts` | `3.9.1` | v1.19.1 | all | | `kyverno_policies` | `kyverno-policies`, `oci://ghcr.io/kyverno/charts` | `3.9.1` | v1.19.1 | all | | `external_secrets` | `external-secrets`, `oci://ghcr.io/external-secrets/charts` | `2.11.0` | v2.11.0 | all (its role and store on aws and gcp) | diff --git a/docs/reference/inputs.md b/docs/reference/inputs.md index d94fc5e9..3506fdd9 100644 --- a/docs/reference/inputs.md +++ b/docs/reference/inputs.md @@ -68,6 +68,9 @@ level deep: a key written alone keeps its siblings at their default. | `keda` | `services` | `[]` | aws and gcp: the services KEDA's own identity may read — on aws from `sqs`, `cloudwatch`, `kinesis`, `dynamodb`, each service also in `aws.crossplane.allowed_services`; on gcp `pubsub`, read through Cloud Monitoring, `roles/monitoring.viewer` also in `gcp.crossplane.allowed_roles`. Needs `crossplane` on | | `metrics_server` | `enabled` | `true` | the `metrics.k8s.io` API behind `kubectl top` and HPAs on CPU or memory. aws only | | `metrics_server` | `ha` | `false` | two replicas on two nodes, with a disruption budget | +| `karpenter` | `enabled` | `false` | Karpenter, the nodes for the workloads. aws only; needs `crossplane` on and `ec2`, `ssm`, `pricing`, `sqs`, `eks` in `aws.crossplane.allowed_services` | +| `karpenter` | `private_pool` | `{ capacity_types = ["spot", "on-demand"], architectures = ["arm64", "amd64"], cpu = 32, memory = "128Gi", consolidate_after = "1m", ami = "al2023@latest" }` | the private NodePool, where a pod goes by default: what it may launch, the ceiling of all its nodes together, how long a node stays empty or under-used before it goes, and the AL2023 alias | +| `karpenter` | `public_pool` | the same | the public NodePool, tainted `socle.do-now.io/node-pool=public`: public subnets, a public IP per node, egress through the Internet Gateway | | `kyverno` | `enabled` | `false` | the Kyverno engine, no policy | | `kyverno_policies` | `enabled` | `false` | the socle's policy set, every policy in Audit; needs `kyverno` | | `kyverno_policies` | `profile` | `"baseline"` | `baseline`, or `restricted` for baseline plus six | diff --git a/oci/.ci/inputs-sample.yaml b/oci/.ci/inputs-sample.yaml index 7789949e..619d2307 100644 --- a/oci/.ci/inputs-sample.yaml +++ b/oci/.ci/inputs-sample.yaml @@ -178,6 +178,25 @@ args: - --v=2 values_secret: metrics-server-values + karpenter: + enabled: true + private_pool: + capacity_types: [spot, on-demand] + architectures: [arm64, amd64] + cpu: 32 + memory: 128Gi + consolidate_after: 1m + ami: al2023@latest + public_pool: + capacity_types: [spot] + architectures: [arm64] + cpu: 16 + memory: 64Gi + consolidate_after: 5m + ami: al2023@latest + values: + logLevel: debug + values_secret: karpenter-values hello: enabled: true replicas: 2 diff --git a/oci/catalog/crossplane/tests/e2e/chainsaw-test.yaml b/oci/catalog/crossplane/tests/e2e/chainsaw-test.yaml index 9de51635..cb1ca3f3 100644 --- a/oci/catalog/crossplane/tests/e2e/chainsaw-test.yaml +++ b/oci/catalog/crossplane/tests/e2e/chainsaw-test.yaml @@ -168,6 +168,26 @@ spec: status: (conditions[?type == 'Healthy'] || `[]`): - status: 'True' + # SQS and EventBridge, for the queues and rules a module owns + # (docs/catalog/karpenter.md). + - assert: + resource: + apiVersion: pkg.crossplane.io/v1 + kind: Provider + metadata: + name: provider-aws-sqs + status: + (conditions[?type == 'Healthy'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: pkg.crossplane.io/v1 + kind: Provider + metadata: + name: provider-aws-cloudwatchevents + status: + (conditions[?type == 'Healthy'] || `[]`): + - status: 'True' # A Provider is Healthy before every CRD it brings is served, and an # assert on a kind not served yet fails at once (run 36983797132, # once the S3 provider joined the family): the CRDs Established first. diff --git a/oci/catalog/karpenter/monitoring/dashboard-karpenter-capacity.yaml b/oci/catalog/karpenter/monitoring/dashboard-karpenter-capacity.yaml new file mode 100644 index 00000000..13c7edb4 --- /dev/null +++ b/oci/catalog/karpenter/monitoring/dashboard-karpenter-capacity.yaml @@ -0,0 +1,1803 @@ +# From https://raw.githubusercontent.com/aws/karpenter-provider-aws/v1.15.0/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json, as it is: written by .github/scripts/upstream-monitoring.sh, +# checked against the source in CI. Do not edit; bump the source. +apiVersion: v1 +kind: ConfigMap +metadata: + name: karpenter-dashboard-karpenter-capacity + labels: + grafana_dashboard: "1" +data: + dashboard-karpenter-capacity.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 32, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Nodes rate", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "Nodes total" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(karpenter_nodes_created_total{nodepool=~\"$nodepool\"}[$__rate_interval])) by(cluster,nodepool)", + "format": "time_series", + "legendFormat": "{{cluster}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(cluster,nodepool) (karpenter_nodes_created_total{nodepool=~\"$nodepool\"})", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Nodes Created by nodepool", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Nodes rate", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "Nodes total" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(karpenter_nodes_terminated_total{nodepool=~\"$nodepool\"}[$__rate_interval])) by(cluster,nodepool)", + "format": "time_series", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(cluster,nodepool) (karpenter_nodes_terminated_total{nodepool=~\"$nodepool\"})", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Nodes Terminated by nodepool", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(reason)(karpenter_voluntary_disruption_eligible_nodes)", + "legendFormat": "reason={{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Nodes Eligible for Disruptions by \"reason\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Disruptions rate", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "Disruptions total" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(karpenter_nodeclaims_disrupted_total{nodepool=~\"$nodepool\"}[$__rate_interval])) by(cluster,nodepool)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(cluster,nodepool)(karpenter_nodeclaims_disrupted_total{nodepool=~\"$nodepool\"})", + "hide": false, + "instant": false, + "legendFormat": "{{label_name}}", + "range": true, + "refId": "B" + } + ], + "title": "Node Disruptions by \"$distribution_filter\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Disruptions rate", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "Disruptions total" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum(rate(karpenter_voluntary_disruption_decisions_total[$__rate_interval])) by(cluster,nodepool,consolidation_type,decision)", + "hide": false, + "legendFormat": "type={{consolidation_type}}, decision={{decision}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(consolidation_type,decision)(karpenter_voluntary_disruption_decisions_total)", + "hide": true, + "instant": false, + "legendFormat": "{{label_name}}", + "range": true, + "refId": "B" + } + ], + "title": "Disruption Actions performed by \"type\" and \"decision\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod Distribution by \"$distribution_filter\"", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum by(phase)(karpenter_pods_state)", + "legendFormat": "{{label_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Pod Phase", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*Utilization$" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + }, + { + "id": "unit", + "value": "percentunit" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Memory Provisioned" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 18, + "x": 0, + "y": 42 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodepools_usage{resource_type=\"cpu\"} / karpenter_nodepools_limit{resource_type=\"cpu\"}", + "format": "table", + "instant": true, + "legendFormat": "CPU Limit Utilization", + "range": false, + "refId": "CPU Limit Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Node Count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodepools_usage{resource_type=\"memory\"} / karpenter_nodepools_limit{resource_type=\"memory\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Limit Utilization", + "range": false, + "refId": "Memory Limit Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "CPU Capacity" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", + "format": "table", + "hide": false, + "instant": true, + "range": false, + "refId": "Memory Capacity" + } + ], + "title": "Nodepool Summary", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "nodepool" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Time 5": true, + "__name__": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "job": true, + "job 1": true, + "job 2": true, + "resource_type": true, + "resource_type 1": true, + "resource_type 2": true + }, + "indexByName": { + "Time 1": 6, + "Time 2": 7, + "Time 3": 11, + "Time 4": 15, + "Time 5": 16, + "Value #CPU Capacity": 2, + "Value #CPU Limit Utilization": 3, + "Value #Memory Capacity": 4, + "Value #Memory Limit Utilization": 5, + "Value #Node Count": 1, + "instance 1": 8, + "instance 2": 12, + "job 1": 9, + "job 2": 13, + "nodepool": 0, + "resource_type 1": 10, + "resource_type 2": 14 + }, + "renameByName": { + "Time 1": "", + "Value": "CPU Utilization", + "Value #CPU Capacity": "CPU Provisioned", + "Value #CPU Limit Utilization": "CPU Limit Utilization", + "Value #CPU Utilization": "CPU Limit Utilization", + "Value #Memory Capacity": "Memory Provisioned", + "Value #Memory Limit Utilization": "Memory Limit Utilization", + "Value #Memory Utilization": "Memory Utilization", + "Value #Node Count": "Node Count", + "instance": "", + "instance 1": "", + "job": "", + "nodepool": "Nodepool" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", + "legendFormat": "Percentage", + "range": true, + "refId": "A" + } + ], + "title": "Spot Node Percentage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "node_name" + }, + "properties": [ + { + "id": "custom.width", + "value": 333 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*Utilization" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "thresholds", + "value": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 75 + } + ] + } + }, + { + "id": "max", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Uptime" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "decimals", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 4, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Uptime" + } + ] + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "CPU Utilization", + "range": false, + "refId": "CPU Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Utilization", + "range": false, + "refId": "Memory Utilization" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Memory Utilization", + "range": false, + "refId": "Pod Count" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "label_replace(\n sum by (node)(node_time_seconds) - sum by (node)(node_boot_time_seconds),\n \"node_name\", \"$1\", \"node\", \"(.+)\"\n)", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "Uptime", + "range": false, + "refId": "Uptime" + } + ], + "title": "Node Summary", + "transformations": [ + { + "id": "seriesToColumns", + "options": { + "byField": "node_name" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time 4": true, + "Value": false, + "Value #Pod Count": false, + "__name__": true, + "arch": true, + "arch 1": true, + "arch 2": true, + "arch 3": true, + "capacity_type 2": true, + "capacity_type 3": true, + "instance": true, + "instance 1": true, + "instance 2": true, + "instance 3": true, + "instance_category 1": true, + "instance_category 2": true, + "instance_category 3": true, + "instance_cpu": true, + "instance_cpu 1": true, + "instance_cpu 2": true, + "instance_cpu 3": true, + "instance_family": true, + "instance_family 1": true, + "instance_family 2": true, + "instance_family 3": true, + "instance_generation 1": true, + "instance_generation 2": true, + "instance_generation 3": true, + "instance_gpu_count": true, + "instance_gpu_count 1": true, + "instance_gpu_count 2": true, + "instance_gpu_count 3": true, + "instance_gpu_manufacturer": true, + "instance_gpu_manufacturer 1": true, + "instance_gpu_manufacturer 2": true, + "instance_gpu_manufacturer 3": true, + "instance_gpu_memory": true, + "instance_gpu_memory 1": true, + "instance_gpu_memory 2": true, + "instance_gpu_memory 3": true, + "instance_gpu_name": true, + "instance_gpu_name 1": true, + "instance_gpu_name 2": true, + "instance_gpu_name 3": true, + "instance_hypervisor": true, + "instance_hypervisor 1": true, + "instance_hypervisor 2": true, + "instance_hypervisor 3": true, + "instance_local_nvme 1": true, + "instance_local_nvme 2": true, + "instance_local_nvme 3": true, + "instance_memory": true, + "instance_memory 1": true, + "instance_memory 2": true, + "instance_memory 3": true, + "instance_pods": true, + "instance_pods 1": true, + "instance_pods 2": true, + "instance_pods 3": true, + "instance_size": true, + "instance_size 1": true, + "instance_size 2": true, + "instance_size 3": true, + "instance_type 1": false, + "instance_type 2": true, + "instance_type 3": true, + "job": true, + "job 1": true, + "job 2": true, + "job 3": true, + "node": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, + "os": true, + "os 1": true, + "os 2": true, + "os 3": true, + "resource_type": true, + "resource_type 1": true, + "resource_type 2": true, + "resource_type 3": true, + "zone 1": false, + "zone 2": true, + "zone 3": true + }, + "indexByName": { + "Time 1": 1, + "Time 2": 25, + "Time 3": 45, + "Time 4": 65, + "Value #CPU Utilization": 10, + "Value #Memory Utilization": 11, + "Value #Pod Count": 9, + "Value #Uptime": 8, + "arch 1": 5, + "arch 2": 26, + "arch 3": 46, + "capacity_type 1": 6, + "capacity_type 2": 27, + "capacity_type 3": 47, + "instance 1": 4, + "instance 2": 28, + "instance 3": 48, + "instance_cpu 1": 12, + "instance_cpu 2": 29, + "instance_cpu 3": 49, + "instance_family 1": 13, + "instance_family 2": 30, + "instance_family 3": 50, + "instance_gpu_count 1": 14, + "instance_gpu_count 2": 31, + "instance_gpu_count 3": 51, + "instance_gpu_manufacturer 1": 15, + "instance_gpu_manufacturer 2": 32, + "instance_gpu_manufacturer 3": 52, + "instance_gpu_memory 1": 16, + "instance_gpu_memory 2": 33, + "instance_gpu_memory 3": 53, + "instance_gpu_name 1": 17, + "instance_gpu_name 2": 34, + "instance_gpu_name 3": 54, + "instance_hypervisor 1": 18, + "instance_hypervisor 2": 35, + "instance_hypervisor 3": 55, + "instance_memory 1": 19, + "instance_memory 2": 36, + "instance_memory 3": 56, + "instance_pods 1": 20, + "instance_pods 2": 37, + "instance_pods 3": 57, + "instance_size 1": 21, + "instance_size 2": 38, + "instance_size 3": 58, + "instance_type 1": 3, + "instance_type 2": 39, + "instance_type 3": 59, + "job 1": 22, + "job 2": 40, + "job 3": 60, + "node": 66, + "node_name": 0, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, + "os 1": 23, + "os 2": 41, + "os 3": 61, + "resource_type 1": 24, + "resource_type 2": 43, + "resource_type 3": 63, + "zone 1": 7, + "zone 2": 44, + "zone 3": 64 + }, + "renameByName": { + "Time": "", + "Time 1": "", + "Value": "CPU Utilization", + "Value #Allocatable": "", + "Value #CPU Utilization": "CPU Utilization", + "Value #Memory Utilization": "Memory Utilization", + "Value #Pod CPU": "", + "Value #Pod Count": "Pods", + "Value #Uptime": "Uptime", + "arch": "Architecture", + "arch 1": "Arch", + "capacity_type": "Capacity Type", + "capacity_type 1": "Capacity Type", + "instance 1": "Instance", + "instance_cpu 1": "vCPU", + "instance_type": "Instance Type", + "instance_type 1": "Instance Type", + "node_name": "Node Name", + "nodepool 1": "Nodepool", + "zone 1": "Zone" + } + } + } + ], + "type": "table" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "nodepool", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, nodepool)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, zone)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "zone", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, zone)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, arch)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "arch", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, arch)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, capacity_type)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "capacity_type", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, capacity_type)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(karpenter_nodes_allocatable, instance_type)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "instance_type", + "options": [], + "query": { + "query": "label_values(karpenter_nodes_allocatable, instance_type)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "nodepool", + "value": "nodepool" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "distribution_filter", + "options": [ + { + "selected": false, + "text": "arch", + "value": "arch" + }, + { + "selected": false, + "text": "capacity_type", + "value": "capacity_type" + }, + { + "selected": false, + "text": "instance_type", + "value": "instance_type" + }, + { + "selected": false, + "text": "namespace", + "value": "namespace" + }, + { + "selected": false, + "text": "node", + "value": "node" + }, + { + "selected": true, + "text": "nodepool", + "value": "nodepool" + }, + { + "selected": false, + "text": "zone", + "value": "zone" + } + ], + "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Karpenter Capacity v1", + "uid": "ta8I9Q67Z", + "version": 7, + "weekStart": "" + } diff --git a/oci/catalog/karpenter/monitoring/dashboard-karpenter-performance.yaml b/oci/catalog/karpenter/monitoring/dashboard-karpenter-performance.yaml new file mode 100644 index 00000000..6ab572f4 --- /dev/null +++ b/oci/catalog/karpenter/monitoring/dashboard-karpenter-performance.yaml @@ -0,0 +1,605 @@ +# From https://raw.githubusercontent.com/aws/karpenter-provider-aws/v1.15.0/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json, as it is: written by .github/scripts/upstream-monitoring.sh, +# checked against the source in CI. Do not edit; bump the source. +apiVersion: v1 +kind: ConfigMap +metadata: + name: karpenter-dashboard-karpenter-performance + labels: + grafana_dashboard: "1" +data: + dashboard-karpenter-performance.json: | + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 33, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_nodes_termination_duration_seconds{quantile=\"0\"})", + "legendFormat": "Min", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_nodes_termination_duration_seconds{quantile=\"0.5\"})", + "hide": false, + "legendFormat": "P50", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_nodes_termination_duration_seconds{quantile=\"0.9\"})", + "hide": false, + "legendFormat": "P90", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_nodes_termination_duration_seconds{quantile=\"0.99\"})", + "hide": false, + "legendFormat": "P99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_nodes_termination_duration_seconds{quantile=\"1\"})", + "hide": false, + "legendFormat": "Max", + "range": true, + "refId": "E" + } + ], + "title": "Node Termination Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_pods_startup_duration_seconds{quantile=\"0\"})", + "format": "time_series", + "legendFormat": "Min", + "range": true, + "refId": "Minimum" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_pods_startup_duration_seconds{quantile=\"0.5\"})", + "hide": false, + "legendFormat": "P50", + "range": true, + "refId": "Median" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_pods_startup_duration_seconds{quantile=\"0.9\"})", + "hide": false, + "legendFormat": "P90", + "range": true, + "refId": "P90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_pods_startup_duration_seconds{quantile=\"0.99\"})", + "hide": false, + "legendFormat": "P99", + "range": true, + "refId": "P99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(karpenter_pods_startup_duration_seconds{quantile=\"1\"})", + "hide": false, + "legendFormat": "Max", + "range": true, + "refId": "Maximum" + } + ], + "title": "Pod Startup Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 13, + "x": 0, + "y": 17 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", + "hide": false, + "legendFormat": "Min", + "range": true, + "refId": "Minimum" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", + "legendFormat": "P50", + "range": true, + "refId": "Median" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", + "hide": false, + "legendFormat": "P90", + "range": true, + "refId": "P90" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", + "hide": false, + "legendFormat": "P99", + "range": true, + "refId": "P99" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, rate(controller_runtime_reconcile_time_seconds_bucket{controller=\"$controller\"}[10m]))", + "hide": false, + "legendFormat": "Max", + "range": true, + "refId": "Maximum" + } + ], + "title": "Controller Reconciliation Latency [$controller]", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 13, + "y": 17 + }, + "id": 8, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(rate(controller_runtime_reconcile_total{job=\"karpenter\"}[10m])) by (controller)", + "legendFormat": "{{controller}}", + "range": true, + "refId": "A" + } + ], + "title": "Controller Reconciliation Rate", + "type": "bargauge" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "disruption", + "value": "disruption" + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(controller_runtime_reconcile_time_seconds_count{job=\"karpenter\"}, controller)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "controller", + "options": [], + "query": { + "query": "label_values(controller_runtime_reconcile_time_seconds_count{job=\"karpenter\"}, controller)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Karpenter Performance v1", + "uid": "fdusq1f2alerke", + "version": 3, + "weekStart": "" + } diff --git a/oci/catalog/karpenter/monitoring/kustomization.yaml b/oci/catalog/karpenter/monitoring/kustomization.yaml new file mode 100644 index 00000000..836eb2dd --- /dev/null +++ b/oci/catalog/karpenter/monitoring/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - dashboard-karpenter-capacity.yaml + - dashboard-karpenter-performance.yaml + - rules.yaml diff --git a/oci/catalog/karpenter/monitoring/rules.yaml b/oci/catalog/karpenter/monitoring/rules.yaml new file mode 100644 index 00000000..d82c1e4b --- /dev/null +++ b/oci/catalog/karpenter/monitoring/rules.yaml @@ -0,0 +1,67 @@ +# karpenter's own alert rules: awesome-prometheus-alerts has no Karpenter +# section. On the metrics Karpenter 1.15.0 exposes, names from its reference +# (website/content/en/preview/reference/metrics.md at v1.15.0). +# Checked in CI with vmalert -dryRun (.github/scripts/check-rules.sh). +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMRule +metadata: + name: socle-karpenter +spec: + groups: + - name: socle-karpenter + rules: + # An EC2NodeClass that does not resolve — no AMI, subnet, security + # group or instance profile found, or one Karpenter may not use — + # launches nothing: its NodePools stay not Ready, and pods Pending. + - alert: KarpenterNodeClassNotReady + expr: >- + max by (name) (operator_ec2nodeclass_status_condition_count{type="Ready", status="False"}) > 0 + for: 10m + labels: + severity: critical + annotations: + summary: >- + The EC2NodeClass {{ $labels.name }} is not Ready: Karpenter launches no node from it + (kubectl describe ec2nodeclass {{ $labels.name }}). + - alert: KarpenterNodePoolNotReady + expr: >- + max by (name) (operator_nodepool_status_condition_count{type="Ready", status="False"}) > 0 + for: 10m + labels: + severity: critical + annotations: + summary: >- + The NodePool {{ $labels.name }} is not Ready: Karpenter launches no node from it. + # The NodePool's limits are its ceiling: at it, the next pod that + # needs a node stays Pending. + - alert: KarpenterNodePoolNearLimit + expr: >- + max by (nodepool, resource_type) (karpenter_nodepools_usage{resource_type=~"cpu|memory"}) + / max by (nodepool, resource_type) (karpenter_nodepools_limit{resource_type=~"cpu|memory"}) + > 0.9 + for: 15m + labels: + severity: warning + annotations: + summary: >- + The NodePool {{ $labels.nodepool }} uses {{ $value | humanizePercentage }} of its {{ $labels.resource_type }} + limit: raise kube.karpenter.node_pool or new pods will stay Pending. + - alert: KarpenterCloudProviderErrors + expr: >- + sum by (method, error) (rate(karpenter_cloudprovider_errors_total{error!="NodeClaimNotFoundError"}[5m])) > 0 + for: 15m + labels: + severity: warning + annotations: + summary: >- + Karpenter's {{ $labels.method }} calls to EC2 fail ({{ $labels.error }}): nodes may not launch or go. + - alert: KarpenterUnschedulablePods + expr: >- + max(karpenter_scheduler_unschedulable_pods_count) > 0 + for: 15m + labels: + severity: warning + annotations: + summary: >- + {{ $value }} pods fit on no node Karpenter may launch: a requirement no NodePool offers, or a + NodePool at its limit. diff --git a/oci/catalog/karpenter/nodes/ec2nodeclass-private.yaml b/oci/catalog/karpenter/nodes/ec2nodeclass-private.yaml new file mode 100644 index 00000000..2dfa91a4 --- /dev/null +++ b/oci/catalog/karpenter/nodes/ec2nodeclass-private.yaml @@ -0,0 +1,52 @@ +# How a node of the private NodePool is launched: the bootstrap nodes' twin +# (opentofu/aws/nodes.tf), in the private subnets, its egress through the +# NAT Gateway. Every selector reads what the foundations already +# lay down — no new tag (docs/catalog/karpenter.md, KARPENTER-02). The +# module's patches replace `socle` with the cluster's name, and the alias and +# the tags with the cluster's. +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: private +spec: + # The bootstrap nodes' role, through the instance profile the foundations + # create for it: the same kubelet, ECR and Cilium ENI rights, and the access + # entry EKS made for that role, so the node joins. A profile named rather + # than a role: Karpenter then creates none, and needs no IAM write. + instanceProfile: socle-bootstrap-node + # AL2023, the bootstrap nodes' family; the alias follows each new AMI + # (Karpenter drifts the nodes onto it) unless the client pins a version. + amiSelectorTerms: + - alias: al2023@latest + # The private subnets: tagged for the cluster and for internal load + # balancers by the foundations, as EKS's own controllers read them. + subnetSelectorTerms: + - tags: + kubernetes.io/cluster/socle: owned + kubernetes.io/role/internal-elb: "1" + # The cluster security group, the bootstrap nodes' one: EKS creates it and + # tags it for the cluster; nothing else in the VPC carries that tag. + securityGroupSelectorTerms: + - tags: + kubernetes.io/cluster/socle: owned + # Instance metadata on the node only, IMDSv2 only, as on the bootstrap + # nodes: a pod that is not hostNetwork cannot borrow the node's role. + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpTokens: required + httpPutResponseHopLimit: 1 + # An encrypted gp3 root volume, as on the bootstrap nodes, larger: workload + # images live here. + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 40Gi + volumeType: gp3 + encrypted: true + deleteOnTermination: true + # The foundations' tags; Karpenter adds the cluster's, its own, and Name, + # the node's. + tags: + owner: platform + environment: dev diff --git a/oci/catalog/karpenter/nodes/ec2nodeclass-public.yaml b/oci/catalog/karpenter/nodes/ec2nodeclass-public.yaml new file mode 100644 index 00000000..0f6bdb56 --- /dev/null +++ b/oci/catalog/karpenter/nodes/ec2nodeclass-public.yaml @@ -0,0 +1,77 @@ +# How a node of the public NodePool is launched: the bootstrap nodes' twin +# (opentofu/aws/nodes.tf) in the public subnets, with a public IP, its egress +# through the Internet Gateway — no NAT Gateway to pay for the traffic — and +# Cilium masquerading its pods' egress to that IP (AWS-23). Every selector reads what the foundations already +# lay down — no new tag (docs/catalog/karpenter.md, KARPENTER-02). The +# module's patches replace `socle` with the cluster's name, and the alias and +# the tags with the cluster's. +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: public +spec: + # The bootstrap nodes' role, through the instance profile the foundations + # create for it: the same kubelet, ECR and Cilium ENI rights, and the access + # entry EKS made for that role, so the node joins. A profile named rather + # than a role: Karpenter then creates none, and needs no IAM write. + instanceProfile: socle-bootstrap-node + # AL2023, the bootstrap nodes' family; the alias follows each new AMI + # (Karpenter drifts the nodes onto it) unless the client pins a version. + amiSelectorTerms: + - alias: al2023@latest + # The public subnets: tagged for the cluster and for internet-facing load + # balancers by the foundations, as EKS's own controllers read them. + subnetSelectorTerms: + - tags: + kubernetes.io/cluster/socle: owned + kubernetes.io/role/elb: "1" + # The public nodes' own security group, which the foundations create: the + # VPC in, nothing from the internet. Not the + # cluster security group, which EKS's load balancer controller opens to + # 0.0.0.0/0 on every NodePort it serves (opentofu/aws/nodes.tf). + securityGroupSelectorTerms: + - name: socle-public-nodes + # The subnet assigns none: the node asks for its own public IP. + associatePublicIPAddress: true + # Labelled by its own kubelet when it registers, so that no load balancer + # ever targets it, not even in the seconds before Karpenter labels it — + # that window was enough, on the sandbox, for the controller to open a + # NodePort to the internet in its security group. Karpenter keeps + # kubernetes.io labels off the kubelet's flags; nodeadm takes this + # NodeConfig beside Karpenter's. + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: application/node.eks.aws + + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + kubelet: + flags: + - --node-labels=node.kubernetes.io/exclude-from-external-load-balancers=true + + --BOUNDARY-- + # Instance metadata on the node only, IMDSv2 only, as on the bootstrap + # nodes: a pod that is not hostNetwork cannot borrow the node's role. + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpTokens: required + httpPutResponseHopLimit: 1 + # An encrypted gp3 root volume, as on the bootstrap nodes, larger: workload + # images live here. + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 40Gi + volumeType: gp3 + encrypted: true + deleteOnTermination: true + # The foundations' tags; Karpenter adds the cluster's, its own, and Name, + # the node's. + tags: + owner: platform + environment: dev diff --git a/oci/catalog/karpenter/nodes/kustomization.yaml b/oci/catalog/karpenter/nodes/kustomization.yaml new file mode 100644 index 00000000..3f378935 --- /dev/null +++ b/oci/catalog/karpenter/nodes/kustomization.yaml @@ -0,0 +1,14 @@ +# The private and public EC2NodeClasses and NodePools of the karpenter +# module, applied by its karpenter-nodes Kustomization, whose patches put in +# what is the cluster's and what kube.karpenter.private_pool and public_pool +# say +# (oci/catalog/karpenter/resourceset.yaml, step nodes). As written here they +# are the catalog's defaults for a cluster named `socle`: valid objects, so +# CI validates them against Karpenter's schemas. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ec2nodeclass-private.yaml + - ec2nodeclass-public.yaml + - nodepool-private.yaml + - nodepool-public.yaml diff --git a/oci/catalog/karpenter/nodes/nodepool-private.yaml b/oci/catalog/karpenter/nodes/nodepool-private.yaml new file mode 100644 index 00000000..8676f95c --- /dev/null +++ b/oci/catalog/karpenter/nodes/nodepool-private.yaml @@ -0,0 +1,61 @@ +# The private NodePool: what every pod that does not fit on the bootstrap +# group gets a node from, unless it asks for the public one — in the private +# subnets, its egress through the NAT Gateway. The module's patches put +# kube.karpenter.private_pool in it (docs/catalog/karpenter.md); a client +# writes more NodePools beside it. +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: private +spec: + template: + metadata: + labels: + socle.do-now.io/node-pool: private + spec: + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: private + requirements: + # Karpenter launches the cheapest instance that fits among these: + # Spot before on demand, Graviton before x86, as for the bootstrap + # group. A pod whose image is x86 only says so with a nodeSelector on + # kubernetes.io/arch. + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["arm64", "amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + # General purpose, compute and memory optimised, recent generations: + # Karpenter's own default, which keeps out GPU, storage and + # previous-generation families. + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["5"] + # Cilium's agent removes it once it runs on the node: no pod lands + # before the node has a network. A startup taint, so Karpenter does not + # count it against scheduling (docs.cilium.io, Karpenter). + startupTaints: + - key: node.cilium.io/agent-not-ready + value: "true" + effect: NoExecute + # A node lives 30 days at most: replaced on the latest AMI and patches. + expireAfter: 720h + limits: + cpu: "32" + memory: 128Gi + disruption: + # Empty nodes go, and nodes whose pods fit elsewhere for less; at most + # one node in ten at a time, Karpenter's default budget. + consolidationPolicy: WhenEmptyOrUnderutilized + consolidateAfter: 1m + budgets: + - nodes: "10%" diff --git a/oci/catalog/karpenter/nodes/nodepool-public.yaml b/oci/catalog/karpenter/nodes/nodepool-public.yaml new file mode 100644 index 00000000..2c9914fe --- /dev/null +++ b/oci/catalog/karpenter/nodes/nodepool-public.yaml @@ -0,0 +1,65 @@ +# The public NodePool: for a workload costly in network traffic, whose egress +# then goes through the Internet Gateway, not the NAT Gateway. Tainted: a pod +# lands here only if it tolerates socle.do-now.io/node-pool=public and +# selects the label — everything else stays private. The module's patches +# put kube.karpenter.public_pool in it (docs/catalog/karpenter.md). +apiVersion: karpenter.sh/v1 +kind: NodePool +metadata: + name: public +spec: + template: + metadata: + labels: + socle.do-now.io/node-pool: public + spec: + nodeClassRef: + group: karpenter.k8s.aws + kind: EC2NodeClass + name: public + taints: + - key: socle.do-now.io/node-pool + value: public + effect: NoSchedule + requirements: + # Karpenter launches the cheapest instance that fits among these: + # Spot before on demand, Graviton before x86, as for the bootstrap + # group. A pod whose image is x86 only says so with a nodeSelector on + # kubernetes.io/arch. + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["arm64", "amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + # General purpose, compute and memory optimised, recent generations: + # Karpenter's own default, which keeps out GPU, storage and + # previous-generation families. + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["5"] + # Cilium's agent removes it once it runs on the node: no pod lands + # before the node has a network. A startup taint, so Karpenter does not + # count it against scheduling (docs.cilium.io, Karpenter). + startupTaints: + - key: node.cilium.io/agent-not-ready + value: "true" + effect: NoExecute + # A node lives 30 days at most: replaced on the latest AMI and patches. + expireAfter: 720h + limits: + cpu: "32" + memory: 128Gi + disruption: + # Empty nodes go, and nodes whose pods fit elsewhere for less; at most + # one node in ten at a time, Karpenter's default budget. + consolidationPolicy: WhenEmptyOrUnderutilized + consolidateAfter: 1m + budgets: + - nodes: "10%" diff --git a/oci/catalog/karpenter/resourceset.yaml b/oci/catalog/karpenter/resourceset.yaml new file mode 100644 index 00000000..f308d25f --- /dev/null +++ b/oci/catalog/karpenter/resourceset.yaml @@ -0,0 +1,361 @@ +# The karpenter module: the autoscaler that owns every node but the bootstrap +# group's. A pod that fits nowhere gets a node sized for it, Spot and +# Graviton first; an empty or under-used node goes. docs/catalog/karpenter.md +# is the module's page, docs/decisions/karpenter.md its decisions; the rules +# of docs/reference/catalog-module-standard.md apply: inputsFrom the one +# provider, the reconcile toggle on EACH resource, values tested never +# presence. +# +# aws only (catalog_clouds): Karpenter is EC2's. Its cloud access — the +# controller's IAM role, its Pod Identity association, the interruption queue +# and the EventBridge rules feeding it — is a step named cloud that aws +# inserts before workload (oci/clusters/aws/karpenter/patch.yaml, SOCLE-32). +# +# Values, in the order helm-controller merges them, later winning +# (docs/decisions/socle.md#socle-06-the-clients-values-win): +# karpenter-socle-values, the socle's defaults below; karpenter-cloud-values, +# the cloud step's (on aws, its ServiceAccount); karpenter-client-values, +# kube.karpenter.values; then the client's Secret when they name one. No +# inline `values:` on the HelmRelease — helm-controller merges it last, so it +# would beat the client on every key. +# +# Steps, each applied once the one before is healthy: namespace; workload — +# the chart, its CRDs among it; nodes — the private and public +# EC2NodeClasses and NodePools, which need those CRDs. Private is where a pod +# goes by default, its egress through the NAT Gateway; public, tainted, takes +# the workloads costly in network traffic that ask for it, their egress +# through the Internet Gateway (KARPENTER-03). +# +# Uninstall order — the one structural decision here (KARPENTER-04). Nodes +# Karpenter launched are in no OpenTofu state: only Karpenter terminates them, +# when their NodePool goes, and only while it runs. flux-operator deletes what +# a ResourceSet rendered in two stages: the Flux objects (Kustomizations, +# HelmReleases) first, waiting up to a minute for them to go, then everything +# else. So the NodePool and the EC2NodeClass are in a Kustomization, +# karpenter-nodes, deleted in the first stage, which waits for them +# (deletionPolicy: WaitForTermination) while Karpenter drains and terminates +# their nodes; and the controller's HelmRelease is in a child ResourceSet, +# karpenter-controller, which is not a Flux object, so it goes in the second. +# Turning the module off and destroying the socle both take that order. +# +# resourcesTemplate rather than resources: the optional Secret entry is +# conditional text, which the structured list cannot carry. +# +# Fixed by decision, not exposed through a named attribute (values can still +# override any of it): +# - the chart from Karpenter's own OCI registry on ECR Public, pinned exactly. +# - the ServiceAccount is karpenter/karpenter, the subject the module's own +# AWS identity is bound to. +# - two replicas, never on a node Karpenter manages, preferably one per +# bootstrap node and zone; one node is enough for both. +# - the CRDs created and replaced with the chart, on install and upgrade +# (Flux's CreateReplace): Karpenter's CRDs ship in the chart's crds/, which +# Helm alone never upgrades. Helm never deletes them either: they stay when +# the module is off, with whatever NodePools a client wrote. +# - the interruption queue named -karpenter, the one the aws step +# creates; the API endpoint and the service range from DescribeCluster +# (eksControlPlane), what nodeadm needs on AL2023. +apiVersion: fluxcd.controlplane.io/v1 +kind: ResourceSet +metadata: + name: karpenter + namespace: flux-system +spec: + inputsFrom: + - apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + name: socle + wait: true + steps: + - name: namespace + resourcesTemplate: | + apiVersion: v1 + kind: Namespace + metadata: + name: karpenter + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + labels: + # The controller runs non-root, read-only, no capabilities, + # default seccomp: restricted holds, and pins it. + pod-security.kubernetes.io/enforce: restricted + - name: workload + resourcesTemplate: | + apiVersion: source.toolkit.fluxcd.io/v1 + kind: OCIRepository + metadata: + name: karpenter-chart + namespace: karpenter + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + spec: + interval: 1h + url: oci://public.ecr.aws/karpenter/karpenter + layerSelector: + mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip + operation: copy + ref: + tag: 1.15.0 # appVersion 1.15.0 + --- + # The socle's defaults. Everything the client may override through values. + apiVersion: v1 + kind: ConfigMap + metadata: + name: karpenter-socle-values + namespace: karpenter + labels: + reconcile.fluxcd.io/watch: Enabled + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + data: + values.yaml: | + # The identity the aws step binds the controller's role to. + # Renaming it through values breaks that binding. + serviceAccount: + name: karpenter + settings: + clusterName: << inputs.cluster.name | quote >> + # The endpoint, the CA and the service range from + # DescribeCluster: AL2023's nodeadm needs the range, which + # nothing else gives Karpenter. + eksControlPlane: true + # The queue the aws step creates and EventBridge fills: Spot + # interruptions, rebalance recommendations, instance state + # changes, scheduled maintenance. Each drains its node ahead + # of the event instead of losing it. + interruptionQueue: << printf "%s-karpenter" inputs.cluster.name | quote >> + replicas: 2 + # Never on a node Karpenter manages, so it is never the one to + # remove the node it runs on; preferably on the bootstrap group, + # one replica per node and per zone. Preferred, not required: a + # cluster of one bootstrap node runs both replicas on it. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: karpenter.sh/nodepool + operator: DoesNotExist + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: eks.amazonaws.com/nodegroup + operator: Exists + podAntiAffinity: + # null removes the chart's own required anti-affinity, which + # Helm would otherwise merge beside this one. + requiredDuringSchedulingIgnoredDuringExecution: null + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: karpenter + app.kubernetes.io/instance: karpenter + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: ScheduleAnyway + # Requests sized for a small fleet; no limit, which would be a + # guess that kills the one thing adding capacity when it is + # needed. ephemeral-storage declared, as everywhere in the socle. + controller: + # Karpenter asks instance metadata for its region when none is + # set, which a pod cannot reach on these nodes (IMDSv2, one hop). + # A client's controller.env replaces this list: keep it there. + env: + - name: AWS_REGION + value: << inputs.cluster.region | quote >> + resources: + requests: + cpu: 100m + memory: 256Mi + ephemeral-storage: 64Mi + # Metrics for the monitoring module's vmagent, over plain HTTP on + # 8080: the scrape of annotated pods. + podAnnotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + --- + # The client's own chart values, as written in kube.karpenter.values. + # OpenTofu refuses credentials in there. + apiVersion: v1 + kind: ConfigMap + metadata: + name: karpenter-client-values + namespace: karpenter + labels: + # helm-controller watches a valuesFrom ConfigMap only with this label; + # without it, a change here waits for the next interval (10m). + reconcile.fluxcd.io/watch: Enabled + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + data: + values.yaml: |<< toYaml inputs.modules.karpenter.values | nindent 8 >> + --- + # The controller, alone in a child ResourceSet: deleted after the + # nodes, never with them (the header, KARPENTER-04). Rendered here, + # nothing is templated inside it. + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter-controller + namespace: flux-system + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + spec: + wait: true + resources: + - apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: karpenter + namespace: karpenter + spec: + interval: 10m + releaseName: karpenter + chartRef: + kind: OCIRepository + name: karpenter-chart + install: + crds: CreateReplace + remediation: + retries: 3 + upgrade: + crds: CreateReplace + remediation: + retries: 3 + # Merged in this order, later wins; nothing inline after them. The + # client's Secret needs the reconcile.fluxcd.io/watch: Enabled label + # too for a change to apply before the next interval. + valuesFrom: + - kind: ConfigMap + name: karpenter-socle-values + valuesKey: values.yaml + # The cloud step's, when the cloud has values for the chart. + - kind: ConfigMap + name: karpenter-cloud-values + valuesKey: values.yaml + optional: true + - kind: ConfigMap + name: karpenter-client-values + valuesKey: values.yaml + << if inputs.modules.karpenter.values_secret >> + - kind: Secret + name: << inputs.modules.karpenter.values_secret | quote >> + valuesKey: values.yaml + optional: true + << end >> + << $monitored := and inputs.modules.karpenter.enabled inputs.modules.monitoring.enabled ->> + --- + # The module's monitoring — its alert rules and its dashboards, from + # monitoring/ in the socle artifact: VMRules and ConfigMaps Grafana's + # sidecar loads. Present only while both this module and the + # monitoring module are on: monitoring brings the VMRule CRD, and + # until it has the Kustomization fails and retries every minute + # (docs/decisions/monitoring.md, MONITORING-12). + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: karpenter-monitoring + namespace: flux-system + annotations: + fluxcd.controlplane.io/reconcile: << if $monitored >>enabled<< else >>disabled<< end >> + spec: + interval: 1h + retryInterval: 1m + sourceRef: + kind: OCIRepository + name: socle + path: ./catalog/karpenter/monitoring + targetNamespace: karpenter + prune: true + wait: true + timeout: 2m + - name: nodes + resourcesTemplate: | + # The private and public EC2NodeClasses and NodePools, from nodes/ in + # the socle artifact, patched with the cluster's names and + # kube.karpenter.private_pool and public_pool. A Kustomization so that + # they go first and are waited for on uninstall (the header). + # wait: false — a NodePool is Ready once its EC2NodeClass has resolved + # AMIs, subnets, security groups and the instance profile against + # EC2; the alert KarpenterNodeClassNotReady says when it has not. + # Flux applies any kind named *Class first and waits for it whatever + # wait says (fluxcd/pkg ssa, ApplyAllStaged): once the EC2NodeClass + # has a status, the NodePools are applied, or changed, only while the + # EC2NodeClasses are Ready. + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: karpenter-nodes + namespace: flux-system + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + spec: + interval: 1h + retryInterval: 1m + sourceRef: + kind: OCIRepository + name: socle + path: ./catalog/karpenter/nodes + prune: true + wait: false + # Deleted, it holds until the NodePool is gone — and the NodePool + # until Karpenter has drained and terminated every node of it. + deletionPolicy: WaitForTermination + timeout: 10m + # What is this cluster's, over the defaults nodes/ holds. JSON + # values, so a list stays a list. + << $cluster := inputs.cluster.name ->> + << $owned := printf "kubernetes.io/cluster/%s" $cluster ->> + << $tags := dict "owner" inputs.cluster.owner "environment" inputs.cluster.environment ->> + << $tiers := dict + "private" (dict "pool" inputs.modules.karpenter.private_pool "role" "kubernetes.io/role/internal-elb") + "public" (dict "pool" inputs.modules.karpenter.public_pool "role" "kubernetes.io/role/elb") ->> + patches: + << range $tier := list "private" "public" ->> + << $t := get $tiers $tier ->> + << $pool := $t.pool ->> + - target: + kind: EC2NodeClass + name: << $tier >> + patch: | + - op: replace + path: /spec/instanceProfile + value: << printf "%s-bootstrap-node" $cluster | toJson >> + - op: replace + path: /spec/amiSelectorTerms/0/alias + value: << $pool.ami | toJson >> + - op: replace + path: /spec/subnetSelectorTerms + value: << list (dict "tags" (dict $owned "owned" $t.role "1")) | toJson >> + - op: replace + path: /spec/securityGroupSelectorTerms + << if eq $tier "public" ->> + value: << list (dict "name" (printf "%s-public-nodes" $cluster)) | toJson >> + << else ->> + value: << list (dict "tags" (dict $owned "owned")) | toJson >> + << end >> + - op: replace + path: /spec/tags + value: << $tags | toJson >> + - target: + kind: NodePool + name: << $tier >> + patch: | + - op: replace + path: /spec/template/spec/requirements/0/values + value: << $pool.capacity_types | toJson >> + - op: replace + path: /spec/template/spec/requirements/1/values + value: << $pool.architectures | toJson >> + - op: replace + path: /spec/limits + value: << dict "cpu" ($pool.cpu | toString) "memory" $pool.memory | toJson >> + - op: replace + path: /spec/disruption/consolidateAfter + value: << $pool.consolidate_after | toJson >> + << end >> diff --git a/oci/catalog/karpenter/tests/e2e/chainsaw-test.yaml b/oci/catalog/karpenter/tests/e2e/chainsaw-test.yaml new file mode 100644 index 00000000..2b950735 --- /dev/null +++ b/oci/catalog/karpenter/tests/e2e/chainsaw-test.yaml @@ -0,0 +1,1482 @@ +# The karpenter module's proof (docs/catalog/karpenter.md). Off by default, +# so `health` proves the disabled shape only. The `module` phase, on aws, in +# two halves: +# +# 1. The module's own AWS resources, with Crossplane on: its Role reaches +# floci's IAM under /socle// with AWS's controller policy scoped +# to this cluster and to the bootstrap nodes' role, its queue reaches +# floci's SQS and its five rules floci's EventBridge — proven from what +# Crossplane reports — and the workload is withheld while the +# association is not Ready. Then off: the managed resources go, with the +# namespace. The floci seams (providerConfigRef: floci, the association +# never syncing) are steps of their own, to delete the day floci serves +# Pod Identity. +# 2. The controller and its nodes' objects, for real, against floci's EC2. +# floci has no Pod Identity, so the controller runs with Crossplane off +# and static test keys in kube.karpenter.values (the plan refuses both; +# this test writes the inputs directly — the seam). The controller +# Available under Pod Security restricted, never on a node it manages; +# the whole valuesFrom order; the EC2NodeClass and the NodePool as the +# inputs say, the client's pools included, the EC2NodeClasses +# reconciled by the controller against floci's EC2; then off, in the +# order that lets Karpenter remove its nodes — the NodePool and the +# EC2NodeClass go before the controller, their finalizers released by +# it — and the CRDs stay. +# +# Not provable on floci, which launches no EC2 node behind its EKS: a node +# launched, joining, Cilium on it, consolidation — karpenter-module-eks, +# last in this file, run by hand on the aws sandbox (docs/catalog/karpenter.md, +# Measured). +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: karpenter-health + labels: + phase: health + cloud: any + platform: any +spec: + description: Off by default, the ResourceSet is Ready and renders nothing. + timeouts: + assert: 3m + error: 1m + steps: + - name: the ResourceSet is Ready with nothing in its inventory + try: + - assert: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter + namespace: flux-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + (length(inventory.entries || `[]`)): 0 + - error: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: karpenter + catch: + - describe: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + namespace: flux-system + name: karpenter +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: karpenter-module-aws + labels: + phase: module + cloud: aws + platform: any +spec: + description: >- + The controller's role, its queue and its rules through Crossplane, the + workload withheld until they are Ready; then the controller and the + private and public EC2NodeClasses and NodePools against floci, removed + nodes first. + timeouts: + assert: 5m + error: 3m + delete: 3m + steps: + # First half: the module's own AWS resources (docs/catalog/karpenter.md, Cloud access). + - name: with Crossplane on, the module declares its Role, its association, its queue and its rules + # Two providers more than velero's four, cold. + timeouts: + assert: 10m + try: + - patch: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + metadata: + name: socle + namespace: flux-system + spec: + defaultValues: + modules: + crossplane: + enabled: true + permissions_boundary: arn:aws:iam::000000000000:policy/socle-e2e-catalog-crossplane-boundary + karpenter: + enabled: true + # In the order their CRDs appear: the release (core), the providers + # Healthy, their CRDs Established (a Provider is Healthy before every + # CRD it brings is served), then the module's objects. + - assert: + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: crossplane + namespace: crossplane-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: pkg.crossplane.io/v1 + kind: Provider + metadata: + name: provider-aws-sqs + status: + (conditions[?type == 'Healthy'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: pkg.crossplane.io/v1 + kind: Provider + metadata: + name: provider-aws-cloudwatchevents + status: + (conditions[?type == 'Healthy'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: roles.iam.aws.m.upbound.io + status: + (conditions[?type == 'Established'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: podidentityassociations.eks.aws.m.upbound.io + status: + (conditions[?type == 'Established'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: queues.sqs.aws.m.upbound.io + status: + (conditions[?type == 'Established'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: rules.cloudwatchevents.aws.m.upbound.io + status: + (conditions[?type == 'Established'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: targets.cloudwatchevents.aws.m.upbound.io + status: + (conditions[?type == 'Established'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: crossplane-provider-config + namespace: flux-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: iam.aws.m.upbound.io/v1beta1 + kind: Role + metadata: + name: karpenter + namespace: karpenter + annotations: + crossplane.io/external-name: (join('', [$values.cluster, '-karpenter'])) + spec: + forProvider: + path: (join('', ['/socle/', $values.cluster, '/'])) + permissionsBoundary: arn:aws:iam::000000000000:policy/socle-e2e-catalog-crossplane-boundary + - assert: + resource: + apiVersion: eks.aws.m.upbound.io/v1beta1 + kind: PodIdentityAssociation + metadata: + name: karpenter + namespace: karpenter + spec: + forProvider: + clusterName: ($values.cluster) + namespace: karpenter + serviceAccount: karpenter + - assert: + resource: + apiVersion: sqs.aws.m.upbound.io/v1beta1 + kind: Queue + metadata: + name: karpenter + namespace: karpenter + spec: + forProvider: + name: (join('', [$values.cluster, '-karpenter'])) + messageRetentionSeconds: 300 + sqsManagedSseEnabled: true + (contains(policy, 'events.amazonaws.com')): true + (contains(policy, 'aws:SecureTransport')): true + - assert: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-spot + namespace: karpenter + annotations: + crossplane.io/external-name: (join('', [$values.cluster, '-karpenter-spot'])) + spec: + forProvider: + (contains(eventPattern, 'EC2 Spot Instance Interruption Warning')): true + catch: + - describe: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + namespace: flux-system + name: karpenter + - describe: + apiVersion: pkg.crossplane.io/v1 + kind: Provider + - podLogs: + namespace: flux-system + selector: app.kubernetes.io/name=flux-operator + tail: 40 + - name: 'floci seam: a ClusterProviderConfig with static keys, and the module''s resources pointed at it' + skipDelete: true + try: + - apply: + resource: + apiVersion: v1 + kind: Secret + metadata: + name: floci-credentials + namespace: crossplane-system + stringData: + credentials: | + [default] + aws_access_key_id = test + aws_secret_access_key = test + - apply: + resource: + apiVersion: aws.m.upbound.io/v1beta1 + kind: ClusterProviderConfig + metadata: + name: floci + spec: + credentials: + source: Secret + secretRef: + namespace: crossplane-system + name: floci-credentials + key: credentials + endpoint: + hostnameImmutable: true + services: [iam, eks, sts, sqs, events] + url: + type: Static + static: (join('', ['http://', $values.floci_ip, ':4566'])) + skip_credentials_validation: true + skip_metadata_api_check: true + skip_region_validation: true + skip_requesting_account_id: true + - patch: + resource: + apiVersion: iam.aws.m.upbound.io/v1beta1 + kind: Role + metadata: + name: karpenter + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: eks.aws.m.upbound.io/v1beta1 + kind: PodIdentityAssociation + metadata: + name: karpenter + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: sqs.aws.m.upbound.io/v1beta1 + kind: Queue + metadata: + name: karpenter + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-health + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-spot + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-rebalance + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-state + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-reservation + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Target + metadata: + name: karpenter-health + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Target + metadata: + name: karpenter-spot + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Target + metadata: + name: karpenter-rebalance + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Target + metadata: + name: karpenter-state + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + - patch: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Target + metadata: + name: karpenter-reservation + namespace: karpenter + spec: + providerConfigRef: {kind: ClusterProviderConfig, name: floci} + # Read from what Crossplane reports (status.atProvider): AWS's controller + # policy, every EC2 write scoped by this cluster's tag, PassRole on the + # bootstrap nodes' role only and to EC2 only, no instance-profile write. + - name: the Role becomes an IAM role under /socle// with AWS's controller policy, scoped to this cluster + try: + - assert: + resource: + apiVersion: iam.aws.m.upbound.io/v1beta1 + kind: Role + metadata: + name: karpenter + namespace: karpenter + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + atProvider: + arn: (join('', ['arn:aws:iam::000000000000:role/socle/', $values.cluster, '/', $values.cluster, '-karpenter'])) + (contains(assumeRolePolicy || '', 'pods.eks.amazonaws.com')): true + (inlinePolicy[?name == 'controller'] || `[]`): + - (contains(policy || '', 'ec2:RunInstances')): true + (contains(policy || '', join('', ['aws:RequestTag/kubernetes.io/cluster/', $values.cluster]))): true + (contains(policy || '', join('', ['role/', $values.cluster, '-bootstrap-node']))): true + (contains(policy || '', 'ec2.amazonaws.com')): true + (contains(policy || '', join('', [':', $values.cluster, '-karpenter"']))): true + (contains(policy || '', 'iam:CreateInstanceProfile')): false + (contains(policy || '', 'iam:AddRoleToInstanceProfile')): false + (contains(policy || '', '"*:*"')): false + catch: + - describe: + apiVersion: iam.aws.m.upbound.io/v1beta1 + kind: Role + namespace: karpenter + name: karpenter + - podLogs: + namespace: crossplane-system + selector: pkg.crossplane.io/provider=provider-aws-iam + tail: 40 + - name: the interruption queue reaches SQS, and the five rules EventBridge, each targeting the queue + try: + - assert: + resource: + apiVersion: sqs.aws.m.upbound.io/v1beta1 + kind: Queue + metadata: + name: karpenter + namespace: karpenter + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + atProvider: + arn: (join('', ['arn:aws:sqs:eu-west-3:000000000000:', $values.cluster, '-karpenter'])) + - assert: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-health + namespace: karpenter + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-spot + namespace: karpenter + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-rebalance + namespace: karpenter + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-state + namespace: karpenter + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-reservation + namespace: karpenter + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Target + metadata: + name: karpenter-spot + namespace: karpenter + spec: + forProvider: + arn: (join('', ['arn:aws:sqs:eu-west-3:000000000000:', $values.cluster, '-karpenter'])) + rule: (join('', [$values.cluster, '-karpenter-spot'])) + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + catch: + - describe: + apiVersion: sqs.aws.m.upbound.io/v1beta1 + kind: Queue + namespace: karpenter + - describe: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + namespace: karpenter + - describe: + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Target + namespace: karpenter + - podLogs: + namespace: crossplane-system + selector: pkg.crossplane.io/provider=provider-aws-cloudwatchevents + tail: 40 + # floci seam, second half: the association never syncs (floci 2.1.0 has + # no association API), so karpenter-access, which waits for it, is never + # Ready, the aws cloud step never healthy, and the workload step never + # applied: one full minute after the role and the queue turned Ready, + # still no controller. The day it syncs this step fails and goes with + # the seam. + - name: 'floci seam: the association cannot sync, so the workload is withheld' + try: + - assert: + resource: + apiVersion: eks.aws.m.upbound.io/v1beta1 + kind: PodIdentityAssociation + metadata: + name: karpenter + namespace: karpenter + status: + (conditions[?type == 'Synced'] || `[]`): + - status: 'False' + - sleep: + duration: 60s + # The ServiceAccount is the aws step's, there before the chart; the + # check that proves the association usable runs under it only once the + # association is Ready (#97): no Job. + - assert: + resource: + apiVersion: v1 + kind: ServiceAccount + metadata: + name: karpenter + namespace: karpenter + annotations: + helm.sh/resource-policy: keep + - error: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: karpenter-pod-identity + namespace: karpenter + - error: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter-controller + namespace: flux-system + - error: + resource: + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: karpenter-nodes + namespace: flux-system + - assert: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter-access + namespace: flux-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'False' + # Not Ready: Unknown while the step's health check waits, False once it + # times out. + - error: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter + namespace: flux-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + catch: + - describe: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + namespace: flux-system + name: karpenter + - describe: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + namespace: flux-system + name: karpenter-access + # The association's finalizer waits on an API floci does not have: + # released by hand. Then the module off: every managed resource is + # deleted (each holds its finalizer until the cloud's object is gone), the + # namespace with them. Crossplane off last. + - name: off, the role, the queue and the rules are deleted, the namespace goes + try: + - patch: + resource: + apiVersion: eks.aws.m.upbound.io/v1beta1 + kind: PodIdentityAssociation + metadata: + name: karpenter + namespace: karpenter + finalizers: [] + # No idle gate: the workload is withheld, there is no release. + - patch: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + metadata: + name: socle + namespace: flux-system + spec: + defaultValues: + modules: + karpenter: + enabled: false + - error: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: karpenter + - delete: + ref: + apiVersion: aws.m.upbound.io/v1beta1 + kind: ClusterProviderConfig + name: floci + - delete: + ref: + apiVersion: v1 + kind: Secret + name: floci-credentials + namespace: crossplane-system + - patch: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + metadata: + name: socle + namespace: flux-system + spec: + defaultValues: + modules: + crossplane: + enabled: false + permissions_boundary: '' + - error: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: crossplane-system + catch: + - describe: + apiVersion: v1 + kind: Namespace + name: karpenter + - podLogs: + namespace: flux-system + selector: app.kubernetes.io/name=flux-operator + tail: 40 + # Second half. floci seam: no Pod Identity, so the controller takes + # static test keys, floci's endpoint as a pod sees it, through the + # controller's environment, and Crossplane stays off so that the cloud + # step renders nothing to wait for. The client's values set a key the + # socle sets too (the controller's memory request, 256Mi): theirs wins. + - name: 'the controller converges, never on a node it manages, the client''s value beats the socle''s' + timeouts: + assert: 6m + try: + - patch: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + metadata: + name: socle + namespace: flux-system + spec: + defaultValues: + modules: + karpenter: + enabled: true + # The client's own NodePools' shape, from the first + # apply: Flux applies a *Class before the rest and + # waits for it to be Ready, and floci's EC2NodeClass + # never is (no tagged subnet), so a later change of + # pool would wait on it here — not on AWS. + private_pool: + capacity_types: [on-demand] + architectures: [arm64, amd64] + cpu: 8 + memory: 32Gi + consolidate_after: 5m + ami: al2023@latest + public_pool: + capacity_types: [spot] + architectures: [arm64] + cpu: 4 + memory: 16Gi + consolidate_after: 2m + ami: al2023@latest + values: + controller: + env: + - name: AWS_REGION + value: eu-west-3 + - name: AWS_ACCESS_KEY_ID + value: test + - name: AWS_SECRET_ACCESS_KEY + value: test + - name: AWS_ENDPOINT_URL + value: (join('', ['http://', $values.floci_ip, ':4566'])) + resources: + requests: + memory: 320Mi + - assert: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: karpenter + labels: + pod-security.kubernetes.io/enforce: restricted + - assert: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter-controller + namespace: flux-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: karpenter + namespace: karpenter + spec: + install: + crds: CreateReplace + upgrade: + crds: CreateReplace + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: karpenter + namespace: karpenter + spec: + replicas: 2 + template: + metadata: + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '8080' + spec: + serviceAccountName: karpenter + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + (nodeSelectorTerms[0].matchExpressions[?key == 'karpenter.sh/nodepool'] || `[]`): + - operator: DoesNotExist + (containers[?name == 'controller'] || `[]`): + - resources: + requests: + cpu: 100m + memory: 320Mi + (env[?name == 'CLUSTER_NAME'] || `[]`): + - value: ($values.cluster) + (env[?name == 'INTERRUPTION_QUEUE'] || `[]`): + - value: (join('', [$values.cluster, '-karpenter'])) + status: + (conditions[?type == 'Available'] || `[]`): + - status: 'True' + readyReplicas: 2 + - assert: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter + namespace: flux-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + # Crossplane off: aws's cloud step renders no access. + - error: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter-access + namespace: flux-system + # The module's monitoring — rules and dashboards — applied with it, + # the monitoring module being on. + - assert: + resource: + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: karpenter-monitoring + namespace: flux-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + catch: + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + namespace: karpenter + name: karpenter + - events: + namespace: karpenter + - podLogs: + namespace: karpenter + tail: 40 + # The order of valuesFrom, proven whole + # (docs/decisions/socle.md#socle-06-the-clients-values-win): the socle's + # document, then kube.karpenter.values, then the Secret the client names + # in values_secret, merged last; the Secret cleared, the client's 320Mi. + - name: kube.karpenter.values_secret is merged last, over values and the socle + try: + - apply: + resource: + apiVersion: v1 + kind: Secret + metadata: + name: karpenter-e2e-values + namespace: karpenter + labels: + reconcile.fluxcd.io/watch: Enabled + stringData: + values.yaml: | + controller: + resources: + requests: + memory: 352Mi + - patch: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + metadata: + name: socle + namespace: flux-system + spec: + defaultValues: + modules: + karpenter: + values_secret: karpenter-e2e-values + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: karpenter + namespace: karpenter + spec: + template: + spec: + (containers[?name == 'controller'] || `[]`): + - resources: + requests: + memory: 352Mi + status: + (conditions[?type == 'Available'] || `[]`): + - status: 'True' + - patch: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + metadata: + name: socle + namespace: flux-system + spec: + defaultValues: + modules: + karpenter: + values_secret: '' + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: karpenter + namespace: karpenter + spec: + template: + spec: + (containers[?name == 'controller'] || `[]`): + - resources: + requests: + memory: 320Mi + status: + (conditions[?type == 'Available'] || `[]`): + - status: 'True' + - delete: + ref: + apiVersion: v1 + kind: Secret + name: karpenter-e2e-values + namespace: karpenter + catch: + - describe: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + namespace: karpenter + name: karpenter + - podLogs: + namespace: flux-system + selector: app=helm-controller + tail: 40 + # The nodes step: the EC2NodeClass and the NodePool as the inputs say — + # the client's pools, written when the module was turned on — every + # selector on what the foundations tag. The controller reconciles + # the EC2NodeClass against floci's EC2 — its subnet and security-group + # conditions are set, False here: floci's VPC carries no cluster tag — + # so it reads EC2 with the selectors the socle wrote. + - name: the default EC2NodeClass and NodePool, as the inputs say, reconciled by the controller + try: + - assert: + resource: + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + metadata: + name: karpenter-nodes + namespace: flux-system + spec: + deletionPolicy: WaitForTermination + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + metadata: + name: private + spec: + instanceProfile: (join('', [$values.cluster, '-bootstrap-node'])) + amiSelectorTerms: + - alias: al2023@latest + # The job's cluster, socle-e2e-catalog: a key cannot be an + # expression in an assertion, a parenthesised key is one. + (subnetSelectorTerms[0].tags): + kubernetes.io/cluster/socle-e2e-catalog: owned + kubernetes.io/role/internal-elb: '1' + (securityGroupSelectorTerms[0].tags): + kubernetes.io/cluster/socle-e2e-catalog: owned + metadataOptions: + httpTokens: required + httpPutResponseHopLimit: 1 + tags: + owner: platform + environment: dev + status: + (conditions[?type == 'SubnetsReady'] || `[]`): + - reason: SubnetsNotFound + - assert: + resource: + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + metadata: + name: public + spec: + instanceProfile: (join('', [$values.cluster, '-bootstrap-node'])) + (subnetSelectorTerms[0].tags): + kubernetes.io/cluster/socle-e2e-catalog: owned + kubernetes.io/role/elb: '1' + securityGroupSelectorTerms: + - name: (join('', [$values.cluster, '-public-nodes'])) + associatePublicIPAddress: true + (contains(userData, '--node-labels=node.kubernetes.io/exclude-from-external-load-balancers=true')): true + metadataOptions: + httpTokens: required + httpPutResponseHopLimit: 1 + - assert: + resource: + apiVersion: karpenter.sh/v1 + kind: NodePool + metadata: + name: private + spec: + template: + metadata: + labels: + socle.do-now.io/node-pool: private + spec: + (length(taints || `[]`)): 0 + (requirements[?key == 'karpenter.sh/capacity-type'] || `[]`): + - values: [on-demand] + (requirements[?key == 'kubernetes.io/arch'] || `[]`): + - values: [arm64, amd64] + (startupTaints[?key == 'node.cilium.io/agent-not-ready'] || `[]`): + - effect: NoExecute + limits: + cpu: '8' + memory: 32Gi + disruption: + consolidationPolicy: WhenEmptyOrUnderutilized + consolidateAfter: 5m + - assert: + resource: + apiVersion: karpenter.sh/v1 + kind: NodePool + metadata: + name: public + spec: + template: + metadata: + labels: + socle.do-now.io/node-pool: public + spec: + nodeClassRef: + name: public + (requirements[?key == 'karpenter.sh/capacity-type'] || `[]`): + - values: [spot] + (requirements[?key == 'kubernetes.io/arch'] || `[]`): + - values: [arm64] + (taints[?key == 'socle.do-now.io/node-pool'] || `[]`): + - value: public + effect: NoSchedule + limits: + cpu: '4' + memory: 16Gi + disruption: + consolidateAfter: 2m + catch: + - describe: + apiVersion: kustomize.toolkit.fluxcd.io/v1 + kind: Kustomization + namespace: flux-system + name: karpenter-nodes + - describe: + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + - describe: + apiVersion: karpenter.sh/v1 + kind: NodePool + - podLogs: + namespace: karpenter + tail: 40 + # The uninstall order (KARPENTER-04): the NodePool and the EC2NodeClass go + # first, their finalizers released by the controller still running — + # with it gone first, the EC2NodeClass would hold forever — then the + # controller and the namespace. The CRDs stay, for whatever NodePools a + # client wrote. Behind the idle gate: no release in flight. + - name: enabled = false removes the nodes' objects before the controller, the CRDs stay + try: + - assert: + resource: + apiVersion: helm.toolkit.fluxcd.io/v2 + kind: HelmRelease + metadata: + name: karpenter + namespace: karpenter + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + (conditions[?type == 'Released'] || `[]`): + - status: 'True' + (length(conditions[?type == 'Reconciling'] || `[]`)): 0 + - patch: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSetInputProvider + metadata: + name: socle + namespace: flux-system + spec: + defaultValues: + modules: + karpenter: + enabled: false + values: {} + private_pool: + capacity_types: [spot, on-demand] + architectures: [arm64, amd64] + cpu: 32 + memory: 128Gi + consolidate_after: 1m + ami: al2023@latest + public_pool: + capacity_types: [spot, on-demand] + architectures: [arm64, amd64] + cpu: 32 + memory: 128Gi + consolidate_after: 1m + ami: al2023@latest + - error: + resource: + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + - error: + resource: + apiVersion: karpenter.sh/v1 + kind: NodePool + - error: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: karpenter + - assert: + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: nodepools.karpenter.sh + - assert: + resource: + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ec2nodeclasses.karpenter.k8s.aws + - assert: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter + namespace: flux-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + (length(inventory.entries || `[]`)): 0 + catch: + - describe: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + namespace: flux-system + name: karpenter + - describe: + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + - podLogs: + namespace: flux-system + selector: app.kubernetes.io/name=flux-operator + tail: 40 +--- +# The proof floci cannot give: run by hand on the aws sandbox, against the +# module as the sandbox's root turned it on (docs/catalog/karpenter.md, +# Measured). The private pool: a Deployment that fits nowhere on the +# bootstrap group gets a node — launched, registered, Cilium's startup taint +# gone (its agent runs there), the pods Running — and, scaled to zero, the +# node is consolidated away. The public pool: a Job that tolerates it gets a +# node in a public subnet, with a public IP, labelled by its kubelet out of +# every load balancer, and reaches the internet from it — the Job completes +# only if the pod's egress does (AWS-23). Selected with platform=eks; the +# floci CI never selects it. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: karpenter-module-eks + labels: + phase: module + cloud: aws + platform: eks +spec: + description: >- + A Deployment gets a private node with Cilium on it, consolidated away at + zero; a Job gets a public node and reaches the internet from it. + timeouts: + apply: 30s + assert: 10m + error: 10m + delete: 2m + cleanup: 5m + exec: 30s + steps: + - name: the module is Ready, both EC2NodeClasses and NodePools too + try: + - assert: + resource: + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter + namespace: flux-system + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + metadata: + name: private + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + metadata: + name: public + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: karpenter.sh/v1 + kind: NodePool + metadata: + name: private + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: karpenter.sh/v1 + kind: NodePool + metadata: + name: public + status: + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + catch: + - describe: + apiVersion: karpenter.k8s.aws/v1 + kind: EC2NodeClass + - podLogs: + namespace: karpenter + tail: 60 + - name: a Deployment that needs a Karpenter node gets a private one, with Cilium on it + try: + - apply: + resource: + apiVersion: v1 + kind: Namespace + metadata: + name: karpenter-e2e + labels: + pod-security.kubernetes.io/enforce: restricted + - apply: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: inflate + namespace: karpenter-e2e + spec: + replicas: 2 + selector: + matchLabels: + app: inflate + template: + metadata: + labels: + app: inflate + spec: + # A Karpenter node, never the bootstrap group. + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: karpenter.sh/nodepool + operator: In + values: [private] + terminationGracePeriodSeconds: 0 + securityContext: + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: pause + image: registry.k8s.io/pause:3.10 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + resources: + requests: + cpu: '1' + memory: 256Mi + ephemeral-storage: 64Mi + - assert: + resource: + apiVersion: karpenter.sh/v1 + kind: NodeClaim + metadata: + labels: + karpenter.sh/nodepool: private + status: + (conditions[?type == 'Launched'] || `[]`): + - status: 'True' + (conditions[?type == 'Registered'] || `[]`): + - status: 'True' + (conditions[?type == 'Initialized'] || `[]`): + - status: 'True' + # Cilium's agent removes its startup taint once it runs on the node: + # the node Ready and untainted is Cilium Ready there. + - assert: + resource: + apiVersion: v1 + kind: Node + metadata: + labels: + karpenter.sh/nodepool: private + socle.do-now.io/node-pool: private + spec: + (length(taints[?key == 'node.cilium.io/agent-not-ready'] || `[]`)): 0 + status: + (length(addresses[?type == 'ExternalIP'] || `[]`)): 0 + (conditions[?type == 'Ready'] || `[]`): + - status: 'True' + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: inflate + namespace: karpenter-e2e + status: + readyReplicas: 2 + catch: + - describe: + apiVersion: karpenter.sh/v1 + kind: NodeClaim + - events: + namespace: karpenter-e2e + - podLogs: + namespace: karpenter + tail: 60 + - name: a Job that tolerates the public pool gets a public node, out of every load balancer, and reaches the internet + try: + - apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: egress + namespace: karpenter-e2e + spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + nodeSelector: + socle.do-now.io/node-pool: public + tolerations: + - key: socle.do-now.io/node-pool + operator: Equal + value: public + effect: NoSchedule + securityContext: + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: wget + image: public.ecr.aws/docker/library/busybox:1.37 + # Completes only if the pod reaches the internet. + command: [wget, -T, '10', -q, -O, /dev/null, https://checkip.amazonaws.com] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + resources: + requests: + cpu: '1' + memory: 64Mi + ephemeral-storage: 16Mi + - assert: + resource: + apiVersion: v1 + kind: Node + metadata: + labels: + karpenter.sh/nodepool: public + socle.do-now.io/node-pool: public + node.kubernetes.io/exclude-from-external-load-balancers: 'true' + spec: + (taints[?key == 'socle.do-now.io/node-pool'] || `[]`): + - value: public + status: + (length(addresses[?type == 'ExternalIP'] || `[]`)): 1 + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: egress + namespace: karpenter-e2e + status: + succeeded: 1 + catch: + - describe: + apiVersion: karpenter.sh/v1 + kind: NodeClaim + - podLogs: + namespace: karpenter-e2e + selector: job-name=egress + tail: 20 + - name: scaled to zero, the nodes are consolidated away + try: + - patch: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: inflate + namespace: karpenter-e2e + spec: + replicas: 0 + - delete: + ref: + apiVersion: batch/v1 + kind: Job + name: egress + namespace: karpenter-e2e + - error: + resource: + apiVersion: karpenter.sh/v1 + kind: NodeClaim + - error: + resource: + apiVersion: v1 + kind: Node + metadata: + labels: + karpenter.sh/registered: 'true' + catch: + - describe: + apiVersion: karpenter.sh/v1 + kind: NodeClaim + - podLogs: + namespace: karpenter + tail: 60 diff --git a/oci/clusters/aws/crossplane/patch.yaml b/oci/clusters/aws/crossplane/patch.yaml index 81429c31..4220f729 100644 --- a/oci/clusters/aws/crossplane/patch.yaml +++ b/oci/clusters/aws/crossplane/patch.yaml @@ -18,8 +18,8 @@ name: providers resourcesTemplate: | # IAM for the modules' roles, EKS for their Pod Identity - # associations: what every module's AWS access is made of; S3 for - # the buckets a module owns. One + # associations: what every module's AWS access is made of; S3, SQS + # and EventBridge for the buckets, queues and rules a module owns. One # ServiceAccount name for every AWS provider pod, so the foundations # can write its Pod Identity association before the cluster has a # node: Crossplane otherwise names it after the ProviderRevision, a @@ -122,6 +122,33 @@ package: xpkg.crossplane.io/crossplane-contrib/provider-aws-s3:v2.8.1 runtimeConfigRef: name: provider-aws + --- + # SQS and EventBridge for the queue and the rules a module owns: the + # karpenter module's interruption queue and what feeds it + # (docs/catalog/karpenter.md). Crossplane's own identity may manage + # queues and rules under the cluster's name prefix only + # (opentofu/aws/iam.tf). + apiVersion: pkg.crossplane.io/v1 + kind: Provider + metadata: + name: provider-aws-sqs + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.crossplane.enabled >>enabled<< else >>disabled<< end >> + spec: + package: xpkg.crossplane.io/crossplane-contrib/provider-aws-sqs:v2.8.1 + runtimeConfigRef: + name: provider-aws + --- + apiVersion: pkg.crossplane.io/v1 + kind: Provider + metadata: + name: provider-aws-cloudwatchevents + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.crossplane.enabled >>enabled<< else >>disabled<< end >> + spec: + package: xpkg.crossplane.io/crossplane-contrib/provider-aws-cloudwatchevents:v2.8.1 + runtimeConfigRef: + name: provider-aws - op: add path: /spec/steps/- value: @@ -156,6 +183,16 @@ name: provider-aws-s3 ready: true readyExpr: status.conditions.exists(c, c.type == 'Healthy' && c.status == 'True') + - apiVersion: pkg.crossplane.io/v1 + kind: Provider + name: provider-aws-sqs + ready: true + readyExpr: status.conditions.exists(c, c.type == 'Healthy' && c.status == 'True') + - apiVersion: pkg.crossplane.io/v1 + kind: Provider + name: provider-aws-cloudwatchevents + ready: true + readyExpr: status.conditions.exists(c, c.type == 'Healthy' && c.status == 'True') wait: true resources: # The provider's credentials: EKS Pod Identity, through the diff --git a/oci/clusters/aws/karpenter/patch.yaml b/oci/clusters/aws/karpenter/patch.yaml new file mode 100644 index 00000000..5deab7bf --- /dev/null +++ b/oci/clusters/aws/karpenter/patch.yaml @@ -0,0 +1,453 @@ +# The karpenter module on aws: its own AWS access, a step named cloud before +# the workload (docs/catalog/karpenter.md, docs/architecture/module-iam.md). +# The controller's IAM role and its Pod Identity association; the +# interruption queue and the EventBridge rules that fill it; the controller's +# ServiceAccount, the association's subject, and the check that a pod under +# it is given the role. The plan refuses the module with Crossplane off. +- op: add + path: /spec/dependsOn + value: + # The Role, Queue and Rule kinds come from Crossplane's providers. + - apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + name: crossplane + namespace: flux-system + ready: true +- op: test + path: /spec/steps/1/name + value: workload +- op: add + path: /spec/steps/1 + value: + name: cloud + resourcesTemplate: | + # The subject of the Pod Identity association, the socle's on aws rather + # than the chart's (its values below): the access check runs under it + # before the chart is installed. Kept whatever Crossplane says, so turning + # Crossplane on or off never hands it between owners; resource-policy + # keep, so the one upgrade that hands it over from the chart does not see + # Helm delete it (docs/architecture/module-iam.md). + apiVersion: v1 + kind: ServiceAccount + metadata: + name: karpenter + namespace: karpenter + annotations: + helm.sh/resource-policy: keep + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + --- + << if inputs.modules.crossplane.enabled >> + << $cluster := inputs.cluster.name ->> + << $region := inputs.cluster.region ->> + << $account := inputs.cluster.accountId ->> + << $queue := printf "%s-karpenter" $cluster ->> + << $queueArn := printf "arn:aws:sqs:%s:%s:%s" $region $account $queue ->> + # The controller's own AWS access, docs/architecture/module-iam.md. Under + # /socle// and carrying Crossplane's boundary when the root wired + # one: Crossplane's own policy refuses any other role. Inline policy only. + # + # AWS's own controller policy for Karpenter 1.15.0 (the getting-started + # CloudFormation template: NodeLifecycle, IAMIntegration, + # EKSIntegration, Interruption, ResourceDiscovery), statement for + # statement, with what the socle does not use left out + # (docs/decisions/karpenter.md, KARPENTER-01): + # - no instance-profile creation, tagging or deletion: the EC2NodeClass + # names the profile the foundations made, so Karpenter writes none, + # and only reads profiles; + # - PassRole on the bootstrap nodes' role only, to EC2 only; + # - no zonal-shift read: the setting stays off. + # Every EC2 write is scoped by the tags Karpenter stamps on what it + # launches — this cluster's, and a NodePool's — so it can only launch, + # tag and terminate its own. The boundary must allow ec2, ssm, + # pricing, sqs and eks (the client's tfvars); its one IAM exception, + # the node role, exists only with ec2 there (opentofu/aws/iam.tf). + apiVersion: iam.aws.m.upbound.io/v1beta1 + kind: Role + metadata: + name: karpenter + namespace: karpenter + annotations: + # IAM names are account-wide and two clusters may share an account. + crossplane.io/external-name: << $cluster >>-karpenter + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + spec: + forProvider: + path: /socle/<< $cluster >>/ + << if inputs.modules.crossplane.permissions_boundary >> + permissionsBoundary: << inputs.modules.crossplane.permissions_boundary | quote >> + << end >> + assumeRolePolicy: | + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "pods.eks.amazonaws.com"}, + "Action": ["sts:AssumeRole", "sts:TagSession"] + }] + } + inlinePolicy: + - name: controller + policy: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowScopedEC2InstanceAccessActions", + "Effect": "Allow", + "Resource": [ + "arn:aws:ec2:<< $region >>::image/*", + "arn:aws:ec2:<< $region >>::snapshot/*", + "arn:aws:ec2:<< $region >>:*:security-group/*", + "arn:aws:ec2:<< $region >>:*:subnet/*", + "arn:aws:ec2:<< $region >>:*:capacity-reservation/*", + "arn:aws:ec2:<< $region >>:*:placement-group/*" + ], + "Action": ["ec2:RunInstances", "ec2:CreateFleet"] + }, + { + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:aws:ec2:<< $region >>:*:launch-template/*", + "Action": ["ec2:RunInstances", "ec2:CreateFleet"], + "Condition": { + "StringEquals": {"aws:ResourceTag/kubernetes.io/cluster/<< $cluster >>": "owned"}, + "StringLike": {"aws:ResourceTag/karpenter.sh/nodepool": "*"} + } + }, + { + "Sid": "AllowScopedEC2InstanceActionsWithTags", + "Effect": "Allow", + "Resource": [ + "arn:aws:ec2:<< $region >>:*:fleet/*", + "arn:aws:ec2:<< $region >>:*:instance/*", + "arn:aws:ec2:<< $region >>:*:volume/*", + "arn:aws:ec2:<< $region >>:*:network-interface/*", + "arn:aws:ec2:<< $region >>:*:launch-template/*", + "arn:aws:ec2:<< $region >>:*:spot-instances-request/*" + ], + "Action": ["ec2:RunInstances", "ec2:CreateFleet", "ec2:CreateLaunchTemplate"], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/<< $cluster >>": "owned", + "aws:RequestTag/eks:eks-cluster-name": "<< $cluster >>" + }, + "StringLike": {"aws:RequestTag/karpenter.sh/nodepool": "*"} + } + }, + { + "Sid": "AllowScopedResourceCreationTagging", + "Effect": "Allow", + "Resource": [ + "arn:aws:ec2:<< $region >>:*:fleet/*", + "arn:aws:ec2:<< $region >>:*:instance/*", + "arn:aws:ec2:<< $region >>:*:volume/*", + "arn:aws:ec2:<< $region >>:*:network-interface/*", + "arn:aws:ec2:<< $region >>:*:launch-template/*", + "arn:aws:ec2:<< $region >>:*:spot-instances-request/*" + ], + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/<< $cluster >>": "owned", + "aws:RequestTag/eks:eks-cluster-name": "<< $cluster >>", + "ec2:CreateAction": ["RunInstances", "CreateFleet", "CreateLaunchTemplate"] + }, + "StringLike": {"aws:RequestTag/karpenter.sh/nodepool": "*"} + } + }, + { + "Sid": "AllowScopedResourceTagging", + "Effect": "Allow", + "Resource": "arn:aws:ec2:<< $region >>:*:instance/*", + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": {"aws:ResourceTag/kubernetes.io/cluster/<< $cluster >>": "owned"}, + "StringLike": {"aws:ResourceTag/karpenter.sh/nodepool": "*"}, + "StringEqualsIfExists": {"aws:RequestTag/eks:eks-cluster-name": "<< $cluster >>"}, + "ForAllValues:StringEquals": {"aws:TagKeys": ["eks:eks-cluster-name", "karpenter.sh/nodeclaim", "Name"]} + } + }, + { + "Sid": "AllowScopedDeletion", + "Effect": "Allow", + "Resource": [ + "arn:aws:ec2:<< $region >>:*:instance/*", + "arn:aws:ec2:<< $region >>:*:launch-template/*" + ], + "Action": ["ec2:TerminateInstances", "ec2:RebootInstances", "ec2:DeleteLaunchTemplate"], + "Condition": { + "StringEquals": {"aws:ResourceTag/kubernetes.io/cluster/<< $cluster >>": "owned"}, + "StringLike": {"aws:ResourceTag/karpenter.sh/nodepool": "*"} + } + }, + { + "Sid": "AllowPassingInstanceRole", + "Effect": "Allow", + "Resource": "arn:aws:iam::<< $account >>:role/<< $cluster >>-bootstrap-node", + "Action": "iam:PassRole", + "Condition": {"StringEquals": {"iam:PassedToService": "ec2.amazonaws.com"}} + }, + { + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "arn:aws:iam::<< $account >>:instance-profile/*", + "Action": ["iam:GetInstanceProfile", "iam:ListInstanceProfiles"] + }, + { + "Sid": "AllowAPIServerEndpointDiscovery", + "Effect": "Allow", + "Resource": "arn:aws:eks:<< $region >>:<< $account >>:cluster/<< $cluster >>", + "Action": "eks:DescribeCluster" + }, + { + "Sid": "AllowInterruptionQueueActions", + "Effect": "Allow", + "Resource": "<< $queueArn >>", + "Action": ["sqs:DeleteMessage", "sqs:GetQueueUrl", "sqs:ReceiveMessage"] + }, + { + "Sid": "AllowRegionalReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "ec2:DescribeCapacityReservations", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribePlacementGroups", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ], + "Condition": {"StringEquals": {"aws:RequestedRegion": "<< $region >>"}} + }, + { + "Sid": "AllowSSMReadActions", + "Effect": "Allow", + "Resource": "arn:aws:ssm:<< $region >>::parameter/aws/service/*", + "Action": "ssm:GetParameter" + }, + { + "Sid": "AllowPricingReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "pricing:GetProducts" + } + ] + } + --- + apiVersion: eks.aws.m.upbound.io/v1beta1 + kind: PodIdentityAssociation + metadata: + name: karpenter + namespace: karpenter + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + spec: + forProvider: + # Required: the socle's ClusterProviderConfig names no region. + region: << $region >> + clusterName: << $cluster >> + namespace: karpenter + serviceAccount: karpenter + roleArnRef: + name: karpenter + --- + # The interruption queue: what EventBridge delivers and Karpenter reads, + # to drain a node ahead of its Spot reclaim, its rebalance, its + # scheduled maintenance, or after it stopped. Five minutes of retention + # (a notice older than that is moot), encrypted at rest by SQS, HTTPS + # only, and only EventBridge may send — AWS's own queue for Karpenter. + apiVersion: sqs.aws.m.upbound.io/v1beta1 + kind: Queue + metadata: + name: karpenter + namespace: karpenter + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + spec: + forProvider: + region: << $region >> + name: << $queue >> + messageRetentionSeconds: 300 + sqsManagedSseEnabled: true + policy: | + { + "Version": "2012-10-17", + "Id": "EC2InterruptionPolicy", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": ["events.amazonaws.com", "sqs.amazonaws.com"]}, + "Action": "sqs:SendMessage", + "Resource": "<< $queueArn >>" + }, + { + "Sid": "DenyHTTP", + "Effect": "Deny", + "Principal": "*", + "Action": "sqs:*", + "Resource": "<< $queueArn >>", + "Condition": {"Bool": {"aws:SecureTransport": "false"}} + } + ] + } + << $rules := list + (list "health" "aws.health" "AWS Health Event") + (list "spot" "aws.ec2" "EC2 Spot Instance Interruption Warning") + (list "rebalance" "aws.ec2" "EC2 Instance Rebalance Recommendation") + (list "state" "aws.ec2" "EC2 Instance State-change Notification") + (list "reservation" "aws.ec2" "EC2 Capacity Reservation Instance Interruption Warning") ->> + << range $rules ->> + << $name := index . 0 ->> + --- + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Rule + metadata: + name: karpenter-<< $name >> + namespace: karpenter + annotations: + crossplane.io/external-name: << $cluster >>-karpenter-<< $name >> + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + spec: + forProvider: + region: << $region >> + eventBusName: default + eventPattern: << dict "source" (list (index . 1)) "detail-type" (list (index . 2)) | toJson | quote >> + --- + apiVersion: cloudwatchevents.aws.m.upbound.io/v1beta1 + kind: Target + metadata: + name: karpenter-<< $name >> + namespace: karpenter + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + spec: + forProvider: + region: << $region >> + eventBusName: default + ruleRef: + name: karpenter-<< $name >> + targetId: karpenter-queue + arn: << $queueArn >> + << end >> + --- + # Healthy only once the role, the association and the queue are Ready, + # and a pod under the module's ServiceAccount has been given the + # association's credentials, so the workload step starts after them: + # EKS Pod Identity injects the credentials at pod admission, the + # webhook learns an association seconds after it is Ready, and a pod + # admitted in between never gets them until it is recreated (#97); + # Karpenter reads its queue at start. A dependsOn with a readyExpr, + # because a managed resource not yet created carries no Ready condition + # at all, which the step's own health check reads as healthy (measured + # on floci). + apiVersion: fluxcd.controlplane.io/v1 + kind: ResourceSet + metadata: + name: karpenter-access + namespace: flux-system + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + spec: + # Ready once the Job below is Complete. + wait: true + dependsOn: + - apiVersion: iam.aws.m.upbound.io/v1beta1 + kind: Role + name: karpenter + namespace: karpenter + ready: true + readyExpr: status.conditions.exists(c, c.type == 'Ready' && c.status == 'True') + - apiVersion: eks.aws.m.upbound.io/v1beta1 + kind: PodIdentityAssociation + name: karpenter + namespace: karpenter + ready: true + readyExpr: status.conditions.exists(c, c.type == 'Ready' && c.status == 'True') + - apiVersion: sqs.aws.m.upbound.io/v1beta1 + kind: Queue + name: karpenter + namespace: karpenter + ready: true + readyExpr: status.conditions.exists(c, c.type == 'Ready' && c.status == 'True') + resources: + # The association, proven usable by the thing itself: a pod under the + # module's ServiceAccount that asks STS who it is, with only the Pod + # Identity credentials to answer (the instance metadata off). + # GetCallerIdentity needs no permission and an explicit deny does not + # stop it (API_GetCallerIdentity), so the boundary's deny on sts + # leaves it whole. Without the injection it exits non-zero, and the + # Job's next pod is a new admission; the wait is exactly as long as + # the propagation. One command, no shell; force, because a Job's + # template is immutable (docs/architecture/module-iam.md). + - apiVersion: batch/v1 + kind: Job + metadata: + name: karpenter-pod-identity + namespace: karpenter + annotations: + fluxcd.controlplane.io/force: enabled + fluxcd.controlplane.io/recreateOnFailure: enabled + spec: + backoffLimit: 6 + template: + spec: + serviceAccountName: karpenter + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: + type: RuntimeDefault + containers: + - name: sts + # 2.38.1 + image: public.ecr.aws/aws-cli/aws-cli@sha256:733d5be1015825e4038c6016f3fa9954868ddee1f611b73be795a9cc2e706bfd + args: ["sts", "get-caller-identity"] + # The one way round the injection: a Secret karpenter-aws + # the client creates, with static keys and, for an + # AWS-compatible endpoint, AWS_ENDPOINT_URL. Absent, + # nothing is set. + envFrom: + - secretRef: + name: karpenter-aws + optional: true + env: + - name: AWS_REGION + value: << $region >> + - name: AWS_EC2_METADATA_DISABLED + value: "true" + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 64Mi + ephemeral-storage: 16Mi + limits: + memory: 128Mi + --- + << end >> + apiVersion: v1 + kind: ConfigMap + metadata: + name: karpenter-cloud-values + namespace: karpenter + labels: + reconcile.fluxcd.io/watch: Enabled + annotations: + fluxcd.controlplane.io/reconcile: << if inputs.modules.karpenter.enabled >>enabled<< else >>disabled<< end >> + data: + values.yaml: | + # The ServiceAccount of the cloud step. + serviceAccount: + create: false + name: karpenter diff --git a/oci/clusters/aws/kustomization.yaml b/oci/clusters/aws/kustomization.yaml index 608c40e2..10e6832b 100644 --- a/oci/clusters/aws/kustomization.yaml +++ b/oci/clusters/aws/kustomization.yaml @@ -24,6 +24,7 @@ resources: - ../../catalog/velero/resourceset.yaml - ../../catalog/hello/resourceset.yaml - ../../catalog/metrics-server/resourceset.yaml + - ../../catalog/karpenter/resourceset.yaml patches: - path: gateway-api/patch.yaml target: @@ -57,3 +58,7 @@ patches: target: kind: ResourceSet name: hello + - path: karpenter/patch.yaml + target: + kind: ResourceSet + name: karpenter diff --git a/opentofu/aws/README.md b/opentofu/aws/README.md index 7649828c..787bba4d 100644 --- a/opentofu/aws/README.md +++ b/opentofu/aws/README.md @@ -74,6 +74,7 @@ No modules. | [aws_eks_node_group.bootstrap](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_eks_pod_identity_association.crossplane](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_pod_identity_association) | resource | | [aws_flow_log.socle](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/flow_log) | resource | +| [aws_iam_instance_profile.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_policy.crossplane_boundary](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.crossplane](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | @@ -95,7 +96,11 @@ No modules. | [aws_route_table.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table) | resource | | [aws_route_table_association.private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource | | [aws_route_table_association.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource | +| [aws_security_group.public_nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.vpc_endpoints](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.cluster_from_public_nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.public_nodes_from_the_vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.public_nodes_to_anywhere](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_subnet.private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | | [aws_subnet.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | | [aws_vpc.socle](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc) | resource | @@ -125,7 +130,7 @@ No modules. | [cluster\_log\_types](#input\_cluster\_log\_types) | Control plane log types shipped to CloudWatch Logs. None by default:
ingestion is billed by the gigabyte, and the audit stream records every
request to the API server — one controller writing in a loop made it
about $40 a day on an idle cluster (#80). A client whose audit expects
the API server's record of who did what (ISO 27001 A.8.15, SOC 2 CC7)
turns on at least audit and authenticator; the log group, its retention
and its key exist either way, so the first line is already covered. | `list(string)` | `[]` | no | | [create\_nat\_gateway](#input\_create\_nat\_gateway) | Create one NAT Gateway per AZ. One per AZ, never a single shared one, to
avoid cross-AZ data transfer charges — not a toggle for disabling NAT
outright, which the private-subnet decision above rules out. Exists
only for the create\_vpc = false case, where the consumer's existing VPC
already manages its own NAT. | `bool` | `true` | no | | [create\_vpc](#input\_create\_vpc) | Create the VPC, or attach to one the consumer already manages. | `bool` | `true` | no | -| [crossplane](#input\_crossplane) | Give the catalog's crossplane module its AWS identity — the one the socle
cannot make for itself: an IAM role, bound through EKS Pod Identity to
crossplane-system/provider-aws, allowed to create roles under
/socle// only, and only carrying the permissions boundary
this module writes. allowed\_services is that boundary: the AWS services
(IAM action prefixes, such as route53 or s3) any module's role may be
granted. Empty grants nothing; iam, sts, organizations, account, sso and
identitystore are refused. Which resources of those services a module
reaches is its own role's policy. Null, the default, creates nothing. The
client root passes the boundary's ARN into kube.crossplane. | 
object({
    allowed_services = optional(list(string), [])
  })
| `null` | no | +| [crossplane](#input\_crossplane) | Give the catalog's crossplane module its AWS identity — the one the socle
cannot make for itself: an IAM role, bound through EKS Pod Identity to
crossplane-system/provider-aws, allowed to create roles under
/socle// only, and only carrying the permissions boundary
this module writes. allowed\_services is that boundary: the AWS services
(IAM action prefixes, such as route53 or s3) any module's role may be
granted. Empty grants nothing; iam, sts, organizations, account, sso and
identitystore are refused. With ec2 in the list, and only then, the
boundary also lets a role pass the bootstrap nodes' role to EC2 and read
instance profiles — what Karpenter's controller needs to launch nodes —
and nothing else of IAM. Which resources of those services a module
reaches is its own role's policy. Null, the default, creates nothing. The
client root passes the boundary's ARN into kube.crossplane. | 
object({
    allowed_services = optional(list(string), [])
  })
| `null` | no | | [force\_update\_version](#input\_force\_update\_version) | Force the control plane version update even if Upgrade Insights reports
blocking findings. Default false: Upgrade Insights is a mandatory
pre-check, never sufficient alone (it only sees the client's own
removed-API usage, over a rolling 30-day audit-log window that both
misses infrequent calls and over-reports fixed ones) — but AWS's own
blocking of `update-cluster-version` on ERROR findings is currently
rolled back, so this module does not assume AWS enforces the check
either. | `bool` | `false` | no | | [gateway\_certificate](#input\_gateway\_certificate) | The ACM certificate the socle's two Gateways terminate TLS with, at the
load balancer: `domain` and `*.domain`, validated by DNS in the public
Route 53 zone named `domain` — found by name, no zone ID to copy. `zone`
names that zone instead when `domain` is a subdomain of it, such as
domain = "sbx.acme.example" in zone = "acme.example". Every route
published through a Gateway — ArgoCD's included — is then
`.`. Null, the default, creates nothing, and the bootstrap
module creates no Gateway: the socle never serves a route in clear text. | 
object({
    domain = string
    zone   = optional(string)
  })
| `null` | no | | [log\_retention\_days](#input\_log\_retention\_days) | Retention for the log groups this module creates — the control plane's and the VPC flow logs'. Set explicitly because a log group left to AWS never expires. | `number` | `90` | no | @@ -133,7 +138,7 @@ No modules. | [public\_subnet\_ids](#input\_public\_subnet\_ids) | Existing public subnet IDs, one per AZ in availability\_zones. Required when create\_vpc is false — same reasoning as private\_subnet\_ids. | `list(string)` | `[]` | no | | [secrets\_encryption\_enabled](#input\_secrets\_encryption\_enabled) | Envelope-encrypt Kubernetes Secrets via KMS. On by default: essentially free (~$1/month per key, negligible per-request cost), and standard on Kubernetes 1.28+ already. | `bool` | `true` | no | | [secrets\_encryption\_kms\_key\_arn](#input\_secrets\_encryption\_kms\_key\_arn) | Existing KMS key for Secrets envelope encryption. Leave unset and the module creates its own key. Ignored when secrets\_encryption\_enabled is false. | `string` | `null` | no | -| [vpc\_cidr](#input\_vpc\_cidr) | CIDR for the VPC when this module creates it. Arbitrary default (not a research decision) — a /16 large enough for any socle estate. | `string` | `"10.0.0.0/16"` | no | +| [vpc\_cidr](#input\_vpc\_cidr) | CIDR for the VPC when this module creates it. Arbitrary default (not a research decision) — a /16 large enough for any socle estate. With create\_vpc = false, set it to the existing VPC's primary CIDR: pods take their addresses from it, and Cilium masquerades only what leaves it (vpc\_cidr output). | `string` | `"10.0.0.0/16"` | no | | [vpc\_flow\_logs\_enabled](#input\_vpc\_flow\_logs\_enabled) | Record accepted and rejected traffic on the VPC this module creates. On by default, aggregated over ten-minute windows to keep the volume down: it is the only record of who talked to whom, and the segmentation this VPC is built around is unauditable without it. | `bool` | `true` | no | | [vpc\_id](#input\_vpc\_id) | Existing VPC to attach to. Required when create\_vpc is false, and incoherent to set when create\_vpc is true — this module cannot both create a VPC and attach to a different one. | `string` | `null` | no | @@ -150,11 +155,14 @@ No modules. | [crossplane\_role\_arn](#output\_crossplane\_role\_arn) | ARN of the IAM role the catalog's crossplane module's AWS providers run as, through Pod Identity. Null when crossplane is not set. | | [gateway\_certificate\_arn](#output\_gateway\_certificate\_arn) | ARN of the issued ACM certificate the socle's Gateways terminate TLS with — what the bootstrap module's gateway\_certificate\_arn takes, and what the client root passes for you. Null when gateway\_certificate is not set. | | [helm\_kubernetes](#output\_helm\_kubernetes) | Drop-in value for the helm provider's kubernetes attribute, so a root configures it in one line. Carries no credential: the exec plugin obtains a short-lived token from the caller's ambient AWS credentials at call time, exactly as the aws provider itself authenticates. | +| [node\_instance\_profile](#output\_node\_instance\_profile) | Name of the instance profile carrying the bootstrap nodes' role, the identity of every node of the cluster: the bootstrap group's, and those the catalog's Karpenter launches (its EC2NodeClass names it). | | [oidc\_issuer\_url](#output\_oidc\_issuer\_url) | The cluster's OIDC issuer. Checklist requirement, not this module's identity mechanism — Pod Identity is, IRSA is absent, and nothing here provisions an OIDC trust relationship against it. Null on an emulated cluster that reports no identity (floci), so an apply there still converges. | | [private\_subnet\_ids](#output\_private\_subnet\_ids) | Private subnet IDs, one per AZ. | +| [public\_nodes\_security\_group](#output\_public\_nodes\_security\_group) | Name of the security group of Karpenter's public nodes, which its public EC2NodeClass selects: from the VPC, nothing from the internet. | | [public\_subnet\_ids](#output\_public\_subnet\_ids) | Public subnet IDs, one per AZ. | | [region](#output\_region) | Region the cluster and VPC were created in, as resolved from the provider. | | [service\_cidr](#output\_service\_cidr) | The Kubernetes service range EKS chose for this cluster (172.20.0.0/16 or 10.100.0.0/16, by VPC CIDR). The bootstrap module gives CoreDNS its .10 address, the one every node's kubelet is told to use. Null on an emulated cluster that reports none (floci). | | [tags](#output\_tags) | The standard tag set applied to every billable resource this module creates. | +| [vpc\_cidr](#output\_vpc\_cidr) | The VPC's CIDR, vpc\_cidr — with create\_vpc = false, as the client declared it. Pods take their addresses from it (Cilium's ENI IPAM): Cilium masquerades only what leaves it. | | [vpc\_id](#output\_vpc\_id) | ID of the VPC the cluster is attached to, whether this module created it or not. | diff --git a/opentofu/aws/iam.tf b/opentofu/aws/iam.tf index 6d2d007b..18b4c386 100644 --- a/opentofu/aws/iam.tf +++ b/opentofu/aws/iam.tf @@ -86,7 +86,9 @@ data "aws_iam_policy_document" "cilium_operator" { # - no way to remove or swap that boundary, no managed-policy attachment, no # policy creation, no access to any role outside the path — its own # included, which lives at /; -# - PassRole to EKS Pod Identity only, associations on this cluster only. +# - PassRole to EKS Pod Identity only, associations on this cluster only; +# - buckets, queues and EventBridge rules under the cluster's name prefix +# only, the resources a module owns beside its role. # What it cannot bound: a role's trust policy. IAM has no condition key on # the document, so a compromised Crossplane could create a role, within the # boundary, that another principal may assume. The boundary is the answer — @@ -110,6 +112,20 @@ locals { # other roles, reach the organisation or the account settings. crossplane_denied_services = ["iam", "sts", "organizations", "account", "sso", "identitystore"] + # The one exception to "never iam", and only once the client allows ec2: a + # role that may launch instances may launch them as the socle's node role + # — the bootstrap nodes' (nodes.tf), the only role an instance of this + # cluster ever runs as — and read the instance profiles that carry it. + # Karpenter's controller is that role (docs/catalog/karpenter.md): EC2 + # refuses RunInstances with an instance profile unless the caller may pass + # its role, and Karpenter checks the profile it is given before using it. + # Nothing else of IAM: no other role to pass, no profile to create or to + # change, so no module can make an instance of more than the nodes already + # are. docs/decisions/aws.md, AWS-21. + crossplane_node_role_exception = contains(try(var.crossplane.allowed_services, []), "ec2") + crossplane_node_role_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.cluster_name}-bootstrap-node" + crossplane_instance_profiles = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:instance-profile/*" + # The ceiling of every role Crossplane creates. The deny is always there, # which also keeps an empty allowlist a valid document. crossplane_boundary_policy = jsonencode({ @@ -121,12 +137,36 @@ locals { Action = "${svc}:*" Resource = "*" }], + [for st in [ + { + Sid = "PassTheNodeRoleToEc2Only" + Effect = "Allow" + Action = "iam:PassRole" + Resource = local.crossplane_node_role_arn + Condition = { StringEquals = { "iam:PassedToService" = "ec2.amazonaws.com" } } + }, + { + Sid = "ReadInstanceProfiles" + Effect = "Allow" + Action = ["iam:GetInstanceProfile", "iam:ListInstanceProfiles"] + Resource = local.crossplane_instance_profiles + }, + ] : st if local.crossplane_node_role_exception], [{ Sid = "NeverIdentityNorAccount" Effect = "Deny" - Action = [for svc in local.crossplane_denied_services : "${svc}:*"] + Action = [for svc in local.crossplane_denied_services : "${svc}:*" if !(local.crossplane_node_role_exception && svc == "iam")] Resource = "*" }], + # With the exception, iam is denied on everything but the node role and + # the instance profiles; on those, only the two statements above allow + # anything, so every other IAM action stays refused there too. + [for st in [{ + Sid = "NeverIamBeyondTheNodeRole" + Effect = "Deny" + Action = "iam:*" + NotResource = [local.crossplane_node_role_arn, local.crossplane_instance_profiles] + }] : st if local.crossplane_node_role_exception], ) }) } @@ -280,6 +320,49 @@ resource "aws_iam_role_policy" "crossplane" { ] Resource = "arn:aws:s3:::${var.cluster_name}-*" }, + { + # A module's own queue, under the cluster's prefix: Karpenter's + # interruption queue (docs/catalog/karpenter.md). Create it, read and + # write its attributes — its access policy among them, which is what + # lets EventBridge deliver to it — tag it, delete it. A queue holds + # notices minutes old, nothing to keep: it goes with the module. + Sid = "ManageQueuesUnderTheClusterPrefix" + Effect = "Allow" + Action = [ + "sqs:CreateQueue", + "sqs:DeleteQueue", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:SetQueueAttributes", + "sqs:ListQueueTags", + "sqs:TagQueue", + "sqs:UntagQueue", + ] + Resource = "arn:aws:sqs:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:${var.cluster_name}-*" + }, + { + # A module's own EventBridge rules on the default bus, under the + # cluster's prefix, and their targets: the EC2 and AWS Health events + # Karpenter reads from its queue. A target needs no role to reach a + # queue — the queue's policy admits events.amazonaws.com — so no + # PassRole here. + Sid = "ManageEventRulesUnderTheClusterPrefix" + Effect = "Allow" + Action = [ + "events:PutRule", + "events:DescribeRule", + "events:DeleteRule", + "events:EnableRule", + "events:DisableRule", + "events:PutTargets", + "events:RemoveTargets", + "events:ListTargetsByRule", + "events:ListTagsForResource", + "events:TagResource", + "events:UntagResource", + ] + Resource = "arn:aws:events:${data.aws_region.current.region}:${data.aws_caller_identity.current.account_id}:rule/${var.cluster_name}-*" + }, ] }) } diff --git a/opentofu/aws/nodes.tf b/opentofu/aws/nodes.tf index f8ed641b..1294c0ac 100644 --- a/opentofu/aws/nodes.tf +++ b/opentofu/aws/nodes.tf @@ -78,6 +78,82 @@ resource "aws_iam_role_policy" "node_cilium_operator" { policy = data.aws_iam_policy_document.cilium_operator.json } +# The same role, as an instance profile: what an instance launched outside +# the node group runs as. The catalog's Karpenter names it in its +# EC2NodeClass (docs/catalog/karpenter.md), so its nodes are the bootstrap +# nodes' peers — the same kubelet, ECR and Cilium ENI rights, the access +# entry EKS made for this role, no second identity to keep in step — and it +# never creates a profile of its own. A profile is not a role: the rule that +# no role is made here for a module stands. Named like the role, which no +# other profile in the account can be. +resource "aws_iam_instance_profile" "node" { + name = aws_iam_role.node.name + role = aws_iam_role.node.name + + tags = local.tags +} + +# The security group of the nodes Karpenter launches in the public subnets, +# with a public IP: what a workload costly in network traffic runs on, its +# egress through the Internet Gateway instead of the NAT Gateway +# (docs/catalog/karpenter.md, AWS-23). They do not carry the cluster +# security group: EKS's in-tree load balancer controller opens a Service's +# NodePort to 0.0.0.0/0 in the security group of every node it targets, and +# on a node with a public IP that is the NodePort itself on the internet — +# measured on the sandbox, 2026-10-17. This group takes no rule from it: +# those nodes register already labelled +# node.kubernetes.io/exclude-from-external-load-balancers, so no load +# balancer targets them. It admits the VPC — the control plane's ENIs, every +# node — and nothing from the internet. Named, not tagged for the cluster: +# Karpenter selects it by name, and a cluster tag would make the controller +# take it for the cluster's. +resource "aws_security_group" "public_nodes" { + name = "${var.cluster_name}-public-nodes" + description = "Karpenter public nodes: from the VPC, to anywhere." + vpc_id = local.vpc_id + + tags = merge(local.tags, { Name = "${var.cluster_name}-public-nodes" }) +} + +resource "aws_security_group_rule" "public_nodes_from_the_vpc" { + description = "From the VPC: the control plane, the other nodes, nothing from the internet" + type = "ingress" + from_port = 0 + to_port = 0 + protocol = "-1" + security_group_id = aws_security_group.public_nodes.id + cidr_blocks = [var.vpc_cidr] +} + +# Their reason to exist: the internet, straight through the Internet +# Gateway. AVD-AWS-0104 flags egress to 0.0.0.0/0; the cluster security +# group EKS gives every other node allows the same. +#trivy:ignore:AVD-AWS-0104 +resource "aws_security_group_rule" "public_nodes_to_anywhere" { + description = "To anywhere: the internet through the Internet Gateway, the VPC" + type = "egress" + from_port = 0 + to_port = 0 + protocol = "-1" + security_group_id = aws_security_group.public_nodes.id + cidr_blocks = ["0.0.0.0/0"] +} + +# And the way back: the control plane and the other nodes accept the public +# nodes, on the cluster security group EKS creates and attaches to both. +# An emulated EKS reports none (floci 2.1.0): the rule then lands on the +# public nodes' own group, where the VPC rule above already admits them — +# an EKS cluster always has one. +resource "aws_security_group_rule" "cluster_from_public_nodes" { + description = "From the Karpenter public nodes" + type = "ingress" + from_port = 0 + to_port = 0 + protocol = "-1" + security_group_id = coalesce(aws_eks_cluster.socle.vpc_config[0].cluster_security_group_id, aws_security_group.public_nodes.id) + source_security_group_id = aws_security_group.public_nodes.id +} + # Instance metadata answers only on the node itself (one hop): a pod that is # not hostNetwork cannot borrow the node's role. IMDSv2 only, and an # encrypted root volume. No variable: none of it is a client's call. diff --git a/opentofu/aws/outputs.tf b/opentofu/aws/outputs.tf index f28cf6bf..cbfc0fd9 100644 --- a/opentofu/aws/outputs.tf +++ b/opentofu/aws/outputs.tf @@ -82,6 +82,21 @@ output "bootstrap_node_group" { } } +output "vpc_cidr" { + description = "The VPC's CIDR, vpc_cidr — with create_vpc = false, as the client declared it. Pods take their addresses from it (Cilium's ENI IPAM): Cilium masquerades only what leaves it." + value = var.vpc_cidr +} + +output "public_nodes_security_group" { + description = "Name of the security group of Karpenter's public nodes, which its public EC2NodeClass selects: from the VPC, nothing from the internet." + value = aws_security_group.public_nodes.name +} + +output "node_instance_profile" { + description = "Name of the instance profile carrying the bootstrap nodes' role, the identity of every node of the cluster: the bootstrap group's, and those the catalog's Karpenter launches (its EC2NodeClass names it)." + value = aws_iam_instance_profile.node.name +} + output "crossplane_role_arn" { description = "ARN of the IAM role the catalog's crossplane module's AWS providers run as, through Pod Identity. Null when crossplane is not set." value = one(aws_iam_role.crossplane[*].arn) diff --git a/opentofu/aws/tests/defaults.tftest.hcl b/opentofu/aws/tests/defaults.tftest.hcl index 761e6201..ff13fc3a 100644 --- a/opentofu/aws/tests/defaults.tftest.hcl +++ b/opentofu/aws/tests/defaults.tftest.hcl @@ -408,7 +408,7 @@ run "crossplane_identity_creates_only_bounded_roles" { error_message = "the boundary must always deny identity and account services." } assert { - condition = alltrue([for st in jsondecode(aws_iam_role_policy.crossplane[0].policy).Statement : st.Resource == "arn:aws:iam::000000000000:role/socle/socle-test/*" if !contains(["PodIdentityAssociationsOnThisCluster", "ReadAnyRoleInTheAccount", "ManageBucketsUnderTheClusterPrefix"], st.Sid)]) + condition = alltrue([for st in jsondecode(aws_iam_role_policy.crossplane[0].policy).Statement : st.Resource == "arn:aws:iam::000000000000:role/socle/socle-test/*" if !contains(["PodIdentityAssociationsOnThisCluster", "ReadAnyRoleInTheAccount", "ManageBucketsUnderTheClusterPrefix", "ManageQueuesUnderTheClusterPrefix", "ManageEventRulesUnderTheClusterPrefix"], st.Sid)]) error_message = "every mutating IAM statement of the Crossplane identity must be scoped to roles under /socle//." } assert { @@ -448,6 +448,108 @@ run "crossplane_identity_creates_only_bounded_roles" { } } +run "without_ec2_the_boundary_denies_all_of_iam" { + command = plan + variables { + crossplane = { allowed_services = ["route53", "sqs"] } + } + + assert { + condition = !anytrue([for st in jsondecode(aws_iam_policy.crossplane_boundary[0].policy).Statement : contains(["PassTheNodeRoleToEc2Only", "ReadInstanceProfiles", "NeverIamBeyondTheNodeRole"], st.Sid)]) + error_message = "without ec2 no role can launch an instance, so the node-role exception must not exist." + } +} + +# docs/catalog/karpenter.md: a role that may launch instances may launch them +# as the nodes' role, and read profiles — nothing else of IAM. +run "with_ec2_the_boundary_lets_a_role_pass_the_node_role_to_ec2_and_nothing_else" { + command = plan + variables { + crossplane = { allowed_services = ["ec2", "ssm", "pricing", "sqs", "eks"] } + } + + assert { + condition = one([ + for st in jsondecode(aws_iam_policy.crossplane_boundary[0].policy).Statement : + st.Effect == "Allow" && st.Action == "iam:PassRole" && st.Resource == "arn:aws:iam::000000000000:role/socle-test-bootstrap-node" && st.Condition.StringEquals["iam:PassedToService"] == "ec2.amazonaws.com" + if st.Sid == "PassTheNodeRoleToEc2Only" + ]) + error_message = "the boundary may let a role pass the bootstrap nodes' role, to EC2 only, and no other role." + } + assert { + condition = one([ + for st in jsondecode(aws_iam_policy.crossplane_boundary[0].policy).Statement : + st.Action == ["iam:GetInstanceProfile", "iam:ListInstanceProfiles"] && st.Resource == "arn:aws:iam::000000000000:instance-profile/*" + if st.Sid == "ReadInstanceProfiles" + ]) + error_message = "the boundary may let a role read instance profiles, never create or change one." + } + assert { + condition = one([ + for st in jsondecode(aws_iam_policy.crossplane_boundary[0].policy).Statement : + st.Effect == "Deny" && st.Action == "iam:*" && st.NotResource == ["arn:aws:iam::000000000000:role/socle-test-bootstrap-node", "arn:aws:iam::000000000000:instance-profile/*"] + if st.Sid == "NeverIamBeyondTheNodeRole" + ]) + error_message = "iam must stay denied on every role but the nodes' and on everything but instance profiles." + } + assert { + condition = one([for st in jsondecode(aws_iam_policy.crossplane_boundary[0].policy).Statement : !contains(st.Action, "iam:*") && contains(st.Action, "sts:*") if st.Sid == "NeverIdentityNorAccount"]) + error_message = "sts and the account services stay denied on everything." + } + assert { + condition = length([for st in jsondecode(aws_iam_policy.crossplane_boundary[0].policy).Statement : st if st.Effect == "Allow" && strcontains(jsonencode(st.Action), "iam:")]) == 2 + error_message = "exactly two statements of the boundary may allow anything of IAM." + } +} + +run "crossplane_may_own_queues_and_event_rules_under_the_cluster_prefix" { + command = plan + variables { + crossplane = { allowed_services = ["sqs"] } + } + override_resource { + target = aws_iam_policy.crossplane_boundary + values = { arn = "arn:aws:iam::000000000000:policy/socle/socle-test/socle-test-crossplane-boundary" } + } + override_resource { + target = aws_eks_cluster.socle + values = { + arn = "arn:aws:eks:eu-west-3:000000000000:cluster/socle-test" + certificate_authority = [{ data = "Y2E=" }] + } + } + + assert { + condition = one([ + for st in jsondecode(aws_iam_role_policy.crossplane[0].policy).Statement : + st.Resource == "arn:aws:sqs:eu-west-3:000000000000:socle-test-*" && contains(st.Action, "sqs:CreateQueue") && !contains(st.Action, "sqs:SendMessage") && !contains(st.Action, "sqs:ReceiveMessage") + if st.Sid == "ManageQueuesUnderTheClusterPrefix" + ]) + error_message = "Crossplane may manage queues only under the cluster's own name prefix, and never read or write their messages." + } + assert { + condition = one([ + for st in jsondecode(aws_iam_role_policy.crossplane[0].policy).Statement : + st.Resource == "arn:aws:events:eu-west-3:000000000000:rule/socle-test-*" && contains(st.Action, "events:PutTargets") + if st.Sid == "ManageEventRulesUnderTheClusterPrefix" + ]) + error_message = "Crossplane may manage EventBridge rules and their targets only under the cluster's own name prefix." + } +} + +run "the_nodes_role_has_an_instance_profile_named_after_it" { + command = plan + + assert { + condition = aws_iam_instance_profile.node.name == "socle-test-bootstrap-node" && aws_iam_instance_profile.node.role == "socle-test-bootstrap-node" + error_message = "Karpenter's EC2NodeClass names the profile -bootstrap-node: it must carry the bootstrap nodes' role, under the role's own name." + } + assert { + condition = output.node_instance_profile == "socle-test-bootstrap-node" + error_message = "the output must name the profile." + } +} + run "crossplane_without_a_service_gets_a_boundary_that_grants_nothing" { command = plan variables { @@ -522,3 +624,34 @@ run "the_crossplane_boundary_is_named_after_its_cluster" { error_message = "the boundary's ARN, built without waiting for AWS, must name the policy as it is created." } } + +# docs/catalog/karpenter.md, AWS-23: the public nodes take nothing from the +# internet, and the cluster takes them back. +run "public_nodes_have_their_own_security_group_closed_to_the_internet" { + command = plan + + assert { + condition = aws_security_group.public_nodes.name == "socle-test-public-nodes" && output.public_nodes_security_group == "socle-test-public-nodes" + error_message = "Karpenter's public EC2NodeClass selects the group by name, -public-nodes." + } + assert { + condition = !contains(keys(aws_security_group.public_nodes.tags), "kubernetes.io/cluster/socle-test") + error_message = "the group must not carry the cluster tag: EKS's load balancer controller would take it for the cluster's and open NodePorts in it." + } + assert { + condition = aws_security_group_rule.public_nodes_from_the_vpc.type == "ingress" && aws_security_group_rule.public_nodes_from_the_vpc.cidr_blocks == tolist(["10.0.0.0/16"]) + error_message = "the public nodes admit the VPC — the control plane and the other nodes — and nothing from the internet." + } + assert { + condition = aws_security_group_rule.public_nodes_to_anywhere.type == "egress" && aws_security_group_rule.public_nodes_to_anywhere.cidr_blocks == tolist(["0.0.0.0/0"]) + error_message = "the public nodes' reason to exist is the internet, through the Internet Gateway." + } + assert { + condition = aws_security_group_rule.cluster_from_public_nodes.type == "ingress" && aws_security_group_rule.cluster_from_public_nodes.protocol == "-1" && aws_security_group_rule.cluster_from_public_nodes.cidr_blocks == null && length([for r in [aws_security_group_rule.public_nodes_from_the_vpc, aws_security_group_rule.public_nodes_to_anywhere, aws_security_group_rule.cluster_from_public_nodes] : r]) == 3 + error_message = "the cluster security group must admit the public nodes, by group, or they cannot join." + } + assert { + condition = output.vpc_cidr == "10.0.0.0/16" + error_message = "the VPC's CIDR is what Cilium masquerades outside of." + } +} diff --git a/opentofu/aws/variables.tf b/opentofu/aws/variables.tf index a2d40ac4..cd1c3f6c 100644 --- a/opentofu/aws/variables.tf +++ b/opentofu/aws/variables.tf @@ -81,7 +81,7 @@ variable "public_subnet_ids" { } variable "vpc_cidr" { - description = "CIDR for the VPC when this module creates it. Arbitrary default (not a research decision) — a /16 large enough for any socle estate." + description = "CIDR for the VPC when this module creates it. Arbitrary default (not a research decision) — a /16 large enough for any socle estate. With create_vpc = false, set it to the existing VPC's primary CIDR: pods take their addresses from it, and Cilium masquerades only what leaves it (vpc_cidr output)." type = string default = "10.0.0.0/16" nullable = false @@ -340,7 +340,10 @@ variable "crossplane" { this module writes. allowed_services is that boundary: the AWS services (IAM action prefixes, such as route53 or s3) any module's role may be granted. Empty grants nothing; iam, sts, organizations, account, sso and - identitystore are refused. Which resources of those services a module + identitystore are refused. With ec2 in the list, and only then, the + boundary also lets a role pass the bootstrap nodes' role to EC2 and read + instance profiles — what Karpenter's controller needs to launch nodes — + and nothing else of IAM. Which resources of those services a module reaches is its own role's policy. Null, the default, creates nothing. The client root passes the boundary's ARN into kube.crossplane. EOT diff --git a/opentofu/bootstrap/README.md b/opentofu/bootstrap/README.md index c40807c5..a440d7f1 100644 --- a/opentofu/bootstrap/README.md +++ b/opentofu/bootstrap/README.md @@ -162,7 +162,7 @@ No modules. | [artifact\_pull\_secret](#input\_artifact\_pull\_secret) | Name of an existing kubernetes.io/dockerconfigjson Secret in flux-system that Flux uses to pull the artifact from a private registry. Empty for a public registry. The Secret is created outside this module — a credential never enters OpenTofu. | `string` | `""` | no | | [artifact\_url](#input\_artifact\_url) | OCI repository the socle artifact is pulled from. Override for a mirror; the tag is socle\_version. | `string` | `"oci://ghcr.io/do-now-io/socle/flux-modules"` | no | | [cilium](#input\_cilium) | The socle's Cilium, on the clouds whose foundations create a cluster with
no CNI — aws and azure — as `{ enabled, hubble, gateway_api }`, every key
optional: `enabled` (true) installs it before Flux; false means the
cluster brings its own CNI and DNS, which only a test double does.
`hubble` (false) adds Hubble Relay and UI. `gateway_api` (true) makes
Cilium serve the `cilium` GatewayClass. `values` ({}) is any Cilium chart
value, merged over the socle's so the client wins; private keys are
refused there, and the chart's `existingSecret` fields name a Secret
instead. Refused on gcp and scaleway, where the cloud operates Cilium.
Typed `any` and validated like `kube`, so a misspelt key is an error at
plan. Chart versions are pinned in cilium.tf. | `any` | `{}` | no | -| [cluster\_network](#input\_cluster\_network) | What Cilium needs to know about the cluster, from the foundations'
outputs, never from the client: `api_endpoint`, the API server as EKS
returns it (https://host) or AKS does (a bare FQDN), for kube-proxy
replacement; `service_cidr`, the service range, whose `.10` is CoreDNS's
address on aws; `pod_cidr`, Cilium's pool on azure, where the VNet holds
nodes only. Required wherever the socle installs Cilium, ignored
elsewhere. | 
object({
    api_endpoint = string
    service_cidr = optional(string)
    pod_cidr     = optional(string)
  })
| `null` | no | +| [cluster\_network](#input\_cluster\_network) | What Cilium needs to know about the cluster, from the foundations'
outputs, never from the client: `api_endpoint`, the API server as EKS
returns it (https://host) or AKS does (a bare FQDN), for kube-proxy
replacement; `service_cidr`, the service range, whose `.10` is CoreDNS's
address on aws; `pod_cidr`, where pods take their addresses: on aws the
VPC's CIDR (ENI IPAM), outside which Cilium masquerades, on azure
Cilium's pool, the VNet holding nodes only. Required wherever the socle
installs Cilium, ignored elsewhere. | 
object({
    api_endpoint = string
    service_cidr = optional(string)
    pod_cidr     = optional(string)
  })
| `null` | no | | [coredns](#input\_coredns) | The CoreDNS the socle installs on aws, right after Cilium, as
`{ values }`: `values` ({}) is any CoreDNS chart value, merged over the
socle's so the client wins — extra zones, forwarders, plugins. The chart
has no value that takes secret material inline; a Secret is mounted by
name through `extraSecrets`, or read through `env[].valueFrom`. Refused
where the socle installs no CoreDNS: every cloud but aws, and aws with
cilium.enabled = false. Chart version pinned in cilium.tf. | `any` | `{}` | no | | [cosign\_identity](#input\_cosign\_identity) | Keyless identity the artifact's signature must match, as issuer and subject regexes. Defaults to the socle's release workflow on main, so production never consumes a branch build by accident. Override on a dev cluster testing a branch. Null means this default. Verification cannot be disabled. | 
object({
    issuer  = string
    subject = string
  })
| 
{
  "issuer": "^https://token\\.actions\\.githubusercontent\\.com$",
  "subject": "^https://github\\.com/do-now-io/socle/\\.github/workflows/publish-artifact\\.yaml@refs/heads/main$"
}
| no | | [eks\_addons](#input\_eks\_addons) | The EKS-managed add-ons the socle installs on aws once the nodes run, as
`{ pod_identity_agent, ebs_csi, efs_csi, snapshot_controller }`, every
key optional:
`pod_identity_agent` (true) is what hands every Pod Identity association
its credentials — Crossplane's AWS providers included — and flux-operator
waits for it; `ebs_csi` (true) is the block-storage driver, with its own
role; `efs_csi` (false) the RWX one, with its own role. Both drivers need
the agent. `snapshot_controller` (true) is the CSI snapshot controller
and its CRDs, what the velero module's EBS snapshots need. Refused on
every cloud but aws. Versions pinned in eks\_addons.tf. | `any` | `{}` | no | diff --git a/opentofu/bootstrap/catalog.tf b/opentofu/bootstrap/catalog.tf index 3114806e..b3742d3b 100644 --- a/opentofu/bootstrap/catalog.tf +++ b/opentofu/bootstrap/catalog.tf @@ -340,6 +340,47 @@ locals { values = {} values_secret = "" } + # Karpenter: the autoscaler that owns every node but the bootstrap group's + # (docs/catalog/karpenter.md). aws only — GKE Autopilot provisions its + # own nodes, and Karpenter has no provider there (catalog_clouds below). + # Off by default: it needs Crossplane, which makes its role and its + # interruption queue, and ec2, ssm, pricing, sqs and eks in the + # foundations' crossplane.allowed_services. Its nodes run as the + # bootstrap nodes' role, through the instance profile the foundations + # create for it. Two NodePools, always: private_pool, where a pod goes by + # default, in the private subnets, its egress through the NAT Gateway; + # public_pool, tainted socle.do-now.io/node-pool=public, for the + # workloads costly in network traffic that tolerate it, in the public + # subnets with a public IP, their egress through the Internet Gateway. + # Each: the capacity_types (spot, on-demand) and architectures (arm64, + # amd64) it may launch — Karpenter launches the cheapest that fits, so + # Spot and Graviton first, as for the bootstrap group — cpu and memory + # the ceiling of all its nodes together, consolidate_after how long a + # node stays empty or under-used before it goes, and ami the EC2NodeClass + # alias (al2023@latest follows each new AMI; al2023@v pins one). A + # client writes more NodePools himself. values and values_secret as every + # module; credentials are refused in values. + karpenter = { + enabled = false + private_pool = { + capacity_types = ["spot", "on-demand"] + architectures = ["arm64", "amd64"] + cpu = 32 + memory = "128Gi" + consolidate_after = "1m" + ami = "al2023@latest" + } + public_pool = { + capacity_types = ["spot", "on-demand"] + architectures = ["arm64", "amd64"] + cpu = 32 + memory = "128Gi" + consolidate_after = "1m" + ami = "al2023@latest" + } + values = {} + values_secret = "" + } } # Which clouds a module exists on. Absent = every cloud. A module listed @@ -351,6 +392,7 @@ locals { # var.kube refuses at plan a module this map does not offer on var.cloud. catalog_clouds = { gateway_api = ["aws", "azure", "gcp", "scaleway"] + karpenter = ["aws"] metrics_server = ["aws"] velero = ["aws", "gcp"] } diff --git a/opentofu/bootstrap/cilium.tf b/opentofu/bootstrap/cilium.tf index 637b3dbd..6d9f3e93 100644 --- a/opentofu/bootstrap/cilium.tf +++ b/opentofu/bootstrap/cilium.tf @@ -131,9 +131,23 @@ locals { # ENIs in the node's own subnet, the operator's default. aksbyocni.enabled # alone yields cluster-pool IPAM over VXLAN; the pool is the foundations' # pod_cidr, because the chart's own default, 10.0.0.0/8, contains the VNet. + # + # On aws, what leaves the VPC is masqueraded to the node's primary address + # (AWS-23). A pod's address is a secondary one of an ENI, which an + # Internet Gateway never translates: on a node in a public subnet, the + # Karpenter public pool's, a pod would reach nothing outside the VPC — + # measured on the sandbox, 2026-10-17, and its egress through the node's + # public IP once masqueraded. With masquerading on, Cilium routes only the + # VPC's range through each ENI and the rest through the primary interface, + # whose name it needs in ENI mode (AL2023 on Nitro: ens5). Inside the VPC + # nothing changes; in a private subnet the NAT Gateway now sees the node's + # address instead of the pod's. cilium_values_cloud = { aws = { - eni = { enabled = true } + eni = { enabled = true } + enableIPv4Masquerade = true + egressMasqueradeInterfaces = "ens+" + ipv4NativeRoutingCIDR = try(var.cluster_network.pod_cidr, null) } azure = { aksbyocni = { enabled = true } diff --git a/opentofu/bootstrap/tests/defaults.tftest.hcl b/opentofu/bootstrap/tests/defaults.tftest.hcl index 02114338..5300d30a 100644 --- a/opentofu/bootstrap/tests/defaults.tftest.hcl +++ b/opentofu/bootstrap/tests/defaults.tftest.hcl @@ -381,6 +381,16 @@ run "aws_installs_cilium_and_coredns_before_flux" { condition = yamldecode(helm_release.cilium[0].values[0]).eni.enabled == true error_message = "aws must run Cilium in ENI mode: pods carry VPC addresses, as the foundations' network design assumes." } + # AWS-23: a pod on a public node reaches the internet only masqueraded to + # its node's primary address; inside the VPC nothing is masqueraded. + assert { + condition = ( + yamldecode(helm_release.cilium[0].values[0]).enableIPv4Masquerade == true + && yamldecode(helm_release.cilium[0].values[0]).ipv4NativeRoutingCIDR == "10.244.0.0/16" + && yamldecode(helm_release.cilium[0].values[0]).egressMasqueradeInterfaces == "ens+" + ) + error_message = "on aws Cilium masquerades what leaves the VPC (pod_cidr), through the primary interface: ENI mode refuses masquerading without egressMasqueradeInterfaces." + } assert { condition = yamldecode(helm_release.cilium[0].values[0]).kubeProxyReplacement == true && yamldecode(helm_release.cilium[0].values[0]).k8sServiceHost == "ABCDEF.gr7.eu-west-3.eks.amazonaws.com" && yamldecode(helm_release.cilium[0].values[0]).k8sServicePort == 443 error_message = "with no kube-proxy, Cilium must replace it and reach the API server by the host the foundations output, port 443." @@ -1661,3 +1671,50 @@ run "flux_version_takes_an_exact_version" { error_message = "an exact flux_version must reach the FluxInstance as written." } } + +# --- karpenter — docs/catalog/karpenter.md --- + +run "karpenter_is_off_by_default_with_a_private_and_a_public_pool" { + command = plan + + assert { + condition = output.inputs.modules.karpenter.enabled == false && output.inputs.modules.karpenter.values == {} && output.inputs.modules.karpenter.values_secret == "" + error_message = "karpenter must be off by default: it needs Crossplane and services the client allows." + } + assert { + condition = alltrue([for p in [output.inputs.modules.karpenter.private_pool, output.inputs.modules.karpenter.public_pool] : + p.capacity_types == ["spot", "on-demand"] && p.architectures == ["arm64", "amd64"] && p.cpu == 32 + && p.memory == "128Gi" && p.consolidate_after == "1m" && p.ami == "al2023@latest" + ]) + error_message = "both NodePools: Spot and on demand, Graviton and x86, 32 vCPU and 128Gi in all, consolidated after a minute, on the latest AL2023 (docs/catalog/karpenter.md)." + } +} + +run "karpenter_pools_keep_their_defaults_beside_what_the_client_writes" { + command = plan + variables { + region = "eu-west-3" + kube = { + crossplane = { enabled = true } + karpenter = { + enabled = true + private_pool = { capacity_types = ["on-demand"], cpu = 8 } + public_pool = { cpu = 4 } + values = { logLevel = "debug" } + } + } + } + + assert { + condition = output.inputs.modules.karpenter.enabled && output.inputs.modules.karpenter.private_pool.capacity_types == ["on-demand"] && output.inputs.modules.karpenter.private_pool.cpu == 8 && output.inputs.modules.karpenter.public_pool.cpu == 4 + error_message = "what the client writes in either pool must reach the inputs." + } + assert { + condition = output.inputs.modules.karpenter.private_pool.architectures == ["arm64", "amd64"] && output.inputs.modules.karpenter.public_pool.capacity_types == ["spot", "on-demand"] && output.inputs.modules.karpenter.public_pool.memory == "128Gi" + error_message = "a key of a pool the client leaves out keeps its default." + } + assert { + condition = output.inputs.modules.karpenter.values.logLevel == "debug" + error_message = "values must flow through untouched." + } +} diff --git a/opentofu/bootstrap/tests/validations.tftest.hcl b/opentofu/bootstrap/tests/validations.tftest.hcl index 73d5888e..98b3e675 100644 --- a/opentofu/bootstrap/tests/validations.tftest.hcl +++ b/opentofu/bootstrap/tests/validations.tftest.hcl @@ -1896,3 +1896,109 @@ run "argocd_image_updater_repositories_need_the_project_on_gcp" { } expect_failures = [var.project] } + +# --- karpenter --- + +run "kube_refuses_karpenter_without_crossplane" { + command = plan + variables { + region = "eu-west-3" + kube = { karpenter = { enabled = true } } + } + expect_failures = [var.kube] +} + +run "kube_refuses_karpenter_without_a_region" { + command = plan + variables { + kube = { crossplane = { enabled = true }, karpenter = { enabled = true } } + } + expect_failures = [var.kube] +} + +run "kube_refuses_karpenter_on_gcp" { + command = plan + variables { + cloud = "gcp" + project = { id = "sandbox-2bace", number = "123456789012" } + kube = { karpenter = { enabled = false } } + } + expect_failures = [var.kube] +} + +run "kube_refuses_a_karpenter_capacity_type_karpenter_does_not_know" { + command = plan + variables { kube = { karpenter = { private_pool = { capacity_types = ["spot", "preemptible"] } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_no_karpenter_capacity_type" { + command = plan + variables { kube = { karpenter = { public_pool = { capacity_types = [] } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_a_karpenter_architecture_that_is_not_kubernetes_arch" { + command = plan + variables { kube = { karpenter = { private_pool = { architectures = ["x86_64"] } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_a_karpenter_architecture_twice" { + command = plan + variables { kube = { karpenter = { public_pool = { architectures = ["arm64", "arm64"] } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_a_fractional_karpenter_cpu_limit" { + command = plan + variables { kube = { karpenter = { private_pool = { cpu = 1.5 } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_a_karpenter_memory_limit_without_a_binary_unit" { + command = plan + variables { kube = { karpenter = { public_pool = { memory = "128G" } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_a_karpenter_consolidate_after_that_is_not_a_duration" { + command = plan + variables { kube = { karpenter = { private_pool = { consolidate_after = "1 minute" } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_a_karpenter_ami_outside_al2023" { + command = plan + variables { kube = { karpenter = { public_pool = { ami = "bottlerocket@latest" } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_an_unknown_karpenter_pool_key" { + command = plan + variables { kube = { karpenter = { private_pool = { instance_types = ["m7g.large"] } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_karpenter_values_carrying_a_literal_credential_env" { + command = plan + variables { kube = { karpenter = { values = { controller = { env = [{ name = "AWS_SECRET_ACCESS_KEY", value = "x" }] } } } } } + expect_failures = [var.kube] +} + +run "kube_refuses_karpenter_values_secret_that_is_not_a_secret_name" { + command = plan + variables { kube = { karpenter = { values_secret = "Karpenter_Values" } } } + expect_failures = [var.kube] +} + +run "cluster_network_refuses_aws_without_the_vpc_cidr" { + command = plan + variables { + cluster_network = { + api_endpoint = "https://ABCDEF.gr7.eu-west-3.eks.amazonaws.com" + service_cidr = "172.20.0.0/16" + } + } + expect_failures = [var.cluster_network] +} diff --git a/opentofu/bootstrap/variables.tf b/opentofu/bootstrap/variables.tf index 33ad7c5e..81ac0d09 100644 --- a/opentofu/bootstrap/variables.tf +++ b/opentofu/bootstrap/variables.tf @@ -1070,6 +1070,84 @@ variable "kube" { ) error_message = "kube.metrics_server.values must not set --kubelet-insecure-tls: it disables verification of kubelet certificates, so metrics-server would present its credentials to anyone impersonating a kubelet. The socle owns the nodes, and a certificate that does not validate is a socle bug to fix, not a client's to work around." } + + # --- karpenter — docs/catalog/karpenter.md --------------------------------- + + # Its role and its interruption queue are Crossplane managed resources: with + # Crossplane off it would install a controller that can launch nothing. + validation { + condition = !can(keys(var.kube)) || !try(var.kube.karpenter.enabled, false) || try(var.kube.crossplane.enabled, false) == true + error_message = "kube.karpenter needs kube.crossplane.enabled = true: its IAM role, Pod Identity association and interruption queue are Crossplane managed resources (docs/catalog/karpenter.md). Turn Crossplane on, and allow ec2, ssm, pricing, sqs and eks in the foundations' aws.crossplane.allowed_services." + } + + validation { + condition = !can(keys(var.kube)) || !try(var.kube.karpenter.enabled, false) || var.region != "" + error_message = "kube.karpenter needs the cluster's region: its queue, its EventBridge rules and its Pod Identity association are regional. Pass region to the bootstrap module (the aws root wires var.aws.region)." + } + + # The same rules for both NodePools, private_pool and public_pool. + validation { + condition = alltrue([for p in ["private_pool", "public_pool"] : !can(var.kube.karpenter[p].capacity_types) || try( + length(tolist(var.kube.karpenter[p].capacity_types)) > 0 + && length(distinct(tolist(var.kube.karpenter[p].capacity_types))) == length(tolist(var.kube.karpenter[p].capacity_types)) + && alltrue([for c in tolist(var.kube.karpenter[p].capacity_types) : contains(["spot", "on-demand"], c)]), + false + )]) + error_message = "kube.karpenter.private_pool.capacity_types and public_pool.capacity_types must list spot, on-demand or both, each once." + } + + validation { + condition = alltrue([for p in ["private_pool", "public_pool"] : !can(var.kube.karpenter[p].architectures) || try( + length(tolist(var.kube.karpenter[p].architectures)) > 0 + && length(distinct(tolist(var.kube.karpenter[p].architectures))) == length(tolist(var.kube.karpenter[p].architectures)) + && alltrue([for a in tolist(var.kube.karpenter[p].architectures) : contains(["arm64", "amd64"], a)]), + false + )]) + error_message = "kube.karpenter.private_pool.architectures and public_pool.architectures must list arm64, amd64 or both, each once: the values of kubernetes.io/arch." + } + + validation { + condition = alltrue([for p in ["private_pool", "public_pool"] : !can(var.kube.karpenter[p].cpu) || try(var.kube.karpenter[p].cpu >= 1 && floor(var.kube.karpenter[p].cpu) == var.kube.karpenter[p].cpu, false)]) + error_message = "kube.karpenter.private_pool.cpu and public_pool.cpu must be a whole number of vCPU, at least 1: the ceiling of all the NodePool's nodes together." + } + + validation { + condition = alltrue([for p in ["private_pool", "public_pool"] : !can(var.kube.karpenter[p].memory) || try(can(regex("^[1-9][0-9]*(Mi|Gi|Ti)$", var.kube.karpenter[p].memory)), false)]) + error_message = "kube.karpenter.private_pool.memory and public_pool.memory must be a Kubernetes quantity in Mi, Gi or Ti, such as 128Gi: the ceiling of all the NodePool's nodes together." + } + + validation { + condition = alltrue([for p in ["private_pool", "public_pool"] : !can(var.kube.karpenter[p].consolidate_after) || try(can(regex("^([0-9]+h)?([0-9]+m)?([0-9]+s)?$", var.kube.karpenter[p].consolidate_after)) && var.kube.karpenter[p].consolidate_after != "", false)]) + error_message = "kube.karpenter.private_pool.consolidate_after and public_pool.consolidate_after must be a duration in hours, minutes and seconds, such as 1m or 1h30m." + } + + # AL2023 only: the bootstrap nodes' AMI family, the one whose nodeadm + # Karpenter configures for this cluster, Cilium is proven on, and the + # public EC2NodeClass's NodeConfig is written for. + validation { + condition = alltrue([for p in ["private_pool", "public_pool"] : !can(var.kube.karpenter[p].ami) || try(can(regex("^al2023@(latest|v[0-9]{8})$", var.kube.karpenter[p].ami)), false)]) + error_message = "kube.karpenter.private_pool.ami and public_pool.ami must be an AL2023 alias, al2023@latest or al2023@v such as al2023@v20261001: the bootstrap nodes' AMI family." + } + + # values is free-form, minus credentials: the controller takes its AWS + # access from Pod Identity, and an access key in controller.env would land + # in the state and in a ConfigMap. Its name says it whatever the case. + validation { + condition = ( + !can(var.kube.karpenter.values) + || !can(keys(var.kube.karpenter.values)) + || alltrue([ + for e in try(tolist(var.kube.karpenter.values.controller.env), []) : + !(can(e.value) && can(regex("(?i)(secret|password|passwd|token|credential|api_?key|access_?key|private_?key)", try(e.name, "")))) + ]) + ) + error_message = "kube.karpenter.values must not carry secrets: a controller.env entry with a literal value named like a password, token, secret, credential or key is refused. Its AWS access is its Pod Identity role; anything else goes in a Secret named in kube.karpenter.values_secret." + } + + validation { + condition = !can(var.kube.karpenter.values_secret) || try(var.kube.karpenter.values_secret == "" || can(regex("^[a-z0-9]([-a-z0-9.]{0,251}[a-z0-9])?$", var.kube.karpenter.values_secret)), true) + error_message = "kube.karpenter.values_secret must be empty or a valid Kubernetes Secret name (lowercase RFC 1123 subdomain)." + } } # --------------------------------------------------------------------------- @@ -1234,9 +1312,10 @@ variable "cluster_network" { outputs, never from the client: `api_endpoint`, the API server as EKS returns it (https://host) or AKS does (a bare FQDN), for kube-proxy replacement; `service_cidr`, the service range, whose `.10` is CoreDNS's - address on aws; `pod_cidr`, Cilium's pool on azure, where the VNet holds - nodes only. Required wherever the socle installs Cilium, ignored - elsewhere. + address on aws; `pod_cidr`, where pods take their addresses: on aws the + VPC's CIDR (ENI IPAM), outside which Cilium masquerades, on azure + Cilium's pool, the VNet holding nodes only. Required wherever the socle + installs Cilium, ignored elsewhere. EOT type = object({ api_endpoint = string @@ -1255,6 +1334,11 @@ variable "cluster_network" { error_message = "cluster_network.service_cidr is required on aws: CoreDNS takes the .10 address of the service range, the one every node's kubelet is told to use." } + validation { + condition = !(local.cilium_installed && var.cloud == "aws") || try(can(cidrhost(var.cluster_network.pod_cidr, 0)), false) + error_message = "cluster_network.pod_cidr is required on aws: the VPC's CIDR, which pods take their addresses from and outside which Cilium masquerades. Pass the foundations' vpc_cidr output (the aws root wires it)." + } + validation { condition = !(local.cilium_installed && var.cloud == "azure") || try(var.cluster_network.pod_cidr != null, false) error_message = "cluster_network.pod_cidr is required on azure: Cilium's cluster pool must not default to 10.0.0.0/8, which contains the VNet." diff --git a/opentofu/clusters/aws/main.tf b/opentofu/clusters/aws/main.tf index 631a7612..15ce3452 100644 --- a/opentofu/clusters/aws/main.tf +++ b/opentofu/clusters/aws/main.tf @@ -81,6 +81,8 @@ module "socle" { cluster_network = { api_endpoint = module.foundations.cluster_endpoint service_cidr = module.foundations.service_cidr + # The VPC's: pods take their addresses from it, Cilium masquerades the rest. + pod_cidr = module.foundations.vpc_cidr } # The EKS-managed add-ons, after the network: the Pod Identity Agent @@ -147,3 +149,17 @@ check "crossplane_has_an_identity" { error_message = "kube.crossplane.enabled is set without aws.crossplane: the AWS provider runs with no identity, and every CloudAccess will fail. Set aws.crossplane, or keep Crossplane off." } } + +# A warning, not an error: the module converges, but Karpenter's controller +# is refused by IAM on every call to a service the boundary does not allow — +# and without ec2 there, the boundary has no node-role exception either, so +# no node is ever launched (docs/catalog/karpenter.md). +check "karpenter_has_its_services" { + assert { + condition = ( + !try(var.kube.karpenter.enabled, false) + || length(setsubtract(["ec2", "ssm", "pricing", "sqs", "eks"], try(var.aws.crossplane.allowed_services, []))) == 0 + ) + error_message = "kube.karpenter.enabled is set but aws.crossplane.allowed_services lacks ${join(", ", setsubtract(["ec2", "ssm", "pricing", "sqs", "eks"], try(var.aws.crossplane.allowed_services, [])))}: Karpenter's controller would be refused by IAM. Add them to aws.crossplane.allowed_services." + } +} diff --git a/opentofu/clusters/aws/prod.tfvars.example b/opentofu/clusters/aws/prod.tfvars.example index 317b7c8e..3f5379d8 100644 --- a/opentofu/clusters/aws/prod.tfvars.example +++ b/opentofu/clusters/aws/prod.tfvars.example @@ -106,6 +106,17 @@ kube = { # must also be in aws.crossplane.allowed_services above. Leave it out for # cron, Prometheus, Kafka, RabbitMQ or Redis triggers: no cloud role needed. # keda = { enabled = true, services = ["sqs"] } + # Nodes for the workloads: Karpenter launches what a Pending pod needs, + # Spot and Graviton first, and removes what is empty or under-used — in a + # private pool by default, or a public one for workloads costly in network + # traffic (no NAT Gateway on their egress) that tolerate it + # (docs/catalog/karpenter.md). Needs kube.crossplane on and ec2, ssm, + # pricing, sqs and eks in aws.crossplane.allowed_services above. + # karpenter = { + # enabled = true + # private_pool = { cpu = 64, memory = "256Gi" } + # public_pool = { cpu = 16, memory = "64Gi" } + # } } # Cilium is installed before Flux with these defaults; list only what diff --git a/site/src/components/CatalogBricks.astro b/site/src/components/CatalogBricks.astro index 364944de..5438cb0e 100644 --- a/site/src/components/CatalogBricks.astro +++ b/site/src/components/CatalogBricks.astro @@ -36,7 +36,7 @@ const marks: Record = { // metrics-server is a kubernetes-sigs project. 'metrics-server': kubernetes, }; -const monogram: Record = { 'external-dns': 'DNS', 'external-secrets': 'ES', reloader: 'R', velero: 'V', hello: 'H' }; +const monogram: Record = { 'external-dns': 'DNS', 'external-secrets': 'ES', karpenter: 'K', reloader: 'R', velero: 'V', hello: 'H' }; const categories = ['gitops', 'networking', 'security', 'secrets', 'autoscaling', 'observability', 'backup', 'platform']; const bricks = [...modules].sort( (a, b) => categories.indexOf(a.category) - categories.indexOf(b.category) || a.page.localeCompare(b.page),