From dc5e27af5eec729879958e68d34497050b2bf293 Mon Sep 17 00:00:00 2001 From: Rahul Date: Fri, 23 Oct 2026 20:50:49 +0530 Subject: [PATCH 1/5] feat(helm): run the executor Guaranteed at 2 CPU / 2Gi by default (#513) A default install had no resources, so the executor was BestEffort and its sandboxes could claim memory the node didn't have. requests equal limits on the executor and on the enable-forwarding init container, so the pod is Guaranteed. Co-Authored-By: Claude Opus 5.5 --- .../templates/executor-deployment.yaml | 9 +++++++++ deploy/helm/openplan/values.yaml | 18 ++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/deploy/helm/openplan/templates/executor-deployment.yaml b/deploy/helm/openplan/templates/executor-deployment.yaml index b5254b37..019217ac 100644 --- a/deploy/helm/openplan/templates/executor-deployment.yaml +++ b/deploy/helm/openplan/templates/executor-deployment.yaml @@ -25,6 +25,15 @@ spec: command: ["sysctl", "-w", "net.ipv4.ip_forward=1"] securityContext: privileged: true + # A pod is Guaranteed only if every container, init included, has + # requests equal to limits; sysctl needs next to nothing. + resources: + requests: + cpu: 50m + memory: 32Mi + limits: + cpu: 50m + memory: 32Mi containers: - name: executor image: {{ include "openplan.image" $component }} diff --git a/deploy/helm/openplan/values.yaml b/deploy/helm/openplan/values.yaml index 8a14fd85..1a0c5ba1 100644 --- a/deploy/helm/openplan/values.yaml +++ b/deploy/helm/openplan/values.yaml @@ -28,10 +28,20 @@ executor: # chart sets EXECUTOR_RUN_ROOT and EXECUTOR_SANDBOX_STATE itself. existingSecret: "" replicas: 1 - # Set a memory limit of at least EXECUTOR_SANDBOX_MEMORY times - # EXECUTOR_MAX_SESSIONS, plus the executor's own headroom: every sandbox's - # memory is counted in this pod. - resources: {} + # Every sandbox's memory is counted in this pod, so memory must be at least + # EXECUTOR_MAX_SESSIONS times EXECUTOR_SANDBOX_MEMORY plus 512Mi for the + # executor itself; the default fits the chart's one 1536 MiB session. + # requests equal limits, so the pod is Guaranteed and never uses more than + # the node reserved for it. Override requests and limits together: Helm + # merges maps, so setting only limits.memory keeps the 2Gi request and the + # pod drops to Burstable. + resources: + requests: + cpu: "2" + memory: 2Gi + limits: + cpu: "2" + memory: 2Gi runRoot: # Disk for the workspaces of the runs in flight on one executor. The # kubelet evicts the pod, and every run on it, when they outgrow it. From 212876bf0ff3699426a25ce60e80d7d5a009e9ba Mon Sep 17 00:00:00 2001 From: Rahul Date: Fri, 23 Oct 2026 20:51:30 +0530 Subject: [PATCH 2/5] feat(helm): default the executor to one 1536 MiB session (#513) The executor's built-in 20 sessions of 2048 MiB need 40 GiB, far past the 2Gi pod. A chart ConfigMap sets EXECUTOR_MAX_SESSIONS=1 and EXECUTOR_SANDBOX_MEMORY=1536 ahead of the user's Secret in envFrom, so a key in the Secret still overrides it, and a checksum annotation rolls the pod when the defaults change. Co-Authored-By: Claude Opus 5.5 --- .../openplan/templates/executor-configmap.yaml | 14 ++++++++++++++ .../openplan/templates/executor-deployment.yaml | 8 ++++++++ 2 files changed, 22 insertions(+) create mode 100644 deploy/helm/openplan/templates/executor-configmap.yaml diff --git a/deploy/helm/openplan/templates/executor-configmap.yaml b/deploy/helm/openplan/templates/executor-configmap.yaml new file mode 100644 index 00000000..e4ff257f --- /dev/null +++ b/deploy/helm/openplan/templates/executor-configmap.yaml @@ -0,0 +1,14 @@ +{{- $component := dict "context" $ "component" "executor" -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "openplan.fullname" . }}-executor + labels: + {{- include "openplan.labels" $component | nindent 4 }} +data: + # One session of 1536 MiB fits the default 2Gi pod with 512Mi for the + # executor. The executor's own defaults (20 sessions of 2048 MiB) would need + # 40 GiB. More runs at once come from executor.replicas; a key of the same + # name in the executor's Secret overrides these. + EXECUTOR_MAX_SESSIONS: "1" + EXECUTOR_SANDBOX_MEMORY: "1536" diff --git a/deploy/helm/openplan/templates/executor-deployment.yaml b/deploy/helm/openplan/templates/executor-deployment.yaml index 019217ac..124bd0b6 100644 --- a/deploy/helm/openplan/templates/executor-deployment.yaml +++ b/deploy/helm/openplan/templates/executor-deployment.yaml @@ -14,6 +14,10 @@ spec: metadata: labels: {{- include "openplan.labels" $component | nindent 8 }} + annotations: + # The executor reads its environment only at start, so a change to the + # chart's session defaults rolls the pod. + checksum/config: {{ include (print $.Template.BasePath "/executor-configmap.yaml") . | sha256sum }} spec: # Each sandbox's network is routed through this pod's, which needs # forwarding on. Kubernetes treats net.ipv4.ip_forward as an unsafe @@ -37,7 +41,11 @@ spec: containers: - name: executor image: {{ include "openplan.image" $component }} + # A later source wins a duplicate key, so the Secret overrides the + # chart's session defaults. envFrom: + - configMapRef: + name: {{ include "openplan.fullname" . }}-executor - secretRef: name: {{ required "executor.existingSecret is required: the name of the Secret holding the executor's environment" .Values.executor.existingSecret }} # env wins over envFrom, so a stray key in the Secret can't move the From c1dd41bdfa63a6702951f8d6917f88563817ecc0 Mon Sep 17 00:00:00 2001 From: Rahul Date: Fri, 23 Oct 2026 20:52:57 +0530 Subject: [PATCH 3/5] feat(helm): run the api Guaranteed at 2 CPU / 2Gi by default (#513) A default install gave the api no resources, so it was BestEffort and could take whatever the node had. A new api.resources key defaults to requests equal to limits; the api has no init containers, so the pod is Guaranteed. Co-Authored-By: Claude Opus 5.5 --- deploy/helm/openplan/templates/api-deployment.yaml | 4 ++++ deploy/helm/openplan/values.yaml | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/deploy/helm/openplan/templates/api-deployment.yaml b/deploy/helm/openplan/templates/api-deployment.yaml index 3a298eb7..cf8406b4 100644 --- a/deploy/helm/openplan/templates/api-deployment.yaml +++ b/deploy/helm/openplan/templates/api-deployment.yaml @@ -50,6 +50,10 @@ spec: httpGet: path: /healthz port: http + {{- with .Values.api.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} {{- if .Values.api.trustedCASecret }} volumeMounts: - name: trusted-ca diff --git a/deploy/helm/openplan/values.yaml b/deploy/helm/openplan/values.yaml index 1a0c5ba1..fbcb6a5b 100644 --- a/deploy/helm/openplan/values.yaml +++ b/deploy/helm/openplan/values.yaml @@ -19,6 +19,17 @@ api: # Optional. The name of an existing Secret whose ca.crt the api trusts in # addition to the public roots, for an OIDC issuer behind an internal CA. trustedCASecret: "" + # requests equal limits, so the pod is Guaranteed and never uses more than + # the node reserved for it. Override requests and limits together: Helm + # merges maps, so setting only limits.memory keeps the 2Gi request and the + # pod drops to Burstable. + resources: + requests: + cpu: "2" + memory: 2Gi + limits: + cpu: "2" + memory: 2Gi executor: # Required. The name of an existing Secret in the release namespace. Every From 71b21bcd1ad0d640c20c82787236a51f5ae414af Mon Sep 17 00:00:00 2001 From: Rahul Date: Fri, 23 Oct 2026 20:54:38 +0530 Subject: [PATCH 4/5] docs(helm): document the 2 CPU / 2Gi defaults and scaling by replicas (#513) The README still told readers to give the executor a 41Gi limit for 20 sessions and warned about a BestEffort pod. It now says what the chart defaults to (one 1536 MiB run per 2 CPU / 2Gi Guaranteed executor, the same for the api), that more runs at once come from executor.replicas, how to give a heavy module more memory, and that a run beyond the free executors waits up to 10 minutes, plan or apply, before it fails. Co-Authored-By: Claude Opus 5.5 --- deploy/helm/openplan/README.md | 64 ++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/deploy/helm/openplan/README.md b/deploy/helm/openplan/README.md index f6b3127e..642008b9 100644 --- a/deploy/helm/openplan/README.md +++ b/deploy/helm/openplan/README.md @@ -84,8 +84,8 @@ The executor's Secret: | `VALKEY_URL` | `redis://executor:@:6379/0` | | `ARTIFACT_STORE_KIND` and the `S3_*` keys | The same as the api's | | `TEMPORAL_NAMESPACE` | Optional; the same as the api's | -| `EXECUTOR_MAX_SESSIONS` | Optional; runs at once on one executor, default 20 | -| `EXECUTOR_SANDBOX_MEMORY` | Optional; MiB per sandbox, default 2048 | +| `EXECUTOR_MAX_SESSIONS` | Optional; runs at once on one executor. The chart sets 1; a value here overrides it | +| `EXECUTOR_SANDBOX_MEMORY` | Optional; MiB per sandbox. The chart sets 1536; a value here overrides it. See "Sizing the executor" before you raise it | | `EXECUTOR_SANDBOX_CPUS`, `EXECUTOR_SANDBOX_PIDS` | Optional; per sandbox, default 2 and 1024 | For example, from two env files you keep out of version control: @@ -120,9 +120,6 @@ api: trustedCASecret: openplan-trusted-ca # only for an internal CA executor: existingSecret: openplan-executor-env - resources: - limits: - memory: 41Gi # see "Sizing the executor" ingress: className: nginx host: openplan.example.com @@ -140,8 +137,8 @@ kubectl -n openplan rollout status deployment/openplan-executor ``` `helm install` succeeds before any pod is up. Wait for the executor too: if -it can't be scheduled or refuses to start, the UI still works, but every -plan fails after a minute and every apply after ten, for want of an executor. +it can't be scheduled or refuses to start, the UI still works, but every plan +and apply waits ten minutes for an executor and then fails. **6. Sign in** at `https://` as the user whose `sub` you set in `OPENPLAN_ROOT_SUBJECT`. @@ -154,9 +151,10 @@ plan fails after a minute and every apply after ten, for want of an executor. | `image.tag` | the chart's version | Override only to test a build that isn't a release. | | `api.existingSecret` | required | The api's Secret. | | `api.trustedCASecret` | none | A Secret whose `ca.crt` the api trusts, for Dex behind an internal CA. | +| `api.resources` | 2 CPU, 2Gi, requests equal to limits | Requests and limits for the api container. Override requests and limits together. | | `executor.existingSecret` | required | The executor's Secret. | -| `executor.replicas` | `1` | Executors. Each runs up to `EXECUTOR_MAX_SESSIONS` runs at once. | -| `executor.resources` | none | Requests and limits for the executor container. | +| `executor.replicas` | `1` | Executors. Each runs one run at a time by default (`EXECUTOR_MAX_SESSIONS`), so this is how many runs go at once. | +| `executor.resources` | 2 CPU, 2Gi, requests equal to limits | Requests and limits for the executor container. Override requests and limits together; see "Sizing the executor". | | `executor.runRoot.sizeLimit` | `8Gi` | Disk for the workspaces of the runs in flight on one executor. | | `ingress.className` | the cluster default | The IngressClass. | | `ingress.host` | required | The host openplan is served on. The api's public URL is `https://`. | @@ -167,19 +165,38 @@ the key. ## Sizing the executor -Every sandbox's memory counts against the executor pod. Set the executor's -memory limit to at least `EXECUTOR_SANDBOX_MEMORY` × `EXECUTOR_MAX_SESSIONS`, -plus about 1 GiB for the executor itself. The defaults, 2048 MiB × 20 -sessions, need 40 GiB of runs, so a limit around 41 GiB. To fit a smaller -node, lower `EXECUTOR_MAX_SESSIONS` in the executor's Secret, then the limit. +By default each executor pod gets 2 CPU and 2 GiB, reserved on its node +(requests equal limits, so Kubernetes runs it as `Guaranteed`), and runs one +plan or apply at a time in a 1536 MiB sandbox. The other 512 MiB is for the +executor itself. + +**More runs at once: add replicas.** Set `executor.replicas` to the number of +runs you want going at the same time. Each replica needs another 2 CPU and +2 GiB on some node. + +**When runs outnumber executors**, a new run shows "waiting for executor" and +waits for a free one, up to 10 minutes, whether it's a plan or an apply. If +none frees up in that time, the run fails and you start it again. + +**A heavy module: give each run more memory.** If a run's sandbox runs out of +memory, the run fails, and an apply can stop halfway and leave the state +locked. Raise `EXECUTOR_SANDBOX_MEMORY` in the executor's Secret, and raise +the executor's memory with it, so that memory is at least +`EXECUTOR_MAX_SESSIONS` × `EXECUTOR_SANDBOX_MEMORY` + 512Mi. Set requests and +limits together: Helm merges the map, so setting only `limits.memory` keeps +the 2Gi request and the pod is no longer `Guaranteed`. For 3072 MiB per run, +set `EXECUTOR_SANDBOX_MEMORY=3072` in the Secret and: -A limit set without a request is also the request, so the scheduler needs a -node with that much allocatable memory. With the defaults' 41 GiB limit on a -smaller node, the executor stays `Pending` ("Insufficient memory"). With no -`executor.resources` at all, the pod schedules anywhere, but its runs can -together claim more memory than the node has, and as a `BestEffort` pod the -executor is the first the kubelet evicts under memory pressure, failing every -run on it. +```yaml +executor: + resources: + requests: + cpu: "2" + memory: 3584Mi + limits: + cpu: "2" + memory: 3584Mi +``` The run root holds the checked-out template and providers of every run in flight. When it passes `executor.runRoot.sizeLimit`, the kubelet evicts the @@ -258,8 +275,9 @@ release you checked out without its `v`, such as `1.0.0`. Then open `https://openplan.localhost` and sign in as `admin@openplan.local` / `admin-local-only`. `setup.sh` prints how to trust -the local CA on macOS, so the browser stops warning. Docker Desktop's VM has -about 8 GiB, so the harness runs two sessions at once. To remove everything: +the local CA on macOS, so the browser stops warning. The harness runs the +chart's defaults, which fit in Docker Desktop's VM of about 8 GiB: one run at +a time, and a second run waits for the first. To remove everything: ```sh ./deploy/helm/evaluation/teardown.sh From 5ce0244abb005a998f970c53086835b69113d449 Mon Sep 17 00:00:00 2001 From: Rahul Date: Fri, 23 Oct 2026 22:29:21 +0530 Subject: [PATCH 5/5] docs(helm): make the README's ingress example controller-neutral (#513) ingress-nginx was retired in March 2026. The install example now takes a placeholder IngressClass, the prerequisites say any controller that serves a standard Ingress works and not to start on ingress-nginx, and the evaluation harness line names Traefik. Co-Authored-By: Claude Opus 5.5 --- deploy/helm/openplan/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/deploy/helm/openplan/README.md b/deploy/helm/openplan/README.md index 642008b9..a4f166f6 100644 --- a/deploy/helm/openplan/README.md +++ b/deploy/helm/openplan/README.md @@ -24,7 +24,10 @@ real TLS, real keys and an OIDC provider reachable over HTTPS. executor's security settings, so the chart doesn't work there. Use standard node pools. - **An ingress controller** with an IngressClass, and a TLS certificate for - the host you serve openplan on. + the host you serve openplan on. The chart creates a standard Ingress with + no controller-specific annotations, so any controller that serves one + works. Don't start on ingress-nginx: Kubernetes retired it in March 2026, + and it gets no more releases or security fixes. ### The services you bring @@ -121,7 +124,7 @@ api: executor: existingSecret: openplan-executor-env ingress: - className: nginx + className: # kubectl get ingressclass; omit for the cluster default host: openplan.example.com tlsSecretName: openplan-tls ``` @@ -257,7 +260,7 @@ how it went. `deploy/helm/evaluation/` in the repository sets up everything above on Docker Desktop's Kubernetes, so you can try the chart without your own -services: single-pod Postgres, Temporal, Dex, Valkey and MinIO, ingress-nginx, +services: single-pod Postgres, Temporal, Dex, Valkey and MinIO, Traefik, a local CA, fresh keys and the two Secrets. It is for evaluation only: one replica of everything, no backups, no persistence you should trust.