[RORDEV-1229] Make docker-compose resource limits opt-in and host-size aware

.github/workflows/all-e2e-tests.yml

@@ -7,6 +7,9 @@ on:
   pull_request:
     types: [opened, synchronize, reopened]
 
+env:
+  APPLY_RESOURCE_LIMITS: "auto"
+
 jobs:
   # ==========================================
   # E2E TESTS - MASTER BRANCH

.github/workflows/targeted-e2e-tests.yml

@@ -25,6 +25,9 @@ on:
         default: 'prod'
         type: string
 
+env:
+  APPLY_RESOURCE_LIMITS: "auto"
+
 jobs:
   e2e-tests:
     name: "🔬 E2E Tests"

environments/elk-ror/apm.docker-compose.yml

@@ -15,8 +15,6 @@ services:
         condition: service_healthy
     networks:
       - es-ror-network
-    mem_limit: 1g
-    memswap_limit: 1g
     healthcheck:
       test: [ "CMD", "curl", "-k", "--fail", "https://localhost:8200" ]
       interval: 10s
@@ -40,8 +38,6 @@ services:
       - ./certs:/example-app/certs:z
     networks:
       - es-ror-network
-    mem_limit: 128m
-    memswap_limit: 128m
     healthcheck:
       test: [ "CMD", "curl", "-k", "--http1.1", "--fail", "https://apm-server:8200" ]
       interval: 10s

environments/elk-ror/apm.limits.docker-compose.yml

@@ -0,0 +1,12 @@
+# Resource-limit overlay for the APM services. Applied on top of apm.docker-compose.yml by start.sh
+# when APPLY_RESOURCE_LIMITS resolves to true (default false; `auto` applies them only when the host
+# has less than 12 GB RAM). See base.limits.docker-compose.yml for why it can be opted out of
+# (Docker-in-Docker on cgroup v2 / threaded-mode memory controller).
+services:
+  apm-server:
+    mem_limit: 640m
+    memswap_limit: 640m
+
+  node-apm-app:
+    mem_limit: 128m
+    memswap_limit: 128m

environments/elk-ror/base.docker-compose.yml

@@ -11,16 +11,14 @@ services:
     ports:
       - "9200:9200"
       - "5005:5005"
-    mem_limit: 3g
-    memswap_limit: 3g
     environment:
       - ES_VERSION=$ES_VERSION
       - cluster.name=elk-ror
       - node.name=elk-ror-es-node
       - discovery.type=single-node
       - bootstrap.memory_lock=true
       # for a sake of debugging and profiling
-      - "ES_JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=0 -Dcom.readonlyrest.settings.loading.delay=0s -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.rmi.port=9010 -Djava.rmi.server.hostname=127.0.0.1"
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=0 -Dcom.readonlyrest.settings.loading.delay=0s -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.rmi.port=9010 -Djava.rmi.server.hostname=127.0.0.1"
       # for a sake of ROR settings marked as "<-- related to ECK environment -->"
       - KIBANA_SERVICE_ACCOUNT_TOKEN=UNUSED
       - INTERNAL_PROBE_PASS=UNUSED
@@ -51,8 +49,6 @@ services:
     depends_on:
       es-ror:
         condition: service_healthy
-    mem_limit: 1g
-    memswap_limit: 1g
     deploy:
       mode: replicated
       replicas: 2
@@ -64,7 +60,11 @@ services:
       interval: 10s
       timeout: 10s
       retries: 30
-      start_period: 60s
+      # Two Kibana replicas + ES + APM boot concurrently. On CPU-small CI agents
+      # (e.g. the 2-vCPU Azure host) a replica needs >60s to reach /api/status, so
+      # `docker compose up --wait` was tearing the stack down before tests ran.
+      # start_period only extends the startup grace; it's free on a fast boot.
+      start_period: 180s
     networks:
       - es-ror-network
     ulimits:
@@ -81,8 +81,6 @@ services:
         condition: service_healthy
     ports:
       - "5601:443"
-    mem_limit: 256m
-    memswap_limit: 256m
     healthcheck:
       test: [ "CMD", "curl", "-k", "--fail", "-u", "kibana:kibana", "https://localhost:443/api/status" ]
       interval: 10s

environments/elk-ror/base.limits.docker-compose.yml

@@ -0,0 +1,20 @@
+# Resource-limit overlay for the base cluster, kept separate so it can be opted out of.
+# Applied on top of base.docker-compose.yml by start.sh when APPLY_RESOURCE_LIMITS resolves to true
+# (default false; `auto` applies them only when the host has less than 12 GB RAM).
+# Needed on small CI agents where docker runs on the host directly.
+# It is intentionally NOT applied when the e2e flow runs inside a Docker-in-Docker container on a
+# cgroup v2 host: there BuildKit puts /sys/fs/cgroup/docker into "threaded" mode, and a threaded
+# cgroup can't enable the memory (domain) controller, so a mem_limit makes the container fail to
+# start. Heaps are pinned in base.docker-compose.yml, so dropping the limits there is safe.
+services:
+  es-ror:
+    mem_limit: 2g
+    memswap_limit: 2g
+
+  kbn-ror:
+    mem_limit: 1g
+    memswap_limit: 1g
+
+  kbn-proxy:
+    mem_limit: 256m
+    memswap_limit: 256m

environments/elk-ror/start.sh

@@ -146,14 +146,48 @@ docker pull "$ROR_KBN_IMAGE" || { echo "Failed to pull Kibana image: $ROR_KBN_IM
 echo "Bootstrapping the docker-based environment ..."
 echo "Cluster type: $CLUSTER_TYPE"
 
+# Resource limits live in separate *.limits.docker-compose.yml overlays so they're opt-in.
+# Disabled by default — safe for Docker-in-Docker on a cgroup v2 host, where a threaded
+# /sys/fs/cgroup/docker can't enable the memory controller and a mem_limit would prevent containers
+# from starting. Set APPLY_RESOURCE_LIMITS=true to apply them; needed on small host-docker agents to avoid OOM.
+#
+# Accepts true, false or auto. `auto` applies them only when the host has less than 12 GB RAM.
+APPLY_RESOURCE_LIMITS="${APPLY_RESOURCE_LIMITS:-false}"
+
+AUTO_LIMITS_MEM_THRESHOLD_KB=12000000
+
+# Resolves APPLY_RESOURCE_LIMITS=auto to true or false, and records why in
+# APPLY_RESOURCE_LIMITS_REASON. Leaves an explicit true/false untouched.
+resolve_auto_resource_limits() {
+  [[ "$APPLY_RESOURCE_LIMITS" == "auto" ]] || return 0
+
+  local mem_kb
+  mem_kb="$(awk '/^MemTotal:/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)"
+
+  # mem_kb is 0 where there is no /proc/meminfo (macOS); treat that as "cannot tell" and leave the
+  # limits off, matching the default.
+  if [[ "$mem_kb" -gt 0 && "$mem_kb" -lt "$AUTO_LIMITS_MEM_THRESHOLD_KB" ]]; then
+    APPLY_RESOURCE_LIMITS="true"
+  else
+    APPLY_RESOURCE_LIMITS="false"
+  fi
+
+  APPLY_RESOURCE_LIMITS_REASON=" (auto: host has ${mem_kb} kB, threshold is ${AUTO_LIMITS_MEM_THRESHOLD_KB} kB)"
+}
+
+resolve_auto_resource_limits
+
 # Set compose files based on cluster type
 if [[ "$CLUSTER_TYPE" == "base" ]]; then
   DOCKER_COMPOSE_FILES="-f base.docker-compose.yml"
+  [[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f base.limits.docker-compose.yml"
   echo "Starting base cluster (Elasticsearch + Kibana + ReadonlyREST)"
 elif [[ "$CLUSTER_TYPE" == "apm" ]]; then
   DOCKER_COMPOSE_FILES="-f base.docker-compose.yml -f apm.docker-compose.yml"
+  [[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f base.limits.docker-compose.yml -f apm.limits.docker-compose.yml"
   echo "Starting cluster with APM (Elasticsearch + Kibana + ReadonlyREST + APM Server + APM App)"
 fi
+echo "Resource limits: $([[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && echo "applied" || echo "disabled")${APPLY_RESOURCE_LIMITS_REASON:-}"
 
 if ! docker compose $DOCKER_COMPOSE_FILES config > /dev/null; then
   echo "Cannot validate docker compose configuration."