From e60cbd4582a49c8c20109f2d93fe206c79dad821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Mon, 15 Jun 2026 13:50:18 +0200 Subject: [PATCH 1/7] improvements --- environments/elk-ror/apm.docker-compose.yml | 4 ++-- environments/elk-ror/base.docker-compose.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/environments/elk-ror/apm.docker-compose.yml b/environments/elk-ror/apm.docker-compose.yml index 71ce6dff..037e98c8 100644 --- a/environments/elk-ror/apm.docker-compose.yml +++ b/environments/elk-ror/apm.docker-compose.yml @@ -15,8 +15,8 @@ services: condition: service_healthy networks: - es-ror-network - mem_limit: 1g - memswap_limit: 1g + mem_limit: 640m + memswap_limit: 640m healthcheck: test: [ "CMD", "curl", "-k", "--fail", "https://localhost:8200" ] interval: 10s diff --git a/environments/elk-ror/base.docker-compose.yml b/environments/elk-ror/base.docker-compose.yml index 060ce5c7..a411394f 100644 --- a/environments/elk-ror/base.docker-compose.yml +++ b/environments/elk-ror/base.docker-compose.yml @@ -11,8 +11,8 @@ services: ports: - "9200:9200" - "5005:5005" - mem_limit: 3g - memswap_limit: 3g + mem_limit: 2g + memswap_limit: 2g environment: - ES_VERSION=$ES_VERSION - cluster.name=elk-ror @@ -20,7 +20,7 @@ services: - discovery.type=single-node - bootstrap.memory_lock=true # for a sake of debugging and profiling - - "ES_JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=0 -Dcom.readonlyrest.settings.loading.delay=0s -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.rmi.port=9010 -Djava.rmi.server.hostname=127.0.0.1" + - "ES_JAVA_OPTS=-Xms1g -Xmx1g -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=0 -Dcom.readonlyrest.settings.loading.delay=0s -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.rmi.port=9010 -Djava.rmi.server.hostname=127.0.0.1" # for a sake of ROR settings marked as "<-- related to ECK environment -->" - KIBANA_SERVICE_ACCOUNT_TOKEN=UNUSED - INTERNAL_PROBE_PASS=UNUSED From 9b0f843d0cea5379ceb4fdd1ae4325466d370dbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Mon, 15 Jun 2026 14:41:07 +0200 Subject: [PATCH 2/7] improvements --- environments/elk-ror/base.docker-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/environments/elk-ror/base.docker-compose.yml b/environments/elk-ror/base.docker-compose.yml index a411394f..fc672f03 100644 --- a/environments/elk-ror/base.docker-compose.yml +++ b/environments/elk-ror/base.docker-compose.yml @@ -64,7 +64,11 @@ services: interval: 10s timeout: 10s retries: 30 - start_period: 60s + # Two Kibana replicas + ES + APM boot concurrently. On CPU-small CI agents + # (e.g. the 2-vCPU Azure host) a replica needs >60s to reach /api/status, so + # `docker compose up --wait` was tearing the stack down before tests ran. + # start_period only extends the startup grace; it's free on a fast boot. + start_period: 180s networks: - es-ror-network ulimits: From 57bf92d4d7ce8e7fafbc7facb631500a9d160b64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Tue, 16 Jun 2026 14:16:48 +0200 Subject: [PATCH 3/7] improvements --- environments/elk-ror/apm.docker-compose.yml | 4 ---- .../elk-ror/apm.limits.docker-compose.yml | 11 +++++++++++ environments/elk-ror/base.docker-compose.yml | 6 ------ .../elk-ror/base.limits.docker-compose.yml | 19 +++++++++++++++++++ environments/elk-ror/start.sh | 9 +++++++++ 5 files changed, 39 insertions(+), 10 deletions(-) create mode 100644 environments/elk-ror/apm.limits.docker-compose.yml create mode 100644 environments/elk-ror/base.limits.docker-compose.yml diff --git a/environments/elk-ror/apm.docker-compose.yml b/environments/elk-ror/apm.docker-compose.yml index 037e98c8..2a0c7f98 100644 --- a/environments/elk-ror/apm.docker-compose.yml +++ b/environments/elk-ror/apm.docker-compose.yml @@ -15,8 +15,6 @@ services: condition: service_healthy networks: - es-ror-network - mem_limit: 640m - memswap_limit: 640m healthcheck: test: [ "CMD", "curl", "-k", "--fail", "https://localhost:8200" ] interval: 10s @@ -40,8 +38,6 @@ services: - ./certs:/example-app/certs:z networks: - es-ror-network - mem_limit: 128m - memswap_limit: 128m healthcheck: test: [ "CMD", "curl", "-k", "--http1.1", "--fail", "https://apm-server:8200" ] interval: 10s diff --git a/environments/elk-ror/apm.limits.docker-compose.yml b/environments/elk-ror/apm.limits.docker-compose.yml new file mode 100644 index 00000000..5a71a8e2 --- /dev/null +++ b/environments/elk-ror/apm.limits.docker-compose.yml @@ -0,0 +1,11 @@ +# Resource-limit overlay for the APM services. Applied on top of apm.docker-compose.yml by start.sh +# when APPLY_RESOURCE_LIMITS=true (default). See base.limits.docker-compose.yml for why it can be +# opted out of (Docker-in-Docker on cgroup v2 / threaded-mode memory controller). +services: + apm-server: + mem_limit: 640m + memswap_limit: 640m + + node-apm-app: + mem_limit: 128m + memswap_limit: 128m diff --git a/environments/elk-ror/base.docker-compose.yml b/environments/elk-ror/base.docker-compose.yml index fc672f03..6f2c87c0 100644 --- a/environments/elk-ror/base.docker-compose.yml +++ b/environments/elk-ror/base.docker-compose.yml @@ -11,8 +11,6 @@ services: ports: - "9200:9200" - "5005:5005" - mem_limit: 2g - memswap_limit: 2g environment: - ES_VERSION=$ES_VERSION - cluster.name=elk-ror @@ -51,8 +49,6 @@ services: depends_on: es-ror: condition: service_healthy - mem_limit: 1g - memswap_limit: 1g deploy: mode: replicated replicas: 2 @@ -85,8 +81,6 @@ services: condition: service_healthy ports: - "5601:443" - mem_limit: 256m - memswap_limit: 256m healthcheck: test: [ "CMD", "curl", "-k", "--fail", "-u", "kibana:kibana", "https://localhost:443/api/status" ] interval: 10s diff --git a/environments/elk-ror/base.limits.docker-compose.yml b/environments/elk-ror/base.limits.docker-compose.yml new file mode 100644 index 00000000..d0a80015 --- /dev/null +++ b/environments/elk-ror/base.limits.docker-compose.yml @@ -0,0 +1,19 @@ +# Resource-limit overlay for the base cluster, kept separate so it can be opted out of. +# Applied on top of base.docker-compose.yml by start.sh when APPLY_RESOURCE_LIMITS=true (default). +# Needed on small CI agents (e.g. the ~7.9 GB Azure host) where docker runs on the host directly. +# It is intentionally NOT applied when the e2e flow runs inside a Docker-in-Docker container on a +# cgroup v2 host: there BuildKit puts /sys/fs/cgroup/docker into "threaded" mode, and a threaded +# cgroup can't enable the memory (domain) controller, so a mem_limit makes the container fail to +# start. Heaps are pinned in base.docker-compose.yml, so dropping the limits there is safe. +services: + es-ror: + mem_limit: 2g + memswap_limit: 2g + + kbn-ror: + mem_limit: 1g + memswap_limit: 1g + + kbn-proxy: + mem_limit: 256m + memswap_limit: 256m diff --git a/environments/elk-ror/start.sh b/environments/elk-ror/start.sh index 60926142..5e7240e4 100755 --- a/environments/elk-ror/start.sh +++ b/environments/elk-ror/start.sh @@ -146,14 +146,23 @@ docker pull "$ROR_KBN_IMAGE" || { echo "Failed to pull Kibana image: $ROR_KBN_IM echo "Bootstrapping the docker-based environment ..." echo "Cluster type: $CLUSTER_TYPE" +# Resource limits live in separate *.limits.docker-compose.yml overlays so they can be opted out of. +# Applied by default; set APPLY_RESOURCE_LIMITS=false to skip them (e.g. when running inside a +# Docker-in-Docker container on a cgroup v2 host, where a threaded /sys/fs/cgroup/docker can't enable +# the memory controller and a mem_limit would prevent containers from starting). +APPLY_RESOURCE_LIMITS="${APPLY_RESOURCE_LIMITS:-true}" + # Set compose files based on cluster type if [[ "$CLUSTER_TYPE" == "base" ]]; then DOCKER_COMPOSE_FILES="-f base.docker-compose.yml" + [[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f base.limits.docker-compose.yml" echo "Starting base cluster (Elasticsearch + Kibana + ReadonlyREST)" elif [[ "$CLUSTER_TYPE" == "apm" ]]; then DOCKER_COMPOSE_FILES="-f base.docker-compose.yml -f apm.docker-compose.yml" + [[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f base.limits.docker-compose.yml -f apm.limits.docker-compose.yml" echo "Starting cluster with APM (Elasticsearch + Kibana + ReadonlyREST + APM Server + APM App)" fi +echo "Resource limits: $([[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && echo "applied" || echo "disabled (APPLY_RESOURCE_LIMITS=$APPLY_RESOURCE_LIMITS)")" if ! docker compose $DOCKER_COMPOSE_FILES config > /dev/null; then echo "Cannot validate docker compose configuration." From 408d19a5e262ca1ee29a5e8729b9f1c50555d329 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Wed, 17 Jun 2026 14:26:23 +0200 Subject: [PATCH 4/7] improvements --- environments/elk-ror/start.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/environments/elk-ror/start.sh b/environments/elk-ror/start.sh index 5e7240e4..62281947 100755 --- a/environments/elk-ror/start.sh +++ b/environments/elk-ror/start.sh @@ -146,11 +146,12 @@ docker pull "$ROR_KBN_IMAGE" || { echo "Failed to pull Kibana image: $ROR_KBN_IM echo "Bootstrapping the docker-based environment ..." echo "Cluster type: $CLUSTER_TYPE" -# Resource limits live in separate *.limits.docker-compose.yml overlays so they can be opted out of. -# Applied by default; set APPLY_RESOURCE_LIMITS=false to skip them (e.g. when running inside a -# Docker-in-Docker container on a cgroup v2 host, where a threaded /sys/fs/cgroup/docker can't enable -# the memory controller and a mem_limit would prevent containers from starting). -APPLY_RESOURCE_LIMITS="${APPLY_RESOURCE_LIMITS:-true}" +# Resource limits live in separate *.limits.docker-compose.yml overlays so they're opt-in. +# Disabled by default — safe for Docker-in-Docker on a cgroup v2 host, where a threaded +# /sys/fs/cgroup/docker can't enable the memory controller and a mem_limit would prevent containers +# from starting. Set APPLY_RESOURCE_LIMITS=true to apply them; needed on small host-docker agents +# (e.g. the ~7.9 GB Azure host) to avoid OOM. +APPLY_RESOURCE_LIMITS="${APPLY_RESOURCE_LIMITS:-false}" # Set compose files based on cluster type if [[ "$CLUSTER_TYPE" == "base" ]]; then From 279c22f55ca7324da3be7ed9e5c81816ee54ef16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Thu, 18 Jun 2026 13:09:04 +0200 Subject: [PATCH 5/7] improvements --- .github/workflows/all-e2e-tests.yml | 7 +++++++ .github/workflows/targeted-e2e-tests.yml | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/.github/workflows/all-e2e-tests.yml b/.github/workflows/all-e2e-tests.yml index adf8b459..37d35814 100644 --- a/.github/workflows/all-e2e-tests.yml +++ b/.github/workflows/all-e2e-tests.yml @@ -7,6 +7,13 @@ on: pull_request: types: [opened, synchronize, reopened] +env: + # These runs use docker directly on the GitHub-hosted runner (host docker, not nested), so the + # compose *.limits.docker-compose.yml overlays apply cleanly and protect the ~16 GB runner from OOM. + # start.sh leaves resource limits off by default (safe for the ROR KBN Docker-in-Docker flow); + # opt in here. (eck-env runs don't use docker-compose limits, so this is a no-op for them.) + APPLY_RESOURCE_LIMITS: "true" + jobs: # ========================================== # E2E TESTS - MASTER BRANCH diff --git a/.github/workflows/targeted-e2e-tests.yml b/.github/workflows/targeted-e2e-tests.yml index 2fb686bd..ed582930 100644 --- a/.github/workflows/targeted-e2e-tests.yml +++ b/.github/workflows/targeted-e2e-tests.yml @@ -25,6 +25,12 @@ on: default: 'prod' type: string +env: + # Runs use docker directly on the GitHub-hosted runner (host docker, not nested), so the compose + # *.limits.docker-compose.yml overlays apply cleanly and protect the runner from OOM. start.sh + # leaves resource limits off by default (safe for the ROR KBN Docker-in-Docker flow); opt in here. + APPLY_RESOURCE_LIMITS: "true" + jobs: e2e-tests: name: "🔬 E2E Tests" From c2a4e093e430350232bd462daa1fffb6fb7cfc02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Wed, 29 Jul 2026 17:19:57 +0200 Subject: [PATCH 6/7] wip --- .github/workflows/all-e2e-tests.yml | 6 +--- .github/workflows/targeted-e2e-tests.yml | 5 +--- .../elk-ror/base.limits.docker-compose.yml | 2 +- environments/elk-ror/start.sh | 30 +++++++++++++++++-- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/.github/workflows/all-e2e-tests.yml b/.github/workflows/all-e2e-tests.yml index c4c2a7d3..14c826d6 100644 --- a/.github/workflows/all-e2e-tests.yml +++ b/.github/workflows/all-e2e-tests.yml @@ -8,11 +8,7 @@ on: types: [opened, synchronize, reopened] env: - # These runs use docker directly on the GitHub-hosted runner (host docker, not nested), so the - # compose *.limits.docker-compose.yml overlays apply cleanly and protect the ~16 GB runner from OOM. - # start.sh leaves resource limits off by default (safe for the ROR KBN Docker-in-Docker flow); - # opt in here. (eck-env runs don't use docker-compose limits, so this is a no-op for them.) - APPLY_RESOURCE_LIMITS: "true" + APPLY_RESOURCE_LIMITS: "auto" jobs: # ========================================== diff --git a/.github/workflows/targeted-e2e-tests.yml b/.github/workflows/targeted-e2e-tests.yml index ed582930..fa686f21 100644 --- a/.github/workflows/targeted-e2e-tests.yml +++ b/.github/workflows/targeted-e2e-tests.yml @@ -26,10 +26,7 @@ on: type: string env: - # Runs use docker directly on the GitHub-hosted runner (host docker, not nested), so the compose - # *.limits.docker-compose.yml overlays apply cleanly and protect the runner from OOM. start.sh - # leaves resource limits off by default (safe for the ROR KBN Docker-in-Docker flow); opt in here. - APPLY_RESOURCE_LIMITS: "true" + APPLY_RESOURCE_LIMITS: "auto" jobs: e2e-tests: diff --git a/environments/elk-ror/base.limits.docker-compose.yml b/environments/elk-ror/base.limits.docker-compose.yml index d0a80015..68918f0e 100644 --- a/environments/elk-ror/base.limits.docker-compose.yml +++ b/environments/elk-ror/base.limits.docker-compose.yml @@ -1,6 +1,6 @@ # Resource-limit overlay for the base cluster, kept separate so it can be opted out of. # Applied on top of base.docker-compose.yml by start.sh when APPLY_RESOURCE_LIMITS=true (default). -# Needed on small CI agents (e.g. the ~7.9 GB Azure host) where docker runs on the host directly. +# Needed on small CI agents where docker runs on the host directly. # It is intentionally NOT applied when the e2e flow runs inside a Docker-in-Docker container on a # cgroup v2 host: there BuildKit puts /sys/fs/cgroup/docker into "threaded" mode, and a threaded # cgroup can't enable the memory (domain) controller, so a mem_limit makes the container fail to diff --git a/environments/elk-ror/start.sh b/environments/elk-ror/start.sh index 62281947..209511eb 100755 --- a/environments/elk-ror/start.sh +++ b/environments/elk-ror/start.sh @@ -149,10 +149,34 @@ echo "Cluster type: $CLUSTER_TYPE" # Resource limits live in separate *.limits.docker-compose.yml overlays so they're opt-in. # Disabled by default — safe for Docker-in-Docker on a cgroup v2 host, where a threaded # /sys/fs/cgroup/docker can't enable the memory controller and a mem_limit would prevent containers -# from starting. Set APPLY_RESOURCE_LIMITS=true to apply them; needed on small host-docker agents -# (e.g. the ~7.9 GB Azure host) to avoid OOM. +# from starting. Set APPLY_RESOURCE_LIMITS=true to apply them; needed on small host-docker agents to avoid OOM. +# +# Accepts true, false or auto. `auto` applies them only when the host has less than 12 GB RAM. APPLY_RESOURCE_LIMITS="${APPLY_RESOURCE_LIMITS:-false}" +AUTO_LIMITS_MEM_THRESHOLD_KB=12000000 + +# Resolves APPLY_RESOURCE_LIMITS=auto to true or false, and records why in +# APPLY_RESOURCE_LIMITS_REASON. Leaves an explicit true/false untouched. +resolve_auto_resource_limits() { + [[ "$APPLY_RESOURCE_LIMITS" == "auto" ]] || return 0 + + local mem_kb + mem_kb="$(awk '/^MemTotal:/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)" + + # mem_kb is 0 where there is no /proc/meminfo (macOS); treat that as "cannot tell" and leave the + # limits off, matching the default. + if [[ "$mem_kb" -gt 0 && "$mem_kb" -lt "$AUTO_LIMITS_MEM_THRESHOLD_KB" ]]; then + APPLY_RESOURCE_LIMITS="true" + else + APPLY_RESOURCE_LIMITS="false" + fi + + APPLY_RESOURCE_LIMITS_REASON=" (auto: host has ${mem_kb} kB, threshold is ${AUTO_LIMITS_MEM_THRESHOLD_KB} kB)" +} + +resolve_auto_resource_limits + # Set compose files based on cluster type if [[ "$CLUSTER_TYPE" == "base" ]]; then DOCKER_COMPOSE_FILES="-f base.docker-compose.yml" @@ -163,7 +187,7 @@ elif [[ "$CLUSTER_TYPE" == "apm" ]]; then [[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f base.limits.docker-compose.yml -f apm.limits.docker-compose.yml" echo "Starting cluster with APM (Elasticsearch + Kibana + ReadonlyREST + APM Server + APM App)" fi -echo "Resource limits: $([[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && echo "applied" || echo "disabled (APPLY_RESOURCE_LIMITS=$APPLY_RESOURCE_LIMITS)")" +echo "Resource limits: $([[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && echo "applied" || echo "disabled")${APPLY_RESOURCE_LIMITS_REASON:-}" if ! docker compose $DOCKER_COMPOSE_FILES config > /dev/null; then echo "Cannot validate docker compose configuration." From 855603324a245ef01d7f2756deb71563b4980736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Ko=C5=82odziejczyk?= Date: Wed, 29 Jul 2026 17:38:52 +0200 Subject: [PATCH 7/7] wip --- environments/elk-ror/apm.limits.docker-compose.yml | 5 +++-- environments/elk-ror/base.limits.docker-compose.yml | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/environments/elk-ror/apm.limits.docker-compose.yml b/environments/elk-ror/apm.limits.docker-compose.yml index 5a71a8e2..e14c419a 100644 --- a/environments/elk-ror/apm.limits.docker-compose.yml +++ b/environments/elk-ror/apm.limits.docker-compose.yml @@ -1,6 +1,7 @@ # Resource-limit overlay for the APM services. Applied on top of apm.docker-compose.yml by start.sh -# when APPLY_RESOURCE_LIMITS=true (default). See base.limits.docker-compose.yml for why it can be -# opted out of (Docker-in-Docker on cgroup v2 / threaded-mode memory controller). +# when APPLY_RESOURCE_LIMITS resolves to true (default false; `auto` applies them only when the host +# has less than 12 GB RAM). See base.limits.docker-compose.yml for why it can be opted out of +# (Docker-in-Docker on cgroup v2 / threaded-mode memory controller). services: apm-server: mem_limit: 640m diff --git a/environments/elk-ror/base.limits.docker-compose.yml b/environments/elk-ror/base.limits.docker-compose.yml index 68918f0e..dc81a16c 100644 --- a/environments/elk-ror/base.limits.docker-compose.yml +++ b/environments/elk-ror/base.limits.docker-compose.yml @@ -1,5 +1,6 @@ # Resource-limit overlay for the base cluster, kept separate so it can be opted out of. -# Applied on top of base.docker-compose.yml by start.sh when APPLY_RESOURCE_LIMITS=true (default). +# Applied on top of base.docker-compose.yml by start.sh when APPLY_RESOURCE_LIMITS resolves to true +# (default false; `auto` applies them only when the host has less than 12 GB RAM). # Needed on small CI agents where docker runs on the host directly. # It is intentionally NOT applied when the e2e flow runs inside a Docker-in-Docker container on a # cgroup v2 host: there BuildKit puts /sys/fs/cgroup/docker into "threaded" mode, and a threaded