diff --git a/.github/workflows/all-e2e-tests.yml b/.github/workflows/all-e2e-tests.yml index fb1e412b..14c826d6 100644 --- a/.github/workflows/all-e2e-tests.yml +++ b/.github/workflows/all-e2e-tests.yml @@ -7,6 +7,9 @@ on: pull_request: types: [opened, synchronize, reopened] +env: + APPLY_RESOURCE_LIMITS: "auto" + jobs: # ========================================== # E2E TESTS - MASTER BRANCH diff --git a/.github/workflows/targeted-e2e-tests.yml b/.github/workflows/targeted-e2e-tests.yml index 2fb686bd..fa686f21 100644 --- a/.github/workflows/targeted-e2e-tests.yml +++ b/.github/workflows/targeted-e2e-tests.yml @@ -25,6 +25,9 @@ on: default: 'prod' type: string +env: + APPLY_RESOURCE_LIMITS: "auto" + jobs: e2e-tests: name: "🔬 E2E Tests" diff --git a/environments/elk-ror/apm.docker-compose.yml b/environments/elk-ror/apm.docker-compose.yml index 71ce6dff..2a0c7f98 100644 --- a/environments/elk-ror/apm.docker-compose.yml +++ b/environments/elk-ror/apm.docker-compose.yml @@ -15,8 +15,6 @@ services: condition: service_healthy networks: - es-ror-network - mem_limit: 1g - memswap_limit: 1g healthcheck: test: [ "CMD", "curl", "-k", "--fail", "https://localhost:8200" ] interval: 10s @@ -40,8 +38,6 @@ services: - ./certs:/example-app/certs:z networks: - es-ror-network - mem_limit: 128m - memswap_limit: 128m healthcheck: test: [ "CMD", "curl", "-k", "--http1.1", "--fail", "https://apm-server:8200" ] interval: 10s diff --git a/environments/elk-ror/apm.limits.docker-compose.yml b/environments/elk-ror/apm.limits.docker-compose.yml new file mode 100644 index 00000000..e14c419a --- /dev/null +++ b/environments/elk-ror/apm.limits.docker-compose.yml @@ -0,0 +1,12 @@ +# Resource-limit overlay for the APM services. Applied on top of apm.docker-compose.yml by start.sh +# when APPLY_RESOURCE_LIMITS resolves to true (default false; `auto` applies them only when the host +# has less than 12 GB RAM). See base.limits.docker-compose.yml for why it can be opted out of +# (Docker-in-Docker on cgroup v2 / threaded-mode memory controller). +services: + apm-server: + mem_limit: 640m + memswap_limit: 640m + + node-apm-app: + mem_limit: 128m + memswap_limit: 128m diff --git a/environments/elk-ror/base.docker-compose.yml b/environments/elk-ror/base.docker-compose.yml index 060ce5c7..6f2c87c0 100644 --- a/environments/elk-ror/base.docker-compose.yml +++ b/environments/elk-ror/base.docker-compose.yml @@ -11,8 +11,6 @@ services: ports: - "9200:9200" - "5005:5005" - mem_limit: 3g - memswap_limit: 3g environment: - ES_VERSION=$ES_VERSION - cluster.name=elk-ror @@ -20,7 +18,7 @@ services: - discovery.type=single-node - bootstrap.memory_lock=true # for a sake of debugging and profiling - - "ES_JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=0 -Dcom.readonlyrest.settings.loading.delay=0s -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.rmi.port=9010 -Djava.rmi.server.hostname=127.0.0.1" + - "ES_JAVA_OPTS=-Xms1g -Xmx1g -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 -Dcom.readonlyrest.settings.loading.attempts.count=0 -Dcom.readonlyrest.settings.loading.delay=0s -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=9010 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.rmi.port=9010 -Djava.rmi.server.hostname=127.0.0.1" # for a sake of ROR settings marked as "<-- related to ECK environment -->" - KIBANA_SERVICE_ACCOUNT_TOKEN=UNUSED - INTERNAL_PROBE_PASS=UNUSED @@ -51,8 +49,6 @@ services: depends_on: es-ror: condition: service_healthy - mem_limit: 1g - memswap_limit: 1g deploy: mode: replicated replicas: 2 @@ -64,7 +60,11 @@ services: interval: 10s timeout: 10s retries: 30 - start_period: 60s + # Two Kibana replicas + ES + APM boot concurrently. On CPU-small CI agents + # (e.g. the 2-vCPU Azure host) a replica needs >60s to reach /api/status, so + # `docker compose up --wait` was tearing the stack down before tests ran. + # start_period only extends the startup grace; it's free on a fast boot. + start_period: 180s networks: - es-ror-network ulimits: @@ -81,8 +81,6 @@ services: condition: service_healthy ports: - "5601:443" - mem_limit: 256m - memswap_limit: 256m healthcheck: test: [ "CMD", "curl", "-k", "--fail", "-u", "kibana:kibana", "https://localhost:443/api/status" ] interval: 10s diff --git a/environments/elk-ror/base.limits.docker-compose.yml b/environments/elk-ror/base.limits.docker-compose.yml new file mode 100644 index 00000000..dc81a16c --- /dev/null +++ b/environments/elk-ror/base.limits.docker-compose.yml @@ -0,0 +1,20 @@ +# Resource-limit overlay for the base cluster, kept separate so it can be opted out of. +# Applied on top of base.docker-compose.yml by start.sh when APPLY_RESOURCE_LIMITS resolves to true +# (default false; `auto` applies them only when the host has less than 12 GB RAM). +# Needed on small CI agents where docker runs on the host directly. +# It is intentionally NOT applied when the e2e flow runs inside a Docker-in-Docker container on a +# cgroup v2 host: there BuildKit puts /sys/fs/cgroup/docker into "threaded" mode, and a threaded +# cgroup can't enable the memory (domain) controller, so a mem_limit makes the container fail to +# start. Heaps are pinned in base.docker-compose.yml, so dropping the limits there is safe. +services: + es-ror: + mem_limit: 2g + memswap_limit: 2g + + kbn-ror: + mem_limit: 1g + memswap_limit: 1g + + kbn-proxy: + mem_limit: 256m + memswap_limit: 256m diff --git a/environments/elk-ror/start.sh b/environments/elk-ror/start.sh index 60926142..209511eb 100755 --- a/environments/elk-ror/start.sh +++ b/environments/elk-ror/start.sh @@ -146,14 +146,48 @@ docker pull "$ROR_KBN_IMAGE" || { echo "Failed to pull Kibana image: $ROR_KBN_IM echo "Bootstrapping the docker-based environment ..." echo "Cluster type: $CLUSTER_TYPE" +# Resource limits live in separate *.limits.docker-compose.yml overlays so they're opt-in. +# Disabled by default — safe for Docker-in-Docker on a cgroup v2 host, where a threaded +# /sys/fs/cgroup/docker can't enable the memory controller and a mem_limit would prevent containers +# from starting. Set APPLY_RESOURCE_LIMITS=true to apply them; needed on small host-docker agents to avoid OOM. +# +# Accepts true, false or auto. `auto` applies them only when the host has less than 12 GB RAM. +APPLY_RESOURCE_LIMITS="${APPLY_RESOURCE_LIMITS:-false}" + +AUTO_LIMITS_MEM_THRESHOLD_KB=12000000 + +# Resolves APPLY_RESOURCE_LIMITS=auto to true or false, and records why in +# APPLY_RESOURCE_LIMITS_REASON. Leaves an explicit true/false untouched. +resolve_auto_resource_limits() { + [[ "$APPLY_RESOURCE_LIMITS" == "auto" ]] || return 0 + + local mem_kb + mem_kb="$(awk '/^MemTotal:/ {print $2}' /proc/meminfo 2>/dev/null || echo 0)" + + # mem_kb is 0 where there is no /proc/meminfo (macOS); treat that as "cannot tell" and leave the + # limits off, matching the default. + if [[ "$mem_kb" -gt 0 && "$mem_kb" -lt "$AUTO_LIMITS_MEM_THRESHOLD_KB" ]]; then + APPLY_RESOURCE_LIMITS="true" + else + APPLY_RESOURCE_LIMITS="false" + fi + + APPLY_RESOURCE_LIMITS_REASON=" (auto: host has ${mem_kb} kB, threshold is ${AUTO_LIMITS_MEM_THRESHOLD_KB} kB)" +} + +resolve_auto_resource_limits + # Set compose files based on cluster type if [[ "$CLUSTER_TYPE" == "base" ]]; then DOCKER_COMPOSE_FILES="-f base.docker-compose.yml" + [[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f base.limits.docker-compose.yml" echo "Starting base cluster (Elasticsearch + Kibana + ReadonlyREST)" elif [[ "$CLUSTER_TYPE" == "apm" ]]; then DOCKER_COMPOSE_FILES="-f base.docker-compose.yml -f apm.docker-compose.yml" + [[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && DOCKER_COMPOSE_FILES="$DOCKER_COMPOSE_FILES -f base.limits.docker-compose.yml -f apm.limits.docker-compose.yml" echo "Starting cluster with APM (Elasticsearch + Kibana + ReadonlyREST + APM Server + APM App)" fi +echo "Resource limits: $([[ "$APPLY_RESOURCE_LIMITS" == "true" ]] && echo "applied" || echo "disabled")${APPLY_RESOURCE_LIMITS_REASON:-}" if ! docker compose $DOCKER_COMPOSE_FILES config > /dev/null; then echo "Cannot validate docker compose configuration."