diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 00000000..4c73058e --- /dev/null +++ b/ci/README.md @@ -0,0 +1,73 @@ +# CI memory measurement + +This directory holds the tooling we use to right-size the molecule +container memory limits (`memory_mb` in each `molecule//molecule.yml`) +against real measurements instead of guesswork. + +## Why anon, not usage + +The memory gate (`Oddly/incus-memory-gate`, wired into the workflows by +the gate PR) admits jobs against the sum of committed `limits.memory`, +so an oversized limit costs concurrency rather than safety. The +question "is this limit too high" therefore comes down to how much +memory a container genuinely needs. + +The trap is that a container's total usage climbs toward its limit +because the kernel fills the spare room with reclaimable page cache. +Reading `memory.peak` (or incus `usage_peak`) shows peak ≈ limit even +when the working set is far smaller, so it is useless for +right-sizing. The number that matters is **anon** — anonymous RSS, +which cannot be reclaimed and is what the OOM killer acts on. There is +no kernel high-water counter for anon alone, so we sample it. + +## anon-peak-sampler.sh + +Runs on the incus host (the runner LXC, `incus-ci`). Every second it +reads, for each running container, the `anon` line from the +container's **top-level** `lxc.payload.` cgroup `memory.stat` +(this is hierarchical, so it is the whole-container total), and keeps +the high-water mark per container across its whole life — a peak +survives the container being destroyed. It also records the limit, +peak cache alongside for context, and the cgroup `oom_kill` count. + +Read the top-level payload cgroup, not the container init PID's +cgroup: the init process sits in an `init.scope` leaf that reports a +couple of MiB, so resolving via `/proc//cgroup` badly under-reads. + +Deploy it as a systemd unit on the host so it survives ssh and session +drops: + +```ini +# /etc/systemd/system/anon-peak-sampler.service +[Unit] +Description=Per-container anon RSS peak sampler (incus-ci right-sizing) +[Service] +Environment=INTERVAL=1 +Environment=OUT=/root/mem-peaks +Environment=NDJSON=/root/mem-peaks/ts.ndjson +ExecStart=/usr/bin/bash /root/anon-peak-sampler.sh +Restart=on-failure +RestartSec=2 +[Install] +WantedBy=multi-user.target +``` + +```bash +systemctl daemon-reload && systemctl enable --now anon-peak-sampler +cat /root/mem-peaks/peaks.txt # live table, rewritten every tick +``` + +Outputs in `$OUT`: `peaks.txt` (human table, per scenario-class and +per container), `peaks.tsv` (persistent state, resumed on restart), +and `ts.ndjson` (per-tick time-series when `NDJSON` is set). + +## Interpreting the numbers + +Rank by anon%. A container sitting well under its limit on anon is a +trim candidate; one near its limit is correctly sized or tight and +must be left alone. Trim toward the cross-distro peak plus a margin, +never the average — the heaviest distro sets the floor. Leave real +IO/JVM working-set headroom on top: for Elasticsearch especially, the +page cache backs Lucene, so cutting to the anon floor invites cache +thrash and flaky converges even when it never OOMs. After any trim, +watch `oom_kill` and memory PSI on the next run before trusting it. diff --git a/ci/anon-peak-sampler.sh b/ci/anon-peak-sampler.sh new file mode 100755 index 00000000..8d684f84 --- /dev/null +++ b/ci/anon-peak-sampler.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# anon-peak-sampler.sh — per-container peak anonymous RSS (real memory, excl. page cache). +# +# Runs on the incus host (LXC 305 / incus-ci). Every $INTERVAL seconds it reads, +# for each running incus container, the *anon* line from the container's top-level +# cgroup memory.stat (hierarchical → whole-container total, excludes reclaimable +# page cache) and keeps the high-water mark per container across its whole life. +# Survives container churn: a container's peak is retained after it is destroyed. +# +# Why sample instead of reading a counter: cgroup v2 exposes memory.peak, but that +# is usage INCLUDING page cache, which fills toward the limit without reflecting +# real need. There is no kernel high-water for anon alone, so we poll. +# +# Env: +# INTERVAL seconds between samples (default 1) +# OUT output dir (default /root/mem-peaks) +# NDJSON if set to a path, append one JSON line per container per tick +# (time-series; off by default to keep it light) +# +# Live view: cat $OUT/peaks.txt (rewritten every tick) +# Final view: send SIGTERM/SIGINT (pkill -f anon-peak-sampler) → prints summary +set -uo pipefail +INTERVAL="${INTERVAL:-1}" +OUT="${OUT:-/root/mem-peaks}" +NDJSON="${NDJSON:-}" +mkdir -p "$OUT" +STATE="$OUT/peaks.tsv" +TABLE="$OUT/peaks.txt" +touch "$STATE" + +declare -A PA PC LIM OOM SC # peakAnon, peakCache, limit, oom_kills, scenarioClass (MiB) + +# Resume prior peaks if the sampler is restarted. +while IFS=$'\t' read -r n pa lim pc oom sc; do + [ -n "${n:-}" ] && { PA[$n]=$pa; LIM[$n]=$lim; PC[$n]=$pc; OOM[$n]=$oom; SC[$n]=$sc; } +done < "$STATE" + +scenario_of(){ echo "$1" | sed -E 's/-(debian|rocky|rockylinux|ubuntu)[0-9]*.*$//'; } + +render(){ + : > "$STATE.tmp" + for n in "${!PA[@]}"; do + printf '%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$n" "${PA[$n]}" "${LIM[$n]:-0}" "${PC[$n]:-0}" "${OOM[$n]:-0}" "${SC[$n]:-}" + done > "$STATE.tmp" + mv "$STATE.tmp" "$STATE" + { + echo "# updated $(date -u +%FT%TZ) interval=${INTERVAL}s metric=peak anon RSS (real, excl. cache)" + echo + echo "== per scenario-class (max anon over all distro/release instances) ==" + printf '%-26s %10s %8s %6s %4s\n' scenario peakAnonMiB limitMiB anon% oom + awk -F'\t' '{c=$1; sub(/-(debian|rocky|rockylinux|ubuntu)[0-9]*.*$/,"",c); + a[c]=($2>a[c]?$2:a[c]); l[c]=($3>l[c]?$3:l[c]); o[c]+=$5} + END{for(s in a){p=(l[s]>0?int(a[s]*100/l[s]):0); printf "%-26s %10s %8s %5s%% %4s\n",s,a[s],l[s],p,o[s]}}' \ + "$STATE" | sort -k2 -rn + echo + echo "== per container instance ==" + printf '%-42s %10s %8s %6s %10s %4s\n' container peakAnonMiB limitMiB anon% peakCacheMiB oom + sort -t$'\t' -k2 -rn "$STATE" | while IFS=$'\t' read -r n pa lim pc oom sc; do + p=0; [ "${lim:-0}" -gt 0 ] && p=$(( pa*100/lim )) + printf '%-42s %10s %8s %5s%% %10s %4s\n' "$n" "$pa" "$lim" "$p" "$pc" "$oom" + done + } > "$TABLE" +} + +trap 'render; echo; echo "=== FINAL PEAK SUMMARY ==="; cat "$TABLE"; exit 0' INT TERM + +while true; do + ts=$(date -u +%FT%TZ) + while IFS= read -r d; do + [ -f "$d/memory.stat" ] || continue + n=${d##*/lxc.payload.} + anon=$(( $(awk '/^anon /{print $2; exit}' "$d/memory.stat" 2>/dev/null || echo 0) / 1048576 )) + cache=$(( $(awk '/^file /{print $2; exit}' "$d/memory.stat" 2>/dev/null || echo 0) / 1048576 )) + mx=$(cat "$d/memory.max" 2>/dev/null || echo max) + if [ "$mx" = max ]; then lim=0; else lim=$(( mx / 1048576 )); fi + oom=$(awk '/^oom_kill /{print $2; exit}' "$d/memory.events" 2>/dev/null || echo 0) + (( anon > ${PA[$n]:-0} )) && PA[$n]=$anon + (( cache > ${PC[$n]:-0} )) && PC[$n]=$cache + (( ${oom:-0} > ${OOM[$n]:-0} )) && OOM[$n]=$oom + LIM[$n]=$lim + SC[$n]=$(scenario_of "$n") + [ -n "$NDJSON" ] && printf '{"t":"%s","c":"%s","anon_mb":%s,"cache_mb":%s,"lim_mb":%s,"oom":%s}\n' \ + "$ts" "$n" "$anon" "$cache" "$lim" "$oom" >> "$NDJSON" + done < <(find /sys/fs/cgroup -type d -name 'lxc.payload.*' 2>/dev/null) + render + sleep "$INTERVAL" +done diff --git a/docs/superpowers/specs/2026-08-13-memory-right-sizing-design.md b/docs/superpowers/specs/2026-08-13-memory-right-sizing-design.md new file mode 100644 index 00000000..224bed1f --- /dev/null +++ b/docs/superpowers/specs/2026-08-13-memory-right-sizing-design.md @@ -0,0 +1,161 @@ +# Right-sizing the molecule container memory limits + +Date: 2026-08-13 +Status: instrumentation + proposal (no limit changes yet) + +## Context + +The memory gate (`Oddly/incus-memory-gate`, wired in by the gate PR) +admits jobs against the sum of committed `limits.memory` on the shared +incus-ci host. An oversized `memory_mb` in a scenario's `molecule.yml` +therefore costs concurrency: it reserves ledger capacity the container +never uses, so fewer jobs run at once than the host could safely +carry. This spec is about reclaiming that lost concurrency without +reintroducing the OOM kills the gate work just fixed. + +The measurement principle and the sampler that implements it are +described in `ci/README.md`: right-size against **anon** (irreducible +RSS), captured as a per-container high-water by sampling, never +against total usage or `memory.peak`, which fill with reclaimable page +cache and read near the limit regardless of real need. + +## Measured baseline (full-matrix max-load storm, 2026-08-13) + +Cross-distro peak anon over a six-distro, max-parallel-6 run of the +full-stack matrix. Limits shown as the effective MiB the kernel +enforces (`memory_mb` in MB is ~4.7% higher). Zero OOM kills across +the whole run. + +| node (scenario) | peak anon | limit | anon% | reading | +|-------------------------------------|-----------|--------|-------|---------------------------------| +| logstash node (logstash_elasticsearch) | 939 MiB | 2929 | 32% | clear headroom | +| ES node (logstash_elasticsearch) | 1690 MiB | 3906 | 43% | headroom | +| ES cluster node (elasticstack_default) | 5099 MiB | 9765 | 52% | moderate; cache-productive | +| ES node (es_kibana) | 2110 MiB | 4394 | 48% | moderate | +| ES node (cert_renewal) | 3865 MiB | 5859 | 65% | correctly sized | +| Kibana node (es_kibana) | 1761 MiB | 1953 | 90% | tight — do not touch | + +The peak climbs through converge: the cluster node read 38% early and +52% at its peak, which is why a single snapshot is not enough and the +sampler exists. + +## Proposal + +Land the instrumentation now; defer every limit change until we have +two or three instrumented runs, then trim in a data-backed follow-up. +Trimming off a single storm, without PSI to confirm the container is +not thrashing cache at the smaller size, is exactly the mistake this +work exists to avoid. + +When we do trim, the rules are: cut toward the cross-distro peak plus +a margin (the heaviest distro sets the floor, not the average); leave +IO/JVM working-set headroom on top of anon, more of it for +Elasticsearch nodes because the page cache backs Lucene; and watch +`oom_kill` and memory PSI on the run after each cut. + +First candidates, by current signal: + +- The logstash node in `logstash_elasticsearch` (939 MiB peak against a + ~3072 MB limit) is the clearest cut — it is a Logstash JVM, not + Lucene-backed, so it needs little cache headroom. +- The ES node in `logstash_elasticsearch` (1690 MiB) has room, but as + an ES node it keeps a larger cache margin. + +Explicitly leave alone: the Kibana node in `es_kibana` (90% of its 2 GB +limit — arguably already tight), `cert_renewal` (64-65%), and the +`elasticstack_default` cluster nodes (52% peak but heavily +cache-active during converge). + +## Second dataset: full role matrix (2026-08-14), and why it is not +## yet a safe trim basis + +A labelled run of the whole PR matrix, with the sampler and the +teardown hook both in place, captured 48 node-classes including the +config-only ES scenarios the storm missed (`elasticsearch_default`, +`_custom`, `_cert_content`, `_custom_certs`, `_no-security`, +`roles_calculation`, etc.). Zero OOM across all 115 teardown records. +Measured anon peaks against the current limits showed dramatic +apparent headroom: `roles_calculation` nodes at 4-5%, beats agents at +6-8%, `repos_default` at 9%, `kibana_default` at 22%, the config-only +ES nodes at 45-49%. + +Those numbers are a **lower bound, not a safe trim basis**, because a +`ci:run` label triggers the `pull_request` path and therefore the +reduced PR matrix. Distro coverage in the data was debian13 and +rockylinux10 almost exclusively (rockylinux9 only 3 records, none for +repos), and releases skewed to 9. The memory high-water for these +scenarios is the package-install phase, and EL9's dnf is the hog — +`repos_default` was OOM-killed ~960 MiB on rockylinux9 (the reason the +gate PR raised it to 2048), while rockylinux10's dnf peaked under +96 MiB. rockylinux9 is exactly the distro this run did not measure, so +the low anon peaks are steady-state on the lean distros, not the +install spike on the distro that actually OOMs. Trimming any +package-installing scenario toward these numbers would walk straight +back into that OOM class. + +## Third dataset: forced full distro run (2026-08-15), and the result + +Rather than wait for the nightlies, we dispatched the role and +config-ES workflows on the gate branch, which fall through to the full +seven-distro, both-release matrix on `workflow_dispatch`. The sampler +captured every distro (rockylinux9 165 rows, all others 165-236) and +both releases, with zero OOM across the run. + +The full data inverts the reduced-matrix picture. The memory +high-water is the package-install phase, and EL9's dnf plus the +Elasticsearch/beats install pushes almost every scenario far above the +lean-distro steady state: + +- `roles_calculation`: ~85 MiB on the lean distros, 946 MiB peak on + rockylinux9/r8. +- beats agents: ~130 MiB lean, 1073 MiB peak — trimming to 1024 would + have OOM'd. +- `repos_default`: 961 MiB peak on rockylinux9/r8, which is why the + gate PR raised it to 2048 and why it stays there. + +Against the current limits, the cross-distro peaks land at 40-98% for +all but one scenario. The reduced-matrix "headroom" was an artifact of +the missing distros, and acting on it would have walked straight back +into the OOM class the gate work removed. + +## The one change + +`kibana_default` is the only genuinely over-provisioned scenario: its +single ES+Kibana node peaked at 923 MiB anon across all seven distros +and both releases, against the 4096 MB default it inherited. It is cut +to 3072 MB. The cut is conservative on purpose — that node also holds +a 2.3-2.5 GB page-cache working set, so 3072 reclaims a gigabyte of +gate ledger while keeping the cache room that ES wants; 2048 would +squeeze it and risk cache thrash. Watch memory PSI on the first run at +3072 before considering anything tighter. Every other limit is left as +measured — the current sizing, including the gate PR's bumps, is +correct once rockylinux9 and release 8 are in the data. + +## Teardown telemetry (implemented) + +The sampler is host-side and only records while it happens to be +running. To get a guaranteed per-container record on every CI run, +`molecule/shared/destroy.yml` now runs a best-effort task before each +`incus delete` that reads the container's top-level payload cgroup: +`memory.peak` (cache-inclusive high-water, an upper bound), final +`anon`/`file` from `memory.stat`, the limit from `memory.max`, and +`oom_kill` from `memory.events`. It writes one NDJSON line per +container to an accumulating ledger on the host +(`/root/mem-peaks/teardown.ndjson`) and a copy under +`MOLECULE_EPHEMERAL_DIRECTORY`, and is guarded with +`failed_when: false` so telemetry can never fail a teardown. It +complements the sampler: the sampler gives the true anon peak, the +teardown record guarantees coverage and the definitive `oom_kill` +count for every scenario, including the config-only ones this storm +missed. + +This lands unvalidated against a live container (none were running +when it was written; the read logic was tested against a substitute +cgroup). The first labelled CI run on this branch is what confirms it +end to end; because it is `failed_when: false` it cannot break that +run even if a field reads wrong. + +## Out of scope + +No change to `Oddly/incus-memory-gate` and no change to the gate +ledger's reservation-before-committed read order. diff --git a/molecule/kibana_default/molecule.yml b/molecule/kibana_default/molecule.yml index 385986c5..e1dbc9fc 100644 --- a/molecule/kibana_default/molecule.yml +++ b/molecule/kibana_default/molecule.yml @@ -9,6 +9,15 @@ driver: platforms: - name: "kib-def-${MOLECULE_DISTRO:-debian12}-r${ELASTIC_RELEASE:-9}${MOLECULE_RUN_SUFFIX}" distro: "${MOLECULE_DISTRO:-debian12}" + # 3072 MiB: this single ES+Kibana node peaked at 923 MiB anon RSS + # across all seven distros and both releases (2026-08 full-matrix + # sampling), against the 4096 default it used to inherit. 3072 keeps + # ~2.1 GB for the page cache, close to the observed 2.3-2.5 GB + # working set, so we reclaim a gigabyte of gate ledger without + # squeezing the Lucene cache. The other scenarios measured at + # 40-98% of their limits once rockylinux9's dnf install spike is + # included and are left as-is. + memory_mb: 3072 provisioner: name: ansible env: diff --git a/molecule/shared/destroy.yml b/molecule/shared/destroy.yml index 5b75ea22..5dcedc55 100644 --- a/molecule/shared/destroy.yml +++ b/molecule/shared/destroy.yml @@ -8,6 +8,7 @@ vars: incus_host: "{{ lookup('env', 'INCUS_HOST') | default('172.30.0.172', true) }}" molecule_ssh_key: "{{ lookup('env', 'MOLECULE_SSH_KEY') | default(lookup('env', 'HOME') ~ '/.ssh/molecule_id_ed25519', true) }}" + ci_run_id: "{{ lookup('env', 'GITHUB_RUN_ID') | default('', true) }}" # Resolve ${VAR:-default} shell substitutions in raw molecule.yml _resolve_envsubst: >- python3 -c 'import re,os,sys; @@ -20,6 +21,56 @@ -i {{ molecule_ssh_key }} root@{{ incus_host }} tasks: + # Best-effort per-container memory telemetry, taken before the + # container is deleted so we can right-size memory_mb from real + # numbers (see ci/README.md). Reads the top-level payload cgroup on + # the incus host: memory.peak is the kernel cache-inclusive + # high-water (an upper bound on total need), anon/cache are the + # final split, oom_kill is the definitive kill count. The continuous + # sampler in ci/ captures the true anon *peak*; this task guarantees + # a per-container record on every run and appends it to an + # accumulating ledger on the host. It never fails a teardown. + - name: Record peak memory before delete # noqa: risky-shell-pipe + ansible.builtin.shell: + cmd: | + {{ _ssh_cmd }} bash -s <<'REMOTE' + name="{{ item.name }}" + dir=$(find /sys/fs/cgroup -type d -name "lxc.payload.${name}" 2>/dev/null | head -1) + b2m() { echo $(( ${1:-0} / 1048576 )); } + stamp=$(date -u +%Y-%m-%dT%H:%M:%SZ) + if [ -z "$dir" ]; then + line="{\"@timestamp\":\"${stamp}\",\"run\":\"{{ ci_run_id }}\",\"container\":\"${name}\",\"note\":\"no cgroup at teardown\"}" + else + peak=$(b2m "$(cat "$dir/memory.peak" 2>/dev/null || echo 0)") + anon=$(b2m "$(awk '/^anon /{print $2; exit}' "$dir/memory.stat" 2>/dev/null || echo 0)") + cache=$(b2m "$(awk '/^file /{print $2; exit}' "$dir/memory.stat" 2>/dev/null || echo 0)") + mx=$(cat "$dir/memory.max" 2>/dev/null || echo max) + if [ "$mx" = "max" ]; then lim=0; else lim=$(b2m "$mx"); fi + oom=$(awk '/^oom_kill /{print $2; exit}' "$dir/memory.events" 2>/dev/null || echo 0) + line="{\"@timestamp\":\"${stamp}\",\"run\":\"{{ ci_run_id }}\",\"container\":\"${name}\",\"limit_mb\":${lim},\"peak_mb\":${peak},\"anon_final_mb\":${anon},\"cache_final_mb\":${cache},\"oom_kill\":${oom}}" + fi + mkdir -p /root/mem-peaks + printf '%s\n' "$line" >> /root/mem-peaks/teardown.ndjson + printf '%s\n' "$line" + REMOTE + args: + executable: /bin/bash + loop: "{{ molecule_yml.platforms }}" + loop_control: + label: "{{ item.name }}" + register: _mem_teardown + failed_when: false + changed_when: false + + - name: Save teardown telemetry to the ephemeral dir + ansible.builtin.copy: + dest: "{{ molecule_ephemeral_directory }}/mem-teardown.ndjson" + content: | + {{ _mem_teardown.results | default([]) | map(attribute='stdout') | select('string') | join('\n') }} + mode: "0644" + failed_when: false + changed_when: false + - name: Destroy containers ansible.builtin.command: cmd: "{{ _ssh_cmd }} incus delete {{ item.name }} --force"