From ff03c04cba4264b265db2db933e9979468ebebc8 Mon Sep 17 00:00:00 2001 From: Nat Brown Date: Sat, 22 Aug 2026 21:29:36 -0400 Subject: [PATCH] build: size container build parallelism by container memory, not host cores J defaulted to nproc, which assumes the host's core count is the binding constraint. In a container it is usually memory: kaldi's nnet2/decoder and dxvk's d3d11 peak near 1GB of RAM per compiland, so a host with less than roughly 1GiB per core lets the OOM killer take out compilers at random once enough heavy translation units overlap. That failure is hard to read. The kill shows up as x86_64-linux-gnu-g++: fatal error: Killed signal terminated program cc1plus and make then reports a plain "Error 1", while "Waiting for unfinished jobs..." lets the surviving jobs drain -- so the last line before the failure is whatever recipe happened to finish last, thousands of lines from the real cause. Ask the container engine how much memory it actually has and use one job per GiB, capped by the core count, printing ":: limiting to -jN (M GiB container memory)" when that reduces the count. Hosts with at least 1GiB per core are unaffected, and an unreachable engine falls back to nproc, so this is a no-op for a normally provisioned machine. J=N still overrides. Also warn when -j is passed on a host whose make cannot forward the count: GNU make 4.x keeps it in MAKEFLAGS, but 3.81 records a bare "-j" and keeps the count in the jobserver, so -jN reaches the container unbounded. Signed-off-by: Nat Brown --- Makefile.in | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/Makefile.in b/Makefile.in index 5e477c7f99c..c9c204909a5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1473,8 +1473,6 @@ module: module32 module64 else # outside of the container ############################### -J := $(shell nproc) - # If CC is coming from make's defaults or nowhere, use our own default. Otherwise respect environment. CCACHE_ENV := $(patsubst %,-e %,$(shell env|cut -d= -f1|grep '^CCACHE_')) ifeq ($(ENABLE_CCACHE),1) @@ -1496,6 +1494,45 @@ ifeq ($(CONTAINER_ENGINE),) CONTAINER_ENGINE := docker endif +# Container build parallelism heuristic. +# +# The previous "J := $(shell nproc)" assumed the host's core count is the binding +# constraint. In a container it is usually memory: sub-modules such as kaldi's +# nnet2/decoder and dxvk's d3d11 peak near 1GB of RAM per compiland, so on a +# memory-constrained host -j lets the OOM killer take out compilers at +# random. That surfaces as "cc1plus: fatal error: Killed signal terminated +# program" and an unrelated-looking "Error 1" thousands of lines from the cause. +# +# Default to one job per GiB of container memory, capped by the core count, and +# say so when that bites. Only differs from nproc on hosts with less than 1GiB +# per core; if the engine cannot be reached we fall back to the core count. +# Override with 'make J=N'. +J := $(shell \ + cpus=$$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1); \ + mem=$$($(CONTAINER_ENGINE) info --format '{{json .}}' 2>/dev/null \ + | tr ',{' '\n\n' \ + | sed -n 's/.*"[Mm]em[Tt]otal": *\([0-9][0-9]*\).*/\1/p' | head -1); \ + jobs=$$cpus; \ + if [ -n "$$mem" ] && [ "$$mem" -gt 0 ] 2>/dev/null; then \ + memgib=$$(( mem / 1073741824 )); \ + [ $$memgib -lt 1 ] && memgib=1; \ + if [ $$memgib -lt $$cpus ]; then \ + jobs=$$memgib; \ + echo ":: limiting to -j$$jobs ($$memgib GiB container memory)" >&2; \ + fi; \ + fi; \ + echo $$jobs) + +# GNU make 4.x keeps an explicit -jN count in MAKEFLAGS so it survives into the +# container; GNU make 3.81 (still shipped by macOS) records only a bare "-j" and +# keeps the count in the jobserver, so -jN arrives unbounded. Harmless on Linux, +# a good way to OOM a small VM elsewhere, so warn rather than override the user. +# +# Deliberately recursive (=), not simple (:=): make 3.81 leaves MAKEFLAGS empty +# until recipe-expansion time, so a parse-time test would never see the -j. +J_HOST_OS := $(shell uname) +J_WARNING = $(if $(filter-out Linux,$(J_HOST_OS)),$(if $(findstring j,$(MAKEFLAGS)),@echo >&2 ":: warning: -j may reach the container unbounded on $(J_HOST_OS) -- use J=N to bound it or omit -j to auto-size to -j$(J)")) + DOCKER_BASE = $(CONTAINER_ENGINE) run --rm -v $(SRC):$(SRC)$(CONTAINER_MOUNT_OPTS) -v $(OBJ):$(OBJ)$(CONTAINER_MOUNT_OPTS) \ -w $(OBJ) -e MAKEFLAGS \ $(DOCKER_OPTS) $(STEAMRT_IMAGE) @@ -1504,6 +1541,7 @@ DOCKER_BASE = $(CONTAINER_ENGINE) run --rm -v $(SRC):$(SRC)$(CONTAINER_MOUNT_OPT .DEFAULT dist deploy redist symstore-tarball: if [ "$(ENABLE_CCACHE)" -eq "1" ]; then mkdir -p $(CCACHE_DIR); fi mkdir -p $(CARGO_HOME) + $(J_WARNING) $(DOCKER_BASE) $(MAKE) -j$(J) $(filter -j%,$(MAKEFLAGS)) -f $(firstword $(MAKEFILE_LIST)) $(MFLAGS) $(MAKEOVERRIDES) CONTAINER=1 $@