perf(config): cut container-start latency — HZ=1000, voluntary preemption, PSI default-off (ABX-498)

configs/arcbox-arm64.config

@@ -22,10 +22,16 @@ CONFIG_TIMERFD=y
 CONFIG_EVENTFD=y
 CONFIG_AIO=y
 CONFIG_NO_HZ_IDLE=y
-# Pressure stall information: in-guest memory/CPU/IO pressure signals.
-# The ArcBox agent's balloon pressure watch upgrades from meminfo/refault
-# sampling to PSI triggers when /proc/pressure is present.
+# Pressure stall information: compiled in for the balloon pressure watch,
+# but DEFAULT-DISABLED. PSI's per-task-state accounting sits on the
+# scheduler wakeup fast path; left on, the container-build gRPC
+# orchestration (thousands of wakeups per step) pays ~0.85 s/build and
+# same-core context switch nearly doubles (ABX-498). Default-off leaves
+# /proc/pressure uncreated, so the agent's balloon watch (which opens it)
+# gets ENOENT and falls back to meminfo/refault sampling — the balloon
+# still functions; boot psi=1 to re-enable the trigger.
 CONFIG_PSI=y
+CONFIG_PSI_DEFAULT_DISABLED=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BPF_SYSCALL=y
 CONFIG_BPF_JIT=y
@@ -68,13 +74,21 @@ CONFIG_64BIT=y
 CONFIG_MMU=y
 CONFIG_SMP=y
 CONFIG_NR_CPUS=256
-# Lazy preemption: near-voluntary throughput, far lower scheduling
-# latency for interactive guest work (container exec, port-forward
-# first byte). Also forward-compatible — 7.0 removes voluntary for
-# lazy-capable arches, and arm64/x86 both select ARCH_HAS_PREEMPT_LAZY.
-CONFIG_PREEMPT_LAZY=y
-CONFIG_HZ_100=y
-CONFIG_HZ=100
+# Voluntary preemption, NOT lazy. Lazy pulls in PREEMPT_RCU, which makes
+# rcu_barrier() on the veth/netns teardown path (every container stop)
+# wait for preemptible-RCU callbacks — measured ~74 ms/veth vs ~0 ms
+# under voluntary — and adds ~25% to same-core context-switch cost. The
+# "near-voluntary throughput" premise did not hold for this guest's
+# container-churn workload. See ABX-498 for the four-kernel A/B matrix.
+CONFIG_PREEMPT_VOLUNTARY=y
+# HZ=1000 (1 ms tick): container start is a chain of jiffy-rounded
+# scheduler waits (containerd poll, shim handshake, runc, dockerd↔
+# containerd gRPC backoff). At HZ=100 each rounds up to 10 ms, making
+# container churn several times slower than an HZ=1000 guest — the
+# dominant per-step cost in multi-step `docker build`. NO_HZ_IDLE keeps
+# the idle-CPU target intact (tickless when idle). See ABX-496/ABX-498.
+CONFIG_HZ_1000=y
+CONFIG_HZ=1000
 
 # Virtualization guest support
 CONFIG_PARAVIRT=y

configs/arcbox-x86_64.config

@@ -20,10 +20,16 @@ CONFIG_TIMERFD=y
 CONFIG_EVENTFD=y
 CONFIG_AIO=y
 CONFIG_NO_HZ_IDLE=y
-# Pressure stall information: in-guest memory/CPU/IO pressure signals.
-# The ArcBox agent's balloon pressure watch upgrades from meminfo/refault
-# sampling to PSI triggers when /proc/pressure is present.
+# Pressure stall information: compiled in for the balloon pressure watch,
+# but DEFAULT-DISABLED. PSI's per-task-state accounting sits on the
+# scheduler wakeup fast path; left on, the container-build gRPC
+# orchestration (thousands of wakeups per step) pays ~0.85 s/build and
+# same-core context switch nearly doubles (ABX-498). Default-off leaves
+# /proc/pressure uncreated, so the agent's balloon watch (which opens it)
+# gets ENOENT and falls back to meminfo/refault sampling — the balloon
+# still functions; boot psi=1 to re-enable the trigger.
 CONFIG_PSI=y
+CONFIG_PSI_DEFAULT_DISABLED=y
 CONFIG_HIGH_RES_TIMERS=y
 CONFIG_BPF_SYSCALL=y
 CONFIG_BPF_JIT=y
@@ -67,13 +73,21 @@ CONFIG_X86=y
 CONFIG_MMU=y
 CONFIG_SMP=y
 CONFIG_NR_CPUS=256
-# Lazy preemption: near-voluntary throughput, far lower scheduling
-# latency for interactive guest work (container exec, port-forward
-# first byte). Also forward-compatible — 7.0 removes voluntary for
-# lazy-capable arches, and arm64/x86 both select ARCH_HAS_PREEMPT_LAZY.
-CONFIG_PREEMPT_LAZY=y
-CONFIG_HZ_100=y
-CONFIG_HZ=100
+# Voluntary preemption, NOT lazy. Lazy pulls in PREEMPT_RCU, which makes
+# rcu_barrier() on the veth/netns teardown path (every container stop)
+# wait for preemptible-RCU callbacks — measured ~74 ms/veth vs ~0 ms
+# under voluntary — and adds ~25% to same-core context-switch cost. The
+# "near-voluntary throughput" premise did not hold for this guest's
+# container-churn workload. See ABX-498 for the four-kernel A/B matrix.
+CONFIG_PREEMPT_VOLUNTARY=y
+# HZ=1000 (1 ms tick): container start is a chain of jiffy-rounded
+# scheduler waits (containerd poll, shim handshake, runc, dockerd↔
+# containerd gRPC backoff). At HZ=100 each rounds up to 10 ms, making
+# container churn several times slower than an HZ=1000 guest — the
+# dominant per-step cost in multi-step `docker build`. NO_HZ_IDLE keeps
+# the idle-CPU target intact (tickless when idle). See ABX-496/ABX-498.
+CONFIG_HZ_1000=y
+CONFIG_HZ=1000
 
 # Virtualization guest support
 CONFIG_HYPERVISOR_GUEST=y

scripts/build-kernel.sh

@@ -73,25 +73,22 @@ do_build() {
     # dependency gate in the fragment otherwise degrades silently — 6.18 did
     # exactly that to the legacy iptables stack via NETFILTER_XTABLES_LEGACY).
     for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT \
-               CONFIG_IP6_NF_NAT; do
+               CONFIG_IP6_NF_NAT CONFIG_HZ_1000 \
+               CONFIG_PSI_DEFAULT_DISABLED; do
         grep -q "^$sym=y" .config || {
             echo "ERROR: $sym missing after olddefconfig" >&2
             exit 1
         }
     done
 
-    # PREEMPT_LAZY exists only where the arch wires it up (x86 6.13+,
-    # arm64 6.16+). Assert it there; on an older KERNEL_VERSION override
-    # the choice falls back to its kconfig default, which is worth a
-    # warning but must not fail a bisect build.
-    if grep -q '^CONFIG_ARCH_HAS_PREEMPT_LAZY=y' .config; then
-        grep -q '^CONFIG_PREEMPT_LAZY=y' .config || {
-            echo 'ERROR: CONFIG_PREEMPT_LAZY missing after olddefconfig' >&2
-            exit 1
-        }
-    else
-        echo 'WARN: kernel lacks ARCH_HAS_PREEMPT_LAZY; preemption model falls back to the kconfig default' >&2
-    fi
+    # Voluntary preemption is load-bearing for container-teardown latency
+    # (see the config comment). Assert it survived olddefconfig; a future
+    # KERNEL_VERSION that drops the voluntary choice must fail loudly here
+    # rather than silently fall back to lazy/full.
+    grep -q '^CONFIG_PREEMPT_VOLUNTARY=y' .config || {
+        echo 'ERROR: CONFIG_PREEMPT_VOLUNTARY missing after olddefconfig' >&2
+        exit 1
+    }
 
     # Build.
     echo "Building kernel..."
@@ -132,20 +129,16 @@ cd linux-$KERNEL_VERSION
 sh /workspace/scripts/inject-drivers.sh /workspace
 cp /workspace/configs/arcbox-$TARGET_ARCH.config .config
 make ARCH=$TARGET_ARCH olddefconfig
-for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT CONFIG_IP6_NF_NAT; do
+for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT CONFIG_IP6_NF_NAT CONFIG_HZ_1000 CONFIG_PSI_DEFAULT_DISABLED; do
     grep -q \"^\$sym=y\" .config || {
         echo \"ERROR: \$sym missing after olddefconfig\" >&2
         exit 1
     }
 done
-if grep -q '^CONFIG_ARCH_HAS_PREEMPT_LAZY=y' .config; then
-    grep -q '^CONFIG_PREEMPT_LAZY=y' .config || {
-        echo 'ERROR: CONFIG_PREEMPT_LAZY missing after olddefconfig' >&2
-        exit 1
-    }
-else
-    echo 'WARN: kernel lacks ARCH_HAS_PREEMPT_LAZY; preemption model falls back to the kconfig default' >&2
-fi
+grep -q '^CONFIG_PREEMPT_VOLUNTARY=y' .config || {
+    echo 'ERROR: CONFIG_PREEMPT_VOLUNTARY missing after olddefconfig' >&2
+    exit 1
+}
 echo 'Building kernel...'
 make ARCH=$TARGET_ARCH -j\$(nproc) $KERNEL_IMAGE
 cp arch/$TARGET_ARCH/boot/$KERNEL_IMAGE /output/kernel-$TARGET_ARCH