diff --git a/configs/arcbox-arm64.config b/configs/arcbox-arm64.config index 1878404..b5e0c0b 100644 --- a/configs/arcbox-arm64.config +++ b/configs/arcbox-arm64.config @@ -22,10 +22,16 @@ CONFIG_TIMERFD=y CONFIG_EVENTFD=y CONFIG_AIO=y CONFIG_NO_HZ_IDLE=y -# Pressure stall information: in-guest memory/CPU/IO pressure signals. -# The ArcBox agent's balloon pressure watch upgrades from meminfo/refault -# sampling to PSI triggers when /proc/pressure is present. +# Pressure stall information: compiled in for the balloon pressure watch, +# but DEFAULT-DISABLED. PSI's per-task-state accounting sits on the +# scheduler wakeup fast path; left on, the container-build gRPC +# orchestration (thousands of wakeups per step) pays ~0.85 s/build and +# same-core context switch nearly doubles (ABX-498). Default-off leaves +# /proc/pressure uncreated, so the agent's balloon watch (which opens it) +# gets ENOENT and falls back to meminfo/refault sampling — the balloon +# still functions; boot psi=1 to re-enable the trigger. CONFIG_PSI=y +CONFIG_PSI_DEFAULT_DISABLED=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y @@ -68,13 +74,21 @@ CONFIG_64BIT=y CONFIG_MMU=y CONFIG_SMP=y CONFIG_NR_CPUS=256 -# Lazy preemption: near-voluntary throughput, far lower scheduling -# latency for interactive guest work (container exec, port-forward -# first byte). Also forward-compatible — 7.0 removes voluntary for -# lazy-capable arches, and arm64/x86 both select ARCH_HAS_PREEMPT_LAZY. -CONFIG_PREEMPT_LAZY=y -CONFIG_HZ_100=y -CONFIG_HZ=100 +# Voluntary preemption, NOT lazy. Lazy pulls in PREEMPT_RCU, which makes +# rcu_barrier() on the veth/netns teardown path (every container stop) +# wait for preemptible-RCU callbacks — measured ~74 ms/veth vs ~0 ms +# under voluntary — and adds ~25% to same-core context-switch cost. The +# "near-voluntary throughput" premise did not hold for this guest's +# container-churn workload. See ABX-498 for the four-kernel A/B matrix. +CONFIG_PREEMPT_VOLUNTARY=y +# HZ=1000 (1 ms tick): container start is a chain of jiffy-rounded +# scheduler waits (containerd poll, shim handshake, runc, dockerd↔ +# containerd gRPC backoff). At HZ=100 each rounds up to 10 ms, making +# container churn several times slower than an HZ=1000 guest — the +# dominant per-step cost in multi-step `docker build`. NO_HZ_IDLE keeps +# the idle-CPU target intact (tickless when idle). See ABX-496/ABX-498. +CONFIG_HZ_1000=y +CONFIG_HZ=1000 # Virtualization guest support CONFIG_PARAVIRT=y diff --git a/configs/arcbox-x86_64.config b/configs/arcbox-x86_64.config index 929d3a7..1142915 100644 --- a/configs/arcbox-x86_64.config +++ b/configs/arcbox-x86_64.config @@ -20,10 +20,16 @@ CONFIG_TIMERFD=y CONFIG_EVENTFD=y CONFIG_AIO=y CONFIG_NO_HZ_IDLE=y -# Pressure stall information: in-guest memory/CPU/IO pressure signals. -# The ArcBox agent's balloon pressure watch upgrades from meminfo/refault -# sampling to PSI triggers when /proc/pressure is present. +# Pressure stall information: compiled in for the balloon pressure watch, +# but DEFAULT-DISABLED. PSI's per-task-state accounting sits on the +# scheduler wakeup fast path; left on, the container-build gRPC +# orchestration (thousands of wakeups per step) pays ~0.85 s/build and +# same-core context switch nearly doubles (ABX-498). Default-off leaves +# /proc/pressure uncreated, so the agent's balloon watch (which opens it) +# gets ENOENT and falls back to meminfo/refault sampling — the balloon +# still functions; boot psi=1 to re-enable the trigger. CONFIG_PSI=y +CONFIG_PSI_DEFAULT_DISABLED=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y @@ -67,13 +73,21 @@ CONFIG_X86=y CONFIG_MMU=y CONFIG_SMP=y CONFIG_NR_CPUS=256 -# Lazy preemption: near-voluntary throughput, far lower scheduling -# latency for interactive guest work (container exec, port-forward -# first byte). Also forward-compatible — 7.0 removes voluntary for -# lazy-capable arches, and arm64/x86 both select ARCH_HAS_PREEMPT_LAZY. -CONFIG_PREEMPT_LAZY=y -CONFIG_HZ_100=y -CONFIG_HZ=100 +# Voluntary preemption, NOT lazy. Lazy pulls in PREEMPT_RCU, which makes +# rcu_barrier() on the veth/netns teardown path (every container stop) +# wait for preemptible-RCU callbacks — measured ~74 ms/veth vs ~0 ms +# under voluntary — and adds ~25% to same-core context-switch cost. The +# "near-voluntary throughput" premise did not hold for this guest's +# container-churn workload. See ABX-498 for the four-kernel A/B matrix. +CONFIG_PREEMPT_VOLUNTARY=y +# HZ=1000 (1 ms tick): container start is a chain of jiffy-rounded +# scheduler waits (containerd poll, shim handshake, runc, dockerd↔ +# containerd gRPC backoff). At HZ=100 each rounds up to 10 ms, making +# container churn several times slower than an HZ=1000 guest — the +# dominant per-step cost in multi-step `docker build`. NO_HZ_IDLE keeps +# the idle-CPU target intact (tickless when idle). See ABX-496/ABX-498. +CONFIG_HZ_1000=y +CONFIG_HZ=1000 # Virtualization guest support CONFIG_HYPERVISOR_GUEST=y diff --git a/scripts/build-kernel.sh b/scripts/build-kernel.sh index 1ba6bf4..f64b3fe 100755 --- a/scripts/build-kernel.sh +++ b/scripts/build-kernel.sh @@ -73,25 +73,22 @@ do_build() { # dependency gate in the fragment otherwise degrades silently — 6.18 did # exactly that to the legacy iptables stack via NETFILTER_XTABLES_LEGACY). for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT \ - CONFIG_IP6_NF_NAT; do + CONFIG_IP6_NF_NAT CONFIG_HZ_1000 \ + CONFIG_PSI_DEFAULT_DISABLED; do grep -q "^$sym=y" .config || { echo "ERROR: $sym missing after olddefconfig" >&2 exit 1 } done - # PREEMPT_LAZY exists only where the arch wires it up (x86 6.13+, - # arm64 6.16+). Assert it there; on an older KERNEL_VERSION override - # the choice falls back to its kconfig default, which is worth a - # warning but must not fail a bisect build. - if grep -q '^CONFIG_ARCH_HAS_PREEMPT_LAZY=y' .config; then - grep -q '^CONFIG_PREEMPT_LAZY=y' .config || { - echo 'ERROR: CONFIG_PREEMPT_LAZY missing after olddefconfig' >&2 - exit 1 - } - else - echo 'WARN: kernel lacks ARCH_HAS_PREEMPT_LAZY; preemption model falls back to the kconfig default' >&2 - fi + # Voluntary preemption is load-bearing for container-teardown latency + # (see the config comment). Assert it survived olddefconfig; a future + # KERNEL_VERSION that drops the voluntary choice must fail loudly here + # rather than silently fall back to lazy/full. + grep -q '^CONFIG_PREEMPT_VOLUNTARY=y' .config || { + echo 'ERROR: CONFIG_PREEMPT_VOLUNTARY missing after olddefconfig' >&2 + exit 1 + } # Build. echo "Building kernel..." @@ -132,20 +129,16 @@ cd linux-$KERNEL_VERSION sh /workspace/scripts/inject-drivers.sh /workspace cp /workspace/configs/arcbox-$TARGET_ARCH.config .config make ARCH=$TARGET_ARCH olddefconfig -for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT CONFIG_IP6_NF_NAT; do +for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT CONFIG_IP6_NF_NAT CONFIG_HZ_1000 CONFIG_PSI_DEFAULT_DISABLED; do grep -q \"^\$sym=y\" .config || { echo \"ERROR: \$sym missing after olddefconfig\" >&2 exit 1 } done -if grep -q '^CONFIG_ARCH_HAS_PREEMPT_LAZY=y' .config; then - grep -q '^CONFIG_PREEMPT_LAZY=y' .config || { - echo 'ERROR: CONFIG_PREEMPT_LAZY missing after olddefconfig' >&2 - exit 1 - } -else - echo 'WARN: kernel lacks ARCH_HAS_PREEMPT_LAZY; preemption model falls back to the kconfig default' >&2 -fi +grep -q '^CONFIG_PREEMPT_VOLUNTARY=y' .config || { + echo 'ERROR: CONFIG_PREEMPT_VOLUNTARY missing after olddefconfig' >&2 + exit 1 +} echo 'Building kernel...' make ARCH=$TARGET_ARCH -j\$(nproc) $KERNEL_IMAGE cp arch/$TARGET_ARCH/boot/$KERNEL_IMAGE /output/kernel-$TARGET_ARCH