From 45b1f8c64fd7917d6f90649f05277c6e1e7be208 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Fri, 24 Jul 2026 12:03:07 +0800 Subject: [PATCH 1/3] =?UTF-8?q?perf(config):=20HZ=20100=E2=86=921000=20to?= =?UTF-8?q?=20cut=20container-start=20latency?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Container start is a chain of jiffy-rounded scheduler waits (containerd poll, shim handshake completions, runc, dockerd↔containerd gRPC backoff). At HZ=100 each rounds up to 10 ms; at HZ=1000 to 1 ms. Measured docker-run end-to-end 496→378 ms and multi-step builds proportionally, matching the HZ=1000 guests Colima/OrbStack ship. NO_HZ_IDLE keeps the tickless-idle CPU target intact. Refs ABX-496/ABX-498. --- configs/arcbox-arm64.config | 10 ++++++++-- configs/arcbox-x86_64.config | 10 ++++++++-- scripts/build-kernel.sh | 4 ++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/configs/arcbox-arm64.config b/configs/arcbox-arm64.config index 1878404..419f4b1 100644 --- a/configs/arcbox-arm64.config +++ b/configs/arcbox-arm64.config @@ -73,8 +73,14 @@ CONFIG_NR_CPUS=256 # first byte). Also forward-compatible — 7.0 removes voluntary for # lazy-capable arches, and arm64/x86 both select ARCH_HAS_PREEMPT_LAZY. CONFIG_PREEMPT_LAZY=y -CONFIG_HZ_100=y -CONFIG_HZ=100 +# HZ=1000 (1 ms tick): container start is a chain of jiffy-rounded +# scheduler waits (containerd poll, shim handshake, runc, dockerd↔ +# containerd gRPC backoff). At HZ=100 each rounds up to 10 ms, making +# container churn several times slower than an HZ=1000 guest — the +# dominant per-step cost in multi-step `docker build`. NO_HZ_IDLE keeps +# the idle-CPU target intact (tickless when idle). See ABX-496/ABX-498. +CONFIG_HZ_1000=y +CONFIG_HZ=1000 # Virtualization guest support CONFIG_PARAVIRT=y diff --git a/configs/arcbox-x86_64.config b/configs/arcbox-x86_64.config index 929d3a7..9fb3f81 100644 --- a/configs/arcbox-x86_64.config +++ b/configs/arcbox-x86_64.config @@ -72,8 +72,14 @@ CONFIG_NR_CPUS=256 # first byte). Also forward-compatible — 7.0 removes voluntary for # lazy-capable arches, and arm64/x86 both select ARCH_HAS_PREEMPT_LAZY. CONFIG_PREEMPT_LAZY=y -CONFIG_HZ_100=y -CONFIG_HZ=100 +# HZ=1000 (1 ms tick): container start is a chain of jiffy-rounded +# scheduler waits (containerd poll, shim handshake, runc, dockerd↔ +# containerd gRPC backoff). At HZ=100 each rounds up to 10 ms, making +# container churn several times slower than an HZ=1000 guest — the +# dominant per-step cost in multi-step `docker build`. NO_HZ_IDLE keeps +# the idle-CPU target intact (tickless when idle). See ABX-496/ABX-498. +CONFIG_HZ_1000=y +CONFIG_HZ=1000 # Virtualization guest support CONFIG_HYPERVISOR_GUEST=y diff --git a/scripts/build-kernel.sh b/scripts/build-kernel.sh index 1ba6bf4..a466cc9 100755 --- a/scripts/build-kernel.sh +++ b/scripts/build-kernel.sh @@ -73,7 +73,7 @@ do_build() { # dependency gate in the fragment otherwise degrades silently — 6.18 did # exactly that to the legacy iptables stack via NETFILTER_XTABLES_LEGACY). for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT \ - CONFIG_IP6_NF_NAT; do + CONFIG_IP6_NF_NAT CONFIG_HZ_1000; do grep -q "^$sym=y" .config || { echo "ERROR: $sym missing after olddefconfig" >&2 exit 1 @@ -132,7 +132,7 @@ cd linux-$KERNEL_VERSION sh /workspace/scripts/inject-drivers.sh /workspace cp /workspace/configs/arcbox-$TARGET_ARCH.config .config make ARCH=$TARGET_ARCH olddefconfig -for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT CONFIG_IP6_NF_NAT; do +for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT CONFIG_IP6_NF_NAT CONFIG_HZ_1000; do grep -q \"^\$sym=y\" .config || { echo \"ERROR: \$sym missing after olddefconfig\" >&2 exit 1 From c1aadc342ff8d2c79fad56448830e8e5811e54a7 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Fri, 24 Jul 2026 12:03:58 +0800 Subject: [PATCH 2/3] perf(config): voluntary preemption, not lazy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 6.18 bump switched arm64/x86 to lazy preemption on the premise of near-voluntary throughput. That premise fails for this guest's container-churn workload: lazy pulls in PREEMPT_RCU, so rcu_barrier() on the veth/netns teardown path (every container stop) waits for preemptible-RCU callbacks — ~74 ms/veth vs ~0 ms under voluntary — and same-core context-switch cost rises ~25%. Revert to voluntary; the build assertion now pins it so a future kernel that drops the choice fails loudly instead of silently re-selecting lazy. See ABX-498. --- configs/arcbox-arm64.config | 12 +++++++----- configs/arcbox-x86_64.config | 12 +++++++----- scripts/build-kernel.sh | 32 ++++++++++++-------------------- 3 files changed, 26 insertions(+), 30 deletions(-) diff --git a/configs/arcbox-arm64.config b/configs/arcbox-arm64.config index 419f4b1..a46449e 100644 --- a/configs/arcbox-arm64.config +++ b/configs/arcbox-arm64.config @@ -68,11 +68,13 @@ CONFIG_64BIT=y CONFIG_MMU=y CONFIG_SMP=y CONFIG_NR_CPUS=256 -# Lazy preemption: near-voluntary throughput, far lower scheduling -# latency for interactive guest work (container exec, port-forward -# first byte). Also forward-compatible — 7.0 removes voluntary for -# lazy-capable arches, and arm64/x86 both select ARCH_HAS_PREEMPT_LAZY. -CONFIG_PREEMPT_LAZY=y +# Voluntary preemption, NOT lazy. Lazy pulls in PREEMPT_RCU, which makes +# rcu_barrier() on the veth/netns teardown path (every container stop) +# wait for preemptible-RCU callbacks — measured ~74 ms/veth vs ~0 ms +# under voluntary — and adds ~25% to same-core context-switch cost. The +# "near-voluntary throughput" premise did not hold for this guest's +# container-churn workload. See ABX-498 for the four-kernel A/B matrix. +CONFIG_PREEMPT_VOLUNTARY=y # HZ=1000 (1 ms tick): container start is a chain of jiffy-rounded # scheduler waits (containerd poll, shim handshake, runc, dockerd↔ # containerd gRPC backoff). At HZ=100 each rounds up to 10 ms, making diff --git a/configs/arcbox-x86_64.config b/configs/arcbox-x86_64.config index 9fb3f81..9484d35 100644 --- a/configs/arcbox-x86_64.config +++ b/configs/arcbox-x86_64.config @@ -67,11 +67,13 @@ CONFIG_X86=y CONFIG_MMU=y CONFIG_SMP=y CONFIG_NR_CPUS=256 -# Lazy preemption: near-voluntary throughput, far lower scheduling -# latency for interactive guest work (container exec, port-forward -# first byte). Also forward-compatible — 7.0 removes voluntary for -# lazy-capable arches, and arm64/x86 both select ARCH_HAS_PREEMPT_LAZY. -CONFIG_PREEMPT_LAZY=y +# Voluntary preemption, NOT lazy. Lazy pulls in PREEMPT_RCU, which makes +# rcu_barrier() on the veth/netns teardown path (every container stop) +# wait for preemptible-RCU callbacks — measured ~74 ms/veth vs ~0 ms +# under voluntary — and adds ~25% to same-core context-switch cost. The +# "near-voluntary throughput" premise did not hold for this guest's +# container-churn workload. See ABX-498 for the four-kernel A/B matrix. +CONFIG_PREEMPT_VOLUNTARY=y # HZ=1000 (1 ms tick): container start is a chain of jiffy-rounded # scheduler waits (containerd poll, shim handshake, runc, dockerd↔ # containerd gRPC backoff). At HZ=100 each rounds up to 10 ms, making diff --git a/scripts/build-kernel.sh b/scripts/build-kernel.sh index a466cc9..3cd8f8d 100755 --- a/scripts/build-kernel.sh +++ b/scripts/build-kernel.sh @@ -80,18 +80,14 @@ do_build() { } done - # PREEMPT_LAZY exists only where the arch wires it up (x86 6.13+, - # arm64 6.16+). Assert it there; on an older KERNEL_VERSION override - # the choice falls back to its kconfig default, which is worth a - # warning but must not fail a bisect build. - if grep -q '^CONFIG_ARCH_HAS_PREEMPT_LAZY=y' .config; then - grep -q '^CONFIG_PREEMPT_LAZY=y' .config || { - echo 'ERROR: CONFIG_PREEMPT_LAZY missing after olddefconfig' >&2 - exit 1 - } - else - echo 'WARN: kernel lacks ARCH_HAS_PREEMPT_LAZY; preemption model falls back to the kconfig default' >&2 - fi + # Voluntary preemption is load-bearing for container-teardown latency + # (see the config comment). Assert it survived olddefconfig; a future + # KERNEL_VERSION that drops the voluntary choice must fail loudly here + # rather than silently fall back to lazy/full. + grep -q '^CONFIG_PREEMPT_VOLUNTARY=y' .config || { + echo 'ERROR: CONFIG_PREEMPT_VOLUNTARY missing after olddefconfig' >&2 + exit 1 + } # Build. echo "Building kernel..." @@ -138,14 +134,10 @@ for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT CONFIG_IP6_NF_NA exit 1 } done -if grep -q '^CONFIG_ARCH_HAS_PREEMPT_LAZY=y' .config; then - grep -q '^CONFIG_PREEMPT_LAZY=y' .config || { - echo 'ERROR: CONFIG_PREEMPT_LAZY missing after olddefconfig' >&2 - exit 1 - } -else - echo 'WARN: kernel lacks ARCH_HAS_PREEMPT_LAZY; preemption model falls back to the kconfig default' >&2 -fi +grep -q '^CONFIG_PREEMPT_VOLUNTARY=y' .config || { + echo 'ERROR: CONFIG_PREEMPT_VOLUNTARY missing after olddefconfig' >&2 + exit 1 +} echo 'Building kernel...' make ARCH=$TARGET_ARCH -j\$(nproc) $KERNEL_IMAGE cp arch/$TARGET_ARCH/boot/$KERNEL_IMAGE /output/kernel-$TARGET_ARCH From 2de95786f75f1646e42645f73e4d51dae9fdee1b Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Fri, 24 Jul 2026 12:05:11 +0800 Subject: [PATCH 3/3] perf(config): disable PSI by default (keep it compiled in) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PSI's per-task-state accounting sits on the scheduler wakeup fast path. The container-build gRPC orchestration (buildkit/dockerd/containerd, thousands of wakeups per step) pays ~0.85 s per simple build with it on, and same-core context switch nearly doubles (ABX-498; psi=0 A/B). Keep CONFIG_PSI=y so the balloon pressure watch can still be enabled with psi=1, but default it off. Default-off leaves /proc/pressure uncreated, so the guest agent's balloon watch gets ENOENT opening it and falls back to meminfo/refault sampling — verified in-guest: the agent logs 'PSI unavailable (No such file or directory); staying on the sampling path' and the balloon keeps functioning (event-driven -> 1 s poll). Refs ABX-498. --- configs/arcbox-arm64.config | 12 +++++++++--- configs/arcbox-x86_64.config | 12 +++++++++--- scripts/build-kernel.sh | 5 +++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/configs/arcbox-arm64.config b/configs/arcbox-arm64.config index a46449e..b5e0c0b 100644 --- a/configs/arcbox-arm64.config +++ b/configs/arcbox-arm64.config @@ -22,10 +22,16 @@ CONFIG_TIMERFD=y CONFIG_EVENTFD=y CONFIG_AIO=y CONFIG_NO_HZ_IDLE=y -# Pressure stall information: in-guest memory/CPU/IO pressure signals. -# The ArcBox agent's balloon pressure watch upgrades from meminfo/refault -# sampling to PSI triggers when /proc/pressure is present. +# Pressure stall information: compiled in for the balloon pressure watch, +# but DEFAULT-DISABLED. PSI's per-task-state accounting sits on the +# scheduler wakeup fast path; left on, the container-build gRPC +# orchestration (thousands of wakeups per step) pays ~0.85 s/build and +# same-core context switch nearly doubles (ABX-498). Default-off leaves +# /proc/pressure uncreated, so the agent's balloon watch (which opens it) +# gets ENOENT and falls back to meminfo/refault sampling — the balloon +# still functions; boot psi=1 to re-enable the trigger. CONFIG_PSI=y +CONFIG_PSI_DEFAULT_DISABLED=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y diff --git a/configs/arcbox-x86_64.config b/configs/arcbox-x86_64.config index 9484d35..1142915 100644 --- a/configs/arcbox-x86_64.config +++ b/configs/arcbox-x86_64.config @@ -20,10 +20,16 @@ CONFIG_TIMERFD=y CONFIG_EVENTFD=y CONFIG_AIO=y CONFIG_NO_HZ_IDLE=y -# Pressure stall information: in-guest memory/CPU/IO pressure signals. -# The ArcBox agent's balloon pressure watch upgrades from meminfo/refault -# sampling to PSI triggers when /proc/pressure is present. +# Pressure stall information: compiled in for the balloon pressure watch, +# but DEFAULT-DISABLED. PSI's per-task-state accounting sits on the +# scheduler wakeup fast path; left on, the container-build gRPC +# orchestration (thousands of wakeups per step) pays ~0.85 s/build and +# same-core context switch nearly doubles (ABX-498). Default-off leaves +# /proc/pressure uncreated, so the agent's balloon watch (which opens it) +# gets ENOENT and falls back to meminfo/refault sampling — the balloon +# still functions; boot psi=1 to re-enable the trigger. CONFIG_PSI=y +CONFIG_PSI_DEFAULT_DISABLED=y CONFIG_HIGH_RES_TIMERS=y CONFIG_BPF_SYSCALL=y CONFIG_BPF_JIT=y diff --git a/scripts/build-kernel.sh b/scripts/build-kernel.sh index 3cd8f8d..f64b3fe 100755 --- a/scripts/build-kernel.sh +++ b/scripts/build-kernel.sh @@ -73,7 +73,8 @@ do_build() { # dependency gate in the fragment otherwise degrades silently — 6.18 did # exactly that to the legacy iptables stack via NETFILTER_XTABLES_LEGACY). for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT \ - CONFIG_IP6_NF_NAT CONFIG_HZ_1000; do + CONFIG_IP6_NF_NAT CONFIG_HZ_1000 \ + CONFIG_PSI_DEFAULT_DISABLED; do grep -q "^$sym=y" .config || { echo "ERROR: $sym missing after olddefconfig" >&2 exit 1 @@ -128,7 +129,7 @@ cd linux-$KERNEL_VERSION sh /workspace/scripts/inject-drivers.sh /workspace cp /workspace/configs/arcbox-$TARGET_ARCH.config .config make ARCH=$TARGET_ARCH olddefconfig -for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT CONFIG_IP6_NF_NAT CONFIG_HZ_1000; do +for sym in CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU CONFIG_IP_NF_NAT CONFIG_IP6_NF_NAT CONFIG_HZ_1000 CONFIG_PSI_DEFAULT_DISABLED; do grep -q \"^\$sym=y\" .config || { echo \"ERROR: \$sym missing after olddefconfig\" >&2 exit 1