From d035f8ba8413aa75f1bd66bbb61aa36703b76b9a Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Wed, 19 Aug 2026 01:03:24 +0000 Subject: [PATCH] unix: only opt mappings out of THP when the system setting is always With allow_thp off, unix_mmap called madvise(MADV_NOHUGEPAGE) on every mapping it created. The kernel only backs a mapping that did not ask for huge pages with them when /sys/kernel/mm/transparent_hugepage/enabled is [always]. Under [madvise] (the Debian and Ubuntu default) and [never] the call changes nothing and costs one syscall per mmap: three during process initialization and one per arena after that. unix_detect_thp already reads that file. It now also records whether the setting is [always], and unix_mmap makes the madvise call only in that case. When the file cannot be read (no sysfs), the opt-out stays in place. The per-size mTHP settings are not read: that would take more syscalls than it saves. test-thp-optout interposes madvise, counts the MADV_NOHUGEPAGE calls the allocator makes during initialization (the ctest entry runs it with MIMALLOC_ALLOW_THP=0) and for a fresh reservation, and checks them against the setting of the machine it runs on. With allow_thp on it expects none. --- CMakeLists.txt | 4 +- src/prim/unix/prim.c | 19 ++++++-- test/test-thp-optout.c | 107 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 test/test-thp-optout.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 96adcf5b1..568959417 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -874,7 +874,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-delete-race heap-churn fork-user-heap snapshot prof prof-adversarial theap-sentinel purge-holes purge-zero park-handoff) + set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-delete-race heap-churn fork-user-heap snapshot prof prof-adversarial theap-sentinel purge-holes purge-zero park-handoff thp-optout) if (MI_DEBUG_FULL OR CMAKE_BUILD_TYPE MATCHES "Debug") list(APPEND mi_static_tests commit-fail) # uses mi_debug_fail_os_commit_after (MI_DEBUG>0 only) endif() @@ -894,6 +894,8 @@ if (MI_BUILD_TESTS) add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1 $) elseif(TEST_NAME STREQUAL "stress-heaps") add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=1 $) + elseif(TEST_NAME STREQUAL "thp-optout") + add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ALLOW_THP=0 $) # off from the start: covers the mappings made during initialization too else() add_test(NAME test-${TEST_NAME} COMMAND mimalloc-test-${TEST_NAME}) endif() diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 83e1add7e..fbd33d7f9 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -152,6 +152,16 @@ static bool unix_detect_overcommit(void) { return os_overcommit; } +#if defined(__linux__) && defined(MADV_NOHUGEPAGE) +// Whether `unix_mmap` has to opt its mappings out of transparent huge pages with `MADV_NOHUGEPAGE` +// when `allow_thp` is off. The kernel only gives huge pages to a mapping that did not ask for them +// under `enabled = [always]`; under `[madvise]` (the Debian/Ubuntu default) and `[never]` the call +// would just cost one syscall per mmap. Starts out true so the opt-out stays in place where the +// setting cannot be read (no sysfs). The per-size mTHP settings (`hugepages-*kB/enabled`) are not +// read: that takes more syscalls at startup than the opt-outs it could save. +static bool unix_thp_needs_optout = true; +#endif + static bool unix_detect_thp(void) { bool thp_enabled = false; #if defined(__linux__) @@ -165,6 +175,9 @@ static bool unix_detect_thp(void) { if (nread >= 1) { if (nread > 64) { nread = 64; } thp_enabled = (_mi_strnstr(buf,nread,"[never]") == NULL); + #if defined(MADV_NOHUGEPAGE) + unix_thp_needs_optout = (_mi_strnstr(buf,nread,"[always]") != NULL); + #endif } } #endif @@ -477,10 +490,10 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec } #endif #if defined(__linux__) && defined(MADV_NOHUGEPAGE) - if (p != NULL && !mi_option_is_enabled(mi_option_allow_thp)) { + if (p != NULL && !mi_option_is_enabled(mi_option_allow_thp) && unix_thp_needs_optout) { // VMA-level opt-out (dropped at execve, so children keep the system policy). Under THP - // `always`/mTHP this avoids 2MiB faults into sparse regions and huge-page splits on - // sub-2MiB purges, and keeps khugepaged from re-collapsing purged ranges; else a no-op. + // `always` this avoids 2MiB faults into sparse regions and huge-page splits on sub-2MiB + // purges, and keeps khugepaged from re-collapsing purged ranges. (void)unix_madvise(p, size, MADV_NOHUGEPAGE); } #endif diff --git a/test/test-thp-optout.c b/test/test-thp-optout.c new file mode 100644 index 000000000..268270d2c --- /dev/null +++ b/test/test-thp-optout.c @@ -0,0 +1,107 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. +-----------------------------------------------------------------------------*/ + +/* With `allow_thp` off, `unix_mmap` (src/prim/unix/prim.c) opts the mappings it creates out of + transparent huge pages with madvise(MADV_NOHUGEPAGE). The kernel only gives huge pages to a + mapping that did not ask for them when /sys/kernel/mm/transparent_hugepage/enabled is + `[always]`, so that is the only setting under which the allocator may make the call. Under + `[madvise]` and `[never]` it must not: there the call is one wasted syscall per mmap. When the + setting cannot be read, the allocator must keep opting out. + + The test interposes `madvise` (the executable's definition wins over libc's, also for the calls + made from the static library) and counts the MADV_NOHUGEPAGE calls the allocator makes: + + - during process initialization, when `allow_thp` was already off at that point (the ctest entry + runs with MIMALLOC_ALLOW_THP=0; a build with MI_NO_THP gets the same), + - for a fresh OS reservation made with `allow_thp` off, + - and, as a control, for a reservation made with `allow_thp` on: never. + + Expected with the bug present, on a system whose setting is `[madvise]` or `[never]`: the first + two counts are not zero. Linux only: elsewhere the test passes without doing anything. */ + +#if !defined(__linux__) +#include +int main(void) { printf("test-thp-optout: skipped, transparent huge pages are Linux only\n"); return 0; } +#else + +#ifndef _DEFAULT_SOURCE +#define _DEFAULT_SOURCE // madvise, MADV_NOHUGEPAGE +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +static long nohugepage_calls; // counted from process start: the allocator initializes before main + +int madvise(void* addr, size_t len, int advice) { + if (advice == MADV_NOHUGEPAGE) { nohugepage_calls++; } + return (int)syscall(SYS_madvise, addr, len, advice); +} + +// Returns the current value between the brackets, or "" when the setting cannot be read. +static const char* thp_setting(char* buf, size_t bufsize) { + int fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); + if (fd < 0) return ""; + ssize_t n = read(fd, buf, bufsize - 1); + close(fd); + if (n <= 0) return ""; + buf[n] = 0; + char* start = strchr(buf, '['); + char* end = (start == NULL ? NULL : strchr(start, ']')); + if (end == NULL) return ""; + *end = 0; + return start + 1; +} + +static int failures; + +static void check(const char* what, long calls, bool expect_calls) { + const bool ok = (expect_calls ? calls > 0 : calls == 0); + printf("%s %s: %ld MADV_NOHUGEPAGE call(s), expected %s\n", (ok ? "ok " : "FAIL"), what, calls, (expect_calls ? "at least one" : "none")); + if (!ok) failures++; +} + +// A fresh arena reservation always creates at least one new mapping. Uncommitted: only address space. +static long nohugepage_calls_for_one_reservation(void) { + const long before = nohugepage_calls; + if (mi_reserve_os_memory(mi_arena_min_size(), false /* commit */, false /* allow_large */) != 0) { + printf("FAIL mi_reserve_os_memory failed\n"); + failures++; + } + return nohugepage_calls - before; +} + +int main(void) { + char buf[128]; + const char* setting = thp_setting(buf, sizeof(buf)); + const bool readable = (setting[0] != 0); + const bool expect_optout = (!readable || strcmp(setting, "always") == 0); + printf("transparent_hugepage/enabled: %s\n", (readable ? setting : "(not readable)")); + + mi_free(mi_malloc(1)); // make sure the allocator is initialized + const long at_startup = nohugepage_calls; + if (!mi_option_is_enabled(mi_option_allow_thp)) { + check("process initialization with allow_thp off", at_startup, expect_optout); + } + else { + printf("skip process initialization: allow_thp was on (run with MIMALLOC_ALLOW_THP=0 to cover it)\n"); + } + + mi_option_disable(mi_option_allow_thp); + check("reservation with allow_thp off", nohugepage_calls_for_one_reservation(), expect_optout); + + mi_option_enable(mi_option_allow_thp); + check("reservation with allow_thp on", nohugepage_calls_for_one_reservation(), false); + + return (failures == 0 ? 0 : 1); +} + +#endif