diff --git a/CMakeLists.txt b/CMakeLists.txt index a1143193b..2e0957d4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -869,6 +869,33 @@ if (MI_BUILD_TESTS) # and sweep every park, so the reclaim path is hit as hard as possible add_test(NAME test-park-handoff-eager COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES_MIN_INTERVAL=0 $) + # The allocator must never touch one of its own `__thread` variables from inside an allocation: + # where `__thread` is emulated (Android before API 29) the first access on a thread mallocs, and + # with the allocator being malloc that is a re-entrant allocation (oven-sh/bun#38051). Build the + # allocator once more with emulated TLS, as malloc, in the TLS model Android uses, and link + # test/test-emulated-tls.c against that build. Only compilers that can emulate TLS (clang); the + # sanitizers bring their own malloc. + if(MI_OVERRIDE AND NOT WIN32 AND NOT (MI_TRACK_ASAN OR MI_TRACK_VALGRIND OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) + include(CheckCCompilerFlag) + check_c_compiler_flag("-femulated-tls" MI_HAS_EMULATED_TLS) + if(MI_HAS_EMULATED_TLS) + set(mi_emutls_defines ${mi_defines}) + list(FILTER mi_emutls_defines EXCLUDE REGEX "^MI_TLS_MODEL_") + add_library(mimalloc-emulated-tls STATIC ${mi_sources}) + target_compile_definitions(mimalloc-emulated-tls PRIVATE ${mi_emutls_defines} MI_STATIC_LIB MI_MALLOC_OVERRIDE MI_TLS_MODEL_PTHREADS=1) + target_compile_options(mimalloc-emulated-tls PRIVATE ${mi_cflags} -femulated-tls) + target_include_directories(mimalloc-emulated-tls PUBLIC $) + target_link_libraries(mimalloc-emulated-tls PRIVATE ${mi_libraries}) + add_executable(mimalloc-test-emulated-tls test/test-emulated-tls.c) + target_compile_definitions(mimalloc-test-emulated-tls PRIVATE ${mi_emutls_defines}) + target_compile_options(mimalloc-test-emulated-tls PRIVATE ${mi_cflags}) + target_link_libraries(mimalloc-test-emulated-tls PRIVATE mimalloc-emulated-tls ${mi_libraries}) + add_test(NAME test-emulated-tls COMMAND mimalloc-test-emulated-tls) + else() + message(STATUS "The compiler cannot emulate TLS: skipping test-emulated-tls") + endif() + endif() + # heap snapshot reader CLI (standalone, no mimalloc dependency) add_executable(mi-heapview tools/mi-heapview.c) target_compile_options(mi-heapview PRIVATE ${mi_cflags}) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 88d943d10..a5798b0a6 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -889,7 +889,7 @@ static inline bool mi_page_all_free(const mi_page_t* page) { // Page hole purging (see the "Page hole purging" section in `page.c`) // ------------------------------------------------------ -void _mi_page_purge_holes(mi_page_t* page); +void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld); // `tld`: the thread whose sweep this is (see `_mi_page_purge_holes_begin`) void _mi_page_purged_reset(mi_page_t* page); bool _mi_page_unpurge_run(mi_page_t* page); void _mi_page_unpurge_all(mi_page_t* page); @@ -898,12 +898,12 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end); size_t _mi_page_unformed_purged_bytes(const mi_page_t* page); // the bytes of this page's unformed tail that are discarded right now bool _mi_page_purge_os_page_blocks(size_t os_page_size, size_t block_size, uintptr_t page_start, size_t capacity, size_t k, size_t* first, size_t* last); -bool _mi_page_purge_holes_in_progress(void); +bool _mi_page_purge_holes_in_progress(void); // is the calling thread inside a sweep of its own heaps? void _mi_page_holes_count_page_freed(void); void _mi_page_holes_count_ineligible(const mi_page_t* page); void _mi_page_holes_reset_ineligible(void); -void _mi_page_purge_holes_begin(void); -void _mi_page_purge_holes_end(void); +void _mi_page_purge_holes_begin(mi_tld_t* tld); // around each pass of a sweep; `tld` is the thread being swept +void _mi_page_purge_holes_end(mi_tld_t* tld); void _mi_page_purge_holes_sweep_begin(mi_tld_t* tld); // once per idle sweep, before its passes // ------------------------------------------------------ diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index d7c2689be..11fa776bb 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -709,6 +709,16 @@ struct mi_tld_s { mi_tld_t* subproc_next; // list of tlds in the subproc, so the scavenger can find parked threads size_t holes_sweep_seq; // idle sweeps run over THIS tld's heaps (paces `purge_holes_full_every`) mi_msecs_t holes_sweep_last; // when this tld's heaps were last swept (paces `purge_holes_min_interval`) + + // State of the sweep that is currently running over this tld's heaps (on the owner, or on the + // scavenger while the owner is parked). It lives here and not in thread-locals of the sweeping + // thread: `_mi_page_purge_holes_in_progress` is read inside the allocator, and where `__thread` + // is emulated (Android before API 29) the first access to a thread-local on a thread calls + // malloc, which re-enters the very page collect that is reading it (oven-sh/bun#38051). + bool holes_sweeping; // a pass of the sweep is rewriting free lists right now (see `_mi_page_purge_holes_begin`) + bool holes_sweep_full; // this sweep ignores `page->swept_state` (see `_mi_page_purge_holes`) + size_t holes_sweep_skipped; // per-sweep counters, folded into the process-wide ones in `_mi_page_purge_holes_end` + size_t holes_sweep_visited; }; diff --git a/src/arena.c b/src/arena.c index af7e8fed5..7845abbe9 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1329,7 +1329,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { typedef struct mi_purge_holes_arg_s { mi_bitmap_t* bitmap; - mi_tld_t* tld; // whose park we are sweeping under; NULL when not a parked sweep + mi_tld_t* tld; // the thread whose sweep this is (its own, or the parked one the scavenger is sweeping for) } mi_purge_holes_arg_t; static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { @@ -1337,7 +1337,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, mi_purge_holes_arg_t* const parg = (mi_purge_holes_arg_t*)arg; // this pass holds every page that ever became full, so it is most of a cold sweep: an owner // waiting in `_mi_park_leave` cannot allocate until we stop - if (parg->tld != NULL && mi_atomic_load_relaxed(&parg->tld->park_reclaim) != 0) return false; + if (mi_atomic_load_relaxed(&parg->tld->park_reclaim) != 0) return false; mi_bitmap_t* const bitmap = parg->bitmap; // Take the page out of the abandoned map first: this is the reader side of the protocol in @@ -1353,8 +1353,8 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, } // We own the page: no other thread can reclaim, unabandon, or free it now, and only the - // atomic `xthread_free` can still change under us. - _mi_page_free_collect(page, true); + // atomic `xthread_free` can still change under us. (No un-purging: we are about to purge.) + _mi_page_free_collect_no_unpurge(page, true); if (mi_page_all_free(page)) { mi_bitmap_set(bitmap, slice_index); // `_mi_arenas_page_unabandon` expects it in the map _mi_arenas_page_unabandon(page, NULL); @@ -1362,7 +1362,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, _mi_page_holes_count_page_freed(); return true; } - _mi_page_purge_holes(page); + _mi_page_purge_holes(page, parg->tld); mi_bitmap_set(bitmap, slice_index); // back in the map *before* unowning: unown may free the page mi_abandoned_page_unown(page, NULL); return true; @@ -1372,9 +1372,9 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, // A page abandoned while full is not mapped; it has no free blocks at that point, and once // enough blocks are freed in it, `_mi_arenas_page_try_reabandon_to_mapped` puts it in the map. void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { - if (heap == NULL) return; + if (heap == NULL || tld == NULL) return; if (!mi_option_is_enabled(mi_option_purge_holes)) return; - _mi_page_purge_holes_begin(); + _mi_page_purge_holes_begin(tld); mi_forall_arenas(heap, ((mi_arena_t*)NULL), 0, arena) { mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); if (arena_pages != NULL) { @@ -1389,7 +1389,7 @@ void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { } } mi_forall_arenas_end(); - _mi_page_purge_holes_end(); + _mi_page_purge_holes_end(tld); } // The read-only counterpart of the sweep above: account for the holes in the abandoned pages diff --git a/src/page.c b/src/page.c index 382d7d92c..42a659ab5 100644 --- a/src/page.c +++ b/src/page.c @@ -472,44 +472,54 @@ static void mi_holes_count_reuse(size_t bytes, size_t blocks, bool reused) { mi_atomic_addi64_relaxed(&mi_holes_blocks, -(int64_t)blocks); } -// Re-entrancy guard: while the idle sweep is rewriting a page's free list and -// bitmap, a nested `mi_malloc` (only reachable through a user output function -// from a warning message) must not un-purge a hole from under it. -static mi_decl_thread bool mi_purging_holes; - -// Per-sweep counters. The sweep runs over every page of the thread, so a process-wide atomic -// per page would be a real cost on the very path we are making cheap: accumulate thread-locally -// and fold them in once per pass, in `_mi_page_purge_holes_end`. -static mi_decl_thread size_t mi_holes_sweep_skipped; -static mi_decl_thread size_t mi_holes_sweep_visited; - -// Whether THIS sweep ignores `page->swept_state` (see `_mi_page_purge_holes`). Set once per idle -// sweep, in `_mi_page_purge_holes_sweep_begin`; the sequence it is paced by lives on the tld being -// swept, since one scavenger thread runs the sweeps of many. -static mi_decl_thread bool mi_holes_sweep_full; - -bool _mi_page_purge_holes_in_progress(void) { return mi_purging_holes; } -void _mi_page_purge_holes_begin(void) { mi_assert_internal(!mi_purging_holes); mi_purging_holes = true; } - -void _mi_page_purge_holes_end(void) { - mi_assert_internal(mi_purging_holes); - mi_purging_holes = false; - if (mi_holes_sweep_skipped > 0) { - mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)mi_holes_sweep_skipped); - mi_holes_sweep_skipped = 0; - } - if (mi_holes_sweep_visited > 0) { - mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)mi_holes_sweep_visited); - mi_holes_sweep_visited = 0; - } -} - -// Called once per idle sweep of `tld`'s heaps, before its passes (`mi_purge_holes_of`). +// The state of a running sweep lives on the tld being swept (`tld->holes_sweep*`, see `types.h`), +// never in thread-locals of the sweeping thread. Besides the scavenger sweeping many tlds from one +// thread, this must not touch a `__thread` variable at all: `_mi_page_purge_holes_in_progress` is +// read from `mi_page_free_collect_ex`, inside the allocator, and on targets where `__thread` is +// emulated (Android before API 29) the first access on a thread allocates -- re-entering the +// collect that is reading it, without bound (oven-sh/bun#38051). The tld of the calling thread is +// reached through the default theap, which every TLS model can read without allocating. + +// Re-entrancy guard: while a sweep is rewriting a page's free list and bitmap, a nested `mi_malloc` +// on the sweeping thread (only reachable through a user output function from a warning message) +// must not un-purge a hole from under it. That nested allocation comes out of the calling thread's +// own theaps, so it is its own tld that matters here: on the owner that is the tld being swept; +// the scavenger has no theaps of its own being swept (it sweeps a parked thread's theaps, and the +// abandoned pages it touches are claimed), so un-purging there is harmless. +bool _mi_page_purge_holes_in_progress(void) { + mi_theap_t* const theap = _mi_theap_default(); + if (theap == NULL || theap->tld == NULL) return false; + return theap->tld->holes_sweeping; +} + +void _mi_page_purge_holes_begin(mi_tld_t* tld) { + mi_assert_internal(tld != NULL && !tld->holes_sweeping); + tld->holes_sweeping = true; +} + +// Also folds the per-sweep counters into the process-wide ones. The sweep runs over every page of +// the thread, so a process-wide atomic per page would be a real cost on the very path we are making +// cheap: they accumulate on the tld and are folded in once per pass. +void _mi_page_purge_holes_end(mi_tld_t* tld) { + mi_assert_internal(tld != NULL && tld->holes_sweeping); + tld->holes_sweeping = false; + if (tld->holes_sweep_skipped > 0) { + mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)tld->holes_sweep_skipped); + tld->holes_sweep_skipped = 0; + } + if (tld->holes_sweep_visited > 0) { + mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)tld->holes_sweep_visited); + tld->holes_sweep_visited = 0; + } +} + +// Called once per idle sweep of `tld`'s heaps, before its passes (`mi_purge_holes_of`): decides +// whether this sweep ignores `page->swept_state` (see `_mi_page_purge_holes`). void _mi_page_purge_holes_sweep_begin(mi_tld_t* tld) { const long every = mi_option_get(mi_option_purge_holes_full_every); const size_t seq = ++tld->holes_sweep_seq; - mi_holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0); - if (mi_holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); } + tld->holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0); + if (tld->holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); } } static inline bool mi_page_bits_at(const uint64_t* bits, size_t k) { @@ -678,7 +688,7 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end) { // Walk the free list of a page and discard every OS page in it that holds no live block. // Returns false if any discard failed: those blocks went straight back on the free list and the // page must be swept again, so the caller must not record it as swept. -static bool mi_page_purge_holes_walk(mi_page_t* page) { +static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { if (page->free == NULL) return true; // nothing to take off the free list const size_t os_size = _mi_os_page_size(); @@ -703,7 +713,7 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) { nfree[k]++; } } - mi_holes_sweep_visited += nvisited; // folded into the process-wide counter at the end of the sweep + tld->holes_sweep_visited += nvisited; // folded into the process-wide counter at the end of the pass // 2. an OS page can be discarded when *every* block overlapping it is free -- either on the // free list, or purged already. Of the blocks overlapping an OS page, only the first and @@ -789,26 +799,27 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) { // regardless, which caps the delay of a missed discard at N parks for 1/N of the old cost. // (An exact "was anything freed in this page" bit is the alternative, and it costs a store in // `mi_free` itself -- the hot path this whole feature stays off.) -void _mi_page_purge_holes(mi_page_t* page) { - mi_assert_internal(page != NULL); +void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld) { + mi_assert_internal(page != NULL && tld != NULL); + mi_assert_internal(tld->holes_sweeping); if (!mi_option_is_enabled(mi_option_purge_holes)) return; if (mi_page_all_free(page)) return; // the page itself is about to be freed if (mi_option_get(mi_option_purge_delay) < 0) return; // purging disabled mi_page_purge_unformed_tail(page); // the blocks that are not formed yet: resident, but never handed out if (!mi_page_can_purge_holes(page)) { _mi_page_holes_count_ineligible(page); return; } - if (!mi_holes_sweep_full && page->swept_state == mi_page_sweep_state(page)) { - mi_holes_sweep_skipped++; // nothing was allocated or freed in this page since we swept it + if (!tld->holes_sweep_full && page->swept_state == mi_page_sweep_state(page)) { + tld->holes_sweep_skipped++; // nothing was allocated or freed in this page since we swept it return; } // Record the state we LEAVE the page in, read back from the page: a nested `mi_malloc` (see - // `mi_purging_holes`) may have taken a block out of it while we walked. + // `_mi_page_purge_holes_in_progress`) may have taken a block out of it while we walked. // // Only if the walk got everything. A failed `_mi_os_discard` (ENOMEM under pressure) puts its // blocks straight back, and changes neither `capacity` nor `used` -- so recording here would // say "already swept" for a page that still has holes, and the skip check would then park them // until the next full sweep, or forever with `purge_holes_full_every=0`. - if (mi_page_purge_holes_walk(page)) { + if (mi_page_purge_holes_walk(page, tld)) { page->swept_state = mi_page_sweep_state(page); } } diff --git a/src/theap.c b/src/theap.c index 1a0a9463d..fc37cfcdd 100644 --- a/src/theap.c +++ b/src/theap.c @@ -165,20 +165,23 @@ void _mi_theap_collect_abandon(mi_theap_t* theap) { // that are still partially used. Meant to be called when the application knows // it is idle (e.g. from an event loop about to park): it costs a few madvise // calls and nothing on the alloc/free hot path. -static bool mi_theap_page_purge_holes(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg1, void* arg2) { - MI_UNUSED(arg1); MI_UNUSED(arg2); +static bool mi_theap_page_purge_holes(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_t* page, void* arg_tld, void* arg2) { + MI_UNUSED(arg2); + mi_tld_t* const tld = (mi_tld_t*)arg_tld; // the tld being swept (== theap->tld) // When the scavenger is doing this for a parked thread, the owner may wake at any moment and // has to wait for us. Stopping between pages bounds that wait to one page's walk; the pages we // skip are simply swept at the next park (`swept_state` makes the re-walk cheap). if (theap->tld != NULL && mi_atomic_load_relaxed(&theap->tld->park_reclaim) != 0) return false; - _mi_page_free_collect(page, true); // force: fold local_free (and thread_free) into `free` first + // force: fold local_free (and thread_free) into `free` first. Never un-purge here: we are about + // to purge, and a run brought back now would be discarded again right away (see `mi_theap_page_collect`). + _mi_page_free_collect_no_unpurge(page, true); if (mi_page_all_free(page)) { // the forced collect emptied the page: hand it back instead of leaving it resident _mi_page_holes_count_page_freed(); _mi_page_free(page, pq); return true; } - _mi_page_purge_holes(page); + _mi_page_purge_holes(page, tld); mi_assert_expensive(_mi_page_is_valid(page)); return true; // continue } @@ -193,9 +196,10 @@ static void mi_theap_purge_holes(mi_theap_t* theap) mi_attr_noexcept { if (theap->tld == NULL) return; if (theap->tld->thread_id != _mi_thread_id() && mi_atomic_load_acquire(&theap->tld->park_state) != MI_PARK_SWEEPING) return; - _mi_page_purge_holes_begin(); - mi_theap_visit_pages(theap, &mi_theap_page_purge_holes, true /* include full pages */, NULL, NULL); - _mi_page_purge_holes_end(); + mi_tld_t* const tld = theap->tld; + _mi_page_purge_holes_begin(tld); + mi_theap_visit_pages(theap, &mi_theap_page_purge_holes, true /* include full pages */, tld, NULL); + _mi_page_purge_holes_end(tld); } // Purge the holes in every page this thread may safely touch: diff --git a/test/test-emulated-tls.c b/test/test-emulated-tls.c new file mode 100644 index 000000000..277986a3c --- /dev/null +++ b/test/test-emulated-tls.c @@ -0,0 +1,184 @@ +// The allocator must not touch a `__thread` variable of its own from inside an allocation or a +// page collect. On targets where `__thread` is emulated (Android before API 29, which is what +// bun's Android build targets), every access goes through `__emutls_get_address`, and the first +// access on a thread materializes the variable -- and the thread's per-variable address array -- +// with malloc(). With mimalloc being malloc, that re-enters the allocator from wherever the access +// sits. The idle sweep's guard used to be such a variable, read from `mi_page_free_collect_ex`: +// the collect called into emulated TLS, emulated TLS called malloc, malloc had to collect a page +// of exactly the size class it was asked for, that collect read the (still unmaterialized) guard +// again, and so on until the stack was gone (oven-sh/bun#38051). +// +// This binary is linked against a build of the allocator compiled with `-femulated-tls` and +// malloc overriding (see CMakeLists.txt), so emulated TLS allocates from the allocator under test, +// exactly as on Android. It then arranges for a thread's first page collect to happen while the +// page of whichever size class emulated TLS is going to ask for is exhausted. Every small size +// class gets a turn, so the test does not depend on the sizes a particular emutls runtime asks for +// (compiler-rt and libgcc differ), and a bug of this kind shows up as a stack overflow instead of +// a failed check. With a correct allocator nothing here ever reaches emulated TLS at all. +#include +#include +#include +#include +#include +#include +#include +#include + +// Request sizes covering every size bin up to 1 KiB (several of these share a bin; part 2 below +// only uses one request size per bin). +static const size_t sizes[] = { 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, + 320, 384, 448, 512, 640, 768, 896, 1024 }; +#define NSIZES (sizeof(sizes) / sizeof(sizes[0])) +// Per class: stays inside one (64 KiB) small page, yet spans several OS pages even with 16 KiB ones. +#define BYTES_PER_CLASS (48 * 1024) +#define MAX_BLOCKS (BYTES_PER_CLASS / 8) +#define EXTEND_BYTES 4096 // a page formats this many bytes of blocks at a time (MI_MAX_EXTEND_SIZE) + +static void* blocks[MAX_BLOCKS]; +static uintptr_t os_page; // the OS page size: the granularity at which the sweep discards + +static void run_thread(void* (*fun)(void*), void* arg) { + pthread_t th; + if (pthread_create(&th, NULL, fun, arg) != 0) { printf(" pthread_create failed\n"); exit(2); } + pthread_join(th, NULL); +} + +// --------------------------------------------------------------------------------------------- +// 1. A fresh thread allocates one size class until its page runs out of formatted blocks. The +// allocation that runs out collects the page (`mi_page_queue_find_free_ex`), which is the first +// collect on this thread. An optimizing compiler hoists a thread-local read in that collect +// ahead of the conditions guarding it, so with the bug this is where emulated TLS allocates, +// and the thread whose size class matches the emutls request recurses into the same exhausted +// page without bound. +// --------------------------------------------------------------------------------------------- +static void* exhaust_one_class(void* arg) { + const size_t size = *(const size_t*)arg; + const size_t n = BYTES_PER_CLASS / size; + for (size_t i = 0; i < n; i++) { + blocks[i] = malloc(size); + if (blocks[i] == NULL) { printf(" out of memory\n"); exit(2); } + memset(blocks[i], (int)i, size); + } + for (size_t i = 0; i < n; i++) free(blocks[i]); + return NULL; +} + +// --------------------------------------------------------------------------------------------- +// 2. The unoptimized variant of the same read only happens for a page with purged holes, so also +// produce those: a fresh thread leaves, in every size class, a page whose free blocks all sit in +// OS pages without a live block, parks, and lets the scavenger sweep it. The sweep discards +// those OS pages and takes every one of those blocks off the free lists. The next allocation in +// any class has to collect a page with purged holes and an empty free list; with the bug that +// reads the guard, emulated TLS allocates, and that allocation lands on another such page. +// --------------------------------------------------------------------------------------------- +static int cmp_ptr(const void* a, const void* b) { + const uintptr_t x = *(const uintptr_t*)a, y = *(const uintptr_t*)b; + return (x < y ? -1 : (x > y ? 1 : 0)); +} + +typedef struct { void** live; size_t count; } kept_t; + +// Allocate a fresh page's worth of one bin, then keep exactly the blocks lying entirely inside the +// first and the last OS page the blocks touch (those two OS pages can never be discarded, so their +// blocks must not be free either) and free everything else: every OS page in between then holds +// only free blocks. The count is a whole number of extensions so that no formatted block is left +// over next to the kept ones, which would stay on the free list and serve the allocation later. +static void prepare_class(size_t size, kept_t* kept) { + static uintptr_t addrs[MAX_BLOCKS]; + addrs[0] = (uintptr_t)malloc(size); + addrs[1] = (uintptr_t)malloc(size); + if (addrs[0] == 0 || addrs[1] == 0) { printf(" out of memory\n"); exit(2); } + const size_t stride = (addrs[1] > addrs[0] ? addrs[1] - addrs[0] : addrs[0] - addrs[1]); // the block size, padding included + if (stride < size || stride > 2 * size + 64) { printf(" unexpected block layout for size %zu (stride %zu)\n", size, stride); exit(2); } + const size_t per_extension = EXTEND_BYTES / stride; + const size_t n = (BYTES_PER_CLASS / EXTEND_BYTES) * per_extension; + for (size_t i = 2; i < n; i++) { + addrs[i] = (uintptr_t)malloc(size); + if (addrs[i] == 0) { printf(" out of memory\n"); exit(2); } + } + qsort(addrs, n, sizeof(addrs[0]), &cmp_ptr); + const uintptr_t first_page = addrs[0] & ~(os_page - 1); + const uintptr_t last_page = (addrs[n - 1] + stride - 1) & ~(os_page - 1); + kept->count = 0; + for (size_t i = 0; i < n; i++) { + const uintptr_t lo = addrs[i], hi = lo + stride; + const int in_first = (lo >= first_page && hi <= first_page + os_page); + const int in_last = (lo >= last_page && hi <= last_page + os_page); + if (in_first || in_last) { kept->live[kept->count++] = (void*)lo; } + else { free((void*)lo); } + } +} + +static void* purged_pages_in_every_class(void* arg) { + (void)arg; + static void* live[NSIZES][MAX_BLOCKS]; // what prepare_class keeps alive, per class + static size_t class_size[NSIZES]; // one request size per bin + kept_t kept[NSIZES]; + size_t nclasses = 0; + for (size_t s = 0; s < NSIZES; s++) { + if (s > 0 && mi_good_size(sizes[s]) == mi_good_size(sizes[s - 1])) continue; // same bin as the previous size + class_size[nclasses] = sizes[s]; + kept[nclasses].live = live[nclasses]; + prepare_class(sizes[s], &kept[nclasses]); + nclasses++; + } + + mi_purge_holes_stats_t before, now; + mi_purge_holes_stats_get(&before); + if (!mi_on_thread_idle_start()) { + printf(" (no scavenger to hand the heaps to: skipping the purged-pages part)\n"); + } + else { + // Wait for the scavenger to sweep us: until the discard count has risen and then stayed put + // for a while (taking the heaps back stops a sweep that is still running). Nothing in this + // loop may allocate. + const struct timespec ms = { 0, 1000 * 1000 }; + size_t last = before.purged_blocks; + int stable = 0, waited = 0; + do { + nanosleep(&ms, NULL); + mi_purge_holes_stats_get(&now); + if (now.purged_blocks == last) { if (last > before.purged_blocks) stable++; } + else { last = now.purged_blocks; stable = 0; } + } while (stable < 100 && ++waited < 20000); + mi_on_thread_idle_end(); + if (now.purged_blocks <= before.purged_blocks) { + printf(" (the scavenger did not sweep this thread within 20s: skipping the purged-pages part)\n"); + } + else { + printf(" swept: %zu blocks held off the free lists in %zu classes\n", (size_t)(now.purged_blocks - before.purged_blocks), nclasses); + for (size_t c = 0; c < nclasses; c++) { + void* p = malloc(class_size[c]); // collects a page with purged holes and an empty free list + if (p == NULL) { printf(" out of memory\n"); exit(2); } + memset(p, 1, class_size[c]); + free(p); + } + mi_purge_holes_stats_t after; + mi_purge_holes_stats_get(&after); + printf(" handed back: %zu hole runs\n", (size_t)(after.reuse_calls - now.reuse_calls)); + if (after.reuse_calls == now.reuse_calls) { + // then the allocations above never collected a purged page and this part tested nothing + printf(" FAILED: the allocations after the sweep did not reach a page with purged holes\n"); + exit(1); + } + } + } + for (size_t c = 0; c < nclasses; c++) { + for (size_t i = 0; i < kept[c].count; i++) free(kept[c].live[i]); + } + return NULL; +} + +int main(void) { + const long ps = sysconf(_SC_PAGESIZE); + os_page = (ps > 0 ? (uintptr_t)ps : 4096); + printf("exhausting one size class per fresh thread...\n"); + for (size_t s = 0; s < NSIZES; s++) { + size_t size = sizes[s]; + run_thread(&exhaust_one_class, &size); + } + printf("collecting pages with purged holes on a fresh thread...\n"); + run_thread(&purged_pages_in_every_class, NULL); + printf("ok\n"); + return 0; +}