From 787be2a892a5035c95f2057e2cb551f356ffc683 Mon Sep 17 00:00:00 2001 From: robobun Date: Wed, 2 Sep 2026 20:14:05 +0000 Subject: [PATCH 1/7] arena: allocate the per-bin abandoned-page bitmaps on first abandon Every (heap, arena) pair carried MI_ARENA_BIN_COUNT abandoned-page bitmaps next to its `pages` bitmap, laid out and initialized up front: once in the arena's info slices for the main heap, and once more in a zeroed allocation for every other heap that touched the arena (`mi_heap_ensure_arena_pages`). Each bitmap is sized by the arena (2 KiB per GiB of arena), so with the default 1 GiB reservation that is ~110 KiB per heap, and ~410 KiB for a heap in a 4 GiB arena. Writing the header of each bitmap touched one OS page per bitmap, so a heap paid about 50 page faults (or the memset, for external memory that is not known to be zero) before its first allocation, and most heaps never abandon a page at all: short-lived heaps are destroyed, not abandoned. Bun creates one heap per transpile and one for JSC's 4 GiB structure heap, so this was about 250 of the ~1000 page faults of `bun file.js`. Now `pages_abandoned[bin]` starts NULL and is allocated the first time a page of that bin is abandoned, from the subproc meta-data heap (safe on the abandon paths, where a regular heap allocation is not), published with a CAS. Readers treat NULL as all-clear; they were already guarded by `heap->abandoned_count[bin]`. `_mi_arena_pages_free` frees them with the heap's arena pages. If the allocation fails the page is abandoned unmapped, like a full page, and mapped once a free brings it back. The new test-abandoned-lazy exercises the abandon, reclaim, re-abandon, visit and delete paths from several threads at once. --- CMakeLists.txt | 2 +- include/mimalloc/internal.h | 1 + include/mimalloc/types.h | 9 +- src/arena.c | 98 ++++++++++++++---- src/heap.c | 2 +- test/test-abandoned-lazy.c | 200 ++++++++++++++++++++++++++++++++++++ 6 files changed, 287 insertions(+), 25 deletions(-) create mode 100644 test/test-abandoned-lazy.c diff --git a/CMakeLists.txt b/CMakeLists.txt index eee0d7d5b..361ebe0e7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -877,7 +877,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-aba heap-burst-destroy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init) + set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init) if(NOT (MI_DEBUG_TSAN OR MI_TRACK_ASAN OR MI_DEBUG_UBSAN)) list(APPEND mi_static_tests thp-optout) # counts madvise calls by interposing it, which a sanitizer runtime does first endif() diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 903d4a235..65e0b3bcb 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -277,6 +277,7 @@ void _mi_arenas_abandoned_page_free(mi_page_t* page, mi_theap_t* curren void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap); void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */); bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page); +void _mi_arena_pages_free(mi_arena_pages_t* arena_pages); size_t mi_arenas_get_count(mi_subproc_t* subproc); uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index); diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index f3b3b84cd..e03e3bfa0 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -756,8 +756,13 @@ typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in struct mi_arena_pages_s { mi_bitmap_t* pages; // all registered pages (abandoned and owned) - mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page) - // followed by the bitmaps (whose siz`es depend on the arena size) + // Abandoned pages per size bin (a set bit means the start of the page). Each bitmap is + // allocated the first time a page of that bin is abandoned (`mi_arena_pages_abandoned_ensure`); + // NULL means no page of that bin was ever abandoned. Eagerly laying out all MI_ARENA_BIN_COUNT + // of them cost a page fault per bitmap (one header write each, on its own OS page) for every + // heap that touched an arena, and most heaps never abandon a page. + _Atomic(mi_bitmap_t*) pages_abandoned[MI_ARENA_BIN_COUNT]; + // followed by the `pages` bitmap (whose size depends on the arena size) }; diff --git a/src/arena.c b/src/arena.c index 70a567f4d..3c378aba1 100644 --- a/src/arena.c +++ b/src/arena.c @@ -29,6 +29,9 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo #error "The page_t.page_ma_offset field is not large enough to cover a full arena" #endif +static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size_t bin); +static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin); + /* ----------------------------------------------------------- Arena id's ----------------------------------------------------------- */ @@ -726,9 +729,9 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, arena) { mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); - if (arena_pages != NULL) { + mi_bitmap_t* const bitmap = (arena_pages != NULL ? mi_arena_pages_abandoned(arena_pages, bin) : NULL); + if (bitmap != NULL) { size_t slice_index; - mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin]; if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) { // found an abandoned page of the right size @@ -1119,7 +1122,7 @@ static void mi_arenas_page_free_prim(mi_page_t* page, mi_subproc_t* subproc, mi_ const size_t bin = _mi_bin(mi_page_block_size(page)); mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); - mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1)); + mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_arena_pages_abandoned(arena_pages, bin) == NULL || mi_bitmap_is_clearN(mi_arena_pages_abandoned(arena_pages, bin), slice_index, 1)); // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may // be (temporarily) not true if the free happens while trying to reclaim // see `mi_arena_try_claim_abandoned` @@ -1224,13 +1227,18 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); - mi_page_set_abandoned_mapped(page); - const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index); - MI_UNUSED(was_clear); mi_assert_internal(was_clear); - mi_atomic_increment_relaxed(&heap->abandoned_count[bin]); - mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); - mi_abandoned_page_unown(page, current_theapx); - return; + // If the bin's bitmap cannot be allocated the page is abandoned unmapped, like a full + // page: it is still reachable through `pages` and is reclaimed once a block in it is freed. + mi_bitmap_t* const bitmap = mi_arena_pages_abandoned_ensure(arena, arena_pages, bin); + if mi_likely(bitmap != NULL) { + mi_page_set_abandoned_mapped(page); + const bool was_clear = mi_bitmap_set(bitmap, slice_index); + MI_UNUSED(was_clear); mi_assert_internal(was_clear); + mi_atomic_increment_relaxed(&heap->abandoned_count[bin]); + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); + return; + } } } // otherwise, @@ -1298,7 +1306,9 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); // this busy waits until a concurrent reader (from alloc_abandoned) is done - mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index); + mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin); + mi_assert_internal(bitmap != NULL); // a mapped page was set in it + mi_bitmap_clear_once_set(arena->subproc, bitmap, slice_index); mi_page_clear_abandoned_mapped(page); mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]); } @@ -1384,7 +1394,8 @@ void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { // singleton bins have no abandoned bitmap (upstream ad1bcdbf, to shrink arena meta). for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) continue; - mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin]; + mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin); + if (bitmap == NULL) continue; mi_purge_holes_arg_t parg = { bitmap, tld }; (void)_mi_bitmap_forall_set(bitmap, &mi_arena_page_purge_holes_at, arena, &parg); } @@ -1427,7 +1438,8 @@ void _mi_arenas_holes_report(mi_heap_t* heap, mi_holes_report_t* rep) { if (arena_pages != NULL) { for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { // see above: not MI_BIN_COUNT if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) continue; - mi_arena_holes_report_arg_t ra = { arena_pages->pages_abandoned[bin], rep }; + mi_arena_holes_report_arg_t ra = { mi_arena_pages_abandoned(arena_pages, bin), rep }; + if (ra.bitmap == NULL) continue; (void)_mi_bitmap_forall_set(ra.bitmap, &mi_arena_page_holes_report_at, arena, &ra); } } @@ -1651,8 +1663,7 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { if (slice_count == 0) slice_count = MI_BCHUNK_BITS; mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); const size_t base_size = _mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE); - const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned - const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL); + const size_t bitmaps_size = mi_bitmap_size(slice_count, NULL); // pages (the abandoned bitmaps are allocated on demand) const size_t size = base_size + bitmaps_size; if (bitmap_base != NULL) *bitmap_base = base_size; return size; @@ -1662,7 +1673,7 @@ static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_bas if (slice_count == 0) slice_count = MI_BCHUNK_BITS; mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); - const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned + const size_t bitmaps_count = 4; // commit, dirty, purge, and pages (the abandoned bitmaps are allocated on demand) const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free #if MI_PAGE_META_IS_SEPARATED const size_t pages_size = slice_count * sizeof(mi_page_t); @@ -1701,12 +1712,55 @@ static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) { uint8_t* base = (uint8_t*)arena_pages + bitmap_base; mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE)); arena_pages->pages = mi_arena_bitmap_init(slice_count, &base); - for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { - arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); - } + // `pages_abandoned[]` stays NULL (the allocation is zeroed) until a page of that bin is abandoned. return arena_pages; } +// The abandoned-pages bitmap of `bin`, or NULL if no page of that bin was ever abandoned in +// this (heap, arena) pair. A NULL bitmap reads as all-clear. +static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size_t bin) { + mi_assert_internal(bin < MI_ARENA_BIN_COUNT); + return mi_atomic_load_ptr_acquire(mi_bitmap_t, &arena_pages->pages_abandoned[bin]); +} + +// The abandoned-pages bitmap of `bin`, allocated on first use. Allocated from the subproc +// meta-data heap so this is safe on the abandon paths (a thread tearing down its theaps, a +// foreign free re-abandoning a page), where allocating from a regular heap is not. Publishing +// is a CAS so no lock is needed: a loser frees its copy and uses the winner's. Returns NULL only +// if the allocation failed. +static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin) { + mi_bitmap_t* bitmap = mi_arena_pages_abandoned(arena_pages, bin); + if mi_likely(bitmap != NULL) return bitmap; + const size_t slice_count = arena->slice_count; + const size_t size = mi_bitmap_size(slice_count, NULL); + mi_bitmap_t* fresh = (mi_bitmap_t*)_mi_meta_zalloc_aligned(arena->subproc, size, MI_BCHUNK_SIZE, NULL); + if (fresh == NULL) return NULL; + mi_bitmap_init(fresh, slice_count, true /* already zero */); + mi_bitmap_t* expected = NULL; + if (mi_atomic_cas_ptr_strong_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], &expected, fresh)) { + return fresh; + } + // another thread published one first + _mi_free_subproc_safe(fresh); + mi_assert_internal(expected != NULL); + return expected; +} + +// Release the on-demand abandoned bitmaps of a heap's arena pages (the `pages` bitmap is part of +// the `arena_pages` allocation itself). +static void mi_arena_pages_free_abandoned(mi_arena_pages_t* arena_pages) { + for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { + mi_bitmap_t* bitmap = mi_atomic_exchange_ptr_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], NULL); + if (bitmap != NULL) { _mi_free_subproc_safe(bitmap); } + } +} + +void _mi_arena_pages_free(mi_arena_pages_t* arena_pages) { + if (arena_pages == NULL) return; + mi_arena_pages_free_abandoned(arena_pages); + _mi_free_subproc_safe(arena_pages); +} + static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, size_t slice_count, mi_arena_t* parent, size_t total_size, int numa_node, bool exclusive, @@ -1799,7 +1853,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, arena->slices_purge = mi_arena_bitmap_init(slice_count, &base); arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base); for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { - arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); + mi_atomic_store_ptr_relaxed(mi_bitmap_t, &arena->pages_main.pages_abandoned[i], NULL); // allocated on first abandon } #if MI_PAGE_META_IS_SEPARATED arena->pages_meta = (mi_page_t*)base; @@ -2690,7 +2744,9 @@ bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_bloc for (size_t bin = 0; ok && bin < MI_ARENA_BIN_COUNT; bin++) { // todo: if we had a single abandoned page map as well, this can be faster. if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) { - ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info); + mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin); + if (bitmap == NULL) continue; + ok = _mi_bitmap_forall_set(bitmap, &mi_heap_visit_page_at, arena, &visit_info); } } } diff --git a/src/heap.c b/src/heap.c index d9173f7cb..ca7db122b 100644 --- a/src/heap.c +++ b/src/heap.c @@ -229,7 +229,7 @@ static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); if (arena_pages!=NULL) { mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL); - _mi_free_subproc_safe(arena_pages); + _mi_arena_pages_free(arena_pages); } } } diff --git a/test/test-abandoned-lazy.c b/test/test-abandoned-lazy.c new file mode 100644 index 000000000..8fd3f1328 --- /dev/null +++ b/test/test-abandoned-lazy.c @@ -0,0 +1,200 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2025 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. +-----------------------------------------------------------------------------*/ + +/* The per-bin abandoned-page bitmaps of a (heap, arena) pair are allocated the + first time a page of that bin is abandoned, not when the heap first touches + the arena. This drives every path that reads or writes them: + + - worker threads allocate blocks of several size classes from a shared heap + and exit while the blocks are still live, so their pages are abandoned + (the bitmaps are allocated on that path, by several threads at once); + - the main thread then allocates the same sizes from the heap, which reclaims + the abandoned pages, frees every block (un-abandon, page free), collects + and deletes the heap (the bitmaps are freed with it); + - the same with the main heap, whose bitmaps are never freed, and with blocks + freed by a thread that never allocated (re-abandon of a mostly free page). + + A debug build checks the bitmap invariants on each of these paths. + + > mimalloc-test-abandoned-lazy [ITER] +*/ + +#include +#include +#include +#include +#include + +#include "mimalloc.h" + +#if defined(MI_TSAN) || defined(MI_UBSAN) || defined(MI_GUARDED) +static int ITER = 20; +#else +static int ITER = 100; +#endif + +#define NTHREADS 8 +#define NSIZES 6 +#define NBLOCKS 64 // per thread per size: enough to span a few pages of each size class + +static const size_t sizes[NSIZES] = { 16, 96, 512, 2048, 8192, 40000 }; + +typedef void (thread_entry_fun_t)(intptr_t tid); +static void run_os_threads(size_t nthreads, thread_entry_fun_t* entry); + +static mi_heap_t* shared_heap; // NULL: use the main heap +static void* blocks[NTHREADS][NSIZES][NBLOCKS]; + +static void alloc_and_exit(intptr_t tid) { + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + void* p = (shared_heap != NULL ? mi_heap_malloc(shared_heap, sizes[s]) : mi_malloc(sizes[s])); + if (p == NULL) { fprintf(stderr, "allocation failed\n"); abort(); } + memset(p, (int)(tid + s), sizes[s]); + blocks[tid][s][b] = p; + } + } + // exit with everything live: every page this thread used is abandoned +} + +static void free_some(intptr_t tid) { + // free most blocks of a page from a thread that never allocated from the heap: + // a multi-threaded free that can re-abandon the page as mapped + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + if ((b % 8) != 0) { + mi_free(blocks[tid][s][b]); + blocks[tid][s][b] = NULL; + } + } + } +} + +static void check_block(intptr_t tid, int s, void* p) { + const unsigned char* q = (const unsigned char*)p; + for (size_t i = 0; i < sizes[s]; i += 64) { + if (q[i] != (unsigned char)(tid + s)) { fprintf(stderr, "block corrupted\n"); abort(); } + } +} + +static void run_round(bool use_main_heap) { + shared_heap = (use_main_heap ? NULL : mi_heap_new()); + memset(blocks, 0, sizeof(blocks)); + + // 1. abandon pages of several size classes from several threads at once + run_os_threads(NTHREADS, &alloc_and_exit); + + // 2. reclaim: allocating the same sizes must find the abandoned pages again + void* mine[NSIZES][NBLOCKS]; + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + mine[s][b] = (shared_heap != NULL ? mi_heap_malloc(shared_heap, sizes[s]) : mi_malloc(sizes[s])); + if (mine[s][b] == NULL) { fprintf(stderr, "allocation failed\n"); abort(); } + } + } + for (intptr_t t = 0; t < NTHREADS; t++) { + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { check_block(t, s, blocks[t][s][b]); } + } + } + + // 3. frees from threads that never touched the heap, then from this thread + run_os_threads(NTHREADS, &free_some); + for (intptr_t t = 0; t < NTHREADS; t++) { + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + if (blocks[t][s][b] != NULL) { check_block(t, s, blocks[t][s][b]); mi_free(blocks[t][s][b]); } + } + } + } + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { mi_free(mine[s][b]); } + } + + // 4. collect and delete + if (shared_heap != NULL) { + mi_heap_collect(shared_heap, true); + mi_heap_delete(shared_heap); + shared_heap = NULL; + } + else { + mi_collect(true); + } +} + +int main(int argc, char** argv) { + if (argc >= 2) { + char* end; + long n = strtol(argv[1], &end, 10); + if (n > 0) ITER = (int)n; + } + for (int i = 0; i < ITER; i++) { + run_round(false); + run_round(true); + } + printf("test-abandoned-lazy: %d rounds, ok\n", ITER); + mi_stats_print(NULL); + return 0; +} + + +/* ----------------------------------------------------------- + OS threads +----------------------------------------------------------- */ + +#ifdef _WIN32 + +#include + +static thread_entry_fun_t* thread_entry_fun; + +static DWORD WINAPI thread_entry(LPVOID param) { + thread_entry_fun((intptr_t)param); + return 0; +} + +static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { + thread_entry_fun = fun; + DWORD* tids = (DWORD*) calloc(nthreads, sizeof(DWORD)); + HANDLE* thandles = (HANDLE*)calloc(nthreads, sizeof(HANDLE)); + for (size_t i = 0; i < nthreads; i++) { + thandles[i] = CreateThread(0, 8*1024L, &thread_entry, (void*)(i), 0, &tids[i]); + } + for (size_t i = 0; i < nthreads; i++) { + WaitForSingleObject(thandles[i], INFINITE); + } + for (size_t i = 0; i < nthreads; i++) { + CloseHandle(thandles[i]); + } + free(tids); + free(thandles); +} + +#else + +#include + +static thread_entry_fun_t* thread_entry_fun; + +static void* thread_entry(void* param) { + thread_entry_fun((intptr_t)param); + return NULL; +} + +static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { + thread_entry_fun = fun; + pthread_t* threads = (pthread_t*)calloc(nthreads, sizeof(pthread_t)); + memset(threads, 0, sizeof(pthread_t) * nthreads); + for (size_t i = 0; i < nthreads; i++) { + pthread_create(&threads[i], NULL, &thread_entry, (void*)i); + } + for (size_t i = 0; i < nthreads; i++) { + pthread_join(threads[i], NULL); + } + free(threads); +} + +#endif From 91218f309a5acaa5174696cf4737aa3fdaae2e5c Mon Sep 17 00:00:00 2001 From: robobun Date: Thu, 3 Sep 2026 02:44:22 +0000 Subject: [PATCH 2/7] heap: a heap that is being released abandons its pages unmapped `mi_heap_delete` and `mi_heap_destroy` abandon every page of the heap (step 2 of the teardown) and then claim each one back through the `arena_pages->pages` bitmap (step 3). With the per-bin abandoned maps allocated on first abandon, step 2 allocated a map for every size class the heap had used, from the meta-data heap and under its lock, only to free them with the heap a moment later. A process that creates and destroys a heap per job on several threads (bun creates one per transpile, on a thread pool) paid that on every job, and the threads contended on the meta lock: with 300 imports, bun used about 4 ms more CPU than with the maps allocated up front. The heap now records that its release has started (`heap->releasing`). `_mi_arenas_page_abandon` leaves the pages of such a heap out of the per-bin maps, as it already does for a full page, and a concurrent free does not re-map them. Step 3 finds them through `pages` as before, and `_mi_arenas_page_unabandon` already handles an unmapped page. A heap that lives only to be released now never allocates a map, and its `arena_pages` block holds just the `pages` bitmap. test-abandoned-lazy counts the maps published while heaps are created, used, destroyed and deleted on one thread (a debug-build counter, as test-heap-teardown uses a debug hook): 0 now, 600 without this change. It also deletes heaps on four threads while four other threads free their blocks during the delete. --- include/mimalloc/types.h | 1 + src/arena.c | 14 +++- src/heap.c | 4 + test/test-abandoned-lazy.c | 160 +++++++++++++++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 1 deletion(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index e03e3bfa0..a5f41c26b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -628,6 +628,7 @@ typedef struct mi_heap_s { mi_lock_t theaps_lock; // lock for the theaps list operations _Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap + _Atomic(uintptr_t) releasing; // set when `mi_heap_delete`/`mi_heap_destroy` starts: its pages are abandoned unmapped mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations) diff --git a/src/arena.c b/src/arena.c index 3c378aba1..693f77d25 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1211,8 +1211,9 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); // add to abandoned? + // (not for a heap that is being released: its teardown claims every page through `pages`, see `mi_heap_release_pages`) mi_heap_t* heap = mi_page_heap(page); - if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { + if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page) && mi_atomic_load_relaxed(&heap->releasing) == 0) { // make available for allocations size_t bin = _mi_bin(mi_page_block_size(page)); mi_assert_internal(bin < MI_ARENA_BIN_COUNT); @@ -1272,6 +1273,9 @@ bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) { if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) { return false; } + else if (mi_atomic_load_relaxed(&mi_page_heap(page)->releasing) != 0) { + return false; // the heap is being released and claims the page through `pages` (see `mi_heap_release_pages`) + } else { // Account on the heap and not on this thread's theap for it (`_mi_page_associated_theap_peek`): the theap // was only used for its statistics here, and a concurrent `mi_heap_delete` may be detaching and merging it. @@ -1728,6 +1732,10 @@ static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size // foreign free re-abandoning a page), where allocating from a regular heap is not. Publishing // is a CAS so no lock is needed: a loser frees its copy and uses the winner's. Returns NULL only // if the allocation failed. +#if MI_DEBUG > 0 +mi_decl_export _Atomic(uintptr_t) mi_debug_abandoned_maps_allocated; // test hook (test-abandoned-lazy): per-bin abandoned maps published so far +#endif + static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin) { mi_bitmap_t* bitmap = mi_arena_pages_abandoned(arena_pages, bin); if mi_likely(bitmap != NULL) return bitmap; @@ -1738,6 +1746,9 @@ static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_ mi_bitmap_init(fresh, slice_count, true /* already zero */); mi_bitmap_t* expected = NULL; if (mi_atomic_cas_ptr_strong_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], &expected, fresh)) { + #if MI_DEBUG > 0 + mi_atomic_increment_relaxed(&mi_debug_abandoned_maps_allocated); + #endif return fresh; } // another thread published one first @@ -1750,6 +1761,7 @@ static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_ // the `arena_pages` allocation itself). static void mi_arena_pages_free_abandoned(mi_arena_pages_t* arena_pages) { for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { + if (mi_atomic_load_ptr_relaxed(mi_bitmap_t, &arena_pages->pages_abandoned[bin]) == NULL) continue; // the common case mi_bitmap_t* bitmap = mi_atomic_exchange_ptr_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], NULL); if (bitmap != NULL) { _mi_free_subproc_safe(bitmap); } } diff --git a/src/heap.c b/src/heap.c index ca7db122b..f1a049c85 100644 --- a/src/heap.c +++ b/src/heap.c @@ -181,6 +181,10 @@ mi_heap_t* mi_heap_new(void) { static void mi_heap_release_pages(mi_heap_t* heap, mi_heap_t* heap_target) { _mi_heap_detach_theaps(heap); if (_mi_is_heap_main(heap)) return; // (`_mi_heap_force_destroy` of a main heap at sub-process teardown: the arenas go as a whole) + // Step 3 claims every page through the `arena_pages->pages` bitmap, so the pages abandoned in step 2 + // do not need to be findable by size class: `_mi_arenas_page_abandon` leaves them out of the + // per-bin abandoned maps, which are then never allocated for a heap that only lives to be released. + mi_atomic_store_release(&heap->releasing, (uintptr_t)1); mi_lock(&heap->theaps_lock) { for (mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext) { mi_assert_internal(_mi_theap_heap_peek(theap)==NULL); diff --git a/test/test-abandoned-lazy.c b/test/test-abandoned-lazy.c index 8fd3f1328..87ce326ab 100644 --- a/test/test-abandoned-lazy.c +++ b/test/test-abandoned-lazy.c @@ -17,6 +17,15 @@ terms of the MIT license. - the same with the main heap, whose bitmaps are never freed, and with blocks freed by a thread that never allocated (re-abandon of a mostly free page). + A heap that is deleted or destroyed abandons all of its pages during its own + teardown, which claims them back through the `pages` bitmap. Those pages are + not mapped, so a heap that lives only to be released never allocates a map: + + - a debug build counts the maps allocated while heaps are created, used and + destroyed on one thread, and the count must not move; + - several threads create, use and delete heaps while other threads free the + blocks of each heap during its delete (a free that would re-map a page). + A debug build checks the bitmap invariants on each of these paths. > mimalloc-test-abandoned-lazy [ITER] @@ -125,6 +134,116 @@ static void run_round(bool use_main_heap) { } } +/* ----------------------------------------------------------- + Released heaps: no maps +----------------------------------------------------------- */ + +#if !defined(NDEBUG) +#ifdef __cplusplus +#include +extern "C" std::atomic mi_debug_abandoned_maps_allocated; +static uintptr_t maps_allocated(void) { return mi_debug_abandoned_maps_allocated.load(); } +#else +#include +extern _Atomic(uintptr_t) mi_debug_abandoned_maps_allocated; +static uintptr_t maps_allocated(void) { return atomic_load(&mi_debug_abandoned_maps_allocated); } +#endif +#define HAS_MAP_COUNTER 1 +#else +#define HAS_MAP_COUNTER 0 +#endif + +static void fill_heap(mi_heap_t* heap, void** keep) { + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + void* p = mi_heap_malloc(heap, sizes[s]); + if (p == NULL) { fprintf(stderr, "allocation failed\n"); abort(); } + memset(p, s + 1, sizes[s]); + if (keep != NULL) { keep[s*NBLOCKS + b] = p; } + } + } +} + +static void released_heaps_map_nothing(void) { + static void* keep[NSIZES*NBLOCKS]; + // one delete first: the main heap takes the moved pages and maps them, and its maps stay + mi_heap_t* heap = mi_heap_new(); + fill_heap(heap, keep); + mi_heap_delete(heap); + for (int i = 0; i < NSIZES*NBLOCKS; i++) { mi_free(keep[i]); } + + #if HAS_MAP_COUNTER + const uintptr_t before = maps_allocated(); + #endif + for (int n = 0; n < 50; n++) { + heap = mi_heap_new(); + fill_heap(heap, NULL); + mi_heap_destroy(heap); + + heap = mi_heap_new(); + fill_heap(heap, keep); + mi_heap_delete(heap); // the blocks now belong to the main heap + for (int i = 0; i < NSIZES*NBLOCKS; i++) { mi_free(keep[i]); } + } + #if HAS_MAP_COUNTER + const uintptr_t after = maps_allocated(); + if (after != before) { + fprintf(stderr, "released heaps allocated %lu abandoned maps\n", (unsigned long)(after - before)); + abort(); + } + #endif +} + +/* ----------------------------------------------------------- + Concurrent delete: blocks freed by another thread while their heap is deleted +----------------------------------------------------------- */ + +#define NPAIRS 4 // a deleter and a freer each +#define DBLOCKS 8 // per size: the pages stay mostly free, so a free would re-map them + +static void* atomic_exchange_ptr(volatile void** p, void* newval); +static long atomic_load_long(volatile long* p); +static void atomic_store_long(volatile long* p, long x); + +static volatile void* dslots[NPAIRS][NSIZES*DBLOCKS]; +static volatile long dgo[NPAIRS]; +static volatile long ddone[NPAIRS]; +static int dround_count; + +static void delete_pair(intptr_t tid) { + const int pair = (int)(tid / 2); + if ((tid % 2) == 0) { + // deleter: allocate, let the freer start, and delete while it frees + for (int r = 1; r <= dround_count; r++) { + mi_heap_t* heap = mi_heap_new(); + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < DBLOCKS; b++) { + void* p = mi_heap_malloc(heap, sizes[s]); + if (p == NULL) { fprintf(stderr, "allocation failed\n"); abort(); } + memset(p, s + 1, sizes[s]); + atomic_exchange_ptr(&dslots[pair][s*DBLOCKS + b], p); + } + } + atomic_store_long(&dgo[pair], r); + mi_heap_delete(heap); + while (atomic_load_long(&ddone[pair]) != r) { /* spin */ } + } + } + else { + // freer: never allocates from the heap, so it may free into it during the delete + for (int r = 1; r <= dround_count; r++) { + while (atomic_load_long(&dgo[pair]) != r) { /* spin */ } + for (int i = 0; i < NSIZES*DBLOCKS; i++) { + unsigned char* p = (unsigned char*)atomic_exchange_ptr(&dslots[pair][i], NULL); + if (p == NULL) { fprintf(stderr, "missing block\n"); abort(); } + if (p[0] != (unsigned char)(i / DBLOCKS + 1)) { fprintf(stderr, "block corrupted\n"); abort(); } + mi_free(p); + } + atomic_store_long(&ddone[pair], r); + } + } +} + int main(int argc, char** argv) { if (argc >= 2) { char* end; @@ -135,6 +254,9 @@ int main(int argc, char** argv) { run_round(false); run_round(true); } + released_heaps_map_nothing(); + dround_count = ITER; + run_os_threads(NPAIRS*2, &delete_pair); printf("test-abandoned-lazy: %d rounds, ok\n", ITER); mi_stats_print(NULL); return 0; @@ -173,6 +295,20 @@ static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { free(thandles); } +static void* atomic_exchange_ptr(volatile void** p, void* newval) { + #if (INTPTR_MAX == INT32_MAX) + return (void*)InterlockedExchange((volatile LONG*)p, (LONG)newval); + #else + return (void*)InterlockedExchange64((volatile LONG64*)p, (LONG64)newval); + #endif +} +static long atomic_load_long(volatile long* p) { + return InterlockedCompareExchange(p, 0, 0); +} +static void atomic_store_long(volatile long* p, long x) { + InterlockedExchange(p, x); +} + #else #include @@ -197,4 +333,28 @@ static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { free(threads); } +#ifdef __cplusplus +#include +static void* atomic_exchange_ptr(volatile void** p, void* newval) { + return std::atomic_exchange((volatile std::atomic*)p, newval); +} +static long atomic_load_long(volatile long* p) { + return std::atomic_load((volatile std::atomic*)p); +} +static void atomic_store_long(volatile long* p, long x) { + std::atomic_store((volatile std::atomic*)p, x); +} +#else +#include +static void* atomic_exchange_ptr(volatile void** p, void* newval) { + return atomic_exchange((volatile _Atomic(void*)*)p, newval); +} +static long atomic_load_long(volatile long* p) { + return atomic_load((volatile _Atomic(long)*)p); +} +static void atomic_store_long(volatile long* p, long x) { + atomic_store((volatile _Atomic(long)*)p, x); +} +#endif + #endif From a26c5de79862fd134c7fe6d417b75a8b875a19b0 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 2 Sep 2026 22:13:48 -0700 Subject: [PATCH 3/7] heap: order the delete walk against concurrent frees, and test heap release on many threads `mi_heap_delete` walks `arena_pages->pages` and then frees the heap and the bitmap, while a foreign `mi_free` that frees a page of that heap reads both and clears the page's bit as its last access. The walk's loads were relaxed, so nothing formally ordered those reads before the free of the memory they read. They are acquire now. The walk also located the page struct (which reads `block_size` with separated page meta) before pinning the slice, when the slice may already be freed or be a fresh page of another heap. It is located after the pin now. On macOS the theap TLS slot is a raw slot in the thread's control block, which the OS reuses for a later thread: the exiting thread's last store and the next thread's first load are the same address from two threads. Use relaxed atomics there (same single load/store). `test/test-heap-release-mt.c` (new): heaps created and deleted/destroyed on 8 threads while 4 threads free their blocks and batches of short-lived threads allocate from them and exit. Clean under TSAN, ASAN and MI_DEBUG_FULL; before this it produced TSAN reports on the base branch as well. --- CMakeLists.txt | 2 +- include/mimalloc/prim-tls.h | 10 +++ src/arena.c | 27 ++++-- src/bitmap.c | 8 +- test/test-heap-release-mt.c | 167 ++++++++++++++++++++++++++++++++++++ 5 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 test/test-heap-release-mt.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 361ebe0e7..97f0de3aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -877,7 +877,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init) + set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init) if(NOT (MI_DEBUG_TSAN OR MI_TRACK_ASAN OR MI_DEBUG_UBSAN)) list(APPEND mi_static_tests thp-optout) # counts madvise calls by interposing it, which a sanitizer runtime does first endif() diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index 242583574..e11cfb098 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -149,13 +149,23 @@ static inline void* mi_prim_tls_slot(size_t slot) { #else return mi_prim_thread_pointer()[slot]; #endif + #elif defined(__GNUC__) || defined(__clang__) + // The slot lives in the thread's control block, which the OS reuses for a later thread, so the + // exiting thread's last store (`_mi_thread_done`) and a new thread's first load touch the same + // address from different threads. A relaxed atomic is the same single load/store instruction and + // states that this is by design (and keeps TSAN quiet about it). + return __atomic_load_n(&mi_prim_thread_pointer()[slot], __ATOMIC_RELAXED); #else return mi_prim_thread_pointer()[slot]; #endif } static inline void mi_prim_tls_slot_set(size_t slot, void* value) { + #if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(&mi_prim_thread_pointer()[slot], value, __ATOMIC_RELAXED); + #else mi_prim_thread_pointer()[slot] = value; + #endif } #endif diff --git a/src/arena.c b/src/arena.c index 693f77d25..29f83faef 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2640,7 +2640,10 @@ static bool mi_heap_visit_page(mi_page_t* page, mi_heap_visit_info_t* vinfo) { // owned we give the bit back and retry; once we own it nobody else can free it and the bit goes back too. // This is the same protocol the abandoned-page map uses (`mi_arena_try_claim_abandoned`, // `mi_arena_page_purge_holes_at` against `_mi_arenas_page_unabandon`). -// Returns `true` if we own the page, `false` if it is gone. +// The page struct is only located (`mi_arena_page_at_slice`, which reads `block_size` with separated page +// meta) once the slice is pinned: before that a concurrent `mi_free` may be freeing the page and the slice +// may already be a fresh page of another heap. +// Returns the page if we own it, or NULL if it is gone. #if MI_DEBUG > 0 mi_decl_export _Atomic(uintptr_t) mi_debug_stall_in_heap_delete_claim; // test hook (test-heap-teardown): stall while a page is pinned but not yet claimed #endif @@ -2651,11 +2654,13 @@ static void mi_heap_visit_page_seize(mi_page_t* page) { mi_page_set_theap(page, NULL); } -static bool mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_page_t* page, size_t slice_index) { +static mi_page_t* mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_arena_t* arena, size_t slice_index) { mi_bitmap_t* const pages = vinfo->arena_pages->pages; + mi_page_t* page = NULL; for (;;) { - if (!mi_bitmap_clear(pages, slice_index)) return false; // freed by a concurrent `mi_free` - // pinned + if (!mi_bitmap_clear(pages, slice_index)) return NULL; // freed by a concurrent `mi_free` + // pinned: now it is a page of this heap and stays one while we hold the bit + page = mi_arena_page_at_slice(arena, slice_index); #if MI_DEBUG > 0 if (mi_atomic_load_acquire(&mi_debug_stall_in_heap_delete_claim) == 1) { mi_atomic_store_release(&mi_debug_stall_in_heap_delete_claim, (uintptr_t)2); // signal: pinned, not yet claimed @@ -2666,8 +2671,8 @@ static bool mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_page_t* pag // After a multi-threaded fork() the child may inherit a torn snapshot of a page that another // thread was allocating or freeing: the bit propagated but the page-map entry or the owned bit // did not, and that thread is gone. Re-derive what we can and take the page. - if (mi_page_start(page) == NULL) return false; // the page struct never made it across: leave it unpublished - if (_mi_safe_ptr_page(mi_page_start(page)) != page && !_mi_page_map_register(page)) return false; + if (mi_page_start(page) == NULL) return NULL; // the page struct never made it across: leave it unpublished + if (_mi_safe_ptr_page(mi_page_start(page)) != page && !_mi_page_map_register(page)) return NULL; mi_page_claim_ownership(page); // ours now, whether or not the dead thread held it mi_bitmap_set(pages, slice_index); if (!mi_page_is_abandoned(page)) { mi_heap_visit_page_seize(page); } @@ -2693,15 +2698,19 @@ static bool mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_page_t* pag mi_assert_internal(mi_page_is_owned(page) && mi_page_is_abandoned(page)); mi_assert_internal(mi_bitmap_is_set(pages, slice_index)); mi_assert_internal(_mi_ptr_page(mi_page_start(page)) == page && mi_page_heap(page) == vinfo->heap); - return true; + return page; } static bool mi_heap_visit_page_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { MI_UNUSED(slice_count); mi_heap_visit_info_t* vinfo = (mi_heap_visit_info_t*)arg; - mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); + mi_page_t* page; if (vinfo->claim_pages) { - if (!mi_heap_visit_page_claim(vinfo, page, slice_index)) return true; + page = mi_heap_visit_page_claim(vinfo, arena, slice_index); + if (page == NULL) return true; // gone: freed by a concurrent `mi_free` + } + else { + page = mi_arena_page_at_slice(arena, slice_index); } return mi_heap_visit_page(page, vinfo); } diff --git a/src/bitmap.c b/src/bitmap.c index 01de1999a..a1dc112aa 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -1433,12 +1433,16 @@ void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t // Visit all set bits in a bitmap. +// The loads are acquire: `mi_heap_delete` walks its `arena_pages->pages` with this and then frees the heap and +// the bitmap itself, while a concurrent `mi_free` that frees a page of the heap reads both and then clears the +// page's bit (release) as its last access (`mi_arenas_page_free_prim`). Seeing that bit clear here is what +// orders those reads before our free of the memory they read. // todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore? bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) { // for all chunkmap entries const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); for(size_t i = 0; i < chunkmap_max; i++) { - mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + mi_bfield_t cmap_entry = mi_atomic_load_acquire(&bitmap->chunkmap.bfields[i]); size_t cmap_idx; // for each chunk (corresponding to a set bit in a chunkmap entry) while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { @@ -1447,7 +1451,7 @@ bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_a mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); - mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[j]); + mi_bfield_t b = mi_atomic_load_acquire(&chunk->bfields[j]); size_t bidx; while (mi_bfield_foreach_bit(&b, &bidx)) { const size_t idx = base_idx + bidx; diff --git a/test/test-heap-release-mt.c b/test/test-heap-release-mt.c new file mode 100644 index 000000000..79ea9037e --- /dev/null +++ b/test/test-heap-release-mt.c @@ -0,0 +1,167 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. +-----------------------------------------------------------------------------*/ + +/* Release heaps on many threads at once while other threads free into them. + + - NCHURN threads loop: `mi_heap_new`, allocate mixed sizes, hand a third of the blocks to a shared + ring, free a third, leak a third, then `mi_heap_delete` (or `mi_heap_destroy` when nothing was + handed out). Some heaps are also published for a while so that short-lived threads allocate from + them and exit with their blocks live (the abandon path, and the per-bin abandoned maps). + - NFREE threads pop blocks from the ring and `mi_free` them: those frees land before, during and + after the delete of the owning heap, so pages are freed and re-abandoned by a foreign thread while + `mi_heap_delete` claims them (`arena.c:mi_heap_visit_page_claim` against `mi_arenas_page_free_prim`). + - NSPAWN threads keep creating batches of threads that allocate from a published heap (or the main + heap) and exit at once. + + Meant to be run under TSAN and ASAN as well as with MI_DEBUG_FULL: it must finish without a report. + + > mimalloc-test-heap-release-mt [SECONDS] +*/ +#if defined(_WIN32) +#include +int main(void) { printf("test-heap-release-mt: skipped on Windows (uses pthreads)\n"); return 0; } +#else + +#include +#include +#include +#include +#include +#include +#include + +#define NCHURN 8 +#define NFREE 4 +#define NSPAWN 4 +#define RING (1<<16) +static _Atomic(void*) ring[RING]; +static _Atomic unsigned long ring_head, ring_tail; +static _Atomic int stop; +static _Atomic unsigned long heaps_done, frees_done, exits_done; + +static const size_t sizes[] = {8,16,48,96,200,512,1024,2048,5000,8192,20000,40000,70000,200000}; +#define NS (sizeof(sizes)/sizeof(sizes[0])) + +static void push(void* p) { + for (;;) { + unsigned long h = atomic_fetch_add(&ring_head, 1); + void* expect = NULL; + if (atomic_compare_exchange_strong(&ring[h % RING], &expect, p)) return; + // slot busy: free it ourselves instead of spinning + mi_free(p); return; + } +} +static void* pop(void) { + unsigned long t = atomic_fetch_add(&ring_tail, 1); + return atomic_exchange(&ring[t % RING], NULL); +} + +static unsigned rnd(unsigned* s){ *s = *s*1103515245u+12345u; return *s>>8; } + +// shared live heaps for the exiting threads +#define NSHARED 4 +static _Atomic(mi_heap_t*) shared[NSHARED]; +static _Atomic int users[NSHARED]; + +static void* churn(void* arg) { + unsigned seed = (unsigned)(uintptr_t)arg * 7919u + 1; + while (!atomic_load(&stop)) { + mi_heap_t* h = mi_heap_new(); + int slot = -1; + // sometimes publish as a shared heap for exiting threads to allocate from + if ((rnd(&seed) % 4) == 0) { + slot = rnd(&seed) % NSHARED; + mi_heap_t* expect = NULL; + if (!atomic_compare_exchange_strong(&shared[slot], &expect, h)) slot = -1; + } + int handed = 0; + int n = 50 + rnd(&seed) % 400; + for (int i = 0; i < n; i++) { + size_t sz = sizes[rnd(&seed) % NS]; + void* p = mi_heap_malloc(h, sz); + if (!p) { fprintf(stderr,"oom\n"); abort(); } + memset(p, 0xAB, sz < 64 ? sz : 64); + unsigned r = rnd(&seed) % 3; + if (r == 0) { push(p); handed++; } + else if (r == 1) mi_free(p); + // else leak into the heap; delete moves it to main / destroy frees it + } + if (slot >= 0) { + // let exiting threads use it for a bit, then unpublish + usleep(200); + atomic_store(&shared[slot], NULL); + while (atomic_load(&users[slot]) != 0) usleep(10); // in-flight allocators finish (contract: no alloc during delete) + handed = 1; // exiting threads may hold blocks: must delete, not destroy + } + if (handed == 0 && (rnd(&seed) & 1)) mi_heap_destroy(h); + else mi_heap_delete(h); + atomic_fetch_add(&heaps_done, 1); + } + return NULL; +} + +static void* freer(void* arg) { + (void)arg; + while (!atomic_load(&stop) || atomic_load(&ring_tail) < atomic_load(&ring_head)) { + void* p = pop(); + if (p) { mi_free(p); atomic_fetch_add(&frees_done,1); } + else if (atomic_load(&stop)) break; + } + return NULL; +} + +static void* exiter(void* arg) { + unsigned seed = (unsigned)(uintptr_t)arg; + int slot = rnd(&seed) % NSHARED; + atomic_fetch_add(&users[slot], 1); + mi_heap_t* h = atomic_load(&shared[slot]); + for (int i = 0; i < 40; i++) { + size_t sz = sizes[rnd(&seed) % 8]; + void* p = h ? mi_heap_malloc(h, sz) : mi_malloc(sz); + if (p) { memset(p, 0xCD, sz < 64 ? sz : 64); push(p); } + } + atomic_fetch_sub(&users[slot], 1); + atomic_fetch_add(&exits_done,1); + return NULL; // exit with blocks live -> pages abandoned (mapped, lazy bitmap alloc) +} + +static void* spawner(void* arg) { + unsigned k = (unsigned)(uintptr_t)arg * 100000u; + while (!atomic_load(&stop)) { + pthread_t t[8]; + for (int i = 0; i < 8; i++) pthread_create(&t[i], NULL, exiter, (void*)(uintptr_t)(k++)); + for (int i = 0; i < 8; i++) pthread_join(t[i], NULL); + } + return NULL; +} + +int main(int argc, char** argv) { + int secs = (argc > 1 ? atoi(argv[1]) : 0); + if (secs <= 0) { + #if defined(MI_TSAN) || !defined(NDEBUG) + secs = 4; + #else + secs = 6; + #endif + } + pthread_t tc[NCHURN], tf[NFREE], ts[NSPAWN]; + for (long i = 0; i < NFREE; i++) pthread_create(&tf[i], NULL, freer, (void*)i); + for (long i = 0; i < NCHURN; i++) pthread_create(&tc[i], NULL, churn, (void*)(i+1)); + for (long i = 0; i < NSPAWN; i++) pthread_create(&ts[i], NULL, spawner, (void*)(i+1)); + sleep(secs); + atomic_store(&stop, 1); + for (int i = 0; i < NCHURN; i++) pthread_join(tc[i], NULL); + for (int i = 0; i < NSPAWN; i++) pthread_join(ts[i], NULL); + for (int i = 0; i < NFREE; i++) pthread_join(tf[i], NULL); + // drain + void* p; while (atomic_load(&ring_tail) < atomic_load(&ring_head) && (p = pop(), 1)) { if (p) mi_free(p); } + for (unsigned i = 0; i < RING; i++) { void* q = atomic_exchange(&ring[i], NULL); if (q) mi_free(q); } + mi_collect(true); + printf("test-heap-release-mt: ok (heaps=%lu frees=%lu thread-exits=%lu)\n", (unsigned long)heaps_done, (unsigned long)frees_done, (unsigned long)exits_done); + return 0; +} + +#endif From 92930d0c40a3bb6c8ddf4358314c322aa54369cf Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 2 Sep 2026 22:23:31 -0700 Subject: [PATCH 4/7] test-heap-release-mt: also release heaps around a park, so the scavenger sweeps them The second half of the test runs the same churn with each churn thread parked (`mi_on_thread_idle_start`) before its heap goes: the scavenger then sweeps that thread's pages and the abandoned-page maps of its heaps. Half of those heaps are deleted by the owner right after `mi_on_thread_idle_end`, racing the end of the sweep; the other half by a freer thread while the owner is still parked, so the delete's detach has to get past the scavenger holding the owner's theaps. A `-no-scavenger` ctest variant covers the inline sweep. --- CMakeLists.txt | 1 + test/test-heap-release-mt.c | 86 +++++++++++++++++++++++++++++++------ 2 files changed, 74 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 97f0de3aa..8faee230f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -923,6 +923,7 @@ if (MI_BUILD_TESTS) # with no scavenger to hand off to, `mi_on_thread_idle_start` must do the work inline instead add_test(NAME test-park-handoff-no-scavenger COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_SCAVENGER=0 $) + add_test(NAME test-heap-release-mt-no-scavenger COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_SCAVENGER=0 $) # and sweep every park, so the reclaim path is hit as hard as possible add_test(NAME test-park-handoff-eager COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES_MIN_INTERVAL=0 $) diff --git a/test/test-heap-release-mt.c b/test/test-heap-release-mt.c index 79ea9037e..03442b543 100644 --- a/test/test-heap-release-mt.c +++ b/test/test-heap-release-mt.c @@ -16,6 +16,15 @@ terms of the MIT license. - NSPAWN threads keep creating batches of threads that allocate from a published heap (or the main heap) and exit at once. + The second half runs the same churn "parked": before its heap is released, a churn thread hands + its theaps to the scavenger (`mi_on_thread_idle_start`), which then sweeps the holes of that + thread's pages and of the abandoned pages of its heaps -- including the per-bin abandoned maps of + the heap that is about to go (`theap.c:mi_purge_holes_of`, `_mi_arenas_purge_abandoned_holes`). + Half of those heaps are deleted by the parked thread itself right after `mi_on_thread_idle_end` + (racing the tail of the sweep), the other half by a freer thread *while the owner is still parked* + (a heap created on one thread and released on another while the first sits idle in the kernel), + so `mi_heap_delete`'s detach has to get past the scavenger holding that thread's theaps. + Meant to be run under TSAN and ASAN as well as with MI_DEBUG_FULL: it must finish without a report. > mimalloc-test-heap-release-mt [SECONDS] @@ -32,6 +41,7 @@ int main(void) { printf("test-heap-release-mt: skipped on Windows (uses pthreads #include #include #include +#include #define NCHURN 8 #define NFREE 4 @@ -40,7 +50,9 @@ int main(void) { printf("test-heap-release-mt: skipped on Windows (uses pthreads static _Atomic(void*) ring[RING]; static _Atomic unsigned long ring_head, ring_tail; static _Atomic int stop; -static _Atomic unsigned long heaps_done, frees_done, exits_done; +static _Atomic int park_mode; // second half: park around the release, see the header +static _Atomic unsigned long heaps_done, frees_done, exits_done, parks_done, parks_handed_off, remote_deletes; +static _Atomic(mi_heap_t*) to_delete[NCHURN]; // park mode: heap i's owner is parked and waits for a freer to delete it static const size_t sizes[] = {8,16,48,96,200,512,1024,2048,5000,8192,20000,40000,70000,200000}; #define NS (sizeof(sizes)/sizeof(sizes[0])) @@ -67,6 +79,7 @@ static _Atomic(mi_heap_t*) shared[NSHARED]; static _Atomic int users[NSHARED]; static void* churn(void* arg) { + const int self = (int)(uintptr_t)arg - 1; unsigned seed = (unsigned)(uintptr_t)arg * 7919u + 1; while (!atomic_load(&stop)) { mi_heap_t* h = mi_heap_new(); @@ -96,6 +109,28 @@ static void* churn(void* arg) { while (atomic_load(&users[slot]) != 0) usleep(10); // in-flight allocators finish (contract: no alloc during delete) handed = 1; // exiting threads may hold blocks: must delete, not destroy } + if (atomic_load(&park_mode)) { + const bool remote = ((rnd(&seed) & 1) != 0); + if (remote) { atomic_store(&to_delete[self], h); } // a freer deletes it while we are parked + const bool parked = mi_on_thread_idle_start(); + if (parked) { atomic_fetch_add(&parks_handed_off, 1); } + if (remote) { + // idle in the kernel as far as mimalloc is concerned: we do not allocate or free until `_end` + while (atomic_load(&to_delete[self]) != NULL && !atomic_load(&stop)) { usleep(20); } + } + else if ((rnd(&seed) & 3) == 0) { + usleep(300); // sometimes let the sweep get well into our heaps, sometimes race its start + } + mi_on_thread_idle_end(); + if (!parked) { mi_on_thread_idle(); } // no scavenger (MIMALLOC_SCAVENGER=0): sweep inline so the pass is not vacuous + atomic_fetch_add(&parks_done, 1); + if (remote) { + mi_heap_t* const left = atomic_exchange(&to_delete[self], NULL); // only non-NULL if we are stopping + if (left != NULL) { mi_heap_delete(left); } + atomic_fetch_add(&heaps_done, 1); + continue; + } + } if (handed == 0 && (rnd(&seed) & 1)) mi_heap_destroy(h); else mi_heap_delete(h); atomic_fetch_add(&heaps_done, 1); @@ -103,9 +138,19 @@ static void* churn(void* arg) { return NULL; } +// park mode: delete the heaps whose owners are parked and waiting for it +static void delete_parked_heaps(void) { + for (int i = 0; i < NCHURN; i++) { + if (atomic_load(&to_delete[i]) == NULL) continue; + mi_heap_t* const h = atomic_exchange(&to_delete[i], NULL); + if (h != NULL) { mi_heap_delete(h); atomic_fetch_add(&remote_deletes, 1); } + } +} + static void* freer(void* arg) { (void)arg; while (!atomic_load(&stop) || atomic_load(&ring_tail) < atomic_load(&ring_head)) { + delete_parked_heaps(); void* p = pop(); if (p) { mi_free(p); atomic_fetch_add(&frees_done,1); } else if (atomic_load(&stop)) break; @@ -138,29 +183,44 @@ static void* spawner(void* arg) { return NULL; } -int main(int argc, char** argv) { - int secs = (argc > 1 ? atoi(argv[1]) : 0); - if (secs <= 0) { - #if defined(MI_TSAN) || !defined(NDEBUG) - secs = 4; - #else - secs = 6; - #endif - } +static void run(int secs) { + atomic_store(&stop, 0); + atomic_store(&ring_head, 0); atomic_store(&ring_tail, 0); pthread_t tc[NCHURN], tf[NFREE], ts[NSPAWN]; for (long i = 0; i < NFREE; i++) pthread_create(&tf[i], NULL, freer, (void*)i); for (long i = 0; i < NCHURN; i++) pthread_create(&tc[i], NULL, churn, (void*)(i+1)); for (long i = 0; i < NSPAWN; i++) pthread_create(&ts[i], NULL, spawner, (void*)(i+1)); - sleep(secs); + sleep((unsigned)secs); atomic_store(&stop, 1); for (int i = 0; i < NCHURN; i++) pthread_join(tc[i], NULL); for (int i = 0; i < NSPAWN; i++) pthread_join(ts[i], NULL); for (int i = 0; i < NFREE; i++) pthread_join(tf[i], NULL); // drain - void* p; while (atomic_load(&ring_tail) < atomic_load(&ring_head) && (p = pop(), 1)) { if (p) mi_free(p); } for (unsigned i = 0; i < RING; i++) { void* q = atomic_exchange(&ring[i], NULL); if (q) mi_free(q); } mi_collect(true); - printf("test-heap-release-mt: ok (heaps=%lu frees=%lu thread-exits=%lu)\n", (unsigned long)heaps_done, (unsigned long)frees_done, (unsigned long)exits_done); +} + +int main(int argc, char** argv) { + int secs = (argc > 1 ? atoi(argv[1]) : 0); + if (secs <= 0) { + #if defined(MI_TSAN) || !defined(NDEBUG) + secs = 4; + #else + secs = 6; + #endif + } + // 1. plain + run((secs + 1) / 2); + printf("test-heap-release-mt: churn ok (heaps=%lu frees=%lu thread-exits=%lu)\n", + (unsigned long)heaps_done, (unsigned long)frees_done, (unsigned long)exits_done); + // 2. parked around the release, every park swept (no rate limit) + mi_option_set(mi_option_purge_holes_min_interval, 0); + atomic_store(&park_mode, 1); + run((secs + 1) / 2); + printf("test-heap-release-mt: parked ok (heaps=%lu frees=%lu thread-exits=%lu parks=%lu handed-off=%lu remote-deletes=%lu scavenger=%s)\n", + (unsigned long)heaps_done, (unsigned long)frees_done, (unsigned long)exits_done, + (unsigned long)parks_done, (unsigned long)parks_handed_off, (unsigned long)remote_deletes, + mi_option_is_enabled(mi_option_scavenger) ? "on" : "off"); return 0; } From 47851447a2a1a52522ff97027aea8d84883893b8 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 2 Sep 2026 22:24:47 -0700 Subject: [PATCH 5/7] test-heap-release-mt: include for uintptr_t (glibc does not pull it in via the other headers) --- test/test-heap-release-mt.c | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test-heap-release-mt.c b/test/test-heap-release-mt.c index 03442b543..67cc4e0be 100644 --- a/test/test-heap-release-mt.c +++ b/test/test-heap-release-mt.c @@ -37,6 +37,7 @@ int main(void) { printf("test-heap-release-mt: skipped on Windows (uses pthreads #include #include #include +#include #include #include #include From c31ed9ede3c3b55c7010b29b9d291351826db460 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 2 Sep 2026 22:31:36 -0700 Subject: [PATCH 6/7] cmake: compile test-abandoned-lazy as C++ under MI_USE_CXX like the other heap tests (MSVC has no in C mode) --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8faee230f..28d699352 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -660,7 +660,8 @@ if(MI_USE_CXX) set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c test/test-stress-heaps.c test/test-stress-subprocs.c PROPERTIES LANGUAGE CXX ) # the white-box tests call internal functions, whose names are mangled now; the heap tests use C11 atomics, which MSVC lacks in C mode set_source_files_properties(test/test-theap-sentinel.c test/test-purge-holes.c test/test-commit-fail.c - test/test-heap-aba.c test/test-heap-delete-race.c test/test-heap-teardown.c PROPERTIES LANGUAGE CXX ) + test/test-heap-aba.c test/test-heap-delete-race.c test/test-heap-teardown.c + test/test-abandoned-lazy.c PROPERTIES LANGUAGE CXX ) endif() From 1515c3c99e81e3a14917b43c12ea3b4ba44a8da2 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Wed, 2 Sep 2026 22:40:32 -0700 Subject: [PATCH 7/7] windows: give the abandoned-maps test hook C linkage, and test-free-before-init a pre-main hook on MSVC MSVC builds the library as C++ and mangles global variables, so a debug hook the C tests link against has to be declared in the `extern "C"` block in `internal.h` like the other hooks; `mi_debug_abandoned_maps_allocated` was not, and `test-abandoned-lazy` failed to link. `test-free-before-init` only had a pre-main hook for ELF (`.preinit_array`) and GCC/Clang (constructor), so on MSVC it ran zero times and the test failed. Use a `.CRT$XCU` initializer there. --- include/mimalloc/internal.h | 1 + test/test-free-before-init.c | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 65e0b3bcb..f39b1c3df 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -357,6 +357,7 @@ extern "C" { #endif extern mi_decl_export _Atomic(uintptr_t) mi_debug_stall_in_thread_theaps_done; extern mi_decl_export _Atomic(uintptr_t) mi_debug_stall_in_heap_delete_claim; +extern mi_decl_export _Atomic(uintptr_t) mi_debug_abandoned_maps_allocated; extern mi_decl_export volatile long mi_debug_fail_os_commit_after; #ifdef __cplusplus } diff --git a/test/test-free-before-init.c b/test/test-free-before-init.c index 2d6536754..6396bdf5d 100644 --- a/test/test-free-before-init.c +++ b/test/test-free-before-init.c @@ -35,6 +35,13 @@ static void (*mi_test_preinit)(void) = &free_null_before_init; #elif defined(__GNUC__) || defined(__clang__) __attribute__((constructor)) static void free_null_before_init_ctor(void) { free_null_before_init(); } +#elif defined(_MSC_VER) +// MSVC has no constructor attribute: put a pointer in the CRT's C initializer section, which the +// startup code walks before `main` (and, for a static mimalloc, in no guaranteed order relative +// to mimalloc's own initializer there -- the same caveat as the constructor above). +#pragma section(".CRT$XCU", read) +static void __cdecl free_null_before_init_ctor(void) { free_null_before_init(); } +__declspec(allocate(".CRT$XCU")) void (__cdecl* mi_test_free_before_init_ctor)(void) = &free_null_before_init_ctor; #endif int main(void) {