diff --git a/CMakeLists.txt b/CMakeLists.txt index eee0d7d5b..28d699352 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -660,7 +660,8 @@ if(MI_USE_CXX) set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c test/test-stress-heaps.c test/test-stress-subprocs.c PROPERTIES LANGUAGE CXX ) # the white-box tests call internal functions, whose names are mangled now; the heap tests use C11 atomics, which MSVC lacks in C mode set_source_files_properties(test/test-theap-sentinel.c test/test-purge-holes.c test/test-commit-fail.c - test/test-heap-aba.c test/test-heap-delete-race.c test/test-heap-teardown.c PROPERTIES LANGUAGE CXX ) + test/test-heap-aba.c test/test-heap-delete-race.c test/test-heap-teardown.c + test/test-abandoned-lazy.c PROPERTIES LANGUAGE CXX ) endif() @@ -877,7 +878,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-aba heap-burst-destroy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init) + set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init) if(NOT (MI_DEBUG_TSAN OR MI_TRACK_ASAN OR MI_DEBUG_UBSAN)) list(APPEND mi_static_tests thp-optout) # counts madvise calls by interposing it, which a sanitizer runtime does first endif() @@ -923,6 +924,7 @@ if (MI_BUILD_TESTS) # with no scavenger to hand off to, `mi_on_thread_idle_start` must do the work inline instead add_test(NAME test-park-handoff-no-scavenger COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_SCAVENGER=0 $) + add_test(NAME test-heap-release-mt-no-scavenger COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_SCAVENGER=0 $) # and sweep every park, so the reclaim path is hit as hard as possible add_test(NAME test-park-handoff-eager COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES_MIN_INTERVAL=0 $) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 903d4a235..f39b1c3df 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -277,6 +277,7 @@ void _mi_arenas_abandoned_page_free(mi_page_t* page, mi_theap_t* curren void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap); void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */); bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page); +void _mi_arena_pages_free(mi_arena_pages_t* arena_pages); size_t mi_arenas_get_count(mi_subproc_t* subproc); uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index); @@ -356,6 +357,7 @@ extern "C" { #endif extern mi_decl_export _Atomic(uintptr_t) mi_debug_stall_in_thread_theaps_done; extern mi_decl_export _Atomic(uintptr_t) mi_debug_stall_in_heap_delete_claim; +extern mi_decl_export _Atomic(uintptr_t) mi_debug_abandoned_maps_allocated; extern mi_decl_export volatile long mi_debug_fail_os_commit_after; #ifdef __cplusplus } diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index 242583574..e11cfb098 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -149,13 +149,23 @@ static inline void* mi_prim_tls_slot(size_t slot) { #else return mi_prim_thread_pointer()[slot]; #endif + #elif defined(__GNUC__) || defined(__clang__) + // The slot lives in the thread's control block, which the OS reuses for a later thread, so the + // exiting thread's last store (`_mi_thread_done`) and a new thread's first load touch the same + // address from different threads. A relaxed atomic is the same single load/store instruction and + // states that this is by design (and keeps TSAN quiet about it). + return __atomic_load_n(&mi_prim_thread_pointer()[slot], __ATOMIC_RELAXED); #else return mi_prim_thread_pointer()[slot]; #endif } static inline void mi_prim_tls_slot_set(size_t slot, void* value) { + #if defined(__GNUC__) || defined(__clang__) + __atomic_store_n(&mi_prim_thread_pointer()[slot], value, __ATOMIC_RELAXED); + #else mi_prim_thread_pointer()[slot] = value; + #endif } #endif diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index f3b3b84cd..a5f41c26b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -628,6 +628,7 @@ typedef struct mi_heap_s { mi_lock_t theaps_lock; // lock for the theaps list operations _Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap + _Atomic(uintptr_t) releasing; // set when `mi_heap_delete`/`mi_heap_destroy` starts: its pages are abandoned unmapped mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations) @@ -756,8 +757,13 @@ typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in struct mi_arena_pages_s { mi_bitmap_t* pages; // all registered pages (abandoned and owned) - mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page) - // followed by the bitmaps (whose siz`es depend on the arena size) + // Abandoned pages per size bin (a set bit means the start of the page). Each bitmap is + // allocated the first time a page of that bin is abandoned (`mi_arena_pages_abandoned_ensure`); + // NULL means no page of that bin was ever abandoned. Eagerly laying out all MI_ARENA_BIN_COUNT + // of them cost a page fault per bitmap (one header write each, on its own OS page) for every + // heap that touched an arena, and most heaps never abandon a page. + _Atomic(mi_bitmap_t*) pages_abandoned[MI_ARENA_BIN_COUNT]; + // followed by the `pages` bitmap (whose size depends on the arena size) }; diff --git a/src/arena.c b/src/arena.c index 70a567f4d..29f83faef 100644 --- a/src/arena.c +++ b/src/arena.c @@ -29,6 +29,9 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo #error "The page_t.page_ma_offset field is not large enough to cover a full arena" #endif +static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size_t bin); +static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin); + /* ----------------------------------------------------------- Arena id's ----------------------------------------------------------- */ @@ -726,9 +729,9 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, arena) { mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); - if (arena_pages != NULL) { + mi_bitmap_t* const bitmap = (arena_pages != NULL ? mi_arena_pages_abandoned(arena_pages, bin) : NULL); + if (bitmap != NULL) { size_t slice_index; - mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin]; if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) { // found an abandoned page of the right size @@ -1119,7 +1122,7 @@ static void mi_arenas_page_free_prim(mi_page_t* page, mi_subproc_t* subproc, mi_ const size_t bin = _mi_bin(mi_page_block_size(page)); mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); - mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1)); + mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_arena_pages_abandoned(arena_pages, bin) == NULL || mi_bitmap_is_clearN(mi_arena_pages_abandoned(arena_pages, bin), slice_index, 1)); // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may // be (temporarily) not true if the free happens while trying to reclaim // see `mi_arena_try_claim_abandoned` @@ -1208,8 +1211,9 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); // add to abandoned? + // (not for a heap that is being released: its teardown claims every page through `pages`, see `mi_heap_release_pages`) mi_heap_t* heap = mi_page_heap(page); - if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { + if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page) && mi_atomic_load_relaxed(&heap->releasing) == 0) { // make available for allocations size_t bin = _mi_bin(mi_page_block_size(page)); mi_assert_internal(bin < MI_ARENA_BIN_COUNT); @@ -1224,13 +1228,18 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); - mi_page_set_abandoned_mapped(page); - const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index); - MI_UNUSED(was_clear); mi_assert_internal(was_clear); - mi_atomic_increment_relaxed(&heap->abandoned_count[bin]); - mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); - mi_abandoned_page_unown(page, current_theapx); - return; + // If the bin's bitmap cannot be allocated the page is abandoned unmapped, like a full + // page: it is still reachable through `pages` and is reclaimed once a block in it is freed. + mi_bitmap_t* const bitmap = mi_arena_pages_abandoned_ensure(arena, arena_pages, bin); + if mi_likely(bitmap != NULL) { + mi_page_set_abandoned_mapped(page); + const bool was_clear = mi_bitmap_set(bitmap, slice_index); + MI_UNUSED(was_clear); mi_assert_internal(was_clear); + mi_atomic_increment_relaxed(&heap->abandoned_count[bin]); + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); + return; + } } } // otherwise, @@ -1264,6 +1273,9 @@ bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) { if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) { return false; } + else if (mi_atomic_load_relaxed(&mi_page_heap(page)->releasing) != 0) { + return false; // the heap is being released and claims the page through `pages` (see `mi_heap_release_pages`) + } else { // Account on the heap and not on this thread's theap for it (`_mi_page_associated_theap_peek`): the theap // was only used for its statistics here, and a concurrent `mi_heap_delete` may be detaching and merging it. @@ -1298,7 +1310,9 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); // this busy waits until a concurrent reader (from alloc_abandoned) is done - mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index); + mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin); + mi_assert_internal(bitmap != NULL); // a mapped page was set in it + mi_bitmap_clear_once_set(arena->subproc, bitmap, slice_index); mi_page_clear_abandoned_mapped(page); mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]); } @@ -1384,7 +1398,8 @@ void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { // singleton bins have no abandoned bitmap (upstream ad1bcdbf, to shrink arena meta). for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) continue; - mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin]; + mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin); + if (bitmap == NULL) continue; mi_purge_holes_arg_t parg = { bitmap, tld }; (void)_mi_bitmap_forall_set(bitmap, &mi_arena_page_purge_holes_at, arena, &parg); } @@ -1427,7 +1442,8 @@ void _mi_arenas_holes_report(mi_heap_t* heap, mi_holes_report_t* rep) { if (arena_pages != NULL) { for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { // see above: not MI_BIN_COUNT if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) continue; - mi_arena_holes_report_arg_t ra = { arena_pages->pages_abandoned[bin], rep }; + mi_arena_holes_report_arg_t ra = { mi_arena_pages_abandoned(arena_pages, bin), rep }; + if (ra.bitmap == NULL) continue; (void)_mi_bitmap_forall_set(ra.bitmap, &mi_arena_page_holes_report_at, arena, &ra); } } @@ -1651,8 +1667,7 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { if (slice_count == 0) slice_count = MI_BCHUNK_BITS; mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); const size_t base_size = _mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE); - const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned - const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL); + const size_t bitmaps_size = mi_bitmap_size(slice_count, NULL); // pages (the abandoned bitmaps are allocated on demand) const size_t size = base_size + bitmaps_size; if (bitmap_base != NULL) *bitmap_base = base_size; return size; @@ -1662,7 +1677,7 @@ static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_bas if (slice_count == 0) slice_count = MI_BCHUNK_BITS; mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); - const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned + const size_t bitmaps_count = 4; // commit, dirty, purge, and pages (the abandoned bitmaps are allocated on demand) const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free #if MI_PAGE_META_IS_SEPARATED const size_t pages_size = slice_count * sizeof(mi_page_t); @@ -1701,12 +1716,63 @@ static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) { uint8_t* base = (uint8_t*)arena_pages + bitmap_base; mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE)); arena_pages->pages = mi_arena_bitmap_init(slice_count, &base); - for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { - arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); - } + // `pages_abandoned[]` stays NULL (the allocation is zeroed) until a page of that bin is abandoned. return arena_pages; } +// The abandoned-pages bitmap of `bin`, or NULL if no page of that bin was ever abandoned in +// this (heap, arena) pair. A NULL bitmap reads as all-clear. +static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size_t bin) { + mi_assert_internal(bin < MI_ARENA_BIN_COUNT); + return mi_atomic_load_ptr_acquire(mi_bitmap_t, &arena_pages->pages_abandoned[bin]); +} + +// The abandoned-pages bitmap of `bin`, allocated on first use. Allocated from the subproc +// meta-data heap so this is safe on the abandon paths (a thread tearing down its theaps, a +// foreign free re-abandoning a page), where allocating from a regular heap is not. Publishing +// is a CAS so no lock is needed: a loser frees its copy and uses the winner's. Returns NULL only +// if the allocation failed. +#if MI_DEBUG > 0 +mi_decl_export _Atomic(uintptr_t) mi_debug_abandoned_maps_allocated; // test hook (test-abandoned-lazy): per-bin abandoned maps published so far +#endif + +static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin) { + mi_bitmap_t* bitmap = mi_arena_pages_abandoned(arena_pages, bin); + if mi_likely(bitmap != NULL) return bitmap; + const size_t slice_count = arena->slice_count; + const size_t size = mi_bitmap_size(slice_count, NULL); + mi_bitmap_t* fresh = (mi_bitmap_t*)_mi_meta_zalloc_aligned(arena->subproc, size, MI_BCHUNK_SIZE, NULL); + if (fresh == NULL) return NULL; + mi_bitmap_init(fresh, slice_count, true /* already zero */); + mi_bitmap_t* expected = NULL; + if (mi_atomic_cas_ptr_strong_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], &expected, fresh)) { + #if MI_DEBUG > 0 + mi_atomic_increment_relaxed(&mi_debug_abandoned_maps_allocated); + #endif + return fresh; + } + // another thread published one first + _mi_free_subproc_safe(fresh); + mi_assert_internal(expected != NULL); + return expected; +} + +// Release the on-demand abandoned bitmaps of a heap's arena pages (the `pages` bitmap is part of +// the `arena_pages` allocation itself). +static void mi_arena_pages_free_abandoned(mi_arena_pages_t* arena_pages) { + for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { + if (mi_atomic_load_ptr_relaxed(mi_bitmap_t, &arena_pages->pages_abandoned[bin]) == NULL) continue; // the common case + mi_bitmap_t* bitmap = mi_atomic_exchange_ptr_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], NULL); + if (bitmap != NULL) { _mi_free_subproc_safe(bitmap); } + } +} + +void _mi_arena_pages_free(mi_arena_pages_t* arena_pages) { + if (arena_pages == NULL) return; + mi_arena_pages_free_abandoned(arena_pages); + _mi_free_subproc_safe(arena_pages); +} + static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, size_t slice_count, mi_arena_t* parent, size_t total_size, int numa_node, bool exclusive, @@ -1799,7 +1865,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, arena->slices_purge = mi_arena_bitmap_init(slice_count, &base); arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base); for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { - arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); + mi_atomic_store_ptr_relaxed(mi_bitmap_t, &arena->pages_main.pages_abandoned[i], NULL); // allocated on first abandon } #if MI_PAGE_META_IS_SEPARATED arena->pages_meta = (mi_page_t*)base; @@ -2574,7 +2640,10 @@ static bool mi_heap_visit_page(mi_page_t* page, mi_heap_visit_info_t* vinfo) { // owned we give the bit back and retry; once we own it nobody else can free it and the bit goes back too. // This is the same protocol the abandoned-page map uses (`mi_arena_try_claim_abandoned`, // `mi_arena_page_purge_holes_at` against `_mi_arenas_page_unabandon`). -// Returns `true` if we own the page, `false` if it is gone. +// The page struct is only located (`mi_arena_page_at_slice`, which reads `block_size` with separated page +// meta) once the slice is pinned: before that a concurrent `mi_free` may be freeing the page and the slice +// may already be a fresh page of another heap. +// Returns the page if we own it, or NULL if it is gone. #if MI_DEBUG > 0 mi_decl_export _Atomic(uintptr_t) mi_debug_stall_in_heap_delete_claim; // test hook (test-heap-teardown): stall while a page is pinned but not yet claimed #endif @@ -2585,11 +2654,13 @@ static void mi_heap_visit_page_seize(mi_page_t* page) { mi_page_set_theap(page, NULL); } -static bool mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_page_t* page, size_t slice_index) { +static mi_page_t* mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_arena_t* arena, size_t slice_index) { mi_bitmap_t* const pages = vinfo->arena_pages->pages; + mi_page_t* page = NULL; for (;;) { - if (!mi_bitmap_clear(pages, slice_index)) return false; // freed by a concurrent `mi_free` - // pinned + if (!mi_bitmap_clear(pages, slice_index)) return NULL; // freed by a concurrent `mi_free` + // pinned: now it is a page of this heap and stays one while we hold the bit + page = mi_arena_page_at_slice(arena, slice_index); #if MI_DEBUG > 0 if (mi_atomic_load_acquire(&mi_debug_stall_in_heap_delete_claim) == 1) { mi_atomic_store_release(&mi_debug_stall_in_heap_delete_claim, (uintptr_t)2); // signal: pinned, not yet claimed @@ -2600,8 +2671,8 @@ static bool mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_page_t* pag // After a multi-threaded fork() the child may inherit a torn snapshot of a page that another // thread was allocating or freeing: the bit propagated but the page-map entry or the owned bit // did not, and that thread is gone. Re-derive what we can and take the page. - if (mi_page_start(page) == NULL) return false; // the page struct never made it across: leave it unpublished - if (_mi_safe_ptr_page(mi_page_start(page)) != page && !_mi_page_map_register(page)) return false; + if (mi_page_start(page) == NULL) return NULL; // the page struct never made it across: leave it unpublished + if (_mi_safe_ptr_page(mi_page_start(page)) != page && !_mi_page_map_register(page)) return NULL; mi_page_claim_ownership(page); // ours now, whether or not the dead thread held it mi_bitmap_set(pages, slice_index); if (!mi_page_is_abandoned(page)) { mi_heap_visit_page_seize(page); } @@ -2627,15 +2698,19 @@ static bool mi_heap_visit_page_claim(mi_heap_visit_info_t* vinfo, mi_page_t* pag mi_assert_internal(mi_page_is_owned(page) && mi_page_is_abandoned(page)); mi_assert_internal(mi_bitmap_is_set(pages, slice_index)); mi_assert_internal(_mi_ptr_page(mi_page_start(page)) == page && mi_page_heap(page) == vinfo->heap); - return true; + return page; } static bool mi_heap_visit_page_at(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { MI_UNUSED(slice_count); mi_heap_visit_info_t* vinfo = (mi_heap_visit_info_t*)arg; - mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); + mi_page_t* page; if (vinfo->claim_pages) { - if (!mi_heap_visit_page_claim(vinfo, page, slice_index)) return true; + page = mi_heap_visit_page_claim(vinfo, arena, slice_index); + if (page == NULL) return true; // gone: freed by a concurrent `mi_free` + } + else { + page = mi_arena_page_at_slice(arena, slice_index); } return mi_heap_visit_page(page, vinfo); } @@ -2690,7 +2765,9 @@ bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_bloc for (size_t bin = 0; ok && bin < MI_ARENA_BIN_COUNT; bin++) { // todo: if we had a single abandoned page map as well, this can be faster. if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) { - ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info); + mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin); + if (bitmap == NULL) continue; + ok = _mi_bitmap_forall_set(bitmap, &mi_heap_visit_page_at, arena, &visit_info); } } } diff --git a/src/bitmap.c b/src/bitmap.c index 01de1999a..a1dc112aa 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -1433,12 +1433,16 @@ void mi_bitmap_clear_once_set(mi_subproc_t* subproc, mi_bitmap_t* bitmap, size_t // Visit all set bits in a bitmap. +// The loads are acquire: `mi_heap_delete` walks its `arena_pages->pages` with this and then frees the heap and +// the bitmap itself, while a concurrent `mi_free` that frees a page of the heap reads both and then clears the +// page's bit (release) as its last access (`mi_arenas_page_free_prim`). Seeing that bit clear here is what +// orders those reads before our free of the memory they read. // todo: optimize further? maybe use avx512 to directly get all indices using a mask_compressstore? bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_arena_t* arena, void* arg) { // for all chunkmap entries const size_t chunkmap_max = _mi_divide_up(mi_bitmap_chunk_count(bitmap), MI_BFIELD_BITS); for(size_t i = 0; i < chunkmap_max; i++) { - mi_bfield_t cmap_entry = mi_atomic_load_relaxed(&bitmap->chunkmap.bfields[i]); + mi_bfield_t cmap_entry = mi_atomic_load_acquire(&bitmap->chunkmap.bfields[i]); size_t cmap_idx; // for each chunk (corresponding to a set bit in a chunkmap entry) while (mi_bfield_foreach_bit(&cmap_entry, &cmap_idx)) { @@ -1447,7 +1451,7 @@ bool _mi_bitmap_forall_set(mi_bitmap_t* bitmap, mi_forall_set_fun_t* visit, mi_a mi_bchunk_t* const chunk = &bitmap->chunks[chunk_idx]; for (size_t j = 0; j < MI_BCHUNK_FIELDS; j++) { const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); - mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[j]); + mi_bfield_t b = mi_atomic_load_acquire(&chunk->bfields[j]); size_t bidx; while (mi_bfield_foreach_bit(&b, &bidx)) { const size_t idx = base_idx + bidx; diff --git a/src/heap.c b/src/heap.c index d9173f7cb..f1a049c85 100644 --- a/src/heap.c +++ b/src/heap.c @@ -181,6 +181,10 @@ mi_heap_t* mi_heap_new(void) { static void mi_heap_release_pages(mi_heap_t* heap, mi_heap_t* heap_target) { _mi_heap_detach_theaps(heap); if (_mi_is_heap_main(heap)) return; // (`_mi_heap_force_destroy` of a main heap at sub-process teardown: the arenas go as a whole) + // Step 3 claims every page through the `arena_pages->pages` bitmap, so the pages abandoned in step 2 + // do not need to be findable by size class: `_mi_arenas_page_abandon` leaves them out of the + // per-bin abandoned maps, which are then never allocated for a heap that only lives to be released. + mi_atomic_store_release(&heap->releasing, (uintptr_t)1); mi_lock(&heap->theaps_lock) { for (mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext) { mi_assert_internal(_mi_theap_heap_peek(theap)==NULL); @@ -229,7 +233,7 @@ static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); if (arena_pages!=NULL) { mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL); - _mi_free_subproc_safe(arena_pages); + _mi_arena_pages_free(arena_pages); } } } diff --git a/test/test-abandoned-lazy.c b/test/test-abandoned-lazy.c new file mode 100644 index 000000000..87ce326ab --- /dev/null +++ b/test/test-abandoned-lazy.c @@ -0,0 +1,360 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2025 Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. +-----------------------------------------------------------------------------*/ + +/* The per-bin abandoned-page bitmaps of a (heap, arena) pair are allocated the + first time a page of that bin is abandoned, not when the heap first touches + the arena. This drives every path that reads or writes them: + + - worker threads allocate blocks of several size classes from a shared heap + and exit while the blocks are still live, so their pages are abandoned + (the bitmaps are allocated on that path, by several threads at once); + - the main thread then allocates the same sizes from the heap, which reclaims + the abandoned pages, frees every block (un-abandon, page free), collects + and deletes the heap (the bitmaps are freed with it); + - the same with the main heap, whose bitmaps are never freed, and with blocks + freed by a thread that never allocated (re-abandon of a mostly free page). + + A heap that is deleted or destroyed abandons all of its pages during its own + teardown, which claims them back through the `pages` bitmap. Those pages are + not mapped, so a heap that lives only to be released never allocates a map: + + - a debug build counts the maps allocated while heaps are created, used and + destroyed on one thread, and the count must not move; + - several threads create, use and delete heaps while other threads free the + blocks of each heap during its delete (a free that would re-map a page). + + A debug build checks the bitmap invariants on each of these paths. + + > mimalloc-test-abandoned-lazy [ITER] +*/ + +#include +#include +#include +#include +#include + +#include "mimalloc.h" + +#if defined(MI_TSAN) || defined(MI_UBSAN) || defined(MI_GUARDED) +static int ITER = 20; +#else +static int ITER = 100; +#endif + +#define NTHREADS 8 +#define NSIZES 6 +#define NBLOCKS 64 // per thread per size: enough to span a few pages of each size class + +static const size_t sizes[NSIZES] = { 16, 96, 512, 2048, 8192, 40000 }; + +typedef void (thread_entry_fun_t)(intptr_t tid); +static void run_os_threads(size_t nthreads, thread_entry_fun_t* entry); + +static mi_heap_t* shared_heap; // NULL: use the main heap +static void* blocks[NTHREADS][NSIZES][NBLOCKS]; + +static void alloc_and_exit(intptr_t tid) { + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + void* p = (shared_heap != NULL ? mi_heap_malloc(shared_heap, sizes[s]) : mi_malloc(sizes[s])); + if (p == NULL) { fprintf(stderr, "allocation failed\n"); abort(); } + memset(p, (int)(tid + s), sizes[s]); + blocks[tid][s][b] = p; + } + } + // exit with everything live: every page this thread used is abandoned +} + +static void free_some(intptr_t tid) { + // free most blocks of a page from a thread that never allocated from the heap: + // a multi-threaded free that can re-abandon the page as mapped + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + if ((b % 8) != 0) { + mi_free(blocks[tid][s][b]); + blocks[tid][s][b] = NULL; + } + } + } +} + +static void check_block(intptr_t tid, int s, void* p) { + const unsigned char* q = (const unsigned char*)p; + for (size_t i = 0; i < sizes[s]; i += 64) { + if (q[i] != (unsigned char)(tid + s)) { fprintf(stderr, "block corrupted\n"); abort(); } + } +} + +static void run_round(bool use_main_heap) { + shared_heap = (use_main_heap ? NULL : mi_heap_new()); + memset(blocks, 0, sizeof(blocks)); + + // 1. abandon pages of several size classes from several threads at once + run_os_threads(NTHREADS, &alloc_and_exit); + + // 2. reclaim: allocating the same sizes must find the abandoned pages again + void* mine[NSIZES][NBLOCKS]; + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + mine[s][b] = (shared_heap != NULL ? mi_heap_malloc(shared_heap, sizes[s]) : mi_malloc(sizes[s])); + if (mine[s][b] == NULL) { fprintf(stderr, "allocation failed\n"); abort(); } + } + } + for (intptr_t t = 0; t < NTHREADS; t++) { + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { check_block(t, s, blocks[t][s][b]); } + } + } + + // 3. frees from threads that never touched the heap, then from this thread + run_os_threads(NTHREADS, &free_some); + for (intptr_t t = 0; t < NTHREADS; t++) { + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + if (blocks[t][s][b] != NULL) { check_block(t, s, blocks[t][s][b]); mi_free(blocks[t][s][b]); } + } + } + } + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { mi_free(mine[s][b]); } + } + + // 4. collect and delete + if (shared_heap != NULL) { + mi_heap_collect(shared_heap, true); + mi_heap_delete(shared_heap); + shared_heap = NULL; + } + else { + mi_collect(true); + } +} + +/* ----------------------------------------------------------- + Released heaps: no maps +----------------------------------------------------------- */ + +#if !defined(NDEBUG) +#ifdef __cplusplus +#include +extern "C" std::atomic mi_debug_abandoned_maps_allocated; +static uintptr_t maps_allocated(void) { return mi_debug_abandoned_maps_allocated.load(); } +#else +#include +extern _Atomic(uintptr_t) mi_debug_abandoned_maps_allocated; +static uintptr_t maps_allocated(void) { return atomic_load(&mi_debug_abandoned_maps_allocated); } +#endif +#define HAS_MAP_COUNTER 1 +#else +#define HAS_MAP_COUNTER 0 +#endif + +static void fill_heap(mi_heap_t* heap, void** keep) { + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < NBLOCKS; b++) { + void* p = mi_heap_malloc(heap, sizes[s]); + if (p == NULL) { fprintf(stderr, "allocation failed\n"); abort(); } + memset(p, s + 1, sizes[s]); + if (keep != NULL) { keep[s*NBLOCKS + b] = p; } + } + } +} + +static void released_heaps_map_nothing(void) { + static void* keep[NSIZES*NBLOCKS]; + // one delete first: the main heap takes the moved pages and maps them, and its maps stay + mi_heap_t* heap = mi_heap_new(); + fill_heap(heap, keep); + mi_heap_delete(heap); + for (int i = 0; i < NSIZES*NBLOCKS; i++) { mi_free(keep[i]); } + + #if HAS_MAP_COUNTER + const uintptr_t before = maps_allocated(); + #endif + for (int n = 0; n < 50; n++) { + heap = mi_heap_new(); + fill_heap(heap, NULL); + mi_heap_destroy(heap); + + heap = mi_heap_new(); + fill_heap(heap, keep); + mi_heap_delete(heap); // the blocks now belong to the main heap + for (int i = 0; i < NSIZES*NBLOCKS; i++) { mi_free(keep[i]); } + } + #if HAS_MAP_COUNTER + const uintptr_t after = maps_allocated(); + if (after != before) { + fprintf(stderr, "released heaps allocated %lu abandoned maps\n", (unsigned long)(after - before)); + abort(); + } + #endif +} + +/* ----------------------------------------------------------- + Concurrent delete: blocks freed by another thread while their heap is deleted +----------------------------------------------------------- */ + +#define NPAIRS 4 // a deleter and a freer each +#define DBLOCKS 8 // per size: the pages stay mostly free, so a free would re-map them + +static void* atomic_exchange_ptr(volatile void** p, void* newval); +static long atomic_load_long(volatile long* p); +static void atomic_store_long(volatile long* p, long x); + +static volatile void* dslots[NPAIRS][NSIZES*DBLOCKS]; +static volatile long dgo[NPAIRS]; +static volatile long ddone[NPAIRS]; +static int dround_count; + +static void delete_pair(intptr_t tid) { + const int pair = (int)(tid / 2); + if ((tid % 2) == 0) { + // deleter: allocate, let the freer start, and delete while it frees + for (int r = 1; r <= dround_count; r++) { + mi_heap_t* heap = mi_heap_new(); + for (int s = 0; s < NSIZES; s++) { + for (int b = 0; b < DBLOCKS; b++) { + void* p = mi_heap_malloc(heap, sizes[s]); + if (p == NULL) { fprintf(stderr, "allocation failed\n"); abort(); } + memset(p, s + 1, sizes[s]); + atomic_exchange_ptr(&dslots[pair][s*DBLOCKS + b], p); + } + } + atomic_store_long(&dgo[pair], r); + mi_heap_delete(heap); + while (atomic_load_long(&ddone[pair]) != r) { /* spin */ } + } + } + else { + // freer: never allocates from the heap, so it may free into it during the delete + for (int r = 1; r <= dround_count; r++) { + while (atomic_load_long(&dgo[pair]) != r) { /* spin */ } + for (int i = 0; i < NSIZES*DBLOCKS; i++) { + unsigned char* p = (unsigned char*)atomic_exchange_ptr(&dslots[pair][i], NULL); + if (p == NULL) { fprintf(stderr, "missing block\n"); abort(); } + if (p[0] != (unsigned char)(i / DBLOCKS + 1)) { fprintf(stderr, "block corrupted\n"); abort(); } + mi_free(p); + } + atomic_store_long(&ddone[pair], r); + } + } +} + +int main(int argc, char** argv) { + if (argc >= 2) { + char* end; + long n = strtol(argv[1], &end, 10); + if (n > 0) ITER = (int)n; + } + for (int i = 0; i < ITER; i++) { + run_round(false); + run_round(true); + } + released_heaps_map_nothing(); + dround_count = ITER; + run_os_threads(NPAIRS*2, &delete_pair); + printf("test-abandoned-lazy: %d rounds, ok\n", ITER); + mi_stats_print(NULL); + return 0; +} + + +/* ----------------------------------------------------------- + OS threads +----------------------------------------------------------- */ + +#ifdef _WIN32 + +#include + +static thread_entry_fun_t* thread_entry_fun; + +static DWORD WINAPI thread_entry(LPVOID param) { + thread_entry_fun((intptr_t)param); + return 0; +} + +static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { + thread_entry_fun = fun; + DWORD* tids = (DWORD*) calloc(nthreads, sizeof(DWORD)); + HANDLE* thandles = (HANDLE*)calloc(nthreads, sizeof(HANDLE)); + for (size_t i = 0; i < nthreads; i++) { + thandles[i] = CreateThread(0, 8*1024L, &thread_entry, (void*)(i), 0, &tids[i]); + } + for (size_t i = 0; i < nthreads; i++) { + WaitForSingleObject(thandles[i], INFINITE); + } + for (size_t i = 0; i < nthreads; i++) { + CloseHandle(thandles[i]); + } + free(tids); + free(thandles); +} + +static void* atomic_exchange_ptr(volatile void** p, void* newval) { + #if (INTPTR_MAX == INT32_MAX) + return (void*)InterlockedExchange((volatile LONG*)p, (LONG)newval); + #else + return (void*)InterlockedExchange64((volatile LONG64*)p, (LONG64)newval); + #endif +} +static long atomic_load_long(volatile long* p) { + return InterlockedCompareExchange(p, 0, 0); +} +static void atomic_store_long(volatile long* p, long x) { + InterlockedExchange(p, x); +} + +#else + +#include + +static thread_entry_fun_t* thread_entry_fun; + +static void* thread_entry(void* param) { + thread_entry_fun((intptr_t)param); + return NULL; +} + +static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { + thread_entry_fun = fun; + pthread_t* threads = (pthread_t*)calloc(nthreads, sizeof(pthread_t)); + memset(threads, 0, sizeof(pthread_t) * nthreads); + for (size_t i = 0; i < nthreads; i++) { + pthread_create(&threads[i], NULL, &thread_entry, (void*)i); + } + for (size_t i = 0; i < nthreads; i++) { + pthread_join(threads[i], NULL); + } + free(threads); +} + +#ifdef __cplusplus +#include +static void* atomic_exchange_ptr(volatile void** p, void* newval) { + return std::atomic_exchange((volatile std::atomic*)p, newval); +} +static long atomic_load_long(volatile long* p) { + return std::atomic_load((volatile std::atomic*)p); +} +static void atomic_store_long(volatile long* p, long x) { + std::atomic_store((volatile std::atomic*)p, x); +} +#else +#include +static void* atomic_exchange_ptr(volatile void** p, void* newval) { + return atomic_exchange((volatile _Atomic(void*)*)p, newval); +} +static long atomic_load_long(volatile long* p) { + return atomic_load((volatile _Atomic(long)*)p); +} +static void atomic_store_long(volatile long* p, long x) { + atomic_store((volatile _Atomic(long)*)p, x); +} +#endif + +#endif diff --git a/test/test-free-before-init.c b/test/test-free-before-init.c index 2d6536754..6396bdf5d 100644 --- a/test/test-free-before-init.c +++ b/test/test-free-before-init.c @@ -35,6 +35,13 @@ static void (*mi_test_preinit)(void) = &free_null_before_init; #elif defined(__GNUC__) || defined(__clang__) __attribute__((constructor)) static void free_null_before_init_ctor(void) { free_null_before_init(); } +#elif defined(_MSC_VER) +// MSVC has no constructor attribute: put a pointer in the CRT's C initializer section, which the +// startup code walks before `main` (and, for a static mimalloc, in no guaranteed order relative +// to mimalloc's own initializer there -- the same caveat as the constructor above). +#pragma section(".CRT$XCU", read) +static void __cdecl free_null_before_init_ctor(void) { free_null_before_init(); } +__declspec(allocate(".CRT$XCU")) void (__cdecl* mi_test_free_before_init_ctor)(void) = &free_null_before_init_ctor; #endif int main(void) { diff --git a/test/test-heap-release-mt.c b/test/test-heap-release-mt.c new file mode 100644 index 000000000..67cc4e0be --- /dev/null +++ b/test/test-heap-release-mt.c @@ -0,0 +1,228 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. +-----------------------------------------------------------------------------*/ + +/* Release heaps on many threads at once while other threads free into them. + + - NCHURN threads loop: `mi_heap_new`, allocate mixed sizes, hand a third of the blocks to a shared + ring, free a third, leak a third, then `mi_heap_delete` (or `mi_heap_destroy` when nothing was + handed out). Some heaps are also published for a while so that short-lived threads allocate from + them and exit with their blocks live (the abandon path, and the per-bin abandoned maps). + - NFREE threads pop blocks from the ring and `mi_free` them: those frees land before, during and + after the delete of the owning heap, so pages are freed and re-abandoned by a foreign thread while + `mi_heap_delete` claims them (`arena.c:mi_heap_visit_page_claim` against `mi_arenas_page_free_prim`). + - NSPAWN threads keep creating batches of threads that allocate from a published heap (or the main + heap) and exit at once. + + The second half runs the same churn "parked": before its heap is released, a churn thread hands + its theaps to the scavenger (`mi_on_thread_idle_start`), which then sweeps the holes of that + thread's pages and of the abandoned pages of its heaps -- including the per-bin abandoned maps of + the heap that is about to go (`theap.c:mi_purge_holes_of`, `_mi_arenas_purge_abandoned_holes`). + Half of those heaps are deleted by the parked thread itself right after `mi_on_thread_idle_end` + (racing the tail of the sweep), the other half by a freer thread *while the owner is still parked* + (a heap created on one thread and released on another while the first sits idle in the kernel), + so `mi_heap_delete`'s detach has to get past the scavenger holding that thread's theaps. + + Meant to be run under TSAN and ASAN as well as with MI_DEBUG_FULL: it must finish without a report. + + > mimalloc-test-heap-release-mt [SECONDS] +*/ +#if defined(_WIN32) +#include +int main(void) { printf("test-heap-release-mt: skipped on Windows (uses pthreads)\n"); return 0; } +#else + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define NCHURN 8 +#define NFREE 4 +#define NSPAWN 4 +#define RING (1<<16) +static _Atomic(void*) ring[RING]; +static _Atomic unsigned long ring_head, ring_tail; +static _Atomic int stop; +static _Atomic int park_mode; // second half: park around the release, see the header +static _Atomic unsigned long heaps_done, frees_done, exits_done, parks_done, parks_handed_off, remote_deletes; +static _Atomic(mi_heap_t*) to_delete[NCHURN]; // park mode: heap i's owner is parked and waits for a freer to delete it + +static const size_t sizes[] = {8,16,48,96,200,512,1024,2048,5000,8192,20000,40000,70000,200000}; +#define NS (sizeof(sizes)/sizeof(sizes[0])) + +static void push(void* p) { + for (;;) { + unsigned long h = atomic_fetch_add(&ring_head, 1); + void* expect = NULL; + if (atomic_compare_exchange_strong(&ring[h % RING], &expect, p)) return; + // slot busy: free it ourselves instead of spinning + mi_free(p); return; + } +} +static void* pop(void) { + unsigned long t = atomic_fetch_add(&ring_tail, 1); + return atomic_exchange(&ring[t % RING], NULL); +} + +static unsigned rnd(unsigned* s){ *s = *s*1103515245u+12345u; return *s>>8; } + +// shared live heaps for the exiting threads +#define NSHARED 4 +static _Atomic(mi_heap_t*) shared[NSHARED]; +static _Atomic int users[NSHARED]; + +static void* churn(void* arg) { + const int self = (int)(uintptr_t)arg - 1; + unsigned seed = (unsigned)(uintptr_t)arg * 7919u + 1; + while (!atomic_load(&stop)) { + mi_heap_t* h = mi_heap_new(); + int slot = -1; + // sometimes publish as a shared heap for exiting threads to allocate from + if ((rnd(&seed) % 4) == 0) { + slot = rnd(&seed) % NSHARED; + mi_heap_t* expect = NULL; + if (!atomic_compare_exchange_strong(&shared[slot], &expect, h)) slot = -1; + } + int handed = 0; + int n = 50 + rnd(&seed) % 400; + for (int i = 0; i < n; i++) { + size_t sz = sizes[rnd(&seed) % NS]; + void* p = mi_heap_malloc(h, sz); + if (!p) { fprintf(stderr,"oom\n"); abort(); } + memset(p, 0xAB, sz < 64 ? sz : 64); + unsigned r = rnd(&seed) % 3; + if (r == 0) { push(p); handed++; } + else if (r == 1) mi_free(p); + // else leak into the heap; delete moves it to main / destroy frees it + } + if (slot >= 0) { + // let exiting threads use it for a bit, then unpublish + usleep(200); + atomic_store(&shared[slot], NULL); + while (atomic_load(&users[slot]) != 0) usleep(10); // in-flight allocators finish (contract: no alloc during delete) + handed = 1; // exiting threads may hold blocks: must delete, not destroy + } + if (atomic_load(&park_mode)) { + const bool remote = ((rnd(&seed) & 1) != 0); + if (remote) { atomic_store(&to_delete[self], h); } // a freer deletes it while we are parked + const bool parked = mi_on_thread_idle_start(); + if (parked) { atomic_fetch_add(&parks_handed_off, 1); } + if (remote) { + // idle in the kernel as far as mimalloc is concerned: we do not allocate or free until `_end` + while (atomic_load(&to_delete[self]) != NULL && !atomic_load(&stop)) { usleep(20); } + } + else if ((rnd(&seed) & 3) == 0) { + usleep(300); // sometimes let the sweep get well into our heaps, sometimes race its start + } + mi_on_thread_idle_end(); + if (!parked) { mi_on_thread_idle(); } // no scavenger (MIMALLOC_SCAVENGER=0): sweep inline so the pass is not vacuous + atomic_fetch_add(&parks_done, 1); + if (remote) { + mi_heap_t* const left = atomic_exchange(&to_delete[self], NULL); // only non-NULL if we are stopping + if (left != NULL) { mi_heap_delete(left); } + atomic_fetch_add(&heaps_done, 1); + continue; + } + } + if (handed == 0 && (rnd(&seed) & 1)) mi_heap_destroy(h); + else mi_heap_delete(h); + atomic_fetch_add(&heaps_done, 1); + } + return NULL; +} + +// park mode: delete the heaps whose owners are parked and waiting for it +static void delete_parked_heaps(void) { + for (int i = 0; i < NCHURN; i++) { + if (atomic_load(&to_delete[i]) == NULL) continue; + mi_heap_t* const h = atomic_exchange(&to_delete[i], NULL); + if (h != NULL) { mi_heap_delete(h); atomic_fetch_add(&remote_deletes, 1); } + } +} + +static void* freer(void* arg) { + (void)arg; + while (!atomic_load(&stop) || atomic_load(&ring_tail) < atomic_load(&ring_head)) { + delete_parked_heaps(); + void* p = pop(); + if (p) { mi_free(p); atomic_fetch_add(&frees_done,1); } + else if (atomic_load(&stop)) break; + } + return NULL; +} + +static void* exiter(void* arg) { + unsigned seed = (unsigned)(uintptr_t)arg; + int slot = rnd(&seed) % NSHARED; + atomic_fetch_add(&users[slot], 1); + mi_heap_t* h = atomic_load(&shared[slot]); + for (int i = 0; i < 40; i++) { + size_t sz = sizes[rnd(&seed) % 8]; + void* p = h ? mi_heap_malloc(h, sz) : mi_malloc(sz); + if (p) { memset(p, 0xCD, sz < 64 ? sz : 64); push(p); } + } + atomic_fetch_sub(&users[slot], 1); + atomic_fetch_add(&exits_done,1); + return NULL; // exit with blocks live -> pages abandoned (mapped, lazy bitmap alloc) +} + +static void* spawner(void* arg) { + unsigned k = (unsigned)(uintptr_t)arg * 100000u; + while (!atomic_load(&stop)) { + pthread_t t[8]; + for (int i = 0; i < 8; i++) pthread_create(&t[i], NULL, exiter, (void*)(uintptr_t)(k++)); + for (int i = 0; i < 8; i++) pthread_join(t[i], NULL); + } + return NULL; +} + +static void run(int secs) { + atomic_store(&stop, 0); + atomic_store(&ring_head, 0); atomic_store(&ring_tail, 0); + pthread_t tc[NCHURN], tf[NFREE], ts[NSPAWN]; + for (long i = 0; i < NFREE; i++) pthread_create(&tf[i], NULL, freer, (void*)i); + for (long i = 0; i < NCHURN; i++) pthread_create(&tc[i], NULL, churn, (void*)(i+1)); + for (long i = 0; i < NSPAWN; i++) pthread_create(&ts[i], NULL, spawner, (void*)(i+1)); + sleep((unsigned)secs); + atomic_store(&stop, 1); + for (int i = 0; i < NCHURN; i++) pthread_join(tc[i], NULL); + for (int i = 0; i < NSPAWN; i++) pthread_join(ts[i], NULL); + for (int i = 0; i < NFREE; i++) pthread_join(tf[i], NULL); + // drain + for (unsigned i = 0; i < RING; i++) { void* q = atomic_exchange(&ring[i], NULL); if (q) mi_free(q); } + mi_collect(true); +} + +int main(int argc, char** argv) { + int secs = (argc > 1 ? atoi(argv[1]) : 0); + if (secs <= 0) { + #if defined(MI_TSAN) || !defined(NDEBUG) + secs = 4; + #else + secs = 6; + #endif + } + // 1. plain + run((secs + 1) / 2); + printf("test-heap-release-mt: churn ok (heaps=%lu frees=%lu thread-exits=%lu)\n", + (unsigned long)heaps_done, (unsigned long)frees_done, (unsigned long)exits_done); + // 2. parked around the release, every park swept (no rate limit) + mi_option_set(mi_option_purge_holes_min_interval, 0); + atomic_store(&park_mode, 1); + run((secs + 1) / 2); + printf("test-heap-release-mt: parked ok (heaps=%lu frees=%lu thread-exits=%lu parks=%lu handed-off=%lu remote-deletes=%lu scavenger=%s)\n", + (unsigned long)heaps_done, (unsigned long)frees_done, (unsigned long)exits_done, + (unsigned long)parks_done, (unsigned long)parks_handed_off, (unsigned long)remote_deletes, + mi_option_is_enabled(mi_option_scavenger) ? "on" : "off"); + return 0; +} + +#endif