diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 88d943d10..20bb932a8 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -889,7 +889,7 @@ static inline bool mi_page_all_free(const mi_page_t* page) { // Page hole purging (see the "Page hole purging" section in `page.c`) // ------------------------------------------------------ -void _mi_page_purge_holes(mi_page_t* page); +void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld); void _mi_page_purged_reset(mi_page_t* page); bool _mi_page_unpurge_run(mi_page_t* page); void _mi_page_unpurge_all(mi_page_t* page); @@ -898,12 +898,12 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end); size_t _mi_page_unformed_purged_bytes(const mi_page_t* page); // the bytes of this page's unformed tail that are discarded right now bool _mi_page_purge_os_page_blocks(size_t os_page_size, size_t block_size, uintptr_t page_start, size_t capacity, size_t k, size_t* first, size_t* last); -bool _mi_page_purge_holes_in_progress(void); +bool _mi_page_purge_holes_in_progress(const mi_page_t* page); void _mi_page_holes_count_page_freed(void); void _mi_page_holes_count_ineligible(const mi_page_t* page); void _mi_page_holes_reset_ineligible(void); -void _mi_page_purge_holes_begin(void); -void _mi_page_purge_holes_end(void); +void _mi_page_purge_holes_begin(mi_tld_t* tld); +void _mi_page_purge_holes_end(mi_tld_t* tld); void _mi_page_purge_holes_sweep_begin(mi_tld_t* tld); // once per idle sweep, before its passes // ------------------------------------------------------ diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index d7c2689be..330f53e75 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -709,6 +709,15 @@ struct mi_tld_s { mi_tld_t* subproc_next; // list of tlds in the subproc, so the scavenger can find parked threads size_t holes_sweep_seq; // idle sweeps run over THIS tld's heaps (paces `purge_holes_full_every`) mi_msecs_t holes_sweep_last; // when this tld's heaps were last swept (paces `purge_holes_min_interval`) + + // Scratch state of the hole sweep running over THIS tld's heaps (by its own thread, or by the + // scavenger while it is parked). Deliberately not compiler thread-locals: `purging_holes` is + // read on the allocation slow path, and with emulated TLS (Android before API 29) the first + // access to a `__thread` variable itself calls `malloc`, which recursed without bound. + bool purging_holes; // re-entrancy guard: a sweep is rewriting this tld's free lists right now + bool holes_sweep_full; // this sweep ignores `page->swept_state` + size_t holes_sweep_skipped; // per-sweep counters, folded into the process-wide stats at `_end` + size_t holes_sweep_visited; }; diff --git a/src/arena.c b/src/arena.c index af7e8fed5..df39cf961 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1362,7 +1362,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, _mi_page_holes_count_page_freed(); return true; } - _mi_page_purge_holes(page); + _mi_page_purge_holes(page, parg->tld); mi_bitmap_set(bitmap, slice_index); // back in the map *before* unowning: unown may free the page mi_abandoned_page_unown(page, NULL); return true; @@ -1374,7 +1374,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { if (heap == NULL) return; if (!mi_option_is_enabled(mi_option_purge_holes)) return; - _mi_page_purge_holes_begin(); + _mi_page_purge_holes_begin(tld); mi_forall_arenas(heap, ((mi_arena_t*)NULL), 0, arena) { mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); if (arena_pages != NULL) { @@ -1389,7 +1389,7 @@ void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { } } mi_forall_arenas_end(); - _mi_page_purge_holes_end(); + _mi_page_purge_holes_end(tld); } // The read-only counterpart of the sweep above: account for the holes in the abandoned pages diff --git a/src/page.c b/src/page.c index 382d7d92c..874218565 100644 --- a/src/page.c +++ b/src/page.c @@ -472,44 +472,54 @@ static void mi_holes_count_reuse(size_t bytes, size_t blocks, bool reused) { mi_atomic_addi64_relaxed(&mi_holes_blocks, -(int64_t)blocks); } -// Re-entrancy guard: while the idle sweep is rewriting a page's free list and -// bitmap, a nested `mi_malloc` (only reachable through a user output function -// from a warning message) must not un-purge a hole from under it. -static mi_decl_thread bool mi_purging_holes; - -// Per-sweep counters. The sweep runs over every page of the thread, so a process-wide atomic -// per page would be a real cost on the very path we are making cheap: accumulate thread-locally -// and fold them in once per pass, in `_mi_page_purge_holes_end`. -static mi_decl_thread size_t mi_holes_sweep_skipped; -static mi_decl_thread size_t mi_holes_sweep_visited; +// The sweep's scratch state (`purging_holes`, `holes_sweep_*`) lives on the tld whose heaps are being +// swept -- see `mi_tld_s` -- and is passed down explicitly, so nothing here reads compiler thread +// locals (with emulated TLS, e.g. Android before API 29, the first access to a `__thread` variable +// allocates, and `_mi_page_purge_holes_in_progress` is reached from the allocation slow path). +// The sweeping thread owns that tld for the duration: it is either its own, or a parked thread's +// that it claimed (MI_PARK_SWEEPING). +// +// - `purging_holes` is the re-entrancy guard: while the sweep is rewriting a page's free list and +// bitmap, a nested `mi_malloc` on the sweeping thread (only reachable through a user output +// function from a warning message) must not un-purge a hole from under it. Such an allocation +// can only come from a page of the swept tld when the sweeper is the owner itself, which is why +// the reader consults the page owner's tld. +// - `holes_sweep_skipped/visited` are per-sweep counters. The sweep runs over every page of the +// thread, so a process-wide atomic per page would be a real cost on the very path we are making +// cheap: accumulate on the tld and fold them in once per pass, in `_mi_page_purge_holes_end`. +// - `holes_sweep_full` is whether THIS sweep ignores `page->swept_state` (see `_mi_page_purge_holes`). +// Set once per idle sweep, in `_mi_page_purge_holes_sweep_begin`, paced by the same tld's sequence. -// Whether THIS sweep ignores `page->swept_state` (see `_mi_page_purge_holes`). Set once per idle -// sweep, in `_mi_page_purge_holes_sweep_begin`; the sequence it is paced by lives on the tld being -// swept, since one scavenger thread runs the sweeps of many. -static mi_decl_thread bool mi_holes_sweep_full; +bool _mi_page_purge_holes_in_progress(const mi_page_t* page) { + const mi_theap_t* const theap = page->theap; // NULL for an abandoned page: nobody allocates from those + return (theap != NULL && theap->tld != NULL && theap->tld->purging_holes); +} -bool _mi_page_purge_holes_in_progress(void) { return mi_purging_holes; } -void _mi_page_purge_holes_begin(void) { mi_assert_internal(!mi_purging_holes); mi_purging_holes = true; } +void _mi_page_purge_holes_begin(mi_tld_t* tld) { + mi_assert_internal(tld != NULL && !tld->purging_holes); + if (tld == NULL) return; + tld->purging_holes = true; +} -void _mi_page_purge_holes_end(void) { - mi_assert_internal(mi_purging_holes); - mi_purging_holes = false; - if (mi_holes_sweep_skipped > 0) { - mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)mi_holes_sweep_skipped); - mi_holes_sweep_skipped = 0; +void _mi_page_purge_holes_end(mi_tld_t* tld) { + mi_assert_internal(tld != NULL && tld->purging_holes); + if (tld == NULL) return; + tld->purging_holes = false; + if (tld->holes_sweep_skipped > 0) { + mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)tld->holes_sweep_skipped); + tld->holes_sweep_skipped = 0; } - if (mi_holes_sweep_visited > 0) { - mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)mi_holes_sweep_visited); - mi_holes_sweep_visited = 0; + if (tld->holes_sweep_visited > 0) { + mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)tld->holes_sweep_visited); + tld->holes_sweep_visited = 0; } } -// Called once per idle sweep of `tld`'s heaps, before its passes (`mi_purge_holes_of`). void _mi_page_purge_holes_sweep_begin(mi_tld_t* tld) { const long every = mi_option_get(mi_option_purge_holes_full_every); const size_t seq = ++tld->holes_sweep_seq; - mi_holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0); - if (mi_holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); } + tld->holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0); + if (tld->holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); } } static inline bool mi_page_bits_at(const uint64_t* bits, size_t k) { @@ -678,7 +688,7 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end) { // Walk the free list of a page and discard every OS page in it that holds no live block. // Returns false if any discard failed: those blocks went straight back on the free list and the // page must be swept again, so the caller must not record it as swept. -static bool mi_page_purge_holes_walk(mi_page_t* page) { +static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { if (page->free == NULL) return true; // nothing to take off the free list const size_t os_size = _mi_os_page_size(); @@ -703,7 +713,7 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) { nfree[k]++; } } - mi_holes_sweep_visited += nvisited; // folded into the process-wide counter at the end of the sweep + if (tld != NULL) { tld->holes_sweep_visited += nvisited; } // folded into the process-wide counter at the end of the sweep // 2. an OS page can be discarded when *every* block overlapping it is free -- either on the // free list, or purged already. Of the blocks overlapping an OS page, only the first and @@ -789,7 +799,7 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) { // regardless, which caps the delay of a missed discard at N parks for 1/N of the old cost. // (An exact "was anything freed in this page" bit is the alternative, and it costs a store in // `mi_free` itself -- the hot path this whole feature stays off.) -void _mi_page_purge_holes(mi_page_t* page) { +void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld) { mi_assert_internal(page != NULL); if (!mi_option_is_enabled(mi_option_purge_holes)) return; if (mi_page_all_free(page)) return; // the page itself is about to be freed @@ -797,18 +807,18 @@ void _mi_page_purge_holes(mi_page_t* page) { mi_page_purge_unformed_tail(page); // the blocks that are not formed yet: resident, but never handed out if (!mi_page_can_purge_holes(page)) { _mi_page_holes_count_ineligible(page); return; } - if (!mi_holes_sweep_full && page->swept_state == mi_page_sweep_state(page)) { - mi_holes_sweep_skipped++; // nothing was allocated or freed in this page since we swept it + if ((tld == NULL || !tld->holes_sweep_full) && page->swept_state == mi_page_sweep_state(page)) { + if (tld != NULL) { tld->holes_sweep_skipped++; } // nothing was allocated or freed in this page since we swept it return; } // Record the state we LEAVE the page in, read back from the page: a nested `mi_malloc` (see - // `mi_purging_holes`) may have taken a block out of it while we walked. + // `purging_holes`) may have taken a block out of it while we walked. // // Only if the walk got everything. A failed `_mi_os_discard` (ENOMEM under pressure) puts its // blocks straight back, and changes neither `capacity` nor `used` -- so recording here would // say "already swept" for a page that still has holes, and the skip check would then park them // until the next full sweep, or forever with `purge_holes_full_every=0`. - if (mi_page_purge_holes_walk(page)) { + if (mi_page_purge_holes_walk(page, tld)) { page->swept_state = mi_page_sweep_state(page); } } @@ -1210,7 +1220,7 @@ static void mi_page_free_collect_ex(mi_page_t* page, bool force, bool allow_unpu // Free list empty but this page has discarded holes: bring a whole run of them // back. Every caller re-checks `mi_page_immediate_available` after collect, so the // page becomes usable again without touching the other holes. - if (allow_unpurge && page->free == NULL && mi_page_has_purged(page) && !_mi_page_purge_holes_in_progress()) { + if (allow_unpurge && page->free == NULL && mi_page_has_purged(page) && !_mi_page_purge_holes_in_progress(page)) { _mi_page_unpurge_run(page); } } diff --git a/src/theap.c b/src/theap.c index 1a0a9463d..89fe91c65 100644 --- a/src/theap.c +++ b/src/theap.c @@ -178,7 +178,7 @@ static bool mi_theap_page_purge_holes(mi_theap_t* theap, mi_page_queue_t* pq, mi _mi_page_free(page, pq); return true; } - _mi_page_purge_holes(page); + _mi_page_purge_holes(page, theap->tld); mi_assert_expensive(_mi_page_is_valid(page)); return true; // continue } @@ -193,9 +193,9 @@ static void mi_theap_purge_holes(mi_theap_t* theap) mi_attr_noexcept { if (theap->tld == NULL) return; if (theap->tld->thread_id != _mi_thread_id() && mi_atomic_load_acquire(&theap->tld->park_state) != MI_PARK_SWEEPING) return; - _mi_page_purge_holes_begin(); + _mi_page_purge_holes_begin(theap->tld); mi_theap_visit_pages(theap, &mi_theap_page_purge_holes, true /* include full pages */, NULL, NULL); - _mi_page_purge_holes_end(); + _mi_page_purge_holes_end(theap->tld); } // Purge the holes in every page this thread may safely touch: