purge-holes: keep sweep state off __thread statics (per-page guard, counters on the tld)

include/mimalloc/internal.h

@@ -889,7 +889,7 @@ static inline bool mi_page_all_free(const mi_page_t* page) {
 // Page hole purging  (see the "Page hole purging" section in `page.c`)
 // ------------------------------------------------------
 
-void          _mi_page_purge_holes(mi_page_t* page);
+void          _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld);
 void          _mi_page_purged_reset(mi_page_t* page);
 bool          _mi_page_unpurge_run(mi_page_t* page);
 void          _mi_page_unpurge_all(mi_page_t* page);
@@ -898,12 +898,12 @@ void          _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end);
 size_t        _mi_page_unformed_purged_bytes(const mi_page_t* page);            // the bytes of this page's unformed tail that are discarded right now
 bool          _mi_page_purge_os_page_blocks(size_t os_page_size, size_t block_size, uintptr_t page_start,
                                             size_t capacity, size_t k, size_t* first, size_t* last);
-bool          _mi_page_purge_holes_in_progress(void);
+bool          _mi_page_purge_holes_in_progress(const mi_page_t* page);
 void          _mi_page_holes_count_page_freed(void);
 void          _mi_page_holes_count_ineligible(const mi_page_t* page);
 void          _mi_page_holes_reset_ineligible(void);
-void          _mi_page_purge_holes_begin(void);
-void          _mi_page_purge_holes_end(void);
+void          _mi_page_purge_holes_begin(mi_tld_t* tld);
+void          _mi_page_purge_holes_end(mi_tld_t* tld);
 void          _mi_page_purge_holes_sweep_begin(mi_tld_t* tld);   // once per idle sweep, before its passes
 
 // ------------------------------------------------------

include/mimalloc/types.h

@@ -709,6 +709,15 @@ struct mi_tld_s {
   mi_tld_t*             subproc_next;         // list of tlds in the subproc, so the scavenger can find parked threads
   size_t                holes_sweep_seq;      // idle sweeps run over THIS tld's heaps (paces `purge_holes_full_every`)
   mi_msecs_t            holes_sweep_last;     // when this tld's heaps were last swept (paces `purge_holes_min_interval`)
+
+  // Scratch state of the hole sweep running over THIS tld's heaps (by its own thread, or by the
+  // scavenger while it is parked). Deliberately not compiler thread-locals: `purging_holes` is
+  // read on the allocation slow path, and with emulated TLS (Android before API 29) the first
+  // access to a `__thread` variable itself calls `malloc`, which recursed without bound.
+  bool                  purging_holes;        // re-entrancy guard: a sweep is rewriting this tld's free lists right now
+  bool                  holes_sweep_full;     // this sweep ignores `page->swept_state`
+  size_t                holes_sweep_skipped;  // per-sweep counters, folded into the process-wide stats at `_end`
+  size_t                holes_sweep_visited;
 };
 
 

src/arena.c

@@ -1362,7 +1362,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count,
     _mi_page_holes_count_page_freed();
     return true;
   }
-  _mi_page_purge_holes(page);
+  _mi_page_purge_holes(page, parg->tld);
   mi_bitmap_set(bitmap, slice_index);     // back in the map *before* unowning: unown may free the page
   mi_abandoned_page_unown(page, NULL);
   return true;
@@ -1374,7 +1374,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count,
 void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) {
   if (heap == NULL) return;
   if (!mi_option_is_enabled(mi_option_purge_holes)) return;
-  _mi_page_purge_holes_begin();
+  _mi_page_purge_holes_begin(tld);
   mi_forall_arenas(heap, ((mi_arena_t*)NULL), 0, arena) {
     mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena);
     if (arena_pages != NULL) {
@@ -1389,7 +1389,7 @@ void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) {
     }
   }
   mi_forall_arenas_end();
-  _mi_page_purge_holes_end();
+  _mi_page_purge_holes_end(tld);
 }
 
 // The read-only counterpart of the sweep above: account for the holes in the abandoned pages

src/page.c

@@ -472,44 +472,54 @@ static void mi_holes_count_reuse(size_t bytes, size_t blocks, bool reused) {
   mi_atomic_addi64_relaxed(&mi_holes_blocks, -(int64_t)blocks);
 }
 
-// Re-entrancy guard: while the idle sweep is rewriting a page's free list and
-// bitmap, a nested `mi_malloc` (only reachable through a user output function
-// from a warning message) must not un-purge a hole from under it.
-static mi_decl_thread bool mi_purging_holes;
-
-// Per-sweep counters. The sweep runs over every page of the thread, so a process-wide atomic
-// per page would be a real cost on the very path we are making cheap: accumulate thread-locally
-// and fold them in once per pass, in `_mi_page_purge_holes_end`.
-static mi_decl_thread size_t mi_holes_sweep_skipped;
-static mi_decl_thread size_t mi_holes_sweep_visited;
+// The sweep's scratch state (`purging_holes`, `holes_sweep_*`) lives on the tld whose heaps are being
+// swept -- see `mi_tld_s` -- and is passed down explicitly, so nothing here reads compiler thread
+// locals (with emulated TLS, e.g. Android before API 29, the first access to a `__thread` variable
+// allocates, and `_mi_page_purge_holes_in_progress` is reached from the allocation slow path).
+// The sweeping thread owns that tld for the duration: it is either its own, or a parked thread's
+// that it claimed (MI_PARK_SWEEPING).
+//
+// - `purging_holes` is the re-entrancy guard: while the sweep is rewriting a page's free list and
+//   bitmap, a nested `mi_malloc` on the sweeping thread (only reachable through a user output
+//   function from a warning message) must not un-purge a hole from under it. Such an allocation
+//   can only come from a page of the swept tld when the sweeper is the owner itself, which is why
+//   the reader consults the page owner's tld.
+// - `holes_sweep_skipped/visited` are per-sweep counters. The sweep runs over every page of the
+//   thread, so a process-wide atomic per page would be a real cost on the very path we are making
+//   cheap: accumulate on the tld and fold them in once per pass, in `_mi_page_purge_holes_end`.
+// - `holes_sweep_full` is whether THIS sweep ignores `page->swept_state` (see `_mi_page_purge_holes`).
+//   Set once per idle sweep, in `_mi_page_purge_holes_sweep_begin`, paced by the same tld's sequence.
 
-// Whether THIS sweep ignores `page->swept_state` (see `_mi_page_purge_holes`). Set once per idle
-// sweep, in `_mi_page_purge_holes_sweep_begin`; the sequence it is paced by lives on the tld being
-// swept, since one scavenger thread runs the sweeps of many.
-static mi_decl_thread bool   mi_holes_sweep_full;
+bool _mi_page_purge_holes_in_progress(const mi_page_t* page) {
+  const mi_theap_t* const theap = page->theap;   // NULL for an abandoned page: nobody allocates from those
+  return (theap != NULL && theap->tld != NULL && theap->tld->purging_holes);
+}
 
-bool _mi_page_purge_holes_in_progress(void) { return mi_purging_holes; }
-void _mi_page_purge_holes_begin(void) { mi_assert_internal(!mi_purging_holes); mi_purging_holes = true; }
+void _mi_page_purge_holes_begin(mi_tld_t* tld) {
+  mi_assert_internal(tld != NULL && !tld->purging_holes);
+  if (tld == NULL) return;
+  tld->purging_holes = true;
+}
 
-void _mi_page_purge_holes_end(void) {
-  mi_assert_internal(mi_purging_holes);
-  mi_purging_holes = false;
-  if (mi_holes_sweep_skipped > 0) {
-    mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)mi_holes_sweep_skipped);
-    mi_holes_sweep_skipped = 0;
+void _mi_page_purge_holes_end(mi_tld_t* tld) {
+  mi_assert_internal(tld != NULL && tld->purging_holes);
+  if (tld == NULL) return;
+  tld->purging_holes = false;
+  if (tld->holes_sweep_skipped > 0) {
+    mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)tld->holes_sweep_skipped);
+    tld->holes_sweep_skipped = 0;
   }
-  if (mi_holes_sweep_visited > 0) {
-    mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)mi_holes_sweep_visited);
-    mi_holes_sweep_visited = 0;
+  if (tld->holes_sweep_visited > 0) {
+    mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)tld->holes_sweep_visited);
+    tld->holes_sweep_visited = 0;
   }
 }
 
-// Called once per idle sweep of `tld`'s heaps, before its passes (`mi_purge_holes_of`).
 void _mi_page_purge_holes_sweep_begin(mi_tld_t* tld) {
   const long every = mi_option_get(mi_option_purge_holes_full_every);
   const size_t seq = ++tld->holes_sweep_seq;
-  mi_holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0);
-  if (mi_holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); }
+  tld->holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0);
+  if (tld->holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); }
 }
 
 static inline bool mi_page_bits_at(const uint64_t* bits, size_t k) {
@@ -678,7 +688,7 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end) {
 // Walk the free list of a page and discard every OS page in it that holds no live block.
 // Returns false if any discard failed: those blocks went straight back on the free list and the
 // page must be swept again, so the caller must not record it as swept.
-static bool mi_page_purge_holes_walk(mi_page_t* page) {
+static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) {
   if (page->free == NULL) return true;                    // nothing to take off the free list
 
   const size_t os_size = _mi_os_page_size();
@@ -703,7 +713,7 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) {
       nfree[k]++;
     }
   }
-  mi_holes_sweep_visited += nvisited;   // folded into the process-wide counter at the end of the sweep
+  if (tld != NULL) { tld->holes_sweep_visited += nvisited; }   // folded into the process-wide counter at the end of the sweep
 
   // 2. an OS page can be discarded when *every* block overlapping it is free -- either on the
   //    free list, or purged already. Of the blocks overlapping an OS page, only the first and
@@ -789,26 +799,26 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) {
 // regardless, which caps the delay of a missed discard at N parks for 1/N of the old cost.
 // (An exact "was anything freed in this page" bit is the alternative, and it costs a store in
 // `mi_free` itself -- the hot path this whole feature stays off.)
-void _mi_page_purge_holes(mi_page_t* page) {
+void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld) {
   mi_assert_internal(page != NULL);
   if (!mi_option_is_enabled(mi_option_purge_holes)) return;
   if (mi_page_all_free(page)) return;                     // the page itself is about to be freed
   if (mi_option_get(mi_option_purge_delay) < 0) return;   // purging disabled
   mi_page_purge_unformed_tail(page);                      // the blocks that are not formed yet: resident, but never handed out
   if (!mi_page_can_purge_holes(page)) { _mi_page_holes_count_ineligible(page); return; }
 
-  if (!mi_holes_sweep_full && page->swept_state == mi_page_sweep_state(page)) {
-    mi_holes_sweep_skipped++;   // nothing was allocated or freed in this page since we swept it
+  if ((tld == NULL || !tld->holes_sweep_full) && page->swept_state == mi_page_sweep_state(page)) {
+    if (tld != NULL) { tld->holes_sweep_skipped++; }   // nothing was allocated or freed in this page since we swept it
     return;
   }
   // Record the state we LEAVE the page in, read back from the page: a nested `mi_malloc` (see
-  // `mi_purging_holes`) may have taken a block out of it while we walked.
+  // `purging_holes`) may have taken a block out of it while we walked.
   //
   // Only if the walk got everything. A failed `_mi_os_discard` (ENOMEM under pressure) puts its
   // blocks straight back, and changes neither `capacity` nor `used` -- so recording here would
   // say "already swept" for a page that still has holes, and the skip check would then park them
   // until the next full sweep, or forever with `purge_holes_full_every=0`.
-  if (mi_page_purge_holes_walk(page)) {
+  if (mi_page_purge_holes_walk(page, tld)) {
     page->swept_state = mi_page_sweep_state(page);
   }
 }
@@ -1210,7 +1220,7 @@ static void mi_page_free_collect_ex(mi_page_t* page, bool force, bool allow_unpu
   // Free list empty but this page has discarded holes: bring a whole run of them
   // back. Every caller re-checks `mi_page_immediate_available` after collect, so the
   // page becomes usable again without touching the other holes.
-  if (allow_unpurge && page->free == NULL && mi_page_has_purged(page) && !_mi_page_purge_holes_in_progress()) {
+  if (allow_unpurge && page->free == NULL && mi_page_has_purged(page) && !_mi_page_purge_holes_in_progress(page)) {
     _mi_page_unpurge_run(page);
   }
 }

src/theap.c

@@ -178,7 +178,7 @@ static bool mi_theap_page_purge_holes(mi_theap_t* theap, mi_page_queue_t* pq, mi
     _mi_page_free(page, pq);
     return true;
   }
-  _mi_page_purge_holes(page);
+  _mi_page_purge_holes(page, theap->tld);
   mi_assert_expensive(_mi_page_is_valid(page));
   return true; // continue
 }
@@ -193,9 +193,9 @@ static void mi_theap_purge_holes(mi_theap_t* theap) mi_attr_noexcept {
   if (theap->tld == NULL) return;
   if (theap->tld->thread_id != _mi_thread_id() &&
       mi_atomic_load_acquire(&theap->tld->park_state) != MI_PARK_SWEEPING) return;
-  _mi_page_purge_holes_begin();
+  _mi_page_purge_holes_begin(theap->tld);
   mi_theap_visit_pages(theap, &mi_theap_page_purge_holes, true /* include full pages */, NULL, NULL);
-  _mi_page_purge_holes_end();
+  _mi_page_purge_holes_end(theap->tld);
 }
 
 // Purge the holes in every page this thread may safely touch: