arena: allocate the abandoned-page bitmaps on first abandon, and not at all for a heap that is being released

CMakeLists.txt

@@ -877,7 +877,7 @@ if (MI_BUILD_TESTS)
   enable_testing()
 
   # static link tests
-  set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-aba fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init)
+  set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-aba abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init)
   if(NOT (MI_DEBUG_TSAN OR MI_TRACK_ASAN OR MI_DEBUG_UBSAN))
     list(APPEND mi_static_tests thp-optout)  # counts madvise calls by interposing it, which a sanitizer runtime does first
   endif()

include/mimalloc/internal.h

@@ -277,6 +277,7 @@ void          _mi_arenas_abandoned_page_free(mi_page_t* page, mi_theap_t* curren
 void          _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap);
 void          _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx /* can be NULL */);
 bool          _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page);
+void          _mi_arena_pages_free(mi_arena_pages_t* arena_pages);
 size_t        mi_arenas_get_count(mi_subproc_t* subproc);
 uint8_t*      mi_arena_slice_start(mi_arena_t* arena, size_t slice_index);
 

include/mimalloc/types.h

@@ -628,6 +628,7 @@ typedef struct mi_heap_s {
   mi_lock_t             theaps_lock;                    // lock for the theaps list operations
 
   _Atomic(size_t)       abandoned_count[MI_BIN_COUNT];  // total count of abandoned pages in this heap
+  _Atomic(uintptr_t)    releasing;                      // set when `mi_heap_delete`/`mi_heap_destroy` starts: its pages are abandoned unmapped
   mi_page_t*            os_abandoned_pages;             // list of pages that are OS allocated and not in an arena
   mi_lock_t             os_abandoned_pages_lock;        // lock for the os abandoned pages list (this lock protects list operations)
 
@@ -756,8 +757,13 @@ typedef struct mi_bbitmap_s mi_bbitmap_t;   // atomic binned bitmap (defined in
 
 struct mi_arena_pages_s {
   mi_bitmap_t* pages;                // all registered pages (abandoned and owned)
-  mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT];  // abandoned pages per size bin (a set bit means the start of the page)
-  // followed by the bitmaps (whose siz`es depend on the arena size)
+  // Abandoned pages per size bin (a set bit means the start of the page). Each bitmap is
+  // allocated the first time a page of that bin is abandoned (`mi_arena_pages_abandoned_ensure`);
+  // NULL means no page of that bin was ever abandoned. Eagerly laying out all MI_ARENA_BIN_COUNT
+  // of them cost a page fault per bitmap (one header write each, on its own OS page) for every
+  // heap that touched an arena, and most heaps never abandon a page.
+  _Atomic(mi_bitmap_t*) pages_abandoned[MI_ARENA_BIN_COUNT];
+  // followed by the `pages` bitmap (whose size depends on the arena size)
 };
 
 

src/arena.c

@@ -29,6 +29,9 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo
 #error "The page_t.page_ma_offset field is not large enough to cover a full arena"
 #endif
 
+static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size_t bin);
+static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin);
+
 /* -----------------------------------------------------------
   Arena id's
 ----------------------------------------------------------- */
@@ -726,9 +729,9 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl
   mi_forall_suitable_arenas(heap, req_arena, tseq, match_numa, any_numa, allow_large, arena)
   {
     mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena);
-    if (arena_pages != NULL) {
+    mi_bitmap_t* const bitmap = (arena_pages != NULL ? mi_arena_pages_abandoned(arena_pages, bin) : NULL);
+    if (bitmap != NULL) {
       size_t slice_index;
-      mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin];
 
       if (mi_bitmap_try_find_and_claim(bitmap, tseq, &slice_index, &mi_arena_try_claim_abandoned, arena)) {
         // found an abandoned page of the right size
@@ -1119,7 +1122,7 @@ static void mi_arenas_page_free_prim(mi_page_t* page, mi_subproc_t* subproc, mi_
       const size_t bin = _mi_bin(mi_page_block_size(page));
       mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count));
       mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
-      mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1));
+      mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_arena_pages_abandoned(arena_pages, bin) == NULL || mi_bitmap_is_clearN(mi_arena_pages_abandoned(arena_pages, bin), slice_index, 1));
       // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may
       // be (temporarily) not true if the free happens while trying to reclaim
       // see `mi_arena_try_claim_abandoned`
@@ -1208,8 +1211,9 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) {
   // mi_assert_internal(current_theap == _mi_page_associated_theap(page));
 
   // add to abandoned?
+  // (not for a heap that is being released: its teardown claims every page through `pages`, see `mi_heap_release_pages`)
   mi_heap_t* heap = mi_page_heap(page);   
-  if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) {
+  if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page) && mi_atomic_load_relaxed(&heap->releasing) == 0) {
     // make available for allocations
     size_t bin = _mi_bin(mi_page_block_size(page));
     mi_assert_internal(bin < MI_ARENA_BIN_COUNT);
@@ -1224,13 +1228,18 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) {
       mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
       mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count));
 
-      mi_page_set_abandoned_mapped(page);
-      const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index);
-      MI_UNUSED(was_clear); mi_assert_internal(was_clear);
-      mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);
-      mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1);
-      mi_abandoned_page_unown(page, current_theapx);
-      return;
+      // If the bin's bitmap cannot be allocated the page is abandoned unmapped, like a full
+      // page: it is still reachable through `pages` and is reclaimed once a block in it is freed.
+      mi_bitmap_t* const bitmap = mi_arena_pages_abandoned_ensure(arena, arena_pages, bin);
+      if mi_likely(bitmap != NULL) {
+        mi_page_set_abandoned_mapped(page);
+        const bool was_clear = mi_bitmap_set(bitmap, slice_index);
+        MI_UNUSED(was_clear); mi_assert_internal(was_clear);
+        mi_atomic_increment_relaxed(&heap->abandoned_count[bin]);
+        mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1);
+        mi_abandoned_page_unown(page, current_theapx);
+        return;
+      }
     }
   }
   // otherwise,
@@ -1264,6 +1273,9 @@ bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) {
   if (mi_page_is_full(page) || mi_page_is_abandoned_mapped(page) || page->memid.memkind != MI_MEM_ARENA) {
     return false;
   }
+  else if (mi_atomic_load_relaxed(&mi_page_heap(page)->releasing) != 0) {
+    return false;  // the heap is being released and claims the page through `pages` (see `mi_heap_release_pages`)
+  }
   else {
     // Account on the heap and not on this thread's theap for it (`_mi_page_associated_theap_peek`): the theap
     // was only used for its statistics here, and a concurrent `mi_heap_delete` may be detaching and merging it.
@@ -1298,7 +1310,9 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) {
     mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
 
     // this busy waits until a concurrent reader (from alloc_abandoned) is done
-    mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index);
+    mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin);
+    mi_assert_internal(bitmap != NULL);  // a mapped page was set in it
+    mi_bitmap_clear_once_set(arena->subproc, bitmap, slice_index);
     mi_page_clear_abandoned_mapped(page);
     mi_atomic_decrement_relaxed(&heap->abandoned_count[bin]);
   }
@@ -1384,7 +1398,8 @@ void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) {
       // singleton bins have no abandoned bitmap (upstream ad1bcdbf, to shrink arena meta).
       for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {
         if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) continue;
-        mi_bitmap_t* const bitmap = arena_pages->pages_abandoned[bin];
+        mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin);
+        if (bitmap == NULL) continue;
         mi_purge_holes_arg_t parg = { bitmap, tld };
         (void)_mi_bitmap_forall_set(bitmap, &mi_arena_page_purge_holes_at, arena, &parg);
       }
@@ -1427,7 +1442,8 @@ void _mi_arenas_holes_report(mi_heap_t* heap, mi_holes_report_t* rep) {
     if (arena_pages != NULL) {
       for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {   // see above: not MI_BIN_COUNT
         if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) == 0) continue;
-        mi_arena_holes_report_arg_t ra = { arena_pages->pages_abandoned[bin], rep };
+        mi_arena_holes_report_arg_t ra = { mi_arena_pages_abandoned(arena_pages, bin), rep };
+        if (ra.bitmap == NULL) continue;
         (void)_mi_bitmap_forall_set(ra.bitmap, &mi_arena_page_holes_report_at, arena, &ra);
       }
     }
@@ -1651,8 +1667,7 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) {
   if (slice_count == 0) slice_count = MI_BCHUNK_BITS;
   mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);
   const size_t base_size = _mi_align_up(sizeof(mi_arena_pages_t), MI_BCHUNK_SIZE);
-  const size_t bitmaps_count = 1 + MI_ARENA_BIN_COUNT; // pages, and abandoned
-  const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL);
+  const size_t bitmaps_size = mi_bitmap_size(slice_count, NULL); // pages (the abandoned bitmaps are allocated on demand)
   const size_t size = base_size + bitmaps_size;
   if (bitmap_base != NULL) *bitmap_base = base_size;
   return size;
@@ -1662,7 +1677,7 @@ static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_bas
   if (slice_count == 0) slice_count = MI_BCHUNK_BITS;
   mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0);
   const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE);
-  const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned
+  const size_t bitmaps_count = 4; // commit, dirty, purge, and pages (the abandoned bitmaps are allocated on demand)
   const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free
   #if MI_PAGE_META_IS_SEPARATED
   const size_t pages_size = slice_count * sizeof(mi_page_t);
@@ -1701,12 +1716,63 @@ static mi_arena_pages_t* mi_arena_pages_alloc(mi_arena_t* arena) {
   uint8_t* base = (uint8_t*)arena_pages + bitmap_base;
   mi_assert_internal(_mi_is_aligned(base, MI_BCHUNK_SIZE));
   arena_pages->pages = mi_arena_bitmap_init(slice_count, &base);
-  for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
-    arena_pages->pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);
-  }
+  // `pages_abandoned[]` stays NULL (the allocation is zeroed) until a page of that bin is abandoned.
   return arena_pages;
 }
 
+// The abandoned-pages bitmap of `bin`, or NULL if no page of that bin was ever abandoned in
+// this (heap, arena) pair. A NULL bitmap reads as all-clear.
+static mi_bitmap_t* mi_arena_pages_abandoned(mi_arena_pages_t* arena_pages, size_t bin) {
+  mi_assert_internal(bin < MI_ARENA_BIN_COUNT);
+  return mi_atomic_load_ptr_acquire(mi_bitmap_t, &arena_pages->pages_abandoned[bin]);
+}
+
+// The abandoned-pages bitmap of `bin`, allocated on first use. Allocated from the subproc
+// meta-data heap so this is safe on the abandon paths (a thread tearing down its theaps, a
+// foreign free re-abandoning a page), where allocating from a regular heap is not. Publishing
+// is a CAS so no lock is needed: a loser frees its copy and uses the winner's. Returns NULL only
+// if the allocation failed.
+#if MI_DEBUG > 0
+mi_decl_export _Atomic(uintptr_t) mi_debug_abandoned_maps_allocated;   // test hook (test-abandoned-lazy): per-bin abandoned maps published so far
+#endif
+
+static mi_bitmap_t* mi_arena_pages_abandoned_ensure(mi_arena_t* arena, mi_arena_pages_t* arena_pages, size_t bin) {
+  mi_bitmap_t* bitmap = mi_arena_pages_abandoned(arena_pages, bin);
+  if mi_likely(bitmap != NULL) return bitmap;
+  const size_t slice_count = arena->slice_count;
+  const size_t size = mi_bitmap_size(slice_count, NULL);
+  mi_bitmap_t* fresh = (mi_bitmap_t*)_mi_meta_zalloc_aligned(arena->subproc, size, MI_BCHUNK_SIZE, NULL);
+  if (fresh == NULL) return NULL;
+  mi_bitmap_init(fresh, slice_count, true /* already zero */);
+  mi_bitmap_t* expected = NULL;
+  if (mi_atomic_cas_ptr_strong_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], &expected, fresh)) {
+    #if MI_DEBUG > 0
+    mi_atomic_increment_relaxed(&mi_debug_abandoned_maps_allocated);
+    #endif
+    return fresh;
+  }
+  // another thread published one first
+  _mi_free_subproc_safe(fresh);
+  mi_assert_internal(expected != NULL);
+  return expected;
+}
+
+// Release the on-demand abandoned bitmaps of a heap's arena pages (the `pages` bitmap is part of
+// the `arena_pages` allocation itself).
+static void mi_arena_pages_free_abandoned(mi_arena_pages_t* arena_pages) {
+  for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) {
+    if (mi_atomic_load_ptr_relaxed(mi_bitmap_t, &arena_pages->pages_abandoned[bin]) == NULL) continue;  // the common case
+    mi_bitmap_t* bitmap = mi_atomic_exchange_ptr_acq_rel(mi_bitmap_t, &arena_pages->pages_abandoned[bin], NULL);
+    if (bitmap != NULL) { _mi_free_subproc_safe(bitmap); }
+  }
+}
+
+void _mi_arena_pages_free(mi_arena_pages_t* arena_pages) {
+  if (arena_pages == NULL) return;
+  mi_arena_pages_free_abandoned(arena_pages);
+  _mi_free_subproc_safe(arena_pages);
+}
+
 static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
                                         size_t slice_count, mi_arena_t* parent, size_t total_size,
                                         int numa_node, bool exclusive,
@@ -1799,7 +1865,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
   arena->slices_purge = mi_arena_bitmap_init(slice_count, &base);
   arena->pages_main.pages = mi_arena_bitmap_init(slice_count, &base);
   for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) {
-    arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base);
+    mi_atomic_store_ptr_relaxed(mi_bitmap_t, &arena->pages_main.pages_abandoned[i], NULL);  // allocated on first abandon
   }
   #if MI_PAGE_META_IS_SEPARATED
   arena->pages_meta = (mi_page_t*)base;
@@ -2690,7 +2756,9 @@ bool _mi_heap_visit_blocks(mi_heap_t* heap, bool abandoned_only, bool visit_bloc
         for (size_t bin = 0; ok && bin < MI_ARENA_BIN_COUNT; bin++) {
           // todo: if we had a single abandoned page map as well, this can be faster.
           if (mi_atomic_load_relaxed(&heap->abandoned_count[bin]) > 0) {
-            ok = _mi_bitmap_forall_set(arena_pages->pages_abandoned[bin], &mi_heap_visit_page_at, arena, &visit_info);
+            mi_bitmap_t* const bitmap = mi_arena_pages_abandoned(arena_pages, bin);
+            if (bitmap == NULL) continue;
+            ok = _mi_bitmap_forall_set(bitmap, &mi_heap_visit_page_at, arena, &visit_info);
           }
         }
       }

src/heap.c

@@ -181,6 +181,10 @@ mi_heap_t* mi_heap_new(void) {
 static void mi_heap_release_pages(mi_heap_t* heap, mi_heap_t* heap_target) {
   _mi_heap_detach_theaps(heap);
   if (_mi_is_heap_main(heap)) return;  // (`_mi_heap_force_destroy` of a main heap at sub-process teardown: the arenas go as a whole)
+  // Step 3 claims every page through the `arena_pages->pages` bitmap, so the pages abandoned in step 2
+  // do not need to be findable by size class: `_mi_arenas_page_abandon` leaves them out of the
+  // per-bin abandoned maps, which are then never allocated for a heap that only lives to be released.
+  mi_atomic_store_release(&heap->releasing, (uintptr_t)1);
   mi_lock(&heap->theaps_lock) {
     for (mi_theap_t* theap = heap->theaps; theap != NULL; theap = theap->hnext) {
       mi_assert_internal(_mi_theap_heap_peek(theap)==NULL);
@@ -229,7 +233,7 @@ static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) {
         mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]);
         if (arena_pages!=NULL) {
           mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL);
-          _mi_free_subproc_safe(arena_pages);
+          _mi_arena_pages_free(arena_pages);
         }
       }
     }