diff --git a/include/mimalloc.h b/include/mimalloc.h index 932a458a0..413037879 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -367,6 +367,11 @@ typedef struct mi_heap_area_s { typedef bool (mi_cdecl mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg); mi_decl_export bool mi_heap_visit_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); +mi_decl_export void mi_os_hint_floor(void* floor) mi_attr_noexcept; +mi_decl_export bool mi_startup_snapshot_hints_enabled(void) mi_attr_noexcept; // deterministic placement is on for this process (snapshot-capable executable, or MIMALLOC_DETERMINISTIC_HINT=1) +mi_decl_export void mi_arenas_seal_existing(void) mi_attr_noexcept; // snapshot restore: no new allocations inside pre-existing (snapshot) arenas +mi_decl_export void mi_arenas_freeze_pages(void) mi_attr_noexcept; // snapshot build: pages written into the snapshot are never freed into again +mi_decl_export void mi_arenas_visit_free_ranges(mi_heap_t* heap, void (*visit)(void* start, size_t size, void* arg), void* arg) mi_attr_noexcept; // visit maximal runs of arena slices that belong to no page mi_decl_export bool mi_heap_visit_abandoned_blocks(mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg); @@ -443,6 +448,8 @@ mi_decl_export bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visi struct mi_theap_s; typedef struct mi_theap_s mi_theap_t; +mi_decl_export void mi_theap_freeze(mi_theap_t* theap) mi_attr_noexcept; // snapshot restore: never collect/purge/idle-sweep this theap again +mi_decl_export void mi_theap_adopt_current_thread(mi_theap_t* theap) mi_attr_noexcept; // snapshot restore: this theap's thread state now belongs to the calling thread mi_decl_export mi_theap_t* mi_heap_theap(mi_heap_t* heap); mi_decl_export mi_theap_t* mi_theap_set_default(mi_theap_t* theap); diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 6a19d5658..05b6dc26e 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -917,7 +917,7 @@ static inline bool mi_page_all_free(const mi_page_t* page) { // Page hole purging (see the "Page hole purging" section in `page.c`) // ------------------------------------------------------ -void _mi_page_purge_holes(mi_page_t* page); +void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld); // `tld`: the one being swept (owns the sweep state) void _mi_page_purged_reset(mi_page_t* page); bool _mi_page_unpurge_run(mi_page_t* page); void _mi_page_unpurge_all(mi_page_t* page); @@ -926,12 +926,11 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end); size_t _mi_page_unformed_purged_bytes(const mi_page_t* page); // the bytes of this page's unformed tail that are discarded right now bool _mi_page_purge_os_page_blocks(size_t os_page_size, size_t block_size, uintptr_t page_start, size_t capacity, size_t k, size_t* first, size_t* last); -bool _mi_page_purge_holes_in_progress(void); void _mi_page_holes_count_page_freed(void); void _mi_page_holes_count_ineligible(const mi_page_t* page); void _mi_page_holes_reset_ineligible(void); -void _mi_page_purge_holes_begin(void); -void _mi_page_purge_holes_end(void); +void _mi_page_purge_holes_begin(mi_tld_t* tld); +void _mi_page_purge_holes_end(mi_tld_t* tld); void _mi_page_purge_holes_sweep_begin(mi_tld_t* tld); // once per idle sweep, before its passes // ------------------------------------------------------ diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index a1bd1eec2..3244f44bb 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -363,6 +363,9 @@ typedef size_t mi_page_flags_t; #define MI_THREADID_ABANDONED MI_ZU(0) #define MI_THREADID_ABANDONED_MAPPED (MI_ZU(1) << MI_PAGE_FLAG_BITS) #define MI_THREADID_DETACHED (MI_ZU(2) << MI_PAGE_FLAG_BITS) +// Snapshot: pages captured in a snapshot are owned by nobody and never change again; frees of their blocks are dropped +// (see mi_free_block_mt) and no thread id can ever equal this value. +#define MI_THREADID_FROZEN (~MI_ZU(0) & ~MI_PAGE_FLAG_MASK) // Thread free list. // Points to a list of blocks that are freed by other threads. @@ -580,6 +583,7 @@ struct mi_theap_s { long page_full_retain; // how many full pages can be retained per queue (before abandoning them) bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages + bool frozen; // pages are an immutable snapshot: never allocate from, collect, or purge them bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` bool prof_force_slow; // if profiling is enabled: keep `pages_free_direct` poisoned so every malloc routes through `_mi_malloc_generic` @@ -720,6 +724,14 @@ struct mi_tld_s { _Atomic(uint32_t) park_swept; // this park's sweep is done: don't claim it again until the thread re-parks mi_tld_t* subproc_next; // list of tlds in the subproc, so the scavenger can find parked threads size_t holes_sweep_seq; // idle sweeps run over THIS tld's heaps (paces `purge_holes_full_every`) + // State of the hole sweep currently running over this tld's pages (owner or scavenger; one sweeper at a time). + // Kept here, reached through the page or the tld in hand, rather than in `__thread` variables: on macOS a first + // access to a thread-local from inside the allocator makes dyld allocate the thread's TLV block with malloc, which + // re-enters the allocator before the variable exists (unbounded recursion on any new thread's first allocation). + bool holes_sweeping; // `_mi_page_purge_holes_begin/end`: a sweep is rewriting this tld's pages + bool holes_sweep_full; // this sweep ignores `page->swept_state` + size_t holes_sweep_skipped; // per-sweep counters, folded into the process-wide ones at the end + size_t holes_sweep_visited; mi_msecs_t holes_sweep_last; // when this tld's heaps were last swept (paces `purge_holes_min_interval`) }; diff --git a/src/arena.c b/src/arena.c index cae63819c..e436806d0 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1371,7 +1371,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, _mi_page_holes_count_page_freed(); return true; } - _mi_page_purge_holes(page); + _mi_page_purge_holes(page, parg->tld); mi_bitmap_set(bitmap, slice_index); // back in the map *before* unowning: unown may free the page mi_abandoned_page_unown(page, NULL); return true; @@ -1383,7 +1383,7 @@ static bool mi_arena_page_purge_holes_at(size_t slice_index, size_t slice_count, void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { if (heap == NULL) return; if (!mi_option_is_enabled(mi_option_purge_holes)) return; - _mi_page_purge_holes_begin(); + _mi_page_purge_holes_begin(tld); mi_forall_arenas(heap, ((mi_arena_t*)NULL), 0, arena) { mi_arena_pages_t* const arena_pages = mi_heap_arena_pages(heap, arena); if (arena_pages != NULL) { @@ -1398,7 +1398,7 @@ void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld) { } } mi_forall_arenas_end(); - _mi_page_purge_holes_end(); + _mi_page_purge_holes_end(tld); } // The read-only counterpart of the sweep above: account for the holes in the abandoned pages @@ -1469,6 +1469,72 @@ void _mi_arenas_holes_committed(mi_heap_t* heap, mi_holes_report_t* rep) { } +// snapshot restore: every arena that exists right now holds snapshot memory. Make them exclusive (to nobody) so no +// theap on any thread places new blocks in their free space; new memory comes from arenas created after this call. +void mi_arenas_seal_existing(void) mi_attr_noexcept { + mi_subproc_t* subproc = _mi_subproc_main(); + const size_t n = mi_arenas_get_count(subproc); + for (size_t i = 0; i < n; i++) { + mi_arena_t* arena = mi_arena_from_index(subproc, i); + if (arena != NULL) { arena->is_exclusive = true; } + } +} + +static bool mi_arena_abandoned_bit_clear(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { + MI_UNUSED(slice_count); MI_UNUSED(arena); + mi_bitmap_clear((mi_bitmap_t*)arg, slice_index); + return true; +} + +static bool mi_arena_page_freeze(size_t slice_index, size_t slice_count, mi_arena_t* arena, void* arg) { + MI_UNUSED(slice_count); MI_UNUSED(arg); + mi_page_t* page = mi_arena_page_at_slice(arena, slice_index); + mi_atomic_store_release(&page->xthread_id, (mi_threadid_t)MI_THREADID_FROZEN); + return true; +} + +// Snapshot build: every page that exists right now is about to be written into the snapshot. Owned by MI_THREADID_FROZEN, +// frees of their blocks in the restored process (or in what is left of this one) take the cross-thread path and are dropped +// there, so snapshot pages are never dirtied by the allocator. Done here, in the builder, so the restore writes nothing. +void mi_arenas_freeze_pages(void) mi_attr_noexcept { + mi_subproc_t* subproc = _mi_subproc_main(); + const size_t n = mi_arenas_get_count(subproc); + mi_lock(&subproc->heaps_lock) { + for (mi_heap_t* heap = subproc->heaps; heap != NULL; heap = heap->next) { // page bitmaps are kept per heap + for (size_t i = 0; i < n; i++) { + mi_arena_t* arena = mi_arena_from_index(subproc, i); + if (arena == NULL) continue; + mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); + if (arena_pages == NULL) continue; + (void)_mi_bitmap_forall_set(arena_pages->pages, &mi_arena_page_freeze, arena, NULL); + // A frozen page must not be findable as abandoned either: reclaiming it (or sweeping its holes) would allocate into + // and rewrite snapshot memory. Everything abandoned at this point is snapshot memory, so the heap's counts go to zero. + for (size_t bin = 0; bin < MI_ARENA_BIN_COUNT; bin++) { + mi_bitmap_t* const abandoned = arena_pages->pages_abandoned[bin]; + (void)_mi_bitmap_forall_set(abandoned, &mi_arena_abandoned_bit_clear, arena, abandoned); + } + } + for (size_t bin = 0; bin < MI_BIN_COUNT; bin++) { mi_atomic_store_relaxed(&heap->abandoned_count[bin], 0); } + } + } +} + +// Visit every maximal run of free slices (belonging to no page) across all arenas of `heap`'s subproc. +void mi_arenas_visit_free_ranges(mi_heap_t* heap, void (*visit)(void* start, size_t size, void* arg), void* arg) mi_attr_noexcept { + if (heap == NULL || visit == NULL) return; + mi_forall_arenas(heap, ((mi_arena_t*)NULL), 0, arena) { + const size_t slice_count = arena->slice_count; + size_t i = 0; + while (i < slice_count) { + if (!mi_bbitmap_is_setN(arena->slices_free, i, 1)) { i++; continue; } + size_t j = i; while (j < slice_count && mi_bbitmap_is_setN(arena->slices_free, j, 1)) j++; + visit(mi_arena_slice_start(arena, i), mi_size_of_slices(j - i), arg); + i = j; + } + } + mi_forall_arenas_end(); +} + /* ----------------------------------------------------------- Arena free ----------------------------------------------------------- */ diff --git a/src/free.c b/src/free.c index 61e064ed5..7ba01c2be 100644 --- a/src/free.c +++ b/src/free.c @@ -63,6 +63,9 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // Free a block multi-threaded static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_collect) mi_attr_noexcept { + // A page frozen into a snapshot (thread id MI_THREADID_FROZEN, so every free of its blocks lands here) is never written + // again: dropping the block keeps the page clean and file-backed. Costs one compare on a path that is already the slow one. + if mi_unlikely(mi_page_thread_id(page) == MI_THREADID_FROZEN) return; // todo: we cannot safely check for double free in _mt -- should check when collecting the thread_free list if (!was_guarded) { mi_check_padding(page, block); } // checking padding is safe for mt // adjust stats (after padding check ) diff --git a/src/init.c b/src/init.c index 004d3cd7d..272c668b5 100644 --- a/src/init.c +++ b/src/init.c @@ -111,7 +111,9 @@ static mi_decl_cache_align mi_tld_t mi_tld_detached = { NULL, // park_theap0 MI_ATOMIC_VAR_INIT(0), // park_swept NULL, // subproc_next - 0, 0 // holes_sweep_seq / _last + 0, // holes_sweep_seq + false, false, 0, 0, // holes_sweeping / holes_sweep_full / _skipped / _visited + 0 // holes_sweep_last }; mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { @@ -130,6 +132,7 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { NULL, NULL, // hnext, hprev 0, // full page retain false, // allow reclaim + false, // frozen true, // allow abandon true, // is_detached false, 0, // prof_force_slow, prof_countdown (fork) diff --git a/src/os.c b/src/os.c index 4ce683c2f..015e40688 100644 --- a/src/os.c +++ b/src/os.c @@ -4,10 +4,12 @@ This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ +#include // ENOENT #include "mimalloc.h" #include "mimalloc/internal.h" #include "mimalloc/atomic.h" #include "mimalloc/prim.h" +#include #include "mimalloc/prim-tls.h" // _mi_theap_default for random /* ----------------------------------------------------------- @@ -128,12 +130,71 @@ bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero #define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes) #define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) +// snapshot support (see mi_arenas_freeze_pages / mi_os_hint_floor). The embedder can name a function, callable from the +// very first allocation, that says whether this executable can build or map a snapshot (`-DMI_STARTUP_SNAPSHOT_HOST_FN=name`, +// `int name(void)`); such executables get deterministic address hints without any environment. It can also name the +// environment variable that marks a snapshot *build* (`-DMI_STARTUP_SNAPSHOT_BUILD_ENV="VAR"`): a build keeps its heap at the +// default base (that heap becomes the snapshot); any other snapshot-capable process keeps its own early heap above snapshot space. +#if defined(MI_STARTUP_SNAPSHOT_HOST_FN) +#ifdef __cplusplus +extern "C" int MI_STARTUP_SNAPSHOT_HOST_FN(void); +#else +extern int MI_STARTUP_SNAPSHOT_HOST_FN(void); +#endif +#define mi_startup_snapshot_capable() (MI_STARTUP_SNAPSHOT_HOST_FN() != 0) +#else +#define mi_startup_snapshot_capable() (0) +#endif +#if !defined(MI_STARTUP_SNAPSHOT_BUILD_ENV) +#define MI_STARTUP_SNAPSHOT_BUILD_ENV "MIMALLOC_SNAPSHOT_BUILD" +#endif +#define MI_STARTUP_SNAPSHOT_RESTORER_FLOOR ((uintptr_t)0x21000000000ULL) // 64GiB above the default hint base +static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0 (hint bump pointer; file scope so mi_os_hint_floor can move it) +static _Atomic(uintptr_t) hint_floor; // = 0: lowest address hinted allocations may use (also where the pointer restarts when it wraps or is invalid) + +// Snapshot restore: every hinted OS allocation from now on lands at or above `floor` (the snapshot occupies the area below). Also repairs a +// pointer that is unusable (0, past MI_HINT_MAX — e.g. inherited from another process's data segment). +void mi_os_hint_floor(void* floor) mi_attr_noexcept { + uintptr_t f = _mi_align_up((uintptr_t)floor, MI_HINT_ALIGN); + uintptr_t curf = mi_atomic_load_acquire(&hint_floor); + while (curf < f && !mi_atomic_cas_weak_acq_rel(&hint_floor, &curf, f)) { } + uintptr_t cur = mi_atomic_load_acquire(&aligned_base); + while ((cur < f || cur > MI_HINT_MAX) && !mi_atomic_cas_weak_acq_rel(&aligned_base, &cur, f)) { } +} + +// Deterministic hinting: on when the executable is snapshot-capable (MI_STARTUP_SNAPSHOT_HOST_FN) or requested through the +// environment. Decided once; the embedder's own reservations ask the same question (`mi_startup_snapshot_hints_enabled`). +static bool mi_startup_snapshot_hints_decide(void) { + static int decided = -1; + if (decided < 0) { + // _mi_getenv (0 = found), not getenv: this runs inside the very first allocation, and the C runtime's getenv allocates on some platforms. + char buf[64]; + const int from_env = (_mi_getenv("MIMALLOC_DETERMINISTIC_HINT", buf, sizeof(buf)) == 0 && buf[0] == '1') ? 1 : 0; + // MIMALLOC_HINT_FLOOR=0x...: start hinted OS allocations at/above this address (a snapshot will be mapped below it) + if (_mi_getenv("MIMALLOC_HINT_FLOOR", buf, sizeof(buf)) == 0 && buf[0] != 0) { + unsigned long long v = strtoull(buf, NULL, 0); + if (v) mi_os_hint_floor((void*)(uintptr_t)v); + } + // Snapshot-capable and not building: this process may be about to map a snapshot over the default hint area, so nothing it + // allocates before then (libc scratch, dyld) may live there. + // Presence is what matters here; the value (an output path) may well not fit `buf`, which _mi_getenv reports as EAGAIN, not ENOENT. + else if (mi_startup_snapshot_capable() && _mi_getenv(MI_STARTUP_SNAPSHOT_BUILD_ENV, buf, sizeof(buf)) == ENOENT) mi_os_hint_floor((void*)MI_STARTUP_SNAPSHOT_RESTORER_FLOOR); + decided = (from_env || mi_startup_snapshot_capable()) ? 1 : 0; + } + return decided == 1; +} + +bool mi_startup_snapshot_hints_enabled(void) mi_attr_noexcept { + return mi_startup_snapshot_hints_decide(); +} + void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { - static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0 // todo: perhaps only do alignment hints if THP is enabled? - if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL; + const int deterministic_all = mi_startup_snapshot_hints_enabled(); + if (!deterministic_all && (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN)) return NULL; + if (deterministic_all && try_alignment > MI_HINT_ALIGN) return NULL; if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space size = _mi_align_up(size, MI_HINT_ALIGN); #if (MI_SECURE>=1) @@ -142,13 +203,19 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) size += MI_HINT_ALIGN; // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); - if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize + if (hint == 0 || hint > MI_HINT_MAX || hint < mi_atomic_load_relaxed(&hint_floor)) { // wrap or initialize (never below the floor) uintptr_t init = MI_HINT_BASE; + { uintptr_t fl = mi_atomic_load_relaxed(&hint_floor); if (fl > init) init = fl; } + // Experiment: MIMALLOC_DETERMINISTIC_HINT=1 disables hint randomization (snapshot determinism tests). + const int deterministic = deterministic_all; + MI_UNUSED(deterministic); // only consulted in the randomizing configurations below #if (MI_SECURE>=1 || defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode + if (!deterministic) { mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point const uintptr_t r = _mi_theap_random_next(theap); init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + } #endif uintptr_t expected = hint + size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); @@ -163,6 +230,8 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { MI_UNUSED(try_alignment); MI_UNUSED(size); return NULL; } +bool mi_startup_snapshot_hints_enabled(void) mi_attr_noexcept { return false; } +void mi_os_hint_floor(void* floor) mi_attr_noexcept { MI_UNUSED(floor); } #endif diff --git a/src/page.c b/src/page.c index 321939ba9..4c828347e 100644 --- a/src/page.c +++ b/src/page.c @@ -472,35 +472,21 @@ static void mi_holes_count_reuse(size_t bytes, size_t blocks, bool reused) { mi_atomic_addi64_relaxed(&mi_holes_blocks, -(int64_t)blocks); } -// Re-entrancy guard: while the idle sweep is rewriting a page's free list and -// bitmap, a nested `mi_malloc` (only reachable through a user output function -// from a warning message) must not un-purge a hole from under it. -static mi_decl_thread bool mi_purging_holes; +// The sweep state lives on the tld being swept (see `mi_tld_s`); a nested `mi_malloc` on the sweeping thread (only +// reachable through a user output function from a warning message) must not un-purge a hole of a page whose free list +// the sweep is rewriting, which `mi_page_free_collect_ex` checks through the page's own tld. +void _mi_page_purge_holes_begin(mi_tld_t* tld) { mi_assert_internal(tld != NULL && !tld->holes_sweeping); tld->holes_sweeping = true; } -// Per-sweep counters. The sweep runs over every page of the thread, so a process-wide atomic -// per page would be a real cost on the very path we are making cheap: accumulate thread-locally -// and fold them in once per pass, in `_mi_page_purge_holes_end`. -static mi_decl_thread size_t mi_holes_sweep_skipped; -static mi_decl_thread size_t mi_holes_sweep_visited; - -// Whether THIS sweep ignores `page->swept_state` (see `_mi_page_purge_holes`). Set once per idle -// sweep, in `_mi_page_purge_holes_sweep_begin`; the sequence it is paced by lives on the tld being -// swept, since one scavenger thread runs the sweeps of many. -static mi_decl_thread bool mi_holes_sweep_full; - -bool _mi_page_purge_holes_in_progress(void) { return mi_purging_holes; } -void _mi_page_purge_holes_begin(void) { mi_assert_internal(!mi_purging_holes); mi_purging_holes = true; } - -void _mi_page_purge_holes_end(void) { - mi_assert_internal(mi_purging_holes); - mi_purging_holes = false; - if (mi_holes_sweep_skipped > 0) { - mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)mi_holes_sweep_skipped); - mi_holes_sweep_skipped = 0; +void _mi_page_purge_holes_end(mi_tld_t* tld) { + mi_assert_internal(tld->holes_sweeping); + tld->holes_sweeping = false; + if (tld->holes_sweep_skipped > 0) { + mi_atomic_addi64_relaxed(&mi_holes_pages_skipped, (int64_t)tld->holes_sweep_skipped); + tld->holes_sweep_skipped = 0; } - if (mi_holes_sweep_visited > 0) { - mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)mi_holes_sweep_visited); - mi_holes_sweep_visited = 0; + if (tld->holes_sweep_visited > 0) { + mi_atomic_addi64_relaxed(&mi_holes_blocks_visited, (int64_t)tld->holes_sweep_visited); + tld->holes_sweep_visited = 0; } } @@ -508,8 +494,8 @@ void _mi_page_purge_holes_end(void) { void _mi_page_purge_holes_sweep_begin(mi_tld_t* tld) { const long every = mi_option_get(mi_option_purge_holes_full_every); const size_t seq = ++tld->holes_sweep_seq; - mi_holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0); - if (mi_holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); } + tld->holes_sweep_full = (every > 0 && (seq % (size_t)every) == 0); + if (tld->holes_sweep_full) { mi_atomic_addi64_relaxed(&mi_holes_full_sweeps, 1); } } static inline bool mi_page_bits_at(const uint64_t* bits, size_t k) { @@ -678,7 +664,7 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end) { // Walk the free list of a page and discard every OS page in it that holds no live block. // Returns false if any discard failed: those blocks went straight back on the free list and the // page must be swept again, so the caller must not record it as swept. -static bool mi_page_purge_holes_walk(mi_page_t* page) { +static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { if (page->free == NULL) return true; // nothing to take off the free list const size_t os_size = _mi_os_page_size(); @@ -703,7 +689,7 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) { nfree[k]++; } } - mi_holes_sweep_visited += nvisited; // folded into the process-wide counter at the end of the sweep + tld->holes_sweep_visited += nvisited; // folded into the process-wide counter at the end of the sweep // 2. an OS page can be discarded when *every* block overlapping it is free -- either on the // free list, or purged already. Of the blocks overlapping an OS page, only the first and @@ -789,7 +775,7 @@ static bool mi_page_purge_holes_walk(mi_page_t* page) { // regardless, which caps the delay of a missed discard at N parks for 1/N of the old cost. // (An exact "was anything freed in this page" bit is the alternative, and it costs a store in // `mi_free` itself -- the hot path this whole feature stays off.) -void _mi_page_purge_holes(mi_page_t* page) { +void _mi_page_purge_holes(mi_page_t* page, mi_tld_t* tld) { mi_assert_internal(page != NULL); if (!mi_option_is_enabled(mi_option_purge_holes)) return; if (mi_page_all_free(page)) return; // the page itself is about to be freed @@ -797,8 +783,8 @@ void _mi_page_purge_holes(mi_page_t* page) { mi_page_purge_unformed_tail(page); // the blocks that are not formed yet: resident, but never handed out if (!mi_page_can_purge_holes(page)) { _mi_page_holes_count_ineligible(page); return; } - if (!mi_holes_sweep_full && page->swept_state == mi_page_sweep_state(page)) { - mi_holes_sweep_skipped++; // nothing was allocated or freed in this page since we swept it + if (!tld->holes_sweep_full && page->swept_state == mi_page_sweep_state(page)) { + tld->holes_sweep_skipped++; // nothing was allocated or freed in this page since we swept it return; } // Record the state we LEAVE the page in, read back from the page: a nested `mi_malloc` (see @@ -808,7 +794,7 @@ void _mi_page_purge_holes(mi_page_t* page) { // blocks straight back, and changes neither `capacity` nor `used` -- so recording here would // say "already swept" for a page that still has holes, and the skip check would then park them // until the next full sweep, or forever with `purge_holes_full_every=0`. - if (mi_page_purge_holes_walk(page)) { + if (mi_page_purge_holes_walk(page, tld)) { page->swept_state = mi_page_sweep_state(page); } } @@ -1177,6 +1163,14 @@ void _mi_page_holes_report_print(const mi_holes_report_t* rep) { } +// Is a hole sweep rewriting this page right now? Only an owned page can be (the sweep of abandoned pages holds them +// exclusively); an abandoned page may still point at the theap of a thread that has since exited, so it is not looked at. +static inline bool mi_page_holes_sweep_in_progress(const mi_page_t* page) { + if (mi_page_is_abandoned(page) || page->theap == NULL) return false; + const mi_tld_t* const tld = page->theap->tld; + return (tld != NULL && tld->holes_sweeping); +} + static void mi_page_free_collect_ex(mi_page_t* page, bool force, bool allow_unpurge) { mi_assert_internal(page!=NULL); @@ -1210,7 +1204,7 @@ static void mi_page_free_collect_ex(mi_page_t* page, bool force, bool allow_unpu // Free list empty but this page has discarded holes: bring a whole run of them // back. Every caller re-checks `mi_page_immediate_available` after collect, so the // page becomes usable again without touching the other holes. - if (allow_unpurge && page->free == NULL && mi_page_has_purged(page) && !_mi_page_purge_holes_in_progress()) { + if (allow_unpurge && page->free == NULL && mi_page_has_purged(page) && !mi_page_holes_sweep_in_progress(page)) { _mi_page_unpurge_run(page); } } diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index 201c15860..33591ac3f 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -725,4 +725,17 @@ static void _mi_macos_override_malloc(void) { } #endif // MI_OSX_INTERPOSE +// snapshot restore: libsystem_malloc rewrites a registered zone's function table in place (typed-malloc shims whose +// bookkeeping lives in libsystem's own per-process data). The zone as the snapshot builder's libsystem left it therefore +// must not be copied over the one this process registered: the restore keeps these ranges as they are. +#ifdef __cplusplus +extern "C" +#endif +mi_decl_export size_t mi_malloc_zone_process_owned_ranges(uintptr_t (*out)[2], size_t cap) { + size_t n = 0; + if (n < cap) { out[n][0] = (uintptr_t)&mi_malloc_zone; out[n][1] = (uintptr_t)(&mi_malloc_zone + 1); n++; } + if (n < cap) { out[n][0] = (uintptr_t)&mi_introspect; out[n][1] = (uintptr_t)(&mi_introspect + 1); n++; } + return n; +} + #endif // MI_MALLOC_OVERRIDE diff --git a/src/prof.c b/src/prof.c index 9f76c539a..cbb8f7092 100644 --- a/src/prof.c +++ b/src/prof.c @@ -307,6 +307,39 @@ void _mi_prof_init(void) { mi_prof_set_all_theaps(true); } +// snapshot restore (or fork child): whatever thread held the profiler lock does not exist here. +#ifdef __cplusplus +extern "C" +#endif +mi_decl_export void mi_prof_reinit_lock(void) mi_attr_noexcept { + if (mi_prof.ht_cap == 0) return; + mi_lock_init(&mi_prof.lock); +} +// Diagnostic: is the profiler lock free right now? (snapshot code asserts this before freezing) +#ifdef __cplusplus +extern "C" +#endif +mi_decl_export bool mi_prof_lock_is_free(void) mi_attr_noexcept { + if (mi_prof.ht_cap == 0) return true; + if (!mi_lock_try_acquire(&mi_prof.lock)) return false; + mi_lock_release(&mi_prof.lock); + return true; +} + +// Visit live sampled allocations (addr != 0). Callback returns false to stop. +#ifdef __cplusplus +extern "C" +#endif +mi_decl_export void mi_prof_visit_live(bool (*cb)(uintptr_t addr, size_t size, const uintptr_t* frames, uint8_t nframes, void* arg), void* arg) { + mi_lock_acquire(&mi_prof.lock); + for (size_t i = 0; i < mi_prof.sample_count; i++) { + mi_prof_sample_t* smp = &mi_prof.samples[i]; + if (smp->addr == 0) continue; + if (!cb(smp->addr, smp->size, smp->frames, smp->nframes, arg)) break; + } + mi_lock_release(&mi_prof.lock); +} + void mi_prof_reset(void) mi_attr_noexcept { if (mi_prof.ht_cap == 0) return; // never initialized mi_lock(&mi_prof.lock) { @@ -573,7 +606,7 @@ static int mi_prof_dump_pb(mi_pb_t* wp) { // To keep this simple, do two passes: pass 1 collect unique frame addrs, // pass 2 emit samples referencing location ids. // Unique-address collection: - size_t loc_cap = 4096; + size_t loc_cap = 4096; while (loc_cap < mi_prof.sample_count * 8) loc_cap <<= 1; mi_memid_t lm; mi_prof_loc_t* locs = (mi_prof_loc_t*)_mi_os_zalloc(_mi_subproc_main(), loc_cap * sizeof(mi_prof_loc_t), &lm); size_t nlocs = 0; @@ -645,6 +678,7 @@ static int mi_prof_dump_pb(mi_pb_t* wp) { pb_flush(&w); *wp = w; + _mi_os_free(_mi_subproc_main(), locs, loc_cap * sizeof(mi_prof_loc_t), lm); return (w.err ? -1 : 0); } diff --git a/src/theap.c b/src/theap.c index 142c3466c..86ad4f4f3 100644 --- a/src/theap.c +++ b/src/theap.c @@ -99,6 +99,7 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag MI_UNUSED(theap); mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL)); mi_collect_t collect = *((mi_collect_t*)arg_collect); + if (mi_page_thread_id(page) == MI_THREADID_FROZEN) return true; // snapshot page: never freed, abandoned or rewritten // A collect has no allocation to serve, so it must not un-purge: the allocation path // (`mi_page_queue_find_free_ex`) hands a hole back when a page is actually needed. Otherwise // `mi_on_thread_idle`, which collects before it sweeps, un-purges a run of every page whose free @@ -171,6 +172,7 @@ static bool mi_theap_page_purge_holes(mi_theap_t* theap, mi_page_queue_t* pq, mi // has to wait for us. Stopping between pages bounds that wait to one page's walk; the pages we // skip are simply swept at the next park (`swept_state` makes the re-walk cheap). if (theap->tld != NULL && mi_atomic_load_relaxed(&theap->tld->park_reclaim) != 0) return false; + if (mi_page_thread_id(page) == MI_THREADID_FROZEN) return true; // snapshot page: its bytes are the snapshot file's _mi_page_free_collect(page, true); // force: fold local_free (and thread_free) into `free` first if (mi_page_all_free(page)) { // the forced collect emptied the page: hand it back instead of leaving it resident @@ -178,7 +180,7 @@ static bool mi_theap_page_purge_holes(mi_theap_t* theap, mi_page_queue_t* pq, mi _mi_page_free(page, pq); return true; } - _mi_page_purge_holes(page); + _mi_page_purge_holes(page, mi_page_tld(page)); mi_assert_expensive(_mi_page_is_valid(page)); return true; // continue } @@ -193,9 +195,9 @@ static void mi_theap_purge_holes(mi_theap_t* theap) mi_attr_noexcept { if (theap->tld == NULL) return; if (theap->tld->thread_id != _mi_thread_id() && mi_atomic_load_acquire(&theap->tld->park_state) != MI_PARK_SWEEPING) return; - _mi_page_purge_holes_begin(); + _mi_page_purge_holes_begin(theap->tld); mi_theap_visit_pages(theap, &mi_theap_page_purge_holes, true /* include full pages */, NULL, NULL); - _mi_page_purge_holes_end(); + _mi_page_purge_holes_end(theap->tld); } // Purge the holes in every page this thread may safely touch: @@ -225,6 +227,7 @@ static void mi_purge_holes_of(mi_tld_t* tld) mi_attr_noexcept { // hold it. Reading a `heaps[i]` outside the lock is a use-after-free (`heap->subproc`). mi_lock(&tld->theaps_lock) { for (mi_theap_t* theap = tld->theaps; theap != NULL; theap = theap->tnext) { + if (theap->frozen) continue; mi_theap_purge_holes(theap); mi_heap_t* const heap = _mi_theap_heap(theap); if (heap != NULL && heap_count < MI_PURGE_HOLES_MAX_HEAPS) { @@ -247,7 +250,7 @@ void _mi_thread_idle_work(mi_tld_t* tld, mi_theap_t* theap0) mi_attr_noexcept { if (tld == NULL) return; // each phase is a full walk: an owner waiting in `_mi_park_leave` cannot allocate until we stop if (mi_atomic_load_relaxed(&tld->park_reclaim) != 0) return; - if (theap0 != NULL && mi_theap_is_initialized(theap0)) { + if (theap0 != NULL && mi_theap_is_initialized(theap0) && !theap0->frozen) { mi_theap_collect(theap0, false /* not forced */); } if (mi_atomic_load_relaxed(&tld->park_reclaim) != 0) return; @@ -994,3 +997,25 @@ bool mi_theap_visit_blocks(const mi_theap_t* theap, bool visit_blocks, mi_block_ return mi_theap_visit_areas(theap, &mi_theap_area_visitor, &args); } + + +// Mark a theap as an immutable snapshot: idle work, hole purging and collection skip it from now on. +void mi_theap_freeze(mi_theap_t* theap) mi_attr_noexcept { + if (theap == NULL) return; + theap->frozen = true; +} + +// Snapshot restore: the thread state in the snapshot (the main thread's static tld included) records the +// thread id of the process that built it. The calling thread takes it over: without this, every page +// this thread allocates from now on is stamped with a thread id no thread has, so all of its frees +// take the cross-thread path and strand blocks on `xthread_free`, and its idle sweeps refuse to run. +void mi_theap_adopt_current_thread(mi_theap_t* theap) mi_attr_noexcept { + if (theap == NULL || theap->tld == NULL) return; + mi_tld_t* const tld = theap->tld; + tld->thread_id = _mi_thread_id(); + mi_atomic_store_release(&tld->park_state, MI_PARK_RUNNING); + mi_atomic_store_release(&tld->park_reclaim, 0); + mi_atomic_store_release(&tld->park_swept, 0); + tld->park_theap0 = NULL; + mi_lock_init(&tld->theaps_lock); // the builder was quiescent when it froze, but its lock word is opaque state +}