From 46cd81f3789fea76abffcff330e88153689ac473 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Fri, 21 Aug 2026 05:45:36 +0000 Subject: [PATCH] Check the free lists on the cold walks instead of faulting on a corrupted link A free block holds its next link in its own first word. When something writes into a block after it was freed, or a stale free links a live object, the link points anywhere. Release builds do not encode links, so the walks that read a whole list followed it and faulted: the idle sweep (mi_page_purge_holes_walk), the forced collect of local_free in mi_page_free_collect_ex, and mi_page_thread_collect_to_local, which bounded the length of the thread-free list but not where its links point. This is the crash family behind oven-sh/bun BUN-40BH, BUN-40CP, BUN-40SQ and BUN-41H5. The three walks now check every link: it has to be NULL or the start of a formed block of the page. The sweep also tracks which blocks it has seen, so a block listed twice is found instead of counted twice (two counts for one block could make an OS page with one live block look entirely free and get discarded) and a cyclic list ends. A corrupted list is reported through _mi_error_message (EFAULT, as the existing thread-free message does) and cut in front of the block holding the bad link; the blocks cut off are counted as used from then on, so they are never handed out and the page is never returned to the arena while one of them may still be live. The sweep knows how many blocks remain listed and sets used exactly; the collects bump it by one. The thread-free collect keeps its existing recovery and drops the list it took. The scavenger thread blocked every signal, including the ones a fault on the thread itself raises. A blocked SIGSEGV or SIGBUS is not queued: the kernel resets it to the default action and kills the process, so a fault during a sweep the scavenger did for a parked thread ended the process with no report from the host's crash handler (verified on Linux). Leave the thread-directed fault signals unblocked; the process-directed ones stay blocked as before. test-freelist-corruption scribbles a freed block the way the crashes look (a link 8 bytes into the block, the word 0xA0D behind it) on each of the three lists, plus a bad head and a self-linked block, and checks the report, the cut, the block accounting, that a second pass is quiet, that the block is not handed out again and that the live blocks are intact. Without the src changes four of the five tests fault and the self-link test loops. --- CMakeLists.txt | 2 +- src/page.c | 180 +++++++++++++-- src/scavenger.c | 13 ++ test/test-freelist-corruption.c | 388 ++++++++++++++++++++++++++++++++ 4 files changed, 563 insertions(+), 20 deletions(-) create mode 100644 test/test-freelist-corruption.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 96adcf5b1..6d4b2e15b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -874,7 +874,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-delete-race heap-churn fork-user-heap snapshot prof prof-adversarial theap-sentinel purge-holes purge-zero park-handoff) + set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-delete-race heap-churn fork-user-heap snapshot prof prof-adversarial theap-sentinel purge-holes purge-zero park-handoff freelist-corruption) if (MI_DEBUG_FULL OR CMAKE_BUILD_TYPE MATCHES "Debug") list(APPEND mi_static_tests commit-fail) # uses mi_debug_fail_os_commit_after (MI_DEBUG>0 only) endif() diff --git a/src/page.c b/src/page.c index 57a5c3eb1..ad0a0660d 100644 --- a/src/page.c +++ b/src/page.c @@ -252,6 +252,107 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif +/* ----------------------------------------------------------- + Free list corruption. + + A free block carries its `next` link in its own first word, so a write into a block after + it was freed, or a stale free that linked a still-live object, leaves a link that can point + anywhere. The allocation path pops a link without looking at it; the cold paths that read a + whole list (`mi_page_thread_collect_to_local`, the forced collect in `mi_page_free_collect_ex`, + and the idle sweep's `mi_page_purge_holes_walk`) are the ones that reach such a link first + in practice, and without a check they fault on it (oven-sh/bun BUN-40BH and its siblings). + So they check every link: a valid link is NULL or the start of a formed block of this page, + a list never holds more blocks than the page has free, and (in the sweep, which counts the + blocks) no block is on it twice. (`MI_ENCODE_FREELIST` builds already reject a link that + leaves the page in `mi_block_next`; release builds do not encode.) The checks cost a division + per block walked. Of the three walks only the thread-free collect is reached from allocation + (`_mi_malloc_generic`), and it already chases every link it collects; popping `page->free` + in `_mi_page_malloc` is not touched. +----------------------------------------------------------- */ + +#define MI_PAGE_MAX_CAPACITY (1 << 16) // `page->capacity` is a uint16_t (`mi_page_sweep_state_fits` checks that) + +// Is `block` the start of a formed block of `page`? On success `*idx` is its block index. +static inline bool mi_page_block_index_of(const mi_page_t* page, const mi_block_t* block, size_t* idx) { + const uintptr_t offset = (uintptr_t)block - (uintptr_t)mi_page_start(page); // wraps around when `block` lies before the page + const size_t i = (size_t)(offset / page->block_size); + *idx = i; + return (i < page->capacity && (uintptr_t)(i * page->block_size) == offset); +} + +// The most blocks the free lists of this page can hold between them: the blocks not in use. +// (A block whose memory is discarded is free as well but on no list, so this is an upper bound.) +static inline size_t mi_page_max_free_listed(const mi_page_t* page) { + return (page->used <= page->capacity ? (size_t)(page->capacity - page->used) : 0); +} + +// Cut the list rooted at `*list` so that it ends at `last` (NULL empties it). What is cut off +// is the block holding a bad link (possibly a live object, or a block something still writes +// to), a block that is on the list for the second time, or the blocks past the page's free +// count -- and everything linked behind it. Nothing cut off is handed out again: it is counted +// as used from now on, which also keeps the page from being given back to the arena while such +// a block is linked into it (for a double free this simply restores the count; anything else +// is leaked). `listed` is the number of blocks the page's free lists hold after the cut, if the +// caller knows it (the sweep does: it has just collected the other lists into this one), and +// then `used` becomes exact; otherwise (`MI_LISTED_UNKNOWN`) `used` goes up by one for the +// block holding the bad link, and what hung behind it stays uncounted, which only makes the +// page look emptier than it is. The error is reported before the cut, so an error handler that +// aborts sees the list as it was: `bad` is the link or block that failed the check, `holder` +// the block whose link it was (NULL for the head of the list). +#define MI_LISTED_UNKNOWN SIZE_MAX + +static void mi_page_free_list_cut(mi_page_t* page, mi_block_t** list, mi_block_t* last, size_t listed, + const char* what, const char* problem, const mi_block_t* bad, const mi_block_t* holder) { + if (holder == NULL) { + _mi_error_message(EFAULT, "corrupted %s list in page %p (block size %zu): %s %p\n", + what, page, page->block_size, problem, bad); + } + else { + _mi_error_message(EFAULT, "corrupted %s list in page %p (block size %zu): %s %p in block %p\n", + what, page, page->block_size, problem, bad, holder); + } + if (last == NULL) { *list = NULL; } + else { mi_block_set_next(page, last, NULL); } + if (listed == MI_LISTED_UNKNOWN) { + if (page->used < page->capacity) { page->used++; } + } + else { + const size_t off_list = listed + _mi_page_purged_count(page); // free, but on the list or discarded + page->used = (off_list < page->capacity ? (uint32_t)(page->capacity - off_list) : 0); + } +} + +// Walk the list rooted at `*list`, checking it. Returns its last block, or NULL when the list is +// (or, after a cut, has become) empty. +static mi_block_t* mi_page_free_list_checked_tail(mi_page_t* page, mi_block_t** list, const char* what) { + mi_block_t* block = *list; + if (block == NULL) return NULL; + size_t idx; + if mi_unlikely(!mi_page_block_index_of(page, block, &idx)) { + mi_page_free_list_cut(page, list, NULL, MI_LISTED_UNKNOWN, what, "invalid head", block, NULL); + return NULL; + } + const size_t max_count = mi_page_max_free_listed(page); + mi_block_t* prev = NULL; + size_t count = 1; + for (;;) { + if mi_unlikely(count > max_count) { + mi_page_free_list_cut(page, list, prev, MI_LISTED_UNKNOWN, what, "more blocks than the page has free, at block", block, NULL); + return prev; + } + mi_block_t* const next = mi_block_next(page, block); + if (next == NULL) return block; + if mi_unlikely(!mi_page_block_index_of(page, next, &idx)) { + mi_page_free_list_cut(page, list, prev, MI_LISTED_UNKNOWN, what, "invalid link", next, block); + return prev; + } + prev = block; + block = next; + count++; + } +} + + /* ----------------------------------------------------------- Page collect the `local_free` and `thread_free` lists ----------------------------------------------------------- */ @@ -265,7 +366,17 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) size_t count = 1; mi_block_t* last = head; mi_block_t* next; + size_t idx; + if mi_unlikely(!mi_page_block_index_of(page, head, &idx)) { + _mi_error_message(EFAULT, "corrupted thread-free list in page %p (block size %zu): invalid head %p\n", page, page->block_size, head); + return; // the thread-free items cannot be freed + } while ((next = mi_block_next(page, last)) != NULL && count <= max_count) { + // a link that is not a block of this page is followed by no one (see "Free list corruption" above) + if mi_unlikely(!mi_page_block_index_of(page, next, &idx)) { + _mi_error_message(EFAULT, "corrupted thread-free list in page %p (block size %zu): invalid link %p in block %p\n", page, page->block_size, next, last); + return; // the thread-free items cannot be freed + } count++; last = next; } @@ -686,8 +797,10 @@ void _mi_page_unpurge_unformed_upto(mi_page_t* page, uintptr_t end) { // Walk the free list of a page and discard every OS page in it that holds no live block. -// Returns false if any discard failed: those blocks went straight back on the free list and the -// page must be swept again, so the caller must not record it as swept. +// Returns false if any discard failed, or if the free list turned out to be corrupted: in both +// cases the page must be swept again, so the caller must not record it as swept. (A failed +// discard puts its blocks straight back on the free list; a corrupted list is cut at the +// corruption, see "Free list corruption" above, and is walked afresh by the next sweep.) static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { if (page->free == NULL) return true; // nothing to take off the free list @@ -697,14 +810,29 @@ static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { if (nbits > MI_PAGE_PURGE_BITS) return true; bool complete = true; - // 1. count, per OS page, the blocks on the free list that overlap it + // 1. count, per OS page, the blocks on the free list that overlap it. The list is checked on + // the way (see "Free list corruption"), including that no block is on it twice: a block + // listed twice (a double free) would be counted twice, and an OS page holding it and one + // live block would then look entirely free and be discarded from under the live block. + // (`listed` also bounds the walk: a list of distinct blocks of this page ends within + // `capacity` links.) After a cut the counts are not usable; the page is consistent again + // and the next sweep walks it afresh. uint16_t nfree[MI_PAGE_PURGE_BITS]; _mi_memzero(nfree, nbits * sizeof(uint16_t)); + uint64_t listed[MI_PAGE_MAX_CAPACITY / 64]; // by block index: seen on the list already? + _mi_memzero(listed, _mi_divide_up(page->capacity, 64) * sizeof(uint64_t)); size_t nvisited = 0; - for (mi_block_t* b = page->free; b != NULL; b = mi_block_next(page, b)) { - const size_t idx = mi_page_block_index(page, b); + size_t idx; + if mi_unlikely(!mi_page_block_index_of(page, page->free, &idx)) { + mi_page_free_list_cut(page, &page->free, NULL, 0, "free", "invalid head", page->free, NULL); + return false; + } + mi_block_t* prev = NULL; + mi_block_t* b = page->free; + for (;;) { mi_assert_internal(idx < page->capacity); mi_assert_internal(!mi_page_block_index_is_purged(page, idx)); // it is on the free list, so not purged + listed[idx / 64] |= ((uint64_t)1 << (idx % 64)); nvisited++; size_t kfirst, klast; mi_page_block_os_pages(page, idx, &kfirst, &klast); @@ -712,6 +840,22 @@ static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { mi_assert_internal(nfree[k] < UINT16_MAX); nfree[k]++; } + mi_block_t* const next = mi_block_next(page, b); + if (next == NULL) break; + if mi_unlikely(!mi_page_block_index_of(page, next, &idx)) { + // `b` holds the bad link, so it goes as well: `prev` becomes the last block, `nvisited - 1` remain + mi_page_free_list_cut(page, &page->free, prev, nvisited - 1, "free", "invalid link", next, b); + tld->holes_sweep_visited += nvisited; + return false; + } + if mi_unlikely((listed[idx / 64] & ((uint64_t)1 << (idx % 64))) != 0) { + // `next` is on the list for the second time: its first occurrence stays, the list ends at `b` + mi_page_free_list_cut(page, &page->free, b, nvisited, "free", "block listed twice:", next, b); + tld->holes_sweep_visited += nvisited; + return false; + } + prev = b; + b = next; } tld->holes_sweep_visited += nvisited; // folded into the process-wide counter at the end of the pass @@ -738,10 +882,10 @@ static bool mi_page_purge_holes_walk(mi_page_t* page, mi_tld_t* tld) { // 3. rebuild the free list without the blocks that are about to lose memory. This must // happen *before* the discard: it walks `next` pointers that live in the very memory - // we are about to discard. + // we are about to discard. (The list was checked in step 1 and nothing has touched it since.) mi_block_t* keep = NULL; size_t ndropped = 0; - mi_block_t* b = page->free; + b = page->free; while (b != NULL) { mi_block_t* const next = mi_block_next(page, b); if (mi_page_block_overlaps(page, mi_page_block_index(page, b), todo)) { @@ -893,8 +1037,6 @@ void _mi_page_unpurge_all(mi_page_t* page) { one OS page, so nothing is double counted even for a block straddling a boundary. ----------------------------------------------------------- */ -#define MI_HOLES_MAX_CAP (1 << 16) // `page->capacity` is a uint16_t - static void mi_holes_mark_free_list(const mi_page_t* page, mi_block_t* b, uint64_t* set) { const size_t cap = page->capacity; for (size_t n = 0; b != NULL && n <= cap; n++) { // `n` bounds a corrupt or cyclic list @@ -953,7 +1095,7 @@ void _mi_page_holes_report_page(const mi_page_t* page, mi_holes_report_t* rep) { if (page == NULL || rep == NULL) return; const size_t bs = page->block_size; const size_t cap = page->capacity; - if (bs == 0 || cap > MI_HOLES_MAX_CAP) return; + if (bs == 0 || cap > MI_PAGE_MAX_CAPACITY) return; mi_holes_bin_t* const r = &rep->bin[_mi_bin(bs)]; r->pages++; if (bs > r->block_size) { r->block_size = bs; } @@ -963,7 +1105,7 @@ void _mi_page_holes_report_page(const mi_page_t* page, mi_holes_report_t* rep) { rep->unformed_discarded_bytes += _mi_page_unformed_purged_bytes(page); if (cap == 0) return; - uint64_t freelisted[MI_HOLES_MAX_CAP / 64]; + uint64_t freelisted[MI_PAGE_MAX_CAPACITY / 64]; const size_t nwords = _mi_divide_up(cap, 64); _mi_memzero(freelisted, nwords * sizeof(uint64_t)); mi_holes_mark_free_list(page, page->free, freelisted); @@ -1203,16 +1345,16 @@ static void mi_page_free_collect_ex(mi_page_t* page, bool force, bool allow_unpu page->free_is_zero = false; } else if (force) { - // append -- only on shutdown (force) as this is a linear operation - mi_block_t* tail = page->local_free; - mi_block_t* next; - while ((next = mi_block_next(page, tail)) != NULL) { - tail = next; + // append -- only on shutdown and in the idle sweep (force) as this is a linear operation; + // a corrupted `local_free` is cut at the corruption (see "Free list corruption") and what + // is left of it is appended + mi_block_t* const tail = mi_page_free_list_checked_tail(page, &page->local_free, "local free"); + if (tail != NULL) { + mi_block_set_next(page, tail, page->free); + page->free = page->local_free; + page->free_is_zero = false; } - mi_block_set_next(page, tail, page->free); - page->free = page->local_free; page->local_free = NULL; - page->free_is_zero = false; } } diff --git a/src/scavenger.c b/src/scavenger.c index 93c71b3bf..78a4d3321 100644 --- a/src/scavenger.c +++ b/src/scavenger.c @@ -322,8 +322,21 @@ void _mi_scavenger_start(void) { // unblocked will have process-directed signals dispatched to it and silently // discarded, starving signalfd/kqueue consumers. sigfillset on glibc/musl // already excludes the libc-internal realtime signals used for setxid/cancel. + // + // Except the signals a fault on this thread itself raises: a blocked SIGSEGV/SIGBUS + // is not queued, the kernel resets it to its default action and kills the process on + // the spot, so the host's crash handler never runs and a corrupted free list that the + // sweep trips over (see `mi_page_purge_holes_walk`) ends the process without a report. + // These are thread-directed by nature, so leaving them unblocked starves no one. sigset_t all, old; sigfillset(&all); + sigdelset(&all, SIGSEGV); + sigdelset(&all, SIGBUS); + sigdelset(&all, SIGILL); + sigdelset(&all, SIGFPE); + sigdelset(&all, SIGTRAP); + sigdelset(&all, SIGABRT); + sigdelset(&all, SIGSYS); pthread_sigmask(SIG_SETMASK, &all, &old); if (pthread_create(&_mi_scavenger_thread, NULL, &mi_scavenger_thread_main, NULL) != 0) { mi_atomic_store_release(&_mi_scavenger_running, (uintptr_t)0); diff --git a/test/test-freelist-corruption.c b/test/test-freelist-corruption.c new file mode 100644 index 000000000..d8e847d4c --- /dev/null +++ b/test/test-freelist-corruption.c @@ -0,0 +1,388 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2025, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// The walkers that read a whole free list (the idle sweep, the forced collect and the +// thread-free collect; see the "Free list corruption" section in `src/page.c`) must survive a +// corrupted list: report it once through the error handler, cut the list, leave the page's +// accounting consistent, and never hand out the block that held the bad link. +// +// Each test corrupts a list the way the crashes in the field look (oven-sh/bun BUN-40BH and +// its siblings): the first words of a block were overwritten after it was freed. The bad link +// points 8 bytes into the block itself, so that a build that encodes its free lists (whose +// `mi_block_next` already rejects a link into another page) and a release build take the same +// path; the word at that address is 0xA0D, the value seen most often in the field. On a build +// without the checks these tests dereference 0xA0D and fault (and `free-twice` loops forever). + +#include +#include +#include +#include +#include + +#include "mimalloc.h" +#include "mimalloc/internal.h" // _mi_ptr_page, mi_block_next, mi_block_set_next, mi_page_thread_free, _mi_page_purged_count + +#include "testhelper.h" + +// --------------------------------------------------------------------------- +// The error handler and the output hook. A corruption must be reported as an +// EFAULT, and the walker under test must report it exactly once: that is +// counted on the message text, because a debug build also walks the lists in +// its assertions (`_mi_page_is_valid`) and reports a bad link from there first. +// --------------------------------------------------------------------------- + +static int error_count = 0; +static int error_last = 0; +static const char* report_needle = NULL; // the message the walker under test is expected to print ... +static int report_count = 0; // ... and how often it was printed + +static void on_error(int err, void* arg) { + MI_UNUSED(arg); + error_count++; + error_last = err; +} + +static void on_output(const char* msg, void* arg) { + MI_UNUSED(arg); + fputs(msg, stderr); + if (report_needle != NULL && strstr(msg, report_needle) != NULL) { report_count++; } +} + +static void expect_report(const char* needle) { + report_needle = needle; + report_count = 0; + error_count = 0; + error_last = 0; +} + +static bool reported_once(const char* what) { + if (report_count != 1 || error_count < 1 || error_last != EFAULT) { + fprintf(stderr, "\n %s: expected one \"%s\" report and an EFAULT, got %d report(s) and %d error(s), the last one %d\n", + what, report_needle, report_count, error_count, error_last); + return false; + } + return true; +} + +#define REPORT_FREE "corrupted free list in page" +#define REPORT_LOCAL_FREE "corrupted local free list in page" +#define REPORT_THREAD_FREE "corrupted thread-free list in page" + +// --------------------------------------------------------------------------- +// a page we control: NBLOCKS blocks in one page of a fresh heap; the odd ones +// are freed and collected onto `page->free`, the even ones stay live and hold +// a pattern. With every other block live no OS page is ever discardable, so +// the freed blocks stay on the list. +// --------------------------------------------------------------------------- + +#define NBLOCKS (24) +#define BSIZE (1000) + +typedef struct victim_s { + mi_heap_t* heap; + mi_page_t* page; + void* block[NBLOCKS]; + bool live[NBLOCKS]; +} victim_t; + +static uint8_t pattern_byte(size_t id, size_t off) { + return (uint8_t)((id * 131u) ^ (off * 7u) ^ (off >> 8)); +} + +static bool pattern_intact(const victim_t* v) { + for (size_t i = 0; i < NBLOCKS; i++) { + if (!v->live[i]) continue; + const uint8_t* const b = (const uint8_t*)v->block[i]; + for (size_t off = 0; off < BSIZE; off++) { + if (b[off] != pattern_byte(i, off)) { + fprintf(stderr, "\n live block %zu was damaged at offset %zu\n", i, off); + return false; + } + } + } + return true; +} + +static bool victim_setup(victim_t* v) { + memset(v, 0, sizeof(*v)); + v->heap = mi_heap_new(); + if (v->heap == NULL) { fprintf(stderr, "\n mi_heap_new failed\n"); return false; } + for (size_t i = 0; i < NBLOCKS; i++) { + uint8_t* const b = (uint8_t*)mi_heap_malloc(v->heap, BSIZE); + if (b == NULL) { fprintf(stderr, "\n mi_heap_malloc failed\n"); return false; } + for (size_t off = 0; off < BSIZE; off++) { b[off] = pattern_byte(i, off); } + v->block[i] = b; + v->live[i] = true; + } + v->page = _mi_ptr_page(v->block[0]); + for (size_t i = 1; i < NBLOCKS; i++) { + if (_mi_ptr_page(v->block[i]) != v->page) { fprintf(stderr, "\n the blocks do not share one page\n"); return false; } + } + for (size_t i = 1; i < NBLOCKS; i += 2) { + mi_free(v->block[i]); + v->live[i] = false; + } + error_count = 0; + mi_on_thread_idle(); // collects the frees onto `page->free` + if (error_count != 0) { fprintf(stderr, "\n the sweep of an intact page reported an error\n"); return false; } + if (v->page->free == NULL || mi_block_next(v->page, v->page->free) == NULL) { + fprintf(stderr, "\n expected at least two blocks on page->free\n"); + return false; + } + return true; +} + +static void victim_done(victim_t* v) { + if (v->heap != NULL) { mi_heap_destroy(v->heap); } + v->heap = NULL; +} + +// Overwrite the first words of a freed block the way the field crashes look: its link points +// 8 bytes into the block itself, and the word there is 0xA0D. Returns the bad link. +static mi_block_t* scribble(const mi_page_t* page, void* block) { + uintptr_t* const inside = (uintptr_t*)((uint8_t*)block + sizeof(mi_block_t)); + *inside = (uintptr_t)0xA0D; + mi_block_set_next(page, (mi_block_t*)block, (mi_block_t*)inside); + return (mi_block_t*)inside; +} + +// Walks are bounded so that a failing test cannot hang on a list that is still broken. +static size_t list_count(const mi_page_t* page, mi_block_t* head) { + size_t n = 0; + for (mi_block_t* b = head; b != NULL && n <= (size_t)page->capacity + 1; b = mi_block_next(page, b)) { n++; } + return n; +} + +static size_t list_occurrences(const mi_page_t* page, mi_block_t* head, const void* block) { + size_t n = 0, found = 0; + for (mi_block_t* b = head; b != NULL && n <= (size_t)page->capacity + 1; b = mi_block_next(page, b)) { + n++; + if ((const void*)b == block) { found++; } + } + return found; +} + +static bool on_no_list(const victim_t* v, const void* block) { + if (list_occurrences(v->page, v->page->free, block) != 0 || + list_occurrences(v->page, v->page->local_free, block) != 0 || + list_occurrences(v->page, mi_page_thread_free(v->page), block) != 0) { + fprintf(stderr, "\n the block that held the bad link is still on a list\n"); + return false; + } + return true; +} + +// The conservation of blocks that a cut must leave intact: `used` (which includes the blocks +// on the thread-free list) + the blocks on `free` and `local_free` + the discarded blocks. +static bool page_accounts(const victim_t* v, const char* when) { + const mi_page_t* const page = v->page; + const size_t listed = list_count(page, page->free) + list_count(page, page->local_free); + const size_t discarded = _mi_page_purged_count(page); + if ((size_t)page->used + listed + discarded != (size_t)page->capacity) { + fprintf(stderr, "\n %s: used %u + listed %zu + discarded %zu != capacity %u\n", + when, (unsigned)page->used, listed, discarded, (unsigned)page->capacity); + return false; + } + return true; +} + +typedef void (pass_fun_t)(victim_t* v); +static void pass_sweep(victim_t* v) { MI_UNUSED(v); mi_on_thread_idle(); } +static void pass_collect(victim_t* v) { mi_heap_collect(v->heap, true); } + +// After a cut the page must be an ordinary page again: another pass reports nothing, the +// accounting holds, the live blocks are intact, and `culprit` (the block that held the bad +// link; NULL when the head itself was bad) is never handed out again. +static bool page_recovered(victim_t* v, const void* culprit, pass_fun_t* pass) { + error_count = 0; + pass(v); + if (error_count != 0) { fprintf(stderr, "\n the second pass reported %d error(s)\n", error_count); return false; } + if (!page_accounts(v, "after the second pass")) return false; + void* got[4 * NBLOCKS]; + bool handed_out = false; + for (size_t i = 0; i < 4 * NBLOCKS; i++) { + got[i] = mi_heap_malloc(v->heap, BSIZE); + if (culprit != NULL && got[i] == culprit) { handed_out = true; } + } + for (size_t i = 0; i < 4 * NBLOCKS; i++) { mi_free(got[i]); } + if (handed_out) { fprintf(stderr, "\n the block that held the bad link was handed out again\n"); return false; } + return pattern_intact(v); +} + +// --------------------------------------------------------------------------- +// 1. a bad link in the middle of `page->free` (the idle sweep's walk): the +// list is cut in front of the block holding it +// --------------------------------------------------------------------------- + +static bool test_free_link(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + mi_block_t* const first = v.page->free; + mi_block_t* const culprit = mi_block_next(v.page, first); // the second block: something stays in front of the cut + scribble(v.page, culprit); + expect_report(REPORT_FREE); + mi_on_thread_idle(); + ok_ = reported_once("free-link") + && (v.page->free == first && mi_block_next(v.page, first) == NULL) + && on_no_list(&v, culprit) + && page_accounts(&v, "after the cut") + && page_recovered(&v, culprit, &pass_sweep); + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// 2. the head of `page->free` itself is bad: the whole list is dropped +// --------------------------------------------------------------------------- + +static bool test_free_head(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + v.page->free = scribble(v.page, v.page->free); + expect_report(REPORT_FREE); + mi_on_thread_idle(); + ok_ = reported_once("free-head") + && (v.page->free == NULL) + && page_accounts(&v, "after the cut") + && page_recovered(&v, NULL, &pass_sweep); + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// 3. a block that links to itself, as after a double free: the sweep must +// neither loop nor count the block twice (an OS page is discarded when every +// block in it is free, and a block counted twice can make it look that way). +// The block is an ordinary free block afterwards, so it may be handed out. +// --------------------------------------------------------------------------- + +static bool test_free_twice(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + mi_block_t* const twice = mi_block_next(v.page, v.page->free); + mi_block_set_next(v.page, twice, twice); + expect_report(REPORT_FREE); + mi_on_thread_idle(); + ok_ = reported_once("free-twice") + && (list_occurrences(v.page, v.page->free, twice) == 1) // its first occurrence stays ... + && (mi_block_next(v.page, twice) == NULL) // ... as the end of the list + && page_accounts(&v, "after the cut"); + if (ok_) { + error_count = 0; + mi_on_thread_idle(); + ok_ = (error_count == 0) && page_accounts(&v, "after the second sweep") && pattern_intact(&v); + if (error_count != 0) { fprintf(stderr, "\n the second sweep reported %d error(s)\n", error_count); } + } + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// 4. a bad link on `page->local_free`, which the sweep's forced collect +// appends to `free`: the part in front of the cut is still appended +// --------------------------------------------------------------------------- + +static bool test_local_free_link(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + // frees by the owning thread go onto `local_free`, most recent first: block 2 -> block 0 + mi_free(v.block[0]); v.live[0] = false; + mi_free(v.block[2]); v.live[2] = false; + ok_ = (v.page->local_free == (mi_block_t*)v.block[2] && mi_block_next(v.page, v.page->local_free) == (mi_block_t*)v.block[0]); + if (!ok_) { fprintf(stderr, "\n local_free is not [block 2, block 0]\n"); } + } + if (ok_) { + scribble(v.page, v.block[0]); // the last block of the list: the cut drops exactly this one + expect_report(REPORT_LOCAL_FREE); + mi_on_thread_idle(); + ok_ = reported_once("local-free-link") + && on_no_list(&v, v.block[0]) + && (list_occurrences(v.page, v.page->free, v.block[2]) == 1) + && (v.page->local_free == NULL) + && page_accounts(&v, "after the cut") + && page_recovered(&v, v.block[0], &pass_sweep); + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// 5. a bad link on the thread-free list (a block freed by another thread): +// the collect drops the list it took off the page, and the blocks on it +// stay accounted as used +// --------------------------------------------------------------------------- + +static void* freed_by_other_thread = NULL; + +static bool free_on_other_thread(void) { + mi_free(freed_by_other_thread); + return true; +} + +static bool test_thread_free_link(void) { + victim_t v; + bool ok_ = victim_setup(&v); + if (ok_) { + freed_by_other_thread = v.block[0]; + v.live[0] = false; + ok_ = mi_run_on_thread(&free_on_other_thread) && (mi_page_thread_free(v.page) == (mi_block_t*)v.block[0]); + if (!ok_) { fprintf(stderr, "\n the block freed on the other thread is not the head of the thread-free list\n"); } + } + if (ok_) { + const uint32_t used_before = v.page->used; // a block on the thread-free list still counts as used + scribble(v.page, v.block[0]); + expect_report(REPORT_THREAD_FREE); + mi_heap_collect(v.heap, true); + ok_ = reported_once("thread-free-link") + && (mi_page_thread_free(v.page) == NULL) + && on_no_list(&v, v.block[0]) + && (v.page->used == used_before) + && page_accounts(&v, "after the drop") + && page_recovered(&v, v.block[0], &pass_collect); + if (ok_ == false && v.page->used != used_before) { fprintf(stderr, "\n used changed from %u to %u\n", (unsigned)used_before, (unsigned)v.page->used); } + } + victim_done(&v); + return ok_; +} + +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- + +// An optional argument selects a single test, which is how each one is checked to fault (or +// loop) on a build without the checks: the first one to run would otherwise take the rest with it. +static bool selected(int argc, char** argv, const char* name) { + return (argc < 2 || strcmp(argv[1], name) == 0); +} + +int main(int argc, char** argv) { + mi_version(); + mi_register_error(&on_error, NULL); + mi_register_output(&on_output, NULL); + mi_option_set(mi_option_show_errors, 1); // the messages are part of what is tested: they have to format without faulting + mi_option_set(mi_option_purge_holes_full_every, 1); // walk every page on every sweep: a scribbled link changes nothing the skip check looks at + + if (mi_option_is_enabled(mi_option_purge_holes)) { + if (selected(argc, argv, "free-link")) { CHECK("free-link", test_free_link()); } + if (selected(argc, argv, "free-head")) { CHECK("free-head", test_free_head()); } + if (selected(argc, argv, "free-twice")) { CHECK("free-twice", test_free_twice()); } + if (selected(argc, argv, "local-free-link")) { CHECK("local-free-link", test_local_free_link()); } + } + else { + fprintf(stderr, "purge_holes is off: the sweep tests are skipped\n"); + } + if (selected(argc, argv, "thread-free-link")) { CHECK("thread-free-link", test_thread_free_link()); } + + return print_test_summary(); +}