diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d57a14d0e..dd60b12e3 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ permissions: contents: write env: - RELEASE: Release v3.4.3 + RELEASE: Release v3.4.5 FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true name: Release @@ -19,7 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - branch: [v3.4.3] # [dev,dev2,dev3] + branch: [v3.4.5, v2.4.5, v1.9.15] # [dev,dev2,dev3] # we build on the oldest ubuntu version for better binary compatibility. os: [windows-latest, macOS-latest, macos-15-intel, ubuntu-22.04, ubuntu-22.04-arm] diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f2e16903e..734cc8fc1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,5 +1,5 @@ on: - workflow_dispatch: # allow running the workflow manually + workflow_dispatch: # allow running the workflow manually push: branches: - 'dev*' @@ -9,7 +9,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true # Cancel any in-progress runs of this workflow when a new run is triggered on the same ref (branch or tag) - + name: Test jobs: build: @@ -21,8 +21,21 @@ jobs: os: [windows-latest, macos-latest, ubuntu-latest, macos-14, macos-15-intel, ubuntu-22.04-arm, windows-11-arm] exclude: - os: macos-14 - tests: extra - + tests: extra + include: + - os: ubuntu-latest + tests: freebsd-x64 + - os: ubuntu-latest + tests: alpine-x64 + - os: ubuntu-latest + tests: alpine-riscv64 # qemu + - os: ubuntu-latest + tests: alpine-arm32 # qemu + - os: windows-latest + tests: mingw-ucrt64 + # - os: ubuntu-latest # slow, so disable by default + # tests: freebsd-arm64 # qemu + steps: # Checkout: https://github.com/actions/checkout - name: Checkout repository @@ -39,10 +52,18 @@ jobs: - name: Release if: matrix.tests == 'basic' run: | - cmake . -B out/release -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake . -B out/release -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON cmake --build out/release --parallel 4 --config Release ctest --test-dir out/release --verbose --timeout 240 -C Release + - name: Release, asm + if: matrix.tests == 'basic' && runner.os == 'Linux' + uses: actions/upload-artifact@v7 + with: + name: ${{ matrix.os }}-alloc.c.s + path: out/release/CMakeFiles/mimalloc-static.dir/src/alloc.c.s + + - name: Secure if: matrix.tests == 'basic' run: | @@ -65,7 +86,7 @@ jobs: cmake --build out/release-cxx --parallel 8 --config Release ctest --test-dir out/release-cxx --verbose --timeout 240 -C Release - # Extra, Windows x86 + # Extra, Windows x86 - name: Debug, Win32 if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') run: | @@ -92,7 +113,7 @@ jobs: run: | cmake . -B out/release-clang -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl cmake --build out/release-clang --parallel 4 --config Release - ctest --test-dir out/release-clang --verbose --timeout 240 -C Release + ctest --test-dir out/release-clang --verbose --timeout 240 -C Release # Extra, macOS fixed TLS - name: Debug, TLS fixed @@ -149,10 +170,17 @@ jobs: - name: Release, C++, clang++ if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/release-clang-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON + CC=clang CXX=clang++ cmake . -B out/release-clang-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON -DMI_SEE_ASM=ON cmake --build out/release-clang-cxx --parallel 8 --config Release ctest --test-dir out/release-clang-cxx --verbose --timeout 240 -C Release + - name: Release, C++, clang++, asm + if: matrix.tests == 'extra' && runner.os == 'Linux' + uses: actions/upload-artifact@v7 + with: + name: ${{ matrix.os }}-clang-cxx-alloc.s + path: out/release-clang-cxx/alloc.s + - name: Debug, ASAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | @@ -191,3 +219,228 @@ jobs: ctest --test-dir out/release-guarded --verbose --timeout 240 -C Release env: MIMALLOC_GUARDED_SAMPLE_RATE: 100 + + # MSYS2 mingw64 + - name: mingw-ucrt64 x64, Setup + if: matrix.tests == 'mingw-ucrt64' + uses: msys2/setup-msys2@v2 + with: + msystem: UCRT64 + install: | + mingw-w64-ucrt-x86_64-gcc + mingw-w64-ucrt-x86_64-git + mingw-w64-ucrt-x86_64-make + mingw-w64-ucrt-x86_64-cmake + + - name: mingw-ucrt64 x64, Debug + if: matrix.tests == 'mingw-ucrt64' + shell: msys2 {0} + run: | + env + cmake . -B out/debug-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake --build out/debug-mingw-ucrt64-x64 --parallel 4 --config Debug + ctest --test-dir out/debug-mingw-ucrt64-x64 --verbose --timeout 240 -C Debug + + - name: mingw-ucrt64 x64, Release + if: matrix.tests == 'mingw-ucrt64' + shell: msys2 {0} + run: | + cmake . -B out/release-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-mingw-ucrt64-x64 --parallel 4 --config Release + ctest --test-dir out/release-mingw-ucrt64-x64 --verbose --timeout 240 -C Release + + + # VM: FreeBSD (arm64 and x64) + - name: FreeBSD arm64, Setup + if: matrix.tests == 'freebsd-arm64' + uses: vmactions/freebsd-vm@v1 + with: + arch: arm64 + custom-shell-name: freebsd-arm64 + cache-after-prepare: true + prepare: | + uname -a + pkg install -y cmake + + - name: FreeBSD arm64, Debug + if: matrix.tests == 'freebsd-arm64' + shell: freebsd-arm64 {0} + run: | + export MI_TEST_LIGHT=1 + cmake . -B out/debug-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake --build out/debug-arm64 --parallel 4 --config Debug + ctest --test-dir out/debug-arm64 --verbose --timeout 240 -C Debug + + - name: FreeBSD arm64, Release + if: matrix.tests == 'freebsd-arm64' + shell: freebsd-arm64 {0} + run: | + export MI_TEST_LIGHT=1 + cmake . -B out/release-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-arm64 --parallel 4 --config Release + ctest --test-dir out/release-arm64 --verbose --timeout 240 -C Release + + - name: FreeBSD arm64, Secure + if: matrix.tests == 'freebsd-arm64' + shell: freebsd-arm64 {0} + run: | + export MI_TEST_LIGHT=1 + cmake . -B out/secure-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-arm64 --parallel 4 --config Release + ctest --test-dir out/secure-arm64 --verbose --timeout 240 -C Release + + + - name: FreeBSD x64, Setup + if: matrix.tests == 'freebsd-x64' + uses: vmactions/freebsd-vm@v1 + with: + arch: x86_64 + usesh: true + custom-shell-name: freebsd-x64 + cache-after-prepare: true + prepare: | + pkg install -y cmake + + - name: FreeBSD x64, Debug + if: matrix.tests == 'freebsd-x64' + shell: freebsd-x64 {0} + run: | + cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake --build out/debug-x64 --parallel 4 --config Debug + ctest --test-dir out/debug-x64 --verbose --timeout 240 -C Debug + + - name: FreeBSD x64, Release + if: matrix.tests == 'freebsd-x64' + shell: freebsd-x64 {0} + run: | + cmake . -B out/release-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-x64 --parallel 4 --config Release + ctest --test-dir out/release-x64 --verbose --timeout 240 -C Release + + - name: FreeBSD x64, Secure + if: matrix.tests == 'freebsd-x64' + shell: freebsd-x64 {0} + run: | + cmake . -B out/secure-x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-x64 --parallel 4 --config Release + ctest --test-dir out/secure-x64 --verbose --timeout 240 -C Release + + + # VM: Alpine Linux with MUSL libc (riscv64 and x64) + - name: Alpine riscv64, Setup + if: matrix.tests == 'alpine-riscv64' + uses: jirutka/setup-alpine@v1 + with: + arch: riscv64 + shell-name: alpine-riscv64.sh + packages: | + build-base + cmake + ninja + + - name: Alpine riscv64, Debug + if: matrix.tests == 'alpine-riscv64' + shell: alpine-riscv64.sh {0} + env: + MI_TEST_LIGHT: 1 + run: | + uname -m + cmake . -B out/debug-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake --build out/debug-riscv64 --parallel 4 --config Debug + ctest --test-dir out/debug-riscv64 --verbose --timeout 240 -C Debug + + - name: Alpine riscv64, Release + if: matrix.tests == 'alpine-riscv64' + shell: alpine-riscv64.sh {0} + run: | + cmake . -B out/release-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON + cmake --build out/release-riscv64 --parallel 4 --config Release + ctest --test-dir out/release-riscv64 --verbose --timeout 240 -C Release + + - name: Alpine risc64, asm + if: matrix.tests == 'alpine-riscv64' + uses: actions/upload-artifact@v7 + with: + name: alpine-riscv64-alloc.c.s + path: out/release-riscv64/CMakeFiles/mimalloc.dir/src/alloc.c.s + + - name: Alpine riscv64, Secure + if: matrix.tests == 'alpine-riscv64' + shell: alpine-riscv64.sh {0} + run: | + cmake . -B out/secure-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-riscv64 --parallel 4 --config Release + ctest --test-dir out/secure-riscv64 --verbose --timeout 240 -C Release + + + - name: Alpine x64, Setup + if: matrix.tests == 'alpine-x64' + uses: jirutka/setup-alpine@v1 + with: + arch: x86_64 + shell-name: alpine-x64.sh + packages: | + build-base + cmake + + - name: Alpine x64, Debug + if: matrix.tests == 'alpine-x64' + shell: alpine-x64.sh {0} + run: | + cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake --build out/debug-x64 --parallel 4 --config Debug + ctest --test-dir out/debug-x64 --verbose --timeout 240 -C Debug + + - name: Alpine x64, Release + if: matrix.tests == 'alpine-x64' + shell: alpine-x64.sh {0} + run: | + cmake . -B out/release-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-x64 --parallel 4 --config Release + ctest --test-dir out/release-x64 --verbose --timeout 240 -C Release + + - name: Alpine x64, Secure + if: matrix.tests == 'alpine-x64' + shell: alpine-x64.sh {0} + run: | + cmake . -B out/secure-x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-x64 --parallel 4 --config Release + ctest --test-dir out/secure-x64 --verbose --timeout 240 -C Release + + - name: Alpine arm32, Setup + if: matrix.tests == 'alpine-arm32' + uses: jirutka/setup-alpine@v1 + with: + arch: armv7 # or armhf for arm-v6 with hard float + shell-name: alpine-arm32.sh + packages: | + build-base + cmake + + - name: Alpine arm32, Debug + if: matrix.tests == 'alpine-arm32' + shell: alpine-arm32.sh {0} + env: + MI_TEST_LIGHT: 1 + run: | + uname -m + cmake . -B out/debug-arm32 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake --build out/debug-arm32 --parallel 4 --config Debug + ctest --test-dir out/debug-arm32 --verbose --timeout 240 -C Debug + + - name: Alpine arm32, Release + if: matrix.tests == 'alpine-arm32' + shell: alpine-arm32.sh {0} + run: | + cmake . -B out/release-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-arm32 --parallel 4 --config Release + ctest --test-dir out/release-arm32 --verbose --timeout 240 -C Release + + - name: Alpine arm32, Secure + if: matrix.tests == 'alpine-arm32' + shell: alpine-arm32.sh {0} + run: | + cmake . -B out/secure-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-arm32 --parallel 4 --config Release + ctest --test-dir out/secure-arm32 --verbose --timeout 240 -C Release + diff --git a/CMakeLists.txt b/CMakeLists.txt index a1143193b..426a02778 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,14 +39,15 @@ option(MI_LOCAL_DYNAMIC_TLS "Use local-dynamic-tls, a slightly slower but dlopen option(MI_LIBC_MUSL "Enable this when linking with musl libc" OFF) # advanced +option(MI_OPT_FREE_SMALL "Use optimized `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)" OFF) option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG or SECURE mode, or with Valgrind/ASAN)" OFF) -option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) option(MI_NO_PROCESS_DETACH "Skip automatic mi_process_done at process exit (embedder calls it manually or not at all)" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) -option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) +option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) option(MI_NO_THP "Opt the memory mimalloc maps out of transparent huge pages on Linux/Android (sets MI_DEFAULT_ALLOW_THP=0)" OFF) option(MI_EXTRA_CPPDEFS "Extra pre-processor definitions (use as `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)" "") option(MI_SEE_ASM "Generate assembly files" OFF) +option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) # negated options for vcpkg features option(MI_NO_USE_CXX "Use plain C compilation (has priority over MI_USE_CXX)" OFF) @@ -78,7 +79,6 @@ set(mi_sources src/alloc-aligned.c src/alloc-posix.c src/arena.c - src/arena-meta.c src/bitmap.c src/heap.c src/heap-snapshot.c @@ -92,9 +92,11 @@ set(mi_sources src/random.c src/scavenger.c src/stats.c + src/subproc.c src/theap.c src/threadlocal.c - src/prim/prim.c) + src/prim/prim.c + src/prim/prim-tls.c) set(mi_cflags "") set(mi_cflags_static "") # extra flags for a static library build @@ -179,6 +181,10 @@ elseif(MI_ARCH STREQUAL "arm64") set(MI_OPT_ARCH "ON") # enable armv8.1-a by default on arm64 unless MI_NO_OPT_ARCH is set endif() +if(DEFINED ENV{MI_TEST_LIGHT}) + list(APPEND mi_defines MI_TEST_LIGHT=1) # to lighten test load in test integration +endif() + # ----------------------------------------------------------------------------- # Enable the C++ compiler early on if needed # ----------------------------------------------------------------------------- @@ -349,11 +355,7 @@ if (MI_NO_PROCESS_DETACH) list(APPEND mi_defines MI_NO_PROCESS_DETACH=1) endif() -if(MI_NO_DEBUG) - message(STATUS "Set debug level zero, all assertions are disabled (MI_NO_DEBUG=ON)") - set(MI_DEBUG OFF) - list(APPEND mi_defines MI_DEBUG=0) -elseif(MI_DEBUG_FULL) +if(MI_DEBUG_FULL) message(STATUS "Set debug level to full assertion and internal invariant checking (MI_DEBUG_FULL=ON, expensive)") set(MI_DEBUG ON) list(APPEND mi_defines MI_DEBUG=3) # full invariant checking (mi_assert, mi_assert_internal, and mi_assert_expensive) @@ -364,6 +366,9 @@ elseif(MI_DEBUG_INTERNAL) elseif(MI_DEBUG) message(STATUS "Set debug level to assertion checking (MI_DEBUG=ON)") list(APPEND mi_defines MI_DEBUG=1) # assertion checking (mi_assert) +elseif(MI_NO_DEBUG) + message(STATUS "Disable assertion checks (MI_NO_DEBUG=ON)") + list(APPEND mi_defines NDEBUG=1) elseif(CMAKE_BUILD_TYPE MATCHES "Debug") message(STATUS "Set debug level to internal assertion and invariant checking (CMAKE_BUILD_TYPE=Debug)") set(MI_DEBUG_INTERNAL ON) @@ -401,6 +406,18 @@ elseif(MI_OVERRIDE AND NOT MI_NO_FREE_IS_CHECKED) list(APPEND mi_cflags_dynamic -DMI_FREE_IS_CHECKED=1) endif() +if(NOT MI_OPT_FREE_SMALL AND NOT MI_DEBUG AND NOT MI_GUARDED AND NOT MI_SECURE) +set(MI_OPT_FREE_SMALL ON) +endif() +if(MI_OPT_FREE_SMALL) + if(MI_GUARDED OR MI_SECURE) + message(STATUS "Cannot enable MI_OPT_FREE_SMALL in secure or guarded mode (MI_OPT_FREE_SMALL=OFF)") + else() + message(STATUS "Optimize `mi_free_small` to not need a pagemap lookup (MI_OPT_FREE_SMALL=ON)") + list(APPEND mi_defines MI_OPT_FREE_SMALL=1) + endif() +endif() + if(MI_XMALLOC) message(STATUS "Enable abort() calls on memory allocation failure (MI_XMALLOC=ON)") list(APPEND mi_defines MI_XMALLOC=1) @@ -473,6 +490,15 @@ if(MI_TLS_RECURSE_GUARD) list(APPEND mi_defines MI_TLS_RECURSE_GUARD=1) endif() +if("$ENV{MSYSTEM}" STREQUAL "UCRT64") + message(STATUS "Detected MSYS2 UCRT64 environment") + set(MI_MINGW_UCRT64 "ON") + list(APPEND mi_defines MI_MINGW_UCRT64=1) +elseif("$ENV{MSYSTEM}" STREQUAL "MSYS") + message(STATUS "Detected MSYS2 MSYS/Cygwin environment") + set(MI_CYGWIN "ON") +endif() + if(MI_WIN_DIRECT_TLS) message(STATUS "Use only direct TLS slots on Windows to avoid extra tests in the malloc fast path -- this only works if the program uses less than 64 TlsAlloc'd slots in total! (MI_WIN_USE_ONLY_DIRECT_TLS=ON)") list(APPEND mi_defines MI_WIN_DIRECT_TLS=1) @@ -671,6 +697,8 @@ if(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$") else() set(mi_libname "${mi_libname}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version endif() +string(REGEX REPLACE "^mimalloc[-]?" "" mi_libsuffix "${mi_libname}") + if(MI_BUILD_SHARED) list(APPEND mi_build_targets "shared") @@ -734,16 +762,20 @@ if(MI_BUILD_SHARED) # On windows, link and copy the mimalloc redirection dll too. if(CMAKE_GENERATOR_PLATFORM STREQUAL "arm64ec") set(MIMALLOC_REDIRECT_SUFFIX "-arm64ec") + set(MINJECT_SUFFIX "") elseif(MI_ARCH STREQUAL "x64") set(MIMALLOC_REDIRECT_SUFFIX "") + set(MINJECT_SUFFIX "") if(CMAKE_SYSTEM_PROCESSOR STREQUAL "ARM64") message(STATUS "Note: x64 code emulated on Windows for arm64 should use an arm64ec build of 'mimalloc.dll'") message(STATUS " together with 'mimalloc-redirect-arm64ec.dll'. See the 'bin\\readme.md' for more information.") endif() elseif(MI_ARCH STREQUAL "x86") set(MIMALLOC_REDIRECT_SUFFIX "32") + set(MINJECT_SUFFIX "32") else() set(MIMALLOC_REDIRECT_SUFFIX "-${MI_ARCH}") # -arm64 etc. + set(MINJECT_SUFFIX "-${MI_ARCH}") endif() target_link_libraries(mimalloc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bin/mimalloc-redirect${MIMALLOC_REDIRECT_SUFFIX}.lib) # the DLL import library @@ -858,7 +890,13 @@ if (MI_BUILD_TESTS) else() message(STATUS "cannot build TSAN tests without MI_BUILD_SHARED being enabled") endif() + if(MI_GUARDED) + add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1 $) + elseif(TEST_NAME STREQUAL "stress-heaps") + add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=1 $) + else() add_test(NAME test-${TEST_NAME} COMMAND mimalloc-test-${TEST_NAME}) + endif() endforeach() # the same hole-purging tests with the feature turned off: it must be a no-op @@ -874,7 +912,7 @@ if (MI_BUILD_TESTS) target_compile_options(mi-heapview PRIVATE ${mi_cflags}) # dynamic override test - if(MI_BUILD_SHARED AND NOT (MI_TRACK_ASAN OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX)) + if(MI_BUILD_SHARED AND NOT (MI_CYGWIN OR MI_TRACK_ASAN OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX)) add_executable(mimalloc-test-stress-dynamic test/test-stress.c) target_compile_definitions(mimalloc-test-stress-dynamic PRIVATE ${mi_defines} "USE_STD_MALLOC=1") target_compile_options(mimalloc-test-stress-dynamic PRIVATE ${mi_cflags}) @@ -882,6 +920,11 @@ if (MI_BUILD_TESTS) if(WIN32) target_compile_definitions(mimalloc-test-stress-dynamic PRIVATE "MI_LINK_VERSION=1") # link mi_version target_link_libraries(mimalloc-test-stress-dynamic PRIVATE mimalloc ${mi_libraries}) # link mi_version + if(MI_MINGW_UCRT64) + # mingw always links mimalloc after system libraries. + # post-process with `minject` to make sure mimalloc is the first dll in the load order. + add_custom_command(TARGET mimalloc-test-stress-dynamic POST_BUILD COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/bin/minject${MINJECT_SUFFIX} --postfix=${mi_libsuffix} -f -i $) + endif() add_test(NAME test-stress-dynamic COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_VERBOSE=1 $) else() target_link_libraries(mimalloc-test-stress-dynamic PRIVATE ${mi_libraries}) # pthreads, issue 1158 diff --git a/bin/readme.md b/bin/readme.md index 69eed4115..7d426c2c3 100644 --- a/bin/readme.md +++ b/bin/readme.md @@ -83,8 +83,7 @@ if they are linked with the dynamic C runtime (`ucrtbase.dll`) -- just put the `mimalloc.dll` into the import table (and put `mimalloc-redirect.dll` in the same directory) Such patching can be done for example with [CFF Explorer](https://ntcore.com/?page_id=388). -The `minject` program can also do this from the command line -Use `minject --help` for options: +The `minject` program can also do this from the command line. Use `minject --help` for options: ``` > minject --help diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 8ef916cd1..857ef9ab5 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 3) -set(mi_version_minor 4) -set(mi_version_patch 3) +set(mi_version_minor 5) +set(mi_version_patch 0) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/contrib/vcpkg/portfile.cmake b/contrib/vcpkg/portfile.cmake index 554854741..b5d219595 100644 --- a/contrib/vcpkg/portfile.cmake +++ b/contrib/vcpkg/portfile.cmake @@ -3,7 +3,7 @@ vcpkg_from_github( REPO microsoft/mimalloc HEAD_REF master - # The "REF" can be a commit hash, branch name (dev3), or a version (v3.4.1). + # The "REF" can be a commit hash, branch name (dev3), or a version (v3.5.0). REF "v${VERSION}" # The sha512 is the hash of the tar.gz bundle. diff --git a/contrib/vcpkg/vcpkg.json b/contrib/vcpkg/vcpkg.json index b797c18ff..39b0fadaf 100644 --- a/contrib/vcpkg/vcpkg.json +++ b/contrib/vcpkg/vcpkg.json @@ -1,6 +1,6 @@ { "name": "mimalloc", - "version": "3.4.1", + "version": "3.5.0", "port-version": 0, "description": "Compact general purpose allocator with excellent performance", "homepage": "https://github.com/microsoft/mimalloc", diff --git a/doc/doxyfile b/doc/doxyfile index 53f874cfb..ba585c9b7 100644 --- a/doc/doxyfile +++ b/doc/doxyfile @@ -48,7 +48,7 @@ PROJECT_NAME = mi-malloc # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.8/2.1 +PROJECT_NUMBER = 3.4/2.4/1.9 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index 4b537c3d8..f3c71bd4f 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -71,13 +71,13 @@ There are three maintained versions of mimalloc. All versions are mostly equal e how the OS memory is handled. New development is mostly on v3, while v1 and v2 are maintained with security and bug fixes. -- __v1__: initial design of mimalloc (release tags: `v1.9.x`, development branch `dev`). Send PR's against this version if possible. -- __v2__: main mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.2.x`, development branch `dev2`) -- __v3__: simplifies the lock-free design of previous versions, and improves sharing of +- __v3__ __recommended__: simplifies the lock-free design of previous versions, and improves sharing of memory between threads. On certain large workloads this version may use (much) less memory. Also supports true first-class heaps (that can allocate from any thread) and has more efficient heap-walking (for the CPython GC for example). - (release tags: `v3.2.x`, development branch `dev3`). + (release tags: `v3.4.x`, development branch `dev3`). +- __v2__ __stable__: Uses thread-local segments to reduce fragmentation. (release tags: `v2.4.x`, development branch `dev2`) +- __v1__ __legacy__: initial design of mimalloc (release tags: `v1.9.x`, development branch `dev`). Send PR's against this version if possible. You can read more on the design of mimalloc in the [technical report](https://www.microsoft.com/en-us/research/publication/mimalloc-free-list-sharding-in-action) @@ -99,6 +99,7 @@ Further information: - \ref malloc - \ref aligned - \ref typed +- \ref constantsize - \ref zeroinit - \ref heap @@ -107,6 +108,7 @@ Further information: - \ref subproc - \ref extended +- \ref stats - \ref options - \ref theap - \ref posix @@ -342,7 +344,8 @@ void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t of /// ``` /// int* p = mi_malloc_tp(int) /// ``` -/// +/// On __v3__ these can improve performance as they use the `mi__csize` api's that +/// in turn can use mi_malloc_small(), mi_free_small(), etc. /// \{ /// Allocate a block of type \a tp. @@ -354,12 +357,22 @@ void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t of /// ``` /// int* p = mi_malloc_tp(int) /// ``` -/// -/// @see mi_malloc() -#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp))) +/// Using this is more safe, but can also be more efficient since +/// the allocation size is a compile-time constant. +/// @see mi_malloc_csize() +/// @see mi_malloc_small() +#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp))) + +/// Free an object of type \a tp. +/// @param tp The type of the object that is freed. +/// @param p A pointer to an object of type \a tp (or \a NULL ) +/// Can be more efficient as it internally uses mi_free_csize() +/// @see mi_free_csize() +/// @see mi_free_small() +#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp))) /// Allocate a zero-initialized block of type \a tp. -#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp))) +#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp))) /// Allocate \a count zero-initialized blocks of type \a tp. #define mi_calloc_tp(tp,count) ((tp*)mi_calloc(count,sizeof(tp))) @@ -390,6 +403,41 @@ void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t of /// \} +/// \defgroup constantsize Constant size allocation +/// Constant size allocation and freeing +/// For example: +/// ``` +/// long* p = (long*)mi_malloc_csize(sizeof(long)) +/// ``` +/// On __v3__ these can improve performance as they will statically +/// use mi_malloc_small(), mi_free_small(), etc. when possible. +/// This is especially useful for compilers and runtimes where the +/// the size is often statically known. These functions can be used +/// even when the size is not constant but then there is no performance benefit. +/// \{ + +/// @brief Allocate a constant size object. +/// @param size A (compile-time) constant size. +/// @return A pointer to an allocated block of size \a size bytes. +/// @see mi_free_csize() +/// @see mi_malloc() +void* mi_malloc_csize(size_t size); + +void* mi_zalloc_csize(size_t size); + +void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size); + +void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size); + +/// @brief Free a pointer with a known constant size. +/// @param p The pointer to the allocated block (or \a NULL ) +/// @param size The (compile time constant) size with which the pointer was allocated. +/// @see mi_malloc_csize() +/// @see mi_free_size() +void mi_free_csize(void* p, size_t size); + +/// \} + /// \defgroup zeroinit Zero initialized re-allocation /// Re-allocation of zero initialized blocks. /// @@ -1006,13 +1054,14 @@ void* mi_malloc_small(size_t size); /// with care! void* mi_zalloc_small(size_t size); -/// __v3__: Can be used to free an object that was allocated with #mi_malloc_small et al. +/// __v3__: Can be used to free an object that was allocated with mi_malloc_small() or +/// any allocation with a size < MI_SMALL_SIZE_MAX() /// @param p Pointer that was returned from #mi_malloc_small et al. /// @details /// This function is meant for use in run-time systems for best /// performance and does not check if the pointer was _indeed_ allocated -/// with #mi_malloc_small (et al) -- use with care! -/// This function has a small perf benefit but only if mimalloc was built with `MI_PAGE_META_ALIGNED_FREE_SMALL=1` +/// as a small object -- use with care! +/// /// @see mi_malloc_small() /// @see mi_zalloc_small() /// @see mi_heap_malloc_small() @@ -1357,8 +1406,9 @@ void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize); /// /// \{ -/// Just as `free` but also checks if the pointer `p` belongs to our heap. -void mi_cfree(void* p); +/// Checked `free`: just as `mi_free` but always checks if the pointer `p` belongs to our heap +/// and is safe to use when a pointer might be invalid or allocated by another allocator. +void mi_cfree(void* p); void* mi__expand(void* p, size_t newsize); size_t mi_malloc_size(const void* p); @@ -1387,8 +1437,22 @@ int mi_reallocarr(void* p, size_t count, size_t size); void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment); void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset); +/// @brief Free an object that was allocated with a known size. +/// @param p The pointer that was allocated with \a size bytes (or \a NULL ) +/// @param size The size in bytes. +/// __v3__: this can improve performance for objects with a \size < MI_SMALL_SIZE_MAX() +/// as it will use mi_free_small() internally. Always use this if possible. +/// @see mi_free_tp() +/// @see mi_free_csize() void mi_free_size(void* p, size_t size); + +/// @brief Free an object that was allocated aligned. +/// @param p The pointer to the object (or \a NULL ) +/// @param size The size of the object as allocated. +/// @param alignment The requested alignment at the allocation. +/// Currently just defers to mi_free_size() void mi_free_size_aligned(void* p, size_t size, size_t alignment); + void mi_free_aligned(void* p, size_t alignment); /// \} @@ -1670,7 +1734,7 @@ Further options for large workloads and services: at runtime. Setting `N` to 1 may avoid problems in some virtual environments. Also, setting it to a lower number than the actual NUMA nodes is fine and will only cause threads to potentially allocate more memory across actual NUMA nodes (but this can happen in any case as NUMA local allocation is always a best effort but not guaranteed). -- `MIMALLOC_ALLOW_LARGE_OS_PAGES=0`: Set to 1 to use large OS pages (2 or 4MiB) when available; for some workloads this can +- `MIMALLOC_ALLOW_LARGE_OS_PAGES=0`: Set to 1 to use large OS pages (usually 2MiB) when available; for some workloads this can significantly improve performance. However, large OS pages cannot be purged or shared with other processes so may lead to increased memory usage in some cases. Use `MIMALLOC_VERBOSE` to check if the large OS pages are enabled -- usually one needs diff --git a/docs/annotated.html b/docs/annotated.html index 6012e64d0..87d0fb2f6 100644 --- a/docs/annotated.html +++ b/docs/annotated.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/bench.html b/docs/bench.html index 31b5f3bc3..33217d900 100644 --- a/docs/bench.html +++ b/docs/bench.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/build.html b/docs/build.html index b1a728868..78e1cc335 100644 --- a/docs/build.html +++ b/docs/build.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/classes.html b/docs/classes.html index 20d71a10f..acb1e25d5 100644 --- a/docs/classes.html +++ b/docs/classes.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/doxygen_crawl.html b/docs/doxygen_crawl.html index dbae7ec7b..aeea689b4 100644 --- a/docs/doxygen_crawl.html +++ b/docs/doxygen_crawl.html @@ -19,6 +19,7 @@ + @@ -94,6 +95,12 @@ + + + + + + @@ -316,6 +323,7 @@ + diff --git a/docs/environment.html b/docs/environment.html index 28a440993..4aafebc69 100644 --- a/docs/environment.html +++ b/docs/environment.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
@@ -123,7 +123,7 @@

Use caution when using fork in combination with either large or huge OS pages: on a fork, the OS uses copy-on-write for all pages in the original process including the huge OS pages. When any memory is now written in that area, the OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in large increments.

diff --git a/docs/functions.html b/docs/functions.html index 5cfd9d86d..ff9b06858 100644 --- a/docs/functions.html +++ b/docs/functions.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/functions_vars.html b/docs/functions_vars.html index 8d199a936..caa47689a 100644 --- a/docs/functions_vars.html +++ b/docs/functions_vars.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__aligned.html b/docs/group__aligned.html index 68e9614b4..3de1f8731 100644 --- a/docs/group__aligned.html +++ b/docs/group__aligned.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__analysis.html b/docs/group__analysis.html index c63be46a9..62d1187be 100644 --- a/docs/group__analysis.html +++ b/docs/group__analysis.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__arenas.html b/docs/group__arenas.html index f697bde8d..b8fef1965 100644 --- a/docs/group__arenas.html +++ b/docs/group__arenas.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__constantsize.html b/docs/group__constantsize.html new file mode 100644 index 000000000..dc7e32ef6 --- /dev/null +++ b/docs/group__constantsize.html @@ -0,0 +1,267 @@ + + + + + + + +mi-malloc: Constant size allocation + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + +
+
mi-malloc 3.4/2.4/1.9 +
+
+ +   + + + + +
+
+
+ + + + +
+
+ +
+
+
+ +
+ +
+
+ + +
+
+
+
+
+
Loading...
+
Searching...
+
No Matches
+
+
+
+
+ +
+
+Functions
+
Constant size allocation
+
+
+ +

Constant size allocation and freeing For example: +More...

+ + + + + + + + + + + + + + +

+Functions

void * mi_malloc_csize (size_t size)
 Allocate a constant size object.
 
void * mi_zalloc_csize (size_t size)
 
void * mi_theap_malloc_csize (mi_theap_t *theap, size_t size)
 
void * mi_theap_zalloc_csize (mi_theap_t *theap, size_t size)
 
void mi_free_csize (void *p, size_t size)
 Free a pointer with a known constant size.
 
+

Detailed Description

+

Constant size allocation and freeing For example:

+
long* p = (long*)mi_malloc_csize(sizeof(long))
+
mi_malloc_csize
void * mi_malloc_csize(size_t size)
Allocate a constant size object.
+

On v3 these can improve performance as they will statically use mi_malloc_small(), mi_free_small(), etc. when possible. This is especially useful for compilers and runtimes where the the size is often statically known. These functions can be used even when the size is not constant but then there is no performance benefit.

+

Function Documentation

+ +

◆ mi_free_csize()

+ +
+
+ + + + + + + + + + + +
void mi_free_csize (void * p,
size_t size )
+
+ +

Free a pointer with a known constant size.

+
Parameters
+ + + +
pThe pointer to the allocated block (or NULL )
sizeThe (compile time constant) size with which the pointer was allocated.
+
+
+
See also
mi_malloc_csize()
+
+mi_free_size()
+ +
+
+ +

◆ mi_malloc_csize()

+ +
+
+ + + + + + + +
void * mi_malloc_csize (size_t size)
+
+ +

Allocate a constant size object.

+
Parameters
+ + +
sizeA (compile-time) constant size.
+
+
+
Returns
A pointer to an allocated block of size size bytes.
+
See also
mi_free_csize()
+
+mi_malloc()
+ +
+
+ +

◆ mi_theap_malloc_csize()

+ +
+
+ + + + + + + + + + + +
void * mi_theap_malloc_csize (mi_theap_t * theap,
size_t size )
+
+ +
+
+ +

◆ mi_theap_zalloc_csize()

+ +
+
+ + + + + + + + + + + +
void * mi_theap_zalloc_csize (mi_theap_t * theap,
size_t size )
+
+ +
+
+ +

◆ mi_zalloc_csize()

+ +
+
+ + + + + + + +
void * mi_zalloc_csize (size_t size)
+
+ +
+
+
+
+ + + + diff --git a/docs/group__constantsize.js b/docs/group__constantsize.js new file mode 100644 index 000000000..bfaeb70ff --- /dev/null +++ b/docs/group__constantsize.js @@ -0,0 +1,8 @@ +var group__constantsize = +[ + [ "mi_free_csize", "group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5", null ], + [ "mi_malloc_csize", "group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571", null ], + [ "mi_theap_malloc_csize", "group__constantsize.html#ga105bd667da91a15d113fe79be04d625c", null ], + [ "mi_theap_zalloc_csize", "group__constantsize.html#ga1addf755829380581dd9c3c9175e376f", null ], + [ "mi_zalloc_csize", "group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec", null ] +]; \ No newline at end of file diff --git a/docs/group__cpp.html b/docs/group__cpp.html index 33dbbd256..42c217152 100644 --- a/docs/group__cpp.html +++ b/docs/group__cpp.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__extended.html b/docs/group__extended.html index 325588c18..e4eb5ef4b 100644 --- a/docs/group__extended.html +++ b/docs/group__extended.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
@@ -169,7 +169,7 @@  Allocate a zero initialized small object.
  void mi_free_small (void *p) - v3: Can be used to free an object that was allocated with mi_malloc_small et al.
+ v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()
 

Detailed Description

@@ -307,14 +307,15 @@

-

v3: Can be used to free an object that was allocated with mi_malloc_small et al.

+

v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()

Parameters
pPointer that was returned from mi_malloc_small et al.
-

This function is meant for use in run-time systems for best performance and does not check if the pointer was indeed allocated with mi_malloc_small (et al) – use with care! This function has a small perf benefit but only if mimalloc was built with MI_FAST_FREE_SMALL=1

See also
mi_malloc_small()
+

This function is meant for use in run-time systems for best performance and does not check if the pointer was indeed allocated as a small object – use with care!

+
See also
mi_malloc_small()
mi_zalloc_small()
diff --git a/docs/group__heap.html b/docs/group__heap.html index c0ba18bb0..50a85dc5c 100644 --- a/docs/group__heap.html +++ b/docs/group__heap.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__malloc.html b/docs/group__malloc.html index 1038f798d..6b7796979 100644 --- a/docs/group__malloc.html +++ b/docs/group__malloc.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__options.html b/docs/group__options.html index c8c4bfb31..2824760be 100644 --- a/docs/group__options.html +++ b/docs/group__options.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__posix.html b/docs/group__posix.html index 238492f0c..96a40e751 100644 --- a/docs/group__posix.html +++ b/docs/group__posix.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
@@ -116,7 +116,7 @@

Functions

void mi_cfree (void *p) - Just as free but also checks if the pointer p belongs to our heap.
+ Checked free: just as mi_free but always checks if the pointer p belongs to our heap and is safe to use when a pointer might be invalid or allocated by another allocator.
  void * mi__expand (void *p, size_t newsize)   @@ -157,8 +157,10 @@ void * mi_aligned_offset_recalloc (void *p, size_t newcount, size_t size, size_t alignment, size_t offset)   void mi_free_size (void *p, size_t size) + Free an object that was allocated with a known size.
  void mi_free_size_aligned (void *p, size_t size, size_t alignment) + Free an object that was allocated aligned.
  void mi_free_aligned (void *p, size_t alignment)   @@ -316,7 +318,7 @@

-

Just as free but also checks if the pointer p belongs to our heap.

+

Checked free: just as mi_free but always checks if the pointer p belongs to our heap and is safe to use when a pointer might be invalid or allocated by another allocator.

@@ -386,6 +388,18 @@

+

Free an object that was allocated with a known size.

+
Parameters
+ + + +
pThe pointer that was allocated with size bytes (or NULL )
sizeThe size in bytes. v3: this can improve performance for objects with a \size < MI_SMALL_SIZE_MAX() as it will use mi_free_small() internally. Always use this if possible.
+
+
+
See also
mi_free_tp()
+
+mi_free_csize()
+

@@ -412,6 +426,16 @@

+

Free an object that was allocated aligned.

+
Parameters
+ + + + +
pThe pointer to the object (or NULL )
sizeThe size of the object as allocated.
alignmentThe requested alignment at the allocation. Currently just defers to mi_free_size()
+
+
+

diff --git a/docs/group__stats.html b/docs/group__stats.html index bfb8127cc..529d18d89 100644 --- a/docs/group__stats.html +++ b/docs/group__stats.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
@@ -260,7 +260,7 @@

if (mi_stats_get(&stats)) {

...
}
-
mi_stats_t_decl
#define mi_stats_t_decl(name)
Helper to declare a properly initialized mi_stats_t() local variable as name where the size and versi...
Definition mimalloc-doc.h:1050
+
mi_stats_t_decl
#define mi_stats_t_decl(name)
Helper to declare a properly initialized mi_stats_t() local variable as name where the size and versi...
Definition mimalloc-doc.h:1099
mi_stats_get
bool mi_stats_get(mi_stats_t *stats)
Get the statistics for the current subprocess aggregated over all its heaps.
diff --git a/docs/group__subproc.html b/docs/group__subproc.html index 8812b8776..f3830c87f 100644 --- a/docs/group__subproc.html +++ b/docs/group__subproc.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__theap.html b/docs/group__theap.html index ffd316aad..e8abeb2c7 100644 --- a/docs/group__theap.html +++ b/docs/group__theap.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/group__typed.html b/docs/group__typed.html index 54deadc2d..bbd77cafb 100644 --- a/docs/group__typed.html +++ b/docs/group__typed.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
@@ -118,6 +118,9 @@ #define mi_malloc_tp(tp)  Allocate a block of type tp.
  +#define mi_free_tp(tp, p) + Free an object of type tp.
+  #define mi_zalloc_tp(tp)  Allocate a zero-initialized block of type tp.
  @@ -152,8 +155,9 @@

Detailed Description

Typed allocation macros.

For example:

int* p = mi_malloc_tp(int)
-
mi_malloc_tp
#define mi_malloc_tp(tp)
Allocate a block of type tp.
Definition mimalloc-doc.h:359
-

Macro Definition Documentation

+
mi_malloc_tp
#define mi_malloc_tp(tp)
Allocate a block of type tp.
Definition mimalloc-doc.h:364
+

On v3 these can improve performance as they use the mi_<malloc>_csize api's that in turn can use mi_malloc_small(), mi_free_small(), etc.

+

Macro Definition Documentation

◆ mi_calloc_tp

@@ -175,6 +179,39 @@

+

◆ mi_free_tp

+ +
+
+ + + + + + + + + + + +
#define mi_free_tp( tp,
p )
+
+ +

Free an object of type tp.

+
Parameters
+ + + +
tpThe type of the object that is freed.
pA pointer to an object of type tp (or NULL ) Can be more efficient as it internally uses mi_free_csize()
+
+
+
See also
mi_free_csize()
+
+mi_free_small()
+
@@ -369,7 +406,9 @@

Returns
A pointer to an object of type tp, or NULL if out of memory.

Example:

int* p = mi_malloc_tp(int)
-
See also
mi_malloc()
+

Using this is more safe, but can also be more efficient since the allocation size is a compile-time constant.

See also
mi_malloc_csize()
+
+mi_malloc_small()
diff --git a/docs/group__typed.js b/docs/group__typed.js index 61cdfda63..2207e4b55 100644 --- a/docs/group__typed.js +++ b/docs/group__typed.js @@ -1,6 +1,7 @@ var group__typed = [ [ "mi_calloc_tp", "group__typed.html#gae80c47c9d4cab10961fff1a8ac98fc07", null ], + [ "mi_free_tp", "group__typed.html#gafc80000c300f02bdd4fe695eb53ca70a", null ], [ "mi_heap_calloc_tp", "group__typed.html#ga953f22101bb354a8b6de2a6d6fda9b98", null ], [ "mi_heap_malloc_tp", "group__typed.html#ga02d3e85bf26aa3120132f2552c408a88", null ], [ "mi_heap_mallocn_tp", "group__typed.html#gaa58690658cc6f5dd54c99a9971d4aa7d", null ], diff --git a/docs/group__zeroinit.html b/docs/group__zeroinit.html index a10f3a92c..ecfa05d49 100644 --- a/docs/group__zeroinit.html +++ b/docs/group__zeroinit.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
diff --git a/docs/index.html b/docs/index.html index c266b86e4..25354e879 100644 --- a/docs/index.html +++ b/docs/index.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 1.8/2.1 +
mi-malloc 3.4/2.4/1.9
@@ -122,9 +122,9 @@

There are three maintained versions of mimalloc. All versions are mostly equal except for how the OS memory is handled. New development is mostly on v3, while v1 and v2 are maintained with security and bug fixes.

    -
  • v1: initial design of mimalloc (release tags: v1.9.x, development branch dev). Send PR's against this version if possible.
    • -
  • v2: main mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: v2.2.x, development branch dev2)
    • -
  • v3: simplifies the lock-free design of previous versions, and improves sharing of memory between threads. On certain large workloads this version may use (much) less memory. Also supports true first-class heaps (that can allocate from any thread) and has more efficient heap-walking (for the CPython GC for example). (release tags: v3.2.x, development branch dev3).
    • +
  • v3 recommended: simplifies the lock-free design of previous versions, and improves sharing of memory between threads. On certain large workloads this version may use (much) less memory. Also supports true first-class heaps (that can allocate from any thread) and has more efficient heap-walking (for the CPython GC for example). (release tags: v3.4.x, development branch dev3).
    • +
  • v2 stable: Uses thread-local segments to reduce fragmentation. (release tags: v2.4.x, development branch dev2)
    • +
  • v1 legacy: initial design of mimalloc (release tags: v1.9.x, development branch dev). Send PR's against this version if possible.

You can read more on the design of mimalloc in the technical report which also has detailed benchmark results. To learn more about the internal implementation of mimalloc, see include/mimalloc/types.h.

Further information:

@@ -142,12 +142,14 @@
  • Basic Allocation
  • Aligned Allocation
  • Typed Macros
    • +
  • Constant size allocation
  • Zero initialized re-allocation
  • Heaps
  • Heap Introspection
  • Arenas
  • Subprocesses
  • Extended Functions
    • +
  • Statistics
  • Runtime Options
  • Thread-local heaps
  • Posix
    • diff --git a/docs/mimalloc-doc_8h_source.html b/docs/mimalloc-doc_8h_source.html index e7eea70b9..37d74e056 100644 --- a/docs/mimalloc-doc_8h_source.html +++ b/docs/mimalloc-doc_8h_source.html @@ -34,7 +34,7 @@ Logo -
    mi-malloc 1.8/2.1 +
    mi-malloc 3.4/2.4/1.9
    @@ -108,7 +108,7 @@
    1/* ----------------------------------------------------------------------------
    -
    2Copyright (c) 2018-2025, Microsoft Research, Daan Leijen
    +
    2Copyright (c) 2018-2026, Microsoft Research, Daan Leijen
    3This is free software; you can redistribute it and/or modify it under the
    4terms of the MIT license. A copy of the license can be found in the file
    5"LICENSE" at the root of this distribution.
    @@ -117,517 +117,533 @@
    8#error "documentation file only!"
    9
    10
    -
    121
    -
    122
    -
    126void mi_free(void* p);
    -
    127
    -
    132void* mi_malloc(size_t size);
    -
    133
    -
    138void* mi_zalloc(size_t size);
    -
    139
    -
    149void* mi_calloc(size_t count, size_t size);
    -
    150
    -
    164void* mi_realloc(void* p, size_t newsize);
    -
    165
    -
    179void* mi_expand(void* p, size_t newsize);
    -
    180
    -
    190void* mi_mallocn(size_t count, size_t size);
    -
    191
    -
    201void* mi_reallocn(void* p, size_t count, size_t size);
    -
    202
    -
    219void* mi_reallocf(void* p, size_t newsize);
    -
    220
    -
    221
    -
    230char* mi_strdup(const char* s);
    -
    231
    -
    241char* mi_strndup(const char* s, size_t n);
    -
    242
    -
    255char* mi_realpath(const char* fname, char* resolved_name);
    -
    256
    -
    270size_t mi_usable_size(void* p);
    -
    271
    -
    281size_t mi_good_size(size_t size);
    -
    282
    +
    123
    +
    124
    +
    128void mi_free(void* p);
    +
    129
    +
    134void* mi_malloc(size_t size);
    +
    135
    +
    140void* mi_zalloc(size_t size);
    +
    141
    +
    151void* mi_calloc(size_t count, size_t size);
    +
    152
    +
    166void* mi_realloc(void* p, size_t newsize);
    +
    167
    +
    181void* mi_expand(void* p, size_t newsize);
    +
    182
    +
    192void* mi_mallocn(size_t count, size_t size);
    +
    193
    +
    203void* mi_reallocn(void* p, size_t count, size_t size);
    +
    204
    +
    221void* mi_reallocf(void* p, size_t newsize);
    +
    222
    +
    223
    +
    232char* mi_strdup(const char* s);
    +
    233
    +
    243char* mi_strndup(const char* s, size_t n);
    +
    244
    +
    257char* mi_realpath(const char* fname, char* resolved_name);
    +
    258
    +
    272size_t mi_usable_size(void* p);
    +
    273
    +
    283size_t mi_good_size(size_t size);
    284
    -
    285
    -
    286// ------------------------------------------------------
    -
    287// Aligned allocation
    +
    286
    +
    287
    288// ------------------------------------------------------
    -
    289
    -
    297
    -
    315void* mi_malloc_aligned(size_t size, size_t alignment);
    -
    316void* mi_zalloc_aligned(size_t size, size_t alignment);
    -
    317void* mi_calloc_aligned(size_t count, size_t size, size_t alignment);
    -
    318void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment);
    -
    319
    -
    331void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset);
    -
    332void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset);
    -
    333void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset);
    -
    334void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
    -
    335
    +
    289// Aligned allocation
    +
    290// ------------------------------------------------------
    +
    291
    +
    299
    +
    317void* mi_malloc_aligned(size_t size, size_t alignment);
    +
    318void* mi_zalloc_aligned(size_t size, size_t alignment);
    +
    319void* mi_calloc_aligned(size_t count, size_t size, size_t alignment);
    +
    320void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment);
    +
    321
    +
    333void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset);
    +
    334void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset);
    +
    335void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset);
    +
    336void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
    337
    -
    338
    -
    347
    -
    359#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp)))
    -
    360
    -
    362#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp)))
    -
    363
    -
    365#define mi_calloc_tp(tp,count) ((tp*)mi_calloc(count,sizeof(tp)))
    -
    366
    -
    368#define mi_mallocn_tp(tp,count) ((tp*)mi_mallocn(count,sizeof(tp)))
    -
    369
    -
    371#define mi_reallocn_tp(p,tp,count) ((tp*)mi_reallocn(p,count,sizeof(tp)))
    -
    372
    -
    374#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
    -
    375
    -
    377#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
    -
    378
    -
    380#define mi_heap_calloc_tp(tp,hp,count) ((tp*)mi_heap_calloc(hp,count,sizeof(tp)))
    -
    381
    -
    383#define mi_heap_mallocn_tp(tp,hp,count) ((tp*)mi_heap_mallocn(hp,count,sizeof(tp)))
    -
    384
    -
    386#define mi_heap_reallocn_tp(tp,hp,p,count) ((tp*)mi_heap_reallocn(p,count,sizeof(tp)))
    -
    387
    -
    389#define mi_heap_recalloc_tp(tp,hp,p,count) ((tp*)mi_heap_recalloc(p,count,sizeof(tp)))
    -
    390
    -
    392
    -
    402
    -
    413void* mi_recalloc(void* p, size_t count, size_t size);
    -
    414
    -
    415void* mi_rezalloc(void* p, size_t newsize);
    -
    416void* mi_recalloc(void* p, size_t newcount, size_t size) ;
    -
    417
    -
    418void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment);
    -
    419void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
    -
    420void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment);
    -
    421void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    -
    422
    -
    423void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize);
    -
    424void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size);
    +
    339
    +
    340
    +
    350
    +
    364#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp)))
    +
    365
    +
    372#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp)))
    +
    373
    +
    375#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp)))
    +
    376
    +
    378#define mi_calloc_tp(tp,count) ((tp*)mi_calloc(count,sizeof(tp)))
    +
    379
    +
    381#define mi_mallocn_tp(tp,count) ((tp*)mi_mallocn(count,sizeof(tp)))
    +
    382
    +
    384#define mi_reallocn_tp(p,tp,count) ((tp*)mi_reallocn(p,count,sizeof(tp)))
    +
    385
    +
    387#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
    +
    388
    +
    390#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
    +
    391
    +
    393#define mi_heap_calloc_tp(tp,hp,count) ((tp*)mi_heap_calloc(hp,count,sizeof(tp)))
    +
    394
    +
    396#define mi_heap_mallocn_tp(tp,hp,count) ((tp*)mi_heap_mallocn(hp,count,sizeof(tp)))
    +
    397
    +
    399#define mi_heap_reallocn_tp(tp,hp,p,count) ((tp*)mi_heap_reallocn(p,count,sizeof(tp)))
    +
    400
    +
    402#define mi_heap_recalloc_tp(tp,hp,p,count) ((tp*)mi_heap_recalloc(p,count,sizeof(tp)))
    +
    403
    +
    405
    +
    418
    +
    424void* mi_malloc_csize(size_t size);
    425
    -
    426void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
    -
    427void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
    -
    428void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment);
    -
    429void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    -
    430
    -
    432
    -
    433
    -
    434
    -
    452
    -
    458struct mi_heap_s;
    -
    459
    -
    465typedef struct mi_heap_s mi_heap_t;
    -
    466
    -
    468mi_heap_t* mi_heap_new();
    -
    469
    -
    476void mi_heap_delete(mi_heap_t* heap);
    -
    477
    -
    484void mi_heap_destroy(mi_heap_t* heap);
    -
    485
    -
    490mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
    -
    491
    -
    495mi_heap_t* mi_heap_get_default();
    -
    496
    -
    503mi_heap_t* mi_heap_get_backing();
    -
    504
    -
    508void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
    -
    509
    -
    515mi_heap_t* mi_heap_main(void);
    -
    516
    -
    522mi_heap_t* mi_heap_of(const void* p);
    -
    523
    -
    528bool mi_heap_contains(const mi_heap_t* heap, const void* p);
    -
    529
    -
    533bool mi_any_heap_contains(const void* p);
    -
    534
    -
    535
    -
    541bool mi_is_in_heap_region(const void* p);
    -
    542
    -
    548bool mi_check_owned(const void* p);
    -
    549
    -
    556bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
    +
    426void* mi_zalloc_csize(size_t size);
    +
    427
    +
    428void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size);
    +
    429
    +
    430void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size);
    +
    431
    +
    437void mi_free_csize(void* p, size_t size);
    +
    438
    +
    440
    +
    450
    +
    461void* mi_recalloc(void* p, size_t count, size_t size);
    +
    462
    +
    463void* mi_rezalloc(void* p, size_t newsize);
    +
    464void* mi_recalloc(void* p, size_t newcount, size_t size) ;
    +
    465
    +
    466void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment);
    +
    467void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
    +
    468void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment);
    +
    469void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    +
    470
    +
    471void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize);
    +
    472void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size);
    +
    473
    +
    474void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
    +
    475void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
    +
    476void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment);
    +
    477void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    +
    478
    +
    480
    +
    481
    +
    482
    +
    500
    +
    506struct mi_heap_s;
    +
    507
    +
    513typedef struct mi_heap_s mi_heap_t;
    +
    514
    +
    516mi_heap_t* mi_heap_new();
    +
    517
    +
    524void mi_heap_delete(mi_heap_t* heap);
    +
    525
    +
    532void mi_heap_destroy(mi_heap_t* heap);
    +
    533
    +
    538mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
    +
    539
    +
    543mi_heap_t* mi_heap_get_default();
    +
    544
    +
    551mi_heap_t* mi_heap_get_backing();
    +
    552
    +
    556void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
    557
    -
    564bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
    -
    565
    -
    567void mi_heap_collect(mi_heap_t* heap, bool force);
    -
    568
    -
    571void* mi_heap_malloc(mi_heap_t* heap, size_t size);
    -
    572
    -
    576void* mi_heap_malloc_small(mi_heap_t* heap, size_t size);
    +
    563mi_heap_t* mi_heap_main(void);
    +
    564
    +
    570mi_heap_t* mi_heap_of(const void* p);
    +
    571
    +
    576bool mi_heap_contains(const mi_heap_t* heap, const void* p);
    577
    -
    580void* mi_heap_zalloc(mi_heap_t* heap, size_t size);
    -
    581
    -
    585void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size);
    -
    586
    -
    589void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size);
    +
    581bool mi_any_heap_contains(const void* p);
    +
    582
    +
    583
    +
    589bool mi_is_in_heap_region(const void* p);
    590
    -
    593void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size);
    -
    594
    -
    597char* mi_heap_strdup(mi_heap_t* heap, const char* s);
    -
    598
    -
    601char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n);
    -
    602
    -
    605char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name);
    -
    606
    -
    607void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize);
    -
    608void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size);
    -
    609void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize);
    -
    610
    -
    611void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
    -
    612void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
    -
    613void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
    -
    614void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
    -
    615void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment);
    -
    616void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset);
    -
    617void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
    -
    618void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
    -
    619
    -
    621
    -
    622
    -
    623
    -
    628
    +
    596bool mi_check_owned(const void* p);
    +
    597
    +
    604bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
    +
    605
    +
    612bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
    +
    613
    +
    615void mi_heap_collect(mi_heap_t* heap, bool force);
    +
    616
    +
    619void* mi_heap_malloc(mi_heap_t* heap, size_t size);
    +
    620
    +
    624void* mi_heap_malloc_small(mi_heap_t* heap, size_t size);
    +
    625
    +
    628void* mi_heap_zalloc(mi_heap_t* heap, size_t size);
    629
    -
    -
    632typedef struct mi_heap_area_s {
    -
    633 void* blocks;
    -
    634 size_t reserved;
    -
    635 size_t committed;
    -
    636 size_t used;
    -
    637 size_t block_size;
    -
    638 size_t full_block_size;
    -
    639 int heap_tag;
    -
    640} mi_heap_area_t;
    -
    -
    641
    -
    649typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
    +
    633void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size);
    +
    634
    +
    637void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size);
    +
    638
    +
    641void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size);
    +
    642
    +
    645char* mi_heap_strdup(mi_heap_t* heap, const char* s);
    +
    646
    +
    649char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n);
    650
    -
    665bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
    -
    666
    -
    682bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
    -
    683
    -
    685
    -
    686// ------------------------------------------------------
    -
    687// Arenas
    -
    688// ------------------------------------------------------
    +
    653char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name);
    +
    654
    +
    655void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize);
    +
    656void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size);
    +
    657void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize);
    +
    658
    +
    659void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
    +
    660void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
    +
    661void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
    +
    662void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
    +
    663void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment);
    +
    664void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset);
    +
    665void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
    +
    666void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
    +
    667
    +
    669
    +
    670
    +
    671
    +
    676
    +
    677
    +
    +
    680typedef struct mi_heap_area_s {
    +
    681 void* blocks;
    +
    682 size_t reserved;
    +
    683 size_t committed;
    +
    684 size_t used;
    +
    685 size_t block_size;
    +
    686 size_t full_block_size;
    +
    687 int heap_tag;
    +
    688} mi_heap_area_t;
    +
    689
    -
    697
    -
    699typedef int mi_arena_id_t;
    -
    700
    -
    709int mi_reserve_os_memory(size_t size, bool commit, bool allow_large);
    -
    710
    -
    718int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);
    -
    719
    -
    732int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs);
    +
    697typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
    +
    698
    +
    713bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
    +
    714
    +
    730bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
    +
    731
    733
    -
    746int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs);
    -
    747
    -
    755int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id);
    -
    756
    -
    759size_t mi_arena_min_alignment(void);
    -
    760
    -
    772bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node);
    -
    773
    -
    784bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id);
    -
    785
    -
    786
    -
    788void mi_debug_show_arenas(void);
    -
    789
    -
    794void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
    +
    734// ------------------------------------------------------
    +
    735// Arenas
    +
    736// ------------------------------------------------------
    +
    737
    +
    745
    +
    747typedef int mi_arena_id_t;
    +
    748
    +
    757int mi_reserve_os_memory(size_t size, bool commit, bool allow_large);
    +
    758
    +
    766int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);
    +
    767
    +
    780int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs);
    +
    781
    +
    794int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs);
    795
    -
    796
    -
    797
    -
    801mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
    -
    802
    -
    814mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id);
    -
    815
    -
    817
    -
    818
    -
    827
    -
    832typedef void* mi_subproc_id_t;
    +
    803int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id);
    +
    804
    +
    807size_t mi_arena_min_alignment(void);
    +
    808
    +
    820bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node);
    +
    821
    +
    832bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id);
    833
    -
    835mi_subproc_id_t mi_subproc_main(void);
    -
    836
    -
    838mi_subproc_id_t mi_subproc_current(void);
    -
    839
    -
    842mi_subproc_id_t mi_subproc_new(void);
    +
    834
    +
    836void mi_debug_show_arenas(void);
    +
    837
    +
    842void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
    843
    -
    847void mi_subproc_destroy(mi_subproc_id_t subproc);
    -
    848
    -
    853void mi_subproc_delete(mi_subproc_id_t subproc);
    -
    854
    -
    858void mi_subproc_add_current_thread(mi_subproc_id_t subproc);
    -
    859
    -
    863typedef bool (mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
    -
    864
    -
    870bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
    -
    871
    -
    873
    -
    874
    -
    875// ------------------------------------------------------
    -
    876// Extended functionality
    -
    877// ------------------------------------------------------
    -
    878
    -
    883
    -
    887#define MI_SMALL_SIZE_MAX (128*sizeof(void*))
    -
    888
    -
    892int mi_version(void);
    -
    893
    -
    901void mi_collect(bool force);
    +
    844
    +
    845
    +
    849mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
    +
    850
    +
    862mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id);
    +
    863
    +
    865
    +
    866
    +
    875
    +
    880typedef void* mi_subproc_id_t;
    +
    881
    +
    883mi_subproc_id_t mi_subproc_main(void);
    +
    884
    +
    886mi_subproc_id_t mi_subproc_current(void);
    +
    887
    +
    890mi_subproc_id_t mi_subproc_new(void);
    +
    891
    +
    895void mi_subproc_destroy(mi_subproc_id_t subproc);
    +
    896
    +
    901void mi_subproc_delete(mi_subproc_id_t subproc);
    902
    -
    908void mi_thread_set_in_threadpool(void);
    -
    909
    -
    914bool mi_is_redirected();
    -
    915
    -
    919void mi_thread_init(void);
    -
    920
    -
    925void mi_thread_done(void);
    +
    906void mi_subproc_add_current_thread(mi_subproc_id_t subproc);
    +
    907
    +
    911typedef bool (mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
    +
    912
    +
    918bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
    +
    919
    +
    921
    +
    922
    +
    923// ------------------------------------------------------
    +
    924// Extended functionality
    +
    925// ------------------------------------------------------
    926
    -
    933typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
    -
    934
    -
    950void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg);
    -
    951
    -
    957typedef void (mi_output_fun)(const char* msg, void* arg);
    -
    958
    -
    965void mi_register_output(mi_output_fun* out, void* arg);
    -
    966
    -
    972typedef void (mi_error_fun)(int err, void* arg);
    -
    973
    -
    989void mi_register_error(mi_error_fun* errfun, void* arg);
    -
    990
    -
    998void* mi_malloc_small(size_t size);
    +
    931
    +
    935#define MI_SMALL_SIZE_MAX (128*sizeof(void*))
    +
    936
    +
    940int mi_version(void);
    +
    941
    +
    949void mi_collect(bool force);
    +
    950
    +
    956void mi_thread_set_in_threadpool(void);
    +
    957
    +
    962bool mi_is_redirected();
    +
    963
    +
    967void mi_thread_init(void);
    +
    968
    +
    973void mi_thread_done(void);
    +
    974
    +
    981typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
    +
    982
    +
    998void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg);
    999
    -
    1007void* mi_zalloc_small(size_t size);
    -
    1008
    -
    1022void mi_free_small(void* p);
    -
    1023
    -
    1025
    -
    1026
    -
    1027
    -
    1028
    -
    1029
    -
    1030// ------------------------------------------------------
    -
    1031// Statistics
    -
    1032// ------------------------------------------------------
    -
    1033
    -
    1037
    -
    1039#define MI_STAT_VERSION 4
    -
    1040
    -
    1050#define mi_stats_t_decl(name)
    -
    1051
    -
    -
    1053struct mi_stats_s {
    -
    1055 size_t size;
    -
    1057 size_t version;
    -
    1058};
    -
    -
    1059
    -
    1061typedef struct mi_stats_s mi_stats_t;
    -
    1062
    -
    1063
    -
    1066void mi_process_info_print(void);
    -
    1067
    -
    1072void mi_process_info_print_out(mi_output_fun* out, void* arg);
    -
    1073
    -
    1087void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults);
    -
    1088
    +
    1005typedef void (mi_output_fun)(const char* msg, void* arg);
    +
    1006
    +
    1013void mi_register_output(mi_output_fun* out, void* arg);
    +
    1014
    +
    1020typedef void (mi_error_fun)(int err, void* arg);
    +
    1021
    +
    1037void mi_register_error(mi_error_fun* errfun, void* arg);
    +
    1038
    +
    1046void* mi_malloc_small(size_t size);
    +
    1047
    +
    1055void* mi_zalloc_small(size_t size);
    +
    1056
    +
    1071void mi_free_small(void* p);
    +
    1072
    +
    1074
    +
    1075
    +
    1076
    +
    1077
    +
    1078
    +
    1079// ------------------------------------------------------
    +
    1080// Statistics
    +
    1081// ------------------------------------------------------
    +
    1082
    +
    1086
    +
    1088#define MI_STAT_VERSION 4
    1089
    -
    1094void mi_stats_print(void* out);
    -
    1095
    -
    1101void mi_stats_print_out(mi_output_fun* out, void* arg);
    -
    1102
    -
    1108bool mi_stats_get(mi_stats_t* stats);
    -
    1109
    -
    1115char* mi_stats_get_json(size_t buf_size, char* buf);
    +
    1099#define mi_stats_t_decl(name)
    +
    1100
    +
    +
    1102struct mi_stats_s {
    +
    1104 size_t size;
    +
    1106 size_t version;
    +
    1107};
    +
    +
    1108
    +
    1110typedef struct mi_stats_s mi_stats_t;
    +
    1111
    +
    1112
    +
    1115void mi_process_info_print(void);
    1116
    -
    1118size_t mi_stats_get_bin_size(size_t bin);
    -
    1119
    -
    1126bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats);
    -
    1127
    -
    1134char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
    -
    1135
    -
    1140void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg);
    -
    1141
    -
    1145void mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
    -
    1146
    -
    1153bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats);
    -
    1154
    -
    1161char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
    -
    1162
    -
    1167void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
    +
    1121void mi_process_info_print_out(mi_output_fun* out, void* arg);
    +
    1122
    +
    1136void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults);
    +
    1137
    +
    1138
    +
    1143void mi_stats_print(void* out);
    +
    1144
    +
    1150void mi_stats_print_out(mi_output_fun* out, void* arg);
    +
    1151
    +
    1157bool mi_stats_get(mi_stats_t* stats);
    +
    1158
    +
    1164char* mi_stats_get_json(size_t buf_size, char* buf);
    +
    1165
    +
    1167size_t mi_stats_get_bin_size(size_t bin);
    1168
    -
    1169
    -
    1174void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
    -
    1175
    -
    1182char* mi_stats_as_json( mi_stats_t* stats, size_t buf_size, char* buf);
    -
    1183
    -
    1185void mi_stats_reset(void);
    -
    1186
    -
    1188void mi_stats_merge(void);
    -
    1189
    -
    1191
    -
    1192// ------------------------------------------------------
    -
    1193// Runtime Options
    -
    1194// ------------------------------------------------------
    +
    1175bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats);
    +
    1176
    +
    1183char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
    +
    1184
    +
    1189void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg);
    +
    1190
    +
    1194void mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
    1195
    -
    1200
    -
    1203void mi_options_print(void);
    -
    1204
    -
    1210void mi_options_print_out(mi_output_fun* out, void* arg);
    +
    1202bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats);
    +
    1203
    +
    1210char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
    1211
    -
    1212
    -
    -
    1214typedef enum mi_option_e {
    -
    1215 // stable options
    -
    1216 mi_option_show_errors,
    -
    1217 mi_option_show_stats,
    -
    1218 mi_option_verbose,
    -
    1219 mi_option_max_errors,
    -
    1220 mi_option_max_warnings,
    -
    1221
    -
    1222 // advanced options
    -
    1223 mi_option_reserve_huge_os_pages,
    -
    1224 mi_option_reserve_huge_os_pages_at,
    -
    1225 mi_option_reserve_os_memory,
    -
    1226 mi_option_allow_large_os_pages,
    -
    1227 mi_option_purge_decommits,
    -
    1228 mi_option_arena_reserve,
    -
    1229 mi_option_os_tag,
    -
    1230 mi_option_retry_on_oom,
    -
    1231 mi_option_generic_collect,
    -
    1232 mi_option_allow_thp,
    -
    1233
    -
    1234 // guard pages
    -
    1235 mi_option_guarded_min,
    -
    1236 mi_option_guarded_max,
    -
    1237 mi_option_guarded_precise,
    -
    1238 mi_option_guarded_sample_rate,
    -
    1239 mi_option_guarded_sample_seed,
    +
    1216void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
    +
    1217
    +
    1218
    +
    1223void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
    +
    1224
    +
    1231char* mi_stats_as_json( mi_stats_t* stats, size_t buf_size, char* buf);
    +
    1232
    +
    1234void mi_stats_reset(void);
    +
    1235
    +
    1237void mi_stats_merge(void);
    +
    1238
    1240
    -
    1241 // experimental options
    -
    1242 mi_option_eager_commit,
    -
    1243 mi_option_eager_commit_delay,
    -
    1244 mi_option_arena_eager_commit,
    -
    1245 mi_option_abandoned_page_purge,
    -
    1246 mi_option_purge_delay,
    -
    1247 mi_option_use_numa_nodes,
    -
    1248 mi_option_disallow_os_alloc,
    -
    1249 mi_option_max_segment_reclaim,
    -
    1250 mi_option_destroy_on_exit,
    -
    1251 mi_option_arena_purge_mult,
    -
    1252 mi_option_abandoned_reclaim_on_free,
    -
    1253 mi_option_purge_extend_delay,
    -
    1254 mi_option_disallow_arena_alloc,
    -
    1255 mi_option_visit_abandoned,
    -
    1256 mi_option_target_segments_per_thread,
    -
    1257
    -
    1258 // v3 options
    -
    1259 mi_option_page_reclaim_on_free,
    -
    1260 mi_option_page_full_retain,
    -
    1261 mi_option_page_max_candidates,
    -
    1262 mi_option_max_vabits,
    -
    1263 mi_option_pagemap_commit,
    -
    1264 mi_option_page_commit_on_demand,
    -
    1265 mi_option_page_max_reclaim,
    -
    1266 mi_option_page_cross_thread_max_reclaim,
    -
    1267 mi_option_minimal_purge_size,
    -
    1268 mi_option_arena_max_object_size,
    -
    1269
    -
    1270 _mi_option_last
    -
    1271} mi_option_t;
    +
    1241// ------------------------------------------------------
    +
    1242// Runtime Options
    +
    1243// ------------------------------------------------------
    +
    1244
    +
    1249
    +
    1252void mi_options_print(void);
    +
    1253
    +
    1259void mi_options_print_out(mi_output_fun* out, void* arg);
    +
    1260
    +
    1261
    +
    +
    1263typedef enum mi_option_e {
    +
    1264 // stable options
    +
    1265 mi_option_show_errors,
    +
    1266 mi_option_show_stats,
    +
    1267 mi_option_verbose,
    +
    1268 mi_option_max_errors,
    +
    1269 mi_option_max_warnings,
    +
    1270
    +
    1271 // advanced options
    +
    1272 mi_option_reserve_huge_os_pages,
    +
    1273 mi_option_reserve_huge_os_pages_at,
    +
    1274 mi_option_reserve_os_memory,
    +
    1275 mi_option_allow_large_os_pages,
    +
    1276 mi_option_purge_decommits,
    +
    1277 mi_option_arena_reserve,
    +
    1278 mi_option_os_tag,
    +
    1279 mi_option_retry_on_oom,
    +
    1280 mi_option_generic_collect,
    +
    1281 mi_option_allow_thp,
    +
    1282
    +
    1283 // guard pages
    +
    1284 mi_option_guarded_min,
    +
    1285 mi_option_guarded_max,
    +
    1286 mi_option_guarded_precise,
    +
    1287 mi_option_guarded_sample_rate,
    +
    1288 mi_option_guarded_sample_seed,
    +
    1289
    +
    1290 // experimental options
    +
    1291 mi_option_eager_commit,
    +
    1292 mi_option_eager_commit_delay,
    +
    1293 mi_option_arena_eager_commit,
    +
    1294 mi_option_abandoned_page_purge,
    +
    1295 mi_option_purge_delay,
    +
    1296 mi_option_use_numa_nodes,
    +
    1297 mi_option_disallow_os_alloc,
    +
    1298 mi_option_max_segment_reclaim,
    +
    1299 mi_option_destroy_on_exit,
    +
    1300 mi_option_arena_purge_mult,
    +
    1301 mi_option_abandoned_reclaim_on_free,
    +
    1302 mi_option_purge_extend_delay,
    +
    1303 mi_option_disallow_arena_alloc,
    +
    1304 mi_option_visit_abandoned,
    +
    1305 mi_option_target_segments_per_thread,
    +
    1306
    +
    1307 // v3 options
    +
    1308 mi_option_page_reclaim_on_free,
    +
    1309 mi_option_page_full_retain,
    +
    1310 mi_option_page_max_candidates,
    +
    1311 mi_option_max_vabits,
    +
    1312 mi_option_pagemap_commit,
    +
    1313 mi_option_page_commit_on_demand,
    +
    1314 mi_option_page_max_reclaim,
    +
    1315 mi_option_page_cross_thread_max_reclaim,
    +
    1316 mi_option_minimal_purge_size,
    +
    1317 mi_option_arena_max_object_size,
    +
    1318
    +
    1319 _mi_option_last
    +
    1320} mi_option_t;
    -
    1272
    -
    1273
    -
    1274bool mi_option_is_enabled(mi_option_t option);
    -
    1275void mi_option_enable(mi_option_t option);
    -
    1276void mi_option_disable(mi_option_t option);
    -
    1277void mi_option_set_enabled(mi_option_t option, bool enable);
    -
    1278void mi_option_set_enabled_default(mi_option_t option, bool enable);
    -
    1279
    -
    1280long mi_option_get(mi_option_t option);
    -
    1281long mi_option_get_clamp(mi_option_t option, long min, long max);
    -
    1282size_t mi_option_get_size(mi_option_t option);
    -
    1283
    -
    1284void mi_option_set(mi_option_t option, long value);
    -
    1285void mi_option_set_default(mi_option_t option, long value);
    -
    1286
    -
    1288
    -
    1301
    -
    1303struct mi_theap_s;
    -
    1304
    -
    1306typedef struct mi_theap_s mi_theap_t;
    -
    1307
    -
    1314mi_theap_t* mi_heap_theap(mi_heap_t* heap);
    -
    1315
    -
    1317void mi_theap_collect(mi_theap_t* theap, bool force);
    -
    1318
    -
    1323mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
    -
    1324
    -
    1327mi_theap_t* mi_theap_get_default();
    +
    1321
    +
    1322
    +
    1323bool mi_option_is_enabled(mi_option_t option);
    +
    1324void mi_option_enable(mi_option_t option);
    +
    1325void mi_option_disable(mi_option_t option);
    +
    1326void mi_option_set_enabled(mi_option_t option, bool enable);
    +
    1327void mi_option_set_enabled_default(mi_option_t option, bool enable);
    1328
    -
    1331void* mi_theap_malloc(mi_theap_t* theap, size_t size);
    +
    1329long mi_option_get(mi_option_t option);
    +
    1330long mi_option_get_clamp(mi_option_t option, long min, long max);
    +
    1331size_t mi_option_get_size(mi_option_t option);
    1332
    -
    1336void* mi_theap_malloc_small(mi_theap_t* theap, size_t size);
    +
    1333void mi_option_set(mi_option_t option, long value);
    +
    1334void mi_option_set_default(mi_option_t option, long value);
    +
    1335
    1337
    -
    1340void* mi_theap_zalloc(mi_theap_t* theap, size_t size);
    -
    1341
    -
    1345void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size);
    -
    1346
    -
    1347void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size);
    -
    1348void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment);
    -
    1349void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize);
    1350
    -
    1352
    +
    1352struct mi_theap_s;
    1353
    -
    1359
    -
    1361void mi_cfree(void* p);
    -
    1362void* mi__expand(void* p, size_t newsize);
    -
    1363
    -
    1364size_t mi_malloc_size(const void* p);
    -
    1365size_t mi_malloc_good_size(size_t size);
    -
    1366size_t mi_malloc_usable_size(const void *p);
    +
    1355typedef struct mi_theap_s mi_theap_t;
    +
    1356
    +
    1363mi_theap_t* mi_heap_theap(mi_heap_t* heap);
    +
    1364
    +
    1366void mi_theap_collect(mi_theap_t* theap, bool force);
    1367
    -
    1368int mi_posix_memalign(void** p, size_t alignment, size_t size);
    -
    1369int mi__posix_memalign(void** p, size_t alignment, size_t size);
    -
    1370void* mi_memalign(size_t alignment, size_t size);
    -
    1371void* mi_valloc(size_t size);
    -
    1372void* mi_pvalloc(size_t size);
    -
    1373void* mi_aligned_alloc(size_t alignment, size_t size);
    -
    1374
    -
    1375unsigned short* mi_wcsdup(const unsigned short* s);
    -
    1376unsigned char* mi_mbsdup(const unsigned char* s);
    -
    1377int mi_dupenv_s(char** buf, size_t* size, const char* name);
    -
    1378int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name);
    -
    1379
    -
    1382void* mi_reallocarray(void* p, size_t count, size_t size);
    -
    1383
    -
    1385int mi_reallocarr(void* p, size_t count, size_t size);
    +
    1372mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
    +
    1373
    +
    1376mi_theap_t* mi_theap_get_default();
    +
    1377
    +
    1380void* mi_theap_malloc(mi_theap_t* theap, size_t size);
    +
    1381
    +
    1385void* mi_theap_malloc_small(mi_theap_t* theap, size_t size);
    1386
    -
    1387void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment);
    -
    1388void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    -
    1389
    -
    1390void mi_free_size(void* p, size_t size);
    -
    1391void mi_free_size_aligned(void* p, size_t size, size_t alignment);
    -
    1392void mi_free_aligned(void* p, size_t alignment);
    -
    1393
    +
    1389void* mi_theap_zalloc(mi_theap_t* theap, size_t size);
    +
    1390
    +
    1394void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size);
    1395
    +
    1396void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size);
    +
    1397void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment);
    +
    1398void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize);
    +
    1399
    +
    1401
    +
    1402
    1408
    -
    1410void* mi_new(std::size_t n) noexcept(false);
    -
    1411
    -
    1413void* mi_new_n(size_t count, size_t size) noexcept(false);
    -
    1414
    -
    1416void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false);
    +
    1411void mi_cfree(void* p);
    +
    1412void* mi__expand(void* p, size_t newsize);
    +
    1413
    +
    1414size_t mi_malloc_size(const void* p);
    +
    1415size_t mi_malloc_good_size(size_t size);
    +
    1416size_t mi_malloc_usable_size(const void *p);
    1417
    -
    1419void* mi_new_nothrow(size_t n);
    -
    1420
    -
    1422void* mi_new_aligned_nothrow(size_t n, size_t alignment);
    -
    1423
    -
    1425void* mi_new_realloc(void* p, size_t newsize);
    -
    1426
    -
    1428void* mi_new_reallocn(void* p, size_t newcount, size_t size);
    +
    1418int mi_posix_memalign(void** p, size_t alignment, size_t size);
    +
    1419int mi__posix_memalign(void** p, size_t alignment, size_t size);
    +
    1420void* mi_memalign(size_t alignment, size_t size);
    +
    1421void* mi_valloc(size_t size);
    +
    1422void* mi_pvalloc(size_t size);
    +
    1423void* mi_aligned_alloc(size_t alignment, size_t size);
    +
    1424
    +
    1425unsigned short* mi_wcsdup(const unsigned short* s);
    +
    1426unsigned char* mi_mbsdup(const unsigned char* s);
    +
    1427int mi_dupenv_s(char** buf, size_t* size, const char* name);
    +
    1428int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name);
    1429
    -
    1437template<class T> struct mi_stl_allocator { }
    -
    1438
    -
    1440
    +
    1432void* mi_reallocarray(void* p, size_t count, size_t size);
    +
    1433
    +
    1435int mi_reallocarr(void* p, size_t count, size_t size);
    +
    1436
    +
    1437void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment);
    +
    1438void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
    +
    1439
    +
    1447void mi_free_size(void* p, size_t size);
    +
    1448
    +
    1454void mi_free_size_aligned(void* p, size_t size, size_t alignment);
    +
    1455
    +
    1456void mi_free_aligned(void* p, size_t alignment);
    +
    1457
    +
    1459
    +
    1472
    +
    1474void* mi_new(std::size_t n) noexcept(false);
    +
    1475
    +
    1477void* mi_new_n(size_t count, size_t size) noexcept(false);
    +
    1478
    +
    1480void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false);
    +
    1481
    +
    1483void* mi_new_nothrow(size_t n);
    +
    1484
    +
    1486void* mi_new_aligned_nothrow(size_t n, size_t alignment);
    +
    1487
    +
    1489void* mi_new_realloc(void* p, size_t newsize);
    +
    1490
    +
    1492void* mi_new_reallocn(void* p, size_t newcount, size_t size);
    +
    1493
    +
    1501template<class T> struct mi_stl_allocator { }
    +
    1502
    +
    1504
    mi_malloc_aligned_at
    void * mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset)
    Allocate size bytes aligned by alignment at a specified offset.
    mi_calloc_aligned
    void * mi_calloc_aligned(size_t count, size_t size, size_t alignment)
    mi_realloc_aligned
    void * mi_realloc_aligned(void *p, size_t newsize, size_t alignment)
    @@ -636,17 +652,17 @@
    mi_calloc_aligned_at
    void * mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset)
    mi_zalloc_aligned
    void * mi_zalloc_aligned(size_t size, size_t alignment)
    mi_realloc_aligned_at
    void * mi_realloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset)
    -
    mi_heap_area_t::heap_tag
    int heap_tag
    heap tag associated with this area (see mi_heap_new_ex)
    Definition mimalloc-doc.h:639
    -
    mi_heap_area_t::block_size
    size_t block_size
    size in bytes of one block
    Definition mimalloc-doc.h:637
    -
    mi_heap_area_t::committed
    size_t committed
    current committed bytes of this area
    Definition mimalloc-doc.h:635
    -
    mi_heap_area_t::full_block_size
    size_t full_block_size
    size in bytes of a full block including padding and metadata.
    Definition mimalloc-doc.h:638
    -
    mi_heap_area_t::used
    size_t used
    bytes in use by allocated blocks
    Definition mimalloc-doc.h:636
    -
    mi_heap_area_t::blocks
    void * blocks
    start of the area containing heap blocks
    Definition mimalloc-doc.h:633
    -
    mi_heap_area_t::reserved
    size_t reserved
    bytes reserved for this area
    Definition mimalloc-doc.h:634
    +
    mi_heap_area_t::heap_tag
    int heap_tag
    heap tag associated with this area (see mi_heap_new_ex)
    Definition mimalloc-doc.h:687
    +
    mi_heap_area_t::block_size
    size_t block_size
    size in bytes of one block
    Definition mimalloc-doc.h:685
    +
    mi_heap_area_t::committed
    size_t committed
    current committed bytes of this area
    Definition mimalloc-doc.h:683
    +
    mi_heap_area_t::full_block_size
    size_t full_block_size
    size in bytes of a full block including padding and metadata.
    Definition mimalloc-doc.h:686
    +
    mi_heap_area_t::used
    size_t used
    bytes in use by allocated blocks
    Definition mimalloc-doc.h:684
    +
    mi_heap_area_t::blocks
    void * blocks
    start of the area containing heap blocks
    Definition mimalloc-doc.h:681
    +
    mi_heap_area_t::reserved
    size_t reserved
    bytes reserved for this area
    Definition mimalloc-doc.h:682
    mi_abandoned_visit_blocks
    bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun *visitor, void *arg)
    v1,v2: Visit all areas and blocks in abandoned heaps.
    -
    mi_block_visit_fun
    bool mi_block_visit_fun(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg)
    Visitor function passed to mi_heap_visit_blocks()
    Definition mimalloc-doc.h:649
    +
    mi_block_visit_fun
    bool mi_block_visit_fun(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg)
    Visitor function passed to mi_heap_visit_blocks()
    Definition mimalloc-doc.h:697
    mi_heap_visit_blocks
    bool mi_heap_visit_blocks(const mi_heap_t *heap, bool visit_blocks, mi_block_visit_fun *visitor, void *arg)
    Visit all areas and blocks in a heap.
    -
    mi_heap_area_t
    An area of heap space contains blocks of a single size.
    Definition mimalloc-doc.h:632
    +
    mi_heap_area_t
    An area of heap space contains blocks of a single size.
    Definition mimalloc-doc.h:680
    mi_reserve_os_memory
    int mi_reserve_os_memory(size_t size, bool commit, bool allow_large)
    Reserve OS memory for use by mimalloc.
    mi_reserve_huge_os_pages_interleave
    int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs)
    Reserve pages of huge OS pages (1GiB) evenly divided over numa_nodes nodes, but stops after at most t...
    mi_reserve_os_memory_ex
    int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t *arena_id)
    Reserve OS memory to be managed in an arena.
    @@ -656,10 +672,15 @@
    mi_reserve_huge_os_pages_at_ex
    int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t *arena_id)
    Reserve huge OS pages (1GiB) into a single arena.
    mi_reserve_huge_os_pages_at
    int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs)
    Reserve pages of huge OS pages (1GiB) at a specific numa_node, but stops after at most timeout_msecs ...
    mi_debug_show_arenas
    void mi_debug_show_arenas(void)
    Show all current arena's.
    -
    mi_arena_id_t
    int mi_arena_id_t
    Each arena has an associated identifier.
    Definition mimalloc-doc.h:699
    +
    mi_arena_id_t
    int mi_arena_id_t
    Each arena has an associated identifier.
    Definition mimalloc-doc.h:747
    mi_arena_area
    void * mi_arena_area(mi_arena_id_t arena_id, size_t *size)
    Return the start and size of an arena.
    mi_arena_min_alignment
    size_t mi_arena_min_alignment(void)
    Return the minimal alignment required for managed OS memory.
    mi_heap_new_in_arena
    mi_heap_t * mi_heap_new_in_arena(mi_arena_id_t arena_id)
    Create a new heap that only allocates in the specified arena.
    +
    mi_theap_malloc_csize
    void * mi_theap_malloc_csize(mi_theap_t *theap, size_t size)
    +
    mi_theap_zalloc_csize
    void * mi_theap_zalloc_csize(mi_theap_t *theap, size_t size)
    +
    mi_free_csize
    void mi_free_csize(void *p, size_t size)
    Free a pointer with a known constant size.
    +
    mi_malloc_csize
    void * mi_malloc_csize(size_t size)
    Allocate a constant size object.
    +
    mi_zalloc_csize
    void * mi_zalloc_csize(size_t size)
    mi_new_nothrow
    void * mi_new_nothrow(size_t n)
    like mi_malloc, but when out of memory, use std::get_new_handler but return NULL on failure.
    mi_new
    void * mi_new(std::size_t n) noexcept(false)
    like mi_malloc(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception...
    mi_new_realloc
    void * mi_new_realloc(void *p, size_t newsize)
    like mi_realloc(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exceptio...
    @@ -667,19 +688,19 @@
    mi_new_aligned_nothrow
    void * mi_new_aligned_nothrow(size_t n, size_t alignment)
    like mi_malloc_aligned, but when out of memory, use std::get_new_handler but return NULL on failure.
    mi_new_reallocn
    void * mi_new_reallocn(void *p, size_t newcount, size_t size)
    like mi_reallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc excepti...
    mi_new_n
    void * mi_new_n(size_t count, size_t size) noexcept(false)
    like mi_mallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exceptio...
    -
    mi_stl_allocator
    std::allocator implementation for mimalloc for use in STL containers. For example:
    Definition mimalloc-doc.h:1437
    +
    mi_stl_allocator
    std::allocator implementation for mimalloc for use in STL containers. For example:
    Definition mimalloc-doc.h:1501
    mi_thread_done
    void mi_thread_done(void)
    Uninitialize mimalloc on a thread.
    -
    mi_deferred_free_fun
    void mi_deferred_free_fun(bool force, unsigned long long heartbeat, void *arg)
    Type of deferred free functions.
    Definition mimalloc-doc.h:933
    -
    mi_free_small
    void mi_free_small(void *p)
    v3: Can be used to free an object that was allocated with mi_malloc_small et al.
    +
    mi_deferred_free_fun
    void mi_deferred_free_fun(bool force, unsigned long long heartbeat, void *arg)
    Type of deferred free functions.
    Definition mimalloc-doc.h:981
    +
    mi_free_small
    void mi_free_small(void *p)
    v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a ...
    mi_register_deferred_free
    void mi_register_deferred_free(mi_deferred_free_fun *deferred_free, void *arg)
    Register a deferred free function.
    mi_collect
    void mi_collect(bool force)
    Eagerly free memory.
    mi_zalloc_small
    void * mi_zalloc_small(size_t size)
    Allocate a zero initialized small object.
    mi_malloc_small
    void * mi_malloc_small(size_t size)
    Allocate a small object.
    -
    mi_error_fun
    void mi_error_fun(int err, void *arg)
    Type of error callback functions.
    Definition mimalloc-doc.h:972
    +
    mi_error_fun
    void mi_error_fun(int err, void *arg)
    Type of error callback functions.
    Definition mimalloc-doc.h:1020
    mi_register_error
    void mi_register_error(mi_error_fun *errfun, void *arg)
    Register an error callback function.
    mi_is_redirected
    bool mi_is_redirected()
    Is the C runtime malloc API redirected?
    mi_thread_set_in_threadpool
    void mi_thread_set_in_threadpool(void)
    v3: Communicate that a thread is in a threadpool.
    -
    mi_output_fun
    void mi_output_fun(const char *msg, void *arg)
    Type of output functions.
    Definition mimalloc-doc.h:957
    +
    mi_output_fun
    void mi_output_fun(const char *msg, void *arg)
    Type of output functions.
    Definition mimalloc-doc.h:1005
    mi_register_output
    void mi_register_output(mi_output_fun *out, void *arg)
    Register an output function.
    mi_version
    int mi_version(void)
    Return the mimalloc version.
    mi_thread_init
    void mi_thread_init(void)
    Initialize mimalloc on a thread.
    @@ -691,7 +712,7 @@
    mi_heap_delete
    void mi_heap_delete(mi_heap_t *heap)
    Delete a previously allocated heap.
    mi_heap_malloc_aligned
    void * mi_heap_malloc_aligned(mi_heap_t *heap, size_t size, size_t alignment)
    mi_heap_set_default
    mi_heap_t * mi_heap_set_default(mi_heap_t *heap)
    v1,v2: Set the default heap to use in the current thread for mi_malloc() et al.
    -
    mi_heap_t
    struct mi_heap_s mi_heap_t
    Type of first-class heaps.
    Definition mimalloc-doc.h:465
    +
    mi_heap_t
    struct mi_heap_s mi_heap_t
    Type of first-class heaps.
    Definition mimalloc-doc.h:513
    mi_heap_of
    mi_heap_t * mi_heap_of(const void *p)
    v3: return the heap that contains the given pointer.
    mi_heap_zalloc_aligned_at
    void * mi_heap_zalloc_aligned_at(mi_heap_t *heap, size_t size, size_t alignment, size_t offset)
    mi_heap_realpath
    char * mi_heap_realpath(mi_heap_t *heap, const char *fname, char *resolved_name)
    Resolve a file path name using a specific heap to allocate the result.
    @@ -746,53 +767,53 @@
    mi_option_disable
    void mi_option_disable(mi_option_t option)
    mi_options_print
    void mi_options_print(void)
    Print out all runtime parameters for mimalloc. Also printed with MIMALLOC_VERBOSE=1 at startup.
    mi_option_set
    void mi_option_set(mi_option_t option, long value)
    -
    mi_option_t
    mi_option_t
    Runtime options.
    Definition mimalloc-doc.h:1214
    -
    mi_option_guarded_min
    @ mi_option_guarded_min
    only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
    Definition mimalloc-doc.h:1235
    -
    mi_option_abandoned_reclaim_on_free
    @ mi_option_abandoned_reclaim_on_free
    v1,v2: allow to reclaim an abandoned segment on a free (=1)
    Definition mimalloc-doc.h:1252
    -
    mi_option_purge_extend_delay
    @ mi_option_purge_extend_delay
    v1,v2: extend purge delay on each subsequent delay (=1)
    Definition mimalloc-doc.h:1253
    -
    mi_option_allow_thp
    @ mi_option_allow_thp
    allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the proces...
    Definition mimalloc-doc.h:1232
    -
    mi_option_show_stats
    @ mi_option_show_stats
    Print statistics on termination.
    Definition mimalloc-doc.h:1217
    -
    mi_option_use_numa_nodes
    @ mi_option_use_numa_nodes
    0 = use all available numa nodes, otherwise use at most N nodes.
    Definition mimalloc-doc.h:1247
    -
    mi_option_page_max_reclaim
    @ mi_option_page_max_reclaim
    v3: don't reclaim pages of the same originating theap if we already own N pages (in that size class) ...
    Definition mimalloc-doc.h:1265
    -
    mi_option_abandoned_page_purge
    @ mi_option_abandoned_page_purge
    v1,v2: immediately purge delayed purges on thread termination
    Definition mimalloc-doc.h:1245
    -
    mi_option_eager_commit_delay
    @ mi_option_eager_commit_delay
    v2: the first N segments per thread are not eagerly committed (but per page in the segment on demand)
    Definition mimalloc-doc.h:1243
    -
    mi_option_eager_commit
    @ mi_option_eager_commit
    v1,v2: eager commit segments? (after eager_commit_delay segments) (enabled by default).
    Definition mimalloc-doc.h:1242
    -
    mi_option_guarded_precise
    @ mi_option_guarded_precise
    disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard pa...
    Definition mimalloc-doc.h:1237
    -
    mi_option_visit_abandoned
    @ mi_option_visit_abandoned
    allow visiting heap blocks from abandoned threads (=0)
    Definition mimalloc-doc.h:1255
    -
    mi_option_os_tag
    @ mi_option_os_tag
    tag used for OS logging (macOS only for now) (=100)
    Definition mimalloc-doc.h:1229
    -
    mi_option_arena_max_object_size
    @ mi_option_arena_max_object_size
    v3: set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
    Definition mimalloc-doc.h:1268
    -
    mi_option_minimal_purge_size
    @ mi_option_minimal_purge_size
    v3: set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled....
    Definition mimalloc-doc.h:1267
    -
    _mi_option_last
    @ _mi_option_last
    Definition mimalloc-doc.h:1270
    -
    mi_option_destroy_on_exit
    @ mi_option_destroy_on_exit
    if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
    Definition mimalloc-doc.h:1250
    -
    mi_option_page_cross_thread_max_reclaim
    @ mi_option_page_cross_thread_max_reclaim
    v3: don't reclaim pages across threads if we already own N pages (in that size class) (=16)
    Definition mimalloc-doc.h:1266
    -
    mi_option_page_commit_on_demand
    @ mi_option_page_commit_on_demand
    v3: commit page memory on-demand (=0)
    Definition mimalloc-doc.h:1264
    -
    mi_option_guarded_sample_seed
    @ mi_option_guarded_sample_seed
    can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
    Definition mimalloc-doc.h:1239
    -
    mi_option_verbose
    @ mi_option_verbose
    Print verbose messages.
    Definition mimalloc-doc.h:1218
    -
    mi_option_allow_large_os_pages
    @ mi_option_allow_large_os_pages
    allow large (2 or 4 MiB) OS pages, implies eager commit.
    Definition mimalloc-doc.h:1226
    -
    mi_option_arena_purge_mult
    @ mi_option_arena_purge_mult
    multiplier for purge_delay for the purging delay for arenas (=10)
    Definition mimalloc-doc.h:1251
    -
    mi_option_target_segments_per_thread
    @ mi_option_target_segments_per_thread
    v1,v2: experimental (=0)
    Definition mimalloc-doc.h:1256
    -
    mi_option_generic_collect
    @ mi_option_generic_collect
    collect heaps every N (=10000) generic allocation calls
    Definition mimalloc-doc.h:1231
    -
    mi_option_retry_on_oom
    @ mi_option_retry_on_oom
    retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
    Definition mimalloc-doc.h:1230
    -
    mi_option_guarded_max
    @ mi_option_guarded_max
    only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
    Definition mimalloc-doc.h:1236
    -
    mi_option_purge_decommits
    @ mi_option_purge_decommits
    should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
    Definition mimalloc-doc.h:1227
    -
    mi_option_reserve_huge_os_pages_at
    @ mi_option_reserve_huge_os_pages_at
    Reserve N huge OS pages at a specific NUMA node N.
    Definition mimalloc-doc.h:1224
    -
    mi_option_pagemap_commit
    @ mi_option_pagemap_commit
    v3: commit the full pagemap (to always catch invalid pointer uses) (=0)
    Definition mimalloc-doc.h:1263
    -
    mi_option_max_segment_reclaim
    @ mi_option_max_segment_reclaim
    v2: max. percentage of the abandoned segments can be reclaimed per try (=10%)
    Definition mimalloc-doc.h:1249
    -
    mi_option_max_vabits
    @ mi_option_max_vabits
    v3: max user space virtual address bits to consider (=48)
    Definition mimalloc-doc.h:1262
    -
    mi_option_arena_reserve
    @ mi_option_arena_reserve
    initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use ...
    Definition mimalloc-doc.h:1228
    -
    mi_option_page_reclaim_on_free
    @ mi_option_page_reclaim_on_free
    v3: reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from...
    Definition mimalloc-doc.h:1259
    -
    mi_option_reserve_huge_os_pages
    @ mi_option_reserve_huge_os_pages
    reserve N huge OS pages (1GiB pages) at startup
    Definition mimalloc-doc.h:1223
    -
    mi_option_page_max_candidates
    @ mi_option_page_max_candidates
    v3: max candidate pages to consider for allocation (=4)
    Definition mimalloc-doc.h:1261
    -
    mi_option_disallow_os_alloc
    @ mi_option_disallow_os_alloc
    1 = do not use OS memory for allocation (but only programmatically reserved arenas)
    Definition mimalloc-doc.h:1248
    -
    mi_option_purge_delay
    @ mi_option_purge_delay
    memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all...
    Definition mimalloc-doc.h:1246
    -
    mi_option_disallow_arena_alloc
    @ mi_option_disallow_arena_alloc
    1 = do not use arena's for allocation (except if using specific arena id's)
    Definition mimalloc-doc.h:1254
    -
    mi_option_guarded_sample_rate
    @ mi_option_guarded_sample_rate
    1 out of N allocations in the min/max range will be guarded (=1000)
    Definition mimalloc-doc.h:1238
    -
    mi_option_max_errors
    @ mi_option_max_errors
    issue at most N error messages
    Definition mimalloc-doc.h:1219
    -
    mi_option_page_full_retain
    @ mi_option_page_full_retain
    v3: retain N full (small) pages per size class (=2). Use -1 for infinite (as in v1,...
    Definition mimalloc-doc.h:1260
    -
    mi_option_max_warnings
    @ mi_option_max_warnings
    issue at most N warning messages
    Definition mimalloc-doc.h:1220
    -
    mi_option_show_errors
    @ mi_option_show_errors
    Print error messages.
    Definition mimalloc-doc.h:1216
    -
    mi_option_reserve_os_memory
    @ mi_option_reserve_os_memory
    reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use m...
    Definition mimalloc-doc.h:1225
    -
    mi_option_arena_eager_commit
    @ mi_option_arena_eager_commit
    eager commit arenas? Use 2 to enable just on overcommit systems (=2)
    Definition mimalloc-doc.h:1244
    +
    mi_option_t
    mi_option_t
    Runtime options.
    Definition mimalloc-doc.h:1263
    +
    mi_option_guarded_min
    @ mi_option_guarded_min
    only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
    Definition mimalloc-doc.h:1284
    +
    mi_option_abandoned_reclaim_on_free
    @ mi_option_abandoned_reclaim_on_free
    v1,v2: allow to reclaim an abandoned segment on a free (=1)
    Definition mimalloc-doc.h:1301
    +
    mi_option_purge_extend_delay
    @ mi_option_purge_extend_delay
    v1,v2: extend purge delay on each subsequent delay (=1)
    Definition mimalloc-doc.h:1302
    +
    mi_option_allow_thp
    @ mi_option_allow_thp
    allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the proces...
    Definition mimalloc-doc.h:1281
    +
    mi_option_show_stats
    @ mi_option_show_stats
    Print statistics on termination.
    Definition mimalloc-doc.h:1266
    +
    mi_option_use_numa_nodes
    @ mi_option_use_numa_nodes
    0 = use all available numa nodes, otherwise use at most N nodes.
    Definition mimalloc-doc.h:1296
    +
    mi_option_page_max_reclaim
    @ mi_option_page_max_reclaim
    v3: don't reclaim pages of the same originating theap if we already own N pages (in that size class) ...
    Definition mimalloc-doc.h:1314
    +
    mi_option_abandoned_page_purge
    @ mi_option_abandoned_page_purge
    v1,v2: immediately purge delayed purges on thread termination
    Definition mimalloc-doc.h:1294
    +
    mi_option_eager_commit_delay
    @ mi_option_eager_commit_delay
    v2: the first N segments per thread are not eagerly committed (but per page in the segment on demand)
    Definition mimalloc-doc.h:1292
    +
    mi_option_eager_commit
    @ mi_option_eager_commit
    v1,v2: eager commit segments? (after eager_commit_delay segments) (enabled by default).
    Definition mimalloc-doc.h:1291
    +
    mi_option_guarded_precise
    @ mi_option_guarded_precise
    disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard pa...
    Definition mimalloc-doc.h:1286
    +
    mi_option_visit_abandoned
    @ mi_option_visit_abandoned
    allow visiting heap blocks from abandoned threads (=0)
    Definition mimalloc-doc.h:1304
    +
    mi_option_os_tag
    @ mi_option_os_tag
    tag used for OS logging (macOS only for now) (=100)
    Definition mimalloc-doc.h:1278
    +
    mi_option_arena_max_object_size
    @ mi_option_arena_max_object_size
    v3: set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
    Definition mimalloc-doc.h:1317
    +
    mi_option_minimal_purge_size
    @ mi_option_minimal_purge_size
    v3: set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled....
    Definition mimalloc-doc.h:1316
    +
    _mi_option_last
    @ _mi_option_last
    Definition mimalloc-doc.h:1319
    +
    mi_option_destroy_on_exit
    @ mi_option_destroy_on_exit
    if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
    Definition mimalloc-doc.h:1299
    +
    mi_option_page_cross_thread_max_reclaim
    @ mi_option_page_cross_thread_max_reclaim
    v3: don't reclaim pages across threads if we already own N pages (in that size class) (=16)
    Definition mimalloc-doc.h:1315
    +
    mi_option_page_commit_on_demand
    @ mi_option_page_commit_on_demand
    v3: commit page memory on-demand (=0)
    Definition mimalloc-doc.h:1313
    +
    mi_option_guarded_sample_seed
    @ mi_option_guarded_sample_seed
    can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
    Definition mimalloc-doc.h:1288
    +
    mi_option_verbose
    @ mi_option_verbose
    Print verbose messages.
    Definition mimalloc-doc.h:1267
    +
    mi_option_allow_large_os_pages
    @ mi_option_allow_large_os_pages
    allow large (2 or 4 MiB) OS pages, implies eager commit.
    Definition mimalloc-doc.h:1275
    +
    mi_option_arena_purge_mult
    @ mi_option_arena_purge_mult
    multiplier for purge_delay for the purging delay for arenas (=10)
    Definition mimalloc-doc.h:1300
    +
    mi_option_target_segments_per_thread
    @ mi_option_target_segments_per_thread
    v1,v2: experimental (=0)
    Definition mimalloc-doc.h:1305
    +
    mi_option_generic_collect
    @ mi_option_generic_collect
    collect heaps every N (=10000) generic allocation calls
    Definition mimalloc-doc.h:1280
    +
    mi_option_retry_on_oom
    @ mi_option_retry_on_oom
    retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
    Definition mimalloc-doc.h:1279
    +
    mi_option_guarded_max
    @ mi_option_guarded_max
    only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
    Definition mimalloc-doc.h:1285
    +
    mi_option_purge_decommits
    @ mi_option_purge_decommits
    should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
    Definition mimalloc-doc.h:1276
    +
    mi_option_reserve_huge_os_pages_at
    @ mi_option_reserve_huge_os_pages_at
    Reserve N huge OS pages at a specific NUMA node N.
    Definition mimalloc-doc.h:1273
    +
    mi_option_pagemap_commit
    @ mi_option_pagemap_commit
    v3: commit the full pagemap (to always catch invalid pointer uses) (=0)
    Definition mimalloc-doc.h:1312
    +
    mi_option_max_segment_reclaim
    @ mi_option_max_segment_reclaim
    v2: max. percentage of the abandoned segments can be reclaimed per try (=10%)
    Definition mimalloc-doc.h:1298
    +
    mi_option_max_vabits
    @ mi_option_max_vabits
    v3: max user space virtual address bits to consider (=48)
    Definition mimalloc-doc.h:1311
    +
    mi_option_arena_reserve
    @ mi_option_arena_reserve
    initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use ...
    Definition mimalloc-doc.h:1277
    +
    mi_option_page_reclaim_on_free
    @ mi_option_page_reclaim_on_free
    v3: reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from...
    Definition mimalloc-doc.h:1308
    +
    mi_option_reserve_huge_os_pages
    @ mi_option_reserve_huge_os_pages
    reserve N huge OS pages (1GiB pages) at startup
    Definition mimalloc-doc.h:1272
    +
    mi_option_page_max_candidates
    @ mi_option_page_max_candidates
    v3: max candidate pages to consider for allocation (=4)
    Definition mimalloc-doc.h:1310
    +
    mi_option_disallow_os_alloc
    @ mi_option_disallow_os_alloc
    1 = do not use OS memory for allocation (but only programmatically reserved arenas)
    Definition mimalloc-doc.h:1297
    +
    mi_option_purge_delay
    @ mi_option_purge_delay
    memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all...
    Definition mimalloc-doc.h:1295
    +
    mi_option_disallow_arena_alloc
    @ mi_option_disallow_arena_alloc
    1 = do not use arena's for allocation (except if using specific arena id's)
    Definition mimalloc-doc.h:1303
    +
    mi_option_guarded_sample_rate
    @ mi_option_guarded_sample_rate
    1 out of N allocations in the min/max range will be guarded (=1000)
    Definition mimalloc-doc.h:1287
    +
    mi_option_max_errors
    @ mi_option_max_errors
    issue at most N error messages
    Definition mimalloc-doc.h:1268
    +
    mi_option_page_full_retain
    @ mi_option_page_full_retain
    v3: retain N full (small) pages per size class (=2). Use -1 for infinite (as in v1,...
    Definition mimalloc-doc.h:1309
    +
    mi_option_max_warnings
    @ mi_option_max_warnings
    issue at most N warning messages
    Definition mimalloc-doc.h:1269
    +
    mi_option_show_errors
    @ mi_option_show_errors
    Print error messages.
    Definition mimalloc-doc.h:1265
    +
    mi_option_reserve_os_memory
    @ mi_option_reserve_os_memory
    reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use m...
    Definition mimalloc-doc.h:1274
    +
    mi_option_arena_eager_commit
    @ mi_option_arena_eager_commit
    eager commit arenas? Use 2 to enable just on overcommit systems (=2)
    Definition mimalloc-doc.h:1293
    mi_malloc_usable_size
    size_t mi_malloc_usable_size(const void *p)
    mi_free_aligned
    void mi_free_aligned(void *p, size_t alignment)
    mi_aligned_offset_recalloc
    void * mi_aligned_offset_recalloc(void *p, size_t newcount, size_t size, size_t alignment, size_t offset)
    @@ -802,9 +823,9 @@
    mi_pvalloc
    void * mi_pvalloc(size_t size)
    mi__expand
    void * mi__expand(void *p, size_t newsize)
    mi_wdupenv_s
    int mi_wdupenv_s(unsigned short **buf, size_t *size, const unsigned short *name)
    -
    mi_cfree
    void mi_cfree(void *p)
    Just as free but also checks if the pointer p belongs to our heap.
    +
    mi_cfree
    void mi_cfree(void *p)
    Checked free: just as mi_free but always checks if the pointer p belongs to our heap and is safe to u...
    mi_memalign
    void * mi_memalign(size_t alignment, size_t size)
    -
    mi_free_size_aligned
    void mi_free_size_aligned(void *p, size_t size, size_t alignment)
    +
    mi_free_size_aligned
    void mi_free_size_aligned(void *p, size_t size, size_t alignment)
    Free an object that was allocated aligned.
    mi_mbsdup
    unsigned char * mi_mbsdup(const unsigned char *s)
    mi_reallocarr
    int mi_reallocarr(void *p, size_t count, size_t size)
    Corresponds to reallocarr in NetBSD.
    mi_malloc_good_size
    size_t mi_malloc_good_size(size_t size)
    @@ -813,10 +834,10 @@
    mi_posix_memalign
    int mi_posix_memalign(void **p, size_t alignment, size_t size)
    mi__posix_memalign
    int mi__posix_memalign(void **p, size_t alignment, size_t size)
    mi_reallocarray
    void * mi_reallocarray(void *p, size_t count, size_t size)
    Correspond s to reallocarray in FreeBSD.
    -
    mi_free_size
    void mi_free_size(void *p, size_t size)
    +
    mi_free_size
    void mi_free_size(void *p, size_t size)
    Free an object that was allocated with a known size.
    mi_aligned_recalloc
    void * mi_aligned_recalloc(void *p, size_t newcount, size_t size, size_t alignment)
    -
    mi_stats_s::version
    size_t version
    initialize with `MI_STAT_VERSION (so linking dynamically with a separately compiled mimalloc is safe)
    Definition mimalloc-doc.h:1057
    -
    mi_stats_s::size
    size_t size
    initialize with sizeof(mi_stats_t) (so linking dynamically with a separately compiled mimalloc is saf...
    Definition mimalloc-doc.h:1055
    +
    mi_stats_s::version
    size_t version
    initialize with `MI_STAT_VERSION (so linking dynamically with a separately compiled mimalloc is safe)
    Definition mimalloc-doc.h:1106
    +
    mi_stats_s::size
    size_t size
    initialize with sizeof(mi_stats_t) (so linking dynamically with a separately compiled mimalloc is saf...
    Definition mimalloc-doc.h:1104
    mi_process_info_print_out
    void mi_process_info_print_out(mi_output_fun *out, void *arg)
    Print out all process info.
    mi_subproc_heap_stats_print_out
    void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun *out, void *arg)
    v3: Print statistics for a given subprocess with each heap separately printed.
    mi_heap_stats_get
    bool mi_heap_stats_get(mi_heap_t *heap, mi_stats_t *stats)
    v3: Return statistics for a given heap.
    @@ -836,11 +857,11 @@
    mi_stats_get_bin_size
    size_t mi_stats_get_bin_size(size_t bin)
    v3: Return the block size for the given bin.
    mi_process_info_print
    void mi_process_info_print(void)
    Print out all process info.
    mi_heap_stats_print_out
    void mi_heap_stats_print_out(mi_heap_t *heap, mi_output_fun *out, void *arg)
    v3: Show the heap statistics as JSON.
    -
    mi_stats_s
    Statistics. See include/mimalloc-stats.h for the full definition.
    Definition mimalloc-doc.h:1053
    +
    mi_stats_s
    Statistics. See include/mimalloc-stats.h for the full definition.
    Definition mimalloc-doc.h:1102
    mi_subproc_main
    mi_subproc_id_t mi_subproc_main(void)
    Get the main sub-process identifier.
    -
    mi_heap_visit_fun
    bool mi_heap_visit_fun(mi_heap_t *heap, void *arg)
    The type of a heap visitor function.
    Definition mimalloc-doc.h:863
    +
    mi_heap_visit_fun
    bool mi_heap_visit_fun(mi_heap_t *heap, void *arg)
    The type of a heap visitor function.
    Definition mimalloc-doc.h:911
    mi_subproc_new
    mi_subproc_id_t mi_subproc_new(void)
    Create a fresh sub-process (with no associated threads yet).
    -
    mi_subproc_id_t
    void * mi_subproc_id_t
    A process can associate threads with sub-processes.
    Definition mimalloc-doc.h:832
    +
    mi_subproc_id_t
    void * mi_subproc_id_t
    A process can associate threads with sub-processes.
    Definition mimalloc-doc.h:880
    mi_subproc_delete
    void mi_subproc_delete(mi_subproc_id_t subproc)
    v1,v2: Delete a previously created sub-process.
    mi_subproc_current
    mi_subproc_id_t mi_subproc_current(void)
    Get the current sub-process identifier of this thread.
    mi_subproc_destroy
    void mi_subproc_destroy(mi_subproc_id_t subproc)
    v3: Destroy a previously created sub-process.
    @@ -852,7 +873,7 @@
    mi_theap_get_default
    mi_theap_t * mi_theap_get_default()
    Get the default theap that is used for mi_malloc() et al. for the current thread.
    mi_heap_theap
    mi_theap_t * mi_heap_theap(mi_heap_t *heap)
    Return the thread-local theap for the given heap.
    mi_theap_malloc_aligned
    void * mi_theap_malloc_aligned(mi_theap_t *theap, size_t size, size_t alignment)
    -
    mi_theap_t
    struct mi_theap_s mi_theap_t
    Type of thread-local heaps.
    Definition mimalloc-doc.h:1306
    +
    mi_theap_t
    struct mi_theap_s mi_theap_t
    Type of thread-local heaps.
    Definition mimalloc-doc.h:1355
    mi_theap_collect
    void mi_theap_collect(mi_theap_t *theap, bool force)
    Release outstanding resources in a specific theap.
    mi_theap_zalloc
    void * mi_theap_zalloc(mi_theap_t *theap, size_t size)
    Allocate zero-initialized in a specific heap.
    mi_theap_zalloc_small
    void * mi_theap_zalloc_small(mi_theap_t *theap, size_t size)
    Allocate a small object in a specific theap. size must be smaller or equal to MI_SMALL_SIZE_MAX().
    diff --git a/docs/modes.html b/docs/modes.html index 1d4c9b26e..a2c488c29 100644 --- a/docs/modes.html +++ b/docs/modes.html @@ -34,7 +34,7 @@ Logo -
    mi-malloc 1.8/2.1 +
    mi-malloc 3.4/2.4/1.9
    diff --git a/docs/navtreedata.js b/docs/navtreedata.js index 9dce6fe80..91658f29a 100644 --- a/docs/navtreedata.js +++ b/docs/navtreedata.js @@ -47,7 +47,7 @@ var NAVTREE = var NAVTREEINDEX = [ "annotated.html", -"group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a" +"group__theap.html#ga7905a73865733da8a43339a500533180" ]; var SYNCONMSG = 'click to disable panel synchronisation'; diff --git a/docs/navtreeindex0.js b/docs/navtreeindex0.js index 1faae144d..24e0f8478 100644 --- a/docs/navtreeindex0.js +++ b/docs/navtreeindex0.js @@ -16,96 +16,102 @@ var NAVTREEINDEX0 = "group__aligned.html#ga977f96bd2c5c141bcd70e6685c90d6c3":[7,1,1], "group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82":[7,1,6], "group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6":[7,1,5], -"group__analysis.html":[7,5], -"group__analysis.html#a2b7a0c92ece8daf46b558efc990ebdc1":[7,5,0,4], -"group__analysis.html#a332a6c14d736a99699d5453a1cb04b41":[7,5,0,0], -"group__analysis.html#ab47526df656d8837ec3e97f11b83f835":[7,5,0,2], -"group__analysis.html#ab53664e31d7fe2564f8d42041ef75cb3":[7,5,0,3], -"group__analysis.html#ab820302c5cd0df133eb8e51650a008b4":[7,5,0,6], -"group__analysis.html#ae0085e6e1cf059a4eb7767e30e9991b8":[7,5,0,1], -"group__analysis.html#ae848a3e6840414891035423948ca0383":[7,5,0,5], -"group__analysis.html#ga6a4865a887b2ec5247854af61562503c":[7,5,2], -"group__analysis.html#ga8255dc9371e6b299d9802a610c4e34ec":[7,5,1], -"group__analysis.html#gae69fcff3324f8e419af850e34cf54c46":[7,5,3], -"group__analysis.html#structmi__heap__area__t":[7,5,0], -"group__arenas.html":[7,6], -"group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767":[7,6,11], -"group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50":[7,6,10], -"group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b":[7,6,12], -"group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf":[7,6,4], -"group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e":[7,6,7], -"group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf":[7,6,6], -"group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52":[7,6,9], -"group__arenas.html#ga7795a13d20087447281858d2c771cca1":[7,6,8], -"group__arenas.html#ga98cda17aa7adeda8b213382af8e7810b":[7,6,3], -"group__arenas.html#ga99fe38650d0b02e0e0f89ee024db91d3":[7,6,0], -"group__arenas.html#ga9a25a00a22151619a0be91a10af7787f":[7,6,1], -"group__arenas.html#gaa1a59464dbf3bc972d34fca0f86668fc":[7,6,2], -"group__arenas.html#gaaf2d9976576d5efd5544be12848af949":[7,6,5], -"group__cpp.html":[7,13], -"group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54":[7,13,5], -"group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a":[7,13,1], -"group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0":[7,13,6], -"group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8":[7,13,2], -"group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7":[7,13,3], -"group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67":[7,13,7], -"group__cpp.html#gadd11b85c15d21d308386844b5233856c":[7,13,4], -"group__cpp.html#structmi__stl__allocator":[7,13,0], -"group__extended.html":[7,8], -"group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf":[7,8,11], -"group__extended.html#ga1ea64283508718d9d645c38efc2f4305":[7,8,0], -"group__extended.html#ga292a45f7dbc7cd23c5352ce1f0002816":[7,8,1], -"group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610":[7,8,5], -"group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece":[7,8,8], -"group__extended.html#ga421430e2226d7d468529cec457396756":[7,8,4], -"group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e":[7,8,15], -"group__extended.html#ga7f050bc6b897da82692174f5fce59cde":[7,8,7], -"group__extended.html#ga83fc6a688b322261e1c2deab000b0591":[7,8,2], -"group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45":[7,8,9], -"group__extended.html#gaad25050b19f30cd79397b227e0157a3f":[7,8,6], -"group__extended.html#gada899104276aac52539eb4de7c020f1f":[7,8,13], -"group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d":[7,8,3], -"group__extended.html#gae5b17ff027cd2150b43a33040250cf3f":[7,8,10], -"group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057":[7,8,14], -"group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17":[7,8,12], -"group__heap.html":[7,4], -"group__heap.html#ga012c5c8abe22b10043de39ff95909541":[7,4,18], -"group__heap.html#ga0994f44cc595f9c1840a9af162d98597":[7,4,35], -"group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377":[7,4,6], -"group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909":[7,4,13], -"group__heap.html#ga29589f427702a1b873de8efb884a611b":[7,4,1], -"group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409":[7,4,10], -"group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5":[7,4,16], -"group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a":[7,4,28], -"group__heap.html#ga34a47cde5a5b38c29f1aa3c5e76943c2":[7,4,0], -"group__heap.html#ga424b69f285109b2515631d59f5eecb1b":[7,4,21], -"group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819":[7,4,34], -"group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2":[7,4,27], -"group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a":[7,4,30], -"group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6":[7,4,36], -"group__heap.html#ga628c237489c2679af84a4d0d143b3dd5":[7,4,2], -"group__heap.html#ga6466bde8b5712aa34e081a8317f9f471":[7,4,33], -"group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5":[7,4,24], -"group__heap.html#ga7922f7495cde30b1984d0e6072419298":[7,4,7], -"group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477":[7,4,14], -"group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d":[7,4,11], -"group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4":[7,4,5], -"group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a":[7,4,20], -"group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af":[7,4,9], -"group__heap.html#gaab93abbc0624f570e2900f95cac18591":[7,4,29], -"group__heap.html#gab0f755c0b21c387fe8e9024200faa372":[7,4,19], -"group__heap.html#gab374e206c7034e0d899fb934e4f4a863":[7,4,15], -"group__heap.html#gabebc796399619d964d8db77aa835e8c1":[7,4,32], -"group__heap.html#gac0098aaf231d3e9586c73136d5df95da":[7,4,3], -"group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a":[7,4,22], -"group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a":[7,4,12], -"group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0":[7,4,8], -"group__heap.html#gacafcc26df827c7a7de5e850217566108":[7,4,4], -"group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29":[7,4,26], -"group__heap.html#gaccf8c249872f30bf1c2493a09197d734":[7,4,23], -"group__heap.html#gad224df78f1fbee942df8adf023e12cf3":[7,4,31], -"group__heap.html#gae7cd171425bee04c683c65a3701f0b4a":[7,4,25], -"group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa":[7,4,17], +"group__analysis.html":[7,6], +"group__analysis.html#a2b7a0c92ece8daf46b558efc990ebdc1":[7,6,0,4], +"group__analysis.html#a332a6c14d736a99699d5453a1cb04b41":[7,6,0,0], +"group__analysis.html#ab47526df656d8837ec3e97f11b83f835":[7,6,0,2], +"group__analysis.html#ab53664e31d7fe2564f8d42041ef75cb3":[7,6,0,3], +"group__analysis.html#ab820302c5cd0df133eb8e51650a008b4":[7,6,0,6], +"group__analysis.html#ae0085e6e1cf059a4eb7767e30e9991b8":[7,6,0,1], +"group__analysis.html#ae848a3e6840414891035423948ca0383":[7,6,0,5], +"group__analysis.html#ga6a4865a887b2ec5247854af61562503c":[7,6,2], +"group__analysis.html#ga8255dc9371e6b299d9802a610c4e34ec":[7,6,1], +"group__analysis.html#gae69fcff3324f8e419af850e34cf54c46":[7,6,3], +"group__analysis.html#structmi__heap__area__t":[7,6,0], +"group__arenas.html":[7,7], +"group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767":[7,7,11], +"group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50":[7,7,10], +"group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b":[7,7,12], +"group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf":[7,7,4], +"group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e":[7,7,7], +"group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf":[7,7,6], +"group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52":[7,7,9], +"group__arenas.html#ga7795a13d20087447281858d2c771cca1":[7,7,8], +"group__arenas.html#ga98cda17aa7adeda8b213382af8e7810b":[7,7,3], +"group__arenas.html#ga99fe38650d0b02e0e0f89ee024db91d3":[7,7,0], +"group__arenas.html#ga9a25a00a22151619a0be91a10af7787f":[7,7,1], +"group__arenas.html#gaa1a59464dbf3bc972d34fca0f86668fc":[7,7,2], +"group__arenas.html#gaaf2d9976576d5efd5544be12848af949":[7,7,5], +"group__constantsize.html":[7,3], +"group__constantsize.html#ga105bd667da91a15d113fe79be04d625c":[7,3,2], +"group__constantsize.html#ga1addf755829380581dd9c3c9175e376f":[7,3,3], +"group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5":[7,3,0], +"group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571":[7,3,1], +"group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec":[7,3,4], +"group__cpp.html":[7,14], +"group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54":[7,14,5], +"group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a":[7,14,1], +"group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0":[7,14,6], +"group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8":[7,14,2], +"group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7":[7,14,3], +"group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67":[7,14,7], +"group__cpp.html#gadd11b85c15d21d308386844b5233856c":[7,14,4], +"group__cpp.html#structmi__stl__allocator":[7,14,0], +"group__extended.html":[7,9], +"group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf":[7,9,11], +"group__extended.html#ga1ea64283508718d9d645c38efc2f4305":[7,9,0], +"group__extended.html#ga292a45f7dbc7cd23c5352ce1f0002816":[7,9,1], +"group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610":[7,9,5], +"group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece":[7,9,8], +"group__extended.html#ga421430e2226d7d468529cec457396756":[7,9,4], +"group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e":[7,9,15], +"group__extended.html#ga7f050bc6b897da82692174f5fce59cde":[7,9,7], +"group__extended.html#ga83fc6a688b322261e1c2deab000b0591":[7,9,2], +"group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45":[7,9,9], +"group__extended.html#gaad25050b19f30cd79397b227e0157a3f":[7,9,6], +"group__extended.html#gada899104276aac52539eb4de7c020f1f":[7,9,13], +"group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d":[7,9,3], +"group__extended.html#gae5b17ff027cd2150b43a33040250cf3f":[7,9,10], +"group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057":[7,9,14], +"group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17":[7,9,12], +"group__heap.html":[7,5], +"group__heap.html#ga012c5c8abe22b10043de39ff95909541":[7,5,18], +"group__heap.html#ga0994f44cc595f9c1840a9af162d98597":[7,5,35], +"group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377":[7,5,6], +"group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909":[7,5,13], +"group__heap.html#ga29589f427702a1b873de8efb884a611b":[7,5,1], +"group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409":[7,5,10], +"group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5":[7,5,16], +"group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a":[7,5,28], +"group__heap.html#ga34a47cde5a5b38c29f1aa3c5e76943c2":[7,5,0], +"group__heap.html#ga424b69f285109b2515631d59f5eecb1b":[7,5,21], +"group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819":[7,5,34], +"group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2":[7,5,27], +"group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a":[7,5,30], +"group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6":[7,5,36], +"group__heap.html#ga628c237489c2679af84a4d0d143b3dd5":[7,5,2], +"group__heap.html#ga6466bde8b5712aa34e081a8317f9f471":[7,5,33], +"group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5":[7,5,24], +"group__heap.html#ga7922f7495cde30b1984d0e6072419298":[7,5,7], +"group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477":[7,5,14], +"group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d":[7,5,11], +"group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4":[7,5,5], +"group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a":[7,5,20], +"group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af":[7,5,9], +"group__heap.html#gaab93abbc0624f570e2900f95cac18591":[7,5,29], +"group__heap.html#gab0f755c0b21c387fe8e9024200faa372":[7,5,19], +"group__heap.html#gab374e206c7034e0d899fb934e4f4a863":[7,5,15], +"group__heap.html#gabebc796399619d964d8db77aa835e8c1":[7,5,32], +"group__heap.html#gac0098aaf231d3e9586c73136d5df95da":[7,5,3], +"group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a":[7,5,22], +"group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a":[7,5,12], +"group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0":[7,5,8], +"group__heap.html#gacafcc26df827c7a7de5e850217566108":[7,5,4], +"group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29":[7,5,26], +"group__heap.html#gaccf8c249872f30bf1c2493a09197d734":[7,5,23], +"group__heap.html#gad224df78f1fbee942df8adf023e12cf3":[7,5,31], +"group__heap.html#gae7cd171425bee04c683c65a3701f0b4a":[7,5,25], +"group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa":[7,5,17], "group__malloc.html":[7,0], "group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583":[7,0,6], "group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee":[7,0,12], @@ -121,133 +127,127 @@ var NAVTREEINDEX0 = "group__malloc.html#gae1dd97b542420c87ae085e822b1229e8":[7,0,4], "group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8":[7,0,13], "group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95":[7,0,2], -"group__options.html":[7,10], -"group__options.html#ga04180ae41b0d601421dd62ced40ca050":[7,10,2], -"group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2":[7,10,12], -"group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c":[7,10,5], -"group__options.html#ga459ad98f18b3fc9275474807fe0ca188":[7,10,6], -"group__options.html#ga65518b69ec5d32336b50e07f74b3f629":[7,10,10], -"group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a":[7,10,3], -"group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90":[7,10,8], -"group__options.html#ga96ad9c406338bd314cfe878cfc9bf723":[7,10,4], -"group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed":[7,10,9], -"group__options.html#gaebf6ff707a2e688ebb1a2296ca564054":[7,10,1], -"group__options.html#gaecfcd3ed90af99932670d881de1fa8f1":[7,10,11], -"group__options.html#gaf84921c32375e25754dc2ee6a911fa60":[7,10,7], -"group__options.html#gafebf7ed116adb38ae5218bc3ce06884c":[7,10,0], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca00734c528aae52784654a8db996ae62b":[7,10,0,15], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca009e4b5684922ce664d73d2a8e1698d9":[7,10,0,30], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca02005f164bdf03f5f00c5be726adf487":[7,10,0,31], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca04c45931ce2c82a20d08ca1471e74087":[7,10,0,14], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0957ef73b2550764b4840edf48422fda":[7,10,0,1], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0ac33a18f6b659fcfaf44efb0bab1b74":[7,10,0,25], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11a13739488294318ffc03eea378f5a4":[7,10,0,41], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11e62ed69200a489a5be955582078c0c":[7,10,0,23], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca17a190c25be381142d87e0468c4c068c":[7,10,0,21], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1e8de72c93da7ff22d91e1e27b52ac2b":[7,10,0,20], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1eaf89bb1698f988cec27b66e546760c":[7,10,0,17], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca38c67733a3956a1f4eeaca89fab9e78e":[7,10,0,33], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4b74ae2a69e445de6c2361b73c1d14bf":[7,10,0,11], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4dee58d60bc95ce0eeef759c61fd849b":[7,10,0,44], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5635112095936356425d9a01910afff7":[7,10,0,43], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5b4357b74be0d87568036c32eb1a2e4a":[7,10,0,45], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca6364331e305e7d3c0218b058ff3afc88":[7,10,0,28], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca671e806447c15a53c7d593afdd8d45c5":[7,10,0,42], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca679d9ee91221bffba4d5aafe488a8d7f":[7,10,0,40], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7b1e41ece69f9abbe5ba81d9ea7a3dc0":[7,10,0,19], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7c8b7bf5281c581bad64f5daa6442777":[7,10,0,2], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7cc4804ced69004fa42a9a136a9ba556":[7,10,0,8], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8236501f1ab45d26e6fd885d191a2b5e":[7,10,0,29], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca874f36a7a1e51f7baee4a78b2830b643":[7,10,0,34], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8b5815a577955fe8125f2b964f41ee7f":[7,10,0,13], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8f51df355bf6651db899e6085b54865e":[7,10,0,12], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca96077da4e5c841edeec0c4d6e45a5ffa":[7,10,0,16], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca9d15c5e3d2115eef681c17e4dd5ab9a4":[7,10,0,9], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa13e7926d4339d2aa6fbf61d4473fd5c":[7,10,0,6], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa88f56e3bccce85542dc2b3297024236":[7,10,0,39], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa9ad9005d7017c8c30ad2d6ba31db909":[7,10,0,27], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caabf8c9793d421ce28361b6d93c432de9":[7,10,0,38], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cab1c88e23ae290bbeec824038a97959de":[7,10,0,10], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cac16e3f838e861338fb2357b430f6974c":[7,10,0,35], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caca7ed041be3b0b9d0b82432c7bf41af2":[7,10,0,5], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cad13382757101aa4fbc4984fa0f917133":[7,10,0,37], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadcfb5a09580361b1be65901d2d812de6":[7,10,0,26], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadd351e615acd8563529c20a347be7290":[7,10,0,24], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeae1696100e4057ffc4182730cc04e40":[7,10,0,32], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeb3c75b1a838eb42106750979d0888ec":[7,10,0,18], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caec6ecbe29d46a48205ed8823a8a52a6a":[7,10,0,3], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeede7d0652e391940357e1c054d29829":[7,10,0,36], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caf9595921087e942602ee079158762665":[7,10,0,4], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4822e5c00732c5984b32a032837f0":[7,10,0,0], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4999c828cf79a0fb2de65d23f7333":[7,10,0,7], -"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5":[7,10,0,22], -"group__posix.html":[7,12], -"group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17":[7,12,12], -"group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9":[7,12,7], -"group__posix.html#ga16570deddd559001b44953eedbad0084":[7,12,3], -"group__posix.html#ga430ed1513f0571ff83be00ec58a98ee0":[7,12,2], -"group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de":[7,12,11], -"group__posix.html#ga50cafb9722020402f065de93799f64ca":[7,12,19], -"group__posix.html#ga644bebccdbb2821542dd8c7e7641f476":[7,12,16], -"group__posix.html#ga66bcfeb4faedbb42b796bc680821ef84":[7,12,0], -"group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1":[7,12,21], -"group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7":[7,12,5], -"group__posix.html#ga726867f13fd29ca36064954c0285b1d8":[7,12,14], -"group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc":[7,12,9], -"group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0":[7,12,13], -"group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5":[7,12,17], -"group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071":[7,12,10], -"group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf":[7,12,20], -"group__posix.html#gab41369c1a1da7504013a7a0b1d4dd958":[7,12,6], -"group__posix.html#gacff84f226ba9feb2031b8992e5579447":[7,12,15], -"group__posix.html#gad5a69c8fea96aa2b7a7c818c2130090a":[7,12,1], -"group__posix.html#gadfeccb72748a2f6305474a37d9d57bce":[7,12,18], -"group__posix.html#gae01389eedab8d67341ff52e2aad80ebb":[7,12,8], -"group__posix.html#gaf82cbb4b4f24acf723348628451798d3":[7,12,4], -"group__stats.html":[7,9], -"group__stats.html#aa74fc9c40dfaef5e06dec4019c19de68":[7,9,0,1], -"group__stats.html#ae41e0fff24b988d0a43c34040b04b887":[7,9,0,0], -"group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732":[7,9,9], -"group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b":[7,9,18], -"group__stats.html#ga2654195cef93ed3613e98b59a19a296f":[7,9,3], -"group__stats.html#ga2d126e5c62d3badc35445e5d84166df2":[7,9,15], -"group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99":[7,9,17], -"group__stats.html#ga537f13b299ddf801e49a5a94fde02c79":[7,9,16], -"group__stats.html#ga59ec89969837db414a6bce8b58915624":[7,9,21], -"group__stats.html#ga5a34328f846c661aa560f4769cfc15f1":[7,9,5], -"group__stats.html#ga5ec3f9058eec804924711ab38ba64b51":[7,9,19], -"group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1":[7,9,10], -"group__stats.html#ga7d862c2affd5790381da14eb102a364d":[7,9,7], -"group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1":[7,9,14], -"group__stats.html#ga961d0f9f0327ebc9bd8a0b84f6dcb410":[7,9,2], -"group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086":[7,9,11], -"group__stats.html#gac4411cf3eed23719e511b9e1128888e0":[7,9,1], -"group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8":[7,9,4], -"group__stats.html#gad1cf918f08b6506e65ae134423477a0b":[7,9,20], -"group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd":[7,9,13], -"group__stats.html#gadecd549d1a7216af596322ac27e78b0d":[7,9,12], -"group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536":[7,9,8], -"group__stats.html#gafea365d7842f5f5bc1f235bbc303d585":[7,9,6], -"group__stats.html#structmi__stats__s":[7,9,0], -"group__subproc.html":[7,7], -"group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806":[7,7,6], -"group__subproc.html#ga7dcbc5863df9989b45e7051999448a4a":[7,7,0], -"group__subproc.html#ga8068cac328e41fa2170faef707315243":[7,7,7], -"group__subproc.html#ga8c0bcd1fee27c7641e9c3c0d991b3b7d":[7,7,1], -"group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e":[7,7,4], -"group__subproc.html#gaabc4db6585bf16099e43e60d64364950":[7,7,3], -"group__subproc.html#gada12e4d9bd382935c398ec324d0391f9":[7,7,5], -"group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c":[7,7,2], -"group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93":[7,7,8], -"group__theap.html":[7,11], -"group__theap.html#ga04420ce52785019412397e4de24a984a":[7,11,9], -"group__theap.html#ga11943128dc354313c4189e72f4299dbb":[7,11,8], -"group__theap.html#ga1fc65302a212daaecb21abda96ff247c":[7,11,7], -"group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5":[7,11,4], -"group__theap.html#ga7905a73865733da8a43339a500533180":[7,11,1], -"group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36":[7,11,6], -"group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8":[7,11,0], -"group__theap.html#ga9377a43095b8dd14996613ceb73d9a32":[7,11,3], -"group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7":[7,11,10], -"group__theap.html#gaa85fccf9d40af3e826c5932aded1d534":[7,11,11] +"group__options.html":[7,11], +"group__options.html#ga04180ae41b0d601421dd62ced40ca050":[7,11,2], +"group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2":[7,11,12], +"group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c":[7,11,5], +"group__options.html#ga459ad98f18b3fc9275474807fe0ca188":[7,11,6], +"group__options.html#ga65518b69ec5d32336b50e07f74b3f629":[7,11,10], +"group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a":[7,11,3], +"group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90":[7,11,8], +"group__options.html#ga96ad9c406338bd314cfe878cfc9bf723":[7,11,4], +"group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed":[7,11,9], +"group__options.html#gaebf6ff707a2e688ebb1a2296ca564054":[7,11,1], +"group__options.html#gaecfcd3ed90af99932670d881de1fa8f1":[7,11,11], +"group__options.html#gaf84921c32375e25754dc2ee6a911fa60":[7,11,7], +"group__options.html#gafebf7ed116adb38ae5218bc3ce06884c":[7,11,0], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca00734c528aae52784654a8db996ae62b":[7,11,0,15], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca009e4b5684922ce664d73d2a8e1698d9":[7,11,0,30], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca02005f164bdf03f5f00c5be726adf487":[7,11,0,31], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca04c45931ce2c82a20d08ca1471e74087":[7,11,0,14], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0957ef73b2550764b4840edf48422fda":[7,11,0,1], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0ac33a18f6b659fcfaf44efb0bab1b74":[7,11,0,25], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11a13739488294318ffc03eea378f5a4":[7,11,0,41], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11e62ed69200a489a5be955582078c0c":[7,11,0,23], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca17a190c25be381142d87e0468c4c068c":[7,11,0,21], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1e8de72c93da7ff22d91e1e27b52ac2b":[7,11,0,20], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1eaf89bb1698f988cec27b66e546760c":[7,11,0,17], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca38c67733a3956a1f4eeaca89fab9e78e":[7,11,0,33], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4b74ae2a69e445de6c2361b73c1d14bf":[7,11,0,11], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4dee58d60bc95ce0eeef759c61fd849b":[7,11,0,44], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5635112095936356425d9a01910afff7":[7,11,0,43], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5b4357b74be0d87568036c32eb1a2e4a":[7,11,0,45], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca6364331e305e7d3c0218b058ff3afc88":[7,11,0,28], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca671e806447c15a53c7d593afdd8d45c5":[7,11,0,42], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca679d9ee91221bffba4d5aafe488a8d7f":[7,11,0,40], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7b1e41ece69f9abbe5ba81d9ea7a3dc0":[7,11,0,19], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7c8b7bf5281c581bad64f5daa6442777":[7,11,0,2], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7cc4804ced69004fa42a9a136a9ba556":[7,11,0,8], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8236501f1ab45d26e6fd885d191a2b5e":[7,11,0,29], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca874f36a7a1e51f7baee4a78b2830b643":[7,11,0,34], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8b5815a577955fe8125f2b964f41ee7f":[7,11,0,13], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8f51df355bf6651db899e6085b54865e":[7,11,0,12], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca96077da4e5c841edeec0c4d6e45a5ffa":[7,11,0,16], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca9d15c5e3d2115eef681c17e4dd5ab9a4":[7,11,0,9], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa13e7926d4339d2aa6fbf61d4473fd5c":[7,11,0,6], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa88f56e3bccce85542dc2b3297024236":[7,11,0,39], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa9ad9005d7017c8c30ad2d6ba31db909":[7,11,0,27], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caabf8c9793d421ce28361b6d93c432de9":[7,11,0,38], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cab1c88e23ae290bbeec824038a97959de":[7,11,0,10], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cac16e3f838e861338fb2357b430f6974c":[7,11,0,35], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caca7ed041be3b0b9d0b82432c7bf41af2":[7,11,0,5], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cad13382757101aa4fbc4984fa0f917133":[7,11,0,37], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadcfb5a09580361b1be65901d2d812de6":[7,11,0,26], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadd351e615acd8563529c20a347be7290":[7,11,0,24], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeae1696100e4057ffc4182730cc04e40":[7,11,0,32], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeb3c75b1a838eb42106750979d0888ec":[7,11,0,18], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caec6ecbe29d46a48205ed8823a8a52a6a":[7,11,0,3], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeede7d0652e391940357e1c054d29829":[7,11,0,36], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caf9595921087e942602ee079158762665":[7,11,0,4], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4822e5c00732c5984b32a032837f0":[7,11,0,0], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4999c828cf79a0fb2de65d23f7333":[7,11,0,7], +"group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5":[7,11,0,22], +"group__posix.html":[7,13], +"group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17":[7,13,12], +"group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9":[7,13,7], +"group__posix.html#ga16570deddd559001b44953eedbad0084":[7,13,3], +"group__posix.html#ga430ed1513f0571ff83be00ec58a98ee0":[7,13,2], +"group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de":[7,13,11], +"group__posix.html#ga50cafb9722020402f065de93799f64ca":[7,13,19], +"group__posix.html#ga644bebccdbb2821542dd8c7e7641f476":[7,13,16], +"group__posix.html#ga66bcfeb4faedbb42b796bc680821ef84":[7,13,0], +"group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1":[7,13,21], +"group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7":[7,13,5], +"group__posix.html#ga726867f13fd29ca36064954c0285b1d8":[7,13,14], +"group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc":[7,13,9], +"group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0":[7,13,13], +"group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5":[7,13,17], +"group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071":[7,13,10], +"group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf":[7,13,20], +"group__posix.html#gab41369c1a1da7504013a7a0b1d4dd958":[7,13,6], +"group__posix.html#gacff84f226ba9feb2031b8992e5579447":[7,13,15], +"group__posix.html#gad5a69c8fea96aa2b7a7c818c2130090a":[7,13,1], +"group__posix.html#gadfeccb72748a2f6305474a37d9d57bce":[7,13,18], +"group__posix.html#gae01389eedab8d67341ff52e2aad80ebb":[7,13,8], +"group__posix.html#gaf82cbb4b4f24acf723348628451798d3":[7,13,4], +"group__stats.html":[7,10], +"group__stats.html#aa74fc9c40dfaef5e06dec4019c19de68":[7,10,0,1], +"group__stats.html#ae41e0fff24b988d0a43c34040b04b887":[7,10,0,0], +"group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732":[7,10,9], +"group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b":[7,10,18], +"group__stats.html#ga2654195cef93ed3613e98b59a19a296f":[7,10,3], +"group__stats.html#ga2d126e5c62d3badc35445e5d84166df2":[7,10,15], +"group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99":[7,10,17], +"group__stats.html#ga537f13b299ddf801e49a5a94fde02c79":[7,10,16], +"group__stats.html#ga59ec89969837db414a6bce8b58915624":[7,10,21], +"group__stats.html#ga5a34328f846c661aa560f4769cfc15f1":[7,10,5], +"group__stats.html#ga5ec3f9058eec804924711ab38ba64b51":[7,10,19], +"group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1":[7,10,10], +"group__stats.html#ga7d862c2affd5790381da14eb102a364d":[7,10,7], +"group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1":[7,10,14], +"group__stats.html#ga961d0f9f0327ebc9bd8a0b84f6dcb410":[7,10,2], +"group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086":[7,10,11], +"group__stats.html#gac4411cf3eed23719e511b9e1128888e0":[7,10,1], +"group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8":[7,10,4], +"group__stats.html#gad1cf918f08b6506e65ae134423477a0b":[7,10,20], +"group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd":[7,10,13], +"group__stats.html#gadecd549d1a7216af596322ac27e78b0d":[7,10,12], +"group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536":[7,10,8], +"group__stats.html#gafea365d7842f5f5bc1f235bbc303d585":[7,10,6], +"group__stats.html#structmi__stats__s":[7,10,0], +"group__subproc.html":[7,8], +"group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806":[7,8,6], +"group__subproc.html#ga7dcbc5863df9989b45e7051999448a4a":[7,8,0], +"group__subproc.html#ga8068cac328e41fa2170faef707315243":[7,8,7], +"group__subproc.html#ga8c0bcd1fee27c7641e9c3c0d991b3b7d":[7,8,1], +"group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e":[7,8,4], +"group__subproc.html#gaabc4db6585bf16099e43e60d64364950":[7,8,3], +"group__subproc.html#gada12e4d9bd382935c398ec324d0391f9":[7,8,5], +"group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c":[7,8,2], +"group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93":[7,8,8], +"group__theap.html":[7,12], +"group__theap.html#ga04420ce52785019412397e4de24a984a":[7,12,9], +"group__theap.html#ga11943128dc354313c4189e72f4299dbb":[7,12,8], +"group__theap.html#ga1fc65302a212daaecb21abda96ff247c":[7,12,7], +"group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5":[7,12,4] }; diff --git a/docs/navtreeindex1.js b/docs/navtreeindex1.js index d95bea51f..79286c5a6 100644 --- a/docs/navtreeindex1.js +++ b/docs/navtreeindex1.js @@ -1,32 +1,39 @@ var NAVTREEINDEX1 = { -"group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a":[7,11,5], -"group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc":[7,11,2], +"group__theap.html#ga7905a73865733da8a43339a500533180":[7,12,1], +"group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36":[7,12,6], +"group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8":[7,12,0], +"group__theap.html#ga9377a43095b8dd14996613ceb73d9a32":[7,12,3], +"group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7":[7,12,10], +"group__theap.html#gaa85fccf9d40af3e826c5932aded1d534":[7,12,11], +"group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a":[7,12,5], +"group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc":[7,12,2], "group__typed.html":[7,2], -"group__typed.html#ga02d3e85bf26aa3120132f2552c408a88":[7,2,2], -"group__typed.html#ga0619a62c5fd886f1016030abe91f0557":[7,2,7], -"group__typed.html#ga0aaa1aa747d8863e844c37e424ed096c":[7,2,6], -"group__typed.html#ga1158b49a55dfa81f58a4426a7578f523":[7,2,9], -"group__typed.html#ga7bf06ad296e20d1d74c90d64294eecec":[7,2,5], -"group__typed.html#ga817e315c24133294caad4d50470ff313":[7,2,4], -"group__typed.html#ga953f22101bb354a8b6de2a6d6fda9b98":[7,2,1], -"group__typed.html#gaa58690658cc6f5dd54c99a9971d4aa7d":[7,2,3], -"group__typed.html#gac77a61bdaf680a803785fe307820b48c":[7,2,10], -"group__typed.html#gae5cb6e0fafc9f23169c5622e077afe8b":[7,2,8], +"group__typed.html#ga02d3e85bf26aa3120132f2552c408a88":[7,2,3], +"group__typed.html#ga0619a62c5fd886f1016030abe91f0557":[7,2,8], +"group__typed.html#ga0aaa1aa747d8863e844c37e424ed096c":[7,2,7], +"group__typed.html#ga1158b49a55dfa81f58a4426a7578f523":[7,2,10], +"group__typed.html#ga7bf06ad296e20d1d74c90d64294eecec":[7,2,6], +"group__typed.html#ga817e315c24133294caad4d50470ff313":[7,2,5], +"group__typed.html#ga953f22101bb354a8b6de2a6d6fda9b98":[7,2,2], +"group__typed.html#gaa58690658cc6f5dd54c99a9971d4aa7d":[7,2,4], +"group__typed.html#gac77a61bdaf680a803785fe307820b48c":[7,2,11], +"group__typed.html#gae5cb6e0fafc9f23169c5622e077afe8b":[7,2,9], "group__typed.html#gae80c47c9d4cab10961fff1a8ac98fc07":[7,2,0], -"group__zeroinit.html":[7,3], -"group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496":[7,3,2], -"group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc":[7,3,6], -"group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9":[7,3,5], -"group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e":[7,3,7], -"group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa":[7,3,10], -"group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7":[7,3,4], -"group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270":[7,3,11], -"group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32":[7,3,1], -"group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d":[7,3,3], -"group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750":[7,3,8], -"group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde":[7,3,0], -"group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756":[7,3,9], +"group__typed.html#gafc80000c300f02bdd4fe695eb53ca70a":[7,2,1], +"group__zeroinit.html":[7,4], +"group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496":[7,4,2], +"group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc":[7,4,6], +"group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9":[7,4,5], +"group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e":[7,4,7], +"group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa":[7,4,10], +"group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7":[7,4,4], +"group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270":[7,4,11], +"group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32":[7,4,1], +"group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d":[7,4,3], +"group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750":[7,4,8], +"group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde":[7,4,0], +"group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756":[7,4,9], "index.html":[], "modes.html":[4], "overrides.html":[3], diff --git a/docs/overrides.html b/docs/overrides.html index 0ce8b6f67..d5736734f 100644 --- a/docs/overrides.html +++ b/docs/overrides.html @@ -34,7 +34,7 @@ Logo -
    mi-malloc 1.8/2.1 +
    mi-malloc 3.4/2.4/1.9
    diff --git a/docs/pages.html b/docs/pages.html index 55ca001f4..c1d698676 100644 --- a/docs/pages.html +++ b/docs/pages.html @@ -34,7 +34,7 @@ Logo -
    mi-malloc 1.8/2.1 +
    mi-malloc 3.4/2.4/1.9
    diff --git a/docs/search/all_1.js b/docs/search/all_1.js index d96b393af..609010d11 100644 --- a/docs/search/all_1.js +++ b/docs/search/all_1.js @@ -2,6 +2,6 @@ var searchData= [ ['aligned_20allocation_0',['Aligned Allocation',['../group__aligned.html',1,'']]], ['allocation_1',['Allocation',['../group__aligned.html',1,'Aligned Allocation'],['../group__malloc.html',1,'Basic Allocation']]], - ['allocation_2',['Zero initialized re-allocation',['../group__zeroinit.html',1,'']]], + ['allocation_2',['allocation',['../group__constantsize.html',1,'Constant size allocation'],['../group__zeroinit.html',1,'Zero initialized re-allocation']]], ['arenas_3',['Arenas',['../group__arenas.html',1,'']]] ]; diff --git a/docs/search/all_3.js b/docs/search/all_3.js index 76374a610..50577eec2 100644 --- a/docs/search/all_3.js +++ b/docs/search/all_3.js @@ -1,5 +1,6 @@ var searchData= [ ['c_20wrappers_0',['C++ wrappers',['../group__cpp.html',1,'']]], - ['committed_1',['committed',['../group__analysis.html#ab47526df656d8837ec3e97f11b83f835',1,'mi_heap_area_t']]] + ['committed_1',['committed',['../group__analysis.html#ab47526df656d8837ec3e97f11b83f835',1,'mi_heap_area_t']]], + ['constant_20size_20allocation_2',['Constant size allocation',['../group__constantsize.html',1,'']]] ]; diff --git a/docs/search/all_9.js b/docs/search/all_9.js index 0b7a6daf4..54d47ba9a 100644 --- a/docs/search/all_9.js +++ b/docs/search/all_9.js @@ -29,226 +29,232 @@ var searchData= ['mi_5fexpand_26',['mi_expand',['../group__malloc.html#ga19299856216cfbb08e2628593654dfb0',1,'mimalloc-doc.h']]], ['mi_5ffree_27',['mi_free',['../group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95',1,'mimalloc-doc.h']]], ['mi_5ffree_5faligned_28',['mi_free_aligned',['../group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_29',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_5faligned_30',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsmall_31',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], - ['mi_5fgood_5fsize_32',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], - ['mi_5fheap_5farea_5ft_33',['mi_heap_area_t',['../group__analysis.html#structmi__heap__area__t',1,'']]], - ['mi_5fheap_5fcalloc_34',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_35',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_5fat_36',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5ftp_37',['mi_heap_calloc_tp',['../group__typed.html#ga953f22101bb354a8b6de2a6d6fda9b98',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcheck_5fowned_38',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcollect_39',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_40',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_5fblock_41',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdelete_42',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdestroy_43',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fbacking_44',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fdefault_45',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmain_46',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_47',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_48',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_5fat_49',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5fsmall_50',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5ftp_51',['mi_heap_malloc_tp',['../group__typed.html#ga02d3e85bf26aa3120132f2552c408a88',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmallocn_52',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmallocn_5ftp_53',['mi_heap_mallocn_tp',['../group__typed.html#gaa58690658cc6f5dd54c99a9971d4aa7d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_54',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fex_55',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fin_5farena_56',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fof_57',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_58',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_59',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_5fat_60',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocf_61',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocn_62',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocn_5ftp_63',['mi_heap_reallocn_tp',['../group__typed.html#ga817e315c24133294caad4d50470ff313',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealpath_64',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_65',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_66',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_5fat_67',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5ftp_68',['mi_heap_recalloc_tp',['../group__typed.html#ga7bf06ad296e20d1d74c90d64294eecec',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_69',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_70',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_5fat_71',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fdefault_72',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fnuma_5faffinity_73',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_74',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_5fjson_75',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_76',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fprint_5fout_77',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrdup_78',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrndup_79',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], - ['mi_5fheap_5ft_80',['mi_heap_t',['../group__heap.html#ga34a47cde5a5b38c29f1aa3c5e76943c2',1,'mimalloc-doc.h']]], - ['mi_5fheap_5ftheap_81',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fvisit_5fblocks_82',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fvisit_5ffun_83',['mi_heap_visit_fun',['../group__subproc.html#ga7dcbc5863df9989b45e7051999448a4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_84',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_85',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_5fat_86',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5fsmall_87',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5ftp_88',['mi_heap_zalloc_tp',['../group__typed.html#ga0aaa1aa747d8863e844c37e424ed096c',1,'mimalloc-doc.h']]], - ['mi_5fis_5fin_5fheap_5fregion_89',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], - ['mi_5fis_5fredirected_90',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_91',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_92',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_5fat_93',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fgood_5fsize_94',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsize_95',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsmall_96',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5ftp_97',['mi_malloc_tp',['../group__typed.html#ga0619a62c5fd886f1016030abe91f0557',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fusable_5fsize_98',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], - ['mi_5fmallocn_99',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], - ['mi_5fmallocn_5ftp_100',['mi_mallocn_tp',['../group__typed.html#gae5cb6e0fafc9f23169c5622e077afe8b',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_101',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_5fex_102',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], - ['mi_5fmbsdup_103',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], - ['mi_5fmemalign_104',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], - ['mi_5fnew_105',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_106',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_5fnothrow_107',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fn_108',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fnothrow_109',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], - ['mi_5fnew_5frealloc_110',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], - ['mi_5fnew_5freallocn_111',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], - ['mi_5foption_5fabandoned_5fpage_5fpurge_112',['mi_option_abandoned_page_purge',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11e62ed69200a489a5be955582078c0c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fabandoned_5freclaim_5fon_5ffree_113',['mi_option_abandoned_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca009e4b5684922ce664d73d2a8e1698d9',1,'mimalloc-doc.h']]], - ['mi_5foption_5fallow_5flarge_5fos_5fpages_114',['mi_option_allow_large_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7cc4804ced69004fa42a9a136a9ba556',1,'mimalloc-doc.h']]], - ['mi_5foption_5fallow_5fthp_115',['mi_option_allow_thp',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca04c45931ce2c82a20d08ca1471e74087',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5feager_5fcommit_116',['mi_option_arena_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5fmax_5fobject_5fsize_117',['mi_option_arena_max_object_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4dee58d60bc95ce0eeef759c61fd849b',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5fpurge_5fmult_118',['mi_option_arena_purge_mult',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8236501f1ab45d26e6fd885d191a2b5e',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5freserve_119',['mi_option_arena_reserve',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cab1c88e23ae290bbeec824038a97959de',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdestroy_5fon_5fexit_120',['mi_option_destroy_on_exit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca6364331e305e7d3c0218b058ff3afc88',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisable_121',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisallow_5farena_5falloc_122',['mi_option_disallow_arena_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeae1696100e4057ffc4182730cc04e40',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisallow_5fos_5falloc_123',['mi_option_disallow_os_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadcfb5a09580361b1be65901d2d812de6',1,'mimalloc-doc.h']]], - ['mi_5foption_5feager_5fcommit_124',['mi_option_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1e8de72c93da7ff22d91e1e27b52ac2b',1,'mimalloc-doc.h']]], - ['mi_5foption_5feager_5fcommit_5fdelay_125',['mi_option_eager_commit_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca17a190c25be381142d87e0468c4c068c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fenable_126',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], - ['mi_5foption_5fgeneric_5fcollect_127',['mi_option_generic_collect',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8b5815a577955fe8125f2b964f41ee7f',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_128',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fclamp_129',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fsize_130',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fmax_131',['mi_option_guarded_max',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca96077da4e5c841edeec0c4d6e45a5ffa',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fmin_132',['mi_option_guarded_min',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca00734c528aae52784654a8db996ae62b',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fprecise_133',['mi_option_guarded_precise',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1eaf89bb1698f988cec27b66e546760c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fsample_5frate_134',['mi_option_guarded_sample_rate',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeb3c75b1a838eb42106750979d0888ec',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fsample_5fseed_135',['mi_option_guarded_sample_seed',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7b1e41ece69f9abbe5ba81d9ea7a3dc0',1,'mimalloc-doc.h']]], - ['mi_5foption_5fis_5fenabled_136',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5ferrors_137',['mi_option_max_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caec6ecbe29d46a48205ed8823a8a52a6a',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5fsegment_5freclaim_138',['mi_option_max_segment_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa9ad9005d7017c8c30ad2d6ba31db909',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5fvabits_139',['mi_option_max_vabits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caabf8c9793d421ce28361b6d93c432de9',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5fwarnings_140',['mi_option_max_warnings',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caf9595921087e942602ee079158762665',1,'mimalloc-doc.h']]], - ['mi_5foption_5fminimal_5fpurge_5fsize_141',['mi_option_minimal_purge_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5635112095936356425d9a01910afff7',1,'mimalloc-doc.h']]], - ['mi_5foption_5fos_5ftag_142',['mi_option_os_tag',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4b74ae2a69e445de6c2361b73c1d14bf',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fcommit_5fon_5fdemand_143',['mi_option_page_commit_on_demand',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca679d9ee91221bffba4d5aafe488a8d7f',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fcross_5fthread_5fmax_5freclaim_144',['mi_option_page_cross_thread_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca671e806447c15a53c7d593afdd8d45c5',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5ffull_5fretain_145',['mi_option_page_full_retain',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeede7d0652e391940357e1c054d29829',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fmax_5fcandidates_146',['mi_option_page_max_candidates',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cad13382757101aa4fbc4984fa0f917133',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fmax_5freclaim_147',['mi_option_page_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11a13739488294318ffc03eea378f5a4',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5freclaim_5fon_5ffree_148',['mi_option_page_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cac16e3f838e861338fb2357b430f6974c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpagemap_5fcommit_149',['mi_option_pagemap_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa88f56e3bccce85542dc2b3297024236',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpurge_5fdecommits_150',['mi_option_purge_decommits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca9d15c5e3d2115eef681c17e4dd5ab9a4',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpurge_5fdelay_151',['mi_option_purge_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadd351e615acd8563529c20a347be7290',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpurge_5fextend_5fdelay_152',['mi_option_purge_extend_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca02005f164bdf03f5f00c5be726adf487',1,'mimalloc-doc.h']]], - ['mi_5foption_5freserve_5fhuge_5fos_5fpages_153',['mi_option_reserve_huge_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caca7ed041be3b0b9d0b82432c7bf41af2',1,'mimalloc-doc.h']]], - ['mi_5foption_5freserve_5fhuge_5fos_5fpages_5fat_154',['mi_option_reserve_huge_os_pages_at',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa13e7926d4339d2aa6fbf61d4473fd5c',1,'mimalloc-doc.h']]], - ['mi_5foption_5freserve_5fos_5fmemory_155',['mi_option_reserve_os_memory',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4999c828cf79a0fb2de65d23f7333',1,'mimalloc-doc.h']]], - ['mi_5foption_5fretry_5fon_5foom_156',['mi_option_retry_on_oom',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8f51df355bf6651db899e6085b54865e',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_157',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fdefault_158',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_159',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_5fdefault_160',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], - ['mi_5foption_5fshow_5ferrors_161',['mi_option_show_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4822e5c00732c5984b32a032837f0',1,'mimalloc-doc.h']]], - ['mi_5foption_5fshow_5fstats_162',['mi_option_show_stats',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0957ef73b2550764b4840edf48422fda',1,'mimalloc-doc.h']]], - ['mi_5foption_5ft_163',['mi_option_t',['../group__options.html#gafebf7ed116adb38ae5218bc3ce06884c',1,'mimalloc-doc.h']]], - ['mi_5foption_5ftarget_5fsegments_5fper_5fthread_164',['mi_option_target_segments_per_thread',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca874f36a7a1e51f7baee4a78b2830b643',1,'mimalloc-doc.h']]], - ['mi_5foption_5fuse_5fnuma_5fnodes_165',['mi_option_use_numa_nodes',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0ac33a18f6b659fcfaf44efb0bab1b74',1,'mimalloc-doc.h']]], - ['mi_5foption_5fverbose_166',['mi_option_verbose',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7c8b7bf5281c581bad64f5daa6442777',1,'mimalloc-doc.h']]], - ['mi_5foption_5fvisit_5fabandoned_167',['mi_option_visit_abandoned',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca38c67733a3956a1f4eeaca89fab9e78e',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_168',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_5fout_169',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], - ['mi_5foutput_5ffun_170',['mi_output_fun',['../group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d',1,'mimalloc-doc.h']]], - ['mi_5fposix_5fmemalign_171',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_172',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_173',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_5fout_174',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], - ['mi_5fpvalloc_175',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], - ['mi_5frealloc_176',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_177',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_5fat_178',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], - ['mi_5freallocarr_179',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], - ['mi_5freallocarray_180',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], - ['mi_5freallocf_181',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], - ['mi_5freallocn_182',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], - ['mi_5freallocn_5ftp_183',['mi_reallocn_tp',['../group__typed.html#ga1158b49a55dfa81f58a4426a7578f523',1,'mimalloc-doc.h']]], - ['mi_5frealpath_184',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_185',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_186',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_5fat_187',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], - ['mi_5fregister_5fdeferred_5ffree_188',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], - ['mi_5fregister_5ferror_189',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], - ['mi_5fregister_5foutput_190',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_191',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_192',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_193',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_194',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_5fex_195',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_196',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_197',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_5fat_198',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], - ['mi_5fsmall_5fsize_5fmax_199',['MI_SMALL_SIZE_MAX',['../group__extended.html#ga1ea64283508718d9d645c38efc2f4305',1,'mimalloc-doc.h']]], - ['mi_5fstat_5fversion_200',['MI_STAT_VERSION',['../group__stats.html#gac4411cf3eed23719e511b9e1128888e0',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fas_5fjson_201',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_202',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fbin_5fsize_203',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fjson_204',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fmerge_205',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_206',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_5fout_207',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], - ['mi_5fstats_5freset_208',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fs_209',['mi_stats_s',['../group__stats.html#structmi__stats__s',1,'']]], - ['mi_5fstats_5ft_5fdecl_210',['mi_stats_t_decl',['../group__stats.html#ga961d0f9f0327ebc9bd8a0b84f6dcb410',1,'mimalloc-doc.h']]], - ['mi_5fstl_5fallocator_211',['mi_stl_allocator',['../group__cpp.html#structmi__stl__allocator',1,'']]], - ['mi_5fstrdup_212',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], - ['mi_5fstrndup_213',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fadd_5fcurrent_5fthread_214',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fcurrent_215',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdelete_216',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdestroy_217',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_218',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fid_5ft_219',['mi_subproc_id_t',['../group__subproc.html#ga8c0bcd1fee27c7641e9c3c0d991b3b7d',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fmain_220',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fnew_221',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_222',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_5fjson_223',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fprint_5fout_224',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fvisit_5fheaps_225',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcalloc_226',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcollect_227',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fget_5fdefault_228',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_229',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5faligned_230',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fsmall_231',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5frealloc_232',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fset_5fdefault_233',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5ft_234',['mi_theap_t',['../group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_235',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fsmall_236',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fdone_237',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], - ['mi_5fthread_5finit_238',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fset_5fin_5fthreadpool_239',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], - ['mi_5fusable_5fsize_240',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], - ['mi_5fvalloc_241',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], - ['mi_5fversion_242',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], - ['mi_5fwcsdup_243',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], - ['mi_5fwdupenv_5fs_244',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_245',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_246',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_5fat_247',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fsmall_248',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5ftp_249',['mi_zalloc_tp',['../group__typed.html#gac77a61bdaf680a803785fe307820b48c',1,'mimalloc-doc.h']]], - ['modes_250',['Build Modes',['../modes.html',1,'']]] + ['mi_5ffree_5fcsize_29',['mi_free_csize',['../group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_30',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5faligned_31',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_32',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], + ['mi_5ffree_5ftp_33',['mi_free_tp',['../group__typed.html#gafc80000c300f02bdd4fe695eb53ca70a',1,'mimalloc-doc.h']]], + ['mi_5fgood_5fsize_34',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], + ['mi_5fheap_5farea_5ft_35',['mi_heap_area_t',['../group__analysis.html#structmi__heap__area__t',1,'']]], + ['mi_5fheap_5fcalloc_36',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_37',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_5fat_38',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5ftp_39',['mi_heap_calloc_tp',['../group__typed.html#ga953f22101bb354a8b6de2a6d6fda9b98',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcheck_5fowned_40',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcollect_41',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_42',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_5fblock_43',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdelete_44',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdestroy_45',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fbacking_46',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fdefault_47',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmain_48',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_49',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_50',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_5fat_51',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5fsmall_52',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5ftp_53',['mi_heap_malloc_tp',['../group__typed.html#ga02d3e85bf26aa3120132f2552c408a88',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmallocn_54',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmallocn_5ftp_55',['mi_heap_mallocn_tp',['../group__typed.html#gaa58690658cc6f5dd54c99a9971d4aa7d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_56',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fex_57',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fin_5farena_58',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fof_59',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_60',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_61',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_5fat_62',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocf_63',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocn_64',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocn_5ftp_65',['mi_heap_reallocn_tp',['../group__typed.html#ga817e315c24133294caad4d50470ff313',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealpath_66',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_67',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_68',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_5fat_69',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5ftp_70',['mi_heap_recalloc_tp',['../group__typed.html#ga7bf06ad296e20d1d74c90d64294eecec',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_71',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_72',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_5fat_73',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fdefault_74',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fnuma_5faffinity_75',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_76',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_5fjson_77',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_78',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fprint_5fout_79',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrdup_80',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrndup_81',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], + ['mi_5fheap_5ft_82',['mi_heap_t',['../group__heap.html#ga34a47cde5a5b38c29f1aa3c5e76943c2',1,'mimalloc-doc.h']]], + ['mi_5fheap_5ftheap_83',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fvisit_5fblocks_84',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fvisit_5ffun_85',['mi_heap_visit_fun',['../group__subproc.html#ga7dcbc5863df9989b45e7051999448a4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_86',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_87',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_5fat_88',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5fsmall_89',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5ftp_90',['mi_heap_zalloc_tp',['../group__typed.html#ga0aaa1aa747d8863e844c37e424ed096c',1,'mimalloc-doc.h']]], + ['mi_5fis_5fin_5fheap_5fregion_91',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], + ['mi_5fis_5fredirected_92',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_93',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_94',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_5fat_95',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fcsize_96',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fgood_5fsize_97',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsize_98',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsmall_99',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5ftp_100',['mi_malloc_tp',['../group__typed.html#ga0619a62c5fd886f1016030abe91f0557',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fusable_5fsize_101',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], + ['mi_5fmallocn_102',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], + ['mi_5fmallocn_5ftp_103',['mi_mallocn_tp',['../group__typed.html#gae5cb6e0fafc9f23169c5622e077afe8b',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_104',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_5fex_105',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], + ['mi_5fmbsdup_106',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], + ['mi_5fmemalign_107',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], + ['mi_5fnew_108',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_109',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_5fnothrow_110',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fn_111',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fnothrow_112',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], + ['mi_5fnew_5frealloc_113',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], + ['mi_5fnew_5freallocn_114',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], + ['mi_5foption_5fabandoned_5fpage_5fpurge_115',['mi_option_abandoned_page_purge',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11e62ed69200a489a5be955582078c0c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fabandoned_5freclaim_5fon_5ffree_116',['mi_option_abandoned_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca009e4b5684922ce664d73d2a8e1698d9',1,'mimalloc-doc.h']]], + ['mi_5foption_5fallow_5flarge_5fos_5fpages_117',['mi_option_allow_large_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7cc4804ced69004fa42a9a136a9ba556',1,'mimalloc-doc.h']]], + ['mi_5foption_5fallow_5fthp_118',['mi_option_allow_thp',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca04c45931ce2c82a20d08ca1471e74087',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5feager_5fcommit_119',['mi_option_arena_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5fmax_5fobject_5fsize_120',['mi_option_arena_max_object_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4dee58d60bc95ce0eeef759c61fd849b',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5fpurge_5fmult_121',['mi_option_arena_purge_mult',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8236501f1ab45d26e6fd885d191a2b5e',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5freserve_122',['mi_option_arena_reserve',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cab1c88e23ae290bbeec824038a97959de',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdestroy_5fon_5fexit_123',['mi_option_destroy_on_exit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca6364331e305e7d3c0218b058ff3afc88',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisable_124',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisallow_5farena_5falloc_125',['mi_option_disallow_arena_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeae1696100e4057ffc4182730cc04e40',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisallow_5fos_5falloc_126',['mi_option_disallow_os_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadcfb5a09580361b1be65901d2d812de6',1,'mimalloc-doc.h']]], + ['mi_5foption_5feager_5fcommit_127',['mi_option_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1e8de72c93da7ff22d91e1e27b52ac2b',1,'mimalloc-doc.h']]], + ['mi_5foption_5feager_5fcommit_5fdelay_128',['mi_option_eager_commit_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca17a190c25be381142d87e0468c4c068c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fenable_129',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], + ['mi_5foption_5fgeneric_5fcollect_130',['mi_option_generic_collect',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8b5815a577955fe8125f2b964f41ee7f',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_131',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fclamp_132',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fsize_133',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fmax_134',['mi_option_guarded_max',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca96077da4e5c841edeec0c4d6e45a5ffa',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fmin_135',['mi_option_guarded_min',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca00734c528aae52784654a8db996ae62b',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fprecise_136',['mi_option_guarded_precise',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1eaf89bb1698f988cec27b66e546760c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fsample_5frate_137',['mi_option_guarded_sample_rate',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeb3c75b1a838eb42106750979d0888ec',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fsample_5fseed_138',['mi_option_guarded_sample_seed',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7b1e41ece69f9abbe5ba81d9ea7a3dc0',1,'mimalloc-doc.h']]], + ['mi_5foption_5fis_5fenabled_139',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5ferrors_140',['mi_option_max_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caec6ecbe29d46a48205ed8823a8a52a6a',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5fsegment_5freclaim_141',['mi_option_max_segment_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa9ad9005d7017c8c30ad2d6ba31db909',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5fvabits_142',['mi_option_max_vabits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caabf8c9793d421ce28361b6d93c432de9',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5fwarnings_143',['mi_option_max_warnings',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caf9595921087e942602ee079158762665',1,'mimalloc-doc.h']]], + ['mi_5foption_5fminimal_5fpurge_5fsize_144',['mi_option_minimal_purge_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5635112095936356425d9a01910afff7',1,'mimalloc-doc.h']]], + ['mi_5foption_5fos_5ftag_145',['mi_option_os_tag',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4b74ae2a69e445de6c2361b73c1d14bf',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fcommit_5fon_5fdemand_146',['mi_option_page_commit_on_demand',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca679d9ee91221bffba4d5aafe488a8d7f',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fcross_5fthread_5fmax_5freclaim_147',['mi_option_page_cross_thread_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca671e806447c15a53c7d593afdd8d45c5',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5ffull_5fretain_148',['mi_option_page_full_retain',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeede7d0652e391940357e1c054d29829',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fmax_5fcandidates_149',['mi_option_page_max_candidates',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cad13382757101aa4fbc4984fa0f917133',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fmax_5freclaim_150',['mi_option_page_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11a13739488294318ffc03eea378f5a4',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5freclaim_5fon_5ffree_151',['mi_option_page_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cac16e3f838e861338fb2357b430f6974c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpagemap_5fcommit_152',['mi_option_pagemap_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa88f56e3bccce85542dc2b3297024236',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpurge_5fdecommits_153',['mi_option_purge_decommits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca9d15c5e3d2115eef681c17e4dd5ab9a4',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpurge_5fdelay_154',['mi_option_purge_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadd351e615acd8563529c20a347be7290',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpurge_5fextend_5fdelay_155',['mi_option_purge_extend_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca02005f164bdf03f5f00c5be726adf487',1,'mimalloc-doc.h']]], + ['mi_5foption_5freserve_5fhuge_5fos_5fpages_156',['mi_option_reserve_huge_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caca7ed041be3b0b9d0b82432c7bf41af2',1,'mimalloc-doc.h']]], + ['mi_5foption_5freserve_5fhuge_5fos_5fpages_5fat_157',['mi_option_reserve_huge_os_pages_at',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa13e7926d4339d2aa6fbf61d4473fd5c',1,'mimalloc-doc.h']]], + ['mi_5foption_5freserve_5fos_5fmemory_158',['mi_option_reserve_os_memory',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4999c828cf79a0fb2de65d23f7333',1,'mimalloc-doc.h']]], + ['mi_5foption_5fretry_5fon_5foom_159',['mi_option_retry_on_oom',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8f51df355bf6651db899e6085b54865e',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_160',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fdefault_161',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_162',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_5fdefault_163',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], + ['mi_5foption_5fshow_5ferrors_164',['mi_option_show_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4822e5c00732c5984b32a032837f0',1,'mimalloc-doc.h']]], + ['mi_5foption_5fshow_5fstats_165',['mi_option_show_stats',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0957ef73b2550764b4840edf48422fda',1,'mimalloc-doc.h']]], + ['mi_5foption_5ft_166',['mi_option_t',['../group__options.html#gafebf7ed116adb38ae5218bc3ce06884c',1,'mimalloc-doc.h']]], + ['mi_5foption_5ftarget_5fsegments_5fper_5fthread_167',['mi_option_target_segments_per_thread',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca874f36a7a1e51f7baee4a78b2830b643',1,'mimalloc-doc.h']]], + ['mi_5foption_5fuse_5fnuma_5fnodes_168',['mi_option_use_numa_nodes',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0ac33a18f6b659fcfaf44efb0bab1b74',1,'mimalloc-doc.h']]], + ['mi_5foption_5fverbose_169',['mi_option_verbose',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7c8b7bf5281c581bad64f5daa6442777',1,'mimalloc-doc.h']]], + ['mi_5foption_5fvisit_5fabandoned_170',['mi_option_visit_abandoned',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca38c67733a3956a1f4eeaca89fab9e78e',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_171',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_5fout_172',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], + ['mi_5foutput_5ffun_173',['mi_output_fun',['../group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d',1,'mimalloc-doc.h']]], + ['mi_5fposix_5fmemalign_174',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_175',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_176',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_5fout_177',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], + ['mi_5fpvalloc_178',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], + ['mi_5frealloc_179',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_180',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_5fat_181',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], + ['mi_5freallocarr_182',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], + ['mi_5freallocarray_183',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], + ['mi_5freallocf_184',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], + ['mi_5freallocn_185',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], + ['mi_5freallocn_5ftp_186',['mi_reallocn_tp',['../group__typed.html#ga1158b49a55dfa81f58a4426a7578f523',1,'mimalloc-doc.h']]], + ['mi_5frealpath_187',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_188',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_189',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_5fat_190',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], + ['mi_5fregister_5fdeferred_5ffree_191',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], + ['mi_5fregister_5ferror_192',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], + ['mi_5fregister_5foutput_193',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_194',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_195',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_196',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_197',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_5fex_198',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_199',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_200',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_5fat_201',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], + ['mi_5fsmall_5fsize_5fmax_202',['MI_SMALL_SIZE_MAX',['../group__extended.html#ga1ea64283508718d9d645c38efc2f4305',1,'mimalloc-doc.h']]], + ['mi_5fstat_5fversion_203',['MI_STAT_VERSION',['../group__stats.html#gac4411cf3eed23719e511b9e1128888e0',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fas_5fjson_204',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_205',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fbin_5fsize_206',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fjson_207',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fmerge_208',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_209',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_5fout_210',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], + ['mi_5fstats_5freset_211',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fs_212',['mi_stats_s',['../group__stats.html#structmi__stats__s',1,'']]], + ['mi_5fstats_5ft_5fdecl_213',['mi_stats_t_decl',['../group__stats.html#ga961d0f9f0327ebc9bd8a0b84f6dcb410',1,'mimalloc-doc.h']]], + ['mi_5fstl_5fallocator_214',['mi_stl_allocator',['../group__cpp.html#structmi__stl__allocator',1,'']]], + ['mi_5fstrdup_215',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], + ['mi_5fstrndup_216',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fadd_5fcurrent_5fthread_217',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fcurrent_218',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdelete_219',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdestroy_220',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_221',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fid_5ft_222',['mi_subproc_id_t',['../group__subproc.html#ga8c0bcd1fee27c7641e9c3c0d991b3b7d',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fmain_223',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fnew_224',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_225',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_5fjson_226',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fprint_5fout_227',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fvisit_5fheaps_228',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcalloc_229',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcollect_230',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fget_5fdefault_231',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_232',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5faligned_233',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fcsize_234',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fsmall_235',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5frealloc_236',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fset_5fdefault_237',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5ft_238',['mi_theap_t',['../group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_239',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fcsize_240',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fsmall_241',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fdone_242',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], + ['mi_5fthread_5finit_243',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fset_5fin_5fthreadpool_244',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], + ['mi_5fusable_5fsize_245',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], + ['mi_5fvalloc_246',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], + ['mi_5fversion_247',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], + ['mi_5fwcsdup_248',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], + ['mi_5fwdupenv_5fs_249',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_250',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_251',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_5fat_252',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fcsize_253',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fsmall_254',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5ftp_255',['mi_zalloc_tp',['../group__typed.html#gac77a61bdaf680a803785fe307820b48c',1,'mimalloc-doc.h']]], + ['modes_256',['Build Modes',['../modes.html',1,'']]] ]; diff --git a/docs/search/all_d.js b/docs/search/all_d.js index d2f41fbac..a8589b7a8 100644 --- a/docs/search/all_d.js +++ b/docs/search/all_d.js @@ -1,6 +1,7 @@ var searchData= [ ['size_0',['size',['../group__stats.html#ae41e0fff24b988d0a43c34040b04b887',1,'mi_stats_s']]], - ['statistics_1',['Statistics',['../group__stats.html',1,'']]], - ['subprocesses_2',['Subprocesses',['../group__subproc.html',1,'']]] + ['size_20allocation_1',['Constant size allocation',['../group__constantsize.html',1,'']]], + ['statistics_2',['Statistics',['../group__stats.html',1,'']]], + ['subprocesses_3',['Subprocesses',['../group__subproc.html',1,'']]] ]; diff --git a/docs/search/functions_0.js b/docs/search/functions_0.js index 2cac704d2..d16b716ee 100644 --- a/docs/search/functions_0.js +++ b/docs/search/functions_0.js @@ -20,158 +20,163 @@ var searchData= ['mi_5fexpand_17',['mi_expand',['../group__malloc.html#ga19299856216cfbb08e2628593654dfb0',1,'mimalloc-doc.h']]], ['mi_5ffree_18',['mi_free',['../group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95',1,'mimalloc-doc.h']]], ['mi_5ffree_5faligned_19',['mi_free_aligned',['../group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_20',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_5faligned_21',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsmall_22',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], - ['mi_5fgood_5fsize_23',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_24',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_25',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_5fat_26',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcheck_5fowned_27',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcollect_28',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_29',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_5fblock_30',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdelete_31',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdestroy_32',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fbacking_33',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fdefault_34',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmain_35',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_36',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_37',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_5fat_38',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5fsmall_39',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmallocn_40',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_41',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fex_42',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fin_5farena_43',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fof_44',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_45',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_46',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_5fat_47',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocf_48',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocn_49',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealpath_50',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_51',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_52',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_5fat_53',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_54',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_55',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_5fat_56',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fdefault_57',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fnuma_5faffinity_58',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_59',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_5fjson_60',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_61',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fprint_5fout_62',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrdup_63',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrndup_64',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], - ['mi_5fheap_5ftheap_65',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fvisit_5fblocks_66',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_67',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_68',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_5fat_69',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5fsmall_70',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], - ['mi_5fis_5fin_5fheap_5fregion_71',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], - ['mi_5fis_5fredirected_72',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_73',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_74',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_5fat_75',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fgood_5fsize_76',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsize_77',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsmall_78',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fusable_5fsize_79',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], - ['mi_5fmallocn_80',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_81',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_5fex_82',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], - ['mi_5fmbsdup_83',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], - ['mi_5fmemalign_84',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], - ['mi_5fnew_85',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_86',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_5fnothrow_87',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fn_88',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fnothrow_89',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], - ['mi_5fnew_5frealloc_90',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], - ['mi_5fnew_5freallocn_91',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisable_92',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], - ['mi_5foption_5fenable_93',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_94',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fclamp_95',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fsize_96',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fis_5fenabled_97',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_98',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fdefault_99',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_100',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_5fdefault_101',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_102',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_5fout_103',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], - ['mi_5fposix_5fmemalign_104',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_105',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_106',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_5fout_107',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], - ['mi_5fpvalloc_108',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], - ['mi_5frealloc_109',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_110',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_5fat_111',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], - ['mi_5freallocarr_112',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], - ['mi_5freallocarray_113',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], - ['mi_5freallocf_114',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], - ['mi_5freallocn_115',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], - ['mi_5frealpath_116',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_117',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_118',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_5fat_119',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], - ['mi_5fregister_5fdeferred_5ffree_120',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], - ['mi_5fregister_5ferror_121',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], - ['mi_5fregister_5foutput_122',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_123',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_124',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_125',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_126',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_5fex_127',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_128',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_129',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_5fat_130',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fas_5fjson_131',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_132',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fbin_5fsize_133',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fjson_134',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fmerge_135',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_136',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_5fout_137',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], - ['mi_5fstats_5freset_138',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], - ['mi_5fstrdup_139',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], - ['mi_5fstrndup_140',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fadd_5fcurrent_5fthread_141',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fcurrent_142',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdelete_143',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdestroy_144',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_145',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fmain_146',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fnew_147',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_148',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_5fjson_149',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fprint_5fout_150',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fvisit_5fheaps_151',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcalloc_152',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcollect_153',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fget_5fdefault_154',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_155',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5faligned_156',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fsmall_157',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5frealloc_158',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fset_5fdefault_159',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_160',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fsmall_161',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fdone_162',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], - ['mi_5fthread_5finit_163',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fset_5fin_5fthreadpool_164',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], - ['mi_5fusable_5fsize_165',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], - ['mi_5fvalloc_166',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], - ['mi_5fversion_167',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], - ['mi_5fwcsdup_168',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], - ['mi_5fwdupenv_5fs_169',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_170',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_171',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_5fat_172',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fsmall_173',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]] + ['mi_5ffree_5fcsize_20',['mi_free_csize',['../group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_21',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5faligned_22',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_23',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], + ['mi_5fgood_5fsize_24',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_25',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_26',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_5fat_27',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcheck_5fowned_28',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcollect_29',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_30',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_5fblock_31',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdelete_32',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdestroy_33',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fbacking_34',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fdefault_35',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmain_36',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_37',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_38',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_5fat_39',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5fsmall_40',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmallocn_41',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_42',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fex_43',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fin_5farena_44',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fof_45',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_46',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_47',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_5fat_48',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocf_49',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocn_50',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealpath_51',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_52',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_53',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_5fat_54',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_55',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_56',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_5fat_57',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fdefault_58',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fnuma_5faffinity_59',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_60',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_5fjson_61',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_62',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fprint_5fout_63',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrdup_64',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrndup_65',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], + ['mi_5fheap_5ftheap_66',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fvisit_5fblocks_67',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_68',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_69',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_5fat_70',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5fsmall_71',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], + ['mi_5fis_5fin_5fheap_5fregion_72',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], + ['mi_5fis_5fredirected_73',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_74',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_75',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_5fat_76',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fcsize_77',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fgood_5fsize_78',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsize_79',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsmall_80',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fusable_5fsize_81',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], + ['mi_5fmallocn_82',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_83',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_5fex_84',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], + ['mi_5fmbsdup_85',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], + ['mi_5fmemalign_86',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], + ['mi_5fnew_87',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_88',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_5fnothrow_89',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fn_90',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fnothrow_91',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], + ['mi_5fnew_5frealloc_92',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], + ['mi_5fnew_5freallocn_93',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisable_94',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], + ['mi_5foption_5fenable_95',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_96',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fclamp_97',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fsize_98',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fis_5fenabled_99',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_100',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fdefault_101',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_102',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_5fdefault_103',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_104',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_5fout_105',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], + ['mi_5fposix_5fmemalign_106',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_107',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_108',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_5fout_109',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], + ['mi_5fpvalloc_110',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], + ['mi_5frealloc_111',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_112',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_5fat_113',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], + ['mi_5freallocarr_114',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], + ['mi_5freallocarray_115',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], + ['mi_5freallocf_116',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], + ['mi_5freallocn_117',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], + ['mi_5frealpath_118',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_119',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_120',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_5fat_121',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], + ['mi_5fregister_5fdeferred_5ffree_122',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], + ['mi_5fregister_5ferror_123',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], + ['mi_5fregister_5foutput_124',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_125',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_126',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_127',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_128',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_5fex_129',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_130',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_131',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_5fat_132',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fas_5fjson_133',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_134',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fbin_5fsize_135',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fjson_136',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fmerge_137',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_138',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_5fout_139',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], + ['mi_5fstats_5freset_140',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], + ['mi_5fstrdup_141',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], + ['mi_5fstrndup_142',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fadd_5fcurrent_5fthread_143',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fcurrent_144',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdelete_145',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdestroy_146',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_147',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fmain_148',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fnew_149',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_150',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_5fjson_151',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fprint_5fout_152',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fvisit_5fheaps_153',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcalloc_154',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcollect_155',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fget_5fdefault_156',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_157',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5faligned_158',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fcsize_159',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fsmall_160',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5frealloc_161',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fset_5fdefault_162',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_163',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fcsize_164',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fsmall_165',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fdone_166',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], + ['mi_5fthread_5finit_167',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fset_5fin_5fthreadpool_168',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], + ['mi_5fusable_5fsize_169',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], + ['mi_5fvalloc_170',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], + ['mi_5fversion_171',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], + ['mi_5fwcsdup_172',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], + ['mi_5fwdupenv_5fs_173',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_174',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_175',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_5fat_176',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fcsize_177',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fsmall_178',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]] ]; diff --git a/docs/search/groups_0.js b/docs/search/groups_0.js index d96b393af..609010d11 100644 --- a/docs/search/groups_0.js +++ b/docs/search/groups_0.js @@ -2,6 +2,6 @@ var searchData= [ ['aligned_20allocation_0',['Aligned Allocation',['../group__aligned.html',1,'']]], ['allocation_1',['Allocation',['../group__aligned.html',1,'Aligned Allocation'],['../group__malloc.html',1,'Basic Allocation']]], - ['allocation_2',['Zero initialized re-allocation',['../group__zeroinit.html',1,'']]], + ['allocation_2',['allocation',['../group__constantsize.html',1,'Constant size allocation'],['../group__zeroinit.html',1,'Zero initialized re-allocation']]], ['arenas_3',['Arenas',['../group__arenas.html',1,'']]] ]; diff --git a/docs/search/groups_2.js b/docs/search/groups_2.js index d260dec8a..d84bd6faa 100644 --- a/docs/search/groups_2.js +++ b/docs/search/groups_2.js @@ -1,4 +1,5 @@ var searchData= [ - ['c_20wrappers_0',['C++ wrappers',['../group__cpp.html',1,'']]] + ['c_20wrappers_0',['C++ wrappers',['../group__cpp.html',1,'']]], + ['constant_20size_20allocation_1',['Constant size allocation',['../group__constantsize.html',1,'']]] ]; diff --git a/docs/search/groups_c.js b/docs/search/groups_c.js index 14bfafdcf..f65813812 100644 --- a/docs/search/groups_c.js +++ b/docs/search/groups_c.js @@ -1,5 +1,6 @@ var searchData= [ - ['statistics_0',['Statistics',['../group__stats.html',1,'']]], - ['subprocesses_1',['Subprocesses',['../group__subproc.html',1,'']]] + ['size_20allocation_0',['Constant size allocation',['../group__constantsize.html',1,'']]], + ['statistics_1',['Statistics',['../group__stats.html',1,'']]], + ['subprocesses_2',['Subprocesses',['../group__subproc.html',1,'']]] ]; diff --git a/docs/tools.html b/docs/tools.html index ec6a43d2b..fa662b9fa 100644 --- a/docs/tools.html +++ b/docs/tools.html @@ -34,7 +34,7 @@ Logo -
    mi-malloc 1.8/2.1 +
    mi-malloc 3.4/2.4/1.9
    diff --git a/docs/topics.html b/docs/topics.html index 0c97f690d..bbbae22f2 100644 --- a/docs/topics.html +++ b/docs/topics.html @@ -34,7 +34,7 @@ Logo -
    mi-malloc 1.8/2.1 +
    mi-malloc 3.4/2.4/1.9
    @@ -112,17 +112,18 @@  Basic AllocationThe basic allocation interface  Aligned AllocationAllocating aligned memory blocks  Typed MacrosTyped allocation macros - Zero initialized re-allocationRe-allocation of zero initialized blocks - HeapsFirst-class heaps - Heap IntrospectionWalk areas and blocks of the heap at runtime - ArenasArenas are large memory areas (usually 1GiB+) from which mimalloc allocates memory - SubprocessesA sub-process contains its own arena's and heaps that are fully separate from the main (sub) process - Extended FunctionsInternal functionality - StatisticsPrint out allocation statistics - Runtime OptionsSet runtime parameters - Thread-local heapsv3: Thread local heaps - Posixmi_ prefixed implementations of various Posix, Unix, and C++ allocation functions. Defined for convenience as all redirect to the regular mimalloc API - C++ wrappersmi_ prefixed implementations of various C++ allocation functions + Constant size allocationConstant size allocation and freeing For example: + Zero initialized re-allocationRe-allocation of zero initialized blocks + HeapsFirst-class heaps + Heap IntrospectionWalk areas and blocks of the heap at runtime + ArenasArenas are large memory areas (usually 1GiB+) from which mimalloc allocates memory + SubprocessesA sub-process contains its own arena's and heaps that are fully separate from the main (sub) process + Extended FunctionsInternal functionality + StatisticsPrint out allocation statistics + Runtime OptionsSet runtime parameters + Thread-local heapsv3: Thread local heaps + Posixmi_ prefixed implementations of various Posix, Unix, and C++ allocation functions. Defined for convenience as all redirect to the regular mimalloc API + C++ wrappersmi_ prefixed implementations of various C++ allocation functions
    diff --git a/docs/topics.js b/docs/topics.js index a1bd9f7d1..99067add9 100644 --- a/docs/topics.js +++ b/docs/topics.js @@ -3,6 +3,7 @@ var topics = [ "Basic Allocation", "group__malloc.html", "group__malloc" ], [ "Aligned Allocation", "group__aligned.html", "group__aligned" ], [ "Typed Macros", "group__typed.html", "group__typed" ], + [ "Constant size allocation", "group__constantsize.html", "group__constantsize" ], [ "Zero initialized re-allocation", "group__zeroinit.html", "group__zeroinit" ], [ "Heaps", "group__heap.html", "group__heap" ], [ "Heap Introspection", "group__analysis.html", "group__analysis" ], diff --git a/docs/using.html b/docs/using.html index 6891dc68e..87f479dce 100644 --- a/docs/using.html +++ b/docs/using.html @@ -34,7 +34,7 @@ Logo -
    mi-malloc 1.8/2.1 +
    mi-malloc 3.4/2.4/1.9
    diff --git a/ide/vs2022/mimalloc-lib.vcxproj b/ide/vs2022/mimalloc-lib.vcxproj index 6c1ed4ba2..216f418c0 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj +++ b/ide/vs2022/mimalloc-lib.vcxproj @@ -433,7 +433,6 @@ - false @@ -451,6 +450,8 @@ true + + diff --git a/ide/vs2022/mimalloc-lib.vcxproj.filters b/ide/vs2022/mimalloc-lib.vcxproj.filters index 1f7a2268f..0c23b2834 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj.filters +++ b/ide/vs2022/mimalloc-lib.vcxproj.filters @@ -58,15 +58,18 @@ Sources - - Sources - Sources Sources + + Sources + + + Sources + diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj b/ide/vs2022/mimalloc-override-dll.vcxproj index 3c432780f..18fce6938 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj +++ b/ide/vs2022/mimalloc-override-dll.vcxproj @@ -466,7 +466,6 @@ - @@ -480,6 +479,8 @@ true + + diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj.filters b/ide/vs2022/mimalloc-override-dll.vcxproj.filters index e0016d040..f57933c17 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj.filters +++ b/ide/vs2022/mimalloc-override-dll.vcxproj.filters @@ -58,15 +58,18 @@ Sources - - Sources - Sources Sources + + Sources + + + Sources + diff --git a/ide/vs2022/mimalloc-override-static-lib.vcxproj b/ide/vs2022/mimalloc-override-static-lib.vcxproj index a5a39452b..58f2ff9b4 100644 --- a/ide/vs2022/mimalloc-override-static-lib.vcxproj +++ b/ide/vs2022/mimalloc-override-static-lib.vcxproj @@ -442,7 +442,6 @@ true true - false @@ -463,6 +462,7 @@ + true @@ -489,6 +489,7 @@ + diff --git a/ide/vs2022/mimalloc-override-static-lib.vcxproj.filters b/ide/vs2022/mimalloc-override-static-lib.vcxproj.filters index aab2cfb4f..06aea7fed 100644 --- a/ide/vs2022/mimalloc-override-static-lib.vcxproj.filters +++ b/ide/vs2022/mimalloc-override-static-lib.vcxproj.filters @@ -64,10 +64,13 @@ Sources - + Sources - + + Sources + + Sources diff --git a/ide/vs2022/mimalloc-test-stress.vcxproj b/ide/vs2022/mimalloc-test-stress.vcxproj index d6af71cee..128a4ff6c 100644 --- a/ide/vs2022/mimalloc-test-stress.vcxproj +++ b/ide/vs2022/mimalloc-test-stress.vcxproj @@ -282,8 +282,8 @@ - - {abb5eae7-b3e6-432e-b636-333449892ea6} + + {abb5eae7-b3e6-432e-b636-333449892ea7} diff --git a/include/mimalloc-new-delete.h b/include/mimalloc-new-delete.h index c16f4a665..aaf185bb1 100644 --- a/include/mimalloc-new-delete.h +++ b/include/mimalloc-new-delete.h @@ -48,7 +48,7 @@ terms of the MIT license. A copy of the license can be found in the file void operator delete[](void* p, std::size_t n) noexcept { mi_free_size(p,n); }; #endif - #if (__cplusplus > 201402L || defined(__cpp_aligned_new)) + #if (__cplusplus > 201402L && defined(__cpp_aligned_new)) void operator delete (void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } void operator delete[](void* p, std::align_val_t al) noexcept { mi_free_aligned(p, static_cast(al)); } void operator delete (void* p, std::size_t n, std::align_val_t al) noexcept { mi_free_size_aligned(p, n, static_cast(al)); }; diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index 4fc4c7129..853ff9c68 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -22,7 +22,7 @@ terms of the MIT license. A copy of the license can be found in the file #elif __cplusplus >= 201103L #define mi_decl_align(a) alignas(a) #else -#define mi_decl_align(a) +#define mi_decl_align(a) _Alignas(a) #endif diff --git a/include/mimalloc.h b/include/mimalloc.h index d41a1920e..932a458a0 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 30403 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 30500 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes @@ -133,6 +133,9 @@ mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; +// `mi_free_size` can be more efficient (as at calls mi_free_small internally). +mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; + // `mi_free_small` is for special applications like language runtimes. // it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free` mi_decl_export void mi_free_small(void* p) mi_attr_noexcept; @@ -157,12 +160,13 @@ mi_decl_nodiscard mi_decl_export void* mi_realloc_aligned_at(void* p, size_t new // Typed allocation, the type is always the first parameter // ------------------------------------------------------ -#define mi_malloc_tp(tp) ((tp*)mi_malloc(sizeof(tp))) -#define mi_zalloc_tp(tp) ((tp*)mi_zalloc(sizeof(tp))) +#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp))) +#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp))) #define mi_calloc_tp(tp,n) ((tp*)mi_calloc(n,sizeof(tp))) #define mi_mallocn_tp(tp,n) ((tp*)mi_mallocn(n,sizeof(tp))) #define mi_reallocn_tp(tp,p,n) ((tp*)mi_reallocn(p,n,sizeof(tp))) #define mi_recalloc_tp(tp,p,n) ((tp*)mi_recalloc(p,n,sizeof(tp))) +#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp))) #define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp))) #define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp))) @@ -451,8 +455,29 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_calloc(mi_theap mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); +mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(2) mi_attr_alloc_align(3); mi_decl_nodiscard mi_decl_export void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept mi_attr_alloc_size(3); +// ------------------------------------------------------ +// Fast constant size allocations. +// ------------------------------------------------------ + +static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_malloc_small(size); } else { return mi_malloc(size); } +} +static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_zalloc_small(size); } else { return mi_zalloc(size); } +} +static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_malloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } +} +static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_zalloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } +} +static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } +} // ------------------------------------------------------ // Experimental @@ -606,7 +631,6 @@ mi_decl_nodiscard mi_decl_export int mi_reallocarr(void* ptrp, size_t count, s mi_decl_nodiscard mi_decl_export void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset) mi_attr_noexcept; -mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; mi_decl_export void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept; mi_decl_export void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept; mi_decl_export int mi_dupenv_s(char** buf, size_t* size, const char* name) mi_attr_noexcept; diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index 565cbd0f1..ffde71d0c 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -9,12 +9,13 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ATOMIC_H // include windows.h or pthreads.h -#if defined(_WIN32) +#if defined(_WIN32) || defined(__CYGWIN__) // we use windows locks on cygwin, but otherwise treat it at unix #ifndef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN #endif #include -#elif MI_TLS_MODEL_PTHREADS || (!defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) +#endif +#if MI_TLS_MODEL_PTHREADS || (!defined(_WIN32) && !defined(__wasi__) && (!defined(__EMSCRIPTEN__) || defined(__EMSCRIPTEN_PTHREADS__))) #define MI_USE_PTHREADS 1 #include #endif @@ -215,16 +216,12 @@ static inline uintptr_t mi_atomic_exchange_explicit(_Atomic(uintptr_t)*p, uintpt (void)(mo); return (uintptr_t)MI_MSC_64(_InterlockedExchange)((volatile msc_intptr_t*)p, (msc_intptr_t)exchange); } -static inline void mi_atomic_thread_fence(mi_memory_order mo) { - (void)(mo); - _Atomic(uintptr_t) x = 0; - mi_atomic_exchange_explicit(&x, 1, mo); -} static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_memory_order mo) { (void)(mo); // assert(mo<=mi_memory_order_acquire); // others are not used by mimalloc #if defined(_M_IX86) || defined(_M_X64) + // on x86/x64 we have a strong memory model so any load is acquire return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); #elif defined(_M_ARM) || defined(_M_ARM64) if (mo == mi_memory_order_relaxed) { @@ -335,18 +332,18 @@ static inline void mi_atomic_void_addi64_relaxed(volatile int64_t* p, const vola } } -static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)*p, int64_t x) { +static inline void mi_atomic_maxi64_relaxed(volatile _Atomic(int64_t)* p, int64_t x) { int64_t current; do { current = *p; } while (current < x && _InterlockedCompareExchange64(p, x, current) != current); } -static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t*)p, int64_t i) { +static inline void mi_atomic_addi64_acq_rel(volatile _Atomic(int64_t)* p, int64_t i) { mi_atomic_addi64_relaxed(p, i); } -static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, int64_t* exp, int64_t des) { +static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t)* p, int64_t* exp, int64_t des) { const int64_t read = _InterlockedCompareExchange64(p, des, *exp); if (read == *exp) { return true; @@ -358,17 +355,17 @@ static inline bool mi_atomic_casi64_strong_acq_rel(volatile _Atomic(int64_t*)p, } // The pointer macros cast to `uintptr_t`. -#define mi_atomic_load_ptr_acquire(tp,p) (tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p)) -#define mi_atomic_load_ptr_relaxed(tp,p) (tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p)) +#define mi_atomic_load_ptr_acquire(tp,p) ((tp*)mi_atomic_load_acquire((_Atomic(uintptr_t)*)(p))) +#define mi_atomic_load_ptr_relaxed(tp,p) ((tp*)mi_atomic_load_relaxed((_Atomic(uintptr_t)*)(p))) #define mi_atomic_store_ptr_release(tp,p,x) mi_atomic_store_release((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) #define mi_atomic_store_ptr_relaxed(tp,p,x) mi_atomic_store_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)(x)) #define mi_atomic_cas_ptr_weak_release(tp,p,exp,des) mi_atomic_cas_weak_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) #define mi_atomic_cas_ptr_weak_acq_rel(tp,p,exp,des) mi_atomic_cas_weak_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) #define mi_atomic_cas_ptr_strong_release(tp,p,exp,des) mi_atomic_cas_strong_release((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) #define mi_atomic_cas_ptr_strong_acq_rel(tp,p,exp,des) mi_atomic_cas_strong_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t*)exp,(uintptr_t)des) -#define mi_atomic_exchange_ptr_relaxed(tp,p,x) (tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x) -#define mi_atomic_exchange_ptr_release(tp,p,x) (tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x) -#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) (tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x) +#define mi_atomic_exchange_ptr_relaxed(tp,p,x) ((tp*)mi_atomic_exchange_relaxed((_Atomic(uintptr_t)*)(p),(uintptr_t)x)) +#define mi_atomic_exchange_ptr_release(tp,p,x) ((tp*)mi_atomic_exchange_release((_Atomic(uintptr_t)*)(p),(uintptr_t)x)) +#define mi_atomic_exchange_ptr_acq_rel(tp,p,x) ((tp*)mi_atomic_exchange_acq_rel((_Atomic(uintptr_t)*)(p),(uintptr_t)x)) #define mi_atomic_loadi64_acquire(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(acquire)) #define mi_atomic_loadi64_relaxed(p) mi_atomic(loadi64_explicit)(p,mi_memory_order(relaxed)) @@ -417,7 +414,7 @@ typedef _Atomic(uintptr_t) mi_atomic_guard_t; #define mi_lock_maybe(lock,acquire) for(bool _mi_go = (acquire ? (mi_lock_acquire(lock),true) : true); _mi_go; _mi_go = (acquire ? (mi_lock_release(lock),false) : false) ) -#if defined(_WIN32) +#if defined(_WIN32) || defined(__CYGWIN__) typedef struct mi_lock_s { SRWLOCK mutex; // slim reader-writer lock diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index 86f173e54..8c3e4710f 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -103,13 +103,13 @@ typedef int32_t mi_ssize_t; #include #endif -#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // msvc +#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__BMI2__) // avx2 implies bmi2 #define __BMI2__ 1 #endif -#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__)) && !defined(__BMI1__) // msvc +#if MI_ARCH_X64 && (defined(__AVX2__) || defined(__BMI2__) || defined(__BMI__)) && !defined(__BMI1__) // bmi2 implies bmi1 #define __BMI1__ 1 #endif -#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // msvc +#if MI_ARCH_X64 && defined(__AVX2__) && !defined(__LZCNT__) // avx2 implies lzcnt #define __LZCNT__ 1 #endif @@ -138,7 +138,7 @@ typedef int32_t mi_ssize_t; // use a flat page-map or a 2-level one #ifndef MI_PAGE_MAP_FLAT -#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 && !MI_PAGE_META_IS_SEPARATED +#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 // && !MI_PAGE_META_IS_SEPARATED #define MI_PAGE_MAP_FLAT 1 #else #define MI_PAGE_MAP_FLAT 0 @@ -193,7 +193,7 @@ static inline size_t mi_popcount(size_t x) { #if mi_has_builtinz(popcount) return mi_builtinz(popcount)(x); #elif defined(_MSC_VER) && (MI_ARCH_ARM64 || MI_ARCH_ARM32) - return mi_msc_builtinz(__popcnt)(x); + return mi_msc_builtinz(_CountOneBits)(x); #elif defined(_MSC_VER) && (MI_ARCH_X64 || MI_ARCH_X86) if (_mi_cpu_has_popcnt) { return mi_msc_builtinz(__popcnt)(x); } else { return _mi_popcount_generic(x); } // see issue #1291 diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 88d943d10..6a19d5658 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -40,7 +40,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold -#define mi_decl_unused #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #if !MI_TRACK_ASAN #define mi_decl_forceinline __attribute__((always_inline)) inline @@ -51,8 +50,11 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_thread __thread #define mi_decl_noreturn __attribute__((noreturn)) #define mi_decl_weak __attribute__((weak)) +#if defined(__MINGW32__) || defined(__CYGWIN__) +#define mi_decl_hidden +#else #define mi_decl_hidden __attribute__((visibility("hidden"))) -#define mi_decl_unused __attribute__((unused)) +#endif #if (__GNUC__ >= 4) || defined(__clang__) #define mi_decl_cold __attribute__((cold)) #else @@ -66,11 +68,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold -#if __cplusplus >= 201703L // c++17 -#define mi_decl_unused [[maybe_unused]] -#else -#define mi_decl_unused -#endif #else #define mi_decl_forceinline inline #define mi_decl_noinline @@ -79,7 +76,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold -#define mi_decl_unused #endif #if defined(__GNUC__) || defined(__clang__) @@ -93,16 +89,6 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_likely(x) (x) #endif -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - -#if defined(__cplusplus) -#define mi_decl_externc extern "C" -#else -#define mi_decl_externc -#endif - #if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc #define mi_decl_maybe_unused __attribute__((unused)) #elif __cplusplus >= 201703L // c++17 @@ -111,10 +97,16 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_maybe_unused #endif +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + #if defined(__cplusplus) #define mi_decl_externc extern "C" +#define mi_init_struct_zero { } #else #define mi_decl_externc +#define mi_init_struct_zero { 0 } #endif @@ -134,19 +126,20 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...); char _mi_toupper(char c); int _mi_strnicmp(const char* s, const char* t, size_t n); -void _mi_strlcpy(char* dest, const char* src, size_t dest_size); -void _mi_strlcat(char* dest, const char* src, size_t dest_size); +bool _mi_strlcpy(char* dest, const char* src, size_t dest_size); // returns true if the entire src was copied +bool _mi_strlcat(char* dest, const char* src, size_t dest_size); // returns true if the entire src was appended size_t _mi_strlen(const char* s); size_t _mi_strnlen(const char* s, size_t max_len); char* _mi_strnstr(char* s, size_t max_len, const char* pat); bool _mi_streq(const char* s, const char* t); int _mi_getenv(const char* name, char* result, size_t result_size); +void _mi_detect_cpu_features(void); // "options.c" void _mi_fputs(mi_output_fun* out, void* arg, const char* prefix, const char* message); void _mi_fprintf(mi_output_fun* out, void* arg, const char* fmt, ...); void _mi_raw_message(const char* fmt, ...); -void _mi_message(const char* fmt, ...); +void _mi_message(const char* fmt, ...); // fork: informational output that is not gated on verbose void _mi_warning_message(const char* fmt, ...); void _mi_verbose_message(const char* fmt, ...); void _mi_trace_message(const char* fmt, ...); @@ -165,38 +158,51 @@ uintptr_t _mi_theap_random_next(mi_theap_t* theap); uintptr_t _mi_os_random_weak(uintptr_t extra_seed); static inline uintptr_t _mi_random_shuffle(uintptr_t x); +// prim-tls.c +void _mi_tls_slots_init(void); +void _mi_tls_slots_done(void); +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; +void _mi_theap_default_set(mi_theap_t* theap); +void _mi_theap_cached_set(mi_theap_t* theap); + +// subproc.c +mi_subproc_t* _mi_subproc_main_init(void); +void _mi_subproc_main_done(void); +mi_subproc_t* _mi_subproc_main(void); +bool _mi_subproc_is_main(mi_subproc_t* subproc); +mi_subproc_t* _mi_subproc(void); // current subproc of this thread +mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc); +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); +void _mi_subprocs_unsafe_destroy_all(void); + +void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ); +void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* p, size_t newsize, mi_memid_t* memid ); +void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t alignment, mi_memid_t* memid ); +void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid); +bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* p); + + // init.c -extern mi_decl_hidden mi_decl_cache_align const mi_page_t _mi_page_empty; +mi_page_t* _mi_page_empty_get(void); void _mi_auto_process_init(void); void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept; bool _mi_is_redirected(void); bool _mi_allocator_init(const char** message); void _mi_allocator_done(void); -bool _mi_is_main_thread(void); -bool _mi_is_process_heap_main(const mi_heap_t* heap); bool _mi_preloading(void); // true while the C runtime is not initialized yet void _mi_thread_done(mi_theap_t* theap); mi_theap_t* _mi_thread_init(void); +mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap); bool _mi_is_empty_theap(const mi_theap_t* theap); -mi_subproc_t* _mi_subproc(void); -mi_subproc_t* _mi_subproc_main(void); void _mi_process_fork_prepare(void); +extern mi_decl_hidden bool _mi_process_is_initialized; extern mi_decl_hidden bool _mi_process_is_forked_child; void _mi_process_fork_parent(void); void _mi_process_fork_child(void); void _mi_thread_locals_fork_prepare(void); void _mi_thread_locals_fork_parent(void); void _mi_thread_locals_fork_child(void); -mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc); -mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id); - -mi_threadid_t _mi_thread_id(void) mi_attr_noexcept; -size_t _mi_thread_seq_id(void) mi_attr_noexcept; -bool _mi_is_theap_main(const mi_theap_t* theap); -void _mi_theap_guarded_init(mi_theap_t* theap); -void _mi_theap_options_init(mi_theap_t* theap); - // os.c void _mi_os_init(void); // called from process init void* _mi_os_alloc(mi_subproc_t* subproc, size_t size, mi_memid_t* memid); @@ -260,7 +266,6 @@ bool _mi_arena_memid_is_suitable(mi_memid_t memid, mi_arena_t* request_ void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); void* _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alignment, size_t align_offset, bool commit, bool allow_pinned, mi_arena_t* req_arena, size_t tseq, int numa_node, mi_memid_t* memid); void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); -bool _mi_arenas_contain(const void* p); void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld); void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld); void _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq); @@ -295,11 +300,6 @@ void _mi_prof_theap_lazy_enable(mi_theap_t* theap); size_t _mi_prof_rate(void); void _mi_prof_on_exit(void); -// arena-meta.c -void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ); -void _mi_meta_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); -bool _mi_meta_is_meta_page(mi_subproc_t* subproc, void* p); - // "page-map.c" bool _mi_page_map_init(void); mi_decl_nodiscard bool _mi_page_map_register(mi_page_t* page); @@ -309,16 +309,13 @@ mi_page_t* _mi_safe_ptr_page(const void* p); void _mi_page_map_unsafe_destroy(void); // "page.c" -void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_theap_t* heap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free the page void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... - -size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append); void _mi_deferred_free(mi_theap_t* theap, bool force); - void _mi_page_free_collect(mi_page_t* page, bool force); void _mi_page_free_collect_no_unpurge(mi_page_t* page, bool force); // for read-only heap inspection: never un-purges a hole void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); @@ -330,20 +327,15 @@ size_t _mi_bin_size(size_t bin); // for stats size_t _mi_bin(size_t size); // for stats // "theap.c" +void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld); +mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld); mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld); -void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock); -void _mi_theap_default_set(mi_theap_t* theap); -void _mi_theap_cached_set(mi_theap_t* theap); void _mi_theap_collect_retired(mi_theap_t* theap, bool force); void _mi_theap_collect_abandon(mi_theap_t* theap); // May the calling thread touch the owner-only state of `theap` (its plain free lists, page // queues and non-atomic stats)? Either we own it, or its owner parked and handed it to us for // the duration of a sweep (see `mi_on_thread_idle_start`). Both mean: no concurrent owner. -static inline bool _mi_theap_can_touch(const mi_theap_t* theap) { - if (theap == NULL || theap->tld == NULL) return true; - if (theap->tld->thread_id == _mi_thread_id()) return true; - return (mi_atomic_load_acquire(&theap->tld->park_state) == MI_PARK_SWEEPING); -} + void _mi_thread_idle_work(mi_tld_t* tld, mi_theap_t* theap0) mi_attr_noexcept; void _mi_theap_sweep_parked(mi_subproc_t* subproc); @@ -351,17 +343,20 @@ void _mi_park_leave(mi_tld_t* tld); void _mi_scavenger_forked_child(void); bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi_block_visit_fun* visitor, void* arg); void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page); -bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock); + +void _mi_heap_detach_theaps( mi_heap_t* heap ); +void _mi_tld_detach_theaps( mi_tld_t* tld ); void _mi_theap_incref(mi_theap_t* theap); void _mi_theap_decref(mi_theap_t* theap); -bool _mi_page_visit_blocks( mi_page_t* page, mi_block_visit_fun* visitor, void* arg ); +void _mi_theap_merge_stats(mi_theap_t* theap); // "heap.c" +void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id); void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page); mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap); // get (and possible create) the theap belonging to a heap void _mi_heap_move_pages(mi_heap_t* heap_from, mi_heap_t* heap_to); // in "arena.c" void _mi_heap_destroy_pages(mi_heap_t* heap_from); // in "arena.c" -void _mi_heap_force_destroy(mi_heap_t* heap); // allow destroying the main heap +void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock); // allow destroying the main heap mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id, bool is_heap_main); bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap); @@ -375,15 +370,18 @@ mi_msecs_t _mi_clock_start(void); // "alloc.c" void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept; -void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept; +void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); // "free.c" -void _mi_free_subproc_safe(void* p); +void _mi_free_in_page(void* p, mi_page_t* page) mi_attr_noexcept; +void _mi_free_subproc_safe_in_page(void* p, mi_page_t* page) mi_attr_noexcept; +void _mi_free_subproc_safe(void* p) mi_attr_noexcept; void _mi_page_unguard_all(mi_page_t* page); +size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept; #if MI_DEBUG>1 bool _mi_page_is_valid(mi_page_t* page); @@ -478,7 +476,7 @@ static inline void* mi_pthread_key_get(pthread_key_t key) { #if !MI_PTHREADS_GET_INVALID_KEY_IS_NULL if mi_unlikely(key==MI_PTHREAD_KEY_INVALID) return NULL; #endif - return pthread_getspecific(key); + return pthread_getspecific(key); } static inline bool mi_pthread_key_set(pthread_key_t* pkey, void* val) { @@ -546,6 +544,11 @@ static inline bool _mi_is_power_of_two(uintptr_t x) { return ((x & (x - 1)) == 0); } +// valid alignment values are as posix memalign: +static inline bool mi_alignment_is_valid(size_t alignment) { + return ((alignment!=0) && _mi_is_power_of_two(alignment)); +} + // Is a pointer aligned? static inline bool _mi_is_aligned(const void* p, size_t alignment) { return (alignment==0 || ((uintptr_t)p % alignment) == 0); @@ -554,7 +557,7 @@ static inline bool _mi_is_aligned(const void* p, size_t alignment) { // Align upwards static inline uintptr_t _mi_align_up(uintptr_t sz, size_t alignment) { mi_assert_internal(alignment != 0); - uintptr_t mask = alignment - 1; + const uintptr_t mask = alignment - 1; if ((alignment & mask) == 0) { // power of two? return ((sz + mask) & ~mask); } @@ -568,18 +571,19 @@ static inline void* _mi_align_up_ptr(const void* p, size_t alignment) { return (void*)_mi_align_up((uintptr_t)p, alignment); } +// Align down static inline uintptr_t _mi_align_down(uintptr_t sz, size_t alignment) { mi_assert_internal(alignment != 0); - uintptr_t mask = alignment - 1; - if ((alignment & mask) == 0) { // power of two? + const uintptr_t mask = alignment - 1; + if ((alignment & mask) == 0) { // power of two? return (sz & ~mask); } else { - return ((sz / alignment) * alignment); + return ((sz/alignment)*alignment); } } -// align a pointer downwards +// Align a pointer downwards static inline void* _mi_align_down_ptr(const void* p, size_t alignment) { return (void*)_mi_align_down((uintptr_t)p, alignment); } @@ -620,10 +624,10 @@ static inline size_t _mi_wsize_from_size(size_t size) { #undef _CLOCK_T #endif static inline bool mi_mul_overflow(size_t count, size_t size, size_t* total) { - #if (SIZE_MAX == ULONG_MAX) - return __builtin_umull_overflow(count, size, (unsigned long *)total); - #elif (SIZE_MAX == UINT_MAX) + #if (SIZE_MAX == UINT_MAX) return __builtin_umul_overflow(count, size, (unsigned int *)total); + #elif (SIZE_MAX == ULONG_MAX) + return __builtin_umull_overflow(count, size, (unsigned long *)total); #else return __builtin_umulll_overflow(count, size, (unsigned long long *)total); #endif @@ -663,8 +667,8 @@ static inline bool mi_count_size_overflow(size_t count, size_t size, size_t* tot Heap functions ------------------------------------------------------------------------------------------- */ -extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL) -extern mi_decl_hidden const mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated +extern mi_decl_hidden const mi_theap_t _mi_theap_empty; // read-only empty theap, initial value of the thread local default theap (in the MI_TLS_MODEL_LOCAL) +extern mi_decl_hidden mi_theap_t _mi_theap_empty_wrong; // read-only empty theap used to signal that a theap for a heap could not be allocated static inline mi_heap_t* _mi_theap_heap_peek(const mi_theap_t* theap) { @@ -694,12 +698,21 @@ static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t return theap->pages_free_direct[idx]; } -//static inline uintptr_t _mi_ptr_cookie(const void* p) { -// extern mi_theap_t _mi_theap_main; -// mi_assert_internal(_mi_theap_main.cookie != 0); -// return ((uintptr_t)p ^ _mi_theap_main.cookie); -//} +static inline bool mi_theap_is_detached(mi_theap_t* theap) { + return (theap!=NULL && theap->tld->thread_id == MI_THREADID_DETACHED); +} + +static inline bool _mi_theap_can_touch(const mi_theap_t* theap) { + if (theap == NULL || theap->tld == NULL) return true; + if (theap->tld->thread_id == _mi_thread_id()) return true; + if (mi_theap_is_detached((mi_theap_t*)theap)) return true; // upstream's detached theaps (meta-data / heaps without a thread) belong to no thread + return (mi_atomic_load_acquire(&theap->tld->park_state) == MI_PARK_SWEEPING); +} +static inline bool mi_theap_matches_thread(mi_theap_t* theap) { + const mi_threadid_t tid = _mi_thread_id(); + return (theap==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); +} /* ----------------------------------------------------------- The page map maps addresses to `mi_page_t` pointers @@ -755,11 +768,17 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { #define MI_PAGE_MAP_SUB_SHIFT (13) #define MI_PAGE_MAP_SUB_COUNT (MI_ZU(1) << MI_PAGE_MAP_SUB_SHIFT) #define MI_PAGE_MAP_SHIFT (MI_MAX_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT) -#define MI_PAGE_MAP_COUNT (MI_ZU(1) << MI_PAGE_MAP_SHIFT) typedef mi_page_t** mi_submap_t; -extern mi_decl_hidden _Atomic(mi_submap_t)* _mi_page_map; -extern mi_decl_hidden _Atomic(void*) _mi_page_map_max_address; +typedef struct mi_page_map_s { + _Atomic(size_t) committed_count; // currently committed entries + size_t reserved_size; // full reserved size (mi_page_map_t + submaps) + mi_memid_t memid; // provenance + mi_lock_t lock; // used when allocating new submaps + mi_decl_cache_align _Atomic(mi_submap_t) submaps[1]; +} mi_page_map_t; + +extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map; static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) { const size_t u = (size_t)((uintptr_t)p / MI_ARENA_SLICE_SIZE); @@ -767,25 +786,28 @@ static inline size_t _mi_page_map_index(const void* p, size_t* sub_idx) { return (u / MI_PAGE_MAP_SUB_COUNT); } -static inline mi_submap_t _mi_page_map_at(size_t idx) { - return mi_atomic_load_ptr_relaxed(mi_page_t*, &_mi_page_map[idx]); +static inline mi_page_map_t* _mi_page_map(void) { + return mi_atomic_load_ptr_relaxed(mi_page_map_t,&__mi_page_map); +} + +static inline mi_submap_t _mi_page_map_at(const mi_page_map_t* pmap, size_t idx) { + return mi_atomic_load_ptr_relaxed(mi_page_t*, &pmap->submaps[idx]); } static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { + const mi_page_map_t* pmap = _mi_page_map(); size_t sub_idx; const size_t idx = _mi_page_map_index(p, &sub_idx); - return (_mi_page_map_at(idx))[sub_idx]; // NULL if p==NULL + return _mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL } static inline mi_page_t* _mi_checked_ptr_page(const void* p) { - #if MI_MIN_VABITS < MI_INTPTR_BITS - if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { - if (p > mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; - } - #endif + const mi_page_map_t* pmap = _mi_page_map(); + const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); size_t sub_idx; const size_t idx = _mi_page_map_index(p, &sub_idx); - mi_submap_t const sub = _mi_page_map_at(idx); + if mi_unlikely(idx > committed_count) return NULL; + mi_submap_t const sub = _mi_page_map_at(pmap,idx); if mi_unlikely(sub == NULL) return NULL; return sub[sub_idx]; } @@ -811,7 +833,7 @@ static inline size_t mi_page_block_size(const mi_page_t* page) { // Page start static inline uint8_t* mi_page_start(const mi_page_t* page) { // multiplication must be done in `size_t`; in a 32-bit multiplication the offset wraps for pages whose blocks start 4 GiB or more after the page meta info - return (uint8_t*)page + ((size_t)page->page_woffset * MI_SIZE_SIZE); + return (uint8_t*)page + (((size_t)page->page_ma_offset) * MI_MAX_ALIGN_SIZE); } static inline size_t mi_page_size(const mi_page_t* page) { @@ -873,9 +895,15 @@ static inline size_t mi_page_slice_offset_of(const mi_page_t* page, size_t offse return (mi_page_start(page) - mi_page_slice_start(page)) + offset_relative_to_page_start; } +// How much of the page is committed relative to the slice start? (or 0 if fully committed already) +static inline size_t mi_page_slice_committed(const mi_page_t* page) { + return ((size_t)page->slice_pcommitted * _mi_os_page_size()); +} + // Currently committed part of a page static inline size_t mi_page_committed(const mi_page_t* page) { - return (page->slice_committed == 0 ? mi_page_size(page) : page->slice_committed - mi_page_slice_offset_of(page,0)); + const size_t slice_committed = mi_page_slice_committed(page); + return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0)); } // are all blocks in a page freed? @@ -1088,6 +1116,10 @@ static inline mi_page_queue_t* mi_page_queue(const mi_theap_t* theap, size_t siz return pq; } +static inline size_t mi_page_min_commit_size(void) { + const size_t psize = _mi_os_page_size(); + return (MI_PAGE_MIN_COMMIT_SIZE >= psize ? MI_PAGE_MIN_COMMIT_SIZE : psize); +} //----------------------------------------------------------- // Page thread id and flags @@ -1125,7 +1157,8 @@ static inline void mi_page_set_in_full(mi_page_t* page, bool in_full) { mi_theap_t* const theap = page->theap; mi_assert_internal(theap!=NULL); if (theap != NULL) { - const size_t size = page->capacity * mi_page_block_size(page); + mi_assert_internal(page->capacity==page->reserved); + const size_t size = page->reserved * mi_page_block_size(page); if (in_full) { theap->pages_full_size += size; } else { mi_assert_internal(size <= theap->pages_full_size); theap->pages_full_size -= size; } } @@ -1211,7 +1244,10 @@ static inline bool _mi_is_heap_main(const mi_heap_t* heap) { return (mi_heap_get_heap_main(heap) == heap); } - +static inline bool _mi_is_process_heap_main(const mi_heap_t* heap) { + mi_assert_internal(heap!=NULL); + return (_mi_subproc_main()->heap_main == heap); +} //----------------------------------------------------------- // Thread free list and ownership @@ -1280,23 +1316,28 @@ static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) { theap->guarded_sample_count = count; return false; } - else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { - // use guarded allocation - theap->guarded_sample_count = theap->guarded_sample_rate; // reset - return (theap->guarded_sample_rate != 0); - } else { - // failed size criteria, rewind count (but don't write to an empty theap) - if (theap->guarded_sample_rate != 0) { theap->guarded_sample_count = 1; } - return false; + // count == 0 + const size_t rate = theap->guarded_sample_rate; + if (rate == 0) { + return false; // don't write to an empty theap + } + else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { + // use guarded allocation + theap->guarded_sample_count = rate; // reset + return true; + } + else { + // failed size criteria, rewind count + theap->guarded_sample_count = 1; + return false; + } } } -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept; - +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; #endif - /* ------------------------------------------------------------------- Encoding/Decoding the free list next pointers @@ -1397,6 +1438,8 @@ static inline void mi_block_set_next(const mi_page_t* page, mi_block_t* block, c #endif } + + /* ----------------------------------------------------------- arena blocks ----------------------------------------------------------- */ @@ -1437,17 +1480,40 @@ static inline mi_memid_t _mi_memid_create_os(void* base, size_t size, bool commi return memid; } -static inline mi_memid_t _mi_memid_create_meta(mi_meta_page_t* mpage, size_t block_idx, size_t block_count) { - mi_memid_t memid = _mi_memid_create(MI_MEM_META); - memid.mem.meta.meta_page = mpage; - memid.mem.meta.block_index = (uint32_t)block_idx; - memid.mem.meta.block_count = (uint32_t)block_count; +static inline mi_memid_t _mi_memid_create_static(void* p, size_t size) { + mi_memid_t memid = _mi_memid_create(MI_MEM_STATIC); + memid.mem.malloc.base = p; + memid.mem.malloc.size = size; + memid.initially_committed = true; + memid.is_pinned = true; + return memid; +} + +static inline mi_memid_t _mi_memid_create_malloc(void* p, size_t size, bool iszero) { + mi_memid_t memid = _mi_memid_create(MI_MEM_MALLOC); + memid.mem.malloc.base = p; + memid.mem.malloc.size = size; memid.initially_committed = true; - memid.initially_zero = true; + memid.initially_zero = iszero; memid.is_pinned = true; return memid; } +static inline size_t _mi_memid_size(mi_memid_t memid) { + if (mi_memid_is_os(memid)) { + return memid.mem.os.size; + } + else if (memid.memkind == MI_MEM_ARENA) { + return mi_size_of_slices(memid.mem.arena.slice_count); + } + else if (memid.memkind == MI_MEM_MALLOC) { + return memid.mem.malloc.size; + } + else { + mi_assert_internal(mi_memid_needs_no_free(memid)); + return 0; + } +} // ------------------------------------------------------------------- // Fast "random" shuffle diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index c9c71e14b..a5fb4439f 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -16,19 +16,20 @@ terms of the MIT license. A copy of the license can be found in the file // to a thread-local theap pointer (in `alloc.c:mi_malloc`). // // For performance, we tend to use specialized code for various platforms. -// This leads to quite a few ifdefs but it is just for performance and there +// This leads to quite a few ifdefs but it is just for performance and there // is always a portable fallback (based on regular thread local variables). // // Windows : use NtCurrentTeB and TlsAlloc (MI_TLS_MODEL_WIN32) -// Linux,FreeBSD : use thread locals with the initial-exec model (MI_TLS_MODEL_LOCAL) -// macOS : use pthread locals with assembly for the thread-id (MI_TLS_MODEL_PTHREADS) +// Linux,FreeBSD : use thread locals with the initial-exec model (MI_TLS_MODEL_LOCAL) +// macOS : use pthread locals with assembly for the thread-id (MI_TLS_MODEL_PTHREADS) // Android,OpenBSD : use pthread locals (MI_TLS_MODEL_PTHREADS). todo: maybe on Android MI_TLS_MODEL_LOCAL is better? // -------------------------------------------------------------------------- -static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept; // directly read an entry from the thread local storage (or thread control block) -static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept; -static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept; // get a unique id for a thread +// static inline void** mi_prim_thread_pointer(void); // get a pointer to the thread local storage +// static inline void* mi_prim_tls_slot(size_t slot); // directly read an entry from the thread local storage block +// static inline void mi_prim_tls_slot_set(size_t slot, void* value); +static inline mi_threadid_t _mi_prim_thread_id(void); // get a unique id for a thread static inline mi_theap_t* _mi_theap_default(void); // the default thread local theap static inline mi_theap_t* _mi_theap_cached(void); // last used thread local theap using the _heap_ api static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap @@ -52,123 +53,7 @@ static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page); // //------------------------------------------------------------------- -// Access to TLS (thread local storage) slots. -//------------------------------------------------------------------- - -// On some libc + platform combinations we can directly access a thread-local storage (TLS) slot. -// The TLS layout depends on both the OS and libc implementation so we use specific tests for each main platform. -// If you test on another platform and it works please send a PR :-) -// see also https://akkadia.org/drepper/tls.pdf for more info on the TLS register. -// -// Note: we would like to prefer `__builtin_thread_pointer()` nowadays instead of using assembly, -// but unfortunately we can not detect support reliably (see issue #883) -#if (defined(_WIN32)) || \ - (defined(__GNUC__) && ( \ - (defined(__GLIBC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__) || defined(__riscv))) \ - || (defined(__APPLE__) && (defined(__x86_64__) || defined(__aarch64__) || defined(__POWERPC__))) \ - || (defined(__BIONIC__) && (defined(__x86_64__) || defined(__i386__) || (defined(__arm__) && __ARM_ARCH >= 7) || defined(__aarch64__))) \ - || (defined(__FreeBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ - || (defined(__OpenBSD__) && (defined(__x86_64__) || defined(__i386__) || defined(__aarch64__))) \ - )) - -static inline void* mi_prim_tls_slot(size_t slot) mi_attr_noexcept { - void* res; - const size_t ofs = (slot*sizeof(void*)); - #if defined(_WIN32) - #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC) - res = (void*)__readgsqword((unsigned long)ofs); // direct load at offset from gs - #elif _M_IX86 && !defined(_M_ARM64EC) - res = (void*)__readfsdword((unsigned long)ofs); // direct load at offset from fs - #else - res = ((void**)NtCurrentTeb())[slot]; MI_UNUSED(ofs); - #endif - #elif defined(__i386__) - __asm__("movl %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86 32-bit always uses GS - #elif defined(__APPLE__) && defined(__x86_64__) - __asm__("movq %%gs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 macOSX uses GS - #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x32 ABI - #elif defined(__x86_64__) - __asm__("movq %%fs:%1, %0" : "=r" (res) : "m" (*((void**)ofs)) : ); // x86_64 Linux, BSD uses FS - #elif defined(__arm__) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); - res = tcb[slot]; - #elif defined(__aarch64__) - void** tcb; MI_UNUSED(ofs); - #if defined(__APPLE__) // M1, issue #343 - __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); - #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); - #endif - res = tcb[slot]; - #elif defined(__riscv) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mv %0, tp" : "=r" (tcb)); - res = tcb[slot]; - #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781 - MI_UNUSED(ofs); - res = pthread_getspecific(slot); - #else - #define MI_HAS_TLS_SLOT 0 - MI_UNUSED(ofs); - res = NULL; - #endif - return res; -} - -#ifndef MI_HAS_TLS_SLOT -#define MI_HAS_TLS_SLOT 1 -#endif - -// setting a tls slot is only used with TLS_MODEL_FIXED (which is not used by default on any platform) -static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexcept { - const size_t ofs = (slot*sizeof(void*)); - #if defined(_WIN32) - ((void**)NtCurrentTeb())[slot] = value; MI_UNUSED(ofs); - #elif defined(__i386__) - __asm__("movl %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // 32-bit always uses GS - #elif defined(__APPLE__) && defined(__x86_64__) - __asm__("movq %1,%%gs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 macOS uses GS - #elif defined(__x86_64__) && (MI_INTPTR_SIZE==4) - __asm__("movl %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x32 ABI - #elif defined(__x86_64__) - __asm__("movq %1,%%fs:%0" : "=m" (*((void**)ofs)) : "rn" (value) : ); // x86_64 Linux, BSD uses FS - #elif defined(__arm__) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); - tcb[slot] = value; - #elif defined(__aarch64__) - void** tcb; MI_UNUSED(ofs); - #if defined(__APPLE__) // M1, issue #343 - __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); - #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); - #endif - tcb[slot] = value; - #elif defined(__riscv) - void** tcb; MI_UNUSED(ofs); - __asm__ volatile ("mv %0, tp" : "=r" (tcb)); - tcb[slot] = value; - #elif defined(__APPLE__) && defined(__POWERPC__) // ppc, issue #781 - MI_UNUSED(ofs); - pthread_setspecific(slot, value); - #else - MI_UNUSED(ofs); MI_UNUSED(value); - #endif -} - -#endif - - -//------------------------------------------------------------------- -// Get a fast unique thread id. -// -// Getting the thread id should be performant as it is called in the -// fast path of `_mi_free` and we specialize for various platforms as -// inlined definitions. Regular code should call `init.c:_mi_thread_id()`. -// We only require _mi_prim_thread_id() to return a unique id -// for each thread (unequal to zero) with the bottom 2 bits clear. +// Access to the thread pointer //------------------------------------------------------------------- // Do we have __builtin_thread_pointer? This would be the preferred way to get a unique thread id @@ -189,69 +74,122 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce #endif #endif -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept; - -static inline mi_threadid_t _mi_prim_thread_id(void) mi_attr_noexcept { - const mi_threadid_t tid = __mi_prim_thread_id(); - mi_assert_internal(tid > 1); - mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear? - return tid; +#if defined(MI_PRIM_THREAD_POINTER) // potential user override +static inline void** mi_prim_thread_pointer(void) { + return MI_PRIM_THREAD_POINTER(); } - -// Get a unique id for the current thread. -#if defined(MI_PRIM_THREAD_ID) - -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { - return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488) -} - #elif defined(_WIN32) - -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { - // Windows: works on Intel and ARM in both 32- and 64-bit - return (uintptr_t)NtCurrentTeb(); +static inline void** mi_prim_thread_pointer(void) { + return (void**)NtCurrentTeb(); } - #elif MI_USE_BUILTIN_THREAD_POINTER - -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { - // Works on most Unix based platforms with recent compilers - return (uintptr_t)__builtin_thread_pointer(); +static inline void** mi_prim_thread_pointer(void) { + return (void**)__builtin_thread_pointer(); } - -#elif MI_HAS_TLS_SLOT - -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { - #if defined(__BIONIC__) - // issue #384, #495: on the Bionic libc (Android), slot 1 is the thread id - // see: https://github.com/aosp-mirror/platform_bionic/blob/c44b1d0676ded732df4b3b21c5f798eacae93228/libc/platform/bionic/tls_defines.h#L86 - return (uintptr_t)mi_prim_tls_slot(1); +#elif defined(__GNUC__) && !defined(__CYGWIN__) + #if defined(__aarch64__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + #if defined(__APPLE__) // M1, issue rgb(62, 76, 62) + __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + #else + __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + #endif + return tcb; + } + #elif defined(__riscv) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + __asm__ volatile ("mv %0, tp" : "=r" (tcb)); + return tcb; + } + #elif defined(__arm__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + return tcb; + } + #elif defined(__i386__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + __asm__("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS + return tcb; + } + #elif defined(__x86_64__) + static inline void** mi_prim_thread_pointer(void) { + void** tcb; + #if defined(__APPLE__) + __asm__("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS + #elif (MI_INTPTR_SIZE==4) + __asm__("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI + #else + __asm__("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS + #endif + return tcb; + } #else - // in all our other targets, slot 0 is the thread id - // glibc: https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=sysdeps/x86_64/nptl/tls.h - // apple: https://github.com/apple/darwin-xnu/blob/main/libsyscall/os/tsd.h#L36 - return (uintptr_t)mi_prim_tls_slot(0); + #define MI_NO_THREAD_POINTER (1) #endif +#elif MI_USE_PTHREADS && defined(__APPLE__) +static inline void** mi_prim_thread_pointer(void) { + return (void**)pthread_self(); } +#else +#define MI_NO_THREAD_POINTER (1) +#endif -#elif defined(MI_USE_PTHREADS) && defined(__APPLE__) - -// on macOS, pthread_t is pointer -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { - return (uintptr_t)((void*)pthread_self()); +#if !MI_NO_THREAD_POINTER +#define MI_HAS_TLS_SLOT (1) +static inline void* mi_prim_tls_slot(size_t slot) { + #if defined(_WIN32) + #if (_M_X64 || _M_AMD64) && !defined(_M_ARM64EC) + return (void*)__readgsqword((unsigned long)(slot*sizeof(void*))); // direct load at offset from gs + #elif _M_IX86 && !defined(_M_ARM64EC) + return (void*)__readfsdword((unsigned long)(slot*sizeof(void*))); // direct load at offset from fs + #else + return mi_prim_thread_pointer()[slot]; + #endif + #else + return mi_prim_thread_pointer()[slot]; + #endif } -#else +static inline void mi_prim_tls_slot_set(size_t slot, void* value) { + mi_prim_thread_pointer()[slot] = value; +} +#endif -extern mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper; +/* ---------------------------------------------------------------------------------------- + Get a unique thread-id for a thread +---------------------------------------------------------------------------------------- */ +// Get a unique id for the current thread. +#if defined(MI_PRIM_THREAD_ID) +static inline mi_threadid_t __mi_prim_thread_id(void) { + return MI_PRIM_THREAD_ID(); // used for example by CPython for a free threaded build (see python/cpython#115488) +} +#elif !MI_NO_THREAD_POINTER +static inline mi_threadid_t __mi_prim_thread_id(void) { + #if defined(__BIONIC__) + return (mi_threadid_t)mi_prim_tls_slot(1); + #else + return (mi_threadid_t)mi_prim_thread_pointer(); + #endif +} +#else // otherwise use portable C, taking the address of a thread local variable (this is still very fast on most platforms). -static inline mi_threadid_t __mi_prim_thread_id(void) mi_attr_noexcept { +extern mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper; +static inline mi_threadid_t __mi_prim_thread_id(void) { return (uintptr_t)&__mi_thread_id_helper; } - #endif +static inline mi_threadid_t _mi_prim_thread_id(void) { + const mi_threadid_t tid = __mi_prim_thread_id(); + mi_assert_internal(tid > MI_THREADID_DETACHED); + mi_assert_internal((tid & MI_PAGE_FLAG_MASK) == 0); // bottom 2 bits are clear? + return tid; +} /* ---------------------------------------------------------------------------------------- @@ -279,7 +217,7 @@ We have 4 models: where the underlying TLS implementation (or the loader) will call itself `malloc` on a first access to a thread local (and recurse in the MI_TLS_MODEL_LOCAL). This goes wrong though if the OS or a library uses the same fixed slot, and also - prevents multiple instances of mimalloc in the same process. + prevents multiple instances of mimalloc in the same process. - MI_TLS_MODEL_WIN32: use a dynamically allocated slot with TlsAlloc. (default on Windows) We use TlsAlloc'd slot. First tries to use one of the "direct" first 64 slots which @@ -333,26 +271,28 @@ static inline mi_theap_t* _mi_theap_cached(void) { // can be avoided with pthreads. #define MI_THEAP_INITASNULL 1 -extern mi_decl_hidden pthread_key_t _mi_theap_default_key; -extern mi_decl_hidden pthread_key_t _mi_theap_cached_key; +extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key; +extern mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key; static inline mi_theap_t* _mi_theap_default(void) { + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key); #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT // on apple arm64, the pthread specific slots are direct slots; inline it to avoid a stack frame setup in `mi_malloc` - // todo: this is probably also the case on x64 and power pc? - if (_mi_theap_default_key == MI_PTHREAD_KEY_INVALID) return NULL; - return (mi_theap_t*)mi_prim_tls_slot(_mi_theap_default_key); + // todo: this is probably also the case on x64 and power pc? + if (key == MI_PTHREAD_KEY_INVALID) return NULL; + return (mi_theap_t*)mi_prim_tls_slot(key); #else - return (mi_theap_t*)mi_pthread_key_get(_mi_theap_default_key); + return (mi_theap_t*)mi_pthread_key_get(key); #endif } static inline mi_theap_t* _mi_theap_cached(void) { + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key); #if defined(__APPLE__) && defined(__aarch64__) && MI_HAS_TLS_SLOT - if (_mi_theap_cached_key == MI_PTHREAD_KEY_INVALID) return NULL; - return (mi_theap_t*)mi_prim_tls_slot(_mi_theap_cached_key); + if (key == MI_PTHREAD_KEY_INVALID) return NULL; + return (mi_theap_t*)mi_prim_tls_slot(key); #else - return (mi_theap_t*)mi_pthread_key_get(_mi_theap_cached_key); + return (mi_theap_t*)mi_pthread_key_get(key); #endif } @@ -368,19 +308,19 @@ static inline mi_theap_t* _mi_theap_cached(void) { #define MI_TLS_EXPANSION_SLOT (0x1780 / MI_INTPTR_SIZE) #endif -extern mi_decl_hidden size_t _mi_theap_default_slot; -extern mi_decl_hidden size_t _mi_theap_cached_slot; -extern mi_decl_hidden size_t _mi_theap_default_expansion_slot; -extern mi_decl_hidden size_t _mi_theap_cached_expansion_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot; +extern mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot; static inline mi_theap_t* _mi_theap_default(void) { - const size_t slot = _mi_theap_default_slot; + const size_t slot = mi_atomic_load_relaxed(&_mi_theap_default_slot); mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot); #if !MI_WIN_DIRECT_TLS if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ? mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is actually the expansion slot entry if mi_likely(eslots!=NULL) { // is it initialized? (on this thread) - theap = eslots[_mi_theap_default_expansion_slot]; + theap = eslots[mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot)]; } } #endif @@ -388,13 +328,13 @@ static inline mi_theap_t* _mi_theap_default(void) { } static inline mi_theap_t* _mi_theap_cached(void) { - const size_t slot = _mi_theap_cached_slot; + const size_t slot = mi_atomic_load_relaxed(&_mi_theap_cached_slot); mi_theap_t* theap = (mi_theap_t*)mi_prim_tls_slot(slot); #if !MI_WIN_DIRECT_TLS if mi_unlikely(slot==MI_TLS_EXPANSION_SLOT) { // in TlsExpansionSlots ? mi_theap_t** const eslots = (mi_theap_t**)theap; // theap is the expansion slot entry if mi_likely(eslots!=NULL) { // is it initialized? (on this thread) - theap = eslots[_mi_theap_cached_expansion_slot]; + theap = eslots[mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot)]; } } #endif @@ -403,10 +343,14 @@ static inline mi_theap_t* _mi_theap_cached(void) { #elif MI_TLS_MODEL_FIXED // Fixed TLS slot. Can be the fastest approach, but does not work if there are multiple instances of -// mimalloc in the same process. Most OS's do not have official user reserved fixed slots so this cannot be +// mimalloc in the same process. Most OS's do not have official user reserved fixed slots so this cannot be // guaranteed to work in general. #define MI_THEAP_INITASNULL 1 +#if !MI_HAS_TLS_SLOT +#error this platform cannot support MI_TLS_MODEL_FIXED without defining mi_prim_tls_slot +#endif + #if !defined(MI_TLS_MODEL_FIXED_DEFAULT) #if defined(__APPLE__) && !defined(__POWERPC__) // macOS on arm64 or x64 // 96-97 sit in the never-assigned gap between __PTK_FRAMEWORK_CORETEXT_KEY0 (95) and @@ -475,7 +419,8 @@ static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) { mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); if (theap==NULL) return NULL; if (theap->heap != heap) return NULL; // should never happen, but can happen for a free across subprocesses, which can happen during pthread tls storage deallocation - mi_assert_internal(!_mi_is_empty_theap(theap) && _mi_thread_id()==theap->tld->thread_id); + mi_assert_internal(!_mi_is_empty_theap(theap) && mi_theap_matches_thread(theap)); + // note: for pages allocated by a detached theap, the returned theap may not be detached return theap; } diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index b2336aa70..449682881 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -24,7 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file // OS memory configuration typedef struct mi_os_mem_config_s { size_t page_size; // default to 4KiB - size_t large_page_size; // 0 if not supported, usually 2MiB (4MiB on Windows) + size_t large_page_size; // 0 if not supported, usually 2MiB size_t alloc_granularity; // smallest allocation size (usually 4KiB, on Windows 64KiB) size_t physical_memory_in_kib; // physical memory size in KiB size_t virtual_address_bits; // usually 48 or 56 bits on 64-bit systems. (used to determine secure randomization) diff --git a/include/mimalloc/track.h b/include/mimalloc/track.h index 1f4021297..753363366 100644 --- a/include/mimalloc/track.h +++ b/include/mimalloc/track.h @@ -135,16 +135,16 @@ defined, undefined, or not accessible at all: #if MI_PADDING #define mi_track_malloc(p,reqsize,zero) \ - if ((p)!=NULL) { \ + do { if ((p)!=NULL) { \ mi_assert_internal(mi_usable_size(p)==(reqsize)); \ mi_track_malloc_size(p,reqsize,reqsize,zero); \ - } + } } while(0) #else #define mi_track_malloc(p,reqsize,zero) \ - if ((p)!=NULL) { \ + do { if ((p)!=NULL) { \ mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ - } + } } while(0) #endif #endif // MI_TRACK_H diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index d7c2689be..a1bd1eec2 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -87,7 +87,7 @@ terms of the MIT license. A copy of the license can be found in the file #endif // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) -#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_PAGE_META_ALIGNED_FREE_SMALL +#if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_OPT_FREE_SMALL #define MI_GUARDED 1 #endif @@ -109,6 +109,10 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ENCODE_FREELIST 1 #endif +#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) +#define MI_CHECK_DOUBLE_FREE 1 +#endif + // Enable large pages for objects between 64KiB and 512KiB. // This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB // are quite random which can lead to too many partially used large pages (but see issue #1104). @@ -131,7 +135,9 @@ terms of the MIT license. A copy of the license can be found in the file // We can choose to only put page info of small pages at the start of the page area. // This can be used to have a slightly faster `mi_free_small` function for specialized // cases (like language runtime systems). -#if !defined(MI_PAGE_META_ALIGNED_FREE_SMALL) +#if MI_OPT_FREE_SMALL && !defined(MI_PAGE_META_ALIGNED_FREE_SMALL) +#define MI_PAGE_META_ALIGNED_FREE_SMALL 1 +#elif !defined(MI_PAGE_META_ALIGNED_FREE_SMALL) #define MI_PAGE_META_ALIGNED_FREE_SMALL 0 #endif @@ -140,7 +146,7 @@ terms of the MIT license. A copy of the license can be found in the file #error "secure mode should use separated page infos" #endif #if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE -#error "secure mode cannot use MI_PAGE_META_ALIGNED_FREE_SMALL" +#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL)" #endif #if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT #error "cannot have a flat page map with separated page infos" @@ -159,14 +165,14 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MI_ARENA_SLICE_SHIFT #ifdef MI_SMALL_PAGE_SHIFT // backward compatibility #define MI_ARENA_SLICE_SHIFT MI_SMALL_PAGE_SHIFT - #elif MI_SECURE>=5 && __APPLE__ && MI_ARCH_ARM64 + #elif MI_SECURE>=5 && ((__APPLE__ && MI_ARCH_ARM64) || (defined(PAGE_SIZE) && PAGE_SIZE >= 16*MI_KiB)) #define MI_ARENA_SLICE_SHIFT (17) // 128 KiB to not waste too much due to 16 KiB guard pages #else #define MI_ARENA_SLICE_SHIFT (13 + MI_SIZE_SHIFT) // 64 KiB (32 KiB on 32-bit) #endif #endif -#if MI_ARENA_SLICE_SHIFT < 12 -#error Arena slices should be at least 4KiB +#if MI_ARENA_SLICE_SHIFT < 13 +#error Arena slices should be at least 8KiB #endif #ifndef MI_BCHUNK_BITS_SHIFT @@ -218,8 +224,8 @@ terms of the MIT license. A copy of the license can be found in the file // We never allocate more than PTRDIFF_MAX (see also ) #define MI_MAX_ALLOC_SIZE PTRDIFF_MAX -// Minimal commit for a page on-demand commit (should be >= OS page size) -#define MI_PAGE_MIN_COMMIT_SIZE MI_ARENA_SLICE_SIZE +// Minimal commit for a page on-demand commit +#define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ // ------------------------------------------------------ @@ -258,15 +264,14 @@ typedef struct mi_subproc_s mi_subproc_t; // Memory can reside in arena's, direct OS allocated, meta-data pages, or statically allocated. // The memid keeps track of this. typedef enum mi_memkind_e { - MI_MEM_NONE, // not allocated + MI_MEM_NONE, // not allocated (or static) MI_MEM_EXTERNAL, // not owned by mimalloc but provided externally (via `mi_manage_os_memory` for example) MI_MEM_STATIC, // allocated in a static area and should not be freed (the initial main theap data for example (`init.c`)) - MI_MEM_META, // allocated with the meta data allocator (`arena-meta.c`) MI_MEM_OS, // allocated from the OS MI_MEM_OS_HUGE, // allocated as huge OS pages (usually 1GiB, pinned to physical memory) MI_MEM_OS_REMAP, // allocated in a remapable area (i.e. using `mremap`) MI_MEM_ARENA, // allocated from an arena (the usual case) (`arena.c`) - MI_MEM_HEAP_MAIN // allocated in the main heap (for theaps) + MI_MEM_MALLOC // allocated with mi_malloc } mi_memkind_t; static inline bool mi_memkind_is_os(mi_memkind_t memkind) { @@ -291,17 +296,16 @@ typedef struct mi_memid_arena_info { uint32_t slice_count; // allocated slices } mi_memid_arena_info_t; -typedef struct mi_memid_meta_info { - mi_meta_page_t* meta_page; // meta-page that contains the block - uint32_t block_index; // block index in the meta-data page - uint32_t block_count; // allocated blocks -} mi_memid_meta_info_t; +typedef struct mi_memid_malloc_info { + void* base; // returned pointer + size_t size; // allocated size +} mi_memid_malloc_info_t; typedef struct mi_memid_s { union { - mi_memid_os_info_t os; // only used for MI_MEM_OS - mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA - mi_memid_meta_info_t meta; // only used for MI_MEM_META + mi_memid_os_info_t os; // only used for MI_MEM_OS(_HUGE/_REMAP) + mi_memid_arena_info_t arena; // only used for MI_MEM_ARENA + mi_memid_malloc_info_t malloc; // only used for MI_MEM_MALLOC } mem; mi_memkind_t memkind; bool is_pinned; // `true` if we cannot decommit/reset/protect in this memory (e.g. when allocated using large (2Mib) or huge (1GiB) OS pages) @@ -309,6 +313,8 @@ typedef struct mi_memid_s { bool initially_zero; // `true` if the memory was originally zero initialized } mi_memid_t; +#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ } +#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC) static inline bool mi_memid_is_os(mi_memid_t memid) { return mi_memkind_is_os(memid.memkind); @@ -347,6 +353,7 @@ typedef struct mi_block_s { #define MI_PAGE_HAS_INTERIOR_POINTERS MI_ZU(0x02) #define MI_PAGE_HAS_PROF_SAMPLES MI_ZU(0x04) // page contains at least one profiled allocation; routes free to the generic path #define MI_PAGE_FLAG_MASK MI_ZU(0x07) +#define MI_PAGE_FLAG_BITS (3) typedef size_t mi_page_flags_t; // There are two special threadid's: 0 for pages that are abandoned (and not in a theap queue), @@ -354,7 +361,8 @@ typedef size_t mi_page_flags_t; // in an arena (in `mi_heap_t.arena_pages.pages_abandoned`) so these can be quickly found for reuse. // Abandoning partially used pages allows for sharing of this memory between threads (in particular if threads are blocked) #define MI_THREADID_ABANDONED MI_ZU(0) -#define MI_THREADID_ABANDONED_MAPPED (MI_PAGE_FLAG_MASK + 1) +#define MI_THREADID_ABANDONED_MAPPED (MI_ZU(1) << MI_PAGE_FLAG_BITS) +#define MI_THREADID_DETACHED (MI_ZU(2) << MI_PAGE_FLAG_BITS) // Thread free list. // Points to a list of blocks that are freed by other threads. @@ -395,23 +403,19 @@ typedef struct mi_page_s { _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) - uint16_t used; // number of blocks in use (including blocks in `thread_free`) + uint32_t used; // number of blocks in use (including blocks in `thread_free`) uint16_t capacity; // number of blocks committed - uint16_t reserved; // number of blocks reserved in memory uint8_t retire_expire; // expiration count for retired blocks bool free_is_zero; // `true` if the blocks in the free list are zero initialized - + mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`) size_t block_size; // const: size available in each block (always `>0`) - uint32_t page_woffset; // const: offset relative to the page (in machine words) to the start of the blocks - uint32_t slice_committed; // committed size relative to the first arena slice of the page data (or 0 if the page is fully committed already) - - #if (MI_ENCODE_FREELIST || MI_PADDING) - uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary - #endif - + uint32_t page_ma_offset; // const: offset relative to the page (in MI_MAX_ALIGN_SIZE parts) to the start of the blocks + uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) + uint16_t reserved; // number of blocks reserved in memory + mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages) mi_heap_t* heap; // const: the heap owning this page @@ -419,6 +423,10 @@ typedef struct mi_page_s { struct mi_page_s* prev; // previous page owned by the theap with the same `block_size` mi_memid_t memid; // const: provenance of the page memory + #if (MI_ENCODE_FREELIST || MI_PADDING) + uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary + #endif + // The OS pages inside this page whose memory was discarded to the OS. A block is // "purged" exactly when it overlaps one of them (we discard an OS page only when every // block overlapping it is free), and a purged block is held OFF every free list: mimalloc @@ -440,11 +448,11 @@ typedef struct mi_page_s { // A sweep that finds it unchanged skips the page without walking its free list at all: nothing // was allocated or freed in it since, so the sweep has nothing new to discard (see // `_mi_page_purge_holes`). `MI_PAGE_SWEPT_NONE` means "unknown". Cold, like `purged` above. - uint32_t swept_state; + uint64_t swept_state; } mi_page_t; // An impossible `(capacity,used)` (`used > capacity` never holds): "this page has no sweep state". -#define MI_PAGE_SWEPT_NONE ((uint32_t)0x0000FFFF) +#define MI_PAGE_SWEPT_NONE (~(uint64_t)0) // capacity is 16-bit, so a state with all upper bits set cannot occur // ------------------------------------------------------ @@ -536,7 +544,7 @@ typedef struct mi_random_cxt_s { #if MI_PADDING typedef struct mi_padding_s { uint32_t canary; // encoded block value to check validity of the padding (in case of overflow) - uint32_t delta; // padding bytes before the block. (mi_usable_size(p) - delta == exact allocated bytes) + uint32_t delta; // padding bytes before the block. (mi_full_usable_size(p) - delta == exact allocated bytes) } mi_padding_t; #define MI_PADDING_SIZE (sizeof(mi_padding_t)) #define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE) @@ -554,7 +562,7 @@ struct mi_theap_s { _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) _Atomic(size_t) refcount; // reference count - _Atomic(size_t) freed; // ensure atomic free-ing + unsigned long long heartbeat; // monotonic heartbeat count uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) mi_random_ctx_t random; // random number context used for secure allocation @@ -573,6 +581,7 @@ struct mi_theap_s { long page_full_retain; // how many full pages can be retained per queue (before abandoning them) bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint + bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` bool prof_force_slow; // if profiling is enabled: keep `pages_free_direct` poisoned so every malloc routes through `_mi_malloc_generic` intptr_t prof_countdown; // bytes until next profiling sample (only consulted in `_mi_malloc_generic`; 0 if profiling is off) #if MI_GUARDED @@ -624,7 +633,7 @@ typedef struct mi_heap_s { _Atomic(mi_arena_pages_t*) arena_pages[MI_MAX_ARENAS]; // track owned and abandoned pages in the arenas (entries can be NULL) mi_lock_t arena_pages_lock; // lock to update the arena_pages array - + mi_memid_t memid; // provenance of the heap memory mi_stats_t stats; // statistics for this heap; periodically updated by merging from each theap } mi_heap_t; @@ -655,6 +664,9 @@ struct mi_subproc_s { mi_heap_t* heaps; // heaps belonging to this sub-process mi_lock_t heaps_lock; + mi_theap_t* theap_meta; // detached theap for allocating meta-data + mi_lock_t theap_meta_lock; // all allocations in theap_meta need a lock + mi_tld_t* tlds; // list of tlds of this sub-process (walked by the scavenger for parked threads) mi_lock_t tlds_lock; // guards the `tlds` list structure only -- never held across a sweep _Atomic(size_t) parked_count; // threads currently parked; lets the scavenger skip the walk entirely @@ -726,16 +738,16 @@ struct mi_tld_s { #define MI_ARENA_BIN_COUNT (MI_MAX_SINGLETON_BIN+1) #define MI_ARENA_MIN_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) // 32 MiB (or 8 MiB on 32-bit) -#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE) +#define MI_ARENA_MAX_SIZE (MI_BITMAP_MAX_BIT_COUNT * MI_ARENA_SLICE_SIZE) // 16 GiB typedef struct mi_bitmap_s mi_bitmap_t; // atomic bitmap (defined in `src/bitmap.h`) typedef struct mi_bbitmap_s mi_bbitmap_t; // atomic binned bitmap (defined in `src/bitmap.h`) -typedef struct mi_arena_pages_s { +struct mi_arena_pages_s { mi_bitmap_t* pages; // all registered pages (abandoned and owned) mi_bitmap_t* pages_abandoned[MI_ARENA_BIN_COUNT]; // abandoned pages per size bin (a set bit means the start of the page) // followed by the bitmaps (whose siz`es depend on the arena size) -} mi_arena_pages_t; +}; // A memory arena diff --git a/readme.md b/readme.md index 84ab37af7..560dabe89 100644 --- a/readme.md +++ b/readme.md @@ -15,9 +15,9 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.4.3` (2026-07-20) recommended. -Latest v2 release: `v2.4.1` (2026-07-14) stable. -Latest v1 release: `v1.9.11` (2026-07-14) legacy. +Latest release : `v3.4.5` (2026-08-05) recommended. +Latest v2 release: `v2.4.5` (2026-08-05) stable. +Latest v1 release: `v1.9.15` (2026-08-05) legacy. mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -85,9 +85,18 @@ New development is mostly on v3, while v1 and v2 are maintained with security an and has more efficient heap-walking (for the CPython GC for example). (release tags: `v3.x`, development branch `dev3`). - __v2__: stable mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.x`, development branch `dev2` and `main`) -- __v1__: legacy version: initial design of mimalloc (release tags: `v1.9.x`, development branch `dev`). Send PR's against this version if possible. +- __v1__: legacy version: initial design of mimalloc (release tags: `v1.9.x`, development branch `dev`). + __Send PR's against this version if possible.__ ### Releases + +* 2026-08-05, `v1.9.15`, `v2.4.5`, `v3.4.5`: Fix compilation with xmalloc (#1353), make the build deterministic (#1355), + mi_zalloc_aligned fix (#763), improve support for mingw (ucrt64), fix fputs fallback on windows (#1354), (v3): use proper + lock backoff for first-class heap deletion, improved riscv64 codegen. +* 2026-08-01, `v1.9.14`, `v2.4.4`, `v3.4.4`: various bug and security fixes through Opus 5 LLM audit (issue #1271, by @Zoxc). + (v3): use pthreads by default on macOS (issue #1333, issue #1327), fix glibc 2.44 crash (issue #1341), fix alignment check for realloc_aligned. (v1,v2,v3): Add initial mingw support, + set `errno` on allocation errors, improved double-free checks and size checks in secure mode, enable + build for universal windows platform and xbox (pr #1340), fix numa node detection when numa nodes are sparse. * 2026-07-20, `v3.4.3`: revert TLS slots on macOS to 108/109 (issue #1333). * 2026-07-14, `v1.9.11`, `v2.4.1`, `v3.4.1`: various bug and security fixes through LLM audit (by @Zoxc). Fix issue with using OS memory instead of arenas for > 4GiB memory usage (v3), fix concurrency bug in concurrent @@ -122,24 +131,6 @@ New development is mostly on v3, while v1 and v2 are maintained with security an v3 uses faster TLS access on Windows, and has improved performance for `mi_calloc` and aligned allocations. Fixed rare race condition on older v3, fixed potential buffer overflow in debug statistics, add API for returning allocated sizes on allocation and free. -* 2025-06-09, `v1.9.4`, `v2.2.4`, `v3.1.4` (beta) : Some important bug fixes, including a case where OS memory - was not always fully released. Improved v3 performance, build on XBox, fix build on Android, support interpose - for older macOS versions, use MADV_FREE_REUSABLE on macOS, always check commit success, better support for Windows - fixed TLS offset, etc. -* 2025-03-28, `v1.9.3`, `v2.2.3`, `v3.0.3` (beta) : Various small bug and build fixes, including: - fix arm32 pre v7 builds, fix mingw build, get runtime statistics, improve statistic commit counts, - fix execution on non BMI1 x64 systems. -* 2025-03-06, `v1.9.2`, `v2.2.2`, `v3.0.2-beta`: Various small bug and build fixes. - Add `mi_options_print`, `mi_arenas_print`, and the experimental `mi_stat_get` and `mi_stat_get_json`. - Add `mi_thread_set_in_threadpool` and `mi_heap_set_numa_affinity` (v3 only). Add vcpkg portfile. - Upgrade mimalloc-redirect to v1.3.2. `MI_OPT_ARCH` is off by default now but still assumes armv8.1-a on arm64 - for fast atomic operations. Add QNX support. -* 2025-01-03, `v1.8.9`, `v2.1.9`, `v3.0.1-alpha`: Interim release. Support Windows arm64. New [guarded](#guarded) build that can place OS - guard pages behind objects to catch buffer overflows as they occur. - Many small fixes: build on Windows arm64, cygwin, riscV, and dragonfly; fix Windows static library initialization to account for - thread local destructors (in Rust/C++); macOS tag change; macOS TLS slot fix; improve stats; - consistent `mimalloc.dll` on Windows (instead of `mimalloc-override.dll`); fix mimalloc-redirect on Win11 H2; - add 0-byte to canary; upstream CPython fixes; reduce .bss size; allow fixed TLS slot on Windows for improved performance. * [Older release notes](#older-release-notes) @@ -925,6 +916,25 @@ provided by the bot. You will only need to do this once across all repos using o # Older Release Notes +* 2025-06-09, `v1.9.4`, `v2.2.4`, `v3.1.4` (beta) : Some important bug fixes, including a case where OS memory + was not always fully released. Improved v3 performance, build on XBox, fix build on Android, support interpose + for older macOS versions, use MADV_FREE_REUSABLE on macOS, always check commit success, better support for Windows + fixed TLS offset, etc. +* 2025-03-28, `v1.9.3`, `v2.2.3`, `v3.0.3` (beta) : Various small bug and build fixes, including: + fix arm32 pre v7 builds, fix mingw build, get runtime statistics, improve statistic commit counts, + fix execution on non BMI1 x64 systems. +* 2025-03-06, `v1.9.2`, `v2.2.2`, `v3.0.2-beta`: Various small bug and build fixes. + Add `mi_options_print`, `mi_arenas_print`, and the experimental `mi_stat_get` and `mi_stat_get_json`. + Add `mi_thread_set_in_threadpool` and `mi_heap_set_numa_affinity` (v3 only). Add vcpkg portfile. + Upgrade mimalloc-redirect to v1.3.2. `MI_OPT_ARCH` is off by default now but still assumes armv8.1-a on arm64 + for fast atomic operations. Add QNX support. +* 2025-01-03, `v1.8.9`, `v2.1.9`, `v3.0.1-alpha`: Interim release. Support Windows arm64. New [guarded](#guarded) build that can place OS + guard pages behind objects to catch buffer overflows as they occur. + Many small fixes: build on Windows arm64, cygwin, riscV, and dragonfly; fix Windows static library initialization to account for + thread local destructors (in Rust/C++); macOS tag change; macOS TLS slot fix; improve stats; + consistent `mimalloc.dll` on Windows (instead of `mimalloc-override.dll`); fix mimalloc-redirect on Win11 H2; + add 0-byte to canary; upstream CPython fixes; reduce .bss size; allow fixed TLS slot on Windows for improved performance. + * 2024-05-21, `v1.8.7`, `v2.1.7`: Fix build issues on less common platforms. Started upstreaming patches from the CPython [integration](https://github.com/python/cpython/issues/113141#issuecomment-2119255217). Upstream `vcpkg` patches. * 2024-05-13, `v1.8.6`, `v2.1.6`: Fix build errors on various (older) platforms. Refactored aligned allocation. diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index b4ab89c26..111d63a8b 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -18,7 +18,7 @@ terms of the MIT license. A copy of the license can be found in the file static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { // certain blocks are always allocated at a certain natural alignment. // (see also `arena.c:mi_arenas_page_alloc_fresh`). - mi_assert_internal(_mi_is_power_of_two(alignment) && (alignment > 0)); + mi_assert_internal(mi_alignment_is_valid(alignment)); if (alignment > size) return false; const size_t bsize = mi_good_size(size); const bool ok = (bsize <= MI_PAGE_MAX_START_BLOCK_ALIGN2 && _mi_is_power_of_two(bsize)) || // power-of-two under N @@ -28,7 +28,7 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { } #if MI_GUARDED -static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero) mi_attr_noexcept { +static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero, mi_page_t** ppage) mi_attr_noexcept { // use over allocation for guarded blocksl #if MI_THEAP_INITASNULL if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } @@ -39,7 +39,7 @@ static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, return NULL; } const size_t oversize = size + alignment - 1; - void* const base = _mi_theap_malloc_guarded(theap, oversize, zero); + void* const base = _mi_theap_malloc_guarded(theap, oversize, zero, ppage); if (base==NULL) return NULL; void* const p = _mi_align_up_ptr(base, alignment); mi_track_align(base, p, (uint8_t*)p - (uint8_t*)base, size); @@ -48,31 +48,32 @@ static mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, return p; } -static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) { +static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { #if MI_THEAP_INITASNULL if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } #endif const size_t rate = theap->guarded_sample_rate; // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty` if (rate != 0) { theap->guarded_sample_rate = 0; } - void* p = _mi_theap_malloc_zero(theap, size, zero, usable); + void* p = _mi_theap_malloc_zero(theap, size, zero, ppage); if (rate != 0) { theap->guarded_sample_rate = rate; } return p; } #else -static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, size_t* usable) { - return _mi_theap_malloc_zero(theap, size, zero, usable); +static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { + return _mi_theap_malloc_zero(theap, size, zero, ppage); } #endif // Fallback aligned allocation that over-allocates -- split out for better codegen -static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept +static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)); - mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); + mi_assert_internal(mi_alignment_is_valid(alignment)); void* p; size_t oversize; + mi_page_t* page; if mi_unlikely(alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { // use OS allocation for large alignments and allocate inside a singleton page (not in an arena) // This can support alignments >= MI_PAGE_ALIGN by ensuring the object can be aligned @@ -84,7 +85,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); // note: no guarded as alignment > 0 - p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, usable); // the page block size should be large enough to align in the single huge page block + p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block if (p == NULL) return NULL; } else { @@ -92,9 +93,11 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap mi_assert_internal(size <= (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) && alignment <= MI_PAGE_MAX_OVERALLOC_ALIGN); mi_assert_internal(size < SIZE_MAX - alignment); // `oversize` cannot overflow oversize = (size < MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : size) + alignment - 1; // adjust for size <= 16; with size 0 and alignment 64k, we would allocate a 64k block and pointing just beyond that. - p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, usable); + p = mi_theap_malloc_zero_no_guarded(theap, oversize, zero, &page); if (p == NULL) return NULL; } + mi_assert_internal(page == _mi_ptr_page(p)); + if (ppage!=NULL) { *ppage = page; } // .. and align within the allocation const uintptr_t align_mask = alignment - 1; // for any x, `(x & align_mask) == (x % alignment)` @@ -107,14 +110,13 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap // and we no longer own it. We should be careful to only read constant fields in the page, // or use safe atomic access as in `mi_page_set_has_interior_pointers`. // (we can access the page though since the just allocated pointer keeps it alive) - mi_page_t* page = _mi_ptr_page(p); if (aligned_p != p) { mi_page_set_has_interior_pointers(page, true); - if (usable!=NULL) { - mi_assert_internal(*usable > adjust); - if (*usable > adjust) { *usable = *usable - adjust; } - mi_assert_internal(*usable >= size); - } + // if (usable!=NULL) { + // mi_assert_internal(*usable > adjust); + // if (*usable > adjust) { *usable = *usable - adjust; } + // mi_assert_internal(*usable >= size); + // } #if MI_GUARDED // set tag to aligned so mi_usable_size works with guard pages if (adjust >= sizeof(mi_block_t)) { @@ -141,7 +143,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap // // for the tracker, on huge aligned allocations only from the start of the large block is defined // mi_track_mem_undefined(aligned_p, size); // if (zero) { - // _mi_memzero_aligned(aligned_p, mi_usable_size(aligned_p)); + // _mi_memzero_(aligned_p, mi_usable_size(aligned_p)); // } //} @@ -155,12 +157,12 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap } // Generic primitive aligned allocation -- split out for better codegen -static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept +static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { - mi_assert_internal(alignment != 0 && _mi_is_power_of_two(alignment)); + mi_assert_internal(mi_alignment_is_valid(alignment)); // we don't allocate more than MI_MAX_ALLOC_SIZE (see ) if mi_unlikely(size > (MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE)) { - _mi_error_message(EOVERFLOW, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); + _mi_error_message(EINVAL, "aligned allocation request is too large (size %zu, alignment %zu)\n", size, alignment); return NULL; } @@ -168,7 +170,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, usable); + void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, ppage); mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { @@ -182,19 +184,21 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t } // fall back to over-allocation - return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,usable); + return mi_theap_malloc_zero_aligned_at_overalloc(theap,size,alignment,offset,zero,ppage); } +static mi_decl_cold mi_decl_noinline void* mi_error_bad_alignment(size_t size, size_t alignment, size_t offset) { + _mi_error_message(EINVAL, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); + return NULL; +} + // Primitive aligned allocation -static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, size_t* usable) mi_attr_noexcept +static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, const size_t size, const size_t alignment, const size_t offset, const bool zero, mi_page_t** ppage) mi_attr_noexcept { // note: we don't require `size > offset`, we just guarantee that the address at offset is aligned regardless of the allocated size. - if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see ) - #if MI_DEBUG > 0 - _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu, offset %zu)\n", size, alignment, offset); - #endif - return NULL; + if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two and multiple of void* (see ) + return mi_error_bad_alignment(size, alignment, offset); } #if MI_GUARDED @@ -202,7 +206,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con if mi_likely(theap!=NULL) #endif if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) { - return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero); + return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero, ppage); } #endif @@ -221,7 +225,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; if mi_likely(is_aligned) { - if (usable!=NULL) { *usable = mi_page_usable_block_size(page); } + if (ppage!=NULL) { *ppage = page; } void* p = _mi_page_malloc_zero(theap, page, padsize, zero); mi_assert_internal(p != NULL); mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); @@ -233,7 +237,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con } // fallback to generic aligned allocation - return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, usable); + return mi_theap_malloc_zero_aligned_at_generic(theap, size, alignment, offset, zero, ppage); } @@ -253,7 +257,7 @@ static mi_decl_restrict void* mi_theap_zalloc_aligned_at(mi_theap_t* theap, size return mi_theap_malloc_zero_aligned_at(theap, size, alignment, offset, true, NULL); } -static mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { +mi_decl_restrict void* mi_theap_zalloc_aligned(mi_theap_t* theap, size_t size, size_t alignment) mi_attr_noexcept { return mi_theap_zalloc_aligned_at(theap, size, alignment, 0); } @@ -280,8 +284,11 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_aligned(size_t size, size_t a return mi_theap_malloc_aligned(_mi_theap_default(), size, alignment); } -mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept { - return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, block_size); +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, false, &page); + if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } + return p; } mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset) mi_attr_noexcept { @@ -292,8 +299,11 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_aligned(size_t size, size_t a return mi_theap_zalloc_aligned(_mi_theap_default(), size, alignment); } -mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* block_size) mi_attr_noexcept { - return mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, block_size); +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_aligned(size_t size, size_t alignment, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_zero_aligned_at(_mi_theap_default(), size, alignment, 0, true, &page); + if (p!=NULL && pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } + return p; } mi_decl_nodiscard mi_decl_restrict void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset) mi_attr_noexcept { @@ -335,29 +345,30 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc_aligned(mi_heap_t* heap, // ------------------------------------------------------ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, size_t offset, bool zero) mi_attr_noexcept { - mi_assert(alignment > 0 && _mi_is_power_of_two(alignment)); - if mi_unlikely(alignment == 0 || !_mi_is_power_of_two(alignment)) { // require power-of-two (see ) - #if MI_DEBUG > 0 - _mi_error_message(EOVERFLOW, "aligned allocation requires the alignment to be a power-of-two (size %zu, alignment %zu, offset %zu)\n", newsize, alignment, offset); - #endif - return NULL; + mi_assert(mi_alignment_is_valid(alignment)); + if mi_unlikely(!mi_alignment_is_valid(alignment)) { // require power-of-two (see ) + return mi_error_bad_alignment(newsize,alignment,offset); } - if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL); + if (alignment <= sizeof(uintptr_t) && offset==0) return _mi_theap_realloc_zero(theap,p,newsize,zero); if (p == NULL) return mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,zero,NULL); - size_t size = mi_usable_size(p); + const size_t size = mi_usable_size(p); if (newsize <= size && newsize >= (size - (size / 2)) && (((uintptr_t)p + offset) & (alignment-1)) == 0) { - return p; // reallocation still fits, is aligned and not more than 25% waste + return p; // reallocation still fits, is aligned and not more than 50% waste } else { - // note: we don't zero allocate upfront so we only zero initialize the expanded part - void* newp = mi_theap_malloc_aligned_at(theap,newsize,alignment,offset); + // note: we don't zero allocate upfront so we only zero initialize the expanded part (at the cost of calling mi_usable_size) + mi_page_t* newpage; + void* const newp = mi_theap_malloc_zero_aligned_at(theap,newsize,alignment,offset,false/*zero?*/,&newpage); if (newp != NULL) { - if (zero && newsize > size) { + const size_t copy_size = (newsize > size ? size : newsize); + const size_t zero_start = (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + const size_t usable = _mi_page_usable_size(newpage,newp); + mi_assert_internal(usable >= newsize); // use usable for zero'ing, issue #763 + if (zero && usable > zero_start) { // also set last word in the previous allocation to zero to ensure any padding is zero-initialized - size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); - _mi_memzero((uint8_t*)newp + start, newsize - start); + _mi_memzero((uint8_t*)newp + zero_start, usable - zero_start); } - _mi_memcpy_aligned(newp, p, (newsize > size ? size : newsize)); + _mi_memcpy(newp, p, copy_size); // cannot be aligned due to abitrary offset... (todo: require offset to be a multiple of sizeof(void*)?) mi_free(p); // only free if successful } return newp; @@ -366,7 +377,7 @@ static void* mi_theap_realloc_zero_aligned_at(mi_theap_t* theap, void* p, size_t static void* mi_theap_realloc_zero_aligned(mi_theap_t* theap, void* p, size_t newsize, size_t alignment, bool zero) mi_attr_noexcept { mi_assert(alignment > 0); - if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero,NULL,NULL); + if (alignment <= sizeof(uintptr_t)) return _mi_theap_realloc_zero(theap,p,newsize,zero); return mi_theap_realloc_zero_aligned_at(theap,p,newsize,alignment,0,zero); } diff --git a/src/alloc-override.c b/src/alloc-override.c index 93d066dc1..04da5f293 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -127,7 +127,7 @@ typedef void* mi_nothrow_t; #elif defined(_MSC_VER) _Check_return_ _Ret_maybenull_ _Post_writable_byte_size_(_Size) _ACRTIMP _CRTALLOCATOR _CRT_HYBRIDPATCHABLE void* __cdecl _expand(_Pre_notnull_ void* _Block, _In_ _CRT_GUARDOVERFLOW size_t _Size) { - return mi_expand(_Block, _Size); + return mi__expand(_Block, _Size); } _Check_return_ _ACRTIMP size_t __cdecl _msize_base(_Pre_notnull_ void* _Block) _CRT_NOEXCEPT { @@ -324,14 +324,25 @@ typedef void* mi_nothrow_t; extern "C" { #endif +// defined here instead alloc-posix so we can alias it +mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept { + if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); +} + +mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { + if (!mi_is_in_heap_region(p)) return 0; + return mi_usable_size(p); +} + #ifndef MI_OSX_IS_INTERPOSED // Forward Posix/Unix calls as well void* reallocf(void* p, size_t newsize) MI_FORWARD2(mi_reallocf,p,newsize) - size_t malloc_size(const void* p) MI_FORWARD1(mi_usable_size,p) + size_t malloc_size(const void* p) MI_FORWARD1(mi_malloc_size,p) #if !defined(__ANDROID__) && !defined(__FreeBSD__) && !defined(__DragonFly__) - size_t malloc_usable_size(void *p) MI_FORWARD1(mi_usable_size,p) + size_t malloc_usable_size(void *p) MI_FORWARD1(mi_malloc_usable_size,p) #else - size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_usable_size,p) + size_t malloc_usable_size(const void *p) MI_FORWARD1(mi_malloc_usable_size,p) #endif // No forwarding here due to aliasing/name mangling issues diff --git a/src/alloc-posix.c b/src/alloc-posix.c index d4cf31dfe..4e1d215f1 100644 --- a/src/alloc-posix.c +++ b/src/alloc-posix.c @@ -31,7 +31,7 @@ terms of the MIT license. A copy of the license can be found in the file #define ENOMEM 12 #endif - +#if !(defined(MI_MALLOC_OVERRIDE) && !defined(_DLL)) // otherwise defined in alloc-override.c, where malloc_size/malloc_usable_size alias them mi_decl_nodiscard size_t mi_malloc_size(const void* p) mi_attr_noexcept { if (!mi_is_in_heap_region(p)) return 0; return mi_usable_size(p); @@ -41,24 +41,18 @@ mi_decl_nodiscard size_t mi_malloc_usable_size(const void *p) mi_attr_noexcept { if (!mi_is_in_heap_region(p)) return 0; return mi_usable_size(p); } +#endif mi_decl_nodiscard size_t mi_malloc_good_size(size_t size) mi_attr_noexcept { return mi_good_size(size); } -void mi_cfree(void* p) mi_attr_noexcept { - if (mi_is_in_heap_region(p)) { - mi_free(p); - } -} - int mi_posix_memalign(void** p, size_t alignment, size_t size) { // mi_attr_noexcept (issue #794) // Note: The spec dictates we should not modify `*p` on an error. (issue#27) // if (p == NULL) return EINVAL; - if ((alignment % sizeof(void*)) != 0) return EINVAL; // natural alignment - // it is also required that alignment is a power of 2 and > 0; this is checked in `mi_malloc_aligned` - if (alignment==0 || !_mi_is_power_of_two(alignment)) return EINVAL; // not a power of 2 + // it is required that alignment is a power of 2 and a multiple of sizeof(void*) + if (alignmentblock_size != 0) { // not the empty theap mi_assert_internal(mi_page_block_size(page) >= size); @@ -40,10 +40,10 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // check the free list mi_block_t* const block = page->free; if mi_unlikely(block == NULL) { - return _mi_malloc_generic(theap, size, (zero ? 1 : 0), usable); + return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); - if (usable != NULL) { *usable = mi_page_usable_block_size(page); }; + if (ppage != NULL) { *ppage = page; }; // pop from the free list page->free = mi_block_next(page, block); @@ -120,33 +120,28 @@ extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t siz return mi_page_malloc_zero(theap, page, size, zero, NULL); } -#if MI_GUARDED -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept; -#endif - // main allocation primitives for small and generic allocation // internal small size allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert(theap != NULL); mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG - const uintptr_t tid = _mi_thread_id(); - mi_assert(theap->tld->thread_id == 0 || theap->tld->thread_id == tid); // theaps are thread local + mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local #endif #if (MI_PADDING || MI_GUARDED) if mi_unlikely(size == 0) { size = sizeof(void*); } #endif #if MI_GUARDED if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) { - return _mi_theap_malloc_guarded(theap, size, zero); + return _mi_theap_malloc_guarded(theap, size, zero, ppage); } #endif // get page in constant time, and allocate from it mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE); - void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, usable); + void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, ppage); mi_track_malloc(p,size,zero); #if MI_DEBUG>3 @@ -158,22 +153,22 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non } // internal generic allocation -static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept +static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { #if MI_GUARDED #if MI_THEAP_INITASNULL if (theap!=NULL) #endif if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) { - return _mi_theap_malloc_guarded(theap, size, zero); + return _mi_theap_malloc_guarded(theap, size, zero, ppage); } #endif #if !MI_THEAP_INITASNULL mi_assert(theap!=NULL); #endif - mi_assert(theap==NULL || theap->tld->thread_id == 0 || theap->tld->thread_id == _mi_thread_id()); // theaps are thread local + mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local mi_assert((huge_alignment & 1)==0); - void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, usable); // note: size can overflow but it is detected in malloc_generic + void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p, size, zero); #if MI_DEBUG>3 @@ -185,16 +180,16 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size } // internal small allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept { +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { #if MI_THEAP_INITASNULL if (theap!=NULL) { - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, 0, usable); // tailcall + return mi_theap_malloc_generic(theap, size, zero, 0, ppage); // tailcall } #else - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); #endif } @@ -213,18 +208,18 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, s } // The main internal allocation functions -static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept { +static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable); + return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, size_t* usable) mi_attr_noexcept { +extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { // fast path for small objects #if MI_THEAP_INITASNULL if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX) @@ -233,22 +228,22 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, siz #endif { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, usable); + return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, huge_alignment, usable); + return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); } } -void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t* usable) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, zero, 0, usable); +inline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + return _mi_theap_malloc_zero_ex(theap, size, zero, 0, ppage); } // Main allocation functions mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(theap, size, false, NULL); + return _mi_theap_malloc_zero_ex(theap, size, false, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { @@ -302,27 +297,43 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_calloc(mi_heap_t* heap, size_t return mi_heap_zalloc(heap, total); } +static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { + mi_assert_internal(page == _mi_ptr_page(p)); + if (pblock_size!=NULL) { + if (p!=NULL) { *pblock_size = mi_page_block_size(page); } + } + return p; +} + // Return usable size -mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* usable) mi_attr_noexcept { - return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, usable); -} +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, &page); + return mi_ublock_size(p,page,pblock_size); +} -mi_decl_nodiscard mi_decl_restrict void* mi_theap_umalloc(mi_theap_t* theap, size_t size, size_t* usable) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, false, 0, usable); +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, true, &page); + return mi_ublock_size(p,page,pblock_size); } -mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* usable) mi_attr_noexcept { - return mi_theap_umalloc(_mi_theap_default(), size, usable); +mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, &page); + return mi_ublock_size(p,page,pblock_size); } -mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* usable) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(_mi_theap_default(), size, true, 0, usable); +mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { + mi_page_t* page; + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, &page); + return mi_ublock_size(p,page,pblock_size); } -mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* usable) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_ucalloc(size_t count, size_t size, size_t* pblock_size) mi_attr_noexcept { size_t total; if (mi_count_size_overflow(count,size,&total)) return NULL; - return mi_uzalloc(total, usable); + return mi_uzalloc(total, pblock_size); } // Uninitialized `calloc` @@ -351,14 +362,14 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { return NULL; #else if (p == NULL) return NULL; - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand"); - const size_t size = _mi_usable_size(p,page); + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand"); + const size_t size = _mi_page_usable_size(page,p); if (newsize > size) return NULL; return p; // it fits #endif } -void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept { +static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, void* p, size_t newsize, bool zero, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { // if p == NULL then behave as malloc. // else if size == 0 then reallocate to a zero-sized block (and don't return NULL, just as mi_malloc(0)). // (this means that returning NULL always indicates an error, and `p` will not have been freed in that case.) @@ -367,17 +378,17 @@ void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool ze if (p==NULL) { page = NULL; size = 0; - if (usable_pre!=NULL) { *usable_pre = 0; } + if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } } else { page = mi_validate_ptr_page(p,"mi_realloc"); if mi_unlikely(page==NULL) { // invalid pointer - if (usable_pre!=NULL) { *usable_pre = 0; } - if (usable_post!=NULL) { *usable_post = 0; } + if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } + if (pblock_size_post!=NULL) { *pblock_size_post = 0; } return NULL; } - size = _mi_usable_size(p,page); - if (usable_pre!=NULL) { *usable_pre = mi_page_usable_block_size(page); } + size = _mi_page_usable_size(page,p); + if (pblock_size_pre!=NULL) { *pblock_size_pre = mi_page_block_size(page); } } // check if we can reuse the existing block if mi_unlikely(newsize<=size && newsize>=(size/2) && newsize>0) { // note: newsize must be > 0 or otherwise we return NULL for realloc(NULL,0) @@ -391,34 +402,47 @@ void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool ze // todo: do not track as the usable size is still the same in the free; adjust potential padding? // mi_track_resize(p,size,newsize) // if (newsize < size) { mi_track_mem_noaccess((uint8_t*)p + newsize, size - newsize); } - if (usable_post!=NULL) { *usable_post = mi_page_usable_block_size(page); } + if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(page); } return p; // reallocation still fits and not more than 50% waste } } } - // otherwise allocate a fresh block - void* newp = mi_theap_umalloc(theap,newsize,usable_post); + // note: we don't zero allocate upfront so we only zero initialize the expanded part + mi_page_t* newpage; // use block_size for zero-ing, issue #763 + void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,&newpage); if mi_likely(newp != NULL) { - if (zero && newsize > size) { - // also set last word in the previous allocation to zero to ensure any padding is zero-initialized - const size_t start = (size >= sizeof(intptr_t) ? size - sizeof(intptr_t) : 0); - _mi_memzero((uint8_t*)newp + start, newsize - start); + if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } + const size_t copy_size = (newsize > size ? size : newsize); + const size_t zero_start = _mi_align_down( (copy_size >= sizeof(intptr_t) ? copy_size - sizeof(intptr_t) : 0), sizeof(intptr_t)); // also set last word in the previous allocation to zero to ensure any padding is zero-initialized + const size_t usable = _mi_page_usable_size(newpage,newp); + mi_assert_internal(usable >= newsize); + if (zero && usable > zero_start) { + _mi_memzero_aligned((uint8_t*)newp + zero_start, usable - zero_start); } else if (newsize == 0) { ((uint8_t*)newp)[0] = 0; // work around for applications that expect zero-reallocation to be zero initialized (issue #725) } if mi_likely(p != NULL) { - const size_t copysize = (newsize > size ? size : newsize); - mi_track_mem_defined(p,copysize); // _mi_useable_size may be too large for byte precise memory tracking.. - _mi_memcpy(newp, p, copysize); - mi_free(p); // only free the original pointer if successful // todo: optimize since page is known? + mi_track_mem_defined(p,copy_size); // _mi_useable_size may be too large for byte precise memory tracking.. + _mi_memcpy_aligned(newp, p, copy_size); + mi_free(p); // only free the original pointer if successful } } return newp; } +void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept { + return mi_theap_realloc_zero_ex(theap,p,newsize,zero,NULL,NULL); +} + mi_decl_nodiscard void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { - return _mi_theap_realloc_zero(theap, p, newsize, false, NULL, NULL); + // optimize p==NULL + if (p==NULL) { + return mi_theap_malloc(theap,newsize); + } + else { + return _mi_theap_realloc_zero(theap, p, newsize, false); + } } static void* mi_theap_reallocn(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept { @@ -435,8 +459,14 @@ static void* mi_theap_reallocf(mi_theap_t* theap, void* p, size_t newsize) mi_at return newp; } -static void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { - return _mi_theap_realloc_zero(theap, p, newsize, true, NULL, NULL); +mi_decl_nodiscard void* mi_theap_rezalloc(mi_theap_t* theap, void* p, size_t newsize) mi_attr_noexcept { + // optimize p==NULL + if (p==NULL) { + return mi_theap_zalloc(theap,newsize); + } + else { + return _mi_theap_realloc_zero(theap, p, newsize, true); + } } static void* mi_theap_recalloc(mi_theap_t* theap, void* p, size_t count, size_t size) mi_attr_noexcept { @@ -454,8 +484,8 @@ mi_decl_nodiscard void* mi_reallocn(void* p, size_t count, size_t size) mi_attr_ return mi_theap_reallocn(_mi_theap_default(),p,count,size); } -mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* usable_pre, size_t* usable_post) mi_attr_noexcept { - return _mi_theap_realloc_zero(_mi_theap_default(),p,newsize, false, usable_pre, usable_post); +mi_decl_nodiscard void* mi_urealloc(void* p, size_t newsize, size_t* pblock_size_pre, size_t* pblock_size_post) mi_attr_noexcept { + return mi_theap_realloc_zero_ex(_mi_theap_default(),p,newsize, false, pblock_size_pre, pblock_size_post); } // Reallocate but free `p` on errors @@ -548,13 +578,30 @@ mi_decl_nodiscard mi_decl_restrict char* mi_heap_strndup(mi_heap_t* heap, const mi_decl_nodiscard static mi_decl_restrict char* mi_theap_realpath(mi_theap_t* theap, const char* fname, char* resolved_name) mi_attr_noexcept { // todo: use GetFullPathNameW to allow longer file names + if (fname==NULL || *fname==0) { + errno = EINVAL; + return NULL; + } char buf[PATH_MAX]; DWORD res = GetFullPathNameA(fname, PATH_MAX, (resolved_name == NULL ? buf : resolved_name), NULL); if (res == 0) { - errno = GetLastError(); return NULL; + DWORD err = GetLastError(); + switch (err) { + case ERROR_LOCK_VIOLATION: + case ERROR_SHARING_VIOLATION: + case ERROR_INVALID_ACCESS: errno = EACCES; break; + case ERROR_INVALID_HANDLE: + case ERROR_INVALID_FUNCTION: errno = EINVAL; break; + case ERROR_PATH_NOT_FOUND: errno = ENOTDIR; break; + case ERROR_FILE_NOT_FOUND: errno = ENOENT; break; + case ERROR_NOT_ENOUGH_MEMORY: errno = ENOMEM; break; + default: errno = EIO; + } + return NULL; } else if (res > PATH_MAX) { - errno = EINVAL; return NULL; + errno = ENAMETOOLONG; + return NULL; } else if (resolved_name != NULL) { return resolved_name; @@ -628,8 +675,11 @@ The standard requires calling into `get_new_handler` and throwing the bad_alloc exception on failure. If we compile with a C++ compiler we can implement this precisely. If we use a C compiler we cannot throw a `bad_alloc` exception -but we call `exit` instead (i.e. not returning). +but we call `abort` instead (i.e. not returning). +Also, the standard requires calling the new handler until +it returns false, but we limit the total calls. -------------------------------------------------------*/ +#define MI_TRY_NEW_MAX (4) #ifdef __cplusplus #include @@ -651,10 +701,23 @@ static bool mi_try_new_handler(bool nothrow) { #endif return false; } - else { + else if (!nothrow) { h(); return true; } + else { + #if defined(_CPPUNWIND) || defined(__cpp_exceptions) // as above: this file is also built as C++ with exceptions disabled + try { + h(); + } + catch(...) { // swallow std::bad_alloc + return false; // stop trying + } + #else + h(); // without exceptions a handler can only return (or terminate) + #endif + return true; + } } #else typedef void (*std_new_handler_t)(void); @@ -691,7 +754,8 @@ static bool mi_try_new_handler(bool nothrow) { static mi_decl_noinline void* mi_theap_try_new(mi_theap_t* theap, size_t size, bool nothrow ) { void* p = NULL; - while(p == NULL && mi_try_new_handler(nothrow)) { + for(int i = 0; i < MI_TRY_NEW_MAX && p == NULL && mi_try_new_handler(nothrow); i++) { + if (size > MI_MAX_ALLOC_SIZE) return NULL; // call try_new_handler at least once p = mi_theap_malloc(theap,size); } return p; @@ -705,7 +769,6 @@ static mi_decl_noinline void* mi_heap_try_new(mi_heap_t* heap, size_t size, bool return mi_theap_try_new(_mi_heap_theap(heap), size, nothrow); } - mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new(mi_theap_t* theap, size_t size) { void* p = mi_theap_malloc(theap,size); if mi_unlikely(p == NULL) return mi_theap_try_new(theap, size, false); @@ -722,7 +785,6 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new(mi_heap_t* heap, size return p; } - mi_decl_nodiscard static mi_decl_restrict void* mi_theap_alloc_new_n(mi_theap_t* theap, size_t count, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(count, size, &total)) { @@ -742,43 +804,52 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_alloc_new_n(mi_heap_t* heap, si return mi_theap_alloc_new_n(_mi_heap_theap(heap), count, size); } - mi_decl_nodiscard mi_decl_restrict void* mi_new_nothrow(size_t size) mi_attr_noexcept { void* p = mi_malloc(size); if mi_unlikely(p == NULL) return mi_try_new(size, true); return p; } -mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { - void* p; - do { - p = mi_malloc_aligned(size, alignment); +static mi_decl_noinline void* mi_try_new_aligned(size_t size, size_t alignment, bool nothrow) { + void* p = NULL; + for(int i = 0; i < MI_TRY_NEW_MAX && p==NULL && mi_try_new_handler(nothrow); i++) { + if (!mi_alignment_is_valid(alignment)) return NULL; + p = mi_malloc_aligned(size,alignment); } - while(p == NULL && mi_try_new_handler(false)); + return p; +} + +mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned(size_t size, size_t alignment) { + void* p = mi_malloc_aligned(size, alignment); + if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,false); return p; } mi_decl_nodiscard mi_decl_restrict void* mi_new_aligned_nothrow(size_t size, size_t alignment) mi_attr_noexcept { - void* p; - do { - p = mi_malloc_aligned(size, alignment); - } - while(p == NULL && mi_try_new_handler(true)); + void* p = mi_malloc_aligned(size, alignment); + if mi_unlikely(p==NULL) return mi_try_new_aligned(size,alignment,true); return p; } +static mi_decl_noinline void* mi_try_new_realloc(void* p, size_t newsize) { + void* q = NULL; + for(int i = 0; i < MI_TRY_NEW_MAX && q==NULL && mi_try_new_handler(false); i++) { + if (newsize > MI_MAX_ALLOC_SIZE) return NULL; + q = mi_realloc(p,newsize); + } + return q; +} + mi_decl_nodiscard void* mi_new_realloc(void* p, size_t newsize) { - void* q; - do { - q = mi_realloc(p, newsize); - } while (q == NULL && mi_try_new_handler(false)); + void* q = mi_realloc(p, newsize); + if (q == NULL) return mi_try_new_realloc(p,newsize); return q; } mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { size_t total; if mi_unlikely(mi_count_size_overflow(newcount, size, &total)) { - mi_try_new_handler(false); // on overflow we invoke the try_new_handler once to potentially throw std::bad_alloc + mi_try_new_handler(false); return NULL; } else { @@ -836,7 +907,7 @@ static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t return p; } -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero) mi_attr_noexcept +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { // allocate multiple of page size ending in a guard page // ensure minimal alignment requirement? @@ -848,13 +919,13 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, NULL); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, ppage); if (block==NULL) return NULL; size_t usable_size = 0; void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); - if (p == NULL) return p; - if (zero) { - _mi_memzero_aligned(p,obj_size); // we have to zero here as padding might have written here (if the blocksize > reqsize + os_page_size) + if (p == NULL) return NULL; + if (zero) { + _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) } // stats @@ -887,6 +958,8 @@ void* _mi_externs[] = { (void*)&mi_theap_malloc, (void*)&mi_theap_zalloc, (void*)&mi_theap_malloc_small, + (void*)&mi_theap_zalloc_small, + (void*)&mi_theap_calloc, (void*)&mi_malloc, (void*)&mi_malloc_small, (void*)&mi_zalloc, diff --git a/src/arena-meta.c b/src/arena-meta.c deleted file mode 100644 index 28f1a01ff..000000000 --- a/src/arena-meta.c +++ /dev/null @@ -1,183 +0,0 @@ -/* ---------------------------------------------------------------------------- -Copyright (c) 2019-2024, Microsoft Research, Daan Leijen -This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. ------------------------------------------------------------------------------*/ - -/* ---------------------------------------------------------------------------- - We have a special "mini" allocator just for allocation of meta-data like - the theap (`mi_theap_t`) or thread-local data (`mi_tld_t`). - - We reuse the bitmap of the arena's for allocation of 64b blocks inside - an arena slice (64KiB). - We always ensure that meta data is zero'd (we zero on `free`) ------------------------------------------------------------------------------*/ - -#include "mimalloc.h" -#include "mimalloc/internal.h" -#include "bitmap.h" - -/* ----------------------------------------------------------- - Meta data allocation ------------------------------------------------------------ */ - -#define MI_META_PAGE_SIZE MI_ARENA_SLICE_SIZE -#define MI_META_PAGE_ALIGN MI_ARENA_SLICE_ALIGN - -// large enough such that META_MAX_SIZE > 4k (even on 32-bit) -#define MI_META_BLOCK_SIZE (1 << (16 - MI_BCHUNK_BITS_SHIFT)) // 128 on 64-bit -#define MI_META_BLOCK_ALIGN MI_META_BLOCK_SIZE -#define MI_META_BLOCKS_PER_PAGE (MI_META_PAGE_SIZE / MI_META_BLOCK_SIZE) // 512 -#define MI_META_MAX_SIZE (MI_BCHUNK_SIZE * MI_META_BLOCK_SIZE) - -#if MI_META_MAX_SIZE <= 4096 -#error "max meta object size should be at least 4KiB" -#endif -#if MI_META_BLOCK_ALIGN < MI_BCHUNK_SIZE -#error "minimal meta object alignment should be at least MI_BCHUNK_SIZE (for thread locals)" -#endif - -struct mi_meta_page_s { - _Atomic(struct mi_meta_page_s*) next; // a linked list of meta-data pages (never released) - mi_memid_t memid; // provenance of the meta-page memory itself - // mi_subproc_t* subproc; // subprocess this page belongs to - mi_bbitmap_t blocks_free; // a small bitmap with 1 bit per block. -}; - -#if MI_DEBUG > 1 -static mi_meta_page_t* mi_meta_page_of_ptr(void* p, size_t* block_idx) { - mi_meta_page_t* mpage = (mi_meta_page_t*)((uint8_t*)_mi_align_down_ptr(p,MI_META_PAGE_ALIGN) + _mi_os_secure_guard_page_size()); - if (block_idx != NULL) { - *block_idx = ((uint8_t*)p - (uint8_t*)mpage) / MI_META_BLOCK_SIZE; - } - return mpage; -} -#endif - -static mi_meta_page_t* mi_meta_page_next( mi_meta_page_t* mpage ) { - return mi_atomic_load_ptr_acquire(mi_meta_page_t, &mpage->next); -} - -static void* mi_meta_block_start( mi_meta_page_t* mpage, size_t block_idx ) { - mi_assert_internal(_mi_is_aligned((uint8_t*)mpage - _mi_os_secure_guard_page_size(), MI_META_PAGE_ALIGN)); - mi_assert_internal(block_idx < MI_META_BLOCKS_PER_PAGE); - void* p = ((uint8_t*)mpage - _mi_os_secure_guard_page_size() + (block_idx * MI_META_BLOCK_SIZE)); - mi_assert_internal(mpage == mi_meta_page_of_ptr(p,NULL)); - return p; -} - -// allocate a fresh meta page and add it to the global list. -static mi_meta_page_t* mi_meta_page_zalloc(mi_subproc_t* subproc) { - mi_assert_internal(subproc!=NULL); - mi_assert_internal(subproc->heap_main!=NULL); - // allocate a fresh arena slice - // note: careful with _mi_subproc as it may recurse into mi_tld and meta_page_zalloc again.. (same with _mi_os_numa_node()...) - mi_memid_t memid; - uint8_t* base = (uint8_t*)_mi_arenas_alloc_aligned(subproc->heap_main, MI_META_PAGE_SIZE, MI_META_PAGE_ALIGN, 0, - true /* commit*/, (MI_SECURE==0) /* allow large? */, - NULL /* req arena */, 0 /* thread_seq */, -1 /* numa node */, &memid); - if (base == NULL) return NULL; - mi_assert_internal(_mi_is_aligned(base,MI_META_PAGE_ALIGN)); - if (!memid.initially_zero) { - _mi_memzero_aligned(base, MI_ARENA_SLICE_SIZE); - } - - // guard pages - #if MI_SECURE >= 1 - _mi_os_secure_guard_page_set_at(subproc, base, memid); - _mi_os_secure_guard_page_set_before(subproc, base + MI_META_PAGE_SIZE, memid); - #endif - - // initialize the page and free block bitmap - mi_meta_page_t* mpage = (mi_meta_page_t*)(base + _mi_os_secure_guard_page_size()); - mpage->memid = memid; - mi_bbitmap_init(subproc, &mpage->blocks_free, MI_META_BLOCKS_PER_PAGE, true /* already_zero */); - const size_t mpage_size = offsetof(mi_meta_page_t,blocks_free) + mi_bbitmap_size(MI_META_BLOCKS_PER_PAGE, NULL); - const size_t info_blocks = _mi_divide_up(mpage_size,MI_META_BLOCK_SIZE); - const size_t guard_blocks = _mi_divide_up(_mi_os_secure_guard_page_size(), MI_META_BLOCK_SIZE); - mi_assert_internal(info_blocks + 2*guard_blocks < MI_META_BLOCKS_PER_PAGE); - mi_bbitmap_unsafe_setN(&mpage->blocks_free, info_blocks + guard_blocks, MI_META_BLOCKS_PER_PAGE - info_blocks - 2*guard_blocks); - - // push atomically in front of the meta page list - // (note: there is no ABA issue since we never free meta-pages) - mi_meta_page_t* old = mi_atomic_load_ptr_acquire(mi_meta_page_t,&subproc->meta_pages); - do { - mi_atomic_store_ptr_release(mi_meta_page_t, &mpage->next, old); - } while(!mi_atomic_cas_ptr_weak_acq_rel(mi_meta_page_t,&subproc->meta_pages,&old,mpage)); - return mpage; -} - - -// allocate meta-data -mi_decl_noinline void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* pmemid ) -{ - mi_assert_internal(pmemid != NULL); - mi_assert_internal(subproc!=NULL); - size = _mi_align_up(size,MI_META_BLOCK_SIZE); - if (size == 0 || size > MI_META_MAX_SIZE) return NULL; - const size_t block_count = _mi_divide_up(size,MI_META_BLOCK_SIZE); - mi_assert_internal(block_count > 0 && block_count < MI_BCHUNK_BITS); - mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t,&subproc->meta_pages); - mi_meta_page_t* mpage = mpage0; - while (mpage != NULL) { - size_t block_idx; - if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) { - // found and claimed `block_count` blocks - *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count); - return mi_meta_block_start(mpage,block_idx); - } - else { - mpage = mi_meta_page_next(mpage); - } - } - // failed to find space in existing pages - if (mi_atomic_load_ptr_acquire(mi_meta_page_t,&subproc->meta_pages) != mpage0) { - // the page list was updated by another thread in the meantime, retry - return _mi_meta_zalloc(subproc,size,pmemid); - } - // otherwise, allocate a fresh metapage and try once more - mpage = mi_meta_page_zalloc(subproc); - if (mpage != NULL) { - size_t block_idx; - if (mi_bbitmap_try_find_and_clearN(&mpage->blocks_free, 0, block_count, &block_idx)) { - // found and claimed `block_count` blocks - *pmemid = _mi_memid_create_meta(mpage, block_idx, block_count); - return mi_meta_block_start(mpage,block_idx); - } - } - // if all this failed, allocate from the OS - return _mi_os_alloc(subproc, size, pmemid); -} - -// free meta-data -mi_decl_noinline void _mi_meta_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid) { - if (p==NULL) return; - if (memid.memkind == MI_MEM_META) { - mi_assert_internal(_mi_divide_up(size, MI_META_BLOCK_SIZE) == memid.mem.meta.block_count); - const size_t block_count = memid.mem.meta.block_count; - const size_t block_idx = memid.mem.meta.block_index; - mi_meta_page_t* mpage = (mi_meta_page_t*)memid.mem.meta.meta_page; - mi_assert_internal(mi_meta_page_of_ptr(p,NULL) == mpage); - mi_assert_internal(block_idx + block_count <= MI_META_BLOCKS_PER_PAGE); - mi_assert_internal(mi_bbitmap_is_clearN(&mpage->blocks_free, block_idx, block_count)); - // we zero on free (and on the initial page allocation) so we don't need a "dirty" map - _mi_memzero_aligned(mi_meta_block_start(mpage, block_idx), block_count*MI_META_BLOCK_SIZE); - mi_bbitmap_setN(&mpage->blocks_free, block_idx, block_count); - } - else { - _mi_arenas_free(subproc, p, size, memid); - } -} - -// used for debug output -bool _mi_meta_is_meta_page(mi_subproc_t* subproc, void* p) -{ - mi_meta_page_t* mpage0 = mi_atomic_load_ptr_acquire(mi_meta_page_t, &subproc->meta_pages); - mi_meta_page_t* mpage = mpage0; - while (mpage != NULL) { - if ((void*)mpage == p) return true; - mpage = mi_meta_page_next(mpage); - } - return false; -} diff --git a/src/arena.c b/src/arena.c index af7e8fed5..cae63819c 100644 --- a/src/arena.c +++ b/src/arena.c @@ -25,8 +25,8 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo #include "mimalloc/prim-tls.h" #include "bitmap.h" -#if (MI_ARENA_MAX_SIZE >= MI_SIZE_SIZE*UINT32_MAX) -#error "The page_t.page_woffset field is not large enough to cover a full arena (redefine it to be an offset in MI_MAX_ALIGN_SIZE sizes?)" +#if (MI_ARENA_MAX_SIZE > MI_MAX_ALIGN_SIZE*UINT32_MAX) +#error "The page_t.page_ma_offset field is not large enough to cover a full arena" #endif /* ----------------------------------------------------------- @@ -104,8 +104,8 @@ static size_t mi_arena_max_object_size(void) { if (max_size <= MI_ARENA_MIN_OBJ_SIZE) { return MI_ARENA_MIN_OBJ_SIZE; } - else if (max_size >= MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE) { // minus a bchunk to accommodate meta info - return (MI_ARENA_MAX_SIZE - MI_BCHUNK_SIZE); + else if (max_size >= MI_ARENA_MAX_SIZE - (MI_BCHUNK_BITS*MI_ARENA_SLICE_SIZE)) { // minus an initial chunk to accommodate meta info + return (MI_ARENA_MAX_SIZE - (MI_BCHUNK_BITS*MI_ARENA_SLICE_SIZE)); } else { return max_size; @@ -239,6 +239,10 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( mi_assert_internal(already_dirty <= touched_slices); touched_slices -= already_dirty; } + else { + // todo: properly count touched pages with a separate bitmap? + touched_slices = 0; + } // set commit state if (commit) { @@ -741,7 +745,7 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl _mi_page_free_collect(page, false); // update `used` count mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); @@ -813,31 +817,33 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ static size_t mi_page_block_start(size_t block_size, bool os_align) { + size_t offset; #if MI_GUARDED // in a guarded build, we align pages with blocks a multiple of an OS page size, to the OS page size // this ensures that all blocks in such pages are OS page size aligned (which is needed for the guard pages) const size_t os_page_size = _mi_os_page_size(); mi_assert_internal(MI_PAGE_ALIGN >= os_page_size); if (!os_align && block_size % os_page_size == 0 && block_size > os_page_size /* at least 2 or more */ ) { - return _mi_align_up(mi_page_info_size(), os_page_size); + offset = _mi_align_up(mi_page_info_size(), os_page_size); } else #endif if (os_align) { - return MI_PAGE_ALIGN; + offset = MI_PAGE_ALIGN; } else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB) - return _mi_align_up(mi_page_info_size(), block_size); + offset = _mi_align_up(mi_page_info_size(), block_size); } else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) { // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB) - return _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2); + offset = _mi_align_up(mi_page_info_size(), MI_PAGE_OSPAGE_BLOCK_ALIGN2); } else { // otherwise start after the info - return mi_page_info_size(); + offset = mi_page_info_size(); } + return _mi_align_up(offset,MI_MAX_ALIGN_SIZE); } // Free a page without modifying page_bin stats @@ -900,11 +906,12 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou page = (mi_page_t*)slice_start; block_start = mi_page_block_start(block_size, os_align); } + mi_assert_internal(block_start % MI_MAX_ALIGN_SIZE == 0); // commit first block? size_t commit_size = 0; if (!memid.initially_committed) { - commit_size = _mi_align_up(block_start + block_size, MI_PAGE_MIN_COMMIT_SIZE); + commit_size = _mi_align_up(block_start + block_size, mi_page_min_commit_size()); if (commit_size > page_noguard_size) { commit_size = page_noguard_size; } bool is_zero = false; if mi_unlikely(!mi_arena_commit( _mi_theap_subproc(theap), mi_memid_arena(memid), slice_start, commit_size, &is_zero, 0)) { @@ -922,7 +929,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou // set the guard page #if MI_SECURE>=5 if (memid.initially_committed) { - _mi_os_secure_guard_page_set_at(slice_start + page_noguard_size, memid); + _mi_os_secure_guard_page_set_at(_mi_theap_subproc(theap), slice_start + page_noguard_size, memid); } #endif @@ -949,8 +956,8 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou uint8_t* const start = slice_start + block_start; mi_assert_internal(start > (uint8_t*)page); const size_t offset = start - (uint8_t*)page; - mi_assert_internal((offset % MI_SIZE_SIZE) == 0 && (offset / MI_SIZE_SIZE) < UINT32_MAX); - page->page_woffset = (uint32_t)(offset / MI_SIZE_SIZE); + mi_assert_internal((offset % MI_MAX_ALIGN_SIZE) == 0 && (offset / MI_MAX_ALIGN_SIZE) <= UINT32_MAX); + page->page_ma_offset = (uint32_t)(offset / MI_MAX_ALIGN_SIZE); // initialize page meta-data page->reserved = (uint16_t)reserved; @@ -958,8 +965,8 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou page->memid = memid; page->free_is_zero = memid.initially_zero; - mi_assert_internal((commit && commit_size==0) || (!commit && commit_size < UINT32_MAX)); - page->slice_committed = (uint32_t)commit_size; + mi_assert_internal((commit && commit_size==0) || (!commit && (commit_size <= UINT16_MAX * _mi_os_page_size()))); + page->slice_pcommitted = (uint16_t)(commit_size / _mi_os_page_size()); page->heap = _mi_theap_heap(theap); mi_page_set_theap(page,theap); @@ -968,6 +975,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou mi_assert_internal(page->free==NULL); mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page)); mi_assert_internal(mi_page_slice_start(page) == slice_start); + mi_assert_internal(mi_page_size(page) <= page_noguard_size); // now register in the arena_pages if (arena_pages!=NULL) { @@ -1016,15 +1024,15 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c // 2. find a free block, potentially allocating a new arena const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); - const bool commit = (slice_count <= mi_slice_count_of_size(MI_PAGE_MIN_COMMIT_SIZE) || // always commit small pages - (slice_count >= mi_slice_count_of_size(UINT32_MAX)) || // always commit pages too large to hold a 32-bit slice_committed + const bool commit = (slice_count <= mi_slice_count_of_size(mi_page_min_commit_size()) || // always commit small pages + (slice_count >= mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit); if (page == NULL) return NULL; mi_assert_internal(page->memid.memkind != MI_MEM_ARENA || page->memid.mem.arena.slice_count == slice_count); if (!_mi_page_init(theap, page)) { - mi_arenas_page_free_prim(page); + _mi_arenas_page_free(page,theap); return NULL; } @@ -1047,7 +1055,7 @@ static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block mi_assert(page->reserved == 1); if (!_mi_page_init(theap, page)) { - mi_arenas_page_free_prim(page); + _mi_arenas_page_free(page,theap); return NULL; } @@ -1103,7 +1111,7 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { mi_arena_pages_t* arena_pages = NULL; mi_arena_t* const arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(bin >= MI_ARENA_BIN_COUNT || mi_bitmap_is_clearN(arena_pages->pages_abandoned[bin], slice_index, 1)); mi_assert_internal(mi_bitmap_is_setN(arena_pages->pages, slice_index, 1)); // note: we cannot check for `!mi_page_is_abandoned_and_mapped` since that may @@ -1116,7 +1124,7 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { // we must do this since we may later allocate large spans over this page and cannot have a guard page in between #if MI_SECURE >= 5 if (!page->memid.is_pinned) { - _mi_os_secure_guard_page_reset_before(mi_page_slice_start(page) + mi_page_full_size(page), page->memid); + _mi_os_secure_guard_page_reset_before(mi_page_subproc(page), mi_page_slice_start(page) + mi_page_full_size(page), page->memid); } #endif @@ -1131,17 +1139,18 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { mi_assert_internal(arena_pages!=NULL); mi_assert_internal(arena->subproc == mi_page_subproc(page)); mi_bitmap_clear(arena_pages->pages, slice_index); - if (page->slice_committed > 0) { + const size_t slice_committed = mi_page_slice_committed(page); + if (slice_committed > 0) { // if committed on-demand, set the commit bits to account commit properly - mi_assert_internal(mi_page_full_size(page) >= page->slice_committed); - const size_t total_slices = page->slice_committed / MI_ARENA_SLICE_SIZE; // conservative + mi_assert_internal(mi_page_full_size(page) >= slice_committed); + const size_t total_slices = slice_committed / MI_ARENA_SLICE_SIZE; // conservative //mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, slice_index, total_slices)); mi_assert_internal(slice_count >= total_slices); if (total_slices > 0) { mi_bitmap_setN(arena->slices_committed, slice_index, total_slices, NULL); } // any left over? - const size_t extra = page->slice_committed % MI_ARENA_SLICE_SIZE; + const size_t extra = slice_committed % MI_ARENA_SLICE_SIZE; if (extra > 0) { // pretend it was decommitted already mi_subproc_stat_decrease(arena->subproc, committed, extra); @@ -1194,7 +1203,7 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) { mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(!mi_page_all_free(page)); mi_assert_internal(page->next==NULL && page->prev == NULL); - mi_assert_internal(_mi_thread_id()==current_theap->tld->thread_id); + mi_assert_internal(mi_theap_matches_thread(current_theap)); // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); // add to abandoned? @@ -1212,7 +1221,7 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) { mi_assert_internal(!mi_page_is_singleton(page)); mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); mi_assert_internal(mi_bitmap_is_setN(arena->slices_dirty, slice_index, slice_count)); mi_page_set_abandoned_mapped(page); @@ -1290,7 +1299,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_arena_t* arena = mi_page_arena_pages(page, &slice_index, &slice_count, &arena_pages); MI_UNUSED(arena); mi_assert_internal(mi_bbitmap_is_clearN(arena->slices_free, slice_index, slice_count)); - mi_assert_internal(page->slice_committed > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); + mi_assert_internal(mi_page_slice_committed(page) > 0 || mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); // this busy waits until a concurrent reader (from alloc_abandoned) is done mi_bitmap_clear_once_set(arena->subproc, arena_pages->pages_abandoned[bin], slice_index); @@ -1512,8 +1521,8 @@ void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t mem return; }; } - else if (memid.memkind == MI_MEM_META) { - _mi_meta_free(subproc, p, size, memid); + else if (memid.memkind == MI_MEM_MALLOC) { + _mi_free_subproc_safe(p); } else { // arena was none, external, or static; nothing to do @@ -1554,10 +1563,10 @@ static bool mi_arenas_contain_ex(const void* p, mi_arena_t* parent) { return false; } -// Is a pointer inside any of our arenas? -bool _mi_arenas_contain(const void* p) { - return mi_arenas_contain_ex(p, NULL); -} +// // Is a pointer inside any of our arenas? +// bool _mi_arenas_contain(const void* p) { +// return mi_arenas_contain_ex(p, NULL); +// } // Is a pointer contained in the given arena area? bool mi_arena_contains(mi_arena_id_t arena_id, const void* p) { @@ -1914,7 +1923,13 @@ bool mi_manage_memory(void* start, size_t size, bool is_committed, bool is_pinne // Reserve a range of regular OS memory static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id) { if (arena_id != NULL) *arena_id = _mi_arena_id_none(); - size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice + if (size <= MI_MAX_ALLOC_SIZE) { + size = _mi_align_up(size, MI_ARENA_SLICE_SIZE); // at least one slice + } + if (size > MI_MAX_ALLOC_SIZE) { + _mi_error_message(EOVERFLOW, "memory reservation request is too large (size %zu)\n", size); + return ENOMEM; + } mi_memid_t memid; void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid); if (start == NULL) return ENOMEM; @@ -1995,7 +2010,6 @@ static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) { } static int mi_page_commit_usage(mi_page_t* page) { - // if (mi_page_size(page) <= MI_PAGE_MIN_COMMIT_SIZE) return 100; const size_t committed_size = mi_page_committed(page); const size_t used_size = page->used * mi_page_block_size(page); return (int)(used_size * 100 / committed_size); @@ -2016,7 +2030,8 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, bit_set_count++; c = 'p'; color = MI_GRAY; - if (mi_page_is_singleton(page)) { c = 's'; } + if (_mi_meta_is_meta_page(arena->subproc,page)) { c = 'm'; } + else if (mi_page_is_singleton(page)) { c = 's'; } else if (mi_page_is_full(page)) { c = 'f'; } if (!mi_page_is_abandoned(page)) { c = _mi_toupper(c); } int commit_usage = mi_page_commit_usage(page); @@ -2029,7 +2044,7 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, else { c = '?'; if (bit_of_page > 0) { c = '-'; } - else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } + // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; } else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) { @@ -2137,7 +2152,7 @@ static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narro size_t page_total = 0; for (size_t i = 0; i < max_arenas; i++) { mi_arena_t* arena = mi_atomic_load_ptr_acquire(mi_arena_t, &subproc->arenas[i]); - if (arena == NULL) break; + if (arena == NULL) continue; mi_assert(arena->subproc == subproc); // slice_total += arena->slice_count; _mi_raw_message("%sarena %zu at %p: %zu slices (%zu MiB)%s%s, subproc: %zu, numa: %i\n", @@ -2158,7 +2173,7 @@ static void mi_debug_show_arenas_ex(mi_heap_t* heap, bool show_pages, bool narro // mi_arena_pages_t* arena_pages = mi_heap_arena_pages(heap, arena); // if (arena_pages != NULL) { - const char* header1 = "chunks (p:page, f:full, s:singleton, P,F,S:not abandoned, i:arena-info, m:meta-data, ~:free-purgable, _:free-committed, .:free-reserved)"; + const char* header1 = "chunks (p:page, f:full, s:single, m:meta-data, i:arena-info, P,F,S,M:not abandoned, ~:free-purgable, _:free-committed, .:free-reserved)"; const char* header2 = (narrow ? "\n " : " "); const char* header3 = "(chunk bin: S:small, M : medium, L : large, X : other)"; page_total += mi_debug_show_chunks(header1, header2, header3, arena->slice_count, @@ -2773,7 +2788,7 @@ static void mi_heap_delete_pages(mi_heap_t* heap, mi_heap_t* heap_target) { _mi_heap_visit_blocks(heap, false, false, true /* claim each page */, &mi_heap_delete_page, &info); #if MI_DEBUG>1 // no more arena pages? - for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { + for (size_t i = 0; i < MI_MAX_ARENAS; i++) { mi_arena_pages_t* const arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); if (arena_pages!=NULL) { mi_assert_internal(mi_bitmap_is_all_clear(arena_pages->pages)); diff --git a/src/bitmap.c b/src/bitmap.c index 40e770db4..01de1999a 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2019-2024 Microsoft Research, Daan Leijen +Copyright (c) 2019-2026 Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -107,8 +107,8 @@ static inline bool mi_bfield_atomic_clear(_Atomic(mi_bfield_t)*b, size_t idx, bo } // Clear a bit but only when/once it is set. This is used by concurrent free's while -// the page is abandoned and mapped. This can incure a busy wait :-( but it should -// happen almost never (and is accounted for in the stats) +// the page is abandoned and mapped. This can incur a busy wait :-( but it should +// be quite rare (and is accounted for in the stats) static inline void mi_bfield_atomic_clear_once_set(mi_subproc_t* subproc, _Atomic(mi_bfield_t)*b, size_t idx) { mi_assert_internal(idx < MI_BFIELD_BITS); const mi_bfield_t mask = mi_bfield_mask(1, idx);; @@ -133,8 +133,7 @@ static inline void mi_bfield_atomic_clear_once_set(mi_subproc_t* subproc, _Atomi // statistics correctly). static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, size_t* already_set) { mi_assert_internal(mask != 0); - mi_bfield_t old = mi_atomic_load_relaxed(b); - while (!mi_atomic_cas_weak_acq_rel(b, &old, old|mask)) {}; // try to atomically set the mask bits until success + const mi_bfield_t old = mi_atomic_or_acq_rel(b,mask); if (already_set!=NULL) { *already_set = mi_bfield_popcount(old&mask); } return ((old&mask) == 0); } @@ -143,8 +142,7 @@ static inline bool mi_bfield_atomic_set_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t // `all_clear` is set to `true` if the new bfield became zero. static inline bool mi_bfield_atomic_clear_mask(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, bool* all_clear) { mi_assert_internal(mask != 0); - mi_bfield_t old = mi_atomic_load_relaxed(b); - while (!mi_atomic_cas_weak_acq_rel(b, &old, old&~mask)) {}; // try to atomically clear the mask bits until success + const mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask); if (all_clear != NULL) { *all_clear = ((old&~mask)==0); } return ((old&mask) == mask); } @@ -155,65 +153,79 @@ static inline bool mi_bfield_atomic_setX(_Atomic(mi_bfield_t)*b, size_t* already return (old==0); } -// static inline bool mi_bfield_atomic_clearX(_Atomic(mi_bfield_t)*b, bool* all_clear) { -// const mi_bfield_t old = mi_atomic_exchange_release(b, mi_bfield_zero()); -// if (all_clear!=NULL) { *all_clear = true; } -// return (~old==0); -// } // ------- mi_bfield_atomic_try_clear --------------------------------------- - // Tries to clear a mask atomically, and returns true if the mask bits atomically transitioned from mask to 0 -// and false otherwise (leaving the bit field as is). +// and false otherwise (leaving the bit field as is). Returns the acquired value back in `expect` (regardless of success). // `all_clear` is set to `true` if the new bfield became zero. -static inline bool mi_bfield_atomic_try_clear_mask_of(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t expect, bool* all_clear) { +// `did_temp_clear_bits` is set to `true` some bits were temporarily cleared. +static inline bool mi_bfield_atomic_try_clear_mask_optimistic(_Atomic(mi_bfield_t)*b, mi_bfield_t mask, mi_bfield_t* previous, bool* did_temp_clear_bits) { mi_assert_internal(mask != 0); + // note: we could also use a strong cas but generally an optimistic atomic and/or is more efficient (at least on arm64) // try to atomically clear the mask bits - do { - if ((expect & mask) != mask) { // are all bits still set? - if (all_clear != NULL) { *all_clear = (expect == 0); } - return false; + mi_bfield_t old = mi_atomic_and_acq_rel(b,~mask); + if (previous!=NULL) { *previous = old; } + if mi_likely((old&mask)==mask) { + // all bits in the mask transitioned from 1 to 0 atomically + return true; + } + else { + // failed to transition all + if ((old&mask)!=0) { + // restore accidentally cleared ones + mi_atomic_or_acq_rel(b, old&mask); + if (did_temp_clear_bits!=NULL) { *did_temp_clear_bits = true; } } - } while (!mi_atomic_cas_weak_acq_rel(b, &expect, expect & ~mask)); - if (all_clear != NULL) { *all_clear = ((expect & ~mask) == 0); } - return true; -} - -static inline bool mi_bfield_atomic_try_clear_mask(_Atomic(mi_bfield_t)* b, mi_bfield_t mask, bool* all_clear) { - mi_assert_internal(mask != 0); - const mi_bfield_t expect = mi_atomic_load_relaxed(b); - return mi_bfield_atomic_try_clear_mask_of(b, mask, expect, all_clear); + return false; + } } -// Tries to clear a bit atomically. Returns `true` if the bit transitioned from 1 to 0 -// and `false` otherwise leaving the bfield `b` as-is. +// Tries to clear a bit atomically. For performance, it assumes there is a good chance of success. +// Returns `true` if the bit transitioned from 1 to 0 +// and `false` otherwise (leaving the bfield `b` as-is). // `all_clear` is set to true if the new bfield became zero (and false otherwise) -mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear(_Atomic(mi_bfield_t)* b, size_t idx, bool* all_clear) { +mi_decl_maybe_unused static inline bool mi_bfield_atomic_try_clear_optimistic(_Atomic(mi_bfield_t)* b, size_t idx, mi_bfield_t* previous) { mi_assert_internal(idx < MI_BFIELD_BITS); - const mi_bfield_t mask = mi_bfield_one()<bfields[field], mask, &already_set) : mi_bfield_atomic_clear_mask(&chunk->bfields[field], mask, &all_clear)); - mi_assert_internal((transition && already_set == 0) || (!transition && already_set > 0)); + mi_assert_internal(!set || ((transition && already_set == 0) || (!transition && already_set > 0))); all_transition = all_transition && transition; total_already_set += already_set; maybe_all_clear = maybe_all_clear && all_clear; @@ -445,14 +457,14 @@ static inline bool mi_bchunk_is_xsetN(mi_xset_t set, const mi_bchunk_t* chunk, s // ------- mi_bchunk_try_clear --------------------------------------- // Clear `0 < n <= MI_BITFIELD_BITS`. Can cross over a bfield boundary. -static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) { +static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) { mi_assert_internal(cidx < MI_BCHUNK_BITS); mi_assert_internal(n <= MI_BFIELD_BITS); const size_t i = cidx / MI_BFIELD_BITS; const size_t idx = cidx % MI_BFIELD_BITS; if mi_likely(idx + n <= MI_BFIELD_BITS) { // within one field - return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear); + return mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(n, idx), pmaybe_all_clear, did_temp_clear_bits); } else { // spanning two fields (todo: use double-word atomic ops?) @@ -460,14 +472,15 @@ static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t mi_assert_internal(m < n); mi_assert_internal(i < MI_BCHUNK_FIELDS - 1); bool field1_is_clear; - if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear)) return false; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), &field1_is_clear, did_temp_clear_bits)) return false; // try the second field as well mi_assert_internal(n - m > 0); mi_assert_internal(n - m < MI_BFIELD_BITS); bool field2_is_clear; - if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear)) { + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[i+1], mi_bfield_mask(n - m, 0), &field2_is_clear, did_temp_clear_bits)) { // we failed to clear the second field, restore the first one mi_bfield_atomic_set_mask(&chunk->bfields[i], mi_bfield_mask(m, idx), NULL); + if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; } return false; } if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = field1_is_clear && field2_is_clear; } @@ -488,7 +501,7 @@ static inline bool mi_bchunk_try_clearNX(mi_bchunk_t* chunk, size_t cidx, size_t // and false otherwise leaving all bit fields as is. // Note: this is the complex one as we need to unwind partial atomic operations if we fail halfway.. // `maybe_all_clear` is set to `true` if all the bfields involved become zero. -mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear) { +mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* pmaybe_all_clear, bool* did_temp_clear_bits) { mi_assert_internal(cidx + n <= MI_BCHUNK_BITS); mi_assert_internal(n>0); if (pmaybe_all_clear != NULL) { *pmaybe_all_clear = true; } @@ -504,7 +517,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci mi_assert_internal(start_field < MI_BCHUNK_FIELDS); const mi_bfield_t mask_start = mi_bfield_mask(m, start_idx); bool maybe_all_clear; - if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear)) return false; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_start, &maybe_all_clear, did_temp_clear_bits)) return false; // done? mi_assert_internal(m <= n); @@ -528,7 +541,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci mi_assert_internal(field < MI_BCHUNK_FIELDS); const mi_bfield_t mask_end = mi_bfield_mask(n, 0); bool field_is_clear; - if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear)) goto restore; + if (!mi_bfield_atomic_try_clear_mask(&chunk->bfields[field], mask_end, &field_is_clear, did_temp_clear_bits)) goto restore; maybe_all_clear = maybe_all_clear && field_is_clear; } @@ -538,6 +551,7 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci restore: // `field` is the index of the field that failed to set atomically; we need to restore all previous fields mi_assert_internal(field > start_field); + if (did_temp_clear_bits != NULL) { *did_temp_clear_bits = true; } while( field > start_field) { field--; if (field == start_field) { @@ -551,11 +565,11 @@ mi_decl_noinline static bool mi_bchunk_try_clearNC(mi_bchunk_t* chunk, size_t ci } -static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear) { +static inline bool mi_bchunk_try_clearN(mi_bchunk_t* chunk, size_t cidx, size_t n, bool* maybe_all_clear, bool* did_temp_clear_bits) { mi_assert_internal(n>0); // if (n==MI_BFIELD_BITS) return mi_bchunk_try_clearX(chunk, cidx, maybe_all_clear); - if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear); - return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear); + if (n<=MI_BFIELD_BITS) return mi_bchunk_try_clearNX(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits); + return mi_bchunk_try_clearNC(chunk, cidx, n, maybe_all_clear, did_temp_clear_bits); } @@ -576,19 +590,24 @@ mi_decl_maybe_unused static inline bool mi_mm256_is_zero( __m256i vec) { } #endif + static inline bool mi_bchunk_try_find_and_clear_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) { mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS); - // note: this must be acquire (and not relaxed), or otherwise the AVX code below can loop forever - // as the compiler won't reload the registers vec1 and vec2 from memory again. - const mi_bfield_t b = mi_atomic_load_acquire(&chunk->bfields[chunk_idx]); - size_t idx; - if (mi_bfield_find_least_bit(b, &idx)) { // find the least bit - if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], mi_bfield_mask(1,idx), b, NULL)) { // clear it atomically - *pidx = (chunk_idx*MI_BFIELD_BITS) + idx; + _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx]; + mi_bfield_t b = mi_atomic_load_relaxed(bfield); + if (b==0) return false; + int tries = 0; + do { + const mi_bfield_t mask = (b & (~b+1)); // == (b & -b) but avoids a compiler warning -- clear all bits except the least-significant one + b = mi_atomic_and_acq_rel(bfield,~mask); // clear the bit and set `b` to the previous value + if mi_likely((b&mask)==mask) { // if we transitioned from 1 to 0, we actually cleared it + size_t bitidx = 0; + mi_bfield_find_least_bit(mask,&bitidx); + *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); return true; - } - } + } + } while (b!=0 && ++tries <= 4); // limit tries to reduce possible contention return false; } @@ -686,28 +705,36 @@ static inline bool mi_bchunk_try_find_and_clear(mi_bchunk_t* chunk, size_t* pidx return false; } -static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx) { - mi_assert_internal(n==1); MI_UNUSED(n); +static inline bool mi_bchunk_try_find_and_clear_1(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { + mi_assert_internal(n==1); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits); return mi_bchunk_try_find_and_clear(chunk, pidx); } -mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx) { - const mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[chunk_idx]); - // has_set8 has low bit in each byte set if the byte in x == 0xFF - const mi_bfield_t has_set8 = - ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F - (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80 - >> 7; // shift high bit to low bit - size_t idx; - if (mi_bfield_find_least_bit(has_set8, &idx)) { // find least 1-bit - mi_assert_internal(idx <= (MI_BFIELD_BITS - 8)); - mi_assert_internal((idx%8)==0); - if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[chunk_idx], (mi_bfield_t)0xFF << idx, b, NULL)) { // unset the byte atomically - *pidx = (chunk_idx*MI_BFIELD_BITS) + idx; - mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS); - return true; +mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchunk_t* chunk, size_t chunk_idx, size_t* pidx, bool* did_temp_clear_bits) { + _Atomic(mi_bfield_t)* const bfield = &chunk->bfields[chunk_idx]; + mi_bfield_t b = mi_atomic_load_relaxed(bfield); + if (b==0) return false; + int tries = 0; + do { + // has_set8 has low bit in each byte set if the byte in x == 0xFF + const mi_bfield_t has_set8 = + ((~b - MI_BFIELD_LO_BIT8) & // high bit set if byte in x is 0xFF or < 0x7F + (b & MI_BFIELD_HI_BIT8)) // high bit set if byte in x is >= 0x80 + >> 7; // shift high bit to low bit + size_t bitidx; + if (mi_bfield_find_least_bit(has_set8, &bitidx)) { // find least 1-bit + mi_assert_internal(bitidx <= (MI_BFIELD_BITS - 8)); + mi_assert_internal((bitidx%8)==0); + if mi_likely(mi_bfield_atomic_try_clear8_optimistic(bfield,bitidx,&b,did_temp_clear_bits)) { // try to clear atomically (and update b) + *pidx = (chunk_idx*MI_BFIELD_BITS) + bitidx; + mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS); + return true; + } } - } + else { + return false; + } + } while(b!=0 && ++tries <= 4); // limit tries to reduce possible contention return false; } @@ -715,9 +742,9 @@ mi_decl_maybe_unused static inline bool mi_bchunk_try_find_and_clear8_at(mi_bchu // set `*pidx` to its bit index (0 <= *pidx < MI_BCHUNK_BITS) on success. // Used to find medium size pages in the free blocks. // todo: try neon version -static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx) { +static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, size_t* pidx, bool* did_temp_clear_bits) { #if MI_OPT_SIMD && defined(__AVX2__) && (MI_BCHUNK_BITS==512) - while (true) { + for(int tries=0; tries<4; tries++) { // paranoia: at most 4 tries // since a cache-line is 64b, load all at once const __m256i vec1 = _mm256_load_si256((const __m256i*)chunk->bfields); const __m256i vec2 = _mm256_load_si256((const __m256i*)chunk->bfields+1); @@ -733,25 +760,29 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clear8(mi_bchunk_t* chunk, s const size_t chunk_idx = bidx / 8; const size_t idx = (bidx % 8)*8; mi_assert_internal(chunk_idx < MI_BCHUNK_FIELDS); - if mi_likely(mi_bfield_atomic_try_clear8(&chunk->bfields[chunk_idx], idx, NULL)) { // clear it atomically + if mi_likely(mi_bfield_atomic_try_clear8_optimistic(&chunk->bfields[chunk_idx], idx, NULL, did_temp_clear_bits)) { // clear it atomically *pidx = (chunk_idx*MI_BFIELD_BITS) + idx; mi_assert_internal(*pidx + 8 <= MI_BCHUNK_BITS); return true; } // try again - // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded } + // note: there must be an atomic release/acquire in between or otherwise the registers may not be reloaded + // we add an explicit memory barrier as older gcc compilers do not reload the registers even with an atomic acquire (issue #1206) + #if defined(__GNUC__) + __asm __volatile ("" : : "g"(chunk) : "memory"); + #endif } #else for (int i = 0; i < MI_BCHUNK_FIELDS; i++) { - if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx)) return true; + if (mi_bchunk_try_find_and_clear8_at(chunk, i, pidx, did_temp_clear_bits)) return true; } - return false; #endif + return false; } -static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx) { - mi_assert_internal(n==8); MI_UNUSED(n); - return mi_bchunk_try_find_and_clear8(chunk, pidx); +static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { + mi_assert_internal(n==8); MI_UNUSED(n); MI_UNUSED(did_temp_clear_bits); + return mi_bchunk_try_find_and_clear8(chunk, pidx, did_temp_clear_bits); } @@ -759,7 +790,7 @@ static inline bool mi_bchunk_try_find_and_clear_8(mi_bchunk_t* chunk, size_t n, // and try to clear them atomically. // set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success. // will cross bfield boundaries. -mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx) { +mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { if (n == 0 || n > MI_BFIELD_BITS) return false; const mi_bfield_t mask = mi_bfield_mask(n, 0); // for all fields in the chunk @@ -775,7 +806,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, const size_t bmask = mask<>idx == mask); if ((b&bmask) == bmask) { // found a match with all bits set, try clearing atomically - if mi_likely(mi_bfield_atomic_try_clear_mask_of(&chunk->bfields[i], bmask, b0, NULL)) { + if mi_likely(mi_bfield_atomic_try_clear_mask_optimistic(&chunk->bfields[i], bmask, &b0, did_temp_clear_bits)) { // updates b0 with the previous value *pidx = (i*MI_BFIELD_BITS) + idx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); mi_assert_internal(*pidx + n <= MI_BCHUNK_BITS); @@ -783,7 +814,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, } else { // if we failed to atomically commit, reload b and try again from the start - b = b0 = mi_atomic_load_acquire(&chunk->bfields[i]); + b = b0; // = mi_atomic_load_acquire(&chunk->bfields[i]); } } else { @@ -803,7 +834,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, if (post + pre >= n) { // it fits -- try to claim it atomically const size_t cidx = (i*MI_BFIELD_BITS) + (MI_BFIELD_BITS - post); - if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL)) { + if (mi_bchunk_try_clearNX(chunk, cidx, n, NULL, did_temp_clear_bits)) { // we cleared all atomically *pidx = cidx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); @@ -821,7 +852,7 @@ mi_decl_noinline static bool mi_bchunk_try_find_and_clearNX(mi_bchunk_t* chunk, // and try to clear them atomically. // set `*pidx` to its bit index (0 <= *pidx <= MI_BCHUNK_BITS - n) on success. // This can cross bfield boundaries. -static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx) { +static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, size_t n, size_t* pidx, bool* did_temp_clear_bits) { if (n == 0 || n > MI_BCHUNK_BITS) return false; // cannot be more than a chunk // we first scan ahead to see if there is a range of `n` set bits, and only then try to clear atomically @@ -870,7 +901,7 @@ static mi_decl_noinline bool mi_bchunk_try_find_and_clearNC(mi_bchunk_t* chunk, // did we find a range? if (m==0) { - if (mi_bchunk_try_clearN(chunk, cidx, n, NULL)) { + if (mi_bchunk_try_clearN(chunk, cidx, n, NULL, did_temp_clear_bits)) { // we cleared all atomically *pidx = cidx; mi_assert_internal(*pidx < MI_BCHUNK_BITS); @@ -963,7 +994,7 @@ static bool mi_bchunk_bsr(mi_bchunk_t* chunk, size_t* pidx) { return false; } -static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) { +static bool mi_bchunk_bsr_inv(mi_bchunk_t* chunk, size_t* pidx) { for (size_t i = MI_BCHUNK_FIELDS; i > 0; ) { i--; mi_bfield_t b = mi_atomic_load_relaxed(&chunk->bfields[i]); @@ -1323,9 +1354,10 @@ static bool mi_bitmap_try_find_and_claim_visit(mi_bitmap_t* bitmap, size_t chunk return true; } else { - // failed to claim it, set abandoned mapping again (unless the page was freed) + // failed to claim it, set abandoned mapping again (unless the page was freed and keep_set will be false) if (keep_set) { const bool wasclear = mi_bchunk_set(&bitmap->chunks[chunk_idx], cidx, NULL); + mi_bitmap_chunkmap_set(bitmap, chunk_idx); mi_assert_internal(wasclear); MI_UNUSED(wasclear); } } @@ -1459,6 +1491,8 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis mi_assert_internal(rng>=1 && rng<=MI_BFIELD_BITS); mi_assert_internal((idx % MI_BFIELD_BITS) + rng <= MI_BFIELD_BITS); mi_assert_internal((idx / MI_BCHUNK_BITS) < mi_bitmap_chunk_count(bitmap)); + // clear rng bits in b + b = b & ~mi_bfield_mask(rng, bidx); if (!visit(idx, rng, arena, arg)) { // break early: reset the non-visited bits if (b!=0) { @@ -1466,8 +1500,6 @@ bool _mi_bitmap_forall_setc_ranges(mi_bitmap_t* bitmap, mi_forall_set_fun_t* vis } return false; } - // clear rng bits in b - b = b & ~mi_bfield_mask(rng, bidx); } mi_assert_internal(rngcount == bpopcount); } @@ -1505,14 +1537,15 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_fo const size_t base_idx = (chunk_idx*MI_BCHUNK_BITS) + (j*MI_BFIELD_BITS); mi_bfield_t b = mi_atomic_exchange_relaxed(&chunk->bfields[j], (mi_bfield_t)0); // atomic clear mi_bfield_t skipped = 0; // but track which bits we skip so we can restore them - for(size_t shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment + size_t shift; + for(shift = 0; rngslices + shift <= MI_BFIELD_BITS; shift += rngslices) { // per `rngslices` to keep alignment const mi_bfield_t rngmask = mi_bfield_mask(rngslices, shift); if ((b & rngmask) == rngmask) { const size_t idx = base_idx + shift; if (!visit(idx, rngslices, arena, arg)) { // break early: restore non-visited entries mi_bfield_t notyet_visited = 0; - if (shift + rngslices < MI_BFIELD_BITS) { + if (rngslices + shift < MI_BFIELD_BITS) { notyet_visited = (b & (~(mi_bfield_t)0 << (shift + rngslices))); } mi_assert_internal((notyet_visited & skipped) == 0); @@ -1526,8 +1559,13 @@ bool _mi_bitmap_forall_setc_rangesn(mi_bitmap_t* bitmap, size_t rngslices, mi_fo skipped = skipped | (b & rngmask); } } - + if (shift < MI_BFIELD_BITS) { + // there are some non-visited top bits when `MI_BFIELD_BITS % rngslices != 0`. + mi_assert_internal(MI_BFIELD_BITS % rngslices != 0); + skipped = skipped | (b & (~(mi_bfield_t)0 << shift)); + } if (skipped != 0) { + // restore non-visited entries mi_atomic_or_relaxed(&chunk->bfields[j], skipped); } } @@ -1578,6 +1616,7 @@ void mi_bbitmap_unsafe_setN(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { bool mi_bbitmap_bsr_inv(mi_bbitmap_t* bbitmap, size_t* idx) { // scan for highest zero bit in the bitmap // note: we cannot use the chunkmap since that only conservatively denotes if there might be a set bit in a chuck + // todo: bbitmap_init rounds up the bitcount to BCHUNK_BITS and we should skip the top-padding! const size_t chunk_count = mi_bbitmap_chunk_count(bbitmap); for(size_t i = chunk_count; i > 0; ) { i--; @@ -1702,8 +1741,15 @@ bool mi_bbitmap_try_clearNC(mi_bbitmap_t* bbitmap, size_t idx, size_t n) { mi_assert_internal(chunk_idx < mi_bbitmap_chunk_count(bbitmap)); if (cidx + n > MI_BCHUNK_BITS) return false; bool maybe_all_clear = false; - const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear); - if (cleared && maybe_all_clear) { mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); } + bool did_temp_clear_bits = false; + const bool cleared = mi_bchunk_try_clearN(&bbitmap->chunks[chunk_idx], cidx, n, &maybe_all_clear, &did_temp_clear_bits); + if (cleared && maybe_all_clear) { + mi_assert_internal(!did_temp_clear_bits); + mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + } else if (did_temp_clear_bits) { + // may have raced with a clearer (in on_find) so set the chunkmap bit conservatively + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); + } // note: we don't set the size class for an explicit try_clearN (only used by purging) return cleared; } @@ -1745,14 +1791,14 @@ bool mi_bbitmap_is_xsetN(mi_xset_t set, mi_bbitmap_t* bbitmap, size_t idx, size_ (used to find free pages) -------------------------------------------------------------------------------- */ -typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx); +typedef bool (mi_bchunk_try_find_and_clear_fun_t)(mi_bchunk_t* chunk, size_t n, size_t* idx, bool* did_temp_clear_bits); // Go through the bbitmap and for every sequence of `n` set bits, call the visitor function. // If it returns `true` stop the search. // // This is used for finding free blocks and it is important to be efficient (with 2-level bitscan) // but also reduce fragmentation (through size bins). -static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find) +static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t tseq, size_t n, size_t* pidx, mi_bchunk_try_find_and_clear_fun_t* on_find) { // we space out threads to reduce contention const size_t cmap_max_count = _mi_divide_up(mi_bbitmap_chunk_count(bbitmap),MI_BFIELD_BITS); @@ -1806,7 +1852,8 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx]; size_t cidx; - if ((*on_find)(chunk, n, &cidx)) { + bool did_temp_clear_bits = false; + if ((*on_find)(chunk, n, &cidx, &did_temp_clear_bits)) { if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin // this chunk is now reserved for the `bbin` size class mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin); @@ -1816,9 +1863,17 @@ static inline bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, return true; } else { + // note: we may find that all are cleared only on a second iteration (when we fail to clear any bits) + // but that is ok as the chunkmap is a conservative approximation. // todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?` - // we may find that all are cleared only on a second iteration but that is ok as the chunkmap is a conservative approximation. - mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + // probably not as we already only call `try_clear` once we fail to clear any bits. + if (did_temp_clear_bits) { + // a concurrent find_and_claim may have cleared the chunkmap bit, restore it now + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false); + } + else { + mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx); + } } } mi_bfield_cycle_iterate_end(Y); @@ -1863,12 +1918,11 @@ bool mi_bbitmap_try_find_and_clearNC(mi_bbitmap_t* bbitmap, size_t tseq, size_t // Try to atomically clear `n` bits starting at `chunk_idx` where `n` can span over multiple chunks static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_t n) { mi_assert_internal((chunk_idx * MI_BCHUNK_BITS) + n <= mi_bbitmap_max_bits(bbitmap)); - size_t m = n; // bits to go size_t count = 0; // chunk count while (m > 0) { mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; - if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL)) { + if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL, NULL)) { goto rollback; } m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS); @@ -1882,10 +1936,8 @@ static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_ count--; mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count]; mi_bchunk_setN(chunk, 0, MI_BCHUNK_BITS, NULL); - // restore the chunkmap bit too (a concurrent clear may have observed the - // all-zero chunk between our clear and this set and dropped the cmap bit; - // without this the chunk's free slices become invisible to non-huge searches) - mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, true /* check_all_set */); + // since we may race with clearing, we need to set the chunkmap conservatively + mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, false); } return false; } @@ -1922,7 +1974,7 @@ bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t // did we find a suitable range? if (count == chunk_req) { - // now try to claim it! + // now try to claim it! if (mi_bchunk_try_clearN_(bbitmap, chunk_idx, n)) { *pidx = (chunk_idx * MI_BCHUNK_BITS); for (size_t i = 0; i < count; i++) { @@ -1931,6 +1983,11 @@ bool mi_bbitmap_try_find_and_clearN_(mi_bbitmap_t* bbitmap, size_t tseq, size_t mi_assert_internal(*pidx + n <= mi_bbitmap_max_bits(bbitmap)); return true; } + else { + // contended: we reset count to retry from the first + // (we still skip the first chunk to guarantee progress) + count = 0; + } } // keep searching but skip the scanned range diff --git a/src/free.c b/src/free.c index e26795434..61e064ed5 100644 --- a/src/free.c +++ b/src/free.c @@ -33,7 +33,9 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool if (!was_guarded) { mi_check_padding(page, block); } if (track_stats) { mi_stat_free(page, block); } #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - memset(block, MI_DEBUG_FREED, mi_page_block_size(page)); + size_t dbgsize = mi_page_block_size(page); + if (dbgsize > 1*MI_MiB) { dbgsize = 1*MI_MiB; } + _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); #endif if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned @@ -61,19 +63,20 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // Free a block multi-threaded static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_collect) mi_attr_noexcept { - MI_UNUSED(was_guarded); - // adjust stats (after padding check and potentially recursive `mi_free` above) + // todo: we cannot safely check for double free in _mt -- should check when collecting the thread_free list + if (!was_guarded) { mi_check_padding(page, block); } // checking padding is safe for mt + // adjust stats (after padding check ) mi_stat_free(page, block); // stat_free may access the padding mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); // _mi_padding_shrink(page, block, sizeof(mi_block_t)); -#if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading if (!was_guarded) { size_t dbgsize = mi_usable_size(block); - if (dbgsize > MI_MiB) { dbgsize = MI_MiB; } + if (dbgsize > 1*MI_MiB) { dbgsize = 1*MI_MiB; } _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); } -#endif + #endif // push atomically on the page thread free list mi_thread_free_t tf_new; @@ -201,11 +204,14 @@ static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) // Free a block // Fast path written carefully to prevent register spilling on the stack -static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* page, bool allow_collect) +static mi_decl_forceinline void mi_free_ex(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) { - if mi_unlikely(page==NULL) return; // page will be NULL if p==NULL + if mi_unlikely(page==NULL) { // page will be NULL if p==NULL + if (pblock_size!=NULL) { *pblock_size = 0; } + return; + } mi_assert_internal(p!=NULL && page!=NULL); - if (usable!=NULL) { *usable = mi_page_usable_block_size(page); } + if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page)); if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` @@ -229,14 +235,18 @@ static mi_decl_forceinline void mi_free_ex(void* p, size_t* usable, mi_page_t* p } } +void _mi_free_in_page(void* p, mi_page_t* page) mi_attr_noexcept { + mi_free_ex(p, page, NULL, true); +} + void mi_free(void* p) mi_attr_noexcept { mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); - mi_free_ex(p, NULL, page, true); + mi_free_ex(p, page, NULL, true); } -void mi_ufree(void* p, size_t* usable) mi_attr_noexcept { +void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree"); - mi_free_ex(p, usable, page, true); + mi_free_ex(p, page, pblock_size, true); } void mi_free_small(void* p) mi_attr_noexcept { @@ -245,27 +255,31 @@ void mi_free_small(void* p) mi_attr_noexcept { // just by aligning down the pointer instead of looking it up in the page map. #if MI_PAGE_META_ALIGNED_FREE_SMALL #if MI_GUARDED - #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as MI_GUARDED is defined" + #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL) ignored as MI_GUARDED is defined" mi_free(p); #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE - #warning "MI_PAGE_META_ALIGNED_FREE_SMALL ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size" + #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL) ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size" mi_free(p); #else mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); mi_assert(page == mi_validate_ptr_page(p,"mi_free_small")); mi_assert((void*)page == _mi_align_down_ptr(mi_page_start(page),MI_SMALL_PAGE_SIZE)); - mi_assert(page->block_size <= MI_SMALL_SIZE_MAX); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small` - mi_free_ex(p, NULL, page, true); + mi_assert(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX)); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small` + mi_free_ex(p, page, NULL, true); #endif #else - mi_free(p); + mi_free(p); #endif } +void _mi_free_subproc_safe_in_page(void* p, mi_page_t* page) mi_attr_noexcept { + mi_free_ex(p, page, NULL, false); +} + // Free a pointer that is potentially allocated in a different sub-process -void _mi_free_subproc_safe(void* p) { +void _mi_free_subproc_safe(void* p) mi_attr_noexcept { mi_page_t* const page = mi_validate_ptr_page(p,"_mi_free_subproc_safe"); - mi_free_ex(p, NULL, page, false); + mi_free_ex(p, page, NULL, false); } @@ -424,7 +438,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // ------------------------------------------------------ -// Usable size +// Usable size // ------------------------------------------------------ // Bytes available in a block @@ -439,8 +453,9 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p return aligned_size; } -static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_attr_noexcept { +size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept { if mi_unlikely(page==NULL) return 0; + mi_assert_internal(mi_validate_ptr_page(p,"_mi_page_usable_size") == page); if mi_likely(!mi_page_has_interior_pointers(page)) { const mi_block_t* block = mi_validate_block_from_ptr(page,p); return mi_page_usable_size_of(page, block, false /* is guarded */); @@ -453,7 +468,7 @@ static inline size_t _mi_usable_size(const void* p, const mi_page_t* page) mi_at mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size"); - return _mi_usable_size(p,page); + return _mi_page_usable_size(page,p); } @@ -464,11 +479,30 @@ mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { void mi_free_size(void* p, size_t size) mi_attr_noexcept { MI_UNUSED_RELEASE(size); #if MI_DEBUG - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); - const size_t available = _mi_usable_size(p,page); - mi_assert(p == NULL || size <= available || available == 0 /* invalid pointer */ ); + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); + if (page==NULL) return; + mi_assert(p!=NULL); + const size_t usable = _mi_page_usable_size(page,p); + if (size > usable) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); + mi_free(p); + return; + } + else if (size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); + mi_free(p); + return; + } #endif - mi_free(p); + #if MI_OPT_FREE_SMALL + if mi_likely(size <= MI_SMALL_SIZE_MAX) { + mi_free_small(p); + } + else + #endif + { + mi_free(p); + } } void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { @@ -483,18 +517,35 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { mi_free(p); } +// checked free +void mi_cfree(void* p) mi_attr_noexcept { + mi_page_t* const page = _mi_checked_ptr_page(p); + if mi_likely(page!=NULL) { + _mi_free_in_page(p,page); + } + // if (mi_is_in_heap_region(p)) { + // mi_free(p); + // } +} + // ------------------------------------------------------ // Check for double free in secure and debug mode // This is somewhat expensive so only enabled for secure mode 4 // ------------------------------------------------------ -#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) +#if MI_CHECK_DOUBLE_FREE // linear check if the free list contains a specific element -static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem) { - while (list != NULL) { +static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) { + const size_t max_count = page->capacity; // can never hold more blocks than the capacity + size_t count = 0; + while (list != NULL && count <= max_count) { // double-free can create cycles so we limit the number of iterations if (elem==list) return true; list = mi_block_next(page, list); + count++; + } + if mi_unlikely(count > max_count) { + _mi_error_message(EFAULT, "corrupted %s list (possibly due to a double free)\n", list_kind); } return false; } @@ -502,9 +553,9 @@ static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, cons static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) { // The decoded value is in the same page (or NULL). // Walk the free lists to verify positively if it is already freed - if (mi_list_contains(page, page->free, block) || - mi_list_contains(page, page->local_free, block) || - mi_list_contains(page, mi_page_thread_free(page), block)) + if (mi_list_contains(page, page->free, block, "free") || + mi_list_contains(page, page->local_free, block, "local free") || + mi_list_contains(page, mi_page_thread_free(page), block, "thread free")) { _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); return true; @@ -512,25 +563,28 @@ static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, con return false; } -#define mi_track_page(page,access) { size_t psize; void* pstart = _mi_page_start(_mi_page_segment(page),page,&psize); mi_track_mem_##access( pstart, psize); } +// Used for double free checking to avoid checking free lists too frequently +static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { + mi_block_t* n = mi_block_nextx(page,block,page->keys); + return (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? + (n==NULL || mi_is_in_same_page(block,n))); // quick check: in the same page or NULL? +} +// check if `block` was free'd before static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { - bool is_double_free = false; // A purged block is free but held off every free list and its memory is discarded, so // neither the decoded-`next` heuristic nor the free-list walk below can recognize it. if mi_unlikely(mi_page_block_is_purged(page, block)) { _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); return true; } - mi_block_t* n = mi_block_nextx(page, block, page->keys); // pretend it is freed, and get the decoded first field - if (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? - (n==NULL || mi_is_in_same_page(block, n))) // quick check: in same page or NULL? + if mi_unlikely(mi_block_could_be_double_free(page,block)) // quick check: next field is aligned in the same page or NULL? { // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free? // (continue in separate function to improve code generation) - is_double_free = mi_check_is_double_freex(page, block); + return mi_check_is_double_freex(page, block); } - return is_double_free; + else return false; } #else static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { @@ -657,9 +711,18 @@ static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { // only maintain stats for smaller objects if requested #if (MI_STAT>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { - MI_UNUSED(block); - mi_theap_t* const theap = _mi_theap_default(); - if (!mi_theap_is_initialized(theap)) return; // (for now) skip statistics if free'd after thread_done was called (usually a thread cleanup call by the OS) + MI_UNUSED(block); + mi_theap_t* theap = _mi_theap_default(); + mi_lock_t* lock = NULL; + mi_subproc_t* const subproc = mi_page_subproc(page); + mi_theap_t* const theap_meta = subproc->theap_meta; + if mi_unlikely(!mi_theap_is_initialized(theap) || // can happen if free'd after thread_done was called (usually a thread cleanup call by the OS) + // page->theap == subproc->theap_meta .. but we cannot read `theap` if we don't own the page + (theap_meta != NULL && mi_page_thread_id(page) == theap_meta->tld->thread_id)) { + theap = theap_meta; + lock = &subproc->theap_meta_lock; + mi_lock_acquire(lock); + } const size_t bsize = mi_page_usable_block_size(page); // #if (MI_STAT>1) @@ -676,6 +739,10 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { const size_t bpsize = mi_page_block_size(page); // match stat in page.c:mi_huge_page_alloc mi_theap_stat_decrease(theap, malloc_huge, bpsize); } + + if mi_unlikely(lock!=NULL) { + mi_lock_release(lock); + } } #else void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { diff --git a/src/heap.c b/src/heap.c index 5e6544c2b..3e7611b5e 100644 --- a/src/heap.c +++ b/src/heap.c @@ -31,7 +31,7 @@ void mi_heap_stats_merge_to_subproc(mi_heap_t* heap) { void mi_heap_stats_merge_to_main(mi_heap_t* heap) { if (heap==NULL) return; - _mi_stats_merge_into(&mi_heap_main()->stats, &heap->stats); + _mi_stats_merge_into(&mi_heap_get_heap_main(heap)->stats, &heap->stats); } bool _mi_heap_theap_set(mi_heap_t* heap, mi_theap_t* theap) { @@ -62,14 +62,6 @@ static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_he mi_heap_t* heap = (mi_heap_t*)const_heap; mi_assert_internal(heap!=NULL); - // if (_mi_is_process_heap_main(heap)) { - // // this can be called if the (main) thread is not yet initialized (as no allocation happened) - // // but `theap_main_init_get()` will call `mi_thread_init()` - // mi_theap_t* const theap = _mi_theap_main_safe(); - // mi_assert_internal(theap!=NULL && _mi_is_heap_main(_mi_theap_heap(theap))); - // return theap; - // } - // initialize thread first in case this is the main heap // (which may allocate the default theap already for the main heap) if (!_mi_thread_is_initialized()) { @@ -107,7 +99,7 @@ mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) return theap; } -static void mi_heap_initialize(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id) +void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* subproc, mi_arena_id_t exclusive_arena_id) { // init fields heap->theap = theap_slot; @@ -151,11 +143,14 @@ mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusi mi_assert_internal(subproc->heap_main == NULL); subproc->heap_main = heap; } - mi_heap_initialize(heap, theap_slot, subproc, exclusive_arena_id); + _mi_heap_init(heap, theap_slot, subproc, exclusive_arena_id); return heap; } mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t exclusive_arena_id) { + // `mi_heap_new` may be the very first mimalloc call in a process, in which case the + // main heap does not exist yet and `_mi_heap_new_for_subproc` would allocate from a NULL `subproc->heap_main`. + mi_thread_init(); return _mi_heap_new_for_subproc(_mi_subproc(), exclusive_arena_id, false); } @@ -167,63 +162,68 @@ mi_heap_t* mi_heap_new(void) { static void mi_heap_free_theaps(mi_heap_t* heap) { // This can run concurrently with a thread that terminates (see `init.c:mi_thread_theaps_done`), // and we need to ensure we free theaps atomically. - // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock - // potential concurrent thread termination which tries to remove the theap from our theaps list. - bool all_freed; - do { - all_freed = true; - mi_theap_t* theap = NULL; - mi_lock(&heap->theaps_lock) { - theap = heap->theaps; - while(theap != NULL) { - mi_theap_t* next = theap->hnext; - if (!_mi_theap_free(theap, false /* dont re-acquire the heap->theaps_lock */, true /* acquire the tld->theaps_lock though */ )) { - all_freed = false; - } - theap = next; - } - } - if (!all_freed) { - mi_heap_stat_counter_increase(heap,heaps_delete_wait,1); - _mi_prim_thread_yield(); - } - else { - mi_assert_internal(heap->theaps==NULL); + + // We first detach our theaps list from any thread local lists + _mi_heap_detach_theaps(heap); + + // Now we can safely free the theaps + mi_lock(&heap->theaps_lock) { // paranoia + mi_theap_t* theap = heap->theaps; + heap->theaps = NULL; + while(theap != NULL) { + mi_theap_t* next = theap->hnext; + theap->hnext = NULL; + theap->hprev = NULL; + mi_assert_internal(theap->tld==NULL); + // merge stats into the owning heap stats + _mi_stats_merge_into(&heap->stats, &theap->stats); + // and free + _mi_theap_decref(theap); // a cached entry can still point to the theap + theap = next; } - } - while(!all_freed); + } } // free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed) -static void mi_heap_free(mi_heap_t* heap) { - mi_assert_internal(heap!=NULL && !_mi_is_heap_main(heap)); +static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { + mi_assert_internal(heap!=NULL); // && !_mi_is_process_heap_main(heap)); // free all arena pages infos - mi_lock(&heap->arena_pages_lock) { - for (size_t i = 0; i < MI_MAX_ARENAS; i++) { - mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); - if (arena_pages!=NULL) { - mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL); - _mi_free_subproc_safe(arena_pages); + const bool is_main = _mi_is_heap_main(heap); + if (!is_main) { // pages for the main heap are pre-allocated in the arenas + mi_lock(&heap->arena_pages_lock) { + for (size_t i = 0; i < MI_MAX_ARENAS; i++) { + mi_arena_pages_t* arena_pages = mi_atomic_load_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i]); + if (arena_pages!=NULL) { + mi_atomic_store_ptr_relaxed(mi_arena_pages_t, &heap->arena_pages[i], NULL); + _mi_free_subproc_safe(arena_pages); + } } } } // remove the heap from the subproc - mi_heap_stats_merge_to_main(heap); + if (!is_main) { + mi_heap_stats_merge_to_main(heap); + } + else { + _mi_stats_merge_into(&heap->subproc->stats,&heap->stats); + } mi_atomic_decrement_relaxed(&heap->subproc->heap_count); mi_subproc_stat_decrease(heap->subproc, heaps, 1); - mi_lock(&heap->subproc->heaps_lock) { + mi_lock_maybe(&heap->subproc->heaps_lock, acquire_heaps_lock) { if (heap->next!=NULL) { heap->next->prev = heap->prev; } if (heap->prev!=NULL) { heap->prev->next = heap->next; } else { heap->subproc->heaps = heap->next; } } - _mi_thread_local_free(heap->theap); mi_lock_done(&heap->theaps_lock); mi_lock_done(&heap->os_abandoned_pages_lock); mi_lock_done(&heap->arena_pages_lock); - _mi_free_subproc_safe(heap); + if (!_mi_is_process_heap_main(heap)) { + _mi_thread_local_free(heap->theap); + _mi_free_subproc_safe(heap); + } } void mi_heap_delete(mi_heap_t* heap) { @@ -235,14 +235,20 @@ void mi_heap_delete(mi_heap_t* heap) { } mi_heap_free_theaps(heap); _mi_heap_move_pages(heap, heap_main); - mi_heap_free(heap); + mi_heap_free(heap,true /* acquire subproc->heaps_lock */); } -void _mi_heap_force_destroy(mi_heap_t* heap) { +void _mi_heap_force_destroy(mi_heap_t* heap, bool acquire_heaps_lock) { if (heap==NULL) return; mi_heap_free_theaps(heap); _mi_heap_destroy_pages(heap); - if (!_mi_is_heap_main(heap)) { mi_heap_free(heap); } // todo: release locks of the main heap? + // if (_mi_subproc_main()->heap_main == heap) { + // _mi_stats_merge_into(&heap->subproc->stats,&heap->stats); + // } + // else + { + mi_heap_free(heap, acquire_heaps_lock); // todo: release locks of the main heap? + } } void mi_heap_destroy(mi_heap_t* heap) { @@ -251,7 +257,7 @@ void mi_heap_destroy(mi_heap_t* heap) { _mi_warning_message("cannot destroy the main heap\n"); return; } - _mi_heap_force_destroy(heap); + _mi_heap_force_destroy(heap,true /* acquire subproc->heaps_lock */); } mi_heap_t* mi_heap_of(const void* p) { diff --git a/src/init.c b/src/init.c index e34f21b27..004d3cd7d 100644 --- a/src/init.c +++ b/src/init.c @@ -11,42 +11,39 @@ terms of the MIT license. A copy of the license can be found in the file #include // memcpy, memset #include // atexit - -#if MI_DEBUG > 0 -mi_decl_export volatile int mi_debug_stall_in_thread_theaps_done = 0; +#if !defined(_WIN32) && !defined(__wasi__) +#include // pthread_atfork (fork handlers, subproc.c) #endif -#define MI_MEMID_INIT(kind) {{{NULL,0}}, kind, true /* pinned */, true /* committed */, false /* zero */ } -#define MI_MEMID_STATIC MI_MEMID_INIT(MI_MEM_STATIC) +static void mi_tld_register(mi_tld_t* tld); // fork: tld registry for the scavenger (defined with mi_tld_free) // Empty page used to initialize the small free pages array -const mi_page_t _mi_page_empty = { +static const mi_page_t mi_page_empty = { MI_ATOMIC_VAR_INIT(0), // xthread_id NULL, // free 0, // used 0, // capacity - 0, // reserved capacity 0, // retire_expire false, // is_zero NULL, // local_free MI_ATOMIC_VAR_INIT(0), // xthread_free 0, // block_size - 0, // page_woffset - MI_ARENA_SLICE_SIZE, // page_committed - #if (MI_PADDING || MI_ENCODE_FREELIST) - { 0, 0 }, // keys - #endif + 0, // page_ma_offset + 0, // slice_pcommitted + 0, // reserved capacity NULL, // theap NULL, // heap NULL, NULL, // next, prev MI_MEMID_STATIC, // memid - { 0 }, // purged - 0, // unformed_purged_lo - 0, // unformed_purged_hi - MI_PAGE_SWEPT_NONE // swept_state + #if (MI_PADDING || MI_ENCODE_FREELIST) + { 0, 0 }, // keys + #endif + { 0 }, // purged (fork) + 0, 0, // unformed_purged_lo/hi (fork) + MI_PAGE_SWEPT_NONE // swept_state (fork) }; -#define MI_PAGE_EMPTY() ((mi_page_t*)&_mi_page_empty) +#define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty) #if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) #define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } @@ -79,15 +76,16 @@ const mi_page_t _mi_page_empty = { #define MI_STAT_COUNT(stat) {0,0,0}, #define MI_STAT_COUNTER(stat) {0}, -#define MI_STATS_NULL \ - MI_STAT_FIELDS() \ - \ - { MI_INIT4(MI_STAT_COUNT_NULL) }, \ - { { 0 }, { 0 }, { 0 }, { 0 } }, \ - \ - { MI_INIT74(MI_STAT_COUNT_NULL) }, \ - { MI_INIT74(MI_STAT_COUNT_NULL) }, \ - { MI_INIT5(MI_STAT_COUNT_NULL) } +#define MI_STATS_FIELDS_NULL \ + MI_STAT_FIELDS() /* regular stat fields */ \ + { MI_INIT4(MI_STAT_COUNT_NULL) }, /* stat reserved */ \ + { { 0 }, { 0 }, { 0 }, { 0 } }, /* stat counter reserved */ \ + { MI_INIT74(MI_STAT_COUNT_NULL) }, /* malloc_bins */ \ + { MI_INIT74(MI_STAT_COUNT_NULL) }, /* page bins */ \ + { MI_INIT5(MI_STAT_COUNT_NULL) } /* chunk bins */ + +#define MI_STATS_NULL \ + { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_FIELDS_NULL } // -------------------------------------------------------- // Statically allocate an empty theap as the initial @@ -98,34 +96,29 @@ const mi_page_t _mi_page_empty = { // may lead to allocation itself on some platforms) // -------------------------------------------------------- -static mi_decl_cache_align mi_subproc_t subproc_main -#if __cplusplus - = { }; // empty initializer to prevent running the constructor (with msvc) -#else - = { 0 }; // C zero initialize -#endif - -static mi_subproc_t* subprocs = &subproc_main; -static mi_lock_t subprocs_lock; - -static mi_decl_cache_align mi_tld_t tld_empty = { - 0, // thread_id +static mi_decl_cache_align mi_tld_t mi_tld_detached = { + MI_THREADID_DETACHED, // thread_id 0, // thread_seq 0, // default numa node - &subproc_main, // subproc + NULL, // subproc NULL, // theaps list MI_LOCK_INITIALIZER, // theaps lock false, // recurse false, // is_in_threadpool - MI_MEMID_STATIC // memid + MI_MEMID_STATIC, // memid + MI_ATOMIC_VAR_INIT(0), // park_state = running (fork) + MI_ATOMIC_VAR_INIT(0), // park_reclaim + NULL, // park_theap0 + MI_ATOMIC_VAR_INIT(0), // park_swept + NULL, // subproc_next + 0, 0 // holes_sweep_seq / _last }; -mi_decl_cache_align const mi_theap_t _mi_theap_empty = { - &tld_empty, // tld +mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { + &mi_tld_detached, // tld MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount - MI_ATOMIC_VAR_INIT(0), // freed 0, // heartbeat 0, // cookie { {0}, {0}, 0, true }, // random @@ -138,158 +131,44 @@ mi_decl_cache_align const mi_theap_t _mi_theap_empty = { 0, // full page retain false, // allow reclaim true, // allow abandon - false, 0, // prof_force_slow, prof_countdown + true, // is_detached + false, 0, // prof_force_slow, prof_countdown (fork) #if MI_GUARDED - 1, 0, 0, 1, // min>max and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) + 0, 0, 0, 1, // rate is 0 and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) #endif MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY, MI_MEMID_STATIC, - { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats + MI_STATS_NULL, // stats }; -mi_decl_cache_align const mi_theap_t _mi_theap_empty_wrong = { - &tld_empty, // tld - MI_ATOMIC_VAR_INIT(NULL), // heap - MI_ATOMIC_VAR_INIT(NULL), // subproc - MI_ATOMIC_VAR_INIT(1), // refcount - MI_ATOMIC_VAR_INIT(0), // freed - 0, // heartbeat - 1, // cookie (see issue #1343) - { {0}, {0}, 0, true }, // random - 0, // page count - MI_BIN_FULL, 0, // page retired min/max - 0, // pages_full_size - 0, 0, // generic count - NULL, NULL, // tnext, tprev - NULL, NULL, // hnext, hprev - 0, // full page retain - false, // allow reclaim - true, // allow abandon - false, 0, // prof_force_slow, prof_countdown - #if MI_GUARDED - 1, 0, 0, 1, // min>max and sample count is 1 so we never write to it (see `internal.h:mi_theap_malloc_use_guarded`) - #endif - MI_SMALL_PAGES_EMPTY, - MI_PAGE_QUEUES_EMPTY, - MI_MEMID_STATIC, - { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats -}; - -// Heap for the main thread - -#define MI_THREADID_INVALID ((mi_threadid_t)(~0)) - -extern mi_decl_hidden mi_decl_cache_align mi_theap_t mi_theap_main; // theap of the main thread (belonging to the `mi_process_heap_main`) -extern mi_decl_hidden mi_decl_cache_align mi_heap_t mi_process_heap_main; // main heap of the main subproc - -static mi_decl_cache_align mi_tld_t mi_process_tld_main = { - 0, // thread_id - 0, // thread_seq - 0, // numa node - &subproc_main, // subproc - &mi_theap_main, // theaps list - MI_LOCK_INITIALIZER, // theaps lock - false, // recurse - false, // is_in_threadpool - MI_MEMID_STATIC // memid -}; - -mi_decl_cache_align mi_theap_t mi_theap_main = { - &mi_process_tld_main, // thread local data - MI_ATOMIC_VAR_INIT(&mi_process_heap_main), // main heap - MI_ATOMIC_VAR_INIT(&subproc_main), // main subproc - MI_ATOMIC_VAR_INIT(1), // refcount - MI_ATOMIC_VAR_INIT(0), // freed - 0, // heartbeat - 0, // initial cookie - { {0x846ca68b}, {0}, 0, true }, // random - 0, // page count - MI_BIN_FULL, 0, // page retired min/max - 0, // pages_full_size - 0, 0, // generic count - NULL, NULL, // tnext, tprev - NULL, NULL, // hnext, hprev - 2, // full page retain - true, // allow page reclaim - true, // allow page abandon - false, 0, // prof_force_slow, prof_countdown - #if MI_GUARDED - 0, 0, 0, 0, - #endif - MI_SMALL_PAGES_EMPTY, - MI_PAGE_QUEUES_EMPTY, - MI_MEMID_STATIC, - { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }, // stats -}; - -mi_decl_cache_align mi_heap_t mi_process_heap_main -#if __cplusplus - = { }; // empty initializer to prevent running the constructor (with msvc) -#else - = { 0 }; // C zero initialize -#endif +#undef MI_STAT_COUNT +#undef MI_STAT_COUNTER -mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { - const mi_threadid_t tid = _mi_prim_thread_id(); - mi_assert_internal( (tid & 0x03) == 0 ); // mimalloc reserves the bottom 2 bits - return tid; -} -mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper = NULL; +// pre-allocate the process heap, and meta-data theap +static mi_decl_cache_align mi_heap_t mi_process_heap_main = mi_init_struct_zero; +static mi_decl_cache_align mi_theap_t mi_process_theap_meta = mi_init_struct_zero; -#if MI_TLS_MODEL_LOCAL -// the thread-local main theap for allocation -mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty; -// the last used non-main theap -mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty; -#endif +// pre-allocate the initial tld and theap for the main thread (this is not strictly needed but nice for stats) +static mi_decl_cache_align mi_tld_t mi_process_tld_main = mi_init_struct_zero; +static mi_decl_cache_align mi_theap_t mi_process_theap_main = mi_init_struct_zero; +mi_decl_hidden mi_decl_cache_align mi_theap_t _mi_theap_empty_wrong = mi_init_struct_zero; // used for error paths mi_decl_hidden bool _mi_process_is_initialized = false; // set to `true` in `mi_process_init`. -mi_stats_t _mi_stats_main = { sizeof(mi_stats_t), MI_STAT_VERSION, MI_STATS_NULL }; -#undef MI_STAT_COUNT -#undef MI_STAT_COUNTER - - -#if MI_GUARDED -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - theap->guarded_sample_rate = sample_rate; - theap->guarded_sample_count = sample_rate; // count down samples - if (theap->guarded_sample_rate > 1) { - if (seed == 0) { - seed = _mi_theap_random_next(theap); - } - theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` - } +mi_page_t* _mi_page_empty_get(void) { + return (mi_page_t*)&mi_page_empty; } -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - theap->guarded_size_min = min; - theap->guarded_size_max = (min > max ? min : max); -} - -void _mi_theap_guarded_init(mi_theap_t* theap) { - mi_theap_guarded_set_sample_rate(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), - (size_t)mi_option_get(mi_option_guarded_sample_seed)); - mi_theap_guarded_set_size_bound(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), - (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); -} -#else -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed); +mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) { + return (mi_theap_t*)&_mi_theap_empty; } -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max); -} -void _mi_theap_guarded_init(mi_theap_t* theap) { - MI_UNUSED(theap); +bool _mi_is_empty_theap(const mi_theap_t* theap) { + return (theap == &_mi_theap_empty); } -#endif /* ----------------------------------------------------------- Initialization @@ -297,177 +176,38 @@ void _mi_theap_guarded_init(mi_theap_t* theap) { can cause allocation and induce recursion during initialization. ----------------------------------------------------------- */ - -// Initialize main subproc -static void mi_subproc_main_init(void) { - if (subproc_main.memid.memkind != MI_MEM_STATIC) { - subproc_main.memid = _mi_memid_create(MI_MEM_STATIC); - subproc_main.heaps = &mi_process_heap_main; - subproc_main.heap_total_count = 1; - subproc_main.heap_count = 1; - mi_atomic_store_ptr_release(mi_heap_t, &subproc_main.heap_main, &mi_process_heap_main); - __mi_stat_increase_mt(&subproc_main.stats.heaps, 1); - mi_stats_header_init(&subproc_main.stats); - mi_lock_init(&subproc_main.arena_reserve_lock); - mi_lock_init(&subproc_main.heaps_lock); - mi_lock_init(&subproc_main.tlds_lock); - mi_lock_init(&subprocs_lock); - mi_lock_init(&tld_empty.theaps_lock); - } -} - -// Initialize main tld -static void mi_tld_register(mi_tld_t* tld); // defined below, with the rest of the registry - -static void mi_tld_main_init(void) { - if (mi_process_tld_main.thread_id == 0) { - mi_process_tld_main.thread_id = _mi_prim_thread_id(); - mi_lock_init(&mi_process_tld_main.theaps_lock); - } - // The main thread's tld is static and never goes through `mi_tld_alloc`, so register it here - // instead -- otherwise it can never hand its theaps to the scavenger. `mi_subproc_main_init` - // runs just before this and initializes the lock the registry takes. - mi_tld_register(&mi_process_tld_main); -} - -void _mi_theap_options_init(mi_theap_t* theap) { - theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0); - theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); - theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32); - _mi_prof_theap_init(theap); -} - -// Initialization of the (statically allocated) main theap, and the main tld and subproc. -static void mi_theap_main_init(void) { - if mi_unlikely(mi_theap_main.memid.memkind != MI_MEM_STATIC) { - // theap - mi_theap_main.memid = _mi_memid_create(MI_MEM_STATIC); - #if defined(__APPLE__) || defined(_WIN32) && !defined(MI_SHARED_LIB) - _mi_random_init_weak(&mi_theap_main.random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185) - #else - _mi_random_init(&mi_theap_main.random); - #endif - mi_theap_main.cookie = _mi_theap_random_next(&mi_theap_main); - _mi_theap_options_init(&mi_theap_main); - _mi_theap_guarded_init(&mi_theap_main); - } -} +static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc); // Initialize main heap -static void mi_heap_main_init(void) { - if mi_unlikely(mi_process_heap_main.subproc == NULL) { - mi_process_heap_main.subproc = &subproc_main; - mi_process_heap_main.theaps = &mi_theap_main; - mi_process_heap_main.theap = mi_thread_local_key_fast; - - mi_theap_main_init(); - mi_subproc_main_init(); - mi_tld_main_init(); - // mi_heap_theap_set(&mi_process_heap_main,&mi_theap_main); // set in `mi_thread_init(_theap_default)` - - mi_lock_init(&mi_process_heap_main.theaps_lock); - mi_lock_init(&mi_process_heap_main.os_abandoned_pages_lock); - mi_lock_init(&mi_process_heap_main.arena_pages_lock); - } -} +static void mi_heap_main_init_once(void) { + mi_memid_t memid_static = _mi_memid_create(MI_MEM_STATIC); + _mi_memcpy(&_mi_theap_empty_wrong,&_mi_theap_empty,sizeof(_mi_theap_empty_wrong)); + // initialize the main subprocess + mi_subproc_t* subproc_main = _mi_subproc_main_init(); -/* ----------------------------------------------------------- - Thread local data ------------------------------------------------------------ */ + // detached tld for mi_theap_empty (and theap_meta) + mi_tld_detached.memid = memid_static; + mi_tld_init(&mi_tld_detached, 0, subproc_main); -// Register a tld with its subproc so the scavenger can find it when it parks -// (`mi_on_thread_idle_start`). Idempotent: the main thread's static tld can come through -// `mi_tld_alloc` more than once if the thread is re-initialized after `_mi_thread_done`. -static void mi_tld_register(mi_tld_t* tld) { - if (tld == NULL) return; - mi_subproc_t* const subproc = tld->subproc; - mi_lock(&subproc->tlds_lock) { - for (mi_tld_t* t = subproc->tlds; t != NULL; t = t->subproc_next) { - if (t == tld) goto done; // already registered - } - tld->subproc_next = subproc->tlds; - subproc->tlds = tld; - done: ; - } -} - -// Unlink before the tld memory goes away. The tld cannot be mid-sweep here: only a PARKED tld is -// ever claimed, and a thread running its own teardown is RUNNING by definition -- assert it. -static void mi_tld_unregister(mi_tld_t* tld) { - if (tld == NULL) return; - mi_assert_internal(mi_atomic_load_acquire(&tld->park_state) != MI_PARK_SWEEPING); - mi_subproc_t* const subproc = tld->subproc; - mi_lock(&subproc->tlds_lock) { - mi_tld_t** prev = &subproc->tlds; - while (*prev != NULL) { - if (*prev == tld) { *prev = tld->subproc_next; break; } - prev = &(*prev)->subproc_next; - } - tld->subproc_next = NULL; - } -} - -// Allocate fresh tld -static mi_tld_t* mi_tld_alloc(mi_subproc_t* subproc) { - // if (_mi_is_main_thread()) { - // mi_atomic_increment_relaxed(&tld_main.subproc->thread_count); - // return &tld_main; - // } - // else - { - // allocate tld meta-data - // note: we need to be careful to not access the tld from `_mi_meta_zalloc` - // (and in turn from `_mi_arena_alloc_aligned` and `_mi_os_alloc_aligned`). - mi_memid_t memid; - mi_tld_t* tld = (mi_tld_t*)_mi_meta_zalloc(subproc, sizeof(mi_tld_t), &memid); - if (tld==NULL) { - _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n"); - return NULL; - } - tld->memid = memid; - tld->theaps = NULL; - mi_lock_init(&tld->theaps_lock); - tld->subproc = subproc; - tld->numa_node = _mi_os_numa_node(); - tld->thread_id = _mi_prim_thread_id(); - tld->thread_seq = mi_atomic_increment_relaxed(&tld->subproc->thread_total_count); - tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool(); - mi_atomic_increment_relaxed(&tld->subproc->thread_count); - mi_tld_register(tld); - return tld; - } -} - -#define MI_TLD_INVALID ((mi_tld_t*)1) - -mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) { - if (tld==NULL || tld==MI_TLD_INVALID) return; - mi_tld_unregister(tld); - mi_atomic_decrement_relaxed(&tld->subproc->thread_count); - tld->thread_id = MI_THREADID_INVALID; // note: not 0 as that would re-initialize tld_main - // we also need to set an invalid tid for tld_main as sometimes the same thread-id - // is reused by the OS after a thread has terminated. (see issue #1287) - mi_lock_done(&tld->theaps_lock); - _mi_meta_free(tld->subproc, tld, sizeof(mi_tld_t), tld->memid); // note: safe for static tld_main -} + // main process heap + mi_process_heap_main.memid = memid_static; + mi_atomic_store_ptr_release(mi_heap_t,&subproc_main->heap_main,&mi_process_heap_main); + _mi_heap_init(&mi_process_heap_main,mi_thread_local_key_fast,subproc_main,0); + // detached theap for allocating meta-data (we can allocate on this without having an initialized thread) + mi_process_theap_meta.memid = memid_static; + _mi_theap_init(&mi_process_theap_meta,&mi_process_heap_main,&mi_tld_detached); + mi_process_theap_meta.allow_page_abandon = false; // for security, don't share with other threads + mi_process_theap_meta.page_full_retain = 2; + subproc_main->theap_meta = &mi_process_theap_meta; -mi_subproc_t* _mi_subproc_main(void) { - return &subproc_main; + // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)` } -mi_subproc_t* _mi_subproc(void) { - // should work without doing initialization (as it may be called from `_mi_tld -> mi_tld_alloc ... -> os_alloc -> _mi_subproc()` - // todo: this will still fail on OS systems where the first access to a thread-local causes allocation. - // on such systems we can check for this with the _mi_prim_get_default_theap as those are protected (by being - // stored in a TLS slot for example) - mi_theap_t* theap = _mi_theap_default(); - if (theap == NULL || theap->tld == NULL) { // see issue #1289 - return _mi_subproc_main(); - } - else { - return theap->tld->subproc; // avoid using thread local storage (`thread_tld`) +static void mi_heap_main_init(void) { + mi_atomic_do_once { + mi_heap_main_init_once(); } } @@ -476,7 +216,7 @@ mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) { if mi_likely(heap!=NULL) { return heap; } - else if (subproc==_mi_subproc_main()) { + else if (_mi_subproc_is_main(subproc)) { mi_heap_main_init(); mi_assert_internal(mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main) != NULL); return mi_atomic_load_ptr_acquire(mi_heap_t,&subproc->heap_main); @@ -487,180 +227,97 @@ mi_heap_t* _mi_subproc_heap_main(mi_subproc_t* subproc) { } } -mi_heap_t* mi_heap_main(void) { - return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init -} - -bool _mi_is_process_heap_main(const mi_heap_t* heap) { - return (heap == NULL || heap == &mi_process_heap_main); -} - -bool _mi_is_theap_main(const mi_theap_t* theap) { - return (mi_theap_is_initialized(theap) && _mi_is_heap_main(_mi_theap_heap(theap))); -} - - - /* ----------------------------------------------------------- - Sub process + Thread local data ----------------------------------------------------------- */ - -mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) { - return (mi_subproc_t*)(subproc_id._mi_subproc_id); -} - -mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) { - mi_subproc_id_t id = { subproc }; - return id; -} - -mi_subproc_id_t mi_subproc_main(void) { - return _mi_subproc_to_id(_mi_subproc_main()); -} - -mi_subproc_id_t mi_subproc_current(void) { - return _mi_subproc_to_id(_mi_subproc()); -} - -mi_subproc_id_t mi_subproc_new(void) { - static _Atomic(size_t) subproc_total_count; - mi_subproc_t* const parent = _mi_subproc(); - mi_memid_t memid; - mi_subproc_t* subproc = (mi_subproc_t*)_mi_meta_zalloc(parent, sizeof(mi_subproc_t),&memid); - if (subproc == NULL) { return _mi_subproc_to_id(NULL); } - - // init subproc - subproc->memid = memid; - subproc->parent = parent; - subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count) + 1; - mi_stats_header_init(&subproc->stats); - mi_lock_init(&subproc->arena_reserve_lock); - mi_lock_init(&subproc->heaps_lock); - mi_lock_init(&subproc->tlds_lock); - mi_lock(&subprocs_lock) { - // push on subproc list - subproc->next = subprocs; - if (subprocs!=NULL) { subprocs->prev = subproc; } - subprocs = subproc; +static mi_tld_t* mi_tld_init(mi_tld_t* tld, size_t tseq, mi_subproc_t* subproc) { + tld->subproc = subproc; + tld->theaps = NULL; + mi_lock_init(&tld->theaps_lock); + if (tld->thread_id == MI_THREADID_DETACHED) { + tld->numa_node = -1; } - - // init main heap - mi_heap_t* heap_main = _mi_heap_new_for_subproc(subproc,0,true); - if (heap_main==NULL) { - mi_subproc_destroy(_mi_subproc_to_id(subproc)); - return _mi_subproc_to_id(NULL); + else { + tld->numa_node = _mi_os_numa_node(); + tld->thread_id = _mi_prim_thread_id(); + tld->is_in_threadpool = _mi_prim_thread_is_in_threadpool(); + tld->thread_seq = tseq; + mi_atomic_increment_relaxed(&tld->subproc->thread_count); + mi_tld_register(tld); } - mi_assert_internal(subproc->heap_main == heap_main); - - return _mi_subproc_to_id(subproc); + return tld; } -// destroy all subproc resources including arena's, heap's etc. -static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock) -{ - if (subproc==NULL) return; - - // remove from the subproc list - mi_lock_maybe(&subprocs_lock, acquire_subprocs_lock) { - if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; } - if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; } - else { mi_assert_internal(subprocs==subproc); subprocs = subproc->next; } - } - - // destroy all subproc heaps - mi_lock(&subproc->heaps_lock) { - mi_heap_t* heap = subproc->heaps; - while (heap != NULL) { - mi_heap_t* next = heap->next; - if (heap!=subproc->heap_main) { mi_heap_destroy(heap); } - heap = next; - } - mi_assert_internal(subproc->heap_main==NULL || subproc->heaps == subproc->heap_main); - if (subproc->heap_main!=NULL) { - _mi_heap_force_destroy(subproc->heap_main); // no warning if destroying the main heap - } - } - - // remove associated arenas - _mi_arenas_unsafe_destroy_all(subproc); - - // merge stats back into the main subproc? - if (subproc!=&subproc_main) { - _mi_stats_merge_into(&subproc_main.stats, &subproc->stats); - } +// Allocate fresh tld +static mi_tld_t* mi_tld_create(mi_subproc_t* subproc) { + mi_assert_internal(subproc->theap_meta != NULL); // should be initialized on the main thread before other threads allocate + const size_t tseq = mi_atomic_increment_relaxed(&subproc->thread_total_count); - // safe to release - // todo: should we refcount subprocesses? - mi_lock_done(&subproc->arena_reserve_lock); - mi_lock_done(&subproc->heaps_lock); - if (subproc!=&subproc_main) { - _mi_meta_free( subproc->parent, subproc, sizeof(mi_subproc_t), subproc->memid); + mi_memid_t memid; + mi_tld_t* tld; + if (_mi_subproc_is_main(subproc) && tseq==0 /* first tld */) { + tld = &mi_process_tld_main; + memid = _mi_memid_create_static(tld,sizeof(*tld)); } else { - // for the main subproc, also release the global page map - _mi_page_map_unsafe_destroy(); + tld = (mi_tld_t*)_mi_meta_zalloc(subproc, sizeof(mi_tld_t), &memid); } + if (tld==NULL) { + _mi_error_message(ENOMEM, "unable to allocate memory for thread local data\n"); + return NULL; + } + tld->memid = memid; + return mi_tld_init(tld,tseq,subproc); } -void mi_subproc_destroy(mi_subproc_id_t subproc_id) { - mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - if (subproc==NULL || subproc==&subproc_main) return; - mi_subproc_unsafe_destroy(subproc, true /* take lock */); -} - -static void mi_subprocs_unsafe_destroy_all(void) { - mi_lock(&subprocs_lock) { - mi_subproc_t* subproc = subprocs; - while (subproc!=NULL) { - mi_subproc_t* next = subproc->next; - if (subproc!=&subproc_main) { - mi_subproc_unsafe_destroy(subproc, false /* take subprocs lock */); - } - subproc = next; +// fork: register a tld with its subproc so the scavenger can find it when it parks (`mi_on_thread_idle_start`). +// Idempotent: a tld can come through allocation more than once if the thread is re-initialized after `_mi_thread_done`. +static void mi_tld_register(mi_tld_t* tld) { + if (tld == NULL) return; + mi_subproc_t* const subproc = tld->subproc; + mi_lock(&subproc->tlds_lock) { + for (mi_tld_t* t = subproc->tlds; t != NULL; t = t->subproc_next) { + if (t == tld) goto done; // already registered } + tld->subproc_next = subproc->tlds; + subproc->tlds = tld; + done: ; } - mi_subproc_unsafe_destroy(&subproc_main, true /* take subprocs lock */); } -static mi_theap_t* mi_thread_init_ex(mi_heap_t* heap_main) mi_attr_noexcept; - -void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) { - mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - mi_assert_internal(subproc!=NULL); - if (subproc==NULL) return; - mi_assert_internal(subproc->heap_main!=NULL); - if (subproc->heap_main==NULL) return; - mi_theap_t* theap = _mi_theap_default(); - if (mi_theap_is_initialized(theap)) { - if (theap->tld!=NULL && theap->tld->subproc != subproc) { - _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (at %p)\n", theap->tld->subproc); +// fork: unlink before the tld memory goes away. The tld cannot be mid-sweep here: only a PARKED tld is ever claimed, +// and a thread running its own teardown is RUNNING by definition. +static void mi_tld_unregister(mi_tld_t* tld) { + if (tld == NULL) return; + mi_assert_internal(mi_atomic_load_acquire(&tld->park_state) != MI_PARK_SWEEPING); + mi_subproc_t* const subproc = tld->subproc; + mi_lock(&subproc->tlds_lock) { + mi_tld_t** prev = &subproc->tlds; + while (*prev != NULL) { + if (*prev == tld) { *prev = tld->subproc_next; break; } + prev = &(*prev)->subproc_next; } - return; + tld->subproc_next = NULL; } - - // initialize this thread tld & theap - mi_thread_init_ex(subproc->heap_main); } - -bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) { - mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - if (subproc==NULL) return false; - bool ok = true; - mi_lock(&subproc->heaps_lock) { - for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) { - ok = (*visitor)(heap, arg); - } - } - return ok; +mi_decl_noinline static void mi_tld_free(mi_tld_t* tld) { + mi_tld_unregister(tld); + if (tld==NULL) return; + mi_atomic_decrement_relaxed(&tld->subproc->thread_count); + tld->thread_id = (mi_threadid_t)(~0); // it is best to set an invalid tid for tld_main as sometimes the same thread-id + // is reused by the OS after a thread has terminated. (see issue #1287) + mi_lock_done(&tld->theaps_lock); + _mi_meta_free(tld->subproc, tld, tld->memid); // note: safe for static tld } /* ----------------------------------------------------------- - Allocate theap data + Thread Init ----------------------------------------------------------- */ +#if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD) static mi_theap_t* mi_heap_check_for_existing_theap(mi_heap_t* heap) { const mi_threadid_t tid = _mi_thread_id(); mi_theap_t* thread_theap = NULL; @@ -674,51 +331,85 @@ static mi_theap_t* mi_heap_check_for_existing_theap(mi_heap_t* heap) { } return thread_theap; } +#endif + +// Initialize thread +mi_theap_t* _mi_thread_init_with_heap(mi_heap_t* heap_main) +{ + // ensure our process has started already + mi_process_init(); -// Initialize the thread local default theap, called from `mi_thread_init` -static mi_theap_t* _mi_thread_init_theap_default(mi_heap_t* heap_main) { + // if the theap_default is already set we have already initialized mi_theap_t* theap = _mi_theap_default(); if (mi_theap_is_initialized(theap)) return theap; - if (_mi_is_main_thread() && heap_main==NULL) { - heap_main = &mi_process_heap_main; - theap = &mi_theap_main; - mi_heap_main_init(); + + // initialize the default theap + // note: we cannot access thread-locals yet as that can cause (recursive) allocation + // (on macOS <= 14 for example where the loader allocates thread-local data on demand). + if (heap_main==NULL) { + heap_main = mi_heap_main(); + mi_assert_internal(heap_main == &mi_process_heap_main); } - else { - // allocates tld data - // note: we cannot access thread-locals yet as that can cause (recursive) allocation - // (on macOS <= 14 for example where the loader allocates thread-local data on demand). - if (heap_main==NULL) { - heap_main = mi_heap_main(); - mi_assert_internal(heap_main == &mi_process_heap_main); + mi_assert_internal(heap_main!=NULL); + + #if MI_DEBUG || defined(MI_TLS_RECURSE_GUARD) + theap = mi_heap_check_for_existing_theap(heap_main); // recursion check + #if !defined(MI_TLS_RECURSE_GUARD) + mi_assert_internal(theap==NULL); + #endif + #else + theap = NULL; + #endif + + if (theap==NULL) { + // allocated the tld + mi_tld_t* tld = mi_tld_create(heap_main->subproc); + if (tld==NULL) return NULL; // out-of-memory on tld allocation + // allocate and initialize the theap for the main heap + if (tld==&mi_process_tld_main) { + theap = &mi_process_theap_main; // initial theap is pre-allocated + theap->memid = _mi_memid_create_static(theap,sizeof(*theap)); } - mi_assert_internal(heap_main!=NULL); - theap = mi_heap_check_for_existing_theap(heap_main); - if (theap==NULL) - { - // allocated the tld - mi_tld_t* tld = mi_tld_alloc(heap_main->subproc); - if (tld==NULL) return NULL; // out-of-memory on tld allocation - - // allocate and initialize the theap for the main heap - theap = _mi_theap_create( heap_main, tld); - if (theap==NULL) return NULL; // out-of-memory on theap allocation + else { + theap = _mi_theap_alloc(heap_main,tld); // otherwise meta allocate + if (theap==NULL) { mi_tld_free(tld); return NULL; } // out-of-memory on theap allocation } + _mi_theap_init(theap,heap_main,tld); } + // now initialize the thread _mi_theap_default_set(theap); - // and only then set the heap_theap field as that accesses thread locals _mi_heap_theap_set(heap_main, theap); // todo: can fail! mi_assert_internal(mi_theap_is_initialized(theap)); mi_theap_t* const heap_theap = (heap_main==NULL ? NULL : (mi_theap_t*)_mi_thread_local_get(heap_main->theap)); mi_assert_internal(heap_main==NULL || heap_theap == theap); MI_UNUSED_RELEASE(heap_theap); + + mi_subproc_stat_increase(_mi_theap_subproc(theap), threads, 1); // or theap stats and wait for merge? + // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); return theap; } +mi_theap_t* _mi_thread_init(void) { + return _mi_thread_init_with_heap(NULL); +} + +void mi_decl_noinline mi_thread_init(void) mi_attr_noexcept { + _mi_thread_init(); +} + + + +/* ----------------------------------------------------------- + Theaps done +----------------------------------------------------------- */ // Free the thread local theaps +#if MI_DEBUG > 0 +mi_decl_export volatile int mi_debug_stall_in_thread_theaps_done = 0; // fork test hook (test-theap-sentinel): stall teardown while holding the tld lock +#endif + static void mi_thread_theaps_done(mi_tld_t* tld) { // abandon the pages of all theaps in this thread @@ -735,9 +426,6 @@ static void mi_thread_theaps_done(mi_tld_t* tld) // never destroy theaps; if a dll is linked statically with mimalloc, // there may still be delete/free calls after the mi_fls_done is called. Issue #207 _mi_theap_collect_abandon(theap); - // A concurrent `mi_heap_free_theaps` may NULL theap->heap (so collect no-ops) - // and then CAS it back if its inner try-acquire fails; in that window the - // page_count invariant doesn't hold here. The second loop re-collects. theap = next; } } @@ -748,39 +436,26 @@ static void mi_thread_theaps_done(mi_tld_t* tld) _mi_theap_default_set((mi_theap_t*)&_mi_theap_empty); _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty); - // free the theaps of this thread. - // This can run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically. - // We do this in a loop where we release the theaps_lock at every potential re-iteration to unblock - // potential concurrent `mi_heap_free_theaps` which tries to remove the theap from our theaps list. - bool all_freed; - do { - all_freed = true; - mi_lock(&tld->theaps_lock) { - mi_theap_t* theap = tld->theaps; - while (theap != NULL) { - mi_theap_t* next = theap->tnext; - // A concurrent `mi_heap_free_theaps` may have transiently NULL'd - // theap->heap (and CAS'd it back on inner-lock failure or page_count>0), - // causing our first-loop collect to no-op. Re-collect; idempotent and - // cheap if already done. `_mi_theap_free` below also guards against - // freeing with pages from the non-owning side. - if (theap->page_count != 0) { - _mi_theap_collect_abandon(theap); - } - if (!_mi_theap_free(theap, true /* acquire heap->theaps_lock */, false /* dont re-acquire the tld->theaps_lock*/ )) { - all_freed = false; - } - theap = next; - } - } - if (!all_freed) { - mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1); - _mi_prim_thread_yield(); - } - else { - mi_assert_internal(tld->theaps==NULL); + // We might run concurrently with a `mi_heap_free_theaps` and we need to ensure we free theaps atomically. + // we first detach our theaps list from any heaps + _mi_tld_detach_theaps(tld); + + // no heaps point to our theaps anymore, free them + mi_lock(&tld->theaps_lock) { // paranoia + mi_theap_t* theap = tld->theaps; + tld->theaps = NULL; + while (theap != NULL) { + mi_theap_t* next = theap->tnext; + mi_assert_internal(theap->page_count==0); + mi_assert_internal(_mi_theap_heap_peek(theap)==NULL); + theap->tld = NULL; + theap->tnext = NULL; + theap->tprev = NULL; + mi_assert_internal(mi_atomic_load_relaxed(&theap->refcount) == 1); // as the cached entry is set to empty + _mi_theap_decref(theap); + theap = next; } - } while (!all_freed); + } mi_assert(_mi_theap_default()==(mi_theap_t*)&_mi_theap_empty); // careful to not re-initialize the default theap during theap_delete mi_assert(!mi_theap_is_initialized(_mi_theap_default())); @@ -803,46 +478,13 @@ static void mi_thread_theaps_done(mi_tld_t* tld) // to set up the thread local keys. // -------------------------------------------------------- -// Set up handlers so `mi_thread_done` is called automatically +// Set up hooks so `mi_thread_done` is called automatically static void mi_process_setup_auto_thread_done(void) { mi_atomic_do_once { _mi_prim_thread_init_auto_done(); - _mi_theap_default_set(&mi_theap_main); } } - -bool _mi_is_main_thread(void) { - return (mi_process_tld_main.thread_id==0 || mi_process_tld_main.thread_id == _mi_thread_id()); -} - -// Initialize thread -static mi_theap_t* mi_thread_init_ex(mi_heap_t* heap_main) mi_attr_noexcept -{ - // ensure our process has started already - mi_process_init(); - - // if the theap_default is already set we have already initialized - mi_theap_t* theap = _mi_theap_default(); - if (mi_theap_is_initialized(theap)) return theap; - - // otherwise initialize the default theap - theap = _mi_thread_init_theap_default(heap_main); - if (theap == NULL) return NULL; // out-of-memory on tld/theap allocation - - mi_subproc_stat_increase(_mi_theap_subproc(theap), threads, 1); // or theap stats and wait for merge? - // _mi_verbose_message("thread init: 0x%zx\n", _mi_thread_id()); - return theap; -} - -mi_theap_t* _mi_thread_init(void) { - return mi_thread_init_ex(NULL); -} - -void mi_decl_noinline mi_thread_init(void) mi_attr_noexcept { - _mi_thread_init(); -} - void mi_thread_done(void) mi_attr_noexcept { _mi_thread_done(NULL); } @@ -859,7 +501,7 @@ void _mi_thread_done(mi_theap_t* _theap_main) return; } - // note: we store the tld as we should avoid reading `thread_tld` at this point (to avoid reinitializing the thread local storage) + // get the current tld mi_tld_t* const tld = _theap_main->tld; // release dynamic thread_local's @@ -891,220 +533,16 @@ void _mi_thread_done(mi_theap_t* _theap_main) mi_tld_free(tld); } - -mi_decl_cold mi_decl_noinline mi_theap_t* _mi_theap_empty_get(void) { - return (mi_theap_t*)&_mi_theap_empty; -} - -bool _mi_is_empty_theap(const mi_theap_t* theap) { - return (theap == &_mi_theap_empty); -} - - -#if MI_TLS_MODEL_WIN32 - -// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed) -// See for the offsets. -#if MI_SIZE_SIZE==4 -#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_INTPTR_SIZE) -#else -#define MI_TLS_DIRECT_FIRST (0x1480 / MI_INTPTR_SIZE) -#endif -#define MI_TLS_DIRECT_SLOTS (64) -#define MI_TLS_EXPANSION_SLOTS (1024) - -// We initially use the last of the expansion slots as the default NULL. -// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc -// before we are initialized :-( (but this seems quite unlikely). -// (todo: another approach could be to use slot 7 (EnvironmentPointer) as the initial slot as that seems to be always NULL) -#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT -#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) - -// in case of errors assign fixed slots (but since we use EFAULT the program should fail anyways) -#define MI_TLS_ERROR_SLOT (5) // arbitrary user pointer -#define MI_TLS_ERROR_EXPANSION_SLOT (7) // environment pointer (only used for OS/2 emulation) - - -mi_decl_hidden mi_decl_cache_align size_t _mi_theap_default_slot = MI_TLS_INITIAL_SLOT; -mi_decl_hidden size_t _mi_theap_default_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT; -mi_decl_hidden size_t _mi_theap_cached_slot = MI_TLS_INITIAL_SLOT; -mi_decl_hidden size_t _mi_theap_cached_expansion_slot = MI_TLS_INITIAL_EXPANSION_SLOT; - -static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES; -static DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES; - -static bool mi_win_tls_slot_alloc(size_t* slot, size_t* extended, DWORD* raw_index) { - const DWORD index = TlsAlloc(); - *raw_index = index; - if (index==TLS_OUT_OF_INDEXES) { - *extended = MI_TLS_ERROR_EXPANSION_SLOT; - *slot = MI_TLS_ERROR_SLOT; - return false; - } - else if (index= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT); - if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) { - mi_prim_tls_slot_set(slot, value); - } - else { - mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS); - TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed - } -} - -#elif MI_TLS_MODEL_PTHREADS - -// only for pthreads for now -mi_decl_hidden pthread_key_t _mi_theap_default_key = MI_PTHREAD_KEY_INVALID; -mi_decl_hidden pthread_key_t _mi_theap_cached_key = MI_PTHREAD_KEY_INVALID; - -static void mi_theap_cached_key_destroy(void* theapv) { - mi_theap_t* theap = (mi_theap_t*)theapv; - if (theap!=NULL) { - _mi_theap_decref(theap); - } -} - -static void mi_tls_slots_init(void) { - mi_atomic_do_once { - _mi_pthread_key_create(&_mi_theap_default_key,NULL,NULL); - _mi_pthread_key_create(&_mi_theap_cached_key,&mi_theap_cached_key_destroy,NULL); - } -} - -static void mi_tls_slots_done(void) { - mi_pthread_key_delete(&_mi_theap_default_key); - mi_pthread_key_delete(&_mi_theap_cached_key); -} - -#elif MI_TLS_MODEL_FIXED - -static void mi_tls_slots_init(void) { - mi_atomic_do_once { - mi_theap_t* theap = _mi_theap_default(); - if (theap!=NULL) { - _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_DEFAULT, theap); - } - theap = _mi_theap_cached(); - if (theap!=NULL) { - _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_CACHED, theap); - } - } -} - -static void mi_tls_slots_done(void) { - // nothing -} - - -#else - -static void mi_tls_slots_init(void) { - // nothing -} - -static void mi_tls_slots_done(void) { - // nothing -} - -#endif - -void _mi_theap_cached_set(mi_theap_t* theap) { - mi_theap_t* prev = _mi_theap_cached(); - if (prev==theap) return; - // set - mi_tls_slots_init(); - #if MI_TLS_MODEL_LOCAL - __mi_theap_cached = theap; - #elif MI_TLS_MODEL_FIXED - mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_CACHED, theap); - #elif MI_TLS_MODEL_WIN32 - mi_win_tls_slot_set(_mi_theap_cached_slot, _mi_theap_cached_expansion_slot, theap); - #elif MI_TLS_MODEL_PTHREADS - mi_pthread_key_set(&_mi_theap_cached_key, theap); - #endif - // update refcounts (so cached theap memory keeps available until no longer cached) - _mi_theap_incref(theap); - _mi_theap_decref(prev); -} - -void _mi_theap_default_set(mi_theap_t* theap) { - mi_assert_internal(theap != NULL); - mi_assert_internal(theap->tld != NULL); - mi_assert_internal(theap->tld->thread_id==0 || theap->tld->thread_id==_mi_thread_id()); - mi_tls_slots_init(); - #if MI_TLS_MODEL_LOCAL - __mi_theap_default = theap; - #elif MI_TLS_MODEL_FIXED - mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_DEFAULT, theap); - #elif MI_TLS_MODEL_WIN32 - mi_win_tls_slot_set(_mi_theap_default_slot, _mi_theap_default_expansion_slot, theap); - #elif MI_TLS_MODEL_PTHREADS - mi_pthread_key_set(&_mi_theap_default_key, theap); - #endif - - // set theap main if needed - if (mi_theap_is_initialized(theap)) { - // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again) - _mi_prim_thread_associate_default_theap(theap); - } -} - void mi_thread_set_in_threadpool(void) mi_attr_noexcept { mi_theap_t* theap = mi_theap_get_default(); theap->tld->is_in_threadpool = true; } + // -------------------------------------------------------- -// Run functions on process init/done, and thread init/done +// Process init and done // -------------------------------------------------------- + static bool os_preloading = true; // true until this module is initialized // Returns true if this module has not been initialized; Don't use C runtime routines until it returns false. @@ -1117,21 +555,12 @@ mi_decl_nodiscard bool mi_is_redirected(void) mi_attr_noexcept { return _mi_is_redirected(); } -// Called once by the process loader from `src/prim/prim.c` +// Called once by the process loader from `src/prim/prim.c` before `main` is called. void _mi_auto_process_init(void) { - // mi_heap_main_init(); - // #if defined(__APPLE__) || defined(MI_TLS_RECURSE_GUARD) - // volatile mi_theap_t* dummy = __mi_theap_default; // access TLS to allocate it before setting tls_initialized to true; - // if (dummy == NULL) return; // use dummy or otherwise the access may get optimized away (issue #697) - // #endif - os_preloading = false; - mi_assert_internal(_mi_is_main_thread()); mi_process_init(); - mi_tls_slots_init(); mi_process_setup_auto_thread_done(); - _mi_thread_locals_init(); _mi_options_post_init(); // now we can print to stderr if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n"); @@ -1144,102 +573,41 @@ void _mi_auto_process_init(void) { } // reseed random - _mi_random_reinit_if_weak(&mi_theap_main.random); -} - -// CPU features -mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast -mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast -mi_decl_cache_align bool _mi_cpu_has_popcnt = false; - -#if (MI_ARCH_X64 || MI_ARCH_X86) -#if defined(__GNUC__) -// #include -static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { - // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx) - // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect). - uint32_t eax, ebx, ecx, edx; - __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : ); - regs4[0] = eax; - regs4[1] = ebx; - regs4[2] = ecx; - regs4[3] = edx; - return true; -} - -#elif defined(_MSC_VER) -static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { - __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel); - return true; -} -#else -static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { - MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel); - return false; -} -#endif - -static void mi_detect_cpu_features(void) { - // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) - // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?) - // FSRS for fast short rep stosb - bool amd = false; - bool fsrm = false; - // bool erms = false; - bool fsrs = false; - uint32_t cpu_info[4]; - if (mi_cpuid(cpu_info, 0, 0)) { - amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD) - } - if (mi_cpuid(cpu_info, 7, 0)) { - fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see - // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see - } - if (mi_cpuid(cpu_info, 7, 1)) { - fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see - } - if (mi_cpuid(cpu_info, 1, 0)) { - _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see - } - - if (fsrm) { - _mi_cpu_movsb_max = 127; - } - if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803 - _mi_cpu_stosb_max = 127; + mi_theap_t* theap = _mi_theap_default(); + if (theap != NULL) { + _mi_random_reinit_if_weak(&theap->random); + mi_subproc_t* subproc = _mi_theap_subproc(theap); + if (subproc->theap_meta != NULL) { + mi_lock(&subproc->theap_meta_lock) { + _mi_random_reinit_if_weak(&subproc->theap_meta->random); + } + } } } -#else -static void mi_detect_cpu_features(void) { - #if MI_ARCH_ARM64 - _mi_cpu_has_popcnt = true; + +// Initialize the process; called by thread_init, the process loader, or an initial allocation (perhaps by the loader or a system library) +static void mi_process_init_once(void) { + #if defined(__CYGWIN__) // we need to kickstart the cygwin runtime + __mi_thread_id_helper = NULL; #endif -} -#endif + _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); + _mi_detect_cpu_features(); + _mi_options_init(); // read environment (if possible) + _mi_stats_init(); // start timer + _mi_os_init(); // primitive dependent -// Initialize the process; called by thread_init or the process loader -static void mi_process_init_once(void) mi_attr_noexcept { - _mi_process_is_initialized = true; - _mi_verbose_message("process init: 0x%zx\n", _mi_thread_id()); + mi_heap_main_init(); // before page_map_init so stats are working + _mi_page_map_init(); // todo: this could fail.. should we abort in that case? + mi_thread_init(); - mi_detect_cpu_features(); - _mi_options_init(); - _mi_stats_init(); - _mi_os_init(); // the following can potentially allocate (on freeBSD for pthread keys) - // todo: do 2-phase so we can use stats at first, then later init the keys? - mi_heap_main_init(); // before page_map_init so stats are working - _mi_page_map_init(); // todo: this could fail.. should we abort in that case? - mi_thread_init(); + _mi_tls_slots_init(); // pthread key create + _mi_thread_locals_init(); // pthread key create _mi_process_is_initialized = true; - #if !defined(_WIN32) && !defined(__wasi__) - pthread_atfork(&_mi_process_fork_prepare, &_mi_process_fork_parent, &_mi_process_fork_child); - #endif - - #if defined(_WIN32) && defined(MI_WIN_USE_FLS) + #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS) // On windows, when building as a static lib the FLS cleanup happens to early for the main thread. // To avoid this, set the FLS value for the main thread to NULL so the fls cleanup // will not call _mi_thread_done on the (still executing) main thread. See issue #508. @@ -1275,119 +643,23 @@ static void mi_process_init_once(void) mi_attr_noexcept { by other threads must be reinitialized rather than released. ----------------------------------------------------------- */ -static _Atomic(uintptr_t) mi_fork_depth; // allow nested prepare/parent calls (e.g., zone hooks + pthread_atfork) -bool _mi_process_is_forked_child; // set once in fork_child, never cleared; lets visitors avoid waiting on dead-thread state - -void _mi_process_fork_prepare(void) { - if (!_mi_process_is_initialized) return; - if (mi_atomic_increment_acq_rel(&mi_fork_depth) != 0) return; - // fork() does not allocate, so a caller may reach it still parked with the scavenger mid-way - // through rewriting this thread's own pages. The clone would freeze that torn heap into a child - // where no thread will ever finish the sweep -- take our heaps back first, while we hold none - // of the locks below and the scavenger we wait on is still live in this (the parent) process. - { - mi_theap_t* const theap = _mi_theap_default(); - if (theap != NULL && mi_theap_is_initialized(theap)) { _mi_park_leave(theap->tld); } - } - mi_subproc_t* const subproc = _mi_subproc_main(); - // Acquire outermost-first: locks that can be held across an allocation (and so may - // nest into arena_reserve_lock) must be taken before arena_reserve_lock. - mi_lock_acquire(&subprocs_lock); - _mi_thread_locals_fork_prepare(); // held across mi_rezalloc_aligned - mi_lock_acquire(&mi_process_heap_main.arena_pages_lock); // held across mi_malloc in mi_heap_ensure_arena_pages - mi_lock_acquire(&subproc->heaps_lock); - // Quiesce every heap's list locks, not just heap_main: _mi_theap_init/_mi_theap_free - // mutate heap->theaps under heap->theaps_lock without taking subproc->heaps_lock. - for (mi_heap_t* h = subproc->heaps; h != NULL; h = h->next) { - if (h == &mi_process_heap_main) continue; // heap_main handled explicitly for ordering - mi_lock_acquire(&h->arena_pages_lock); - mi_lock_acquire(&h->theaps_lock); - mi_lock_acquire(&h->os_abandoned_pages_lock); - } - mi_lock_acquire(&mi_process_heap_main.theaps_lock); - mi_lock_acquire(&mi_process_heap_main.os_abandoned_pages_lock); - mi_lock_acquire(&subproc->tlds_lock); // leaf: never held across an allocation or a sweep - mi_lock_acquire(&subproc->arena_reserve_lock); -} - -void _mi_process_fork_parent(void) { - if (!_mi_process_is_initialized) return; - if (mi_atomic_decrement_acq_rel(&mi_fork_depth) != 1) return; - mi_subproc_t* const subproc = _mi_subproc_main(); - mi_lock_release(&subproc->arena_reserve_lock); - mi_lock_release(&subproc->tlds_lock); - mi_lock_release(&mi_process_heap_main.os_abandoned_pages_lock); - mi_lock_release(&mi_process_heap_main.theaps_lock); - for (mi_heap_t* h = subproc->heaps; h != NULL; h = h->next) { - if (h == &mi_process_heap_main) continue; - mi_lock_release(&h->os_abandoned_pages_lock); - mi_lock_release(&h->theaps_lock); - mi_lock_release(&h->arena_pages_lock); - } - mi_lock_release(&subproc->heaps_lock); - mi_lock_release(&mi_process_heap_main.arena_pages_lock); - _mi_thread_locals_fork_parent(); - mi_lock_release(&subprocs_lock); -} - -void _mi_process_fork_child(void) { - if (!_mi_process_is_initialized) return; - if (mi_atomic_exchange_acq_rel(&mi_fork_depth, 0) == 0) return; - _mi_process_is_forked_child = true; - _mi_scavenger_forked_child(); // the thread did not survive; clear the state that says it did - // single-threaded here: just reinitialize every lock we know about - mi_lock_init(&subprocs_lock); - for (mi_subproc_t* sp = subprocs; sp != NULL; sp = sp->next) { - mi_lock_init(&sp->arena_reserve_lock); - mi_lock_init(&sp->heaps_lock); - mi_lock_init(&sp->tlds_lock); - // a wake that was in flight at fork() leaves this at 1 with nobody to clear it, and the - // 0->1 edge in `_mi_scavenger_wake` would then never fire again - mi_atomic_store_relaxed(&sp->scavenger_wake, (uint32_t)0); - // no scavenger in the child: nothing will ever finish a claimed sweep - mi_atomic_store_relaxed(&sp->parked_count, 0); - for (mi_tld_t* t = sp->tlds; t != NULL; t = t->subproc_next) { - mi_atomic_store_relaxed(&t->park_state, (uint32_t)MI_PARK_RUNNING); - mi_atomic_store_relaxed(&t->park_reclaim, (uint32_t)0); - mi_atomic_store_relaxed(&t->park_swept, (uint32_t)0); - } - for (mi_heap_t* h = sp->heaps; h != NULL; h = h->next) { - mi_lock_init(&h->theaps_lock); - mi_lock_init(&h->arena_pages_lock); - mi_lock_init(&h->os_abandoned_pages_lock); - } - } - mi_lock_init(&mi_process_tld_main.theaps_lock); - mi_theap_t* const theap = _mi_theap_default(); - if (theap != NULL && mi_theap_is_initialized(theap) && theap->tld != &mi_process_tld_main) { - mi_lock_init(&theap->tld->theaps_lock); - } - // Re-init tld->theaps_lock for every theap still on a heap list: those tlds belong to - // threads that vanished at fork() and may have held the lock (e.g. inside _mi_theap_init - // or mi_thread_theaps_done). _mi_theap_free in the child takes that lock at theap.c:322. - for (mi_subproc_t* sp = subprocs; sp != NULL; sp = sp->next) { - for (mi_heap_t* h = sp->heaps; h != NULL; h = h->next) { - for (mi_theap_t* t = h->theaps; t != NULL; t = t->hnext) { - if (t->tld != NULL) { mi_lock_init(&t->tld->theaps_lock); } - } - } - } - _mi_thread_locals_fork_child(); -} // Initialize the process; called by thread_init or the process loader void mi_process_init(void) mi_attr_noexcept { - // #if _MSC_VER < 1920 - // mi_heap_main_init(); // vs2017 can dynamically re-initialize _mi_heap_main - // #endif mi_atomic_do_once { mi_process_init_once(); } + #if !defined(_WIN32) && !defined(__wasi__) + pthread_atfork(&_mi_process_fork_prepare, &_mi_process_fork_parent, &_mi_process_fork_child); // fork: see subproc.c + #endif } // Called when the process is done static void mi_process_done_once(void) { + _mi_scavenger_stop(); // fork: stop the background scavenger before any teardown + _mi_heap_snapshot_on_exit(); // fork: heap snapshot / profile at exit, before anything is torn down + _mi_prof_on_exit(); // only shutdown if we were initialized if (!_mi_process_is_initialized) return; // ensure we are called once @@ -1395,15 +667,8 @@ static void mi_process_done_once(void) { if (process_done) return; process_done = true; - // stop the background scavenger before any teardown - _mi_scavenger_stop(); - - // write a heap snapshot / heap profile if requested (before any cleanup so the live state is captured) - _mi_heap_snapshot_on_exit(); - _mi_prof_on_exit(); - - // free dynamic thread locals (if used at all) - _mi_thread_locals_done(); + // decref any cached theap + _mi_theap_cached_set(_mi_theap_empty_get()); // release any thread specific resources and ensure _mi_thread_done is called on all but the main thread _mi_prim_thread_done_auto_done(); @@ -1423,37 +688,44 @@ static void mi_process_done_once(void) { // Forcefully release all retained memory; this can be dangerous in general if overriding regular malloc/free // since after process_done there might still be other code running that calls `free` (like at_exit routines, // or C-runtime termination code. + mi_subproc_t* subproc_main = _mi_subproc_main(); if (mi_option_is_enabled(mi_option_destroy_on_exit)) { - mi_subprocs_unsafe_destroy_all(); // destroys all subprocs, arenas, and the page_map! + _mi_subprocs_unsafe_destroy_all(); // destroys all mi_subprocs, arenas, thread locals, and the page_map! } - else if (subproc_main.heap_main != NULL) { - mi_heap_stats_merge_to_subproc(subproc_main.heap_main); + else { + // free dynamic thread locals (if used at all) + _mi_thread_locals_thread_done(); + _mi_thread_locals_done(); + if (subproc_main->heap_main != NULL) { + if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + _mi_theap_merge_stats(subproc_main->theap_meta); + _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free + mi_heap_stats_merge_to_subproc(subproc_main->heap_main); + mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); + } + } } - // careful now to no longer access any allocator functionality - if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { - mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); - } - mi_lock_done(&subprocs_lock); - mi_tls_slots_done(); + _mi_tls_slots_done(); + _mi_subproc_main_done(); _mi_allocator_done(); - _mi_verbose_message("process done: 0x%zx\n", mi_process_tld_main.thread_id); + _mi_verbose_message("process done\n"); // : 0x%zx\n", mi_process_tld_main.thread_id); os_preloading = true; // don't call the C runtime anymore } + +// Call when the process is done (cdecl as it is used with `at_exit` on some platforms) void mi_cdecl mi_process_done(void) mi_attr_noexcept { mi_atomic_do_once { mi_process_done_once(); } } +// Called automatically when the process is done (cdecl as it is used with `at_exit` on some platforms) void mi_cdecl _mi_auto_process_done(void) mi_attr_noexcept { #ifdef MI_NO_PROCESS_DETACH - // Embedder opted out of automatic exit-time teardown entirely. The process is going - // away; skipping mi_process_done avoids touching allocator state after other static - // destructors may have already run. mi_process_done() can still be called manually. - return; + return; // fork: the embedder tears down explicitly or not at all; touching allocator state this late is unsafe under other static destructors #endif - if (_mi_option_get_fast(mi_option_destroy_on_exit)>1) return; + if (_mi_option_get_fast(mi_option_destroy_on_exit)>=2) return; // allow disabling auto process done mi_process_done(); } diff --git a/src/libc.c b/src/libc.c index f7b8e5592..17dd1d1dc 100644 --- a/src/libc.c +++ b/src/libc.c @@ -1,12 +1,12 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2024, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. -----------------------------------------------------------------------------*/ // -------------------------------------------------------- -// This module defines various std libc functions to reduce +// This module defines various standard libc functions to reduce // the dependency on libc, and also prevent errors caused // by some libc implementations when called before `main` // executes (due to malloc redirection) @@ -39,8 +39,8 @@ bool _mi_streq(const char* s, const char* t) { return (*s == *t); } -void _mi_strlcpy(char* dest, const char* src, size_t dest_size) { - if (dest==NULL || src==NULL || dest_size == 0) return; +bool _mi_strlcpy(char* dest, const char* src, size_t dest_size) { + if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0); // copy until end of src, or when dest is (almost) full while (*src != 0 && dest_size > 1) { *dest++ = *src++; @@ -48,17 +48,18 @@ void _mi_strlcpy(char* dest, const char* src, size_t dest_size) { } // always zero terminate *dest = 0; + return (*src == 0); } -void _mi_strlcat(char* dest, const char* src, size_t dest_size) { - if (dest==NULL || src==NULL || dest_size == 0) return; +bool _mi_strlcat(char* dest, const char* src, size_t dest_size) { + if (dest==NULL || src==NULL || dest_size == 0) return (src==NULL || *src==0); // find end of string in the dest buffer while (*dest != 0 && dest_size > 1) { dest++; dest_size--; } // and catenate - _mi_strlcpy(dest, src, dest_size); + return _mi_strlcpy(dest, src, dest_size); } size_t _mi_strnlen(const char* s, size_t max_len) { @@ -96,7 +97,7 @@ int _mi_getenv(const char* name, char* result, size_t result_size) { } #else int _mi_getenv(const char* name, char* result, size_t result_size) { - if (name==NULL || result == NULL || result_size < 64) return false; + if (name==NULL || result == NULL || result_size < 64) return ENOENT; // change the result of _mi_prim_getenv to an errno result const int res = _mi_prim_getenv(name,result,result_size); return (res > 0 ? 0 : (res == 0 ? ENOENT : EAGAIN)); @@ -118,7 +119,7 @@ bool _mi_atomic_once_enter(mi_atomic_once_t* once) { } const mi_threadid_t current_tid = _mi_thread_id(); if (once_tid == current_tid) { - return false; // recursive invocation; we need this for process_init for example + return false; // recursive invocation; don't block on ourselves } mi_lock_acquire(&once->lock); @@ -156,6 +157,79 @@ mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruc } #endif +// -------------------------------------------------------- +// Detect CPU features +// -------------------------------------------------------- +mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast +mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast +mi_decl_cache_align bool _mi_cpu_has_popcnt = false; + +#if (MI_ARCH_X64 || MI_ARCH_X86) +#if defined(__GNUC__) +// #include +static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { + // note: use explicit assembly instead of __get_cpuid as we need the sublevel (in ecx) + // (on Ubuntu 22 with WSL the __get_cpuid does not clear ecx for level 7 which is incorrect). + uint32_t eax, ebx, ecx, edx; + __asm __volatile("cpuid" : "=a"(eax), "=b"(ebx), "=c"(ecx), "=d"(edx) : "a"(level), "c"(sublevel) : ); + regs4[0] = eax; + regs4[1] = ebx; + regs4[2] = ecx; + regs4[3] = edx; + return true; +} + +#elif defined(_MSC_VER) +static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { + __cpuidex((int32_t*)regs4, (int32_t)level, (int32_t)sublevel); + return true; +} +#else +static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { + MI_UNUSED(regs4); MI_UNUSED(level); MI_UNUSED(sublevel); + return false; +} +#endif + +void _mi_detect_cpu_features(void) { + // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) + // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?) + // FSRS for fast short rep stosb + bool amd = false; + bool fsrm = false; + // bool erms = false; + bool fsrs = false; + uint32_t cpu_info[4]; + if (mi_cpuid(cpu_info, 0, 0)) { + amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD) + } + if (mi_cpuid(cpu_info, 7, 0)) { + fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see + // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see + } + if (mi_cpuid(cpu_info, 7, 1)) { + fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see + } + if (mi_cpuid(cpu_info, 1, 0)) { + _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see + } + + if (fsrm) { + _mi_cpu_movsb_max = 127; + } + if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803 + _mi_cpu_stosb_max = 127; + } +} + +#else +void _mi_detect_cpu_features(void) { + #if MI_ARCH_ARM64 + _mi_cpu_has_popcnt = true; + #endif +} +#endif + // -------------------------------------------------------- // Define our own limited `_mi_vsnprintf` and `_mi_snprintf` @@ -215,22 +289,20 @@ static void mi_out_num(uintmax_t x, size_t base, char prefix, char** out, char* mi_outc('0',out,end); } else { - // output digits in reverse - char* start = *out; - while (x > 0) { + #define MI_MAX_OUT_DIGITS (160) /* a 512 bit number has 155 digits */ + char num[MI_MAX_OUT_DIGITS]; + int dcount = 0; + while(x>0 && dcount < MI_MAX_OUT_DIGITS) { char digit = (char)(x % base); - mi_outc((digit <= 9 ? '0' + digit : 'A' + digit - 10),out,end); + num[dcount++] = (digit <= 9 ? '0' + digit : 'A' + digit - 10); x = x / base; } + if (dcount>=MI_MAX_OUT_DIGITS) return; // don't output anything? if (prefix != 0) { mi_outc(prefix, out, end); } - size_t len = *out - start; - // and reverse in-place - for (size_t i = 0; i < (len / 2); i++) { - char c = start[len - i - 1]; - start[len - i - 1] = start[i]; - start[i] = c; + while(dcount-- > 0) { + mi_outc(num[dcount], out, end); } } } @@ -277,7 +349,10 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { if (c >= '1' && c <= '9') { width = (c - '0'); MI_NEXTC(); while (c >= '0' && c <= '9') { - width = (10 * width) + (c - '0'); MI_NEXTC(); + if (width < SIZE_MAX/1024) { // no overflow + width = (10 * width) + (c - '0'); + } + MI_NEXTC(); } if (c == 0) break; // extra check due to while } @@ -316,7 +391,7 @@ int _mi_vsnprintf(char* buf, size_t bufsize, const char* fmt, va_list args) { if (width == 0 && (c == 'x' || c == 'p')) { if (c == 'p') { width = 2 * (x <= UINT32_MAX ? 4 : ((x >> 16) <= UINT32_MAX ? 6 : sizeof(void*))); } if (width == 0) { width = 2; } - fill = '0'; + if (alignright) { fill = '0'; } } mi_out_num(x, (c == 'x' || c == 'p' ? 16 : 10), numplus, &out, end); } @@ -369,6 +444,7 @@ int _mi_snprintf(char* buf, size_t buflen, const char* fmt, ...) { return written; } +#undef MI_NEXTC // -------------------------------------------------------- @@ -384,7 +460,7 @@ static size_t mi_ctz_generic32(uint32_t x) { 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 }; if (x==0) return 32; - return debruijn[(uint32_t)((x & -(int32_t)x) * (uint32_t)(0x077CB531U)) >> 27]; + return debruijn[(uint32_t)((x & (~x + 1U)) * (uint32_t)(0x077CB531U)) >> 27]; } static size_t mi_clz_generic32(uint32_t x) { diff --git a/src/options.c b/src/options.c index 9a526c811..bc5e12fa6 100644 --- a/src/options.c +++ b/src/options.c @@ -85,7 +85,7 @@ int mi_version(void) { #endif #ifndef MI_DEFAULT_PAGEMAP_COMMIT -#if defined(__APPLE__) // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access +#if defined(__APPLE__) && MI_PAGE_MAP_FLAT // when overloading malloc, we still get mixed pointers sometimes on macOS; this avoids a bad access #define MI_DEFAULT_PAGEMAP_COMMIT 1 #else #define MI_DEFAULT_PAGEMAP_COMMIT 0 @@ -224,7 +224,7 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ const int vermajor = MI_MALLOC_VERSION/10000; const int verminor = (MI_MALLOC_VERSION%10000)/100; const int verpatch = (MI_MALLOC_VERSION%100); - _mi_fprintf(out, arg, "v%i.%i.%i%s%s (built on %s, %s)\n", vermajor, verminor, verpatch, + _mi_fprintf(out, arg, "v%i.%i.%i%s%s\n", vermajor, verminor, verpatch, #if defined(MI_CMAKE_BUILD_TYPE) ", " mi_stringify(MI_CMAKE_BUILD_TYPE) #else @@ -236,7 +236,7 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ #else "" #endif - , __DATE__, __TIME__); + ); // show options for (int i = 0; i < _mi_option_last; i++) { @@ -508,7 +508,7 @@ void _mi_fprintf( mi_output_fun* out, void* arg, const char* fmt, ... ) { } static void mi_vfprintf_thread(mi_output_fun* out, void* arg, const char* prefix, const char* fmt, va_list args) { - if (prefix != NULL && _mi_strnlen(prefix,33) <= 32 && !_mi_is_main_thread()) { + if (prefix != NULL && _mi_strnlen(prefix,33) <= 32) { // && !_mi_is_main_thread()) { char tprefix[64]; _mi_snprintf(tprefix, sizeof(tprefix), "%sthread 0x%tx: ", prefix, (uintptr_t)_mi_thread_id()); mi_vfprintf(out, arg, tprefix, fmt, args); @@ -525,6 +525,7 @@ void _mi_raw_message(const char* fmt, ...) { va_end(args); } +// fork: informational output (heap snapshot / profile file names and the like) that is not gated on the verbose option void _mi_message(const char* fmt, ...) { va_list args; va_start(args, fmt); @@ -584,24 +585,27 @@ static _Atomic(void*) mi_error_arg; // = NULL static void mi_error_default(int err) { MI_UNUSED(err); -#if (MI_DEBUG>0) - if (err==EFAULT) { - #ifdef _MSC_VER - __debugbreak(); - #endif - abort(); - } -#endif -#if (MI_SECURE>0) - if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data) - abort(); - } -#endif -#if defined(MI_XMALLOC) - if (err==ENOMEM || err==EOVERFLOW) { // abort on memory allocation fails in xmalloc mode - abort(); + #if (MI_DEBUG>0) + if (err==EFAULT) { + #ifdef _MSC_VER + __debugbreak(); + #endif + abort(); + } + #endif + #if (MI_SECURE>0) + if (err==EFAULT) { // abort on serious errors in secure mode (corrupted meta-data) + abort(); + } + #endif + #if defined(MI_XMALLOC) + if (err==ENOMEM || err==EOVERFLOW || err==EINVAL) { // abort on memory allocation fails in xmalloc mode + abort(); + } + #endif + if (errno==0) { + errno = (err==EINVAL ? EINVAL : ENOMEM /* compatibility */ ); } -#endif } void mi_register_error(mi_error_fun* fun, void* arg) { @@ -615,7 +619,7 @@ void _mi_error_message(int err, const char* fmt, ...) { va_start(args, fmt); mi_show_error_message(fmt, args); va_end(args); - // and call the error handler which may abort (or return normally) + // and call the error handler which may abort (or return normally, potentially setting errno) if (mi_error_handler != NULL) { mi_error_handler(err, mi_atomic_load_ptr_acquire(void,&mi_error_arg)); } @@ -663,8 +667,9 @@ static void mi_option_init(mi_option_desc_t* desc) { } else { char* end = buf; + errno = 0; long value = strtol(buf, &end, 10); - if (mi_option_has_size_in_kib(desc->option)) { + if (errno==0 && mi_option_has_size_in_kib(desc->option)) { // this option is interpreted in KiB to prevent overflow of `long` for large allocations // (long is 32-bit on 64-bit windows, which allows for 4TiB max.) size_t size = (value < 0 ? 0 : (size_t)value); @@ -679,7 +684,7 @@ static void mi_option_init(mi_option_desc_t* desc) { if (overflow || size > (MI_MAX_ALLOC_SIZE / MI_KiB)) { size = (MI_MAX_ALLOC_SIZE / MI_KiB); } value = (size > LONG_MAX ? LONG_MAX : (long)size); } - if (*end == 0) { + if (errno==0 && *end == 0) { mi_option_set(desc->option, value); } else { diff --git a/src/os.c b/src/os.c index 5bda5bdd7..4ce683c2f 100644 --- a/src/os.c +++ b/src/os.c @@ -386,7 +386,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t // note: Windows VirtualFree needs the actual base pointer // this is handled though by having the `base` field in the memid os_base = p; // remember the base - os_size = over_size; + os_size = over_size; // todo: use size instead as now we over-decrement commit stats on free? p = _mi_align_up_ptr(p, alignment); // explicitly commit only the aligned part @@ -512,6 +512,7 @@ void* _mi_os_alloc_aligned_at_offset(mi_subproc_t* subproc, size_t size, size_t void* const p = (uint8_t*)start + extra; mi_assert(_mi_is_aligned((uint8_t*)p + offset, alignment)); // decommit the overallocation at the start + // note: this double counts the decommit when freeing `memid`. Should we keep commit size in the memid as well? if (commit && extra >= _mi_os_page_size()) { _mi_os_decommit(subproc, start, extra); } @@ -796,7 +797,11 @@ static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0 // Claim an aligned address range for huge pages static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { if (total_size != NULL) *total_size = 0; - const size_t size = pages * MI_HUGE_OS_PAGE_SIZE; + size_t size = 0; + if (mi_mul_overflow(pages,MI_HUGE_OS_PAGE_SIZE,&size)) { + _mi_warning_message("too many huge pages requested: %zu\n", pages); + return NULL; + } uintptr_t start = 0; uintptr_t end = 0; @@ -805,7 +810,7 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) { start = huge_start; if (start == 0) { // Initialize the start address after the 32TiB area - start = ((uintptr_t)8 << 40); // 8TiB virtual start address + start = ((uintptr_t)32 << 40); // 32TiB virtual start address (after addresses returned by _mi_os_get_aligned_hint) #if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (mi_theap_is_initialized(theap)) { // todo: or no hint at all if we lack randomness? @@ -877,7 +882,7 @@ void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_n if (max_msecs > 0) { mi_msecs_t elapsed = _mi_clock_end(start_t); if (page >= 1) { - mi_msecs_t estimate = ((elapsed / (page+1)) * pages); + mi_msecs_t estimate = ((elapsed / (page==0 ? 1 : page)) * pages); if (estimate > 2*max_msecs) { // seems like we are going to timeout, break elapsed = max_msecs + 1; } diff --git a/src/page-map.c b/src/page-map.c index 4bda10db3..b4ee0189a 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -38,7 +38,7 @@ static mi_bitmap_t* mi_page_map_commit; // one bit per committed 64 KiB entries mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, size_t slice_count); bool _mi_page_map_init(void) { - size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS); + size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS); if (vbits == 0) { vbits = _mi_os_virtual_address_bits(); #if MI_ARCH_X64 // canonical address is limited to the first 128 TiB @@ -51,6 +51,9 @@ bool _mi_page_map_init(void) { if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr vbits = MI_MIN_VABITS; } + if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available + vbits = MI_MAX_VABITS; + } // Allocate the page map and commit bits mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits))); @@ -210,47 +213,64 @@ mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_att // A 2-level page map #define MI_PAGE_MAP_SUB_SIZE (MI_PAGE_MAP_SUB_COUNT * sizeof(mi_page_t*)) -#define MI_PAGE_MAP_ENTRIES_PER_CBIT (MI_PAGE_MAP_COUNT < MI_BFIELD_BITS ? 1 : (MI_PAGE_MAP_COUNT / MI_BFIELD_BITS)) // Use an initial empty page map so `free(NULL)` works even if mimalloc is not yet initialized (issue #1341) -static mi_page_t* mi_submap_empty[1] = { NULL }; -static _Atomic(mi_submap_t) mi_page_map_empty[1] = { MI_ATOMIC_VAR_INIT(mi_submap_empty) }; - -mi_decl_hidden mi_decl_cache_align _Atomic(mi_submap_t)* _mi_page_map = mi_page_map_empty; -mi_decl_hidden _Atomic(void*) _mi_page_map_max_address; // = NULL (static zero-init; a `= NULL` initializer does not compile as C++) -static size_t mi_page_map_count; -static mi_memid_t mi_page_map_memid; -static mi_lock_t mi_page_map_lock; +static mi_page_map_t mi_page_map_empty = { + MI_ATOMIC_VAR_INIT(1), + sizeof(mi_page_map_t), + MI_MEMID_STATIC, + MI_LOCK_INITIALIZER, + { MI_ATOMIC_VAR_INIT(NULL) } +}; + +mi_decl_hidden mi_decl_cache_align _Atomic(mi_page_map_t*) __mi_page_map = MI_ATOMIC_VAR_INIT(&mi_page_map_empty); + +static size_t mi_page_map_count_of_size(size_t size) { + return (size < sizeof(mi_page_map_t) ? 0 : 1 + (size - sizeof(mi_page_map_t))/sizeof(mi_submap_t)); +} -// divide the main map in 64 (`MI_BFIELD_BITS`) parts commit those parts on demand -static _Atomic(mi_bfield_t) mi_page_map_commit; +// static void* mi_page_map_addr_of_index(size_t idx) { +// return (void*)((uintptr_t)idx * MI_PAGE_MAP_SUB_COUNT * MI_ARENA_SLICE_SIZE); +// } -mi_decl_nodiscard static inline bool mi_page_map_is_committed(size_t idx, size_t* pbit_idx) { - mi_bfield_t commit = mi_atomic_load_relaxed(&mi_page_map_commit); - const size_t bit_idx = idx/MI_PAGE_MAP_ENTRIES_PER_CBIT; - mi_assert_internal(bit_idx < MI_BFIELD_BITS); - if (pbit_idx != NULL) { *pbit_idx = bit_idx; } - return ((commit & (MI_ZU(1) << bit_idx)) != 0); +mi_decl_nodiscard static mi_decl_noinline bool mi_page_map_commit_entries(mi_page_map_t* pmap, size_t required_idx) { + const size_t reserved_count = mi_page_map_count_of_size(pmap->reserved_size); + if mi_unlikely(required_idx >= reserved_count) { + mi_page_map_cannot_commit(); + return false; + } + size_t commit_size = _mi_align_up( sizeof(mi_page_map_t) + (required_idx * sizeof(mi_submap_t)), MI_ARENA_SLICE_SIZE ); + if (pmap->reserved_size < commit_size) { commit_size = pmap->reserved_size; } + const size_t commit_count = mi_page_map_count_of_size(commit_size); + mi_assert_internal(commit_count > required_idx); + mi_assert_internal(commit_count <= reserved_count); + // note: we rely on uncommitted memory to be zero initialized on the first commit (and further concurrent commits leave the memory as is). + bool is_zero; + if mi_unlikely(!_mi_os_commit(_mi_subproc_main(), pmap, commit_size, &is_zero)) { + mi_page_map_cannot_commit(); + return false; + } + mi_assert_internal(is_zero || pmap->memid.initially_zero); + mi_atomic_store_release(&pmap->committed_count, commit_count); + // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); + return true; } -mi_decl_nodiscard static bool mi_page_map_ensure_committed(size_t idx, mi_submap_t* submap) { +mi_decl_nodiscard static bool mi_page_map_ensure_committed(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) { mi_assert_internal(submap!=NULL && *submap==NULL); - size_t bit_idx; - if mi_unlikely(!mi_page_map_is_committed(idx, &bit_idx)) { - uint8_t* start = (uint8_t*)&_mi_page_map[bit_idx * MI_PAGE_MAP_ENTRIES_PER_CBIT]; - if (!_mi_os_commit(_mi_subproc_main(), start, MI_PAGE_MAP_ENTRIES_PER_CBIT * sizeof(mi_submap_t), NULL)) { - mi_page_map_cannot_commit(); - return false; + if mi_unlikely(idx >= mi_atomic_load_relaxed(&pmap->committed_count)) { + if (idx >= mi_atomic_load_acquire(&pmap->committed_count)) { + if (!mi_page_map_commit_entries(pmap,idx)) return false; + mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); } - mi_atomic_or_acq_rel(&mi_page_map_commit, MI_ZU(1) << bit_idx); } - *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // acquire _mi_page_map_at(idx); + *submap = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); return true; } -// initialize the page map -bool _mi_page_map_init(void) { - size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_SIZE_BITS); +// initialize the page map +static bool mi_page_map_init_once(void) { + size_t vbits = (size_t)mi_option_get_clamp(mi_option_max_vabits, 0, MI_MAX_VABITS); if (vbits == 0) { vbits = _mi_os_virtual_address_bits(); #if MI_ARCH_X64 // canonical address is limited to the first 128 TiB @@ -263,114 +283,141 @@ bool _mi_page_map_init(void) { if (vbits < MI_MIN_VABITS) { // cover at least this much for a faster _mi_checked_ptr vbits = MI_MIN_VABITS; } + if (vbits > MI_MAX_VABITS) { // limit page map size even if more virtual addresses are available + vbits = MI_MAX_VABITS; + } // Allocate the page map and commit bits mi_assert(MI_MAX_VABITS >= vbits); - mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, (void*)(vbits >= MI_SIZE_BITS ? (SIZE_MAX - MI_ARENA_SLICE_SIZE + 1) : (MI_PU(1) << vbits))); - mi_page_map_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); - mi_assert(mi_page_map_count <= MI_PAGE_MAP_COUNT); - const size_t os_page_size = _mi_os_page_size(); - const size_t page_map_size = _mi_align_up( mi_page_map_count * sizeof(mi_page_t**), os_page_size); - const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; - const size_t reserve_size = page_map_size + submap_size; - #if MI_SECURE + const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + const size_t os_page_size = _mi_os_page_size(); + const size_t reserve_size = _mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_submap_t)), os_page_size); + const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; + const size_t extra_reserve_size = reserve_size + submap_size; + #if MI_SECURE const bool commit = true; // the whole page map is valid and we can reliably check any pointer #else - const bool commit = page_map_size <= 64*MI_KiB || - mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); + const bool commit = false; // reserve_size <= 256*MI_KiB || // 40 virtual address bits + // mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); #endif mi_subproc_t* const subproc = _mi_subproc_main(); - _mi_page_map = (_Atomic(mi_page_t**)*)_mi_os_alloc_aligned(subproc, reserve_size, 1, commit, true /* allow large */, &mi_page_map_memid); - if (_mi_page_map==NULL) { - _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", page_map_size / MI_KiB); - _mi_page_map = mi_page_map_empty; + mi_memid_t memid; + mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_aligned(subproc, extra_reserve_size, 1, commit, true /* allow large */, &memid); + if mi_unlikely(pmap==NULL) { + _mi_error_message(ENOMEM, "unable to reserve virtual memory for the page map (%zu KiB)\n", extra_reserve_size / MI_KiB); return false; } - if (mi_page_map_memid.initially_committed && !mi_page_map_memid.initially_zero) { - _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); - _mi_memzero_aligned(_mi_page_map, page_map_size); + + // commit + size_t commit_count; + if (memid.initially_committed) { + if (!memid.initially_zero) { + _mi_warning_message("internal: the page map was committed but not zero initialized!\n"); + _mi_memzero_aligned(pmap, extra_reserve_size); + memid.initially_zero = true; + } + commit_count = mi_page_map_count_of_size(reserve_size); } - mi_atomic_store_release(&mi_page_map_commit, (mi_page_map_memid.initially_committed ? ~MI_ZU(0) : MI_ZU(0))); - + else { + // commit first page + bool is_zero; + if (!_mi_os_commit(subproc,pmap,os_page_size,&is_zero)) { + mi_page_map_cannot_commit(); + _mi_os_free(subproc,pmap,extra_reserve_size,memid); + return false; + }; + mi_assert_internal(is_zero || memid.initially_zero); + commit_count = mi_page_map_count_of_size(os_page_size); + } + // ensure there is a submap for the NULL address - mi_page_t** const sub0 = (mi_page_t**)((uint8_t*)_mi_page_map + page_map_size); // we reserved a submap part at the end already - if (!mi_page_map_memid.initially_committed) { + mi_page_t** const sub0 = (mi_submap_t)((uint8_t*)pmap + reserve_size); // we reserved a submap part at the end already + if (!memid.initially_committed) { if (!_mi_os_commit(subproc, sub0, submap_size, NULL)) { // commit full submap (issue #1087) mi_page_map_cannot_commit(); + _mi_os_free(subproc,pmap,extra_reserve_size,memid); return false; } } - if (!mi_page_map_memid.initially_zero) { // initialize low addresses with NULL + if (!memid.initially_zero) { // initialize low addresses with NULL _mi_memzero_aligned(sub0, submap_size); } - mi_submap_t nullsub = NULL; - if (!mi_page_map_ensure_committed(0,&nullsub)) { - mi_page_map_cannot_commit(); - return false; - } - mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[0], sub0); - mi_lock_init(&mi_page_map_lock); // initialize late in case the lock init causes allocation - + + // initialize the fields + pmap->memid = memid; + pmap->reserved_size = reserve_size; + mi_lock_init(&pmap->lock); + mi_atomic_store_release(&pmap->committed_count, commit_count); + // mi_atomic_store_release(&pmap->committed_addr, mi_page_map_addr_of_index(commit_count)); + mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[0], sub0); + mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, pmap); mi_assert_internal(_mi_ptr_page(NULL)==NULL); return true; } +bool _mi_page_map_init(void) { + bool ok = true; + mi_atomic_do_once { + ok = mi_page_map_init_once(); + } + return ok; +} void _mi_page_map_unsafe_destroy(void) { - mi_assert_internal(_mi_page_map != NULL); - if (_mi_page_map == NULL) return; + mi_page_map_t* const pmap = _mi_page_map(); + mi_assert_internal(pmap != NULL); + if (pmap == NULL || pmap == &mi_page_map_empty) return; mi_subproc_t* const subproc = _mi_subproc_main(); - mi_lock_done(&mi_page_map_lock); - for (size_t idx = 1; idx < mi_page_map_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map) - // free all sub-maps - if (mi_page_map_is_committed(idx, NULL)) { - mi_submap_t sub = _mi_page_map_at(idx); - if (sub != NULL) { - mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false); - _mi_os_free_ex(subproc, memid.mem.os.base, memid.mem.os.size, true, memid); - mi_atomic_store_ptr_release(mi_page_t*, &_mi_page_map[idx], NULL); - } + mi_lock_done(&pmap->lock); + for (size_t idx = 1; idx < pmap->committed_count; idx++) { // skip entry 0 (as we allocate that submap at the end of the page_map) + // free all sub-maps + mi_submap_t sub = _mi_page_map_at(pmap,idx); + if (sub != NULL) { + mi_memid_t memid = _mi_memid_create_os(sub, MI_PAGE_MAP_SUB_SIZE, true, false, false); + _mi_os_free_ex(subproc, memid.mem.os.base, memid.mem.os.size, true, memid); + mi_atomic_store_ptr_release(mi_page_t*, &pmap->submaps[idx], NULL); } } - _mi_os_free_ex(subproc, _mi_page_map, mi_page_map_memid.mem.os.size, true, mi_page_map_memid); - _mi_page_map = NULL; - mi_page_map_count = 0; - mi_page_map_memid = _mi_memid_none(); - mi_atomic_store_ptr_release(void, &_mi_page_map_max_address, NULL); - mi_atomic_store_release(&mi_page_map_commit, (mi_bfield_t)0); + _mi_os_free_ex(subproc, pmap, pmap->reserved_size, true, pmap->memid); + mi_atomic_store_ptr_release(mi_page_map_t, &__mi_page_map, &mi_page_map_empty); } - -mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap_t* submap) { - mi_assert_internal(submap!=NULL && *submap==NULL); +mi_decl_nodiscard static mi_decl_noinline mi_submap_t mi_page_map_alloc_submap_at(mi_page_map_t* pmap, size_t idx) { + // sub map not yet allocated, alloc now mi_submap_t sub = NULL; - if (!mi_page_map_ensure_committed(idx, &sub)) { - return false; - } - if mi_unlikely(sub == NULL) { - // sub map not yet allocated, alloc now - mi_lock(&mi_page_map_lock) + mi_lock(&pmap->lock) + { + sub = mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); // reload + if (sub==NULL) // not yet allocated by another thread? { - sub = mi_atomic_load_ptr_acquire(mi_page_t*, &_mi_page_map[idx]); // reload - if (sub==NULL) // not yet allocated by another thread? - { - mi_subproc_t* const subproc = _mi_subproc_main(); - mi_memid_t memid; - const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; - sub = (mi_submap_t)_mi_os_zalloc(subproc, submap_size, &memid); - if (sub==NULL) { - _mi_warning_message("internal error: unable to extend the page map\n"); - } - else { - mi_submap_t expect = NULL; - if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &_mi_page_map[idx], &expect, sub)) { - // another thread already allocated it.. free and continue - _mi_os_free(subproc, sub, submap_size, memid); - sub = expect; - } + mi_subproc_t* const subproc = _mi_subproc_main(); + mi_memid_t memid; + const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; + sub = (mi_submap_t)_mi_os_zalloc(subproc, submap_size, &memid); + if (sub==NULL) { + _mi_warning_message("internal error: unable to extend the page map\n"); + } + else { + mi_submap_t expect = NULL; + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_page_t*, &pmap->submaps[idx], &expect, sub)) { + // another thread already allocated it.. free and continue + _mi_os_free(subproc, sub, submap_size, memid); + sub = expect; } } } + } + return sub; +} + +mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(mi_page_map_t* pmap, size_t idx, mi_submap_t* submap) { + mi_assert_internal(submap!=NULL && *submap==NULL); + mi_submap_t sub = NULL; + if (!mi_page_map_ensure_committed(pmap, idx, &sub)) { + return false; + } + if mi_unlikely(sub==NULL) { + sub = mi_page_map_alloc_submap_at(pmap, idx); if (sub==NULL) return false; // unable to allocate the submap.. } mi_assert_internal(sub!=NULL); @@ -378,11 +425,11 @@ mi_decl_nodiscard static bool mi_page_map_ensure_submap_at(size_t idx, mi_submap return true; } -static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { +static bool mi_page_map_set_range_prim(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { // is the page map area that contains the page address committed? while (slice_count > 0) { mi_submap_t sub = NULL; - if (!mi_page_map_ensure_submap_at(idx, &sub)) { + if (!mi_page_map_ensure_submap_at(pmap, idx, &sub)) { return false; }; mi_assert_internal(sub!=NULL); @@ -398,11 +445,11 @@ static bool mi_page_map_set_range_prim(mi_page_t* page, size_t idx, size_t sub_i return true; } -static bool mi_page_map_set_range(mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { - if mi_unlikely(!mi_page_map_set_range_prim(page,idx,sub_idx,slice_count)) { +static bool mi_page_map_set_range(mi_page_map_t* pmap, mi_page_t* page, size_t idx, size_t sub_idx, size_t slice_count) { + if mi_unlikely(!mi_page_map_set_range_prim(pmap, page,idx,sub_idx,slice_count)) { // failed to commit, call again to reset the page pointer if needed if (page!=NULL) { - mi_page_map_set_range_prim(NULL,idx,sub_idx,slice_count); + mi_page_map_set_range_prim(pmap,NULL,idx,sub_idx,slice_count); } return false; } @@ -420,49 +467,46 @@ static size_t mi_page_map_get_idx(mi_page_t* page, size_t* sub_idx, size_t* slic bool _mi_page_map_register(mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); - mi_assert_internal(_mi_page_map != NULL); // should be initialized before multi-thread access! - if mi_unlikely(_mi_page_map == NULL) { + mi_page_map_t* pmap = _mi_page_map(); + mi_assert_internal(pmap != NULL); // should be initialized before multi-thread access! + if mi_unlikely(pmap == NULL) { if (!_mi_page_map_init()) return false; + pmap = mi_atomic_load_ptr_acquire(mi_page_map_t,&__mi_page_map); } - mi_assert(_mi_page_map!=NULL); + mi_assert(pmap!=NULL); size_t slice_count; size_t sub_idx; const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count); - return mi_page_map_set_range(page, idx, sub_idx, slice_count); + return mi_page_map_set_range(pmap, page, idx, sub_idx, slice_count); } void _mi_page_map_unregister(mi_page_t* page) { mi_assert_internal(_mi_page_map != NULL); mi_assert_internal(page != NULL); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_page_map_t* const pmap = _mi_page_map(); // note: should proceed even if the page was not registered yet (for failure paths in page allocation in `arena.c`) - if mi_unlikely(_mi_page_map == NULL) return; + if mi_unlikely(pmap == NULL) return; // get index and count size_t slice_count; size_t sub_idx; const size_t idx = mi_page_map_get_idx(page, &sub_idx, &slice_count); // unset the offsets - mi_page_map_set_range(NULL, idx, sub_idx, slice_count); + mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); } void _mi_page_map_unregister_range(void* start, size_t size) { - if mi_unlikely(_mi_page_map == NULL) return; + mi_page_map_t* const pmap = _mi_page_map(); + if mi_unlikely(pmap == NULL) return; const size_t slice_count = _mi_divide_up(size, MI_ARENA_SLICE_SIZE); size_t sub_idx; const uintptr_t idx = _mi_page_map_index(start, &sub_idx); - mi_page_map_set_range(NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed? + mi_page_map_set_range(pmap, NULL, idx, sub_idx, slice_count); // todo: avoid committing if not already committed? } // Return NULL for invalid pointers mi_page_t* _mi_safe_ptr_page(const void* p) { - if (p==NULL) return NULL; - if mi_unlikely(p >= mi_atomic_load_ptr_relaxed(void, &_mi_page_map_max_address)) return NULL; - size_t sub_idx; - const size_t idx = _mi_page_map_index(p,&sub_idx); - if mi_unlikely(!mi_page_map_is_committed(idx,NULL)) return NULL; - mi_page_t** const sub = _mi_page_map[idx]; - if mi_unlikely(sub==NULL) return NULL; - return sub[sub_idx]; + return _mi_checked_ptr_page(p); } mi_decl_nodiscard mi_decl_export bool mi_is_in_heap_region(const void* p) mi_attr_noexcept { diff --git a/src/page-queue.c b/src/page-queue.c index eec6f9e3b..29860078c 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -61,7 +61,7 @@ static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) { // Returns MI_BIN_HUGE if the size is too large. // We use `wsize` for the size in "machine word sizes", // i.e. byte size == `wsize*sizeof(void*)`. -static mi_decl_noinline size_t mi_bin(size_t size) { +static size_t mi_bin(size_t size) { size_t wsize = _mi_wsize_from_size(size); #if defined(MI_ALIGN4W) if mi_likely(wsize <= 4) { @@ -212,12 +212,12 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ if (size > MI_SMALL_SIZE_MAX) return; mi_page_t* page = pq->first; - if (pq->first == NULL) page = (mi_page_t*)&_mi_page_empty; + if (pq->first == NULL) page = _mi_page_empty_get(); if mi_unlikely(theap->prof_force_slow) { // profiling: route all allocs through _mi_malloc_generic; lazy-disable here // (cold path) if the global rate has since gone to 0. if (_mi_prof_rate() == 0) { theap->prof_force_slow = false; theap->prof_countdown = 0; } - else { page = (mi_page_t*)&_mi_page_empty; } + else { page = _mi_page_empty_get(); } } // find index in the right direct page array @@ -427,38 +427,3 @@ static void mi_page_queue_enqueue_from_full(mi_page_queue_t* to, mi_page_queue_t // note: we could insert at the front to increase reuse, but it slows down certain benchmarks (like `alloc-test`) mi_page_queue_enqueue_from_ex(to, from, true /* enqueue at the end of the `to` queue? */, page); } - -// Only called from `mi_theap_absorb`. -size_t _mi_page_queue_append(mi_theap_t* theap, mi_page_queue_t* pq, mi_page_queue_t* append) { - mi_assert_internal(mi_theap_contains_queue(theap,pq)); - mi_assert_internal(pq->block_size == append->block_size); - - if (append->first==NULL) return 0; - - // set append pages to new theap and count - size_t count = 0; - for (mi_page_t* page = append->first; page != NULL; page = page->next) { - mi_page_set_theap(page, theap); - count++; - } - mi_assert_internal(count == append->count); - - if (pq->last==NULL) { - // take over afresh - mi_assert_internal(pq->first==NULL); - pq->first = append->first; - pq->last = append->last; - mi_theap_queue_first_update(theap, pq); - } - else { - // append to end - mi_assert_internal(pq->last!=NULL); - mi_assert_internal(append->first!=NULL); - pq->last->next = append->first; - append->first->prev = pq->last; - pq->last = append->last; - } - pq->count += append->count; - - return count; -} diff --git a/src/page.c b/src/page.c index 382d7d92c..321939ba9 100644 --- a/src/page.c +++ b/src/page.c @@ -60,14 +60,14 @@ void _mi_page_purged_reset(mi_page_t* page) { page->swept_state = MI_PAGE_SWEPT_NONE; // a fresh (or recycled) page was never swept } -// The block state of the page, as the sweep sees it (see `_mi_page_purge_holes`). The pack is -// lossless, and `MI_PAGE_SWEPT_NONE` impossible, only while both fields are 16-bit: -typedef char mi_page_sweep_state_fits[(sizeof(((mi_page_t*)0)->capacity) == 2 && - sizeof(((mi_page_t*)0)->used) == 2) ? 1 : -1]; +// The block state of the page, as the sweep sees it (see `_mi_page_purge_holes`): `(capacity << 32) | used`. Lossless +// while `capacity` fits 16 bits and `used` 32, and then `MI_PAGE_SWEPT_NONE` (all ones) is impossible: +typedef char mi_page_sweep_state_fits[(sizeof(((mi_page_t*)0)->capacity) <= 2 && + sizeof(((mi_page_t*)0)->used) <= 4) ? 1 : -1]; -static inline uint32_t mi_page_sweep_state(const mi_page_t* page) { +static inline uint64_t mi_page_sweep_state(const mi_page_t* page) { mi_assert_internal(page->used <= page->capacity); - return (((uint32_t)page->capacity) << 16) | (uint32_t)page->used; + return (((uint64_t)page->capacity) << 32) | (uint64_t)page->used; } // Anything that changes which blocks are free *without* changing `(capacity,used)` must say so, @@ -175,7 +175,7 @@ static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(page->heap!=NULL); mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); - mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap); + mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); // const size_t bsize = mi_page_block_size(page); // uint8_t* start = mi_page_start(page); @@ -239,7 +239,7 @@ bool _mi_page_is_valid(mi_page_t* page) { //mi_assert_internal(!_mi_process_is_initialized); mi_assert_internal(page->heap!=NULL); mi_theap_t* const page_theap = _mi_heap_theap_peek(page->heap); - mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap); + mi_assert_internal(page_theap == NULL || mi_page_theap(page)==page_theap || mi_page_theap(page)->tld->thread_id == MI_THREADID_DETACHED); { mi_page_queue_t* pq = mi_page_queue_of(page); mi_assert_internal(mi_page_queue_contains(pq, page)); @@ -272,7 +272,7 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) // if `count > max_count` there was a memory corruption (possibly infinite list due to double multi-threaded free) if mi_unlikely(count > max_count) { - _mi_error_message(EFAULT, "corrupted thread-free list\n"); + _mi_error_message(EFAULT, "corrupted thread-free list (possibly due to a cross-thread double free)\n"); return; // the thread-free items cannot be freed } // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list) @@ -310,8 +310,8 @@ static void mi_page_thread_free_collect(mi_page_t* page) } // returns `true` if after collection `mi_page_immediate_available` is true. -static bool mi_page_free_quick_collect(mi_page_t* page) { - if (page->free != NULL) return true; +static inline bool mi_page_free_quick_collect(mi_page_t* page) { + if mi_likely(page->free != NULL) return true; if (page->local_free == NULL) return false; // move local_free to free page->free = page->local_free; @@ -1284,7 +1284,7 @@ void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq) { mi_page_set_theap(page, NULL); page->theap = theap; // don't actually set theap to NULL so we can reclaim_on_free within the same theap _mi_arenas_page_abandon(page, theap); - _mi_arenas_collect(false, false, theap->tld); // allow purging + // _mi_arenas_collect(false, false, theap->tld); // allow purging } } @@ -1306,7 +1306,9 @@ static mi_page_t* mi_page_fresh_alloc(mi_theap_t* theap, mi_page_queue_t* pq, si if (!mi_page_immediate_available(page)) { if (mi_page_is_expandable(page)) { if (!mi_page_extend_free(theap, page)) { - return NULL; // cannot commit + // cannot commit + _mi_page_abandon(page,pq); + return NULL; }; } else { @@ -1395,10 +1397,9 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { mi_theap_t* theap = mi_page_theap(page); mi_assert_internal(theap!=NULL); mi_page_set_theap(page,NULL); _mi_arenas_page_free(page, theap); - _mi_arenas_collect(false, false, theap->tld); // allow purging + // _mi_arenas_collect(false, false, theap->tld); // allow purging } -#define MI_MAX_RETIRE_SIZE MI_LARGE_OBJ_SIZE_MAX // should be less than size for MI_BIN_HUGE #define MI_RETIRE_CYCLES (16) // Retire a page with no more used blocks @@ -1445,6 +1446,28 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { _mi_page_free(page, pq); } + +static void mi_theap_collect_full_pages(mi_theap_t* theap) { + // note: normally full pages get immediately abandoned and the full queue is always empty + // this path is only used if abandoning is disabled due to a destroy-able theap or options + // set by the user. + mi_page_queue_t* pq = &theap->pages[MI_BIN_FULL]; + for (mi_page_t* page = pq->first; page != NULL; ) { + mi_page_t* next = page->next; // get next in case we free the page + _mi_page_free_collect(page, false); // register concurrent free's + // no longer full? + if (!mi_page_is_full(page)) { + if (mi_page_all_free(page)) { + _mi_page_free(page, pq); + } + else { + _mi_page_unfull(page); + } + } + page = next; + } +} + // free retired pages: we don't need to look at the entire queues // since we only retire pages that are at the head position in a queue. void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { @@ -1472,30 +1495,12 @@ void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { } theap->page_retired_min = min; theap->page_retired_max = max; -} - -/* -static void mi_theap_collect_full_pages(mi_theap_t* theap) { - // note: normally full pages get immediately abandoned and the full queue is always empty - // this path is only used if abandoning is disabled due to a destroy-able theap or options - // set by the user. - mi_page_queue_t* pq = &theap->pages[MI_BIN_FULL]; - for (mi_page_t* page = pq->first; page != NULL; ) { - mi_page_t* next = page->next; // get next in case we free the page - _mi_page_free_collect(page, false); // register concurrent free's - // no longer full? - if (!mi_page_is_full(page)) { - if (mi_page_all_free(page)) { - _mi_page_free(page, pq); - } - else { - _mi_page_unfull(page); - } - } - page = next; + if (!theap->allow_page_abandon) { + mi_theap_collect_full_pages(theap); } } -*/ + + /* ----------------------------------------------------------- @@ -1509,7 +1514,7 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) { #define MI_MIN_SLICES (2) static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* const page, const size_t bsize, const size_t extend) { - #if (MI_SECURE<3) + #if (MI_SECURE < 2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); #endif @@ -1566,7 +1571,7 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, const size_t bsize, const size_t extend) { - #if (MI_SECURE<3) + #if (MI_SECURE < 2) mi_assert_internal(page->free == NULL); mi_assert_internal(page->local_free == NULL); #endif @@ -1607,7 +1612,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co // extra test in malloc? or cache effects?) static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { mi_assert_expensive(mi_page_is_valid_init(page)); - #if (MI_SECURE<3) + #if (MI_SECURE < 2) mi_assert(page->free == NULL); mi_assert(page->local_free == NULL); if (page->free != NULL) return true; @@ -1640,16 +1645,30 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(extend < (1UL<<16)); // commit on demand? - if (page->slice_committed > 0) { + const size_t slice_committed = mi_page_slice_committed(page); + if (slice_committed > 0) { + // reduce extend if it commits more than an arena slice + if ((extend * bsize) > MI_ARENA_SLICE_SIZE) { + extend = _mi_divide_up(MI_ARENA_SLICE_SIZE, bsize); + } + // commit required size const size_t needed_size = (page->capacity + extend)*bsize; - const size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), MI_PAGE_MIN_COMMIT_SIZE ); - if (needed_commit > page->slice_committed) { - mi_assert_internal(((needed_commit - page->slice_committed) % _mi_os_page_size()) == 0); - if (!_mi_os_commit(_mi_theap_subproc(theap), mi_page_slice_start(page) + page->slice_committed, needed_commit - page->slice_committed, NULL)) { + mi_assert_internal(needed_size <= page_size); + size_t needed_commit = _mi_align_up( mi_page_slice_offset_of(page, needed_size), mi_page_min_commit_size()); + #if MI_SECURE>=5 + // the previous alignup could extend the commit into the guard page; re-adjust if needed + const size_t page_size_commit = _mi_align_up( mi_page_slice_offset_of(page, page_size), _mi_os_page_size() ); + if (needed_commit > page_size_commit) { + needed_commit = page_size_commit; + } + #endif + if (needed_commit > slice_committed) { + mi_assert_internal(((needed_commit - slice_committed) % _mi_os_page_size()) == 0); + if (!_mi_os_commit(_mi_theap_subproc(theap), mi_page_slice_start(page) + slice_committed, needed_commit - slice_committed, NULL)) { return false; } - mi_assert_internal(needed_commit < UINT32_MAX); - page->slice_committed = (uint32_t)needed_commit; + mi_assert_internal(needed_commit <= UINT16_MAX * _mi_os_page_size()); + page->slice_pcommitted = (uint16_t)(needed_commit / _mi_os_page_size()); } } @@ -1821,7 +1840,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, if (page == NULL) { _mi_theap_collect_retired(theap, false); // perhaps make a page available - page = mi_page_fresh(theap, pq); + page = mi_page_fresh(theap, pq); mi_assert_internal(page == NULL || mi_page_immediate_available(page)); if (page == NULL && first_try) { // out-of-memory _or_ an abandoned page with free blocks was reclaimed, try once again @@ -1842,65 +1861,48 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, return page; } - - -// Find a page with free blocks of `size`. -static mi_page_t* mi_find_free_page(mi_theap_t* theap, mi_page_queue_t* pq) { - // mi_page_queue_t* pq = mi_page_queue(theap, size); +// Look for a page with free blocks of `size` but don't try to search or allocate +static inline mi_page_t* mi_page_queue_lookup_free_first(mi_theap_t* theap, mi_page_queue_t* pq) { mi_assert_internal(!mi_page_queue_is_huge(pq)); - - // check the first page: we even do this with candidate search or otherwise we re-search every time mi_page_t* page = pq->first; - if mi_likely(page != NULL && mi_page_free_quick_collect(page)) { + if mi_likely(page!=NULL && mi_page_free_quick_collect(page)) { + // fast path #if (MI_SECURE>=2) // in secure mode, we extend half the time to increase randomness if (page->capacity < page->reserved && ((_mi_theap_random_next(theap) & 1) == 1)) { (void)mi_page_extend_free(theap, page); // ok if this fails mi_assert_internal(mi_page_immediate_available(page)); } + #else + MI_UNUSED(theap); #endif page->retire_expire = 0; - return page; // fast path + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + return page; } else { - return mi_page_queue_find_free_ex(theap, pq, true); + return NULL; } } +// Find a page with free blocks of `size`. +static inline mi_page_t* mi_page_queue_find_free(mi_theap_t* theap, mi_page_queue_t* pq) { + // mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(!mi_page_queue_is_huge(pq)); -/* ----------------------------------------------------------- - Users can register a deferred free function called - when the `free` list is empty. Since the `local_free` - is separate this is deterministically called after - a certain number of allocations. ------------------------------------------------------------ */ - -// The program should only install a single deferred free handler before doing allocation. -static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers) -static _Atomic(void*) deferred_arg; - -void _mi_deferred_free(mi_theap_t* theap, bool force) { - // owner only: `heartbeat` and `recurse` are plain fields, and the handler is thread-affine - if (theap->tld != NULL && theap->tld->thread_id != _mi_thread_id()) return; - theap->heartbeat++; - mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free); - if (fun != NULL && !theap->tld->recurse) { - theap->tld->recurse = true; - void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); - fun(force, theap->heartbeat, arg); - theap->tld->recurse = false; - } -} - -void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { - mi_atomic_store_ptr_release(void,&deferred_arg, arg); - mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); + // check the first page: we even do this with candidate search or otherwise we re-search every time + mi_page_t* page = mi_page_queue_lookup_free_first(theap,pq); + if (page==NULL) { + page = mi_page_queue_find_free_ex(theap, pq, true); + if (page==NULL) return NULL; + } + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + return page; } - -/* ----------------------------------------------------------- - General allocation ------------------------------------------------------------ */ - // Huge pages contain just one block, and the segment contains just that page. // Huge pages are also use if the requested alignment is very large (> MI_BLOCK_ALIGNMENT_MAX) // so their size is not always `> MI_LARGE_OBJ_SIZE_MAX`. @@ -1929,7 +1931,6 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page return page; } - // Allocate a page // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept { @@ -1939,36 +1940,64 @@ static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_align return NULL; } mi_page_queue_t* pq = mi_page_queue(theap, (huge_alignment > 0 ? MI_LARGE_MAX_OBJ_SIZE+1 : size)); + mi_page_t* page; // huge allocation? if mi_unlikely(mi_page_queue_is_huge(pq) || req_size > MI_MAX_ALLOC_SIZE) { - return mi_huge_page_alloc(theap,size,huge_alignment,pq); + page = mi_huge_page_alloc(theap,size,huge_alignment,pq); } else { // otherwise find a page with free blocks in our size segregated queues #if MI_PADDING mi_assert_internal(size >= MI_PADDING_SIZE); #endif - return mi_find_free_page(theap, pq); + page = mi_page_queue_find_free(theap,pq); } + if (page==NULL) return NULL; + mi_assert_internal(mi_page_block_size(page) >= size); + mi_assert_internal(mi_page_immediate_available(page)); + mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); + mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + return page; } -// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. -// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. -// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for -// very large requested alignments in which case we use a huge singleton page. -// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit) -// to use 4 parameters which compiles better on msvc for the malloc fast path. -void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, size_t* usable) mi_attr_noexcept -{ - const bool zero = ((zero_huge_alignment & 1) != 0); - const size_t huge_alignment = (zero_huge_alignment & ~1); - #if !MI_THEAP_INITASNULL - mi_assert_internal(theap != NULL); - #endif +/* ----------------------------------------------------------- + Users can register a deferred free function called + when the `free` list is empty. Since the `local_free` + is separate this is deterministically called after + a certain number of allocations. +----------------------------------------------------------- */ - // initialize if necessary +// The program should only install a single deferred free handler before doing allocation. +static _Atomic(void*) deferred_free; // is `mi_deferred_free_fun*` (but some platforms don't support atomic function pointers) +static _Atomic(void*) deferred_arg; + +void _mi_deferred_free(mi_theap_t* theap, bool force) { + // owner only: `heartbeat` and `recurse` are plain fields, and the handler is thread-affine + if (theap->tld != NULL && theap->tld->thread_id != _mi_thread_id()) return; + theap->heartbeat++; + mi_deferred_free_fun* const fun = (mi_deferred_free_fun*)mi_atomic_load_ptr_acquire(void,&deferred_free); + if (fun != NULL && !theap->tld->recurse) { + theap->tld->recurse = true; + void* const arg = mi_atomic_load_ptr_acquire(void,&deferred_arg); + fun(force, theap->heartbeat, arg); + theap->tld->recurse = false; + } +} + +void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noexcept { + mi_atomic_store_ptr_release(void,&deferred_arg, arg); + mi_atomic_store_ptr_release(void,&deferred_free, (void*)fn); +} + + +/* ----------------------------------------------------------- + Admin +----------------------------------------------------------- */ + +static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) +{ if mi_unlikely(!mi_theap_is_initialized(theap)) { if (theap==&_mi_theap_empty_wrong) { // we were unable to allocate a theap for a first-class heap @@ -1981,22 +2010,48 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm mi_assert_internal(mi_theap_is_initialized(theap)); // do administrative tasks every N generic mallocs - if mi_unlikely(++theap->generic_count >= 1000) { + if mi_unlikely(theap->generic_count >= 1000) { theap->generic_collect_count += theap->generic_count; theap->generic_count = 0; - // call potential deferred free routines - _mi_deferred_free(theap, false); - // free retired pages - _mi_theap_collect_retired(theap, false); - - // collect every once in a while (10000 by default) + + // do a full theap collect every once in a while (10000 by default) const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); if (theap->generic_collect_count >= generic_collect) { theap->generic_collect_count = 0; mi_theap_collect(theap, false /* force? */); } + else { + // otherwise we do a mini-collect + _mi_deferred_free(theap, false); // call potential deferred free routines + _mi_theap_collect_retired(theap, false); // free retired pages + } + } + return theap; +} + +/* ----------------------------------------------------------- + Generic allocation +----------------------------------------------------------- */ + +// Heap profiling (fork): while profiling is on, the direct pages are poisoned so every allocation comes through the generic +// path, and this is checked at each of its exits; zero overhead on the real fast path. Gated on the global rate so a thread +// picks up profiling that another thread enabled. +static inline void mi_prof_note_generic_alloc(mi_theap_t* theap, mi_page_t* page, void* p) { + if mi_unlikely(p != NULL && _mi_prof_rate() != 0) { + if (theap->prof_countdown == 0) { _mi_prof_theap_lazy_enable(theap); } + const size_t bsize = mi_page_block_size(page); + if ((theap->prof_countdown -= (intptr_t)bsize) <= 0) { + _mi_prof_sample(theap, page, p, bsize); + } } +} +static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) +{ + // initialize if necessary + theap = mi_malloc_generic_admin(theap); + if (theap==NULL) return NULL; + // find (or allocate) a page of the right size mi_page_t* page = mi_find_page(theap, size, huge_alignment); if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more @@ -2016,19 +2071,11 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc) - if (usable!=NULL) { *usable = mi_page_usable_block_size(page); } + if (ppage!=NULL) { *ppage = page; } void* const p = _mi_page_malloc_zero(theap,page,size,zero); mi_assert_internal(p != NULL); - // heap profiling: only checked here (slow path); zero overhead on the fast path. - // gate on the global rate so any thread picks up profiling enabled elsewhere. - if mi_unlikely(_mi_prof_rate() != 0) { - if (theap->prof_countdown == 0) { _mi_prof_theap_lazy_enable(theap); } - const size_t bsize = mi_page_block_size(page); - if ((theap->prof_countdown -= (intptr_t)bsize) <= 0) { - _mi_prof_sample(theap, page, p, bsize); - } - } + mi_prof_note_generic_alloc(theap, page, p); // move full pages to the full queue if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) { @@ -2036,3 +2083,40 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm } return p; } + + +// Generic allocation routine if the fast path (`alloc.c:mi_page_malloc`) does not succeed. +// Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. +// The `huge_alignment` is normally 0 but is set to a multiple of MI_SLICE_SIZE for +// very large requested alignments in which case we use a huge singleton page. +// Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit) +// to use 4 parameters which compiles better on msvc for the malloc fast path. +void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept +{ + #if !MI_THEAP_INITASNULL + mi_assert_internal(theap != NULL); + #endif + const bool zero = ((zero_huge_alignment & 1) != 0); + const size_t huge_alignment = (zero_huge_alignment & ~1); + mi_page_t* page = NULL; + + // fast path objects that fit in a small page + if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (req_size < MI_SMALL_MAX_OBJ_SIZE) { + mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); + page = mi_page_queue_find_free(theap,pq); + // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + if (page!=NULL) { + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(mi_page_immediate_available(page)); + void* const p = _mi_page_malloc_zero(theap,page,size,zero); + mi_prof_note_generic_alloc(theap, page, p); + return p; + } + } + } + // otherwise fallback + return mi_malloc_generic_fallback(theap,size,zero, huge_alignment,ppage); +} diff --git a/src/prim/emscripten/prim.c b/src/prim/emscripten/prim.c index ad4f7bef5..1c5a87c77 100644 --- a/src/prim/emscripten/prim.c +++ b/src/prim/emscripten/prim.c @@ -61,7 +61,6 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config) { extern void emmalloc_free(void*); int _mi_prim_free(void* addr, size_t size) { - if (size==0) return 0; emmalloc_free(addr); return 0; } @@ -165,7 +164,8 @@ size_t _mi_prim_numa_node_count(void) { #include mi_msecs_t _mi_prim_clock_now(void) { - return emscripten_date_now(); + // todo: use a monotonic clock instead + return emscripten_date_now(); } diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index bc72148cf..201c15860 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -75,14 +75,17 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) { } static void zone_free(malloc_zone_t* zone, void* p) { - if mi_likely(mi_any_heap_contains(p)) { - if mi_likely(_mi_thread_is_initialized()) { - mi_free(p); // with the page_map and pagemap_commit=1 we can use the regular free - } - else { - // during thread shutdown `_pthread_tsd_cleanup` may call `zone_free` on a pointer that was allocated in another subproc. - _mi_free_subproc_safe(p); - } + mi_page_t* const page = _mi_checked_ptr_page(p); + if mi_likely(page!=NULL) { + mi_assert_internal(_mi_thread_is_initialized()); + _mi_free_in_page(p,page); + // if mi_likely(_mi_thread_is_initialized()) { + // _mi_free_in_page(p,page); + // } + // else { + // // during thread shutdown `_pthread_tsd_cleanup` may call `zone_free` on a pointer that was allocated in another subproc. + // _mi_free_subproc_safe_in_page(p,page); + // } } else if (!is_mimalloc_zone(zone)) { // can happen due to interpose zone->free(zone,p); @@ -225,8 +228,8 @@ static kern_return_t mi_zone_enum_remote_page(task_t task, memory_reader_t reade const size_t ubsize = bsize - MI_PADDING_SIZE; // `lpage` is a local copy of a page that lives in the TARGET process, so the block start // must be rebuilt from the remote page address -- mi_page_start(&lpage) would offset from - // our own copy. (Upstream dropped the cached `page_start` field in favour of `page_woffset`.) - const vm_address_t pstart = (vm_address_t)(rpage + ((size_t)lpage.page_woffset * MI_SIZE_SIZE)); + // our own copy. (The start is stored as an offset from the page struct, in `MI_MAX_ALIGN_SIZE` units.) + const vm_address_t pstart = (vm_address_t)(rpage + ((size_t)lpage.page_ma_offset * MI_MAX_ALIGN_SIZE)); if (e->type_mask & MALLOC_PTR_REGION_RANGE_TYPE) { mi_zone_enum_flush(e, MALLOC_PTR_IN_USE_RANGE_TYPE); diff --git a/src/prim/prim-tls.c b/src/prim/prim-tls.c new file mode 100644 index 000000000..8c8b0c71f --- /dev/null +++ b/src/prim/prim-tls.c @@ -0,0 +1,252 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim.h" +#include "mimalloc/prim-tls.h" + +// -------------------------------------------------------------------------- +// Implement fast access to the thread local storage for `_mi_theap_default()` +// and `mi_theap_cached()`. See `include/mimalloc/prim-tls.h` for more info +// on the TLS models. +// -------------------------------------------------------------------------- + +void _mi_tls_slots_init(void); +void _mi_tls_slots_done(void); +void _mi_theap_default_set(mi_theap_t* theap); +void _mi_theap_cached_set(mi_theap_t* theap); + + +#if MI_TLS_MODEL_LOCAL +// the thread-local main theap for allocation +mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_default = (mi_theap_t*)&_mi_theap_empty; +// the last used non-main theap +mi_decl_hidden mi_decl_thread mi_theap_t* __mi_theap_cached = (mi_theap_t*)&_mi_theap_empty; +#endif + +mi_decl_hidden mi_decl_thread void* __mi_thread_id_helper = NULL; + +mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { + const mi_threadid_t tid = _mi_prim_thread_id(); + mi_assert_internal( (tid & MI_PAGE_FLAG_MASK) == 0 ); // mimalloc reserves the bottom 2 bits + return tid; +} + +// ---------------------------------------------------------------------------- +// Setting the default and cached theap +// ---------------------------------------------------------------------------- + +#if MI_TLS_MODEL_WIN32 + +// If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed) +// See for the offsets. +#if MI_SIZE_SIZE==4 +#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_INTPTR_SIZE) +#else +#define MI_TLS_DIRECT_FIRST (0x1480 / MI_INTPTR_SIZE) +#endif +#define MI_TLS_DIRECT_SLOTS (64) +#define MI_TLS_EXPANSION_SLOTS (1024) + +#if !MI_WIN_DIRECT_TLS +// We initially use the last of the expansion slots as the default NULL. +// note: this will fail if the program allocates exactly 1024+64 slots with TlsAlloc +// before we are initialized :-( (but this seems quite unlikely). +// (todo: another approach could be to use slot 7 (EnvironmentPointer) as the initial slot as that seems to be always NULL) +#define MI_TLS_INITIAL_SLOT MI_TLS_EXPANSION_SLOT +#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) +#else +// With direct tls we need an initial NULL slot outside the expansion slots +#define MI_TLS_INITIAL_SLOT (5) // Arbitrary user pointer +#define MI_TLS_INITIAL_EXPANSION_SLOT (MI_TLS_EXPANSION_SLOTS-1) // unused +#endif + +// in case of errors assign fixed slots (but since we use EFAULT the program should fail anyways) +#define MI_TLS_ERROR_SLOT (5) // arbitrary user pointer +#define MI_TLS_ERROR_EXPANSION_SLOT (7) // environment pointer (only used for OS/2 emulation) + + +mi_decl_hidden mi_decl_cache_align _Atomic(size_t) _mi_theap_default_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT); +mi_decl_hidden _Atomic(size_t) _mi_theap_default_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT); +mi_decl_hidden _Atomic(size_t) _mi_theap_cached_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_SLOT); +mi_decl_hidden _Atomic(size_t) _mi_theap_cached_expansion_slot = MI_ATOMIC_VAR_INIT(MI_TLS_INITIAL_EXPANSION_SLOT); + +static DWORD mi_tls_raw_index_default = TLS_OUT_OF_INDEXES; +static DWORD mi_tls_raw_index_cached = TLS_OUT_OF_INDEXES; + +static bool mi_win_tls_slot_alloc(_Atomic(size_t)* slot, _Atomic(size_t)* extended, DWORD* raw_index) { + // always write slot before extended due to concurrent readers + const DWORD index = TlsAlloc(); + *raw_index = index; + if (index==TLS_OUT_OF_INDEXES) { + mi_atomic_store_release(slot,MI_TLS_ERROR_SLOT); + mi_atomic_store_release(extended,MI_TLS_ERROR_EXPANSION_SLOT); + return false; + } + else if (index= MI_TLS_DIRECT_FIRST && slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) || slot == MI_TLS_EXPANSION_SLOT); + if (slot < MI_TLS_DIRECT_FIRST + MI_TLS_DIRECT_SLOTS) { + mi_prim_tls_slot_set(slot, value); + } + else { + mi_assert_internal(extended_slot < MI_TLS_EXPANSION_SLOTS); + TlsSetValue((DWORD)(extended_slot + MI_TLS_DIRECT_SLOTS), value); // use TlsSetValue to initialize the TlsExpansion array if needed + } +} + +#elif MI_TLS_MODEL_PTHREADS + +// only for pthreads for now +mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_default_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID); +mi_decl_hidden _Atomic(pthread_key_t) _mi_theap_cached_key = MI_ATOMIC_VAR_INIT(MI_PTHREAD_KEY_INVALID); + +static void mi_theap_cached_key_destroy(void* theapv) { + mi_theap_t* theap = (mi_theap_t*)theapv; + if (theap!=NULL) { + _mi_theap_decref(theap); + } +} + +void _mi_tls_slots_init(void) { + mi_atomic_do_once { + pthread_key_t key; + if (_mi_pthread_key_create(&key,NULL,NULL)) { mi_atomic_store_release(&_mi_theap_default_key,key); } + if (_mi_pthread_key_create(&key,&mi_theap_cached_key_destroy,NULL)) { mi_atomic_store_release(&_mi_theap_cached_key,key); } + } +} + +void _mi_tls_slots_done(void) { + pthread_key_t key = mi_atomic_exchange_relaxed(&_mi_theap_default_key,MI_PTHREAD_KEY_INVALID); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); } + key = mi_atomic_exchange_relaxed(&_mi_theap_cached_key,MI_PTHREAD_KEY_INVALID); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_key_delete(key); } +} + +#elif MI_TLS_MODEL_FIXED + +void _mi_tls_slots_init(void) { + mi_atomic_do_once { + mi_theap_t* theap = _mi_theap_default(); + if (theap!=NULL) { + _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_DEFAULT, theap); + } + theap = _mi_theap_cached(); + if (theap!=NULL) { + _mi_error_message(EINVAL,"fixed TLS slot is already in use (slot %d = %p)", MI_TLS_MODEL_FIXED_CACHED, theap); + } + } +} + +void _mi_tls_slots_done(void) { + // nothing +} + + +#else + +void _mi_tls_slots_init(void) { + // nothing +} + +void _mi_tls_slots_done(void) { + // nothing +} + +#endif + +void _mi_theap_cached_set(mi_theap_t* theap) { + mi_theap_t* prev = _mi_theap_cached(); + if (prev==theap) return; + // set + _mi_tls_slots_init(); + #if MI_TLS_MODEL_LOCAL + __mi_theap_cached = theap; + #elif MI_TLS_MODEL_FIXED + mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_CACHED, theap); + #elif MI_TLS_MODEL_WIN32 + mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_cached_slot), mi_atomic_load_relaxed(&_mi_theap_cached_expansion_slot), theap); + #elif MI_TLS_MODEL_PTHREADS + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_cached_key); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); } + #endif + // update refcounts (so cached theap memory keeps available until no longer cached) + _mi_theap_incref(theap); + _mi_theap_decref(prev); +} + +void _mi_theap_default_set(mi_theap_t* theap) { + mi_assert_internal(theap != NULL); + mi_assert_internal(theap->tld != NULL); + mi_assert_internal(mi_theap_matches_thread(theap)); + _mi_tls_slots_init(); + #if MI_TLS_MODEL_LOCAL + __mi_theap_default = theap; + #elif MI_TLS_MODEL_FIXED + mi_prim_tls_slot_set(MI_TLS_MODEL_FIXED_DEFAULT, theap); + #elif MI_TLS_MODEL_WIN32 + mi_win_tls_slot_set(mi_atomic_load_relaxed(&_mi_theap_default_slot), mi_atomic_load_relaxed(&_mi_theap_default_expansion_slot), theap); + #elif MI_TLS_MODEL_PTHREADS + pthread_key_t key = mi_atomic_load_relaxed(&_mi_theap_default_key); + if (key!=MI_PTHREAD_KEY_INVALID) { pthread_setspecific(key, theap); } + #endif + + // set theap main if needed + if (mi_theap_is_initialized(theap)) { + // ensure the default theap is passed to `_mi_thread_done` as on some platforms we cannot access TLS at thread termination (as it would allocate again) + _mi_prim_thread_associate_default_theap(theap); + } +} diff --git a/src/prim/prim.c b/src/prim/prim.c index 533557ab9..d7912dae7 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2023, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. diff --git a/src/prim/readme.md b/src/prim/readme.md index 380dd3a71..02cb1b90c 100644 --- a/src/prim/readme.md +++ b/src/prim/readme.md @@ -6,4 +6,7 @@ This is the portability layer where all primitives needed from the OS are define - `prim.c`: Selects one of `unix/prim.c`, `wasi/prim.c`, or `windows/prim.c` depending on the host platform (and on macOS, `osx/prim.c` defers to `unix/prim.c`). +- `include/mimalloc/prim-tls.h`: primitive TLS model (thread local, pthreads, etc) +- `prim-tls.c`: TLS model implementation + Note: still work in progress, there may still be places in the sources that still depend on OS ifdef's. \ No newline at end of file diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index ca99b51fd..83e1add7e 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -288,7 +288,7 @@ int _mi_prim_free(void* addr, size_t size ) { // return errno on failure static int unix_madvise(void* addr, size_t size, int advice) { - #if defined(__sun) + #if defined(__sun) || defined(_AIX) const int res = madvise((caddr_t)addr, size, advice); // Solaris needs cast (issue #520) return (res==0 ? 0 : errno); #elif defined(__QNX__) @@ -370,6 +370,10 @@ static int unix_mmap_fd(void) { #endif } +#if defined(MAP_ALIGNED_SUPER) || defined(MAP_HUGETLB) || defined(MAP_HUGE_1GB) || defined(MAP_HUGE_2MB) || defined(VM_FLAGS_SUPERPAGE_SIZE_2MB) +#define MI_OS_HAS_HUGE_PAGES 1 +#endif + static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protect_flags, bool large_only, bool allow_large, bool* is_large) { #if !defined(MAP_ANONYMOUS) #define MAP_ANONYMOUS MAP_ANON @@ -387,6 +391,7 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec protect_flags |= PROT_MAX(PROT_READ | PROT_WRITE); // BSD #endif // huge page allocation + #if MI_OS_HAS_HUGE_PAGES if (allow_large && (large_only || (_mi_os_canuse_large_page(size, try_alignment) && mi_option_is_enabled(mi_option_allow_large_os_pages)))) { static _Atomic(size_t) large_page_try_ok; // = 0; size_t try_ok = mi_atomic_load_acquire(&large_page_try_ok); @@ -441,7 +446,10 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec } } } - } + } // huge pages + #else + MI_UNUSED(large_only); + #endif // regular allocation if (p == NULL) { *is_large = false; @@ -702,7 +710,7 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo bool is_large = true; *is_zero = true; *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large); - if (*addr != NULL && numa_node >= 0 && numa_node < 8*MI_INTPTR_SIZE) { // at most 64 nodes + if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_INTPTR_SIZE - 1)) { // at most 63 nodes unsigned long numa_mask = (1UL << numa_node); // todo: does `mbind` work correctly for huge OS pages? should we // use `set_mempolicy` before calling mmap instead? @@ -735,9 +743,9 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo size_t _mi_prim_numa_node(void) { #if defined(MI_HAS_SYSCALL_H) && defined(SYS_getcpu) - unsigned long node = 0; - unsigned long ncpu = 0; - long err = syscall(SYS_getcpu, &ncpu, &node, NULL); + unsigned int node = 0; + unsigned int ncpu = 0; + int err = syscall(SYS_getcpu, &ncpu, &node, NULL); if (err != 0) return 0; return node; #else @@ -747,13 +755,16 @@ size_t _mi_prim_numa_node(void) { size_t _mi_prim_numa_node_count(void) { char buf[128]; - unsigned node = 0; - for(node = 0; node < 256; node++) { + unsigned last_found = 1; + for(unsigned node = 1; node < 256; node++) { // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation) - _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node + 1); - if (mi_prim_access(buf,R_OK) != 0) break; + _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node); + if (mi_prim_access(buf,R_OK) != 0) { + if (node - last_found > 4) break; // allow some sparseness of nodes but not more than 4 + } + else { last_found = node; } // highest found node } - return (node+1); + return last_found; } #elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 @@ -867,66 +878,68 @@ static mi_msecs_t timeval_secs(const struct timeval* tv) { void _mi_prim_process_info(mi_process_info_t* pinfo) { struct rusage rusage; - getrusage(RUSAGE_SELF, &rusage); - pinfo->utime = timeval_secs(&rusage.ru_utime); - pinfo->stime = timeval_secs(&rusage.ru_stime); -#if !defined(__HAIKU__) - pinfo->page_faults = rusage.ru_majflt; -#endif -#if defined(__HAIKU__) - // Haiku does not have (yet?) a way to - // get these stats per process - thread_info tid; - area_info mem; - ssize_t c; - get_thread_info(find_thread(0), &tid); - while (get_next_area_info(tid.team, &c, &mem) == B_OK) { - pinfo->peak_rss += mem.ram_size; - } - pinfo->page_faults = 0; -#elif defined(__APPLE__) - pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes - #ifdef MACH_TASK_BASIC_INFO - struct mach_task_basic_info info; - mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; - if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { - pinfo->current_rss = (size_t)info.resident_size; - } - #else - struct task_basic_info info; - mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT; - if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { - pinfo->current_rss = (size_t)info.resident_size; - } - #endif -#else - pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB - #if defined(__linux__) - // Without this `current_rss` keeps its default, which is `current_commit` -- so anything - // reporting "rss" on Linux prints committed bytes instead. That is not a rounding error: - // a purge that keeps a range committed does not lower commit, so the two diverge by - // gigabytes (a Bun crash report read "RSS: 4.91 GB" against a real 0.56 GB peak). - { - int fd = mi_prim_open("/proc/self/statm", O_RDONLY); - if (fd >= 0) { - char buf[64]; - const ssize_t nread = mi_prim_read(fd, buf, sizeof(buf) - 1); - mi_prim_close(fd); - if (nread > 0) { - buf[nread] = 0; - // "size resident shared text lib data dt" -- all in pages; we want the 2nd field - const char* p = buf; - while (*p == ' ') { p++; } - while (*p != 0 && *p != ' ') { p++; } // skip `size` - while (*p == ' ') { p++; } - size_t resident_pages = 0; - while (*p >= '0' && *p <= '9') { resident_pages = (resident_pages * 10) + (size_t)(*p - '0'); p++; } - if (resident_pages > 0) { pinfo->current_rss = resident_pages * _mi_os_page_size(); } + if (getrusage(RUSAGE_SELF, &rusage) == 0) { + pinfo->utime = timeval_secs(&rusage.ru_utime); + pinfo->stime = timeval_secs(&rusage.ru_stime); + #if !defined(__HAIKU__) + pinfo->page_faults = rusage.ru_majflt; + #endif + #if defined(__APPLE__) + pinfo->peak_rss = rusage.ru_maxrss; // macos reports in bytes + #else + pinfo->peak_rss = rusage.ru_maxrss * 1024; // Linux/BSD report in KiB + #endif + #if defined(__linux__) + // Without this `current_rss` keeps its default, which is `current_commit` -- so anything reporting "rss" on Linux prints + // committed bytes instead; a purge that keeps a range committed does not lower commit, so the two diverge by gigabytes. + { + int fd = mi_prim_open("/proc/self/statm", O_RDONLY); + if (fd >= 0) { + char buf[64]; + const ssize_t nread = mi_prim_read(fd, buf, sizeof(buf) - 1); + mi_prim_close(fd); + if (nread > 0) { + buf[nread] = 0; + const char* q = buf; // "size resident ..." in pages; the 2nd field + while (*q == ' ') { q++; } + while (*q != 0 && *q != ' ') { q++; } + while (*q == ' ') { q++; } + size_t resident_pages = 0; + while (*q >= '0' && *q <= '9') { resident_pages = (resident_pages * 10) + (size_t)(*q - '0'); q++; } + if (resident_pages > 0) { pinfo->current_rss = resident_pages * _mi_os_page_size(); } + } } } + #endif } + + #if defined(__HAIKU__) + // Haiku does not have (yet?) a way to + // get these stats per process + thread_info tid; + if (get_thread_info(find_thread(0), &tid) == B_OK) { + area_info mem; + ssize_t c; + while (get_next_area_info(tid.team, &c, &mem) == B_OK) { + pinfo->peak_rss += mem.ram_size; + } + } + pinfo->page_faults = 0; + #elif defined(__APPLE__) + #ifdef MACH_TASK_BASIC_INFO + struct mach_task_basic_info info; + mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { + pinfo->current_rss = (size_t)info.resident_size; + } + #else + struct task_basic_info info; + mach_msg_type_number_t infoCount = TASK_BASIC_INFO_COUNT; + if (task_info(mach_task_self(), TASK_BASIC_INFO, (task_info_t)&info, &infoCount) == KERN_SUCCESS) { + pinfo->current_rss = (size_t)info.resident_size; + } + #endif #endif -#endif // use defaults for commit } @@ -984,8 +997,8 @@ int _mi_prim_getenv(const char* name, char* result, size_t result_size) { const char* s = env[i]; if (_mi_strnicmp(name, s, len) == 0 && s[len] == '=') { // case insensitive // found it - _mi_strlcpy(result, s + len + 1, result_size); - return 1; // success + if (!_mi_strlcpy(result, s + len + 1, result_size)) return -1; + return 1; // success } } return 0; // not found @@ -1006,8 +1019,8 @@ int _mi_prim_getenv(const char* name, char* result, size_t result_size) { buf[len] = 0; s = getenv(buf); } - if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found - _mi_strlcpy(result, s, result_size); + if (s == NULL || _mi_strnlen(s,result_size) >= result_size) return 0; // not found + if (!_mi_strlcpy(result, s, result_size)) return -1; return 1; // success } #endif // !MI_USE_ENVIRON @@ -1037,7 +1050,7 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { return true; } -#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) // also for old apple versions < 10.7 (issue #829) +#elif defined(__APPLE__) || defined(__linux__) || defined(__HAIKU__) || defined(__CYGWIN__) // also for old apple versions < 10.7 (issue #829) #include #include @@ -1069,7 +1082,10 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { size_t count = 0; while(count < buf_len) { ssize_t ret = mi_prim_read(fd, (char*)buf + count, buf_len - count); - if (ret<=0) { + if (ret==0) { + break; + } + else if (ret<0) { if (errno!=EAGAIN && errno!=EINTR) break; } else { diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 8a0a332d5..5a82a92e8 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -13,9 +13,9 @@ terms of the MIT license. A copy of the license can be found in the file #include // fputs, stderr #include // atexit -// xbox has no console IO -#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP | WINAPI_PARTITION_SYSTEM) -#define MI_HAS_CONSOLE_IO +// xbox has no console IO and cannot use LoadLibrary or GetModuleHandle +#if !defined(WINAPI_FAMILY_PARTITION) || WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP | WINAPI_PARTITION_SYSTEM) +#define MI_WIN_DESKTOP 1 #endif //--------------------------------------------- @@ -87,6 +87,34 @@ typedef BOOL (__stdcall *PGetPhysicallyInstalledSystemMemory)( PULONGLONG TotalM typedef BOOL (__stdcall* PGetVersionExW)(LPOSVERSIONINFOW lpVersionInformation); +// Load a library +static HMODULE mi_win_loadlibrary(const TCHAR* library) { + #if MI_WIN_DESKTOP + return LoadLibrary(library); + #else + return LoadPackagedLibrary(library, 0); + #endif +} + +// Get a library handle (and possibly load it) +static HMODULE mi_win_getlibrary(const TCHAR* library, bool* should_free) { + #if MI_WIN_DESKTOP + // avoid calling LoadLibrary for "kernel32", "ntdll", and "kernelbase" (also to avoid hitting the loader lock) + HMODULE mod = GetModuleHandle(library); + if (mod!=NULL) { + *should_free = false; + return mod; + } + #endif + *should_free = true; + return mi_win_loadlibrary(library); +} + +static void mi_win_freelibrary(HMODULE mod, bool should_free) { + if (should_free) { + FreeLibrary(mod); + } +} //--------------------------------------------- // Enable large page support dynamically (if possible) @@ -103,15 +131,17 @@ static bool win_enable_large_os_pages_once(size_t* large_page_size) unsigned long err = 0; HANDLE token = NULL; BOOL ok = OpenProcessToken(GetCurrentProcess(), TOKEN_ADJUST_PRIVILEGES | TOKEN_QUERY, &token); + err = GetLastError(); if (ok) { TOKEN_PRIVILEGES tp; ok = LookupPrivilegeValue(NULL, TEXT("SeLockMemoryPrivilege"), &tp.Privileges[0].Luid); + err = GetLastError(); if (ok) { tp.PrivilegeCount = 1; tp.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED; ok = AdjustTokenPrivileges(token, FALSE, &tp, 0, (PTOKEN_PRIVILEGES)NULL, 0); + err = GetLastError(); if (ok) { - err = GetLastError(); ok = (err == ERROR_SUCCESS); if (ok && large_page_size != NULL && pGetLargePageMinimum != NULL) { *large_page_size = (*pGetLargePageMinimum)(); @@ -121,17 +151,19 @@ static bool win_enable_large_os_pages_once(size_t* large_page_size) CloseHandle(token); } if (!ok) { - if (err == 0) err = GetLastError(); + if (err == 0) { err = GetLastError(); } _mi_warning_message("cannot enable large OS page support, error %lu\n", err); } return (ok!=0); } static bool win_enable_large_os_pages(size_t* large_page_size) { + static size_t win_large_page_size = 0; mi_atomic_do_once { - win_enable_large_os_pages_once(large_page_size); + win_enable_large_os_pages_once(&win_large_page_size); } - return (_mi_os_large_page_size() > 0); + if (large_page_size != NULL) { *large_page_size = win_large_page_size; } + return (win_large_page_size > 0); } @@ -162,21 +194,22 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) } // get the VirtualAlloc2 function - HINSTANCE hDll = LoadLibrary(TEXT("kernelbase.dll")); + bool hDllFree; + HINSTANCE hDll = mi_win_getlibrary(TEXT("kernelbase.dll"), &hDllFree); if (hDll != NULL) { // use VirtualAlloc2FromApp if possible as it is available to Windows store apps pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2FromApp"); if (pVirtualAlloc2==NULL) pVirtualAlloc2 = (PVirtualAlloc2)(void (*)(void))GetProcAddress(hDll, "VirtualAlloc2"); - FreeLibrary(hDll); + mi_win_freelibrary(hDll, hDllFree); } // NtAllocateVirtualMemoryEx is used for huge page allocation - hDll = LoadLibrary(TEXT("ntdll.dll")); + hDll = mi_win_getlibrary(TEXT("ntdll.dll"), &hDllFree); if (hDll != NULL) { pNtAllocateVirtualMemoryEx = (PNtAllocateVirtualMemoryEx)(void (*)(void))GetProcAddress(hDll, "NtAllocateVirtualMemoryEx"); - FreeLibrary(hDll); + mi_win_freelibrary(hDll, hDllFree); } // Try to use Win7+ numa API - hDll = LoadLibrary(TEXT("kernel32.dll")); + hDll = mi_win_getlibrary(TEXT("kernel32.dll"), &hDllFree); if (hDll != NULL) { pGetCurrentProcessorNumberEx = (PGetCurrentProcessorNumberEx)(void (*)(void))GetProcAddress(hDll, "GetCurrentProcessorNumberEx"); pGetNumaProcessorNodeEx = (PGetNumaProcessorNodeEx)(void (*)(void))GetProcAddress(hDll, "GetNumaProcessorNodeEx"); @@ -205,7 +238,7 @@ void _mi_prim_mem_init( mi_os_mem_config_t* config ) win_minor_version = version.dwMinorVersion; } } - FreeLibrary(hDll); + mi_win_freelibrary(hDll, hDllFree); } // Enable large/huge OS page support? if (mi_option_is_enabled(mi_option_allow_large_os_pages) || mi_option_is_enabled(mi_option_reserve_huge_os_pages)) { @@ -228,8 +261,9 @@ int _mi_prim_free(void* addr, size_t size ) { // the memory region returned by VirtualAlloc; in that case we need to free using // the start of the region. MEMORY_BASIC_INFORMATION info; _mi_memzero_var(info); - VirtualQuery(addr, &info, sizeof(info)); - if (info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) { + err = (VirtualQuery(addr, &info, sizeof(info)) == 0); + if (err) { errcode = GetLastError(); } + if (!err && info.AllocationBase < addr && ((uint8_t*)addr - (uint8_t*)info.AllocationBase) < (ptrdiff_t)(4*MI_MiB)) { errcode = 0; err = (VirtualFree(info.AllocationBase, 0, MEM_RELEASE) == 0); if (err) { errcode = GetLastError(); } @@ -542,8 +576,9 @@ static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { static LARGE_INTEGER mfreq; // = 0 if (mfreq.QuadPart == 0LL) { LARGE_INTEGER f; - QueryPerformanceFrequency(&f); - mfreq.QuadPart = f.QuadPart/1000LL; + if (QueryPerformanceFrequency(&f)) { + mfreq.QuadPart = f.QuadPart/1000LL; + } if (mfreq.QuadPart == 0) mfreq.QuadPart = 1; } return (mi_msecs_t)(t.QuadPart / mfreq.QuadPart); @@ -551,8 +586,12 @@ static mi_msecs_t mi_to_msecs(LARGE_INTEGER t) { mi_msecs_t _mi_prim_clock_now(void) { LARGE_INTEGER t; - QueryPerformanceCounter(&t); - return mi_to_msecs(t); + if (QueryPerformanceCounter(&t)) { + return mi_to_msecs(t); + } + else { + return 0; + } } @@ -579,29 +618,31 @@ void _mi_prim_process_info(mi_process_info_t* pinfo) FILETIME ut; FILETIME st; FILETIME et; - GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut); - pinfo->utime = filetime_msecs(&ut); - pinfo->stime = filetime_msecs(&st); + if (GetProcessTimes(GetCurrentProcess(), &ct, &et, &st, &ut)) { + pinfo->utime = filetime_msecs(&ut); + pinfo->stime = filetime_msecs(&st); + } // load psapi on demand - mi_atomic_do_once { - HINSTANCE hDll = LoadLibrary(TEXT("psapi.dll")); + mi_atomic_do_once{ + HINSTANCE hDll = mi_win_loadlibrary(TEXT("psapi.dll")); if (hDll != NULL) { pGetProcessMemoryInfo = (PGetProcessMemoryInfo)(void (*)(void))GetProcAddress(hDll, "GetProcessMemoryInfo"); - // FreeLibrary(hDll); // don't free + // mi_win_freelibrary(hDll, true); // don't free } } // get process info - PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info); if (pGetProcessMemoryInfo != NULL) { - pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); + PROCESS_MEMORY_COUNTERS info; _mi_memzero_var(info); + if (pGetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info))) { + pinfo->current_rss = (size_t)info.WorkingSetSize; + pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; + pinfo->current_commit = (size_t)info.PagefileUsage; + pinfo->peak_commit = (size_t)info.PeakPagefileUsage; + pinfo->page_faults = (size_t)info.PageFaultCount; + } } - pinfo->current_rss = (size_t)info.WorkingSetSize; - pinfo->peak_rss = (size_t)info.PeakWorkingSetSize; - pinfo->current_commit = (size_t)info.PagefileUsage; - pinfo->peak_commit = (size_t)info.PeakPagefileUsage; - pinfo->page_faults = (size_t)info.PageFaultCount; } //---------------------------------------------------------------- @@ -613,32 +654,37 @@ void _mi_prim_out_stderr( const char* msg ) // on windows with redirection, the C runtime cannot handle locale dependent output // after the main thread closes so we use direct console output. if (!_mi_preloading()) { - // _cputs(msg); // _cputs cannot be used as it aborts when failing to lock the console static HANDLE hcon = INVALID_HANDLE_VALUE; + #if MI_WIN_DESKTOP static bool hconIsConsole = false; + #endif if (hcon == INVALID_HANDLE_VALUE) { - hcon = GetStdHandle(STD_ERROR_HANDLE); - #ifdef MI_HAS_CONSOLE_IO + hcon = GetStdHandle(STD_ERROR_HANDLE); // returns NULL if no stderr is available + #if MI_WIN_DESKTOP + if (hcon==NULL) { + AttachConsole(ATTACH_PARENT_PROCESS); // if started from a parent console, try to attach to that + hcon = GetStdHandle(STD_ERROR_HANDLE); + } CONSOLE_SCREEN_BUFFER_INFO sbi; - hconIsConsole = ((hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); + hconIsConsole = ((hcon != NULL && hcon != INVALID_HANDLE_VALUE) && GetConsoleScreenBufferInfo(hcon, &sbi)); #endif } const size_t len = _mi_strlen(msg); if (len > 0 && len < UINT32_MAX) { DWORD written = 0; - if (hconIsConsole) { - #ifdef MI_HAS_CONSOLE_IO - WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL); + if (hcon != NULL && hcon != INVALID_HANDLE_VALUE) { + #if MI_WIN_DESKTOP + if (hconIsConsole) { + WriteConsoleA(hcon, msg, (DWORD)len, &written, NULL); + } + else #endif + { + // use direct write in case stderr was redirected + WriteFile(hcon, msg, (DWORD)len, &written, NULL); + } } - else if (hcon != INVALID_HANDLE_VALUE) { - // use direct write if stderr was redirected - WriteFile(hcon, msg, (DWORD)len, &written, NULL); - } - else { - // finally fall back to fputs after all - fputs(msg, stderr); - } + // don't fall back to fputs or _cputs as the crt can have it locked } } } @@ -690,10 +736,10 @@ bool _mi_prim_random_buf(void* buf, size_t buf_len) { mi_assert(buf_len <= ULONG_MAX); if (buf_len > ULONG_MAX) return false; mi_atomic_do_once { - HINSTANCE hDll = LoadLibrary(TEXT("bcrypt.dll")); + HINSTANCE hDll = mi_win_loadlibrary(TEXT("bcrypt.dll")); if (hDll != NULL) { pBCryptGenRandom = (PBCryptGenRandom)(void (*)(void))GetProcAddress(hDll, "BCryptGenRandom"); - // FreeLibrary(hDll); // don't free + // mi_win_freelibrary(hDll); // don't free } } if (pBCryptGenRandom == NULL) return false; @@ -754,15 +800,18 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { both static and dynamic linkage (`MI_WIN_INIT_USE_CRT_TLS`). ------------------------------------------------------------------------- */ #if !defined(MI_WIN_INIT_USE_CRT_TLS) && !defined(MI_WIN_INIT_USE_RAW_DLLMAIN) && !defined(MI_WIN_INIT_USE_TLS_DLLMAIN) && !defined(MI_WIN_INIT_USE_FLS) - #if !defined(__INTEL_LLVM_COMPILER) && !defined(__INTEL_COMPILER) - #define MI_WIN_INIT_USE_CRT_TLS 1 + #if defined(__INTEL_LLVM_COMPILER) || defined(__INTEL_COMPILER) + #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* needed for Intel ICX, see issue #1268 */ #else - #define MI_WIN_INIT_USE_TLS_DLLMAIN 1 /* default for Intel ICX, see issue #1268 */ + #define MI_WIN_INIT_USE_CRT_TLS 1 /* default */ #endif #endif #if defined(MI_WIN_INIT_USE_CRT_TLS) - #define MI_PRIM_HAS_PROCESS_ATTACH 1 + #if !defined(__MINGW32__) || defined(MI_MINGW_UCRT64) // on mingw without UCRT use the constructor attribute (in `src/prim/prim.c`) + #define MI_PRIM_HAS_PROCESS_ATTACH 1 + #endif + // nothing to do since `_mi_thread_done` is handled through the DLL_THREAD_DETACH event. void _mi_prim_thread_init_auto_done(void) {} void _mi_prim_thread_done_auto_done(void) {} @@ -835,10 +884,12 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #endif typedef int (mi_cdecl* mi_crt_callback_t)(void); - #if defined(_WIN64) + + #if defined(_WIN64) && defined(_MSC_VER) // 64-bit #pragma comment(linker, "/INCLUDE:_tls_used") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") + #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init") #pragma const_seg(".CRT$XLB") extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[]; const PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach }; @@ -851,10 +902,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern const mi_crt_callback_t _mi_crt_callback_init[]; const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; #pragma const_seg() - #else + #elif defined(_MSC_VER) // 32-bit #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") + #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init") #pragma data_seg(".CRT$XLB") PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_tls_attach }; #pragma data_seg() @@ -864,6 +916,12 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #pragma data_seg(".CRT$XIB") mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; #pragma data_seg() + #elif defined(__MINGW32__) + extern const IMAGE_TLS_DIRECTORY _tls_used; + __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; + __attribute__((used, section(".CRT$XIB"))) mi_crt_callback_t _mi_crt_callback_init = &mi_crt_init; #endif #if defined(__cplusplus) @@ -933,7 +991,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern "C" { #endif - #if defined(_WIN64) + #if defined(_WIN64) && defined(_MSC_VER) // 64-bit #pragma comment(linker, "/INCLUDE:_tls_used") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") @@ -945,7 +1003,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach }; #pragma const_seg() - #else + #elif defined(_MSC_VER) // 32-bit #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") @@ -955,6 +1013,11 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { #pragma data_seg(".CRT$XLY") PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_tls_detach }; #pragma data_seg() + #elif defined(__MINGW32__) + extern const IMAGE_TLS_DIRECTORY _tls_used; + __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; #endif #if defined(__cplusplus) @@ -1002,7 +1065,7 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern "C" { #endif - #if defined(_WIN64) + #if defined(_WIN64) && defined(_MSC_VER) // 64-bit #pragma comment(linker, "/INCLUDE:_tls_used") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:_mi_tls_callback_post") @@ -1014,16 +1077,21 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { extern const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[]; const PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; #pragma const_seg() - #else + #elif defined(_MSC_VER) // 32-bit #pragma comment(linker, "/INCLUDE:__tls_used") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_pre") #pragma comment(linker, "/INCLUDE:__mi_tls_callback_post") #pragma data_seg(".CRT$XLB") PIMAGE_TLS_CALLBACK _mi_tls_callback_pre[] = { &mi_win_main_attach }; #pragma data_seg() - #pragma data_seg(".CRT$XIY") + #pragma data_seg(".CRT$XLY") PIMAGE_TLS_CALLBACK _mi_tls_callback_post[] = { &mi_win_main_detach }; #pragma data_seg() + #elif defined(__MINGW32__) + extern const IMAGE_TLS_DIRECTORY _tls_used; + __attribute__((used)) static const void* const mi_tls_used_ref = &_tls_used; // pull in the CRT tls + __attribute__((used, section(".CRT$XLB"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_pre = &mi_tls_attach; + __attribute__((used, section(".CRT$XLY"))) PIMAGE_TLS_CALLBACK _mi_tls_callback_post = &mi_tls_detach; #endif #if defined(__cplusplus) @@ -1044,21 +1112,31 @@ static void NTAPI mi_win_main(PVOID module, DWORD reason, LPVOID reserved) { // See #define MI_PRIM_HAS_PROCESS_ATTACH 1 - static int mi_process_attach(void) { + static int mi_cdecl mi_crt_init(void) { mi_win_main(NULL,DLL_PROCESS_ATTACH,NULL); atexit(&_mi_auto_process_done); return 0; } - typedef int(*mi_crt_callback_t)(void); - #if defined(_WIN64) - #pragma comment(linker, "/INCLUDE:_mi_tls_callback") - #pragma section(".CRT$XIU", long, read) - #else - #pragma comment(linker, "/INCLUDE:__mi_tls_callback") + + #if defined(__cplusplus) + extern "C" { + #endif + typedef int (mi_cdecl* mi_crt_callback_t)(void); + #if defined(_WIN64) // 64-bit + #pragma comment(linker, "/INCLUDE:_mi_crt_callback_init") + #pragma const_seg(".CRT$XIU") + extern const mi_crt_callback_t _mi_crt_callback_init[]; + const mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; + #pragma const_seg() + #else // 32-bit + #pragma comment(linker, "/INCLUDE:__mi_crt_callback_init") + #pragma data_seg(".CRT$XIU") + mi_crt_callback_t _mi_crt_callback_init[] = { &mi_crt_init }; + #pragma data_seg() + #endif + #if defined(__cplusplus) + } #endif - #pragma data_seg(".CRT$XIU") - mi_decl_externc mi_crt_callback_t _mi_tls_callback[] = { &mi_process_attach }; - #pragma data_seg() #endif // use the fiber api for calling `_mi_thread_done`. diff --git a/src/prof.c b/src/prof.c index b2bb06e0a..9f76c539a 100644 --- a/src/prof.c +++ b/src/prof.c @@ -257,7 +257,7 @@ static void mi_prof_set_theap(mi_theap_t* theap, bool on) { theap->prof_countdown = (on ? mi_prof_next_countdown(theap) : 0); // poison/restore pages_free_direct for (size_t i = 0; i < MI_PAGES_DIRECT; i++) { - theap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty; + theap->pages_free_direct[i] = _mi_page_empty_get(); } // (when turning off, the slots refill lazily via mi_theap_queue_first_update) } diff --git a/src/random.c b/src/random.c index 75b5cfbb0..464627acf 100644 --- a/src/random.c +++ b/src/random.c @@ -134,8 +134,9 @@ static bool mi_random_is_initialized(mi_random_ctx_t* ctx) { void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { mi_assert_internal(mi_random_is_initialized(ctx)); - mi_assert_internal(ctx != ctx_new); - chacha_split(ctx, (uintptr_t)ctx_new /*nonce*/, ctx_new); + mi_assert_internal(ctx != ctx_new); + const uintptr_t nonce_rnd = _mi_random_next(ctx); + chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new); } uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { @@ -193,6 +194,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { ctx->weak = false; } chacha_init(ctx, key, (uintptr_t)ctx /*nonce*/ ); + _mi_memzero(key, sizeof(key)); } void _mi_random_init(mi_random_ctx_t* ctx) { diff --git a/src/static.c b/src/static.c index 8f9cdcc50..49a2568fc 100644 --- a/src/static.c +++ b/src/static.c @@ -24,7 +24,6 @@ terms of the MIT license. A copy of the license can be found in the file #include "alloc-aligned.c" #include "alloc-posix.c" #include "arena.c" -#include "arena-meta.c" #include "bitmap.c" #include "heap.c" #include "heap-snapshot.c" @@ -38,9 +37,11 @@ terms of the MIT license. A copy of the license can be found in the file #include "random.c" #include "scavenger.c" #include "stats.c" +#include "subproc.c" #include "theap.c" #include "threadlocal.c" #include "prim/prim.c" +#include "prim/prim-tls.c" #if MI_OSX_ZONE #include "prim/osx/alloc-override-zone.c" #endif diff --git a/src/stats.c b/src/stats.c index 2c60dbc86..4b07ce07f 100644 --- a/src/stats.c +++ b/src/stats.c @@ -97,15 +97,17 @@ void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) { static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* src) { if (stat==src) return; mi_atomic_void_addi64_relaxed(&stat->total, &src->total); - const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src->current); + const int64_t src_peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->peak); + const int64_t src_current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&src->current); + const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src_current); // Global current plus thread peak approximates new global peak // note: peak scores do really not work across threads. // we used to just add them together but that often overestimates in practice. // similarly, max does not seem to work well. The current approach // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112 - // for a longer description. - mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src->peak); + // for a longer description. + mi_atomic_maxi64_relaxed(&stat->peak, prev_current + src_peak); } static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counter_t* src) { @@ -146,7 +148,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // unit > 0 : size in binary bytes // unit == 0: count as decimal // unit < 0 : count in binary -static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, const char* fmt) { +static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg, bool limitwidth) { char buf[32]; _mi_memzero_var(buf); int len = 32; const char* suffix = (unit <= 0 ? " " : "B"); @@ -171,12 +173,17 @@ static void mi_printf_amount(int64_t n, int64_t unit, mi_output_fun* out, void* _mi_snprintf(unitdesc, 8, "%s%s%s", magnitude, (base==1024 ? "i" : ""), suffix); _mi_snprintf(buf, len, "%ld.%ld %-3s", whole, (frac1 < 0 ? -frac1 : frac1), unitdesc); } - _mi_fprintf(out, arg, (fmt==NULL ? "%12s" : fmt), buf); + if (limitwidth) { + _mi_fprintf(out, arg, "%12s", buf); + } + else { + _mi_fprintf(out, arg, "%s", buf); + } } static void mi_print_amount(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { - mi_printf_amount(n,unit,out,arg,NULL); + mi_printf_amount(n,unit,out,arg,true); } static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* arg) { @@ -210,7 +217,7 @@ static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64 } if (stat->current != 0) { _mi_fprintf(out, arg, " "); - _mi_fprintf(out, arg, (notok == NULL ? "not all freed" : notok)); + _mi_fprintf(out, arg, "%s", (notok == NULL ? "not all freed" : notok)); _mi_fprintf(out, arg, "\n"); } else { @@ -335,10 +342,10 @@ mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_ _mi_fprintf(out, arg, " %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); _mi_fprintf(out, arg, " %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults); - mi_printf_amount((int64_t)peak_rss, 1, out, arg, "%s"); + mi_printf_amount((int64_t)peak_rss, 1, out, arg, false); if (peak_commit > 0) { _mi_fprintf(out, arg, ", peak commit: "); - mi_printf_amount((int64_t)peak_commit, 1, out, arg, "%s"); + mi_printf_amount((int64_t)peak_commit, 1, out, arg, false); } _mi_fprintf(out, arg, "\n"); } @@ -436,7 +443,7 @@ void _mi_stats_merge_into(mi_stats_t* to, mi_stats_t* from) { mi_assert_internal(to != NULL && from != NULL); if (to == from) return; mi_stats_add(to, from); - _mi_memzero(from, sizeof(mi_stats_t)); + mi_stats_init(from); // zero field and keep the header } static const mi_stats_t* mi_stats_merge_theap_to_heap(mi_theap_t* theap) mi_attr_noexcept { @@ -485,6 +492,10 @@ void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* if (subproc==NULL) return; mi_heap_print_visit_info_t vinfo = { out, arg }; mi_subproc_visit_heaps(subproc_id, &mi_heap_print_visitor, &vinfo); + if (subproc->theap_meta!=NULL) { + _mi_stats_print("meta", subproc->subproc_seq, &subproc->theap_meta->stats, out, arg); + } + // if (subproc->theap_meta!=NULL) { _mi_stats_merge_into(&subproc->stats, &subproc->theap_meta->stats); } _mi_stats_print("subproc", subproc->subproc_seq, &subproc->stats, out, arg); } @@ -564,8 +575,10 @@ mi_decl_export void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, s pinfo.elapsed = _mi_clock_end(mi_process_start); { const mi_subproc_t* subproc = _mi_subproc_main(); if (subproc!=NULL) { - pinfo.current_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current))); - pinfo.peak_commit = (size_t)(mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak))); + const int64_t current = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.current)); + const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)(&subproc->stats.committed.peak)); + pinfo.current_commit = (current < 0 ? 0 : (current < PTRDIFF_MAX ? (size_t)current : PTRDIFF_MAX)); + pinfo.peak_commit = (peak < 0 ? 0 : (peak < PTRDIFF_MAX ? (size_t)peak : PTRDIFF_MAX)); } } pinfo.current_rss = pinfo.current_commit; @@ -630,6 +643,8 @@ bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr if (subproc == NULL) return false; if (!mi_stats_copy(stats, &subproc->stats)) return false; mi_subproc_visit_heaps(subproc_id, &mi_heap_aggregate_visitor, stats); + // hide meta data stats + // if (subproc->theap_meta!=NULL) { mi_stats_add_into(stats, &subproc->theap_meta->stats); } return true; } @@ -738,9 +753,6 @@ static void mi_json_buf_print_counter_value(mi_json_buf_t* hbuf, const char* nam mi_json_buf_print_value(hbuf, name, stat->total); } -#define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat); -#define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat); - static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, char* output_buf) mi_attr_noexcept { if (stats==NULL || stats->size!=sizeof(mi_stats_t) || stats->version!=MI_STAT_VERSION) return NULL; mi_json_buf_t hbuf = { NULL, 0, 0, true }; @@ -779,7 +791,13 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, mi_json_buf_print(&hbuf, " },\n"); // statistics + #define MI_STAT_COUNT(stat) mi_json_buf_print_count_value(&hbuf, #stat, &stats->stat); + #define MI_STAT_COUNTER(stat) mi_json_buf_print_counter_value(&hbuf, #stat, &stats->stat); + MI_STAT_FIELDS() + + #undef MI_STAT_COUNT + #undef MI_STAT_COUNTER // size bins mi_json_buf_print(&hbuf, " \"malloc_bins\": [\n"); @@ -798,7 +816,7 @@ static char* mi_stats_get_json_from(const mi_stats_t* stats, size_t output_size, } mi_json_buf_print(&hbuf, " ]\n"); mi_json_buf_print(&hbuf, "}\n"); - if (hbuf.used >= hbuf.size) { + if (hbuf.used + 1 >= hbuf.size) { // failed if (hbuf.can_realloc) { mi_free(hbuf.buf); } return NULL; diff --git a/src/subproc.c b/src/subproc.c new file mode 100644 index 000000000..ccfa0a638 --- /dev/null +++ b/src/subproc.c @@ -0,0 +1,433 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" + +// pre-allocate the main subprocess structure. +static mi_decl_cache_align mi_subproc_t mi_process_subproc_main = mi_init_struct_zero; +static mi_subproc_t* mi_subprocs = NULL; +static mi_lock_t mi_subprocs_lock = MI_LOCK_INITIALIZER; + + +/* ----------------------------------------------------------- + Meta-data allocation + We allocate thread local data and theaps through a dedicated + theap `subproc.theap_meta` which uses a detached tld with + a detached thread id. The initial theap_meta is statically + allocated and can thus be used to allocate on an as yet + uninitialized thread or process. + We need to take a lock though to allocate safely on the + detached `theap_meta`. +----------------------------------------------------------- */ + +void* _mi_meta_zalloc( mi_subproc_t* subproc, size_t size, mi_memid_t* memid ) { + mi_assert_internal(subproc->theap_meta != NULL); + void* p = NULL; + mi_lock(&subproc->theap_meta_lock) { + p = mi_theap_zalloc(subproc->theap_meta, size); + if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); } + } + return p; +} + +void* _mi_meta_zalloc_aligned( mi_subproc_t* subproc, size_t size, size_t aligned, mi_memid_t* memid ) { + mi_assert_internal(subproc->theap_meta != NULL); + void* p = NULL; + mi_lock(&subproc->theap_meta_lock) { + p = mi_theap_zalloc_aligned(subproc->theap_meta, size, aligned); + if (memid != NULL) { *memid = (p==NULL ? _mi_memid_none() : _mi_memid_create_malloc(p,size,true) ); } + } + return p; +} + +void* _mi_meta_rezalloc( mi_subproc_t* subproc, void* oldp, size_t newsize, mi_memid_t* memid ) { + mi_assert_internal(subproc->theap_meta != NULL); + // note: since we take a meta lock we cannot use `mi_theap_rezalloc` as that could call `mi_free` which + // can call `mi_stat_free` which would try to take the meta lock again. See issue #1358. + void* p = NULL; + mi_lock(&subproc->theap_meta_lock) { + p = mi_theap_zalloc(subproc->theap_meta, newsize); + } + if (p!=NULL) { + if (oldp!=NULL) { + const size_t oldsize = mi_usable_size(oldp); + const size_t copysize = (newsize < oldsize ? newsize : oldsize); + _mi_memcpy(p,oldp,copysize); + if (memid!=NULL) { _mi_meta_free(subproc,oldp,*memid); } + else { mi_free(oldp); } + } + if (memid!=NULL) { *memid = _mi_memid_create_malloc(p,newsize,true); } + } + else { + if (memid!=NULL) { *memid = _mi_memid_none(); } + } + return p; +} + +void _mi_meta_free(mi_subproc_t* subproc, void* p, mi_memid_t memid) { + if (p==NULL || mi_memid_needs_no_free(memid)) return; + if (memid.memkind == MI_MEM_MALLOC) { + mi_free(p); + } + else { + mi_assert_internal(subproc!=NULL); + _mi_arenas_free(subproc, p, _mi_memid_size(memid), memid); + } +} + +bool _mi_meta_is_meta_page(const mi_subproc_t* subproc, const mi_page_t* page) { + if (page==NULL) return false; + mi_theap_t* theap = page->theap; + return (theap != NULL && theap == subproc->theap_meta); +} + + +/* ----------------------------------------------------------- + Sub process helpers +----------------------------------------------------------- */ + +mi_subproc_t* _mi_subproc_main(void) { + return &mi_process_subproc_main; +} + +bool _mi_subproc_is_main(mi_subproc_t* subproc) { + return (subproc == &mi_process_subproc_main); +} + +mi_subproc_t* _mi_subproc(void) { + mi_theap_t* theap = _mi_theap_default(); + if (theap == NULL || theap->tld == NULL) { // see issue #1289 + return _mi_subproc_main(); + } + else { + return theap->tld->subproc; + } +} + +mi_heap_t* mi_heap_main(void) { + return _mi_subproc_heap_main(_mi_subproc()); // don't use mi_theap_main_init_get() so this call works during process_init +} + + +mi_subproc_t* _mi_subproc_from_id(mi_subproc_id_t subproc_id) { + return (mi_subproc_t*)(subproc_id._mi_subproc_id); +} + +mi_subproc_id_t _mi_subproc_to_id(mi_subproc_t* subproc) { + mi_subproc_id_t id = { subproc }; + return id; +} + +mi_subproc_id_t mi_subproc_main(void) { + return _mi_subproc_to_id(_mi_subproc_main()); +} + +mi_subproc_id_t mi_subproc_current(void) { + return _mi_subproc_to_id(_mi_subproc()); +} + + +/* ----------------------------------------------------------- + Sub process creation +----------------------------------------------------------- */ + + +static mi_subproc_t* mi_subproc_init(mi_subproc_t* subproc, mi_subproc_t* parent) { + static _Atomic(size_t) subproc_total_count; + subproc->parent = parent; + subproc->subproc_seq = mi_atomic_increment_relaxed(&subproc_total_count); + mi_stats_header_init(&subproc->stats); + mi_lock_init(&subproc->arena_reserve_lock); + mi_lock_init(&subproc->heaps_lock); + mi_lock_init(&subproc->theap_meta_lock); + mi_lock_init(&subproc->tlds_lock); // fork: tld registry + mi_lock(&mi_subprocs_lock) { + // push on subproc list + subproc->next = mi_subprocs; + if (mi_subprocs!=NULL) { mi_subprocs->prev = subproc; } + mi_subprocs = subproc; + } + return subproc; +} + +mi_subproc_id_t mi_subproc_new(void) { + mi_thread_init(); + mi_subproc_t* const parent = _mi_subproc(); + mi_memid_t memid; + mi_subproc_t* const subproc = (mi_subproc_t*)_mi_meta_zalloc(parent, sizeof(mi_subproc_t), &memid); + if (subproc == NULL) { return _mi_subproc_to_id(NULL); } + subproc->memid = memid; + + mi_memid_t theap_memid; + mi_theap_t* const theap_meta = (mi_theap_t*)_mi_meta_zalloc(parent, sizeof(mi_theap_t), &theap_memid); + if (theap_meta==NULL) { + _mi_meta_free(parent, subproc, memid); + return _mi_subproc_to_id(NULL); + } + theap_meta->memid = memid; + + // init subproc + mi_subproc_init(subproc,parent); + + // init main heap + mi_heap_t* heap_main = _mi_heap_new_for_subproc(subproc,0,true); + if (heap_main==NULL) { + _mi_meta_free(parent, theap_meta, theap_meta->memid); + mi_subproc_destroy(_mi_subproc_to_id(subproc)); + return _mi_subproc_to_id(NULL); + } + mi_assert_internal(subproc->heap_main == heap_main); + + // init meta theap + mi_assert_internal(parent->theap_meta!=NULL); + mi_assert_internal(parent->theap_meta->tld!=NULL); + mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREADID_DETACHED); + _mi_theap_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */); + subproc->theap_meta = theap_meta; + + return _mi_subproc_to_id(subproc); +} + + +/* ----------------------------------------------------------- + Sub process destruction +----------------------------------------------------------- */ + +// destroy all subproc resources including arena's, heap's etc. +static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subprocs_lock) +{ + if (subproc==NULL) return; + + // remove from the subproc list + mi_lock_maybe(&mi_subprocs_lock, acquire_subprocs_lock) { + if (subproc->next!=NULL) { subproc->next->prev = subproc->prev; } + if (subproc->prev!=NULL) { subproc->prev->next = subproc->next; } + else { mi_assert_internal(mi_subprocs==subproc); mi_subprocs = subproc->next; } + } + + // destroy all subproc heaps + mi_lock(&subproc->heaps_lock) { + mi_heap_t* heap = subproc->heaps; + while (heap != NULL) { + mi_heap_t* next = heap->next; + if (heap!=subproc->heap_main) { _mi_heap_force_destroy(heap, false /* don't re-acquire the heaps_lock */); } + heap = next; + } + mi_assert_internal(subproc->heap_main==NULL || subproc->heaps == subproc->heap_main); + if (subproc->heap_main!=NULL) { + _mi_thread_locals_thread_done(); // release thread locals that may have been allocated (safe as the main heap uses the fast key) + if (_mi_subproc_is_main(subproc)) { + _mi_thread_locals_done(); + } + _mi_heap_force_destroy(subproc->heap_main, false /* don't re-acquire the heaps_lock */); // no warning if destroying the main heap + } + } + + subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap + + if (!_mi_subproc_is_main(subproc)) { + // merge stats back into the main subproc + _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats); + } + + // remove associated arenas + _mi_arenas_unsafe_destroy_all(subproc); + + // show stats of the main process (at process end) before releasing the heaps lock + if (_mi_subproc_is_main(subproc)) { + if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); + } + } + + // todo: should we refcount subprocesses? + mi_lock_done(&subproc->arena_reserve_lock); + mi_lock_done(&subproc->heaps_lock); + mi_lock_done(&subproc->theap_meta_lock); + _mi_meta_free( subproc->parent, subproc, subproc->memid); + if (_mi_subproc_is_main(subproc)) { + // for the main subproc, also release the global page map + _mi_page_map_unsafe_destroy(); + } +} + +void mi_subproc_destroy(mi_subproc_id_t subproc_id) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL || subproc==&mi_process_subproc_main) return; + mi_subproc_unsafe_destroy(subproc, true /* take lock */); +} + +void _mi_subprocs_unsafe_destroy_all(void) { + mi_lock(&mi_subprocs_lock) { + mi_subproc_t* subproc = mi_subprocs; + while (subproc!=NULL) { + mi_subproc_t* next = subproc->next; + if (subproc!=&mi_process_subproc_main) { + mi_subproc_unsafe_destroy(subproc, false /* take mi_subprocs lock */); + } + subproc = next; + } + } + mi_subproc_unsafe_destroy(&mi_process_subproc_main, true /* take mi_subprocs lock */); +} + + +/* ----------------------------------------------------------- + Sub process various +----------------------------------------------------------- */ + +void mi_subproc_add_current_thread(mi_subproc_id_t subproc_id) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + mi_assert_internal(subproc!=NULL); + if (subproc==NULL) return; + mi_assert_internal(subproc->heap_main!=NULL); + if (subproc->heap_main==NULL) return; + mi_theap_t* theap = _mi_theap_default(); + if (mi_theap_is_initialized(theap)) { + if (theap->tld!=NULL && theap->tld->subproc != subproc) { + _mi_warning_message("unable to add thread to the subprocess as it was already in another subprocess (at %p)\n", theap->tld->subproc); + } + return; + } + + // initialize this thread tld & theap + _mi_thread_init_with_heap(subproc->heap_main); +} + + +bool mi_subproc_visit_heaps(mi_subproc_id_t subproc_id, mi_heap_visit_fun* visitor, void* arg) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return false; + bool ok = true; + mi_lock(&subproc->heaps_lock) { + for (mi_heap_t* heap = subproc->heaps; heap!=NULL && ok; heap = heap->next) { + ok = (*visitor)(heap, arg); + } + } + return ok; +} + + +mi_subproc_t* _mi_subproc_main_init(void) { + mi_lock_init(&mi_subprocs_lock); + mi_memid_t memid = _mi_memid_create_static(&mi_process_subproc_main,sizeof(mi_subproc_t)); + mi_process_subproc_main.memid = memid; + mi_subproc_init(&mi_process_subproc_main,NULL); + return &mi_process_subproc_main; +} + +void _mi_subproc_main_done(void) { + mi_lock_done(&mi_subprocs_lock); +} + +/* ----------------------------------------------------------- + fork() support (fork-only): quiesce every sub-process around a fork so the child inherits consistent allocator state. +----------------------------------------------------------- */ + +static _Atomic(uintptr_t) mi_fork_depth; // allow nested prepare/parent calls (e.g., zone hooks + pthread_atfork) +bool _mi_process_is_forked_child; // set once in fork_child, never cleared; lets visitors avoid waiting on dead threads + +// Lock order (prepare acquires in this order; parent releases in reverse; child re-initializes everything): +// subprocs registry -> thread locals -> per subproc: main heap's arena_pages (it may allocate while held) -> heaps list -> +// every other heap's arena_pages/theaps/os_abandoned -> main heap's theaps/os_abandoned -> tlds registry (leaf) -> +// arena reserve -> meta-data theap. +static void mi_subproc_fork_prepare(mi_subproc_t* sp) { + mi_heap_t* const heap_main = _mi_subproc_heap_main(sp); + if (heap_main != NULL) { mi_lock_acquire(&heap_main->arena_pages_lock); } + mi_lock_acquire(&sp->heaps_lock); + for (mi_heap_t* h = sp->heaps; h != NULL; h = h->next) { + if (h == heap_main) continue; + mi_lock_acquire(&h->arena_pages_lock); + mi_lock_acquire(&h->theaps_lock); + mi_lock_acquire(&h->os_abandoned_pages_lock); + } + if (heap_main != NULL) { + mi_lock_acquire(&heap_main->theaps_lock); + mi_lock_acquire(&heap_main->os_abandoned_pages_lock); + } + mi_lock_acquire(&sp->tlds_lock); + mi_lock_acquire(&sp->arena_reserve_lock); + mi_lock_acquire(&sp->theap_meta_lock); +} + +static void mi_subproc_fork_parent(mi_subproc_t* sp) { + mi_heap_t* const heap_main = _mi_subproc_heap_main(sp); + mi_lock_release(&sp->theap_meta_lock); + mi_lock_release(&sp->arena_reserve_lock); + mi_lock_release(&sp->tlds_lock); + if (heap_main != NULL) { + mi_lock_release(&heap_main->os_abandoned_pages_lock); + mi_lock_release(&heap_main->theaps_lock); + } + for (mi_heap_t* h = sp->heaps; h != NULL; h = h->next) { + if (h == heap_main) continue; + mi_lock_release(&h->os_abandoned_pages_lock); + mi_lock_release(&h->theaps_lock); + mi_lock_release(&h->arena_pages_lock); + } + mi_lock_release(&sp->heaps_lock); + if (heap_main != NULL) { mi_lock_release(&heap_main->arena_pages_lock); } +} + +void _mi_process_fork_prepare(void) { + if (!_mi_process_is_initialized) return; + if (mi_atomic_increment_acq_rel(&mi_fork_depth) != 0) return; + { + // a parked thread's theaps may be mid-sweep by the scavenger; leave the parked state first so nothing is claimed across the fork + mi_theap_t* const theap = _mi_theap_default(); + if (theap != NULL && mi_theap_is_initialized(theap) && theap->tld != NULL) { _mi_park_leave(theap->tld); } + } + mi_lock_acquire(&mi_subprocs_lock); + _mi_thread_locals_fork_prepare(); + for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { mi_subproc_fork_prepare(sp); } +} + +void _mi_process_fork_parent(void) { + if (!_mi_process_is_initialized) return; + if (mi_atomic_decrement_acq_rel(&mi_fork_depth) != 1) return; + // release in reverse: last sub-process first (the registry is a stack: newest first, so walk it into a reversed order) + size_t n = 0; + for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { n++; } + while (n > 0) { + mi_subproc_t* sp = mi_subprocs; + for (size_t k = 1; k < n; k++) { sp = sp->next; } + mi_subproc_fork_parent(sp); + n--; + } + _mi_thread_locals_fork_parent(); + mi_lock_release(&mi_subprocs_lock); +} + +void _mi_process_fork_child(void) { + if (!_mi_process_is_initialized) return; + if (mi_atomic_exchange_acq_rel(&mi_fork_depth, 0) == 0) return; + _mi_process_is_forked_child = true; + _mi_scavenger_forked_child(); // the scavenger thread did not survive the fork; clear the state that says it did + mi_lock_init(&mi_subprocs_lock); + for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { + mi_lock_init(&sp->arena_reserve_lock); + mi_lock_init(&sp->heaps_lock); + mi_lock_init(&sp->tlds_lock); + mi_lock_init(&sp->theap_meta_lock); + mi_atomic_store_relaxed(&sp->scavenger_wake, (uint32_t)0); + mi_atomic_store_relaxed(&sp->parked_count, (size_t)0); + for (mi_tld_t* t = sp->tlds; t != NULL; t = t->subproc_next) { // every tld, the forking thread's included, is registered here + mi_lock_init(&t->theaps_lock); + mi_atomic_store_relaxed(&t->park_state, (uint32_t)MI_PARK_RUNNING); + mi_atomic_store_relaxed(&t->park_reclaim, (uint32_t)0); + mi_atomic_store_relaxed(&t->park_swept, (uint32_t)0); + } + for (mi_heap_t* h = sp->heaps; h != NULL; h = h->next) { + mi_lock_init(&h->theaps_lock); + mi_lock_init(&h->arena_pages_lock); + mi_lock_init(&h->os_abandoned_pages_lock); + } + } + _mi_thread_locals_fork_child(); +} diff --git a/src/theap.c b/src/theap.c index 1a0a9463d..142c3466c 100644 --- a/src/theap.c +++ b/src/theap.c @@ -7,7 +7,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc/internal.h" -#include "mimalloc/prim.h" // _mi_prim_thread_yield +#include "mimalloc/prim.h" // _mi_prim_thread_yield #include "mimalloc/prim-tls.h" // _mi_theap_default #if defined(_MSC_VER) && (_MSC_VER < 1920) @@ -24,7 +24,7 @@ typedef bool (theap_page_visitor_fun)(mi_theap_t* theap, mi_page_queue_t* pq, mi // Visit all pages in a theap; returns `false` if break was called. static bool mi_theap_visit_pages(mi_theap_t* theap, theap_page_visitor_fun* fn, bool include_full, void* arg1, void* arg2) { - if (theap==NULL || theap->page_count==0) return 0; + if (theap==NULL || theap->page_count==0) return true; // visit all pages #if MI_DEBUG>1 @@ -119,7 +119,7 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag return true; // don't break } -static void mi_theap_merge_stats(mi_theap_t* theap) { +void _mi_theap_merge_stats(mi_theap_t* theap) { mi_assert_internal(mi_theap_is_initialized(theap)); mi_heap_t* const heap = _mi_theap_heap(theap); _mi_stats_merge_into(&heap->stats, &theap->stats); @@ -136,8 +136,8 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) // python/cpython#112532: we may be called from a thread that is not the owner of the theap // const bool is_main_thread = (_mi_is_main_thread() && theap->thread_id == _mi_thread_id()); - // collect retired pages - _mi_theap_collect_retired(theap, force); + // collect retired pages (and full pages if theap->allow_page_abandon is false) + _mi_theap_collect_retired(theap, force); // collect all pages owned by this thread mi_theap_visit_pages(theap, &mi_theap_page_collect, (collect!=MI_NORMAL), &collect, NULL); // dont normally visit full pages, see issue #1220 @@ -153,7 +153,7 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) } // merge statistics - mi_theap_merge_stats(theap); + _mi_theap_merge_stats(theap); } void _mi_theap_collect_abandon(mi_theap_t* theap) { @@ -442,6 +442,52 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) { return previous; } +#if MI_GUARDED +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { + theap->guarded_sample_rate = sample_rate; + theap->guarded_sample_count = sample_rate; // count down samples + if (theap->guarded_sample_rate > 1) { + if (seed == 0) { + seed = _mi_theap_random_next(theap); + } + theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` + } +} + +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { + theap->guarded_size_min = min; + theap->guarded_size_max = (min > max ? min : max); +} + +static void mi_theap_guarded_init(mi_theap_t* theap) { + mi_theap_guarded_set_sample_rate(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), + (size_t)mi_option_get(mi_option_guarded_sample_seed)); + mi_theap_guarded_set_size_bound(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), + (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); +} +#else +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { + MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed); +} + +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { + MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max); +} +static void mi_theap_guarded_init(mi_theap_t* theap) { + MI_UNUSED(theap); +} +#endif + +static void mi_theap_options_init(mi_theap_t* theap) { + theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0); + theap->allow_page_abandon = (mi_option_get(mi_option_page_full_retain) >= 0); + theap->page_full_retain = mi_option_get_clamp(mi_option_page_full_retain, -1, 32); + _mi_prof_theap_init(theap); // fork: sampling state + theap->is_detached = (theap->tld->thread_id == MI_THREADID_DETACHED); +} + // todo: make order of parameters consistent (but would that break compat with CPython?) void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) { @@ -452,17 +498,15 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) _mi_memcpy_aligned(theap, &_mi_theap_empty, sizeof(mi_theap_t)); theap->memid = memid; theap->tld = tld; // avoid reading the thread-local tld during initialization - mi_atomic_store_release(&theap->refcount,1); - mi_atomic_store_release(&theap->freed,0); - mi_atomic_store_ptr_release(mi_heap_t,&theap->heap,heap); + mi_atomic_store_release(&theap->refcount,1); mi_atomic_store_ptr_release(mi_subproc_t,&theap->subproc,heap->subproc); mi_assert_internal(theap->stats.size == sizeof(mi_stats_t)); - - _mi_theap_options_init(theap); + mi_theap_options_init(theap); + if (theap->tld->is_in_threadpool) { // if we run as part of a thread pool it is better to not arbitrarily reclaim abandoned pages into our theap. // this is checked in `free.c:mi_free_try_collect_mt` - // .. but abandoning is good in this case: halve the full page retain (possibly to 0) + // .. but abandoning is good in this case: quarter the full page retain (possibly to 0) // (so blocked threads do not hold on to too much memory) if (theap->page_full_retain > 0) { theap->page_full_retain = theap->page_full_retain / 4; @@ -484,20 +528,29 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) } // initialize random if heap==NULL - if (head == NULL) { // first theap in this thread? + if (head==NULL) { // first theap of the first thread? #if defined(_WIN32) && !defined(MI_SHARED_LIB) + if (tld->thread_seq==0) { _mi_random_init_weak(&theap->random); // prevent allocation failure during bcrypt dll initialization with static linking (issue #1185) - #else - _mi_random_init(&theap->random); + } + else #endif + { + _mi_random_init(&theap->random); + } } else { _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random } theap->cookie = _mi_theap_random_next(theap) | 1; - _mi_theap_guarded_init(theap); - mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem + mi_theap_guarded_init(theap); // needs theap->random + if (!theap->is_detached) { + mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem + } + // only now set the heap member as it is used to determine if a theap is initialized + mi_atomic_store_ptr_release(mi_heap_t,&theap->heap,heap); + // push on the heap's theap list mi_lock(&heap->theaps_lock) { head = heap->theaps; @@ -508,10 +561,10 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) } } -mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) { +mi_theap_t* _mi_theap_alloc(mi_heap_t* heap, mi_tld_t* tld) { mi_assert_internal(tld!=NULL); mi_assert_internal(heap!=NULL); - mi_assert_internal(_mi_thread_id() == tld->thread_id); + mi_assert_internal(tld->thread_id == MI_THREADID_DETACHED || _mi_thread_id() == tld->thread_id); // mi_assert_internal(_mi_heap_theap_peek(heap)==NULL); // don't access thread locals as this is called on thread init // allocate and initialize a theap @@ -533,7 +586,13 @@ mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) { } theap->memid = memid; - _mi_theap_init(theap, heap, tld); + return theap; +} + +mi_theap_t* _mi_theap_create(mi_heap_t* heap, mi_tld_t* tld) { + mi_theap_t* theap = _mi_theap_alloc(heap,tld); + if (theap == NULL) return NULL; + _mi_theap_init(theap, heap, tld); return theap; } @@ -543,18 +602,11 @@ uintptr_t _mi_theap_random_next(mi_theap_t* theap) { static void mi_theap_free_mem(mi_theap_t* theap) { if (theap!=NULL) { - mi_subproc_stat_decrease(_mi_theap_subproc(theap),theaps,1); - // free the used memory - if (theap->memid.memkind == MI_MEM_HEAP_MAIN) { // note: for now unused as it would access theap_default stats in mi_free of the current theap - mi_assert_internal(_mi_is_heap_main(mi_heap_of(theap))); - _mi_free_subproc_safe(theap); - } - else if (theap->memid.memkind == MI_MEM_META) { - _mi_meta_free(_mi_theap_subproc(theap), theap, sizeof(*theap), theap->memid); - } - else { - _mi_arenas_free(_mi_theap_subproc(theap), theap, _mi_align_up(sizeof(*theap),MI_ARENA_MIN_OBJ_SIZE), theap->memid ); // issue #1168, avoid assertion failure + mi_subproc_t* const subproc = mi_atomic_load_ptr_relaxed(mi_subproc_t,&theap->subproc); + if (!theap->is_detached) { + mi_subproc_stat_decrease(subproc,theaps,1); } + _mi_meta_free(subproc, theap, theap->memid); } } @@ -573,90 +625,106 @@ void _mi_theap_decref(mi_theap_t* theap) { } } - -// called from `mi_theap_delete` to free the internal theap resources. -bool _mi_theap_free(mi_theap_t* theap, bool acquire_heap_theaps_lock, bool acquire_tld_theaps_lock) { - mi_assert(theap != NULL); - if (theap==NULL) return true; - - // ensure only one thread actually frees the theap - const size_t freed = mi_atomic_exchange_acq_rel( &theap->freed, 1 ); - if (freed!=0) { - // concurrent interaction, retry in an outer loop (as the other thread may be blocked on our lock) - return false; - } - else { - // We won the `freed` exchange, so we own the free. Before doing any blocking work, secure - // both inner locks with try-acquire so the caller's retry loop can fire under contention. - // (Blocking here while the caller still holds its outer lock can complete a wait-for cycle - // with another `mi_heap_delete`, thread exit, or `_mi_process_fork_prepare`.) - mi_heap_t* const heap = _mi_theap_heap(theap); - bool got_hlock = !acquire_heap_theaps_lock; - bool got_tlock = !acquire_tld_theaps_lock; - if (acquire_heap_theaps_lock) { got_hlock = mi_lock_try_acquire(&heap->theaps_lock); } - if (got_hlock && acquire_tld_theaps_lock) { got_tlock = mi_lock_try_acquire(&theap->tld->theaps_lock); } - if (!got_hlock || !got_tlock) { - // back out: release what we took and give the claim back, so the retry can win it again. - // (`theap->heap` is untouched here -- it is only NULLed once the free actually completes.) - if (got_hlock && acquire_heap_theaps_lock) { mi_lock_release(&heap->theaps_lock); } - mi_atomic_store_release(&theap->freed, (size_t)0); - return false; // caller releases its outer lock, yields, and retries +// Thread termination and heap delete/destroy might run concurrently +// and we need to ensure we free the memory correctly. A heap or tld +// will first "detach" its theaps so it has a list with theaps that are +// no longer shared, and only then free's the theaps in that list. +// To detach we need to hold both the `heap->theaps_lock` and the `tld->theaps_lock`. +// Due to lock-inversion we need to use `mi_lock_try_acquire` and if that fails +// we back-off, release the outer lock, and try again until we succeed. + +// Remove the theaps in this heap from any thread local tld lists. +void _mi_heap_detach_theaps( mi_heap_t* heap ) { + bool all_detached; + do { + all_detached = true; + mi_lock(&heap->theaps_lock) { + mi_theap_t* theap = heap->theaps; + while (theap != NULL) { + mi_theap_t* next = theap->hnext; + mi_tld_t* tld = theap->tld; + if (tld != NULL) { + if (mi_lock_try_acquire(&tld->theaps_lock)) { + // remove the theap from the tld theaps list + if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; } + if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; } + else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; } + theap->tnext = theap->tprev = NULL; + theap->tld = NULL; + mi_lock_release(&tld->theaps_lock); + } + else { + all_detached = false; + } + } + theap = next; + } } - - // merge stats to the owning heap - _mi_stats_merge_into(&heap->stats, &theap->stats); - - // remove ourselves from the heap theaps list (lock already held if needed) - if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } - if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } - else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; } - theap->hnext = theap->hprev = NULL; - if (acquire_heap_theaps_lock) { mi_lock_release(&heap->theaps_lock); } - - // remove ourselves from the thread local theaps list (lock already held if needed) - if (theap->tnext != NULL) { theap->tnext->tprev = theap->tprev; } - if (theap->tprev != NULL) { theap->tprev->tnext = theap->tnext; } - else { mi_assert_internal(theap->tld->theaps == theap); theap->tld->theaps = theap->tnext; } - theap->tnext = theap->tprev = NULL; - if (acquire_tld_theaps_lock) { mi_lock_release(&theap->tld->theaps_lock); } - - // Set heap to NULL only after we are removed from the thread local theaps list since - // we may concurrently traverse it to collect (in `init.c:mi_thread_theaps_done`) - // (We need to set it to NULL to avoid an ABA problem where the _mi_theap_cached - // has a heap address that is reused for a newly allocated heap.) - mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL); - theap->tld = NULL; - // clear the per-thread cached theap if it is this one (this only catches the case where - // the *current* thread is the one freeing; cross-thread callers cannot reach the owning - // thread's TLS, but cache lookups still re-validate via `theap->heap` which is now NULL) - if (_mi_theap_cached() == theap) { - _mi_theap_cached_set((mi_theap_t*)&_mi_theap_empty); + if (!all_detached) { + mi_subproc_stat_counter_increase(heap->subproc,heaps_delete_wait,1); + _mi_prim_thread_yield(); } - // leave subproc field as is for free-ing - _mi_theap_decref(theap); - return true; - } + } while (!all_detached); +} + +// Remove the theaps in this thread from the heaps that own them. +void _mi_tld_detach_theaps( mi_tld_t* tld ) { + bool all_detached; + do { + all_detached = true; + mi_lock(&tld->theaps_lock) { + mi_theap_t* theap = tld->theaps; + while (theap != NULL) { + mi_theap_t* next = theap->tnext; + mi_assert_internal(theap->page_count==0); + mi_heap_t* heap = _mi_theap_heap_peek(theap); // now the heap might be NULL from an earlier iteration + if (heap != NULL) { + if (mi_lock_try_acquire(&heap->theaps_lock)) { + // merge stats into the owning heap stats + _mi_stats_merge_into(&heap->stats, &theap->stats); + // remove the theap from the heap list + if (theap->hnext != NULL) { theap->hnext->hprev = theap->hprev; } + if (theap->hprev != NULL) { theap->hprev->hnext = theap->hnext; } + else { mi_assert_internal(heap->theaps == theap); heap->theaps = theap->hnext; } + theap->hnext = theap->hprev = NULL; + // and set `heap` to NULL + mi_atomic_store_ptr_release(mi_heap_t, &theap->heap, NULL); + mi_lock_release(&heap->theaps_lock); + } + else { + all_detached = false; + } + } + theap = next; + } + } + if (!all_detached) { + mi_subproc_stat_counter_increase(tld->subproc,heaps_delete_wait,1); + _mi_prim_thread_yield(); + } + } while (!all_detached); } + /* ----------------------------------------------------------- Safe theap delete ----------------------------------------------------------- */ // Safe delete a theap without freeing any still allocated blocks in that theap. -void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock) -{ - mi_assert(theap != NULL); - mi_assert(mi_theap_is_initialized(theap)); - mi_assert_expensive(mi_theap_is_valid(theap)); - if (theap==NULL || !mi_theap_is_initialized(theap)) return; +// void _mi_theap_delete(mi_theap_t* theap, bool acquire_tld_theaps_lock) +// { +// mi_assert(theap != NULL); +// mi_assert(mi_theap_is_initialized(theap)); +// mi_assert_expensive(mi_theap_is_valid(theap)); +// if (theap==NULL || !mi_theap_is_initialized(theap)) return; - // abandon all pages - _mi_theap_collect_abandon(theap); +// // abandon all pages +// _mi_theap_collect_abandon(theap); - mi_assert_internal(theap->page_count==0); - _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock); -} +// mi_assert_internal(theap->page_count==0); +// _mi_theap_free(theap, true /* acquire heap->theaps_lock */, acquire_tld_theaps_lock); +// } @@ -710,7 +778,7 @@ bool mi_theap_reload(mi_theap_t* theap, mi_arena_id_t arena_id) { // reinit direct pages (as we may be in a different process) mi_assert_internal(theap->page_count == 0); for (size_t i = 0; i < MI_PAGES_DIRECT; i++) { - theap->pages_free_direct[i] = (mi_page_t*)&_mi_page_empty; + theap->pages_free_direct[i] = _mi_page_empty_get(); } // push on the thread local theaps list @@ -871,11 +939,11 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi return true; } -bool _mi_page_visit_blocks( mi_page_t* page, mi_block_visit_fun* visitor, void* arg ) { - mi_heap_area_t area; - _mi_heap_area_init(&area, page); - return _mi_theap_area_visit_blocks(&area, page, visitor, arg); -} +// bool _mi_page_visit_blocks( mi_page_t* page, mi_block_visit_fun* visitor, void* arg ) { +// mi_heap_area_t area; +// _mi_heap_area_init(&area, page); +// return _mi_theap_area_visit_blocks(&area, page, visitor, arg); +// } // Separate struct to keep `mi_page_t` out of the public interface diff --git a/src/threadlocal.c b/src/threadlocal.c index 5149161a6..ee2eafc83 100644 --- a/src/threadlocal.c +++ b/src/threadlocal.c @@ -6,9 +6,9 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ /* ---------------------------------------------------------------------------- -Implement dynamic thread local variables (for heap's). -Unlike most OS native implementations there is no limit on the number -that can be allocated. + Implement dynamic thread local variables (used by heap's for their theap's). + Unlike most OS native implementations there is no limit on the number + that can be allocated. -----------------------------------------------------------------------------*/ #include "mimalloc.h" @@ -27,10 +27,11 @@ typedef struct mi_tls_slot_s { typedef struct mi_thread_locals_s { size_t count; + mi_memid_t memid; mi_tls_slot_t slots[1]; } mi_thread_locals_t; -static mi_thread_locals_t mi_thread_locals_empty = { 0, {{0,NULL}} }; +static mi_thread_locals_t mi_thread_locals_empty = mi_init_struct_zero; /* ----------------------------------------------------------- @@ -54,7 +55,7 @@ static mi_thread_locals_t mi_thread_locals_empty = { 0, {{0,NULL}} }; #define mi_define_thread_local(tp,name,initval) \ static mi_decl_thread tp __##name = initval; \ static inline tp name##_peek(void) { return __##name; } \ - static inline tp name##_get(void) { return __##name; } \ + static inline tp name##_get(void) { tp result = __##name; return (result!=NULL ? result : initval); } \ static inline bool name##_set(tp val) { __##name = val; return true; } \ static inline void name##_delete(void) { } #endif @@ -70,10 +71,12 @@ mi_define_thread_local(void*, mi_slot_fast, NULL) a value, we also set the version of the key. ----------------------------------------------------------- */ -#if MI_SIZE_BITS < 64 -#define MI_TLS_IDX_BITS (MI_SIZE_BITS/2) // half for the index, half for the version +#if MI_SIZE_BITS >= 64 +#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) /* 16 bits for the index, 48 bits for the version */ +#elif MI_SIZE_BITS >= 32 +#define MI_TLS_IDX_BITS (12) /* 12 bits for index, 20 for the version? */ #else -#define MI_TLS_IDX_BITS (MI_SIZE_BITS/4) // 16 bits for the index, 48 bits for the version +#error not enough bits for the version for thread locals #endif #define MI_TLS_IDX_MASK ((MI_ZU(1)< MI_TLS_IDX_MAX) { return NULL; } // too large - mi_thread_locals_t* tls = (mi_thread_locals_t*)mi_rezalloc(tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t)); + // allocate as meta (for secure mode) + // we could also allocate on the main heap; this is recursion safe as that uses the fast local key + mi_memid_t memid = (tls_old==NULL ? _mi_memid_none() : tls_old->memid); + mi_thread_locals_t* tls = (mi_thread_locals_t*)_mi_meta_rezalloc(_mi_subproc(), tls_old, sizeof(mi_thread_locals_t) + count*sizeof(mi_tls_slot_t), &memid); if mi_unlikely(tls==NULL) return NULL; // `mi_rezalloc` copies `mi_usable_size(old)` bytes of the old block — the live // slots *plus* the uninitialized slack between the old requested size and the @@ -130,6 +136,7 @@ static mi_thread_locals_t* mi_thread_locals_expand(size_t least_idx) { // means dereferencing application data as a `mi_theap_t*`. Zero the new slot // range explicitly. (See test/test-heap-churn.c for the reproduction.) _mi_memzero(&tls->slots[count_old], (count - count_old)*sizeof(mi_tls_slot_t)); + tls->memid = memid; tls->count = count; mi_thread_locals_set(tls); return tls; @@ -205,7 +212,7 @@ void* _mi_thread_local_get( mi_thread_local_t key ) { void _mi_thread_locals_thread_done(void) { mi_thread_locals_t* const tls = mi_thread_locals_peek(); if (tls!=NULL && tls->count > 0) { - mi_free(tls); + _mi_meta_free(_mi_subproc(), tls, tls->memid); mi_thread_locals_set(NULL); } if (mi_slot_fast_peek() != NULL) { @@ -236,9 +243,7 @@ void _mi_thread_locals_done(void) { mi_lock(&mi_thread_locals_lock) { mi_bitmap_t* const slots = mi_thread_locals_free; if (slots!=NULL) { - const size_t slots_count = mi_bitmap_max_bits(slots); - const size_t slots_size = mi_bitmap_size(slots_count,NULL); - _mi_meta_free(_mi_subproc_main(), slots,slots_size,mi_thread_locals_memid); + _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid); } } mi_lock_done(&mi_thread_locals_lock); @@ -276,14 +281,14 @@ static bool mi_thread_local_create_expand(void) { const size_t newsize = mi_bitmap_size( newcount, NULL ); // mi_bitmap_t* newslots = (mi_bitmap_t*)mi_zalloc_aligned(newsize, MI_BCHUNK_SIZE); mi_memid_t memid; - mi_bitmap_t* newslots = (mi_bitmap_t*)_mi_meta_zalloc(_mi_subproc_main(), newsize, &memid); // always allocate thread locals in the main subprocess + mi_bitmap_t* newslots = (mi_bitmap_t*)_mi_meta_zalloc_aligned(_mi_subproc_main(), newsize, MI_BCHUNK_SIZE, &memid); // always allocate thread locals in the main subprocess mi_assert_internal(_mi_is_aligned(newslots,MI_BCHUNK_SIZE)); if (newslots==NULL) { return false; } if (slots!=NULL) { // copy over the previous bitmap const size_t oldsize = mi_bitmap_size(oldcount,NULL); _mi_memcpy_aligned(newslots, slots, oldsize); - _mi_meta_free(_mi_subproc_main(), slots,oldsize,mi_thread_locals_memid); + _mi_meta_free(_mi_subproc_main(), slots, mi_thread_locals_memid); } mi_bitmap_init(newslots, newcount, true /* pretend already zero'd so we do not zero out the copied old entries */); mi_bitmap_unsafe_setN(newslots, oldcount, newcount - oldcount); /* set the new expanded slots as available */ diff --git a/test/test-api.c b/test/test-api.c index 275bd1d6e..2d2b988de 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -1,5 +1,5 @@ /* ---------------------------------------------------------------------------- -Copyright (c) 2018-2020, Microsoft Research, Daan Leijen +Copyright (c) 2018-2026, Microsoft Research, Daan Leijen This is free software; you can redistribute it and/or modify it under the terms of the MIT license. A copy of the license can be found in the file "LICENSE" at the root of this distribution. @@ -38,6 +38,7 @@ we therefore test the API over various inputs. Please add more tests :-) #include "testhelper.h" + // --------------------------------------------------------------------------- // Test functions // --------------------------------------------------------------------------- @@ -53,13 +54,18 @@ bool test_stl_theap_allocator2(void); bool test_stl_theap_allocator3(void); bool test_stl_theap_allocator4(void); -bool mem_is_zero(uint8_t* p, size_t size) { +static bool test_zero_aligned_first(void); + +static bool mem_has_vals(const uint8_t* p, size_t size, uint8_t val) { if (p==NULL) return false; for (size_t i = 0; i < size; ++i) { - if (p[i] != 0) return false; + if (p[i] != val) return false; } return true; } +static bool mem_is_zero(const void* p, size_t size) { + return mem_has_vals((const uint8_t*)p,size,0); +} // --------------------------------------------------------------------------- // Main testing @@ -67,14 +73,25 @@ bool mem_is_zero(uint8_t* p, size_t size) { int main(void) { mi_option_disable(mi_option_verbose); - CHECK_BODY("malloc-aligned9a") { // test large alignments - void* p = mi_zalloc_aligned(1024 * 1024, 2); - mi_free(p); - p = mi_zalloc_aligned(1024 * 1024, 2); - mi_free(p); - result = true; - }; - + #if 1 + #if defined(__cplusplus) && !defined(_MSC_VER) + CHECK_BODY("c++ new-handler") { + std::set_new_handler([]{ throw std::bad_alloc(); }); + void* p = mi_new_nothrow(SIZE_MAX/2); + result = (p==NULL); + } + CHECK_BODY("c++ new handler2") { + try { + void* p = mi_new_n(SIZE_MAX/2, 4); + (void)(p); + result = false; + } + catch(std::bad_alloc) { + result = true; + } + } + #endif + #endif // --------------------------------------------------- // Malloc @@ -100,15 +117,18 @@ int main(void) { // use (size_t)&mi_calloc to get some number without triggering compiler warnings result = (mi_calloc((size_t)&mi_calloc,SIZE_MAX/1000) == NULL); }; - CHECK_BODY("calloc0") { - void* p = mi_calloc(0,1000); - result = (mi_usable_size(p) <= 16); - mi_free(p); - }; CHECK_BODY("malloc-large") { // see PR #544. void* p = mi_malloc(67108872); mi_free(p); }; + + CHECK_BODY("calloc0") { + void* p = mi_calloc(0,1000); + const size_t usable = mi_usable_size(p); + result = (usable <= 16); + mi_free(p); + }; + CHECK_BODY("mi_urealloc_invalid") { void* p = mi_malloc(64); size_t pre, post; @@ -116,7 +136,7 @@ int main(void) { mi_free(p); result = (q==NULL || q==(uint8_t*)p+3); } - + // --------------------------------------------------- // Extended // --------------------------------------------------- @@ -214,6 +234,7 @@ int main(void) { } result = ok; }; + CHECK_BODY("malloc-aligned9") { // test large alignments bool ok = true; void* p[8]; @@ -237,6 +258,15 @@ int main(void) { } result = ok; }; + + CHECK_BODY("malloc-aligned9a") { // test large alignments + void* p = mi_zalloc_aligned(1024 * 1024, 2); + mi_free(p); + p = mi_zalloc_aligned(1024 * 1024, 2); + mi_free(p); + result = true; + }; + CHECK_BODY("malloc-aligned10") { bool ok = true; void* p[10+1]; @@ -262,6 +292,7 @@ int main(void) { result = (((uintptr_t)p % 0x100) == 0); // #602 mi_free(p); } + CHECK_BODY("mimalloc-aligned13") { bool ok = true; for( size_t size = 1; size <= (MI_SMALL_SIZE_MAX * 2) && ok; size++ ) { @@ -315,6 +346,21 @@ int main(void) { mi_free(p); }; + CHECK_BODY("rezalloc_aligned_zeros") { // issue #763 + size_t alignment = 1024; + size_t n = 1024 * 6; + void* ptr = mi_zalloc_aligned(n, alignment); + assert(mem_is_zero(ptr,n)); + memset(ptr,123,n/2); + + ptr = mi_rezalloc_aligned(ptr, n/2, alignment); + assert(mem_has_vals((uint8_t*)ptr,n/2,123)); + + ptr = mi_rezalloc_aligned(ptr, n, alignment); + assert(mem_has_vals((uint8_t*)ptr,n/2,123)); + result = mem_is_zero((uint8_t*)ptr + n/2, n/2); + } + // --------------------------------------------------- // Reallocation // --------------------------------------------------- @@ -352,6 +398,29 @@ int main(void) { } } + // --------------------------------------------------- + // Small allocations + // --------------------------------------------------- + CHECK_BODY("free_small1") { + for(size_t n = 1; n < MI_SMALL_SIZE_MAX; n *=2) { + const size_t size = n*sizeof(int); + int* p = (int*)mi_zalloc(size); + p[n-1] = 42; + mi_free_size(p,size); + } + } + + CHECK_BODY("free_small2") { + for(size_t n = 1; n < MI_SMALL_SIZE_MAX; n *=2) { + const size_t size = n*sizeof(int); + int* p = (int*)mi_zalloc(size); + p[n-1] = 42; + p = (int*)mi_rezalloc(p, size + MI_SMALL_SIZE_MAX); + mi_free_size(p,size + MI_SMALL_SIZE_MAX); + } + } + + // --------------------------------------------------- // Returned block sizes // --------------------------------------------------- @@ -391,19 +460,43 @@ int main(void) { CHECK_BODY("heap-os2") { // @zoxc opus bug #3. + mi_collect(true); mi_stats_t_decl(stats0); mi_stats_get(&stats0); - mi_heap_t* h = mi_heap_new(); + mi_heap_t* h = mi_heap_new(); + long failed = 0; for(int i = 0; i < 10; i++) { int* p = (int*)mi_heap_malloc_aligned(h, 1<<20, 2<<20); // forced OS allocation - p[0] = 42; + if (p==NULL) { + failed++; + } + else { + p[0] = 42; + } } mi_heap_destroy(h); + mi_collect(true); mi_stats_t_decl(stats1); - mi_stats_get(&stats1); + mi_stats_get(&stats1); result = (stats0.pages.current == stats1.pages.current); + if (!result) { + fprintf(stderr, "heap-os2: pages: %ld != %ld (failed: %ld)\n", (long)stats0.pages.current, (long)stats1.pages.current, failed); + } + } + + #define NHEAPS (1000) + CHECK_BODY("heap-many") { // check creating many heaps and threadlocals, see issue #1358 + mi_heap_t* heaps[NHEAPS]; + for (size_t i = 0; i < NHEAPS; i++) { + heaps[i] = mi_heap_new(); + if (heaps[i] == NULL) { result = false; break; }; + if (mi_heap_malloc(heaps[i], 32) == NULL) { result = false; break; } + } + for (size_t i = 0; i < NHEAPS; i++) { + mi_heap_destroy(heaps[i]); + } } //CHECK("theap_destroy", test_theap1()); @@ -411,6 +504,14 @@ int main(void) { //CHECK("theap_arena_destroy", test_theap_arena_destroy()); //CHECK("theap_arena_delete", test_theap_arena_delete()); + + // --------------------------------------------------- + // Threads + // --------------------------------------------------- + CHECK_BODY("zero_aligned_first") { + result = mi_run_on_thread(&test_zero_aligned_first); + } + //mi_stats_print(NULL); // --------------------------------------------------- @@ -572,3 +673,20 @@ bool test_stl_theap_allocator4(void) { #endif } */ + +// --------------------------------------------------------------------------- +// Test a zero size aligned allocation as the very first allocation of a fresh thread. +// --------------------------------------------------------------------------- + +static bool test_zero_aligned_first(void) { + void* p = mi_malloc_aligned(0, 16); // must be the first mimalloc call on this thread + bool res = (p != NULL && (uintptr_t)(p) % 16 == 0); + mi_free(p); + p = mi_zalloc_aligned(0, 32); + res = res && (p != NULL && (uintptr_t)(p) % 32 == 0); + mi_free(p); + return res; +} + + + diff --git a/test/test-purge-holes.c b/test/test-purge-holes.c index b8e8a9596..088bea41b 100644 --- a/test/test-purge-holes.c +++ b/test/test-purge-holes.c @@ -1516,7 +1516,7 @@ static bool test_sweep_full_every(void) { if (ptrs[i] != NULL && _mi_ptr_page(ptrs[i]) == page) { mi_free(ptrs[i]); ptrs[i] = NULL; break; } } s0 = hole_stats(); - page->swept_state = (((uint32_t)page->capacity) << 16) | (uint32_t)page->used; + page->swept_state = (((uint64_t)page->capacity) << 32) | (uint64_t)page->used; // must match mi_page_sweep_state() in page.c // with no periodic full sweep the page is now wedged: no amount of parking finds the hole for (int r = 0; r < BOUND_EVERY * 2; r++) { mi_on_thread_idle(); } diff --git a/test/test-stress-heaps.c b/test/test-stress-heaps.c index 11e9de9bf..b12e9493e 100644 --- a/test/test-stress-heaps.c +++ b/test/test-stress-heaps.c @@ -6,6 +6,9 @@ terms of the MIT license. #define TEST_STRESS 1 #define MI_USE_HEAPS 4 + +#if !defined(MI_TEST_LIGHT) // too slow in test integration #define ALLOW_LARGE 1 +#endif #include "test-stress.c" diff --git a/test/test-stress.c b/test/test-stress.c index af6af3ff6..4de1ab96d 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -44,21 +44,24 @@ terms of the MIT license. // argument defaults #if defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the azure pipeline limits static int THREADS = NTHREADS/4; -static int SCALE = 25; -static int ITER = 400; +static int SCALE = 10; +static int ITER = 300; #elif defined(MI_UBSAN) // with undefined behavious sanitizer reduce parameters to stay within the azure pipeline limits static int THREADS = NTHREADS/4; static int SCALE = 25; static int ITER = 20; #elif defined(MI_GUARDED) // with debug guard pages reduce parameters to stay within the azure pipeline limits static int THREADS = NTHREADS/4; -static int SCALE = 50; +static int SCALE = 25; static int ITER = 10; -#elif 0 +#elif MI_DEBUG && MI_TEST_LIGHT +static int THREADS = NTHREADS/4; +static int SCALE = 25; +static int ITER = 10; +#elif MI_DEBUG static int THREADS = NTHREADS; static int SCALE = 25; -static int ITER = 50; -#define ALLOW_LARGE true +static int ITER = 25; #else static int THREADS = NTHREADS; // more repeatable if THREADS <= #processors static int SCALE = 50; // scaling factor @@ -278,7 +281,7 @@ static void run_os_threads(mi_subproc_id_t subproc, size_t nthreads, thread_entr static void test_stress(mi_subproc_id_t subproc) { // printf("test stress: subproc: %p\n", subproc._mi_subproc_id); volatile void* transfers[TRANSFERS]; - memset(transfers,0,sizeof(transfers)); + memset((void**)transfers,0,sizeof(transfers)); #ifdef MI_USE_HEAPS mi_heap_t* prev_heaps[MI_USE_HEAPS] = { NULL }; @@ -298,7 +301,7 @@ static void test_stress(mi_subproc_id_t subproc) { current_heap = mi_heap_new(); #endif - run_os_threads(subproc, THREADS, &stress, transfers); + run_os_threads(subproc, THREADS, &stress, (void**)transfers); #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) // switch between arena and OS allocation for testing @@ -330,10 +333,14 @@ static void test_stress(mi_subproc_id_t subproc) { } #ifndef USE_STD_MALLOC + #ifdef MI_USE_HEAPS + mi_subproc_heap_stats_print_out(mi_subproc_current(),NULL,NULL); + #else mi_stats_print(NULL); #endif + #endif - // clean up (a bit too early to test the final free_items still works correctly) + // clean up (a bit too early in order to test if the final `free_items` still works correctly) #ifdef MI_USE_HEAPS for (int i = 0; i < MI_USE_HEAPS; i++) { mi_heap_delete(prev_heaps[i]); prev_heaps[i] = NULL; @@ -406,7 +413,7 @@ int main(int argc, char** argv) { mi_version(); #endif #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) - mi_option_set(mi_option_arena_reserve, mi_arena_min_size()/1024 /* in KiB ! */); + mi_option_set(mi_option_arena_reserve, (long)(mi_arena_min_size()/1024) /* in KiB ! */); mi_option_set(mi_option_purge_delay,1); #endif #if defined(NDEBUG) && !defined(USE_STD_MALLOC) diff --git a/test/testhelper.h b/test/testhelper.h index a97275841..87b2f3834 100644 --- a/test/testhelper.h +++ b/test/testhelper.h @@ -47,3 +47,52 @@ static inline int print_test_summary(void) } #endif // TESTHELPER_H_ + +// ------------------------------------------------------ +// helper to run on threads +// ------------------------------------------------------ +typedef bool (*mi_thread_fun_t)(void); + +bool mi_run_on_thread(mi_thread_fun_t fun); + +typedef struct mi_thread_fun_args_s { + mi_thread_fun_t fun; + bool result; +} mi_thread_fun_args_t; + +#ifdef _WIN32 +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +static DWORD WINAPI mi_win_thread_entry(LPVOID varg) { + mi_thread_fun_args_t* arg = (mi_thread_fun_args_t*)varg; + arg->result = arg->fun(); + return 0; +} +bool mi_run_on_thread(mi_thread_fun_t fun) { + mi_thread_fun_args_t arg = { fun, false }; + HANDLE thread = CreateThread(NULL, 0, &mi_win_thread_entry, &arg, 0, NULL); + if (thread == NULL) return false; + WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); + return arg.result; +} +#else +#include +static void* mi_pthread_entry(void* varg) { + mi_thread_fun_args_t* arg = (mi_thread_fun_args_t*)varg; + arg->result = arg->fun(); + return NULL; +} +bool mi_run_on_thread(mi_thread_fun_t fun) { + mi_thread_fun_args_t arg = { fun, false }; + pthread_t thread; + if (pthread_create(&thread, NULL, &mi_pthread_entry, &arg) != 0) return false; + pthread_join(thread, NULL); + return arg.result; +} +#endif