From af30828c58e038b11f6d4c096199ac5241b4eb67 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 3 Sep 2024 13:04:47 +0100 Subject: [PATCH 1/9] Use scikit-build-core Signed-off-by: Pablo Galindo --- CMakeLists.txt | 146 +++++++++++++++++++++++++++++ Makefile | 2 +- pyproject.toml | 204 +++++++++++++++++++++-------------------- src/memray/_memray.pyx | 12 +-- 4 files changed, 256 insertions(+), 108 deletions(-) create mode 100644 CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..e0486eff45 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,146 @@ +cmake_minimum_required(VERSION 3.7) +project(memray) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Find Python +find_package(Python COMPONENTS Interpreter Development.Module Development.SABIModule REQUIRED) + +# Find Cython +find_program(CYTHON_EXECUTABLE cython) +if(NOT CYTHON_EXECUTABLE) + message(FATAL_ERROR "Cython not found. Please install Cython.") +endif() + +# Find required packages +find_package(PkgConfig REQUIRED) +pkg_check_modules(LZ4 REQUIRED liblz4) +if(NOT LZ4_FOUND) + message(FATAL_ERROR "LZ4 library not found. Please install liblz4-dev or equivalent.") +endif() + +if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + pkg_check_modules(LIBUNWIND REQUIRED libunwind) +endif() + +# Set compiler flags +set(COMPILER_FLAGS "-Wall") +if(NOT DEFINED ENV{NO_MEMRAY_FAST_TLS}) + add_definitions(-DUSE_MEMRAY_TLS_MODEL=1) +endif() + +if(DEFINED ENV{MEMRAY_MINIMIZE_INLINING}) + set(COMPILER_FLAGS ${COMPILER_FLAGS} -Og) +else() + set(COMPILER_FLAGS ${COMPILER_FLAGS} -flto) + set(LINKER_FLAGS ${LINKER_FLAGS} -flto) +endif() + +if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + set(BINARY_FORMAT "elf") +elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(BINARY_FORMAT "macho") +else() + message(FATAL_ERROR "Unsupported platform: ${CMAKE_SYSTEM_NAME}") +endif() + +# Set up libbacktrace +set(LIBBACKTRACE_DIR ${CMAKE_SOURCE_DIR}/src/vendor/libbacktrace) +set(LIBBACKTRACE_INSTALL_DIR ${LIBBACKTRACE_DIR}/install) +set(LIBBACKTRACE_INCLUDE_DIR ${LIBBACKTRACE_DIR}/install/include) +set(LIBBACKTRACE_LIB_DIR ${LIBBACKTRACE_DIR}/install/lib) + +# Add custom command to build libbacktrace +add_custom_command( + OUTPUT ${LIBBACKTRACE_LIB_DIR}/libbacktrace.a + COMMAND mkdir -p ${LIBBACKTRACE_INSTALL_DIR} + COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build + COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build && + ${LIBBACKTRACE_DIR}/configure + --with-pic + --prefix=${LIBBACKTRACE_INSTALL_DIR} + --includedir=${LIBBACKTRACE_INSTALL_DIR}/include/libbacktrace + COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build && make -j + COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build && make install + DEPENDS ${LIBBACKTRACE_DIR}/configure +) +add_custom_target(libbacktrace DEPENDS ${LIBBACKTRACE_LIB_DIR}/libbacktrace.a) + +# _memray extension + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/_memray.cpp + COMMAND Python::Interpreter -m cython + --cplus + -3 + -I ${CMAKE_SOURCE_DIR}/src/memray/ + ${CMAKE_SOURCE_DIR}/src/memray/_memray.pyx + -o ${CMAKE_CURRENT_BINARY_DIR}/_memray.cpp + DEPENDS ${CMAKE_SOURCE_DIR}/src/memray/_memray.pyx + VERBATIM +) +set(MEMRAY_SOURCES + src/memray/_memray/compat.cpp + src/memray/_memray/hooks.cpp + src/memray/_memray/tracking_api.cpp + src/memray/_memray/${BINARY_FORMAT}_shenanigans.cpp + src/memray/_memray/logging.cpp + src/memray/_memray/python_helpers.cpp + src/memray/_memray/source.cpp + src/memray/_memray/sink.cpp + src/memray/_memray/records.cpp + src/memray/_memray/record_reader.cpp + src/memray/_memray/record_writer.cpp + src/memray/_memray/snapshot.cpp + src/memray/_memray/socket_reader_thread.cpp + src/memray/_memray/native_resolver.cpp +) +python_add_library(_memray MODULE WITH_SOABI ${MEMRAY_SOURCES} ${CMAKE_CURRENT_BINARY_DIR}/_memray.cpp) +target_include_directories(_memray PRIVATE + ${CMAKE_SOURCE_DIR}/src/memray/_memray + ${LIBBACKTRACE_INCLUDE_DIR} + ${LZ4_INCLUDE_DIRS} +) +target_link_libraries(_memray PRIVATE ${LZ4_LIBRARIES} dl ${LIBUNWIND_LIBRARIES} ${LIBBACKTRACE_LIB_DIR}/libbacktrace.a) +target_link_options(_memray PRIVATE ${LZ4_LDFLAGS}) +target_compile_options(_memray PRIVATE ${COMPILER_FLAGS}) +target_link_options(_memray PRIVATE ${LINKER_FLAGS}) +add_dependencies(_memray libbacktrace) + +# _test_utils extension + +add_custom_command( + OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/_memray_test_utils.cpp + COMMAND Python::Interpreter -m cython + --cplus + -3 + -I ${CMAKE_SOURCE_DIR}/src/memray/ + ${CMAKE_SOURCE_DIR}/src/memray/_memray_test_utils.pyx + -o ${CMAKE_CURRENT_BINARY_DIR}/_memray_test_utils.cpp + --module-name memray._test_utils + DEPENDS ${CMAKE_SOURCE_DIR}/src/memray/_memray_test_utils.pyx + VERBATIM +) +python_add_library(_test_utils MODULE WITH_SOABI ${CMAKE_CURRENT_BINARY_DIR}/_memray_test_utils.cpp) +target_include_directories(_test_utils PRIVATE + ${CMAKE_SOURCE_DIR}/src/memray/_memray +) + +# _inject extension + +set(INJECT_SOURCES + src/memray/_memray/inject.cpp +) +python_add_library(_inject MODULE WITH_SOABI USE_SABI 3.7 ${INJECT_SOURCES}) +target_include_directories(_inject PRIVATE + ${CMAKE_SOURCE_DIR}/src/memray +) +target_compile_options(_test_utils PRIVATE ${COMPILER_FLAGS}) +target_link_options(_test_utils PRIVATE ${LINKER_FLAGS}) +target_compile_options(_inject PRIVATE ${COMPILER_FLAGS}) +target_link_options(_inject PRIVATE ${LINKER_FLAGS}) + + +# Install targets +install(TARGETS _memray _test_utils _inject LIBRARY DESTINATION memray) \ No newline at end of file diff --git a/Makefile b/Makefile index 96d141e917..ed13906004 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ build: build-js build-ext ## (default) Build package extensions and assets in-p .PHONY: build-ext build-ext: ## Build package extensions in-place - $(PYTHON) setup.py build_ext --inplace + $(PYTHON) -m pip install --no-build-isolation --config-settings=editable.rebuild=true -Cbuild-dir=build -ve. $(reporters_path)/templates/assets/%.js: $(reporters_path)/assets/%.js $(NPM) install diff --git a/pyproject.toml b/pyproject.toml index a3938cab02..6c5efc71e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,92 @@ -[build-system] +[project] +name = "memray" +version = "1.11.0" +description = "A memory profiler for Python applications" +readme = "README.md" +requires-python = ">=3.7.0" +license = { text = "Apache 2.0" } +authors = [{ name = "Pablo Galindo Salgado" }] +classifiers = [ + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Software Development :: Debuggers", +] +dependencies = [ + "jinja2 >= 2.9", + "typing_extensions; python_version < '3.8.0'", + "rich >= 11.2.0", + "textual >= 0.41.0", +] -requires = [ - "setuptools", - "wheel", - "pkgconfig", - "Cython>=0.29.31" +[project.optional-dependencies] +test = [ + "Cython", + "greenlet; python_version < '3.12'", + "pytest", + "pytest-cov", + "ipython", + "setuptools; python_version >= '3.12'", + "pytest-textual-snapshot", +] +docs = [ + "IPython", + "bump2version", + "sphinx", + "furo", + "sphinx-argparse", + "towncrier", +] +lint = ["black", "flake8", "isort", "mypy", "check-manifest"] +benchmark = ["asv"] +dev = [ + "Cython", + "greenlet; python_version < '3.12'", + "pytest", + "pytest-cov", + "ipython", + "setuptools; python_version >= '3.12'", + "pytest-textual-snapshot", + "black", + "flake8", + "isort", + "mypy", + "check-manifest", + "IPython", + "bump2version", + "sphinx", + "furo", + "sphinx-argparse", + "towncrier", + "asv", ] -build-backend = 'setuptools.build_meta' +[project.urls] +Homepage = "https://github.com/bloomberg/memray" + +[project.scripts] +memray = "memray.__main__:main" +"memray3.7" = "memray.__main__:main" +"memray3.8" = "memray.__main__:main" +"memray3.9" = "memray.__main__:main" +"memray3.10" = "memray.__main__:main" +"memray3.11" = "memray.__main__:main" +"memray3.12" = "memray.__main__:main" + +[build-system] +requires = ["scikit-build-core", "cython"] +build-backend = "scikit_build_core.build" + +[tool.scikit-build] +wheel.packages = ["src/memray"] + [tool.ruff] line-length = 95 @@ -17,7 +96,7 @@ fix = true [tool.ruff.isort] force-single-line = true known-first-party = ["memray"] -known-third-party=["rich", "elftools", "pytest"] +known-third-party = ["rich", "elftools", "pytest"] [tool.ruff.per-file-ignores] "benchmarks/*" = ["C4", "PERF"] @@ -29,8 +108,8 @@ include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true line_length = 88 -known_first_party=["memray"] -known_third_party=["rich", "elftools", "pytest"] +known_first_party = ["memray"] +known_third_party = ["rich", "elftools", "pytest"] [tool.towncrier] package = "memray" @@ -38,34 +117,29 @@ package_dir = "src" filename = "NEWS.rst" directory = "news" type = [ - { name = "Features", directory = "feature", showcontent = true }, + { name = "Features", directory = "feature", showcontent = true }, { name = "Deprecations and Removals", directory = "removal", showcontent = true }, - { name = "Bug Fixes", directory = "bugfix", showcontent = true }, - { name = "Improved Documentation", directory = "doc", showcontent = true }, - { name = "Miscellaneous", directory = "misc", showcontent = true }, + { name = "Bug Fixes", directory = "bugfix", showcontent = true }, + { name = "Improved Documentation", directory = "doc", showcontent = true }, + { name = "Miscellaneous", directory = "misc", showcontent = true }, ] underlines = "-~" [tool.pytest.ini_options] -markers = [ - "valgrind", -] +markers = ["valgrind"] xfail_strict = true [tool.check-manifest] -ignore = [ - "src/memray/reporters/templates/assets/*.js", -] +ignore = ["src/memray/reporters/templates/assets/*.js"] [tool.mypy] -exclude="tests/integration/(native_extension|multithreaded_extension)/" +exclude = "tests/integration/(native_extension|multithreaded_extension)/" [tool.cibuildwheel] build = ["cp38-*", "cp39-*", "cp310-*", "cp311-*"] skip = "*musllinux*{i686,aarch64}*" manylinux-x86_64-image = "manylinux2014" manylinux-i686-image = "manylinux2014" -musllinux-x86_64-image = "musllinux_1_1" [[tool.cibuildwheel.overrides]] select = "cp3{7,8,9,10}-*" @@ -74,37 +148,8 @@ manylinux-i686-image = "manylinux2010" [tool.cibuildwheel.linux] before-all = [ - # Build the latest curl from source. - "yum install -y openssl-devel", - "cd /", - "CURL_VERS=8.7.1", - "curl -LO https://curl.se/download/curl-$CURL_VERS.tar.bz2", - "tar xf ./curl-$CURL_VERS.tar.bz2", - "cd curl-$CURL_VERS", - "./configure --with-openssl", - "make install", - - # Build the latest zstd from source - "yum install -y lz4-devel xz-devel", - "cd /", - "ZSTD_VERS=1.5.6", - "/usr/bin/curl -LO https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-$ZSTD_VERS.tar.gz", - "tar xf ./zstd-$ZSTD_VERS.tar.gz", - "cd zstd-$ZSTD_VERS", - "V=1 LDLIBS=-lrt make install", - - # Build the latest elfutils from source. - "yum install -y lz4-devel", - "cd /", - "VERS=0.191", - "/usr/bin/curl https://sourceware.org/elfutils/ftp/$VERS/elfutils-$VERS.tar.bz2 > ./elfutils.tar.bz2", - "tar -xf elfutils.tar.bz2", - "cd elfutils-$VERS", - "CFLAGS='-Wno-error -g -O3' CXXFLAGS='-Wno-error -g -O3' LDFLAGS=-lrt ./configure --enable-libdebuginfod --disable-debuginfod --disable-nls --with-zstd", - "make install", - - # Install Memray's other build and test dependencies - "yum install -y libunwind-devel", + "yum install -y libunwind-devel lz4-devel gdb", + "if ! yum install -y lldb; then echo lldb is not available; fi", ] [tool.cibuildwheel.macos] @@ -112,8 +157,7 @@ before-all = [ "git clone --depth 1 --branch v1.9.4 https://github.com/lz4/lz4 lz4", "cd lz4", "make", - "make install PREFIX=$LZ4_INSTALL_DIR", - "find $LZ4_INSTALL_DIR", + "make install DESTDIR=/tmp/lz4_install", ] before-test = [ "codesign --remove-signature /Library/Frameworks/Python.framework/Versions/*/bin/python3 || true", @@ -121,18 +165,11 @@ before-test = [ ] [tool.coverage.run] -plugins = [ - "Cython.Coverage", -] -source = [ - "src/memray", - "tests/", -] +plugins = ["Cython.Coverage"] +source = ["src/memray", "tests/"] branch = true parallel = true -omit = [ - "*__init__.py", -] +omit = ["*__init__.py"] [tool.coverage.report] skip_covered = true @@ -142,39 +179,4 @@ show_missing = true # Override the default linux before-all for musl linux [[tool.cibuildwheel.overrides]] select = "*-musllinux*" -before-all = [ - # Remove gettext-dev, which conficts with the musl-libintl, which is a build - # dependency of elfutils. - "apk del gettext-dev glib-dev", - - # Build musl-fts from source. This is a build dependency of elfutils, but - # isn't in the Alpine repos of the musllinux_1_1 image. The build steps come - # from https://git.alpinelinux.org/aports/tree/main/musl-fts/APKBUILD - # Setting PATH before calling boostrap.sh fixes an automake failure. I think - # the failure may be caused by a different pkg-config in /usr/local/bin. - "cd /", - "apk add --update automake autoconf libtool", - "VERS=1.2.7", - "curl -L https://github.com/void-linux/musl-fts/archive/refs/tags/v$VERS.tar.gz > ./musl-fts.tar.gz", - "tar -xf musl-fts.tar.gz", - "cd musl-fts-$VERS", - "PATH=/usr/bin:/bin ./bootstrap.sh", - "CFLAGS=-fPIC ./configure", - "make install", - - # Build the latest elfutils from source. The build steps come from - # https://git.alpinelinux.org/aports/tree/main/elfutils, and the need to - # set the FNM_EXTMATCH macro to get the build to succeed is seen here: - # https://git.alpinelinux.org/aports/tree/main/elfutils/musl-macros.patch - "cd /", - "apk add --update argp-standalone bison bsd-compat-headers bzip2-dev curl-dev flex-dev libtool linux-headers musl-libintl musl-obstack-dev xz-dev zlib-dev zstd-dev", - "VERS=0.191", - "curl https://sourceware.org/elfutils/ftp/$VERS/elfutils-$VERS.tar.bz2 > ./elfutils.tar.bz2", - "tar -xf elfutils.tar.bz2", - "cd elfutils-$VERS", - "CFLAGS='-Wno-error -DFNM_EXTMATCH=0 -g -O3' CXXFLAGS='-Wno-error -g -O3' ./configure --enable-libdebuginfod --disable-debuginfod --disable-nls --with-zstd", - "make install", - - # Install Memray's other build and test dependencies - "apk add --update libunwind-dev lz4-dev" -] +before-all = ["apk add --update libunwind-dev lz4-dev gdb lldb"] diff --git a/src/memray/_memray.pyx b/src/memray/_memray.pyx index 417f6f2657..8d3ef971fa 100644 --- a/src/memray/_memray.pyx +++ b/src/memray/_memray.pyx @@ -69,12 +69,12 @@ from libcpp.unordered_map cimport unordered_map from libcpp.utility cimport move from libcpp.vector cimport vector -from ._destination import Destination -from ._destination import FileDestination -from ._destination import SocketDestination -from ._metadata import Metadata -from ._stats import Stats -from ._thread_name_interceptor import ThreadNameInterceptor +from memray._destination import Destination +from memray._destination import FileDestination +from memray._destination import SocketDestination +from memray._metadata import Metadata +from memray._stats import Stats +from memray._thread_name_interceptor import ThreadNameInterceptor cdef extern from "pthread.h" nogil: From a43f149aa3d96cc3edc2f71ae56173f35b005f2b Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 24 Sep 2024 19:55:39 -0400 Subject: [PATCH 2/9] fixup! Use scikit-build-core --- .github/workflows/coverage.yml | 2 +- CMakeLists.txt | 135 ++++++++++++++++++++++----------- MANIFEST.in | 41 ---------- Makefile | 2 +- news/673.misc.rst | 1 + pyproject.toml | 118 +++++++++++++++++++++++++--- src/memray/_memray.pyx | 12 +-- 7 files changed, 206 insertions(+), 105 deletions(-) delete mode 100644 MANIFEST.in create mode 100644 news/673.misc.rst diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ba03d71f16..d6f11e0ba2 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -48,7 +48,7 @@ jobs: python3.10-dbg - name: Install Python dependencies run: | - python3 -m pip install --upgrade pip cython pkgconfig + python3 -m pip install --upgrade pip scikit-build-core cython pkgconfig make test-install - name: Disable ptrace security restrictions run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index e0486eff45..edc92b1f3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,8 +1,9 @@ -cmake_minimum_required(VERSION 3.7) +cmake_minimum_required(VERSION 3.24...3.26) project(memray) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) # Find Python find_package(Python COMPONENTS Interpreter Development.Module Development.SABIModule REQUIRED) @@ -15,27 +16,40 @@ endif() # Find required packages find_package(PkgConfig REQUIRED) -pkg_check_modules(LZ4 REQUIRED liblz4) +pkg_check_modules(LZ4 liblz4) if(NOT LZ4_FOUND) - message(FATAL_ERROR "LZ4 library not found. Please install liblz4-dev or equivalent.") + message(FATAL_ERROR "liblz4 not found. Please install liblz4-dev or equivalent.") endif() if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") - pkg_check_modules(LIBUNWIND REQUIRED libunwind) + pkg_check_modules(UNWIND libunwind) + if(NOT UNWIND_FOUND) + message(FATAL_ERROR "libunwind not found. Please install libunwind-dev or equivalent.") + endif() + + pkg_check_modules(DEBUGINFOD libdebuginfod) + if(NOT DEBUGINFOD_FOUND) + # Some systems don't have a libdebuginfod.pc file. + # See if there's a libdebuginfod.so wherever liblz4 or libunwind are. + include(CheckLibraryExists) + check_library_exists("libdebuginfod.so" "debuginfod_find_debuginfo" "${LZ4_LIBRARY_DIRS};${UNWIND_LIBRARY_DIRS}" DEBUGINFOD) + if(DEBUGINFOD) + set(DEBUGINFOD_LIBRARIES "debuginfod") + else() + message(FATAL_ERROR "libdebuginfod not found. Please install libdebuginfod-dev or equivalent.") + endif() + endif() endif() # Set compiler flags -set(COMPILER_FLAGS "-Wall") +add_compile_options(-Wall) if(NOT DEFINED ENV{NO_MEMRAY_FAST_TLS}) - add_definitions(-DUSE_MEMRAY_TLS_MODEL=1) + add_compile_definitions(-DUSE_MEMRAY_TLS_MODEL=1) endif() -if(DEFINED ENV{MEMRAY_MINIMIZE_INLINING}) - set(COMPILER_FLAGS ${COMPILER_FLAGS} -Og) -else() - set(COMPILER_FLAGS ${COMPILER_FLAGS} -flto) - set(LINKER_FLAGS ${LINKER_FLAGS} -flto) -endif() +set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -flto") +set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Og") if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") set(BINARY_FORMAT "elf") @@ -48,21 +62,23 @@ endif() # Set up libbacktrace set(LIBBACKTRACE_DIR ${CMAKE_SOURCE_DIR}/src/vendor/libbacktrace) set(LIBBACKTRACE_INSTALL_DIR ${LIBBACKTRACE_DIR}/install) -set(LIBBACKTRACE_INCLUDE_DIR ${LIBBACKTRACE_DIR}/install/include) -set(LIBBACKTRACE_LIB_DIR ${LIBBACKTRACE_DIR}/install/lib) +set(LIBBACKTRACE_INCLUDE_DIR ${LIBBACKTRACE_INSTALL_DIR}/include) +set(LIBBACKTRACE_LIB_DIR ${LIBBACKTRACE_INSTALL_DIR}/lib) # Add custom command to build libbacktrace add_custom_command( OUTPUT ${LIBBACKTRACE_LIB_DIR}/libbacktrace.a + OUTPUT ${LIBBACKTRACE_INCLUDE_DIR}/libbacktrace/backtrace.h + OUTPUT ${LIBBACKTRACE_INCLUDE_DIR}/libbacktrace/internal.h COMMAND mkdir -p ${LIBBACKTRACE_INSTALL_DIR} COMMAND mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build && - ${LIBBACKTRACE_DIR}/configure - --with-pic - --prefix=${LIBBACKTRACE_INSTALL_DIR} - --includedir=${LIBBACKTRACE_INSTALL_DIR}/include/libbacktrace - COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build && make -j - COMMAND cd ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build && make install + ${LIBBACKTRACE_DIR}/configure + --with-pic + --prefix=${LIBBACKTRACE_INSTALL_DIR} + --includedir=${LIBBACKTRACE_INCLUDE_DIR}/libbacktrace + COMMAND make -C ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build -j + COMMAND make -C ${CMAKE_CURRENT_BINARY_DIR}/libbacktrace_build install DEPENDS ${LIBBACKTRACE_DIR}/configure ) add_custom_target(libbacktrace DEPENDS ${LIBBACKTRACE_LIB_DIR}/libbacktrace.a) @@ -72,15 +88,23 @@ add_custom_target(libbacktrace DEPENDS ${LIBBACKTRACE_LIB_DIR}/libbacktrace.a) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/_memray.cpp COMMAND Python::Interpreter -m cython - --cplus - -3 + --cplus + -3 + -X embedsignature=True + -X boundscheck=$,True,False> + -X wraparound=$,True,False> + -X overflowcheck=$,True,False> + -X cdivision=$,False,True> + -X c_string_type=unicode + -X c_string_encoding=utf8 -I ${CMAKE_SOURCE_DIR}/src/memray/ - ${CMAKE_SOURCE_DIR}/src/memray/_memray.pyx + ${CMAKE_SOURCE_DIR}/src/memray/_memray.pyx -o ${CMAKE_CURRENT_BINARY_DIR}/_memray.cpp + --module-name memray._memray DEPENDS ${CMAKE_SOURCE_DIR}/src/memray/_memray.pyx VERBATIM ) -set(MEMRAY_SOURCES +python_add_library(_memray MODULE WITH_SOABI src/memray/_memray/compat.cpp src/memray/_memray/hooks.cpp src/memray/_memray/tracking_api.cpp @@ -95,17 +119,38 @@ set(MEMRAY_SOURCES src/memray/_memray/snapshot.cpp src/memray/_memray/socket_reader_thread.cpp src/memray/_memray/native_resolver.cpp + ${CMAKE_CURRENT_BINARY_DIR}/_memray.cpp ) -python_add_library(_memray MODULE WITH_SOABI ${MEMRAY_SOURCES} ${CMAKE_CURRENT_BINARY_DIR}/_memray.cpp) -target_include_directories(_memray PRIVATE + +target_include_directories(_memray PRIVATE ${CMAKE_SOURCE_DIR}/src/memray/_memray - ${LIBBACKTRACE_INCLUDE_DIR} + ${LIBBACKTRACE_INCLUDE_DIR} ${LZ4_INCLUDE_DIRS} + ${UNWIND_INCLUDE_DIRS} + ${DEBUGINFOD_INCLUDE_DIRS} +) +target_link_libraries(_memray PRIVATE + ${LIBBACKTRACE_LIB_DIR}/libbacktrace.a + ${LZ4_LIBRARIES} + ${UNWIND_LIBRARIES} + ${DEBUGINFOD_LIBRARIES} + dl +) +target_link_directories(_memray PRIVATE + ${LZ4_LIBRARY_DIRS} + ${UNWIND_LIBRARY_DIRS} + ${DEBUGINFOD_LIBRARY_DIRS} +) +target_link_options(_memray PRIVATE + ${LZ4_LDFLAGS} + ${UNWIND_LDFLAGS} + ${DEBUGINFOD_LDFLAGS} +) +target_compile_options(_memray PRIVATE + ${LZ4_CFLAGS} + ${UNWIND_CFLAGS} + ${DEBUGINFOD_CFLAGS} ) -target_link_libraries(_memray PRIVATE ${LZ4_LIBRARIES} dl ${LIBUNWIND_LIBRARIES} ${LIBBACKTRACE_LIB_DIR}/libbacktrace.a) -target_link_options(_memray PRIVATE ${LZ4_LDFLAGS}) -target_compile_options(_memray PRIVATE ${COMPILER_FLAGS}) -target_link_options(_memray PRIVATE ${LINKER_FLAGS}) add_dependencies(_memray libbacktrace) # _test_utils extension @@ -113,34 +158,36 @@ add_dependencies(_memray libbacktrace) add_custom_command( OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/_memray_test_utils.cpp COMMAND Python::Interpreter -m cython - --cplus + --cplus -3 + -X embedsignature=True + -X boundscheck=False + -X wraparound=False + -X cdivision=True + -X c_string_type=unicode + -X c_string_encoding=utf8 -I ${CMAKE_SOURCE_DIR}/src/memray/ - ${CMAKE_SOURCE_DIR}/src/memray/_memray_test_utils.pyx + ${CMAKE_SOURCE_DIR}/src/memray/_memray_test_utils.pyx -o ${CMAKE_CURRENT_BINARY_DIR}/_memray_test_utils.cpp --module-name memray._test_utils DEPENDS ${CMAKE_SOURCE_DIR}/src/memray/_memray_test_utils.pyx VERBATIM ) -python_add_library(_test_utils MODULE WITH_SOABI ${CMAKE_CURRENT_BINARY_DIR}/_memray_test_utils.cpp) -target_include_directories(_test_utils PRIVATE +python_add_library(_test_utils MODULE WITH_SOABI + ${CMAKE_CURRENT_BINARY_DIR}/_memray_test_utils.cpp +) +target_include_directories(_test_utils PRIVATE ${CMAKE_SOURCE_DIR}/src/memray/_memray ) # _inject extension -set(INJECT_SOURCES +python_add_library(_inject MODULE WITH_SOABI USE_SABI 3.7 src/memray/_memray/inject.cpp ) -python_add_library(_inject MODULE WITH_SOABI USE_SABI 3.7 ${INJECT_SOURCES}) -target_include_directories(_inject PRIVATE +target_include_directories(_inject PRIVATE ${CMAKE_SOURCE_DIR}/src/memray ) -target_compile_options(_test_utils PRIVATE ${COMPILER_FLAGS}) -target_link_options(_test_utils PRIVATE ${LINKER_FLAGS}) -target_compile_options(_inject PRIVATE ${COMPILER_FLAGS}) -target_link_options(_inject PRIVATE ${LINKER_FLAGS}) - # Install targets -install(TARGETS _memray _test_utils _inject LIBRARY DESTINATION memray) \ No newline at end of file +install(TARGETS _memray _test_utils _inject LIBRARY DESTINATION memray) diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 68f65802d0..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,41 +0,0 @@ -exclude .clang-format -exclude asv.conf.json -exclude CONTRIBUTING.md -exclude Dockerfile -exclude Jenkinsfile -exclude requirements-*.txt -exclude .medusarc -exclude valgrind.supp - -recursive-exclude src/vendor/libbacktrace/install * -recursive-exclude benchmarks * -recursive-exclude debian * -recursive-exclude docker * -recursive-exclude docs * -recursive-exclude src/memray *.cpp *.h -recursive-exclude src/memray *.md -recursive-exclude tests * -recursive-exclude news * -recursive-exclude vendor * - -include README.md -include Makefile -include pyproject.toml -include package.json -include package-lock.json -include .bumpversion.cfg -include .babelrc -include webpack.config.js -include NEWS.rst -include .flake8 -include src/memray/py.typed -include .pre-commit-config.yaml - -recursive-include src/vendor * -recursive-include src/memray *.py -recursive-include src/memray *.pyi -recursive-include src/memray *.html *.js *.css -recursive-include src/memray *.pyx *.pxd -recursive-include src/memray *.gdb *.lldb -recursive-include src/memray/_memray * -recursive-include tools *.sh diff --git a/Makefile b/Makefile index ed13906004..eebaffeb29 100644 --- a/Makefile +++ b/Makefile @@ -25,7 +25,7 @@ build: build-js build-ext ## (default) Build package extensions and assets in-p .PHONY: build-ext build-ext: ## Build package extensions in-place - $(PYTHON) -m pip install --no-build-isolation --config-settings=editable.rebuild=true -Cbuild-dir=build -ve. + $(PYTHON) -m pip install --no-build-isolation --config-settings=editable.rebuild=true -ve . $(reporters_path)/templates/assets/%.js: $(reporters_path)/assets/%.js $(NPM) install diff --git a/news/673.misc.rst b/news/673.misc.rst new file mode 100644 index 0000000000..e0a84021f1 --- /dev/null +++ b/news/673.misc.rst @@ -0,0 +1 @@ +Memray is now built using ``scikit-build-core`` and CMake instead of ``setuptools``. This should have little to no impact on end users, but redistributors like distro package maintainers will need to ensure that these tools are available when building Memray. diff --git a/pyproject.toml b/pyproject.toml index 6c5efc71e8..c8cce5159d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,21 +1,22 @@ [project] name = "memray" -version = "1.11.0" description = "A memory profiler for Python applications" readme = "README.md" requires-python = ">=3.7.0" license = { text = "Apache 2.0" } -authors = [{ name = "Pablo Galindo Salgado" }] +authors = [{ name = "Pablo Galindo Salgado" }, { name = "Matt Wozniski" }] classifiers = [ "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: POSIX :: Linux", "Operating System :: MacOS", + "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Software Development :: Debuggers", ] @@ -25,6 +26,11 @@ dependencies = [ "rich >= 11.2.0", "textual >= 0.41.0", ] +dynamic = ["version"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "src/memray/_version.py" [project.optional-dependencies] test = [ @@ -73,20 +79,42 @@ Homepage = "https://github.com/bloomberg/memray" [project.scripts] memray = "memray.__main__:main" -"memray3.7" = "memray.__main__:main" -"memray3.8" = "memray.__main__:main" -"memray3.9" = "memray.__main__:main" -"memray3.10" = "memray.__main__:main" -"memray3.11" = "memray.__main__:main" -"memray3.12" = "memray.__main__:main" [build-system] requires = ["scikit-build-core", "cython"] build-backend = "scikit_build_core.build" [tool.scikit-build] +minimum-version = "0.9" +cmake.verbose = true +cmake.version = ">=3.26.1" +cmake.build-type = "RelWithDebInfo" +logging.level = "INFO" +install.strip = false +sdist.exclude = [ + "/*/", + "!/src/", + ".*", + "/src/vendor/libbacktrace/install/**", + "/src/memray/*.cpp", + "/src/memray/*.h", + "/src/memray/**/*.md", + "/.clang-format", + "/asv.conf.json", + "/CONTRIBUTING.md", + "/Dockerfile", + "/Jenkinsfile", + "/requirements-*.txt", + "/.medusarc", + "/valgrind.supp", +] wheel.packages = ["src/memray"] +[[tool.scikit-build.overrides]] +if.state = "editable" +build-dir = "build" +editable.verbose = false +cmake.build-type = "Debug" [tool.ruff] line-length = 95 @@ -140,6 +168,7 @@ build = ["cp38-*", "cp39-*", "cp310-*", "cp311-*"] skip = "*musllinux*{i686,aarch64}*" manylinux-x86_64-image = "manylinux2014" manylinux-i686-image = "manylinux2014" +musllinux-x86_64-image = "musllinux_1_1" [[tool.cibuildwheel.overrides]] select = "cp3{7,8,9,10}-*" @@ -148,8 +177,37 @@ manylinux-i686-image = "manylinux2010" [tool.cibuildwheel.linux] before-all = [ - "yum install -y libunwind-devel lz4-devel gdb", - "if ! yum install -y lldb; then echo lldb is not available; fi", + # Build the latest curl from source. + "yum install -y openssl-devel", + "cd /", + "CURL_VERS=8.7.1", + "curl -LO https://curl.se/download/curl-$CURL_VERS.tar.bz2", + "tar xf ./curl-$CURL_VERS.tar.bz2", + "cd curl-$CURL_VERS", + "./configure --with-openssl", + "make install", + + # Build the latest zstd from source + "yum install -y lz4-devel xz-devel", + "cd /", + "ZSTD_VERS=1.5.6", + "/usr/bin/curl -LO https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-$ZSTD_VERS.tar.gz", + "tar xf ./zstd-$ZSTD_VERS.tar.gz", + "cd zstd-$ZSTD_VERS", + "V=1 LDLIBS=-lrt make install", + + # Build the latest elfutils from source. + "yum install -y lz4-devel", + "cd /", + "VERS=0.191", + "/usr/bin/curl https://sourceware.org/elfutils/ftp/$VERS/elfutils-$VERS.tar.bz2 > ./elfutils.tar.bz2", + "tar -xf elfutils.tar.bz2", + "cd elfutils-$VERS", + "CFLAGS='-Wno-error -g -O3' CXXFLAGS='-Wno-error -g -O3' LDFLAGS=-lrt ./configure --enable-libdebuginfod --disable-debuginfod --disable-nls --with-zstd", + "make install", + + # Install Memray's other build and test dependencies + "yum install -y libunwind-devel", ] [tool.cibuildwheel.macos] @@ -157,7 +215,8 @@ before-all = [ "git clone --depth 1 --branch v1.9.4 https://github.com/lz4/lz4 lz4", "cd lz4", "make", - "make install DESTDIR=/tmp/lz4_install", + "make install PREFIX=$LZ4_INSTALL_DIR", + "find $LZ4_INSTALL_DIR", ] before-test = [ "codesign --remove-signature /Library/Frameworks/Python.framework/Versions/*/bin/python3 || true", @@ -179,4 +238,39 @@ show_missing = true # Override the default linux before-all for musl linux [[tool.cibuildwheel.overrides]] select = "*-musllinux*" -before-all = ["apk add --update libunwind-dev lz4-dev gdb lldb"] +before-all = [ + # Remove gettext-dev, which conficts with the musl-libintl, which is a build + # dependency of elfutils. + "apk del gettext-dev glib-dev", + + # Build musl-fts from source. This is a build dependency of elfutils, but + # isn't in the Alpine repos of the musllinux_1_1 image. The build steps come + # from https://git.alpinelinux.org/aports/tree/main/musl-fts/APKBUILD + # Setting PATH before calling boostrap.sh fixes an automake failure. I think + # the failure may be caused by a different pkg-config in /usr/local/bin. + "cd /", + "apk add --update automake autoconf libtool", + "VERS=1.2.7", + "curl -L https://github.com/void-linux/musl-fts/archive/refs/tags/v$VERS.tar.gz > ./musl-fts.tar.gz", + "tar -xf musl-fts.tar.gz", + "cd musl-fts-$VERS", + "PATH=/usr/bin:/bin ./bootstrap.sh", + "CFLAGS=-fPIC ./configure", + "make install", + + # Build the latest elfutils from source. The build steps come from + # https://git.alpinelinux.org/aports/tree/main/elfutils, and the need to + # set the FNM_EXTMATCH macro to get the build to succeed is seen here: + # https://git.alpinelinux.org/aports/tree/main/elfutils/musl-macros.patch + "cd /", + "apk add --update argp-standalone bison bsd-compat-headers bzip2-dev curl-dev flex-dev libtool linux-headers musl-libintl musl-obstack-dev xz-dev zlib-dev zstd-dev", + "VERS=0.191", + "curl https://sourceware.org/elfutils/ftp/$VERS/elfutils-$VERS.tar.bz2 > ./elfutils.tar.bz2", + "tar -xf elfutils.tar.bz2", + "cd elfutils-$VERS", + "CFLAGS='-Wno-error -DFNM_EXTMATCH=0 -g -O3' CXXFLAGS='-Wno-error -g -O3' ./configure --enable-libdebuginfod --disable-debuginfod --disable-nls --with-zstd", + "make install", + + # Install Memray's other build and test dependencies + "apk add --update libunwind-dev lz4-dev" +] diff --git a/src/memray/_memray.pyx b/src/memray/_memray.pyx index 8d3ef971fa..417f6f2657 100644 --- a/src/memray/_memray.pyx +++ b/src/memray/_memray.pyx @@ -69,12 +69,12 @@ from libcpp.unordered_map cimport unordered_map from libcpp.utility cimport move from libcpp.vector cimport vector -from memray._destination import Destination -from memray._destination import FileDestination -from memray._destination import SocketDestination -from memray._metadata import Metadata -from memray._stats import Stats -from memray._thread_name_interceptor import ThreadNameInterceptor +from ._destination import Destination +from ._destination import FileDestination +from ._destination import SocketDestination +from ._metadata import Metadata +from ._stats import Stats +from ._thread_name_interceptor import ThreadNameInterceptor cdef extern from "pthread.h" nogil: From e27634223d6c4d455bbf98261ae3fb1b59c855ab Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 24 Sep 2024 20:00:40 -0400 Subject: [PATCH 3/9] Drop CYTHON_TEST_MACROS / MEMRAY_MINIMIZE_INLINING These environment variables previously allowed us to conditionally enable extra safety checks in our Cython code, or conditionally disable optimizations that might break tools like Valgrind. Now, we control this based on the CMake build type, so that release builds have the extra optimizations and debug builds don't, and so that debug builds have the extra safety checks and release builds don't. Signed-off-by: Matt Wozniski --- .github/workflows/build.yml | 4 ---- Dockerfile | 3 +-- Makefile | 6 +++--- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4484668b8a..159c845444 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,7 +61,6 @@ jobs: echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope - name: Run coverage env: - CYTHON_TEST_MACROS: 1 PYTHON: ./venv/bin/python run: | make dev-install pycoverage @@ -104,7 +103,6 @@ jobs: /venv/bin/python -m pip install --upgrade pip - name: Run coverage env: - CYTHON_TEST_MACROS: 1 PYTHON: /venv/bin/python GENHTMLOPTS: "--ignore-errors inconsistent" run: | @@ -161,8 +159,6 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install -r requirements-test.txt python3 -m pip install -e . - env: - MEMRAY_MINIMIZE_INLINING: 1 - name: Run Valgrind run: make valgrind - name: Run Helgrind diff --git a/Dockerfile b/Dockerfile index 956c63f091..cbf0bd1bfc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,8 +36,7 @@ ENV VIRTUAL_ENV=/venv \ RUN python3 -m venv "$VIRTUAL_ENV" ENV PATH="${VIRTUAL_ENV}/bin:/usr/lib/ccache:${PATH}" \ - PYTHON="${VIRTUAL_ENV}/bin/python" \ - MEMRAY_MINIMIZE_INLINING="1" + PYTHON="${VIRTUAL_ENV}/bin/python" COPY requirements-test.txt requirements-extra.txt requirements-docs.txt /tmp/ diff --git a/Makefile b/Makefile index eebaffeb29..10639bf747 100644 --- a/Makefile +++ b/Makefile @@ -41,15 +41,15 @@ dist: ## Generate Python distribution files .PHONY: install-sdist install-sdist: dist ## Install from source distribution - $(ENV) $(PIP_INSTALL) $(wildcard dist/*.tar.gz) + $(PIP_INSTALL) $(wildcard dist/*.tar.gz) .PHONY: test-install test-install: build-js ## Install with test dependencies - $(ENV) CYTHON_TEST_MACROS=1 $(PIP_INSTALL) -e .[test] + $(PIP_INSTALL) -e .[test] .PHONY: dev-install dev-install: build-js ## Install with dev dependencies - $(ENV) CYTHON_TEST_MACROS=1 $(PIP_INSTALL) -e .[dev] + $(PIP_INSTALL) -e .[dev] .PHONY: check check: check-python check-js ## Run all the tests From 491c1dca9cf44d07416d14ed139f5d63f7c61b12 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 24 Sep 2024 20:05:33 -0400 Subject: [PATCH 4/9] tests: Avoid clobbering environment variables Some tests were erroneously removing all environment variables from the environment and passing along an environment consisting of only a single variable. Since this removes environment variables like $PATH, this breaks automatic rebuilds -- they can't find the compiler! Signed-off-by: Matt Wozniski --- tests/integration/test_main.py | 34 +++++++++++++++++++++++------- tests/integration/test_tracking.py | 5 ++++- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/integration/test_main.py b/tests/integration/test_main.py index 2df8e97a21..3a7753d1ac 100644 --- a/tests/integration/test_main.py +++ b/tests/integration/test_main.py @@ -1442,7 +1442,11 @@ def test_live_tracking(self, tmp_path, simple_test_file, free_port): assert client.returncode == 0 def test_live_tracking_waits_for_client(self, simple_test_file): - # GIVEN/WHEN + # GIVEN + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + + # WHEN server = subprocess.Popen( [ sys.executable, @@ -1452,7 +1456,7 @@ def test_live_tracking_waits_for_client(self, simple_test_file): "--live-remote", str(simple_test_file), ], - env={"PYTHONUNBUFFERED": "1"}, + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) @@ -1464,7 +1468,11 @@ def test_live_tracking_waits_for_client(self, simple_test_file): @pytest.mark.parametrize("port", [0, 2**16, 1000000]) def test_run_live_tracking_invalid_port(self, simple_test_file, port): - # GIVEN/WHEN + # GIVEN + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + + # WHEN server = subprocess.Popen( [ sys.executable, @@ -1476,7 +1484,7 @@ def test_run_live_tracking_invalid_port(self, simple_test_file, port): str(port), str(simple_test_file), ], - env={"PYTHONUNBUFFERED": "1"}, + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -1489,7 +1497,11 @@ def test_run_live_tracking_invalid_port(self, simple_test_file, port): @pytest.mark.parametrize("port", [0, 2**16, 1000000]) def test_live_tracking_invalid_port(self, port): - # GIVEN/WHEN + # GIVEN + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + + # WHEN server = subprocess.Popen( [ sys.executable, @@ -1498,7 +1510,7 @@ def test_live_tracking_invalid_port(self, port): "live", str(port), ], - env={"PYTHONUNBUFFERED": "1"}, + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -1511,6 +1523,9 @@ def test_live_tracking_invalid_port(self, port): def test_live_tracking_server_when_client_disconnects(self, free_port, tmp_path): # GIVEN + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + test_file = tmp_path / "test.py" test_file.write_text("import time; time.sleep(3)") @@ -1525,7 +1540,7 @@ def test_live_tracking_server_when_client_disconnects(self, free_port, tmp_path) "--live-remote", str(test_file), ], - env={"PYTHONUNBUFFERED": "1"}, + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -1566,6 +1581,9 @@ def test_live_tracking_server_when_client_disconnects(self, free_port, tmp_path) def test_live_tracking_server_exits_properly_on_sigint(self, simple_test_file): # GIVEN + env = os.environ.copy() + env["PYTHONUNBUFFERED"] = "1" + server = subprocess.Popen( [ sys.executable, @@ -1575,7 +1593,7 @@ def test_live_tracking_server_exits_properly_on_sigint(self, simple_test_file): "--live-remote", str(simple_test_file), ], - env={"PYTHONUNBUFFERED": "1"}, + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, # Explicitly reset the signal handler for SIGINT to work around any signal diff --git a/tests/integration/test_tracking.py b/tests/integration/test_tracking.py index dbd0d5cfb6..454504b81d 100644 --- a/tests/integration/test_tracking.py +++ b/tests/integration/test_tracking.py @@ -1,6 +1,7 @@ import collections import datetime import mmap +import os import signal import subprocess import sys @@ -1599,6 +1600,8 @@ def test_get_header_after_snapshot(self, monkeypatch, tmpdir): def test_header_allocator(self, allocator, allocator_name, tmpdir): # GIVEN output = Path(tmpdir) / "test.bin" + env = os.environ.copy() + env["PYTHONMALLOC"] = allocator # WHEN @@ -1617,7 +1620,7 @@ def test_header_allocator(self, allocator, allocator_name, tmpdir): subprocess.run( [sys.executable, "-c", subprocess_code], timeout=5, - env={"PYTHONMALLOC": allocator}, + env=env, ) reader = FileReader(output) From 02a351a3ca9e2845c32c91b33eb9440aa7d0eb50 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 24 Sep 2024 20:07:31 -0400 Subject: [PATCH 5/9] tests: Loosen a check on elapsed time This test failed in CI because the same time was captured in both `end_time` and `start_time`. There's no reason to assume that this took more than a millisecond, or that the system clock can give us better than millisecond granularity, so it seems more reasonable to simply check that the end time is no earlier than the start time. Technically even that assumption could be violated, since we're not using a monotonic clock, but oh well, this is just tests. Signed-off-by: Matt Wozniski --- tests/integration/test_tracking.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_tracking.py b/tests/integration/test_tracking.py index 454504b81d..a1b237c6c6 100644 --- a/tests/integration/test_tracking.py +++ b/tests/integration/test_tracking.py @@ -1551,7 +1551,7 @@ def test_get_header(self, monkeypatch, tmpdir): metadata = reader.metadata # THEN - assert metadata.end_time > metadata.start_time + assert metadata.end_time >= metadata.start_time assert abs(metadata.start_time - start_time).seconds < 1 assert abs(metadata.end_time - end_time).seconds < 1 assert metadata.total_allocations == n_records @@ -1582,7 +1582,7 @@ def test_get_header_after_snapshot(self, monkeypatch, tmpdir): metadata = reader.metadata # THEN - assert metadata.end_time > metadata.start_time + assert metadata.end_time >= metadata.start_time assert abs(metadata.start_time - start_time).seconds < 1 assert abs(metadata.end_time - end_time).seconds < 1 assert metadata.total_allocations == peak.n_allocations From 2d04f8217789777cf2f3d1bfb7646970da2a713e Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 24 Sep 2024 20:10:39 -0400 Subject: [PATCH 6/9] Remove our Makefile from .gitignore We meant to ignore any Makefile generated by CMake, but we shouldn't be ignoring our own hand-written Makefile. Signed-off-by: Matt Wozniski --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 32fc3e5815..7dd5439202 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ CMakeFiles CMakeScripts Testing Makefile +!/Makefile cmake_install.cmake install_manifest.txt compile_commands.json From 7363ed42fdf681ecf5f78758a1a4edbc5f912d5a Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 24 Sep 2024 20:12:12 -0400 Subject: [PATCH 7/9] ci: Set CXXFLAGS in addition to CFLAGS `setuptools` uses CFLAGS for both C and C++ builds, but CMake distinguishes between them, and only uses CXXFLAGS for C++ builds. Whenever we want to set flags that affect both C and C++ builds, ensure that we set both CFLAGS and CXXFLAGS. Signed-off-by: Matt Wozniski --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 10639bf747..9a4234cbde 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ helgrind: ## Run helgrind, with the correct configuration .PHONY: ccoverage ccoverage: ## Run the test suite, with C++ code coverage $(MAKE) clean - CFLAGS="$(CFLAGS) -O0 -pg --coverage" $(MAKE) build + CFLAGS="$(CFLAGS) -O0 -pg --coverage" CXXFLAGS="$(CXXFLAGS) -O0 -pg --coverage" $(MAKE) build $(MAKE) check gcov -i build/*/src/memray/_memray -i -d lcov --capture --directory . --output-file cppcoverage.lcov From 8becdd3dfaa83065f223ef605491a2bb5791c602 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 24 Sep 2024 20:13:54 -0400 Subject: [PATCH 8/9] docs: Drop the memray3.N entry point With `scikit-build-core`, we don't have any way to produce an entry point script called `memray3.N` (where `3.N` is the Python version Memray was installed for). Drop references to it from the docs. Signed-off-by: Matt Wozniski --- docs/examples/README.rst | 6 +++--- docs/getting_started.rst | 14 ++++---------- docs/run.rst | 6 +++--- news/673.removal.rst | 1 + 4 files changed, 11 insertions(+), 16 deletions(-) create mode 100644 news/673.removal.rst diff --git a/docs/examples/README.rst b/docs/examples/README.rst index d43e96ff2b..7be2aa1759 100644 --- a/docs/examples/README.rst +++ b/docs/examples/README.rst @@ -11,11 +11,11 @@ Make sure you install the required dependencies by running directory. The examples below use the project in the ``mandelbrot`` folder, but you can use the same instructions to launch the other examples as well. -To track memory allocations, invoke ``memray3.9 run``: +To track memory allocations, invoke ``memray run``: .. code:: shell - memray3.9 run mandelbrot/mandelbrot.py + memray run mandelbrot/mandelbrot.py Memray will print a message displaying the output file it creates. @@ -28,7 +28,7 @@ graph, use the following command: .. code:: shell - memray3.9 flamegraph mandelbrot/memray-mandelbrot.py.187967.bin + memray flamegraph mandelbrot/memray-mandelbrot.py.187967.bin The HTML file for the flame graph will be generated under ``mandelbrot/memray-flamegraph-mandelbrot.py.187967.html``. The flame graph diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 07e8f3e662..f54473b42e 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -26,19 +26,13 @@ You can invoke Memray the following way: python3.9 -m memray -Or alternatively through the version-qualified ``memrayX.Y`` script: - -.. code:: shell - - memray3.9 - -You can also invoke Memray without version-qualifying it: +Or alternatively through the ``memray`` script: .. code:: shell memray -The downside to the unqualified ``memray`` script is that it's not immediately +The downside to using the ``memray`` script is that it's not immediately clear what Python interpreter will be used to execute Memray. If you're using a virtual environment that's not a problem because you know exactly what interpreter is in use, but otherwise you need to be careful to ensure that ``memray`` is @@ -56,7 +50,7 @@ To run memray on the ``example.py`` script, use :doc:`the run subcommand `. .. code:: shell - memray3.9 run example.py + memray run example.py This will execute the script and track its memory allocations, displaying the name of the file where results are being recorded with a message like: @@ -72,7 +66,7 @@ the results file: .. code:: shell - memray3.9 flamegraph memray-example.py.4131.bin + memray flamegraph memray-example.py.4131.bin This will generate the ``memray-flamegraph-example.py.4131.html`` file in the current directory. See the :doc:`flamegraph` documentation which explains how to interpret flame graphs. diff --git a/docs/run.rst b/docs/run.rst index dee31e00ef..5e5354c94e 100644 --- a/docs/run.rst +++ b/docs/run.rst @@ -140,7 +140,7 @@ You can run a program in live mode using ``run --live``: .. code:: shell - memray3.9 run --live application.py + memray run --live application.py Immediately Memray will start your application in the background and will run a TUI in the foreground that you can use to analyze your application's memory usage. If you don't want to run your program in the background, you can instead @@ -148,7 +148,7 @@ use ``run --live-remote``: .. code:: shell - memray3.9 run --live-remote application.py + memray run --live-remote application.py In this mode, Memray will choose an unused port, bind to it, and display a message saying: @@ -160,7 +160,7 @@ It will wait for you to run: .. code:: shell - memray3.9 live + memray live in another terminal window to attach to it. Regardless of whether you choose to use one terminal or two, the resulting TUI is exactly the same. See :doc:`live` for details on how to interpret and control the TUI. diff --git a/news/673.removal.rst b/news/673.removal.rst new file mode 100644 index 0000000000..93431cf53e --- /dev/null +++ b/news/673.removal.rst @@ -0,0 +1 @@ +Memray no longer provides an entry point named ``memray3.N`` (where "3.N" is the Python version that Memray was installed for). You can use ``python3.N -m memray`` instead. From 44ec99fbb33243b1f5031a180192086eff927161 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 24 Sep 2024 20:18:23 -0400 Subject: [PATCH 9/9] Find `_inject.abi3.so` relative to `_memray.*.so` With an editable `scikit-build-core` install, shared libraries are not in the same directory as Python files, so we can no longer locate it relative to `__init__.py`. Signed-off-by: Matt Wozniski --- src/memray/commands/attach.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memray/commands/attach.py b/src/memray/commands/attach.py index 6e239e4411..c89065a0ad 100644 --- a/src/memray/commands/attach.py +++ b/src/memray/commands/attach.py @@ -130,7 +130,7 @@ def deactivate_because_timer_elapsed(): def inject(debugger: str, pid: int, port: int, verbose: bool) -> str | None: """Executes a file in a running Python process.""" - injecter = pathlib.Path(memray.__file__).parent / "_inject.abi3.so" + injecter = pathlib.Path(memray._memray.__file__).parent / "_inject.abi3.so" assert injecter.exists() gdb_cmd = [