From cda5716b2de912e98bf81b72e3c9a00713c60a8e Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sat, 14 Nov 2026 18:03:35 +0100 Subject: [PATCH 01/13] ci: unlink the brew formulae only when they are installed --- .github/workflows/build-macos.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index cc0d3bf..f1ba535 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -38,11 +38,17 @@ jobs: - name: Install dependencies run: | HOMEBREW_NO_AUTO_UPDATE=1 brew install python boost@1.85 hidapi openssl@3 zmq libpgm miniupnpc expat libunwind-headers protobuf ccache - brew unlink boost || true - brew link boost@1.85 --force + # the image of the runner can have another boost linked, which gets in the way of boost@1.85. They are touched only when + # they are there, brew prints an error for a formula that is not installed (the annotations of the run show it as one) + if brew list --versions boost > /dev/null 2>&1; then + brew unlink boost + brew link boost@1.85 --force + fi # brew links openssl@4 into /opt/homebrew, and the -I/opt/homebrew/include of boost and zeromq comes # before the one of openssl@3, so its headers win over the ones of OPENSSL_ROOT_DIR - brew unlink openssl@4 || true + if brew list --versions openssl@4 > /dev/null 2>&1; then + brew unlink openssl@4 + fi # brew's openssl is OpenSSL 4 on the newer runner images, which boost 1.85 does not build with. # monero-project and monero-cpp take brew's openssl unless they are given another one, so # they get it on the command line, and the module finds it in the environment From 9897a7bc160441171c677385d16beb562f763330 Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 17:56:43 +0100 Subject: [PATCH 02/13] ci: rename build workflow --- .github/workflows/build-linux.yml | 32 +++++---- .github/workflows/build-macos.yml | 29 +++++--- .github/workflows/build-windows.yml | 38 +++++----- .github/workflows/{build.yml => ci.yml} | 94 ++++++++++++++----------- .github/workflows/codacy.yml | 8 +-- .github/workflows/release.yml | 8 +-- README.md | 2 +- bin/install_dependencies_linux.sh | 2 +- bin/show_versions_linux.sh | 11 +++ docs/ci-ccache.md | 4 +- tests/requirements.txt | 2 +- 11 files changed, 137 insertions(+), 93 deletions(-) rename .github/workflows/{build.yml => ci.yml} (81%) create mode 100755 bin/show_versions_linux.sh diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 76de948..5670d13 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -1,7 +1,7 @@ -name: Build Linux Module +name: Build Linux -# Called by build.yml. It builds monero-project, monero-cpp and the module, the last two with the coverage on, and uploads the module -# with the files that the coverage needs. The tests of build.yml run on it, in several jobs. The packages of Linux are the deb ones, +# Called by ci.yml. It builds monero-project, monero-cpp and the module, the last two with the coverage on, and uploads the module +# with the files that the coverage needs. The tests of ci.yml run on it, in several jobs. The packages of Linux are the deb ones, # which build-deb.yml makes. on: workflow_call: @@ -26,12 +26,16 @@ jobs: persist-credentials: false submodules: recursive - - name: Install dependencies - run: | - bin/install_dependencies_linux.sh - pip3 install -r requirements-build.txt --break-system-packages + - name: Install apt packages + run: bin/install_dependencies_linux.sh + + - name: Show the versions of the apt libraries + run: bin/show_versions_linux.sh + + - name: Install the Python build requirements (requirements-build.txt) + run: pip3 install -r requirements-build.txt --break-system-packages - - name: Install expat + - name: Install expat 2.7.1 (static) run: | wget https://github.com/libexpat/libexpat/releases/download/R_2_7_1/expat-2.7.1.tar.bz2 echo "45c98ae1e9b5127325d25186cf8c511fa814078e9efeae7987a574b482b79b3d expat-2.7.1.tar.bz2" | sha256sum -c @@ -42,7 +46,7 @@ jobs: sudo make install cd ../ - - name: Install unbound + - name: Install unbound 1.22.0 (static) run: | wget https://www.nlnetlabs.nl/downloads/unbound/unbound-1.22.0.tar.gz echo "c5dd1bdef5d5685b2cedb749158dd152c52d44f65529a34ac15cd88d4b1b3d43 unbound-1.22.0.tar.gz" | sha256sum -c @@ -65,7 +69,7 @@ jobs: - name: Reset ccache statistics run: ccache --zero-stats - - name: Build monero-project + - name: Build monero-project (static, Release) run: | cd external/monero-cpp/external/monero-project mkdir -p build/release @@ -77,7 +81,7 @@ jobs: make -j3 wallet cryptonote_protocol cd ../../../../../../ - - name: Build monero-cpp libraries + - name: Build monero-cpp (with coverage) run: | cd external/monero-cpp mkdir -p build @@ -87,7 +91,7 @@ jobs: make -j3 cd ../../../ - - name: Build monero-python module + - name: Build the monero-python module (with coverage) run: | mkdir -p build cmake -B build -DENABLE_COVERAGE=ON -Dpybind11_DIR="$(python3 -m pybind11 --cmakedir)" @@ -113,14 +117,14 @@ jobs: key: ccache-linux-test # The module, and the .gcno that lcov reads together with the .gcda that the tests write: the ones of monero-cpp and of the module - # are the only ones, monero-project is not built with the coverage. The tests of build.yml extract it at the root of the + # are the only ones, monero-project is not built with the coverage. The tests of ci.yml extract it at the root of the # workspace, where the paths of the .gcda are the ones that the build had. - name: Package the build for the tests run: | find build external/monero-cpp/build \( -name 'monero*.so' -o -name '*.gcno' \) -print0 | tar --null -czf linux-build.tar.gz -T - ls -l linux-build.tar.gz - - name: Upload monero-python build + - name: Upload the Linux build uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: monero-python-linux-x86_64 diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index f1ba535..17b1345 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -1,6 +1,6 @@ -name: Build MacOS +name: Build macOS -# Called by build.yml, which has the triggers and the concurrency group: a group here would be the one of the caller, and the +# Called by ci.yml, which has the triggers and the concurrency group: a group here would be the one of the caller, and the # jobs of the same run would cancel each other. on: workflow_call: @@ -35,7 +35,7 @@ jobs: persist-credentials: false submodules: recursive - - name: Install dependencies + - name: Install brew formulae (boost@1.85, openssl@3 and the other libraries) run: | HOMEBREW_NO_AUTO_UPDATE=1 brew install python boost@1.85 hidapi openssl@3 zmq libpgm miniupnpc expat libunwind-headers protobuf ccache # the image of the runner can have another boost linked, which gets in the way of boost@1.85. They are touched only when @@ -53,6 +53,15 @@ jobs: # monero-project and monero-cpp take brew's openssl unless they are given another one, so # they get it on the command line, and the module finds it in the environment echo "OPENSSL_ROOT_DIR=$(brew --prefix openssl@3)" >> "$GITHUB_ENV" + + - name: Show the versions of the brew formulae + run: | + brew list --versions python boost@1.85 hidapi openssl@3 zmq libpgm miniupnpc expat libunwind-headers protobuf ccache + cmake --version | head -n 1 + clang --version | head -n 1 + + - name: Create the Python virtual environment and install the build requirements (requirements-build.txt) + run: | # a virtual environment: the python of brew has packages that pip can not replace, such as wheel and typing_extensions on the # Intel runner (they have no RECORD file), and the versions of the requirements files are pinned. The next steps find it first in the PATH python3 -m venv "$RUNNER_TEMP/venv" @@ -60,7 +69,7 @@ jobs: "$RUNNER_TEMP/venv/bin/pip" install -r requirements-build.txt # static, against openssl@3: the unbound of brew is built with OpenSSL 4 - - name: Install unbound (static) + - name: Install unbound 1.22.0 (static) run: bin/install_unbound_macos.sh - name: Restore ccache @@ -74,7 +83,7 @@ jobs: - name: Reset ccache statistics run: ccache --zero-stats - - name: Build monero-project + - name: Build monero-project (static, Release) run: | cd external/monero-cpp/external/monero-project mkdir -p build/release @@ -87,7 +96,7 @@ jobs: make -j3 wallet cryptonote_protocol cd ../../../../../../ - - name: Build monero-cpp libraries + - name: Build monero-cpp run: | cd external/monero-cpp mkdir -p build @@ -97,7 +106,7 @@ jobs: make -j3 cd ../../../ - - name: Build monero-python module + - name: Build the monero-python wheel run: | mkdir -p build python3 -m pip wheel . -w dist --no-build-isolation @@ -113,17 +122,17 @@ jobs: with: key: ccache-${{ matrix.os }} - - name: Repair wheel with delocate + - name: Repair the wheel with delocate run: | delocate-wheel -w wheelhouse -v dist/*.whl # the canary and the exports of the module in the wheel that is published - - name: Verify libraries + - name: Check the hardening of the module in the wheel run: | python3 -m zipfile -e wheelhouse/*.whl wheel-check bin/check_macho_hardening.sh wheel-check/monero*.so - - name: Upload monero-python wheel + - name: Upload the macOS wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: monero-${{ matrix.os }} diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index f6742fd..cbb307c 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -1,6 +1,6 @@ -name: Build Windows Package +name: Build Windows -# Called by build.yml, which has the triggers and the concurrency group: a group here would be the one of the caller, and the +# Called by ci.yml, which has the triggers and the concurrency group: a group here would be the one of the caller, and the # jobs of the same run would cancel each other. on: workflow_call: @@ -29,18 +29,18 @@ jobs: persist-credentials: false submodules: recursive - - name: Setup Python 3.13 + - name: Install Python 3.13 uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: '3.13' architecture: 'x64' - - name: Install Python build dependencies + - name: Install the Python build requirements (requirements-build.txt) shell: bash run: | python -m pip install -r requirements-build.txt - - name: Setup MSYS2 MINGW64 + - name: Install MSYS2 and its MINGW64 packages (openssl, zeromq, libsodium, unbound and protobuf at the version of the repo) uses: msys2/setup-msys2@ec48f7c5447b3140e2b088413ae3a55687bccb6e # v2.33.0 with: msystem: MINGW64 @@ -60,7 +60,7 @@ jobs: base-devel wget - - name: Install pinned MinGW64 toolchain v16.2.0 + - name: Install MinGW64 toolchain 16.2.0 (pinned) run: | wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst echo "acad5748942953f91a0f2bba03e20437b909366e55949f478f1b2f2d30c726a7 mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c @@ -107,13 +107,13 @@ jobs: mingw-w64-x86_64-gcc-libs-16.2.0-3-any.pkg.tar.zst \ mingw-w64-x86_64-gcc-16.2.0-3-any.pkg.tar.zst - - name: Install ICU v78.3 + - name: Install ICU 78.3 (pinned) run: | wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst echo "8486a018aca2e56d1fcd2eed2069011f2e1da02e03af72b38916ee06cbe7203f mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst" | sha256sum -c pacman -U --noconfirm mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst - - name: Install hidapi v0.15.0 and libusb v1.0.30 + - name: Install hidapi 0.15.0 and libusb 1.0.30 (pinned) run: | # dropped from the msys2 mingw64 repo, install the archived packages wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst @@ -122,7 +122,7 @@ jobs: echo "124edc26b009678a4c5e759481a364abf63ed7f2f49a53092e8bd0a47c0c0740 mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst" | sha256sum -c pacman -U --noconfirm mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst - - name: Install boost v1.87.0 + - name: Install boost 1.87.0 (pinned) run: | wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst echo "241919a5885a9270d0e0937e71cf5e146c2a8cabecc2714f09248350181cb1b6 mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst" | sha256sum -c @@ -131,12 +131,18 @@ jobs: # the version that monero-cpp builds with. The boost-libs in the repo is a newer rebuild that needs the split gcc runtime pacman -U --noconfirm mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst - - name: Install pybind11 v2.13.6 + - name: Install pybind11 2.13.6 (pinned) run: | wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst echo "5b56ad87bc544336b2d51edb8bb0614f46542571362daf4d7e53bf23120175a1 mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst" | sha256sum -c pacman -U --noconfirm mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst + - name: Show the versions of the MSYS2 packages + run: | + pacman -Q mingw-w64-x86_64-gcc mingw-w64-x86_64-binutils mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq \ + mingw-w64-x86_64-libsodium mingw-w64-x86_64-unbound mingw-w64-x86_64-protobuf mingw-w64-x86_64-boost mingw-w64-x86_64-icu \ + mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-pybind11 mingw-w64-x86_64-ccache + - name: Restore ccache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -148,7 +154,7 @@ jobs: - name: Reset ccache statistics run: ccache --zero-stats - - name: Build monero-project + - name: Build monero-project (static, Release) run: | cd external/monero-cpp/external/monero-project mkdir -p build/release @@ -164,7 +170,7 @@ jobs: ../../ make -j$(nproc) wallet cryptonote_protocol - - name: Build monero-cpp libraries + - name: Build monero-cpp run: | cd external/monero-cpp mkdir -p build @@ -172,7 +178,7 @@ jobs: cmake .. -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache cmake --build . --parallel $(nproc) - - name: Build monero-python module + - name: Build the monero-python wheel run: | export WIN_PYTHON_EXE=$(cygpath -u "$pythonLocation/python.exe") export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) @@ -192,20 +198,20 @@ jobs: with: key: ccache-windows - - name: Repair wheel with delvewheel + - name: Repair the wheel with delvewheel run: | export WIN_PYTHON_EXE=$(cygpath -u "$pythonLocation/python.exe") MINGW_BIN_WIN=$(cygpath -w "$MINGW_PREFIX/bin") "$WIN_PYTHON_EXE" -m delvewheel repair dist/*.whl -w wheelhouse --add-path "$MINGW_BIN_WIN" # the flags of the image (ASLR, DEP) and the exports of the module in the wheel that is published - - name: Verify libraries + - name: Check the hardening of the module in the wheel run: | export WIN_PYTHON_EXE=$(cygpath -u "$pythonLocation/python.exe") "$WIN_PYTHON_EXE" -m zipfile -e wheelhouse/*.whl wheel-check bash bin/check_pe_hardening.sh wheel-check/monero*.pyd - - name: Upload monero-python wheel + - name: Upload the Windows wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: monero-python-win-amd64 diff --git a/.github/workflows/build.yml b/.github/workflows/ci.yml similarity index 81% rename from .github/workflows/build.yml rename to .github/workflows/ci.yml index eab827c..881cd32 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Build +name: CI # The builds are in build-linux.yml, build-windows.yml and build-macos.yml, called from here, and the tests run on what they built: the # wheel that a build uploads is the one that its tests install, and the module of Linux is the one that its shards download, so nothing @@ -11,7 +11,7 @@ on: branches: - main # Only on pull requests: the release needs a successful run for every commit on main. - # If Build ever becomes a required check, a skipped run stays Pending and blocks the merge. + # If CI ever becomes a required check, a skipped run stays Pending and blocks the merge. paths-ignore: - 'README.md' - 'LICENSE' @@ -38,26 +38,30 @@ permissions: contents: read jobs: - build-ubuntu: + build-linux: + # the name of the caller is the first part of the name of the checks of its job: "build / linux-x86_64" + name: build permissions: contents: read actions: write # build-linux.yml replaces its ccache entries uses: ./.github/workflows/build-linux.yml build-windows: + name: build permissions: contents: read actions: write # build-windows.yml replaces its ccache entries uses: ./.github/workflows/build-windows.yml - build-mac-os: + build-macos: + name: build permissions: contents: read actions: write # build-macos.yml replaces its ccache entries uses: ./.github/workflows/build-macos.yml test-windows: - name: windows (pytest) + name: unit tests / windows needs: build-windows runs-on: windows-latest timeout-minutes: 30 @@ -68,34 +72,34 @@ jobs: with: persist-credentials: false - - name: Setup Python 3.13 + - name: Install Python 3.13 uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: '3.13' architecture: 'x64' - - name: Download monero-python wheel + - name: Download the Windows wheel uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: monero-python-win-amd64 path: wheelhouse - - name: Install dependencies + - name: Install the test requirements and the wheel shell: bash run: | python -m pip install -r tests/requirements.txt python -m pip install wheelhouse/*.whl - - name: Run unit tests + - name: Run unit tests (pytest) shell: bash env: IN_CONTAINER: "true" run: | python -m pytest -m unit - test-mac-os: - name: ${{ matrix.arch }} (pytest) - needs: build-mac-os + test-macos: + name: unit tests / ${{ matrix.arch }} + needs: build-macos runs-on: ${{ matrix.os }} timeout-minutes: 30 @@ -114,23 +118,27 @@ jobs: with: persist-credentials: false - - name: Download monero-python wheel + - name: Download the macOS wheel (${{ matrix.arch }}) uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: monero-${{ matrix.os }} path: wheelhouse - - name: Install dependencies + # the python that the build used, the wheel is for its version + - name: Install Python (brew) run: | - # the python that the build used, the wheel is for its version HOMEBREW_NO_AUTO_UPDATE=1 brew install python - # a virtual environment, as in the build: the python of brew has packages that pip can not replace, and the - # versions of the requirements file are pinned + brew list --versions python + + # a virtual environment, as in the build: the python of brew has packages that pip can not replace, and the + # versions of the requirements file are pinned + - name: Install the test requirements and the wheel in a virtual environment + run: | python3 -m venv "$RUNNER_TEMP/venv" echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" "$RUNNER_TEMP/venv/bin/pip" install -r tests/requirements.txt wheelhouse/*.whl - - name: Run unit tests + - name: Run unit tests (pytest) run: | pytest -m unit @@ -138,9 +146,9 @@ jobs: # wallet rpc servers of docker. Together they are all the tests. Each one uploads its coverage, and the coverage job merges them. # The name of an entry is in the names of files and artifacts, so it has no spaces: lcov splits the path of a tracefile at them, # and the merge of the coverage failed with a name that had one. The title is what the job shows. - test-ubuntu: - name: linux-x86_64 ${{ matrix.title }} (pytest) - needs: build-ubuntu + test-linux: + name: ${{ matrix.title }} / linux-x86_64 + needs: build-linux runs-on: ubuntu-24.04 timeout-minutes: 90 @@ -164,29 +172,34 @@ jobs: persist-credentials: false submodules: recursive - - name: Download monero-python build + - name: Download the Linux build (module and coverage files) uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: monero-python-linux-x86_64 # at the root of the workspace: the module in build/, and the .gcno where the tests will write the .gcda - - name: Extract monero-python build + - name: Extract the Linux build run: tar -xzf linux-build.tar.gz # the module is linked to shared libraries of apt, the ones of the build - - name: Install dependencies - run: | - bin/install_dependencies_linux.sh - pip3 install -r tests/requirements.txt --break-system-packages + - name: Install apt packages + run: bin/install_dependencies_linux.sh + + - name: Show the versions of the apt libraries + run: bin/show_versions_linux.sh + + - name: Install the Python test requirements (tests/requirements.txt) + run: pip3 install -r tests/requirements.txt --break-system-packages - - name: Setup test environment + - name: Start the test environment (2 nodes and 3 wallet rpc servers in docker) if: ${{ matrix.docker }} run: | docker compose -f tests/docker-compose.yml up -d node_1 node_2 xmr_wallet_1 xmr_wallet_2 xmr_wallet_3 + docker compose -f tests/docker-compose.yml images sleep 10 # --durations shows where a shard spends its time, to balance them - - name: Run tests + - name: Run ${{ matrix.title }} (pytest) env: IN_CONTAINER: "true" PYTHONPATH: ${{ github.workspace }}/build @@ -196,7 +209,7 @@ jobs: read -r -a args <<< "$PYTEST_ARGS" pytest --cov=tests --cov-report= --durations=25 "${args[@]}" - - name: Dump container logs + - name: Collect the container logs if: ${{ always() && matrix.docker }} run: | mkdir -p container-logs @@ -211,11 +224,11 @@ jobs: name: container-logs-${{ matrix.name }} path: container-logs/ - - name: Cleanup test environment + - name: Stop the test environment if: ${{ always() && matrix.docker }} run: docker compose -f tests/docker-compose.yml down -v - - name: Upload tests logs + - name: Upload the test logs if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: @@ -250,9 +263,9 @@ jobs: # The coverage of the tests of all the shards, in the same reports that Tests made (the artifact has the same name, and Codacy and the # release read it). It runs when a shard fails too, with the shards that have it, since a failing test is still reported. coverage: - name: coverage - needs: [build-ubuntu, test-ubuntu] - if: ${{ !cancelled() && needs.build-ubuntu.result == 'success' }} + name: coverage report + needs: [build-linux, test-linux] + if: ${{ !cancelled() && needs.build-linux.result == 'success' }} runs-on: ubuntu-24.04 timeout-minutes: 30 @@ -263,11 +276,12 @@ jobs: persist-credentials: false submodules: recursive - - name: Install dependencies + - name: Install lcov and the Python test requirements run: | sudo apt-get update sudo apt-get install -y lcov pip3 install -r tests/requirements.txt --break-system-packages + lcov --version # a folder for each shard, named as the artifact - name: Download the coverage of the shards @@ -276,7 +290,7 @@ jobs: pattern: coverage-* path: shards - - name: Generate python bindings coverage report + - name: Generate the python bindings coverage report run: | args=() for f in shards/*/coverage.info; do args+=(-a "$f"); done @@ -284,13 +298,13 @@ jobs: genhtml coverage.info --output-directory coverage_monero_python_bindings_html --ignore-errors mismatch,inconsistent,source,corrupt --title "monero-python bindings coverage @ ${GITHUB_SHA::7}" lcov --summary coverage.info - - name: Generate python tests coverage report + - name: Generate the python tests coverage report run: | coverage combine shards/*/pytest-coverage-* coverage xml coverage html -d coverage_monero_python_tests_html --title "monero-python tests coverage @ ${GITHUB_SHA::7}" - - name: Generate monero-cpp coverage report + - name: Generate the monero-cpp coverage report run: | args=() for f in shards/*/coverage_monero_cpp.info; do args+=(-a "$f"); done @@ -298,7 +312,7 @@ jobs: genhtml coverage_monero_cpp.info --output-directory coverage_monero_cpp_html --ignore-errors mismatch,inconsistent,source,corrupt --title "monero-cpp coverage @ ${GITHUB_SHA::7}" lcov --summary coverage_monero_cpp.info - - name: Upload coverage report + - name: Upload the coverage reports uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-reports-linux diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml index cbfd2d5..8892eee 100644 --- a/.github/workflows/codacy.yml +++ b/.github/workflows/codacy.yml @@ -1,18 +1,18 @@ name: Codacy Coverage Reporter -# workflow_run is what gives this job the secret after a run of Build on a pull request. It does not check out or run anything +# workflow_run is what gives this job the secret after a run of CI on a pull request. It does not check out or run anything # of the pull request: it downloads the coverage files, and reads them with a reporter pinned to a checksum. on: # zizmor: ignore[dangerous-triggers] workflow_run: workflows: - - Build + - CI types: - completed workflow_dispatch: permissions: contents: read - # needed to download the artifact of the Build run with the default GITHUB_TOKEN + # needed to download the artifact of the CI run with the default GITHUB_TOKEN actions: read jobs: @@ -51,7 +51,7 @@ jobs: if: ${{ !cancelled() && steps.download.outcome == 'success' }} env: CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} - # the commit that Build ran on, which is the head of the pull request. A manual run has no workflow_run + # the commit that CI ran on, which is the head of the pull request. A manual run has no workflow_run COMMIT_UUID: ${{ github.event.workflow_run.head_sha || github.sha }} run: | ./codacy-coverage-reporter report --language python --coverage-reports coverage.xml --partial --commit-uuid "$COMMIT_UUID" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8b69eb1..13834a7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ permissions: actions: read jobs: - # the macos and windows builds, the tests and the coverage are those of the Build run of main, the deb ones are built here + # the macos and windows builds, the tests and the coverage are those of the CI run of main, the deb ones are built here check: name: Check the tag and the builds of main runs-on: ubuntu-latest @@ -37,16 +37,16 @@ jobs: fi # a successful run is one in which every build and every test passed - - name: Find the main-branch Build run for this commit + - name: Find the main-branch CI run for this commit id: find_run env: GH_TOKEN: ${{ github.token }} run: | - run_id=$(gh run list --repo "${{ github.repository }}" --workflow build.yml \ + run_id=$(gh run list --repo "${{ github.repository }}" --workflow ci.yml \ --branch main --status success --limit 200 --json databaseId,headSha \ --jq ".[] | select(.headSha == \"${{ github.sha }}\") | .databaseId" | head -n1) if [ -z "$run_id" ]; then - echo "::error::No successful Build run on main for commit ${{ github.sha }}. Push to main and wait for it to build and pass before tagging." + echo "::error::No successful CI run on main for commit ${{ github.sha }}. Push to main and wait for it to build and pass before tagging." exit 1 fi echo "build_run_id=${run_id}" >> "$GITHUB_OUTPUT" diff --git a/README.md b/README.md index 2df42aa..656bfab 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Codacy Badge](https://app.codacy.com/project/badge/Grade/f6129b0bbe314061b34a262a4cb89109)](https://app.codacy.com/gh/libmonero/monero-python/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/f6129b0bbe314061b34a262a4cb89109)](https://app.codacy.com/gh/libmonero/monero-python/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_coverage) -[![Build](https://github.com/libmonero/monero-python/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/libmonero/monero-python/actions/workflows/build.yml) +[![CI](https://github.com/libmonero/monero-python/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/libmonero/monero-python/actions/workflows/ci.yml) [![Build Debian Packages](https://github.com/libmonero/monero-python/actions/workflows/build-deb.yml/badge.svg?branch=main)](https://github.com/libmonero/monero-python/actions/workflows/build-deb.yml) diff --git a/bin/install_dependencies_linux.sh b/bin/install_dependencies_linux.sh index af30650..7d8fbeb 100755 --- a/bin/install_dependencies_linux.sh +++ b/bin/install_dependencies_linux.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# The packages of apt for the Linux build of build-linux.yml, and for the tests of build.yml: the module is linked to the shared libraries +# The packages of apt for the Linux build of build-linux.yml, and for the tests of ci.yml: the module is linked to the shared libraries # of some of them (the boost, zeromq, sodium and protobuf ones), so a job that only runs the module needs them too. The development packages # bring the libraries, and are the same ones in both so that the module finds at run time what it was linked to. set -euo pipefail diff --git a/bin/show_versions_linux.sh b/bin/show_versions_linux.sh new file mode 100755 index 0000000..c48e255 --- /dev/null +++ b/bin/show_versions_linux.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Prints the versions of the tools and of the libraries that bin/install_dependencies_linux.sh installs with apt, for the log of a job. They are the ones that the +# distribution of the runner has, not pinned ones, so what a run built and tested with is only known from its log. +set -uo pipefail + +# only the installed ones: a pattern also matches the packages of other versions that the distribution has, which are not installed +dpkg-query -W -f='${db:Status-Abbrev} ${Package} ${Version}\n' \ + libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libexpat1-dev libpgm-dev \ + libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler 'libboost1.*-dev' \ + cmake ccache python3 lcov 2> /dev/null | awk '$1 == "ii" { print $2, $3 }' +g++ --version | head -n 1 diff --git a/docs/ci-ccache.md b/docs/ci-ccache.md index 2da26d7..387d033 100644 --- a/docs/ci-ccache.md +++ b/docs/ci-ccache.md @@ -12,7 +12,7 @@ The workflows keep a ccache directory (`.ccache` in the workspace) between runs ### Where the cache lives - Key is `ccache-_`, restore falls back to the newest `ccache-_`. The `_` is there because `macos-15` is a prefix of `macos-15-intel`. -- Each build has its own cache: Linux, Windows and macOS (one for each architecture) in `build.yml`, and the deb builds one per distro and arch. The tests do not compile, they use what the builds uploaded. +- Each build has its own cache: Linux, Windows and macOS (one for each architecture) in `ci.yml`, and the deb builds one per distro and arch. The tests do not compile, they use what the builds uploaded. - Pull requests only restore. Pushes to main and manual runs (`workflow_dispatch`) save. This is on purpose, see quota below. The deb workflow also runs on pull requests that touch its own files, and follows the same rule. - A cache saved on a branch is only visible to that branch. Main's caches are visible to everyone, so a pull request starts from main's. - The save step runs after the build and before the tests, so a flaky test doesn't throw the cache away. Failed builds save too, whatever got compiled is still good. @@ -44,6 +44,6 @@ Measured on `build monero` for Windows: about 9 minutes with a cold cache, about ### Checking it works -Run `build.yml` twice on the same branch with `workflow_dispatch`. Second run should show mostly hits in the "Show ccache statistics" step (for deb it's the end of the build step, printed by `debian/rules`). Stats are zeroed at the start of each job, so they're only for that run. +Run `ci.yml` twice on the same branch with `workflow_dispatch`. Second run should show mostly hits in the "Show ccache statistics" step (for deb it's the end of the build step, printed by `debian/rules`). Stats are zeroed at the start of each job, so they're only for that run. To start over: `gh cache list`, then `gh cache delete `. diff --git a/tests/requirements.txt b/tests/requirements.txt index 4caa5f3..4f0b49a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,4 @@ -# The versions that the tests run with, in CI (.github/workflows/build.yml) and locally: pip install -r tests/requirements.txt +# The versions that the tests run with, in CI (.github/workflows/ci.yml) and locally: pip install -r tests/requirements.txt # Dependabot proposes the updates. coverage==7.17.3 pytest==9.2.0 From 4fbd2066b1e590d370b886f10c57d0c1468fd561 Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 17:59:35 +0100 Subject: [PATCH 03/13] ci: name the steps of the deb build as the ones of the other builds --- .github/workflows/build-deb.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 73068d9..d6f1435 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -84,7 +84,7 @@ jobs: # Installs the Build-Depends of debian/control in the container, sets the version of # debian/changelog from pyproject.toml and runs dpkg-buildpackage. The workspace is # mounted at /github/workspace in the container. - - name: Build package + - name: Build the deb package (dpkg-buildpackage) uses: jtdor/build-deb-action@473136f14a42a231124c1f7cefaedfe941203c38 # v1.9.1 env: CCACHE_DIR: /github/workspace/.ccache @@ -104,7 +104,7 @@ jobs: with: key: ccache-deb-${{ matrix.name }}-${{ matrix.arch }} - - name: Upload monero-python package + - name: Upload the deb package uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: python3-monero_${{ steps.version.outputs.version }}-1${{ matrix.name }}1_${{ matrix.arch }} @@ -113,7 +113,7 @@ jobs: # the segments and the exports of the module in the package, and the stack protector and the branch protection in its code. The # module of the armhf package is read but not disassembled, the objdump of the arm64 runner does not read 32 bit arm. # After the upload, so that a package that fails can be downloaded and looked at. The release needs this job to succeed - - name: Verify libraries + - name: Check the hardening of the module in the package run: | dpkg-deb -x debian/artifacts/python3-monero_*.deb deb-check bin/check_elf_hardening.sh deb-check/usr/lib/python3/dist-packages/monero*.so From 41f47f066a5e2fe813330b996f27951b959c74ae Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 17:59:52 +0100 Subject: [PATCH 04/13] ci: restore the ccache with a local action, as it is saved --- .github/actions/restore-ccache/action.yml | 23 +++++++++++++++++++++++ .github/workflows/build-deb.yml | 5 +---- .github/workflows/build-linux.yml | 5 +---- .github/workflows/build-macos.yml | 5 +---- .github/workflows/build-windows.yml | 5 +---- docs/ci-ccache.md | 2 +- 6 files changed, 28 insertions(+), 17 deletions(-) create mode 100644 .github/actions/restore-ccache/action.yml diff --git a/.github/actions/restore-ccache/action.yml b/.github/actions/restore-ccache/action.yml new file mode 100644 index 0000000..0dc8f81 --- /dev/null +++ b/.github/actions/restore-ccache/action.yml @@ -0,0 +1,23 @@ +name: Restore ccache +description: > + Restores the ccache that save-ccache stored under the key, or else the newest one stored under a key that starts with the key and a `_` + (the `_` is there because `macos-15` is a prefix of `macos-15-intel`). The cache of a name is the one of its last run on main. + +inputs: + key: + description: Cache key, the same one given to save-ccache + required: true + path: + description: Directory of the ccache + default: .ccache + +runs: + using: composite + steps: + - name: Restore the cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: | + ${{ inputs.key }}_ diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index d6f1435..d3ea647 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -74,12 +74,9 @@ jobs: docker system prune -af || true - name: Restore ccache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: ./.github/actions/restore-ccache with: - path: .ccache key: ccache-deb-${{ matrix.name }}-${{ matrix.arch }} - restore-keys: | - ccache-deb-${{ matrix.name }}-${{ matrix.arch }}_ # Installs the Build-Depends of debian/control in the container, sets the version of # debian/changelog from pyproject.toml and runs dpkg-buildpackage. The workspace is diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 5670d13..3e45799 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -59,12 +59,9 @@ jobs: cd ../ - name: Restore ccache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: ./.github/actions/restore-ccache with: - path: .ccache key: ccache-linux-test - restore-keys: | - ccache-linux-test_ - name: Reset ccache statistics run: ccache --zero-stats diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 17b1345..d9ca007 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -73,12 +73,9 @@ jobs: run: bin/install_unbound_macos.sh - name: Restore ccache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: ./.github/actions/restore-ccache with: - path: .ccache key: ccache-${{ matrix.os }} - restore-keys: | - ccache-${{ matrix.os }}_ - name: Reset ccache statistics run: ccache --zero-stats diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index cbb307c..7318679 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -144,12 +144,9 @@ jobs: mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-pybind11 mingw-w64-x86_64-ccache - name: Restore ccache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: ./.github/actions/restore-ccache with: - path: .ccache key: ccache-windows - restore-keys: | - ccache-windows_ - name: Reset ccache statistics run: ccache --zero-stats diff --git a/docs/ci-ccache.md b/docs/ci-ccache.md index 387d033..bc2f2b1 100644 --- a/docs/ci-ccache.md +++ b/docs/ci-ccache.md @@ -11,7 +11,7 @@ The workflows keep a ccache directory (`.ccache` in the workspace) between runs ### Where the cache lives -- Key is `ccache-_`, restore falls back to the newest `ccache-_`. The `_` is there because `macos-15` is a prefix of `macos-15-intel`. +- Key is `ccache-_`, restore falls back to the newest `ccache-_`. The `_` is there because `macos-15` is a prefix of `macos-15-intel`. The local actions `.github/actions/restore-ccache` and `.github/actions/save-ccache` do both, the builds only give them the key. - Each build has its own cache: Linux, Windows and macOS (one for each architecture) in `ci.yml`, and the deb builds one per distro and arch. The tests do not compile, they use what the builds uploaded. - Pull requests only restore. Pushes to main and manual runs (`workflow_dispatch`) save. This is on purpose, see quota below. The deb workflow also runs on pull requests that touch its own files, and follows the same rule. - A cache saved on a branch is only visible to that branch. Main's caches are visible to everyone, so a pull request starts from main's. From 02df2de0c5363de78acd6b58d10ffa2e1c1e5826 Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 18:00:57 +0100 Subject: [PATCH 05/13] ci: install the pinned MSYS2 packages of Windows from bin/msys2_pins.txt, with a script --- .github/workflows/build-windows.yml | 67 ++++------------------------- README.md | 33 ++++++-------- bin/install_msys2_pins.sh | 28 ++++++++++++ bin/msys2_pins.txt | 22 ++++++++++ 4 files changed, 70 insertions(+), 80 deletions(-) create mode 100755 bin/install_msys2_pins.sh create mode 100644 bin/msys2_pins.txt diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 7318679..969b176 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -60,32 +60,9 @@ jobs: base-devel wget + # the packages, with their sha256, are in bin/msys2_pins.txt - name: Install MinGW64 toolchain 16.2.0 (pinned) run: | - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "acad5748942953f91a0f2bba03e20437b909366e55949f478f1b2f2d30c726a7 mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-crt-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "d24b4e7614756414ae3ef58793265448a3ca7d5fd447a991e9b2facde7b59d75 mingw-w64-x86_64-crt-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-binutils-2.47-3-any.pkg.tar.zst - echo "827363748ce3320683319d860ee4fcfcdbb36baf66aac6dab0ec3822be2d4ff7 mingw-w64-x86_64-binutils-2.47-3-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libmangle-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "481babe8c2fdfa4fc7f51bf13939701719af9f8166ef6a91a9a209cc25961b03 mingw-w64-x86_64-libmangle-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-winpthreads-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "7d86a267b6069198d28f3e77314282ff7b4e3c7bb8fe482299ca58f7a5b439db mingw-w64-x86_64-winpthreads-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libwinpthread-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "1d0ed7fda332a5c879de210f3d72c75fa1f3df7cc63dd951e763dec8eb02c1ee mingw-w64-x86_64-libwinpthread-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-winstorecompat-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "7fcafd7ba30bd47000263e76a06bd893406c64f88955825f0a9188407a35c3d3 mingw-w64-x86_64-winstorecompat-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-tools-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "c17557d09371e39db38deb56c7da6ae54e2c65b6f3466ca02f4da26b616ca8c3 mingw-w64-x86_64-tools-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pkgconf-1~3.0.7-1-any.pkg.tar.zst - echo "37b97f372409fa7cf54d709bab892f34e9c62d2658230a09f783a89a55ae590e mingw-w64-x86_64-pkgconf-1~3.0.7-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-make-4.4.1-5-any.pkg.tar.zst - echo "c19e7caf09bbc89b2730556b2da73004118d4e1f967a685cc48524c7ff80d864 mingw-w64-x86_64-make-4.4.1-5-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-gcc-libs-16.2.0-3-any.pkg.tar.zst - echo "f8e25ea67bb796e7f65550f0dca9fce4cdde8aaa3dadafe4d13c6a8233c8de26 mingw-w64-x86_64-gcc-libs-16.2.0-3-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-gcc-16.2.0-3-any.pkg.tar.zst - echo "8b37c953ae12f3d8fa05c854236463231ea0a294dff423007cbd93d4d8089f9f mingw-w64-x86_64-gcc-16.2.0-3-any.pkg.tar.zst" | sha256sum -c # gcc-libs is split into libgcc, libstdc++ and cc-libs since 16.2.0-4 and the # new packages conflict with the pinned gcc-libs, so drop them before the downgrade for p in libgcc libstdc++ cc-libs; do @@ -93,49 +70,21 @@ jobs: pacman -Rdd --noconfirm mingw-w64-x86_64-$p fi done - pacman -U --noconfirm \ - mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-crt-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-binutils-2.47-3-any.pkg.tar.zst \ - mingw-w64-x86_64-libmangle-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-winpthreads-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-libwinpthread-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-winstorecompat-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-tools-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-pkgconf-1~3.0.7-1-any.pkg.tar.zst \ - mingw-w64-x86_64-make-4.4.1-5-any.pkg.tar.zst \ - mingw-w64-x86_64-gcc-libs-16.2.0-3-any.pkg.tar.zst \ - mingw-w64-x86_64-gcc-16.2.0-3-any.pkg.tar.zst + bin/install_msys2_pins.sh toolchain - name: Install ICU 78.3 (pinned) - run: | - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst - echo "8486a018aca2e56d1fcd2eed2069011f2e1da02e03af72b38916ee06cbe7203f mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst" | sha256sum -c - pacman -U --noconfirm mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst + run: bin/install_msys2_pins.sh icu + # dropped from the msys2 mingw64 repo, install the archived packages - name: Install hidapi 0.15.0 and libusb 1.0.30 (pinned) - run: | - # dropped from the msys2 mingw64 repo, install the archived packages - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst - echo "89fb61549a437d65076a092a367fcb264fc7045ee9ebef48cd81f5a76a5588f8 mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst - echo "124edc26b009678a4c5e759481a364abf63ed7f2f49a53092e8bd0a47c0c0740 mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst" | sha256sum -c - pacman -U --noconfirm mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst + run: bin/install_msys2_pins.sh hidapi-libusb + # the version that monero-cpp builds with. The boost-libs in the repo is a newer rebuild that needs the split gcc runtime - name: Install boost 1.87.0 (pinned) - run: | - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst - echo "241919a5885a9270d0e0937e71cf5e146c2a8cabecc2714f09248350181cb1b6 mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst - echo "5fd77df3e5983ba0d9c689f46151119e0e59b3da6850c5a0ba54068eeae7749c mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst" | sha256sum -c - # the version that monero-cpp builds with. The boost-libs in the repo is a newer rebuild that needs the split gcc runtime - pacman -U --noconfirm mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst + run: bin/install_msys2_pins.sh boost - name: Install pybind11 2.13.6 (pinned) - run: | - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst - echo "5b56ad87bc544336b2d51edb8bb0614f46542571362daf4d7e53bf23120175a1 mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst" | sha256sum -c - pacman -U --noconfirm mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst + run: bin/install_msys2_pins.sh pybind11 - name: Show the versions of the MSYS2 packages run: | diff --git a/README.md b/README.md index 656bfab..b0f2b88 100644 --- a/README.md +++ b/README.md @@ -179,28 +179,19 @@ The package is built from [debian/](debian) with `dpkg-buildpackage`, the way CI ``` pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-unbound mingw-w64-x86_64-protobuf git gettext base-devel wget - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst - echo "8486a018aca2e56d1fcd2eed2069011f2e1da02e03af72b38916ee06cbe7203f mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst" | sha256sum -c - pacman -U mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst - # hidapi and libusb were removed from the msys2 mingw64 repo, install the archived packages - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst - echo "89fb61549a437d65076a092a367fcb264fc7045ee9ebef48cd81f5a76a5588f8 mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst" | sha256sum -c - echo "124edc26b009678a4c5e759481a364abf63ed7f2f49a53092e8bd0a47c0c0740 mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst" | sha256sum -c - pacman -U mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst - echo "241919a5885a9270d0e0937e71cf5e146c2a8cabecc2714f09248350181cb1b6 mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst - echo "5fd77df3e5983ba0d9c689f46151119e0e59b3da6850c5a0ba54068eeae7749c mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst" | sha256sum -c - pacman -U mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst - echo "5b56ad87bc544336b2d51edb8bb0614f46542571362daf4d7e53bf23120175a1 mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst" | sha256sum -c - pacman -U mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst ``` - These are the versions CI uses. CI also pins the MinGW toolchain, see [build-windows.yml](.github/workflows/build-windows.yml). - 5. Clone repo: `git clone --recurse-submodules https://github.com/libmonero/monero-python.git` -6. Build monero-cpp, located as a submodule at ./external/monero-cpp: +6. Install the packages at the versions that CI uses (ICU, hidapi and libusb, boost and pybind11). [bin/msys2_pins.txt](bin/msys2_pins.txt) has them with their sha256, and the script downloads, checks and installs a group of them: + + ``` + cd monero-python + for group in icu hidapi-libusb boost pybind11; do bin/install_msys2_pins.sh $group; done + ``` + + hidapi and libusb were removed from the msys2 mingw64 repo, the script installs the archived packages. CI also pins the MinGW toolchain (`bin/install_msys2_pins.sh toolchain`), see [build-windows.yml](.github/workflows/build-windows.yml). + +7. Build monero-cpp, located as a submodule at ./external/monero-cpp: ``` cd external/monero-cpp @@ -208,7 +199,7 @@ The package is built from [debian/](debian) with `dpkg-buildpackage`, the way CI cd ../.. ``` -7. Build monero-python: +8. Build monero-python: ``` mkdir -p build && @@ -217,7 +208,7 @@ The package is built from [debian/](debian) with `dpkg-buildpackage`, the way CI cmake --build . ``` -8. Or run build script: `./bin/build_libmonero_python.sh` +9. Or run build script: `./bin/build_libmonero_python.sh` ## Use RPC servers in your project: diff --git a/bin/install_msys2_pins.sh b/bin/install_msys2_pins.sh new file mode 100755 index 0000000..7561a91 --- /dev/null +++ b/bin/install_msys2_pins.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Installs a group of the packages that bin/msys2_pins.txt pins, in the MSYS2 of the Windows build (build-windows.yml): it downloads +# each one, checks it against its sha256 and installs the group with a single pacman -U, since the packages of a group depend on each other. +# +# bin/install_msys2_pins.sh +set -euo pipefail + +group=${1:?usage: $0 } +pins="$(dirname "${BASH_SOURCE[0]}")/msys2_pins.txt" +repo=${MSYS2_REPO:-https://repo.msys2.org/mingw/mingw64} + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +files=() +while read -r pin_group sha256 file; do + [ "$pin_group" = "$group" ] || continue + wget -nv -P "$work" "$repo/$file" + echo "$sha256 $work/$file" | sha256sum -c + files+=("$work/$file") +done < <(grep -v '^#' "$pins") + +if [ ${#files[@]} -eq 0 ]; then + echo "No package of the group $group in $pins" >&2 + exit 1 +fi + +pacman -U --noconfirm "${files[@]}" diff --git a/bin/msys2_pins.txt b/bin/msys2_pins.txt new file mode 100644 index 0000000..21ee42f --- /dev/null +++ b/bin/msys2_pins.txt @@ -0,0 +1,22 @@ +# The packages of the MSYS2 mingw64 repository that the Windows build installs at an exact version, for bin/install_msys2_pins.sh. The +# versions of the repository change, and some of these have been dropped from it or need a gcc runtime that is not the one pinned here. +# A line is ` `: a group is installed with one pacman -U. To update, put the new file and its hash, and the +# version in the name of the step of the group in .github/workflows/build-windows.yml. +toolchain acad5748942953f91a0f2bba03e20437b909366e55949f478f1b2f2d30c726a7 mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain d24b4e7614756414ae3ef58793265448a3ca7d5fd447a991e9b2facde7b59d75 mingw-w64-x86_64-crt-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 827363748ce3320683319d860ee4fcfcdbb36baf66aac6dab0ec3822be2d4ff7 mingw-w64-x86_64-binutils-2.47-3-any.pkg.tar.zst +toolchain 481babe8c2fdfa4fc7f51bf13939701719af9f8166ef6a91a9a209cc25961b03 mingw-w64-x86_64-libmangle-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 7d86a267b6069198d28f3e77314282ff7b4e3c7bb8fe482299ca58f7a5b439db mingw-w64-x86_64-winpthreads-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 1d0ed7fda332a5c879de210f3d72c75fa1f3df7cc63dd951e763dec8eb02c1ee mingw-w64-x86_64-libwinpthread-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 7fcafd7ba30bd47000263e76a06bd893406c64f88955825f0a9188407a35c3d3 mingw-w64-x86_64-winstorecompat-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain c17557d09371e39db38deb56c7da6ae54e2c65b6f3466ca02f4da26b616ca8c3 mingw-w64-x86_64-tools-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 37b97f372409fa7cf54d709bab892f34e9c62d2658230a09f783a89a55ae590e mingw-w64-x86_64-pkgconf-1~3.0.7-1-any.pkg.tar.zst +toolchain c19e7caf09bbc89b2730556b2da73004118d4e1f967a685cc48524c7ff80d864 mingw-w64-x86_64-make-4.4.1-5-any.pkg.tar.zst +toolchain f8e25ea67bb796e7f65550f0dca9fce4cdde8aaa3dadafe4d13c6a8233c8de26 mingw-w64-x86_64-gcc-libs-16.2.0-3-any.pkg.tar.zst +toolchain 8b37c953ae12f3d8fa05c854236463231ea0a294dff423007cbd93d4d8089f9f mingw-w64-x86_64-gcc-16.2.0-3-any.pkg.tar.zst +icu 8486a018aca2e56d1fcd2eed2069011f2e1da02e03af72b38916ee06cbe7203f mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst +hidapi-libusb 89fb61549a437d65076a092a367fcb264fc7045ee9ebef48cd81f5a76a5588f8 mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst +hidapi-libusb 124edc26b009678a4c5e759481a364abf63ed7f2f49a53092e8bd0a47c0c0740 mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst +boost 241919a5885a9270d0e0937e71cf5e146c2a8cabecc2714f09248350181cb1b6 mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst +boost 5fd77df3e5983ba0d9c689f46151119e0e59b3da6850c5a0ba54068eeae7749c mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst +pybind11 5b56ad87bc544336b2d51edb8bb0614f46542571362daf4d7e53bf23120175a1 mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst From e04df075a2df78206475735c1cedb30ff6134b9b Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 18:01:12 +0100 Subject: [PATCH 06/13] ci: name the artifacts of the builds as build- and wheel- --- .github/workflows/build-linux.yml | 2 +- .github/workflows/build-macos.yml | 2 +- .github/workflows/build-windows.yml | 2 +- .github/workflows/ci.yml | 6 +++--- .github/workflows/release.yml | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 3e45799..33a95e4 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -124,7 +124,7 @@ jobs: - name: Upload the Linux build uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: monero-python-linux-x86_64 + name: build-linux-x86_64 path: linux-build.tar.gz # already compressed compression-level: 0 diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index d9ca007..ec16919 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -132,5 +132,5 @@ jobs: - name: Upload the macOS wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: monero-${{ matrix.os }} + name: wheel-${{ matrix.arch }} path: wheelhouse/*.whl diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 969b176..b97a925 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -160,5 +160,5 @@ jobs: - name: Upload the Windows wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: monero-python-win-amd64 + name: wheel-windows-x86_64 path: wheelhouse/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 881cd32..5fddb9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,7 +81,7 @@ jobs: - name: Download the Windows wheel uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: monero-python-win-amd64 + name: wheel-windows-x86_64 path: wheelhouse - name: Install the test requirements and the wheel @@ -121,7 +121,7 @@ jobs: - name: Download the macOS wheel (${{ matrix.arch }}) uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: monero-${{ matrix.os }} + name: wheel-${{ matrix.arch }} path: wheelhouse # the python that the build used, the wheel is for its version @@ -175,7 +175,7 @@ jobs: - name: Download the Linux build (module and coverage files) uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: monero-python-linux-x86_64 + name: build-linux-x86_64 # at the root of the workspace: the module in build/, and the .gcno where the tests will write the .gcda - name: Extract the Linux build diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 13834a7..002cd16 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -86,7 +86,7 @@ jobs: with: github-token: ${{ github.token }} run-id: ${{ needs.check.outputs.build_run_id }} - pattern: monero-macos-* + pattern: wheel-mac-* path: release-assets merge-multiple: true @@ -95,7 +95,7 @@ jobs: with: github-token: ${{ github.token }} run-id: ${{ needs.check.outputs.build_run_id }} - name: monero-python-win-amd64 + name: wheel-windows-x86_64 path: release-assets merge-multiple: true From 0f3d87768f2c5f3a380c96ae2d7ac4ce552095b5 Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 18:04:05 +0100 Subject: [PATCH 07/13] ci: give the docker compose commands of the Linux tests the file and the services once --- .github/workflows/ci.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5fddb9b..9f75144 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -151,6 +151,10 @@ jobs: needs: build-linux runs-on: ubuntu-24.04 timeout-minutes: 90 + env: + # what the docker compose commands of the steps work on + COMPOSE_FILE: tests/docker-compose.yml + TEST_SERVICES: node_1 node_2 xmr_wallet_1 xmr_wallet_2 xmr_wallet_3 strategy: fail-fast: false @@ -194,8 +198,9 @@ jobs: - name: Start the test environment (2 nodes and 3 wallet rpc servers in docker) if: ${{ matrix.docker }} run: | - docker compose -f tests/docker-compose.yml up -d node_1 node_2 xmr_wallet_1 xmr_wallet_2 xmr_wallet_3 - docker compose -f tests/docker-compose.yml images + read -r -a services <<< "$TEST_SERVICES" + docker compose up -d "${services[@]}" + docker compose images sleep 10 # --durations shows where a shard spends its time, to balance them @@ -213,8 +218,8 @@ jobs: if: ${{ always() && matrix.docker }} run: | mkdir -p container-logs - for svc in node_1 node_2 xmr_wallet_1 xmr_wallet_2 xmr_wallet_3; do - docker compose -f tests/docker-compose.yml logs --no-color --timestamps "$svc" > "container-logs/$svc.log" 2>&1 || true + for svc in $TEST_SERVICES; do + docker compose logs --no-color --timestamps "$svc" > "container-logs/$svc.log" 2>&1 || true done - name: Upload container logs @@ -226,7 +231,7 @@ jobs: - name: Stop the test environment if: ${{ always() && matrix.docker }} - run: docker compose -f tests/docker-compose.yml down -v + run: docker compose down -v - name: Upload the test logs if: always() From 1da2c18d24dabdd358dab11713f641554e1b04bc Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 18:04:06 +0100 Subject: [PATCH 08/13] ci: take the Python version of the Windows build and tests from one file --- .github/python-version | 1 + .github/workflows/build-windows.yml | 5 +++-- .github/workflows/ci.yml | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 .github/python-version diff --git a/.github/python-version b/.github/python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.github/python-version @@ -0,0 +1 @@ +3.13 diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index b97a925..18b891f 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -29,10 +29,11 @@ jobs: persist-credentials: false submodules: recursive - - name: Install Python 3.13 + # the version is in one file: the wheel that CI builds is for it, and the job that tests the wheel needs the same + - name: Install Python (the version of .github/python-version) uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: '3.13' + python-version-file: .github/python-version architecture: 'x64' - name: Install the Python build requirements (requirements-build.txt) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9f75144..a7e789a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,10 +72,11 @@ jobs: with: persist-credentials: false - - name: Install Python 3.13 + # the version is in one file: the wheel that CI builds is for it, and the job that tests the wheel needs the same + - name: Install Python (the version of .github/python-version) uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: '3.13' + python-version-file: .github/python-version architecture: 'x64' - name: Download the Windows wheel From 662854d16608ed7f995d74d59d7ea82787ccfe30 Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 18:07:56 +0100 Subject: [PATCH 09/13] docs: update README --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index b0f2b88..2cf59b6 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,8 @@ # Monero Python Library +[![CI](https://github.com/libmonero/monero-python/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/libmonero/monero-python/actions/workflows/ci.yml) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/f6129b0bbe314061b34a262a4cb89109)](https://app.codacy.com/gh/libmonero/monero-python/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/f6129b0bbe314061b34a262a4cb89109)](https://app.codacy.com/gh/libmonero/monero-python/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_coverage) -[![CI](https://github.com/libmonero/monero-python/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/libmonero/monero-python/actions/workflows/ci.yml) - -[![Build Debian Packages](https://github.com/libmonero/monero-python/actions/workflows/build-deb.yml/badge.svg?branch=main)](https://github.com/libmonero/monero-python/actions/workflows/build-deb.yml) [![Pybind](https://img.shields.io/badge/pybind-v2-blue?logo=python&logoColor=white)](https://github.com/pybind/pybind11) [![Python](https://img.shields.io/badge/python-3.10_%7C_3.11_%7C_3.12_%7C_3.13_%7C_3.14-blue?logo=python&logoColor=white)](https://github.com/libmonero/monero-python#monero-python-library) From ae1cc0dccf7bc4dec82e1ad13f03db99dc75def2 Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 18:32:03 +0100 Subject: [PATCH 10/13] ci: delete only the ccache of the running ref when saving, and do not save on tags --- .github/actions/save-ccache/action.yml | 11 ++++++++--- .github/workflows/build-deb.yml | 5 +++-- docs/ci-ccache.md | 1 + 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/actions/save-ccache/action.yml b/.github/actions/save-ccache/action.yml index 74c2de9..334fe36 100644 --- a/.github/actions/save-ccache/action.yml +++ b/.github/actions/save-ccache/action.yml @@ -1,6 +1,6 @@ name: Save ccache description: > - Replaces the cache stored under the key instead of adding an entry per commit, so the + Replaces the cache stored under the key, for the ref that is running, instead of adding an entry per commit, so the caches of the repository stay below its quota. The job needs the actions: write permission. inputs: @@ -19,8 +19,13 @@ runs: env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ inputs.key }} - # cache keys are immutable, there is nothing to delete the first time - run: gh cache delete "$CACHE_KEY" --repo "$GITHUB_REPOSITORY" || true + # Only the entry of the ref that is running: the API deletes the entries of that key of every ref when it is not given one (`gh cache delete ` does not give one), and a run of a + # tag or of a branch would delete the one of main, which is the one that the runs of main and of the pull requests restore, to + # leave one that they can not read. Cache keys are immutable, there is nothing to delete the first time + run: | + ref=$(printf '%s' "$GITHUB_REF" | jq -sRr @uri) + key=$(printf '%s' "$CACHE_KEY" | jq -sRr @uri) + gh api --method DELETE "repos/$GITHUB_REPOSITORY/actions/caches?key=$key&ref=$ref" > /dev/null || true - name: Save the cache uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index d3ea647..4ad22c3 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -95,8 +95,9 @@ jobs: buildpackage-opts: --build=binary --no-sign --jobs=auto - name: Save ccache - # Pull requests only read the cache, pushes to main and manual runs replace it. - if: ${{ !cancelled() && github.event_name != 'pull_request' }} + # Pull requests only read the cache, pushes to main and manual runs replace it. A tag does not save: what it would save is only + # readable by that tag, and the release builds all the legs from the caches of main + if: ${{ !cancelled() && github.event_name != 'pull_request' && github.ref_type != 'tag' }} uses: ./.github/actions/save-ccache with: key: ccache-deb-${{ matrix.name }}-${{ matrix.arch }} diff --git a/docs/ci-ccache.md b/docs/ci-ccache.md index bc2f2b1..bdf99c6 100644 --- a/docs/ci-ccache.md +++ b/docs/ci-ccache.md @@ -15,6 +15,7 @@ The workflows keep a ccache directory (`.ccache` in the workspace) between runs - Each build has its own cache: Linux, Windows and macOS (one for each architecture) in `ci.yml`, and the deb builds one per distro and arch. The tests do not compile, they use what the builds uploaded. - Pull requests only restore. Pushes to main and manual runs (`workflow_dispatch`) save. This is on purpose, see quota below. The deb workflow also runs on pull requests that touch its own files, and follows the same rule. - A cache saved on a branch is only visible to that branch. Main's caches are visible to everyone, so a pull request starts from main's. +- Saving deletes only the entry of the same key and of the ref that is running (the API call has a `ref`): deleting by key alone removes the entries of every ref, and a manual run on a branch would take away main's. A tag does not save in the deb workflow, its entry would only be readable by that tag, and the release builds from the caches of main (the legs that main does not build, armhf and arm64 on Ubuntu, start with no cache). - The save step runs after the build and before the tests, so a flaky test doesn't throw the cache away. Failed builds save too, whatever got compiled is still good. ### What isn't cached From a9721a7bb15b7d78356cc56943f9abeb18d10767 Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 18:32:26 +0100 Subject: [PATCH 11/13] ci: name the deb packages of main and of pull requests with the commit, and the ones of a tag with the version alone --- .github/workflows/build-deb.yml | 16 ++++++++++++++-- debian/set-version.sh | 6 +++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 4ad22c3..56deaba 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -62,11 +62,23 @@ jobs: persist-credentials: false submodules: recursive + # A tag is a release and its packages have the version of pyproject.toml alone. The others are snapshots of a commit, which have + # ~git after it: it sorts before the release, and the name of the package says what it was built from. The version of the + # module, in pyproject.toml, does not change, so the build does not either and ccache keeps hitting - name: Determine package version id: version + env: + # the commit of the pull request, github.sha is the one of its merge with main + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + REF_TYPE: ${{ github.ref_type }} run: | VERSION=$(sed -n 's/^version = "\(.*\)"$/\1/p' pyproject.toml | head -n 1) - echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + SNAPSHOT="" + if [ "$REF_TYPE" != tag ]; then + SNAPSHOT="git${COMMIT::7}" + fi + echo "version=${VERSION}${SNAPSHOT:+~$SNAPSHOT}" >> "$GITHUB_OUTPUT" + echo "snapshot=${SNAPSHOT}" >> "$GITHUB_OUTPUT" - name: Free up disk space run: | @@ -91,7 +103,7 @@ jobs: # would be a job of its own docker-image: ${{ fromJSON('{"noble":"ubuntu:24.04","jammy":"ubuntu:22.04","trixie":"debian:trixie-slim","bookworm":"debian:bookworm-slim"}')[matrix.name] }} extra-docker-args: --platform ${{ matrix.arch == 'armhf' && 'linux/arm/v7' || format('linux/{0}', matrix.arch) }} - before-build-hook: sh debian/set-version.sh ${{ matrix.name }} + before-build-hook: sh debian/set-version.sh ${{ matrix.name }} ${{ steps.version.outputs.snapshot }} buildpackage-opts: --build=binary --no-sign --jobs=auto - name: Save ccache diff --git a/debian/set-version.sh b/debian/set-version.sh index 1401b2b..8c0cfa0 100755 --- a/debian/set-version.sh +++ b/debian/set-version.sh @@ -1,8 +1,10 @@ #!/bin/sh # Adds an entry to debian/changelog for the version declared in pyproject.toml, so the # version is not repeated here. The package is named like python3-monero_0.9.10-1noble1_amd64.deb. +# With a snapshot, such as the commit that is built, it is python3-monero_0.9.10~git1a2b3c4-1noble1_amd64.deb: the tilde sorts it +# before the release, so a release is always newer than the snapshots that came before it. # -# usage: debian/set-version.sh [distribution] the distribution is the one of the system by default +# usage: debian/set-version.sh [distribution [snapshot]] the distribution is the one of the system by default set -eu cd "$(dirname "$0")/.." @@ -14,6 +16,8 @@ if [ -z "$version" ]; then fi dist=${1:-$(. /etc/os-release && echo "$VERSION_CODENAME")} +snapshot=${2:-} +version=${version}${snapshot:+~$snapshot} full_version="${version}-1${dist}1" if [ "$(dpkg-parsechangelog -S Version)" = "$full_version" ]; then From 25e0290ba40d8a96de55bb889b336db29f873a54 Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 18:37:07 +0100 Subject: [PATCH 12/13] ci: keep the logs, the shards of the coverage and the packages of main for days, not the default 90 --- .github/workflows/build-deb.yml | 2 ++ .github/workflows/build-macos.yml | 1 + .github/workflows/build-windows.yml | 1 + .github/workflows/ci.yml | 6 ++++++ 4 files changed, 10 insertions(+) diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 56deaba..925ef59 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -118,6 +118,8 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: python3-monero_${{ steps.version.outputs.version }}-1${{ matrix.name }}1_${{ matrix.arch }} + # the packages of the release are attached to its release, and the ones of main and of the pull requests are only looked at for a while + retention-days: 14 path: debian/artifacts/python3-monero_*.deb # the segments and the exports of the module in the package, and the stack protector and the branch protection in its code. The diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index ec16919..9101d38 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -133,4 +133,5 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: wheel-${{ matrix.arch }} + # the default retention: the release downloads it from the run of main, and a tag can come days after the commit path: wheelhouse/*.whl diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 18b891f..90d9b6e 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -162,4 +162,5 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: wheel-windows-x86_64 + # the default retention: the release downloads it from the run of main, and a tag can come days after the commit path: wheelhouse/*.whl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7e789a..83ff5dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,6 +228,8 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: container-logs-${{ matrix.name }} + # to look at a failure, not to keep + retention-days: 14 path: container-logs/ - name: Stop the test environment @@ -239,6 +241,7 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: test-logs-${{ matrix.name }} + retention-days: 14 path: monero_tests_*.log # the coverage of this shard, as lcov tracefiles. The coverage job merges them @@ -261,6 +264,8 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-${{ matrix.name }} + # only for the coverage job, and for its reruns + retention-days: 7 path: | coverage.info coverage_monero_cpp.info @@ -322,6 +327,7 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-reports-linux + # the default retention: the release downloads it from the run of main, and Codacy reads it when the run ends path: | coverage.info coverage.xml From 7849e2f4d0cfb2783ab1fdabc3296ef508a6f708 Mon Sep 17 00:00:00 2001 From: everoddandeven Date: Sun, 15 Nov 2026 18:37:29 +0100 Subject: [PATCH 13/13] ci: add the versions of the libraries and the ccache statistics to the summary of each job --- .github/workflows/build-linux.yml | 4 ++-- .github/workflows/build-macos.yml | 10 ++++++---- .github/workflows/build-windows.yml | 4 ++-- .github/workflows/ci.yml | 2 +- bin/job_summary.sh | 25 +++++++++++++++++++++++++ 5 files changed, 36 insertions(+), 9 deletions(-) create mode 100755 bin/job_summary.sh diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 33a95e4..a82452e 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -30,7 +30,7 @@ jobs: run: bin/install_dependencies_linux.sh - name: Show the versions of the apt libraries - run: bin/show_versions_linux.sh + run: bin/show_versions_linux.sh | bin/job_summary.sh "apt libraries" - name: Install the Python build requirements (requirements-build.txt) run: pip3 install -r requirements-build.txt --break-system-packages @@ -104,7 +104,7 @@ jobs: - name: Show ccache statistics if: ${{ !cancelled() }} - run: ccache --show-stats + run: ccache --show-stats | bin/job_summary.sh "ccache" - name: Save ccache # Pull requests only read the cache, pushes to main and manual runs replace it. diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 9101d38..b50425d 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -56,9 +56,11 @@ jobs: - name: Show the versions of the brew formulae run: | - brew list --versions python boost@1.85 hidapi openssl@3 zmq libpgm miniupnpc expat libunwind-headers protobuf ccache - cmake --version | head -n 1 - clang --version | head -n 1 + { + brew list --versions python boost@1.85 hidapi openssl@3 zmq libpgm miniupnpc expat libunwind-headers protobuf ccache + cmake --version | head -n 1 + clang --version | head -n 1 + } | bin/job_summary.sh "brew formulae" - name: Create the Python virtual environment and install the build requirements (requirements-build.txt) run: | @@ -110,7 +112,7 @@ jobs: - name: Show ccache statistics if: ${{ !cancelled() }} - run: ccache --show-stats + run: ccache --show-stats | bin/job_summary.sh "ccache" - name: Save ccache # Pull requests only read the cache, pushes to main and manual runs replace it. diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index 90d9b6e..0d4f6c9 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -91,7 +91,7 @@ jobs: run: | pacman -Q mingw-w64-x86_64-gcc mingw-w64-x86_64-binutils mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq \ mingw-w64-x86_64-libsodium mingw-w64-x86_64-unbound mingw-w64-x86_64-protobuf mingw-w64-x86_64-boost mingw-w64-x86_64-icu \ - mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-pybind11 mingw-w64-x86_64-ccache + mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-pybind11 mingw-w64-x86_64-ccache | bin/job_summary.sh "MSYS2 packages" - name: Restore ccache uses: ./.github/actions/restore-ccache @@ -136,7 +136,7 @@ jobs: - name: Show ccache statistics if: ${{ !cancelled() }} - run: ccache --show-stats + run: ccache --show-stats | bin/job_summary.sh "ccache" - name: Save ccache # Pull requests only read the cache, pushes to main and manual runs replace it. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83ff5dd..b88bbe2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -191,7 +191,7 @@ jobs: run: bin/install_dependencies_linux.sh - name: Show the versions of the apt libraries - run: bin/show_versions_linux.sh + run: bin/show_versions_linux.sh | bin/job_summary.sh "apt libraries" - name: Install the Python test requirements (tests/requirements.txt) run: pip3 install -r tests/requirements.txt --break-system-packages diff --git a/bin/job_summary.sh b/bin/job_summary.sh new file mode 100755 index 0000000..6193c36 --- /dev/null +++ b/bin/job_summary.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Prints what it reads and also adds it to the summary of the job (the page of the run) as a block of code, with a title. It does nothing +# more than print when there is no summary, as outside of GitHub Actions, and it never fails because of the summary. +# +# bin/show_versions_linux.sh | bin/job_summary.sh "apt libraries" +set -uo pipefail + +title=${1:-} +text=$(cat) +printf '%s\n' "$text" + +summary=${GITHUB_STEP_SUMMARY:-} +if [ -n "$summary" ]; then + # in MSYS2 the file of the summary has a path of Windows + if command -v cygpath > /dev/null 2>&1; then + summary=$(cygpath -u "$summary") + fi + { + if [ -n "$title" ]; then + printf '### %s\n\n' "$title" + fi + printf '```\n%s\n```\n\n' "$text" + } >> "$summary" || true +fi +exit 0