diff --git a/.github/actions/restore-ccache/action.yml b/.github/actions/restore-ccache/action.yml new file mode 100644 index 0000000..0dc8f81 --- /dev/null +++ b/.github/actions/restore-ccache/action.yml @@ -0,0 +1,23 @@ +name: Restore ccache +description: > + Restores the ccache that save-ccache stored under the key, or else the newest one stored under a key that starts with the key and a `_` + (the `_` is there because `macos-15` is a prefix of `macos-15-intel`). The cache of a name is the one of its last run on main. + +inputs: + key: + description: Cache key, the same one given to save-ccache + required: true + path: + description: Directory of the ccache + default: .ccache + +runs: + using: composite + steps: + - name: Restore the cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ inputs.path }} + key: ${{ inputs.key }} + restore-keys: | + ${{ inputs.key }}_ diff --git a/.github/actions/save-ccache/action.yml b/.github/actions/save-ccache/action.yml index 74c2de9..334fe36 100644 --- a/.github/actions/save-ccache/action.yml +++ b/.github/actions/save-ccache/action.yml @@ -1,6 +1,6 @@ name: Save ccache description: > - Replaces the cache stored under the key instead of adding an entry per commit, so the + Replaces the cache stored under the key, for the ref that is running, instead of adding an entry per commit, so the caches of the repository stay below its quota. The job needs the actions: write permission. inputs: @@ -19,8 +19,13 @@ runs: env: GH_TOKEN: ${{ github.token }} CACHE_KEY: ${{ inputs.key }} - # cache keys are immutable, there is nothing to delete the first time - run: gh cache delete "$CACHE_KEY" --repo "$GITHUB_REPOSITORY" || true + # Only the entry of the ref that is running: the API deletes the entries of that key of every ref when it is not given one (`gh cache delete ` does not give one), and a run of a + # tag or of a branch would delete the one of main, which is the one that the runs of main and of the pull requests restore, to + # leave one that they can not read. Cache keys are immutable, there is nothing to delete the first time + run: | + ref=$(printf '%s' "$GITHUB_REF" | jq -sRr @uri) + key=$(printf '%s' "$CACHE_KEY" | jq -sRr @uri) + gh api --method DELETE "repos/$GITHUB_REPOSITORY/actions/caches?key=$key&ref=$ref" > /dev/null || true - name: Save the cache uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 diff --git a/.github/python-version b/.github/python-version new file mode 100644 index 0000000..24ee5b1 --- /dev/null +++ b/.github/python-version @@ -0,0 +1 @@ +3.13 diff --git a/.github/workflows/build-deb.yml b/.github/workflows/build-deb.yml index 73068d9..925ef59 100644 --- a/.github/workflows/build-deb.yml +++ b/.github/workflows/build-deb.yml @@ -62,11 +62,23 @@ jobs: persist-credentials: false submodules: recursive + # A tag is a release and its packages have the version of pyproject.toml alone. The others are snapshots of a commit, which have + # ~git after it: it sorts before the release, and the name of the package says what it was built from. The version of the + # module, in pyproject.toml, does not change, so the build does not either and ccache keeps hitting - name: Determine package version id: version + env: + # the commit of the pull request, github.sha is the one of its merge with main + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + REF_TYPE: ${{ github.ref_type }} run: | VERSION=$(sed -n 's/^version = "\(.*\)"$/\1/p' pyproject.toml | head -n 1) - echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + SNAPSHOT="" + if [ "$REF_TYPE" != tag ]; then + SNAPSHOT="git${COMMIT::7}" + fi + echo "version=${VERSION}${SNAPSHOT:+~$SNAPSHOT}" >> "$GITHUB_OUTPUT" + echo "snapshot=${SNAPSHOT}" >> "$GITHUB_OUTPUT" - name: Free up disk space run: | @@ -74,17 +86,14 @@ jobs: docker system prune -af || true - name: Restore ccache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: ./.github/actions/restore-ccache with: - path: .ccache key: ccache-deb-${{ matrix.name }}-${{ matrix.arch }} - restore-keys: | - ccache-deb-${{ matrix.name }}-${{ matrix.arch }}_ # Installs the Build-Depends of debian/control in the container, sets the version of # debian/changelog from pyproject.toml and runs dpkg-buildpackage. The workspace is # mounted at /github/workspace in the container. - - name: Build package + - name: Build the deb package (dpkg-buildpackage) uses: jtdor/build-deb-action@473136f14a42a231124c1f7cefaedfe941203c38 # v1.9.1 env: CCACHE_DIR: /github/workspace/.ccache @@ -94,26 +103,29 @@ jobs: # would be a job of its own docker-image: ${{ fromJSON('{"noble":"ubuntu:24.04","jammy":"ubuntu:22.04","trixie":"debian:trixie-slim","bookworm":"debian:bookworm-slim"}')[matrix.name] }} extra-docker-args: --platform ${{ matrix.arch == 'armhf' && 'linux/arm/v7' || format('linux/{0}', matrix.arch) }} - before-build-hook: sh debian/set-version.sh ${{ matrix.name }} + before-build-hook: sh debian/set-version.sh ${{ matrix.name }} ${{ steps.version.outputs.snapshot }} buildpackage-opts: --build=binary --no-sign --jobs=auto - name: Save ccache - # Pull requests only read the cache, pushes to main and manual runs replace it. - if: ${{ !cancelled() && github.event_name != 'pull_request' }} + # Pull requests only read the cache, pushes to main and manual runs replace it. A tag does not save: what it would save is only + # readable by that tag, and the release builds all the legs from the caches of main + if: ${{ !cancelled() && github.event_name != 'pull_request' && github.ref_type != 'tag' }} uses: ./.github/actions/save-ccache with: key: ccache-deb-${{ matrix.name }}-${{ matrix.arch }} - - name: Upload monero-python package + - name: Upload the deb package uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: python3-monero_${{ steps.version.outputs.version }}-1${{ matrix.name }}1_${{ matrix.arch }} + # the packages of the release are attached to its release, and the ones of main and of the pull requests are only looked at for a while + retention-days: 14 path: debian/artifacts/python3-monero_*.deb # the segments and the exports of the module in the package, and the stack protector and the branch protection in its code. The # module of the armhf package is read but not disassembled, the objdump of the arm64 runner does not read 32 bit arm. # After the upload, so that a package that fails can be downloaded and looked at. The release needs this job to succeed - - name: Verify libraries + - name: Check the hardening of the module in the package run: | dpkg-deb -x debian/artifacts/python3-monero_*.deb deb-check bin/check_elf_hardening.sh deb-check/usr/lib/python3/dist-packages/monero*.so diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index 76de948..a82452e 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -1,7 +1,7 @@ -name: Build Linux Module +name: Build Linux -# Called by build.yml. It builds monero-project, monero-cpp and the module, the last two with the coverage on, and uploads the module -# with the files that the coverage needs. The tests of build.yml run on it, in several jobs. The packages of Linux are the deb ones, +# Called by ci.yml. It builds monero-project, monero-cpp and the module, the last two with the coverage on, and uploads the module +# with the files that the coverage needs. The tests of ci.yml run on it, in several jobs. The packages of Linux are the deb ones, # which build-deb.yml makes. on: workflow_call: @@ -26,12 +26,16 @@ jobs: persist-credentials: false submodules: recursive - - name: Install dependencies - run: | - bin/install_dependencies_linux.sh - pip3 install -r requirements-build.txt --break-system-packages + - name: Install apt packages + run: bin/install_dependencies_linux.sh + + - name: Show the versions of the apt libraries + run: bin/show_versions_linux.sh | bin/job_summary.sh "apt libraries" + + - name: Install the Python build requirements (requirements-build.txt) + run: pip3 install -r requirements-build.txt --break-system-packages - - name: Install expat + - name: Install expat 2.7.1 (static) run: | wget https://github.com/libexpat/libexpat/releases/download/R_2_7_1/expat-2.7.1.tar.bz2 echo "45c98ae1e9b5127325d25186cf8c511fa814078e9efeae7987a574b482b79b3d expat-2.7.1.tar.bz2" | sha256sum -c @@ -42,7 +46,7 @@ jobs: sudo make install cd ../ - - name: Install unbound + - name: Install unbound 1.22.0 (static) run: | wget https://www.nlnetlabs.nl/downloads/unbound/unbound-1.22.0.tar.gz echo "c5dd1bdef5d5685b2cedb749158dd152c52d44f65529a34ac15cd88d4b1b3d43 unbound-1.22.0.tar.gz" | sha256sum -c @@ -55,17 +59,14 @@ jobs: cd ../ - name: Restore ccache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: ./.github/actions/restore-ccache with: - path: .ccache key: ccache-linux-test - restore-keys: | - ccache-linux-test_ - name: Reset ccache statistics run: ccache --zero-stats - - name: Build monero-project + - name: Build monero-project (static, Release) run: | cd external/monero-cpp/external/monero-project mkdir -p build/release @@ -77,7 +78,7 @@ jobs: make -j3 wallet cryptonote_protocol cd ../../../../../../ - - name: Build monero-cpp libraries + - name: Build monero-cpp (with coverage) run: | cd external/monero-cpp mkdir -p build @@ -87,7 +88,7 @@ jobs: make -j3 cd ../../../ - - name: Build monero-python module + - name: Build the monero-python module (with coverage) run: | mkdir -p build cmake -B build -DENABLE_COVERAGE=ON -Dpybind11_DIR="$(python3 -m pybind11 --cmakedir)" @@ -103,7 +104,7 @@ jobs: - name: Show ccache statistics if: ${{ !cancelled() }} - run: ccache --show-stats + run: ccache --show-stats | bin/job_summary.sh "ccache" - name: Save ccache # Pull requests only read the cache, pushes to main and manual runs replace it. @@ -113,17 +114,17 @@ jobs: key: ccache-linux-test # The module, and the .gcno that lcov reads together with the .gcda that the tests write: the ones of monero-cpp and of the module - # are the only ones, monero-project is not built with the coverage. The tests of build.yml extract it at the root of the + # are the only ones, monero-project is not built with the coverage. The tests of ci.yml extract it at the root of the # workspace, where the paths of the .gcda are the ones that the build had. - name: Package the build for the tests run: | find build external/monero-cpp/build \( -name 'monero*.so' -o -name '*.gcno' \) -print0 | tar --null -czf linux-build.tar.gz -T - ls -l linux-build.tar.gz - - name: Upload monero-python build + - name: Upload the Linux build uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: monero-python-linux-x86_64 + name: build-linux-x86_64 path: linux-build.tar.gz # already compressed compression-level: 0 diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index cc0d3bf..b50425d 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -1,6 +1,6 @@ -name: Build MacOS +name: Build macOS -# Called by build.yml, which has the triggers and the concurrency group: a group here would be the one of the caller, and the +# Called by ci.yml, which has the triggers and the concurrency group: a group here would be the one of the caller, and the # jobs of the same run would cancel each other. on: workflow_call: @@ -35,18 +35,35 @@ jobs: persist-credentials: false submodules: recursive - - name: Install dependencies + - name: Install brew formulae (boost@1.85, openssl@3 and the other libraries) run: | HOMEBREW_NO_AUTO_UPDATE=1 brew install python boost@1.85 hidapi openssl@3 zmq libpgm miniupnpc expat libunwind-headers protobuf ccache - brew unlink boost || true - brew link boost@1.85 --force + # the image of the runner can have another boost linked, which gets in the way of boost@1.85. They are touched only when + # they are there, brew prints an error for a formula that is not installed (the annotations of the run show it as one) + if brew list --versions boost > /dev/null 2>&1; then + brew unlink boost + brew link boost@1.85 --force + fi # brew links openssl@4 into /opt/homebrew, and the -I/opt/homebrew/include of boost and zeromq comes # before the one of openssl@3, so its headers win over the ones of OPENSSL_ROOT_DIR - brew unlink openssl@4 || true + if brew list --versions openssl@4 > /dev/null 2>&1; then + brew unlink openssl@4 + fi # brew's openssl is OpenSSL 4 on the newer runner images, which boost 1.85 does not build with. # monero-project and monero-cpp take brew's openssl unless they are given another one, so # they get it on the command line, and the module finds it in the environment echo "OPENSSL_ROOT_DIR=$(brew --prefix openssl@3)" >> "$GITHUB_ENV" + + - name: Show the versions of the brew formulae + run: | + { + brew list --versions python boost@1.85 hidapi openssl@3 zmq libpgm miniupnpc expat libunwind-headers protobuf ccache + cmake --version | head -n 1 + clang --version | head -n 1 + } | bin/job_summary.sh "brew formulae" + + - name: Create the Python virtual environment and install the build requirements (requirements-build.txt) + run: | # a virtual environment: the python of brew has packages that pip can not replace, such as wheel and typing_extensions on the # Intel runner (they have no RECORD file), and the versions of the requirements files are pinned. The next steps find it first in the PATH python3 -m venv "$RUNNER_TEMP/venv" @@ -54,21 +71,18 @@ jobs: "$RUNNER_TEMP/venv/bin/pip" install -r requirements-build.txt # static, against openssl@3: the unbound of brew is built with OpenSSL 4 - - name: Install unbound (static) + - name: Install unbound 1.22.0 (static) run: bin/install_unbound_macos.sh - name: Restore ccache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: ./.github/actions/restore-ccache with: - path: .ccache key: ccache-${{ matrix.os }} - restore-keys: | - ccache-${{ matrix.os }}_ - name: Reset ccache statistics run: ccache --zero-stats - - name: Build monero-project + - name: Build monero-project (static, Release) run: | cd external/monero-cpp/external/monero-project mkdir -p build/release @@ -81,7 +95,7 @@ jobs: make -j3 wallet cryptonote_protocol cd ../../../../../../ - - name: Build monero-cpp libraries + - name: Build monero-cpp run: | cd external/monero-cpp mkdir -p build @@ -91,14 +105,14 @@ jobs: make -j3 cd ../../../ - - name: Build monero-python module + - name: Build the monero-python wheel run: | mkdir -p build python3 -m pip wheel . -w dist --no-build-isolation - name: Show ccache statistics if: ${{ !cancelled() }} - run: ccache --show-stats + run: ccache --show-stats | bin/job_summary.sh "ccache" - name: Save ccache # Pull requests only read the cache, pushes to main and manual runs replace it. @@ -107,18 +121,19 @@ jobs: with: key: ccache-${{ matrix.os }} - - name: Repair wheel with delocate + - name: Repair the wheel with delocate run: | delocate-wheel -w wheelhouse -v dist/*.whl # the canary and the exports of the module in the wheel that is published - - name: Verify libraries + - name: Check the hardening of the module in the wheel run: | python3 -m zipfile -e wheelhouse/*.whl wheel-check bin/check_macho_hardening.sh wheel-check/monero*.so - - name: Upload monero-python wheel + - name: Upload the macOS wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: monero-${{ matrix.os }} + name: wheel-${{ matrix.arch }} + # the default retention: the release downloads it from the run of main, and a tag can come days after the commit path: wheelhouse/*.whl diff --git a/.github/workflows/build-windows.yml b/.github/workflows/build-windows.yml index f6742fd..0d4f6c9 100644 --- a/.github/workflows/build-windows.yml +++ b/.github/workflows/build-windows.yml @@ -1,6 +1,6 @@ -name: Build Windows Package +name: Build Windows -# Called by build.yml, which has the triggers and the concurrency group: a group here would be the one of the caller, and the +# Called by ci.yml, which has the triggers and the concurrency group: a group here would be the one of the caller, and the # jobs of the same run would cancel each other. on: workflow_call: @@ -29,18 +29,19 @@ jobs: persist-credentials: false submodules: recursive - - name: Setup Python 3.13 + # the version is in one file: the wheel that CI builds is for it, and the job that tests the wheel needs the same + - name: Install Python (the version of .github/python-version) uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: '3.13' + python-version-file: .github/python-version architecture: 'x64' - - name: Install Python build dependencies + - name: Install the Python build requirements (requirements-build.txt) shell: bash run: | python -m pip install -r requirements-build.txt - - name: Setup MSYS2 MINGW64 + - name: Install MSYS2 and its MINGW64 packages (openssl, zeromq, libsodium, unbound and protobuf at the version of the repo) uses: msys2/setup-msys2@ec48f7c5447b3140e2b088413ae3a55687bccb6e # v2.33.0 with: msystem: MINGW64 @@ -60,32 +61,9 @@ jobs: base-devel wget - - name: Install pinned MinGW64 toolchain v16.2.0 + # the packages, with their sha256, are in bin/msys2_pins.txt + - name: Install MinGW64 toolchain 16.2.0 (pinned) run: | - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "acad5748942953f91a0f2bba03e20437b909366e55949f478f1b2f2d30c726a7 mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-crt-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "d24b4e7614756414ae3ef58793265448a3ca7d5fd447a991e9b2facde7b59d75 mingw-w64-x86_64-crt-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-binutils-2.47-3-any.pkg.tar.zst - echo "827363748ce3320683319d860ee4fcfcdbb36baf66aac6dab0ec3822be2d4ff7 mingw-w64-x86_64-binutils-2.47-3-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libmangle-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "481babe8c2fdfa4fc7f51bf13939701719af9f8166ef6a91a9a209cc25961b03 mingw-w64-x86_64-libmangle-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-winpthreads-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "7d86a267b6069198d28f3e77314282ff7b4e3c7bb8fe482299ca58f7a5b439db mingw-w64-x86_64-winpthreads-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libwinpthread-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "1d0ed7fda332a5c879de210f3d72c75fa1f3df7cc63dd951e763dec8eb02c1ee mingw-w64-x86_64-libwinpthread-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-winstorecompat-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "7fcafd7ba30bd47000263e76a06bd893406c64f88955825f0a9188407a35c3d3 mingw-w64-x86_64-winstorecompat-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-tools-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst - echo "c17557d09371e39db38deb56c7da6ae54e2c65b6f3466ca02f4da26b616ca8c3 mingw-w64-x86_64-tools-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pkgconf-1~3.0.7-1-any.pkg.tar.zst - echo "37b97f372409fa7cf54d709bab892f34e9c62d2658230a09f783a89a55ae590e mingw-w64-x86_64-pkgconf-1~3.0.7-1-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-make-4.4.1-5-any.pkg.tar.zst - echo "c19e7caf09bbc89b2730556b2da73004118d4e1f967a685cc48524c7ff80d864 mingw-w64-x86_64-make-4.4.1-5-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-gcc-libs-16.2.0-3-any.pkg.tar.zst - echo "f8e25ea67bb796e7f65550f0dca9fce4cdde8aaa3dadafe4d13c6a8233c8de26 mingw-w64-x86_64-gcc-libs-16.2.0-3-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-gcc-16.2.0-3-any.pkg.tar.zst - echo "8b37c953ae12f3d8fa05c854236463231ea0a294dff423007cbd93d4d8089f9f mingw-w64-x86_64-gcc-16.2.0-3-any.pkg.tar.zst" | sha256sum -c # gcc-libs is split into libgcc, libstdc++ and cc-libs since 16.2.0-4 and the # new packages conflict with the pinned gcc-libs, so drop them before the downgrade for p in libgcc libstdc++ cc-libs; do @@ -93,62 +71,37 @@ jobs: pacman -Rdd --noconfirm mingw-w64-x86_64-$p fi done - pacman -U --noconfirm \ - mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-crt-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-binutils-2.47-3-any.pkg.tar.zst \ - mingw-w64-x86_64-libmangle-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-winpthreads-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-libwinpthread-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-winstorecompat-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-tools-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst \ - mingw-w64-x86_64-pkgconf-1~3.0.7-1-any.pkg.tar.zst \ - mingw-w64-x86_64-make-4.4.1-5-any.pkg.tar.zst \ - mingw-w64-x86_64-gcc-libs-16.2.0-3-any.pkg.tar.zst \ - mingw-w64-x86_64-gcc-16.2.0-3-any.pkg.tar.zst - - - name: Install ICU v78.3 - run: | - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst - echo "8486a018aca2e56d1fcd2eed2069011f2e1da02e03af72b38916ee06cbe7203f mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst" | sha256sum -c - pacman -U --noconfirm mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst + bin/install_msys2_pins.sh toolchain - - name: Install hidapi v0.15.0 and libusb v1.0.30 - run: | - # dropped from the msys2 mingw64 repo, install the archived packages - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst - echo "89fb61549a437d65076a092a367fcb264fc7045ee9ebef48cd81f5a76a5588f8 mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst - echo "124edc26b009678a4c5e759481a364abf63ed7f2f49a53092e8bd0a47c0c0740 mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst" | sha256sum -c - pacman -U --noconfirm mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst - - - name: Install boost v1.87.0 - run: | - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst - echo "241919a5885a9270d0e0937e71cf5e146c2a8cabecc2714f09248350181cb1b6 mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst - echo "5fd77df3e5983ba0d9c689f46151119e0e59b3da6850c5a0ba54068eeae7749c mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst" | sha256sum -c - # the version that monero-cpp builds with. The boost-libs in the repo is a newer rebuild that needs the split gcc runtime - pacman -U --noconfirm mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst - - - name: Install pybind11 v2.13.6 + - name: Install ICU 78.3 (pinned) + run: bin/install_msys2_pins.sh icu + + # dropped from the msys2 mingw64 repo, install the archived packages + - name: Install hidapi 0.15.0 and libusb 1.0.30 (pinned) + run: bin/install_msys2_pins.sh hidapi-libusb + + # the version that monero-cpp builds with. The boost-libs in the repo is a newer rebuild that needs the split gcc runtime + - name: Install boost 1.87.0 (pinned) + run: bin/install_msys2_pins.sh boost + + - name: Install pybind11 2.13.6 (pinned) + run: bin/install_msys2_pins.sh pybind11 + + - name: Show the versions of the MSYS2 packages run: | - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst - echo "5b56ad87bc544336b2d51edb8bb0614f46542571362daf4d7e53bf23120175a1 mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst" | sha256sum -c - pacman -U --noconfirm mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst + pacman -Q mingw-w64-x86_64-gcc mingw-w64-x86_64-binutils mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq \ + mingw-w64-x86_64-libsodium mingw-w64-x86_64-unbound mingw-w64-x86_64-protobuf mingw-w64-x86_64-boost mingw-w64-x86_64-icu \ + mingw-w64-x86_64-hidapi mingw-w64-x86_64-libusb mingw-w64-x86_64-pybind11 mingw-w64-x86_64-ccache | bin/job_summary.sh "MSYS2 packages" - name: Restore ccache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + uses: ./.github/actions/restore-ccache with: - path: .ccache key: ccache-windows - restore-keys: | - ccache-windows_ - name: Reset ccache statistics run: ccache --zero-stats - - name: Build monero-project + - name: Build monero-project (static, Release) run: | cd external/monero-cpp/external/monero-project mkdir -p build/release @@ -164,7 +117,7 @@ jobs: ../../ make -j$(nproc) wallet cryptonote_protocol - - name: Build monero-cpp libraries + - name: Build monero-cpp run: | cd external/monero-cpp mkdir -p build @@ -172,7 +125,7 @@ jobs: cmake .. -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache cmake --build . --parallel $(nproc) - - name: Build monero-python module + - name: Build the monero-python wheel run: | export WIN_PYTHON_EXE=$(cygpath -u "$pythonLocation/python.exe") export CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) @@ -183,7 +136,7 @@ jobs: - name: Show ccache statistics if: ${{ !cancelled() }} - run: ccache --show-stats + run: ccache --show-stats | bin/job_summary.sh "ccache" - name: Save ccache # Pull requests only read the cache, pushes to main and manual runs replace it. @@ -192,21 +145,22 @@ jobs: with: key: ccache-windows - - name: Repair wheel with delvewheel + - name: Repair the wheel with delvewheel run: | export WIN_PYTHON_EXE=$(cygpath -u "$pythonLocation/python.exe") MINGW_BIN_WIN=$(cygpath -w "$MINGW_PREFIX/bin") "$WIN_PYTHON_EXE" -m delvewheel repair dist/*.whl -w wheelhouse --add-path "$MINGW_BIN_WIN" # the flags of the image (ASLR, DEP) and the exports of the module in the wheel that is published - - name: Verify libraries + - name: Check the hardening of the module in the wheel run: | export WIN_PYTHON_EXE=$(cygpath -u "$pythonLocation/python.exe") "$WIN_PYTHON_EXE" -m zipfile -e wheelhouse/*.whl wheel-check bash bin/check_pe_hardening.sh wheel-check/monero*.pyd - - name: Upload monero-python wheel + - name: Upload the Windows wheel uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: monero-python-win-amd64 + name: wheel-windows-x86_64 + # the default retention: the release downloads it from the run of main, and a tag can come days after the commit path: wheelhouse/*.whl diff --git a/.github/workflows/build.yml b/.github/workflows/ci.yml similarity index 74% rename from .github/workflows/build.yml rename to .github/workflows/ci.yml index eab827c..b88bbe2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Build +name: CI # The builds are in build-linux.yml, build-windows.yml and build-macos.yml, called from here, and the tests run on what they built: the # wheel that a build uploads is the one that its tests install, and the module of Linux is the one that its shards download, so nothing @@ -11,7 +11,7 @@ on: branches: - main # Only on pull requests: the release needs a successful run for every commit on main. - # If Build ever becomes a required check, a skipped run stays Pending and blocks the merge. + # If CI ever becomes a required check, a skipped run stays Pending and blocks the merge. paths-ignore: - 'README.md' - 'LICENSE' @@ -38,26 +38,30 @@ permissions: contents: read jobs: - build-ubuntu: + build-linux: + # the name of the caller is the first part of the name of the checks of its job: "build / linux-x86_64" + name: build permissions: contents: read actions: write # build-linux.yml replaces its ccache entries uses: ./.github/workflows/build-linux.yml build-windows: + name: build permissions: contents: read actions: write # build-windows.yml replaces its ccache entries uses: ./.github/workflows/build-windows.yml - build-mac-os: + build-macos: + name: build permissions: contents: read actions: write # build-macos.yml replaces its ccache entries uses: ./.github/workflows/build-macos.yml test-windows: - name: windows (pytest) + name: unit tests / windows needs: build-windows runs-on: windows-latest timeout-minutes: 30 @@ -68,34 +72,35 @@ jobs: with: persist-credentials: false - - name: Setup Python 3.13 + # the version is in one file: the wheel that CI builds is for it, and the job that tests the wheel needs the same + - name: Install Python (the version of .github/python-version) uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: - python-version: '3.13' + python-version-file: .github/python-version architecture: 'x64' - - name: Download monero-python wheel + - name: Download the Windows wheel uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: monero-python-win-amd64 + name: wheel-windows-x86_64 path: wheelhouse - - name: Install dependencies + - name: Install the test requirements and the wheel shell: bash run: | python -m pip install -r tests/requirements.txt python -m pip install wheelhouse/*.whl - - name: Run unit tests + - name: Run unit tests (pytest) shell: bash env: IN_CONTAINER: "true" run: | python -m pytest -m unit - test-mac-os: - name: ${{ matrix.arch }} (pytest) - needs: build-mac-os + test-macos: + name: unit tests / ${{ matrix.arch }} + needs: build-macos runs-on: ${{ matrix.os }} timeout-minutes: 30 @@ -114,23 +119,27 @@ jobs: with: persist-credentials: false - - name: Download monero-python wheel + - name: Download the macOS wheel (${{ matrix.arch }}) uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: monero-${{ matrix.os }} + name: wheel-${{ matrix.arch }} path: wheelhouse - - name: Install dependencies + # the python that the build used, the wheel is for its version + - name: Install Python (brew) run: | - # the python that the build used, the wheel is for its version HOMEBREW_NO_AUTO_UPDATE=1 brew install python - # a virtual environment, as in the build: the python of brew has packages that pip can not replace, and the - # versions of the requirements file are pinned + brew list --versions python + + # a virtual environment, as in the build: the python of brew has packages that pip can not replace, and the + # versions of the requirements file are pinned + - name: Install the test requirements and the wheel in a virtual environment + run: | python3 -m venv "$RUNNER_TEMP/venv" echo "$RUNNER_TEMP/venv/bin" >> "$GITHUB_PATH" "$RUNNER_TEMP/venv/bin/pip" install -r tests/requirements.txt wheelhouse/*.whl - - name: Run unit tests + - name: Run unit tests (pytest) run: | pytest -m unit @@ -138,11 +147,15 @@ jobs: # wallet rpc servers of docker. Together they are all the tests. Each one uploads its coverage, and the coverage job merges them. # The name of an entry is in the names of files and artifacts, so it has no spaces: lcov splits the path of a tracefile at them, # and the merge of the coverage failed with a name that had one. The title is what the job shows. - test-ubuntu: - name: linux-x86_64 ${{ matrix.title }} (pytest) - needs: build-ubuntu + test-linux: + name: ${{ matrix.title }} / linux-x86_64 + needs: build-linux runs-on: ubuntu-24.04 timeout-minutes: 90 + env: + # what the docker compose commands of the steps work on + COMPOSE_FILE: tests/docker-compose.yml + TEST_SERVICES: node_1 node_2 xmr_wallet_1 xmr_wallet_2 xmr_wallet_3 strategy: fail-fast: false @@ -164,29 +177,35 @@ jobs: persist-credentials: false submodules: recursive - - name: Download monero-python build + - name: Download the Linux build (module and coverage files) uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - name: monero-python-linux-x86_64 + name: build-linux-x86_64 # at the root of the workspace: the module in build/, and the .gcno where the tests will write the .gcda - - name: Extract monero-python build + - name: Extract the Linux build run: tar -xzf linux-build.tar.gz # the module is linked to shared libraries of apt, the ones of the build - - name: Install dependencies - run: | - bin/install_dependencies_linux.sh - pip3 install -r tests/requirements.txt --break-system-packages + - name: Install apt packages + run: bin/install_dependencies_linux.sh + + - name: Show the versions of the apt libraries + run: bin/show_versions_linux.sh | bin/job_summary.sh "apt libraries" + + - name: Install the Python test requirements (tests/requirements.txt) + run: pip3 install -r tests/requirements.txt --break-system-packages - - name: Setup test environment + - name: Start the test environment (2 nodes and 3 wallet rpc servers in docker) if: ${{ matrix.docker }} run: | - docker compose -f tests/docker-compose.yml up -d node_1 node_2 xmr_wallet_1 xmr_wallet_2 xmr_wallet_3 + read -r -a services <<< "$TEST_SERVICES" + docker compose up -d "${services[@]}" + docker compose images sleep 10 # --durations shows where a shard spends its time, to balance them - - name: Run tests + - name: Run ${{ matrix.title }} (pytest) env: IN_CONTAINER: "true" PYTHONPATH: ${{ github.workspace }}/build @@ -196,12 +215,12 @@ jobs: read -r -a args <<< "$PYTEST_ARGS" pytest --cov=tests --cov-report= --durations=25 "${args[@]}" - - name: Dump container logs + - name: Collect the container logs if: ${{ always() && matrix.docker }} run: | mkdir -p container-logs - for svc in node_1 node_2 xmr_wallet_1 xmr_wallet_2 xmr_wallet_3; do - docker compose -f tests/docker-compose.yml logs --no-color --timestamps "$svc" > "container-logs/$svc.log" 2>&1 || true + for svc in $TEST_SERVICES; do + docker compose logs --no-color --timestamps "$svc" > "container-logs/$svc.log" 2>&1 || true done - name: Upload container logs @@ -209,17 +228,20 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: container-logs-${{ matrix.name }} + # to look at a failure, not to keep + retention-days: 14 path: container-logs/ - - name: Cleanup test environment + - name: Stop the test environment if: ${{ always() && matrix.docker }} - run: docker compose -f tests/docker-compose.yml down -v + run: docker compose down -v - - name: Upload tests logs + - name: Upload the test logs if: always() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: test-logs-${{ matrix.name }} + retention-days: 14 path: monero_tests_*.log # the coverage of this shard, as lcov tracefiles. The coverage job merges them @@ -242,6 +264,8 @@ jobs: uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-${{ matrix.name }} + # only for the coverage job, and for its reruns + retention-days: 7 path: | coverage.info coverage_monero_cpp.info @@ -250,9 +274,9 @@ jobs: # The coverage of the tests of all the shards, in the same reports that Tests made (the artifact has the same name, and Codacy and the # release read it). It runs when a shard fails too, with the shards that have it, since a failing test is still reported. coverage: - name: coverage - needs: [build-ubuntu, test-ubuntu] - if: ${{ !cancelled() && needs.build-ubuntu.result == 'success' }} + name: coverage report + needs: [build-linux, test-linux] + if: ${{ !cancelled() && needs.build-linux.result == 'success' }} runs-on: ubuntu-24.04 timeout-minutes: 30 @@ -263,11 +287,12 @@ jobs: persist-credentials: false submodules: recursive - - name: Install dependencies + - name: Install lcov and the Python test requirements run: | sudo apt-get update sudo apt-get install -y lcov pip3 install -r tests/requirements.txt --break-system-packages + lcov --version # a folder for each shard, named as the artifact - name: Download the coverage of the shards @@ -276,7 +301,7 @@ jobs: pattern: coverage-* path: shards - - name: Generate python bindings coverage report + - name: Generate the python bindings coverage report run: | args=() for f in shards/*/coverage.info; do args+=(-a "$f"); done @@ -284,13 +309,13 @@ jobs: genhtml coverage.info --output-directory coverage_monero_python_bindings_html --ignore-errors mismatch,inconsistent,source,corrupt --title "monero-python bindings coverage @ ${GITHUB_SHA::7}" lcov --summary coverage.info - - name: Generate python tests coverage report + - name: Generate the python tests coverage report run: | coverage combine shards/*/pytest-coverage-* coverage xml coverage html -d coverage_monero_python_tests_html --title "monero-python tests coverage @ ${GITHUB_SHA::7}" - - name: Generate monero-cpp coverage report + - name: Generate the monero-cpp coverage report run: | args=() for f in shards/*/coverage_monero_cpp.info; do args+=(-a "$f"); done @@ -298,10 +323,11 @@ jobs: genhtml coverage_monero_cpp.info --output-directory coverage_monero_cpp_html --ignore-errors mismatch,inconsistent,source,corrupt --title "monero-cpp coverage @ ${GITHUB_SHA::7}" lcov --summary coverage_monero_cpp.info - - name: Upload coverage report + - name: Upload the coverage reports uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: coverage-reports-linux + # the default retention: the release downloads it from the run of main, and Codacy reads it when the run ends path: | coverage.info coverage.xml diff --git a/.github/workflows/codacy.yml b/.github/workflows/codacy.yml index cbfd2d5..8892eee 100644 --- a/.github/workflows/codacy.yml +++ b/.github/workflows/codacy.yml @@ -1,18 +1,18 @@ name: Codacy Coverage Reporter -# workflow_run is what gives this job the secret after a run of Build on a pull request. It does not check out or run anything +# workflow_run is what gives this job the secret after a run of CI on a pull request. It does not check out or run anything # of the pull request: it downloads the coverage files, and reads them with a reporter pinned to a checksum. on: # zizmor: ignore[dangerous-triggers] workflow_run: workflows: - - Build + - CI types: - completed workflow_dispatch: permissions: contents: read - # needed to download the artifact of the Build run with the default GITHUB_TOKEN + # needed to download the artifact of the CI run with the default GITHUB_TOKEN actions: read jobs: @@ -51,7 +51,7 @@ jobs: if: ${{ !cancelled() && steps.download.outcome == 'success' }} env: CODACY_PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }} - # the commit that Build ran on, which is the head of the pull request. A manual run has no workflow_run + # the commit that CI ran on, which is the head of the pull request. A manual run has no workflow_run COMMIT_UUID: ${{ github.event.workflow_run.head_sha || github.sha }} run: | ./codacy-coverage-reporter report --language python --coverage-reports coverage.xml --partial --commit-uuid "$COMMIT_UUID" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8b69eb1..002cd16 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ permissions: actions: read jobs: - # the macos and windows builds, the tests and the coverage are those of the Build run of main, the deb ones are built here + # the macos and windows builds, the tests and the coverage are those of the CI run of main, the deb ones are built here check: name: Check the tag and the builds of main runs-on: ubuntu-latest @@ -37,16 +37,16 @@ jobs: fi # a successful run is one in which every build and every test passed - - name: Find the main-branch Build run for this commit + - name: Find the main-branch CI run for this commit id: find_run env: GH_TOKEN: ${{ github.token }} run: | - run_id=$(gh run list --repo "${{ github.repository }}" --workflow build.yml \ + run_id=$(gh run list --repo "${{ github.repository }}" --workflow ci.yml \ --branch main --status success --limit 200 --json databaseId,headSha \ --jq ".[] | select(.headSha == \"${{ github.sha }}\") | .databaseId" | head -n1) if [ -z "$run_id" ]; then - echo "::error::No successful Build run on main for commit ${{ github.sha }}. Push to main and wait for it to build and pass before tagging." + echo "::error::No successful CI run on main for commit ${{ github.sha }}. Push to main and wait for it to build and pass before tagging." exit 1 fi echo "build_run_id=${run_id}" >> "$GITHUB_OUTPUT" @@ -86,7 +86,7 @@ jobs: with: github-token: ${{ github.token }} run-id: ${{ needs.check.outputs.build_run_id }} - pattern: monero-macos-* + pattern: wheel-mac-* path: release-assets merge-multiple: true @@ -95,7 +95,7 @@ jobs: with: github-token: ${{ github.token }} run-id: ${{ needs.check.outputs.build_run_id }} - name: monero-python-win-amd64 + name: wheel-windows-x86_64 path: release-assets merge-multiple: true diff --git a/README.md b/README.md index 2df42aa..2cf59b6 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,8 @@ # Monero Python Library +[![CI](https://github.com/libmonero/monero-python/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/libmonero/monero-python/actions/workflows/ci.yml) [![Codacy Badge](https://app.codacy.com/project/badge/Grade/f6129b0bbe314061b34a262a4cb89109)](https://app.codacy.com/gh/libmonero/monero-python/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade) [![Codacy Badge](https://app.codacy.com/project/badge/Coverage/f6129b0bbe314061b34a262a4cb89109)](https://app.codacy.com/gh/libmonero/monero-python/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_coverage) -[![Build](https://github.com/libmonero/monero-python/actions/workflows/build.yml/badge.svg?branch=main)](https://github.com/libmonero/monero-python/actions/workflows/build.yml) - -[![Build Debian Packages](https://github.com/libmonero/monero-python/actions/workflows/build-deb.yml/badge.svg?branch=main)](https://github.com/libmonero/monero-python/actions/workflows/build-deb.yml) [![Pybind](https://img.shields.io/badge/pybind-v2-blue?logo=python&logoColor=white)](https://github.com/pybind/pybind11) [![Python](https://img.shields.io/badge/python-3.10_%7C_3.11_%7C_3.12_%7C_3.13_%7C_3.14-blue?logo=python&logoColor=white)](https://github.com/libmonero/monero-python#monero-python-library) @@ -179,28 +177,19 @@ The package is built from [debian/](debian) with `dpkg-buildpackage`, the way CI ``` pacman -S mingw-w64-x86_64-toolchain make mingw-w64-x86_64-cmake mingw-w64-x86_64-openssl mingw-w64-x86_64-zeromq mingw-w64-x86_64-libsodium mingw-w64-x86_64-unbound mingw-w64-x86_64-protobuf git gettext base-devel wget - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst - echo "8486a018aca2e56d1fcd2eed2069011f2e1da02e03af72b38916ee06cbe7203f mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst" | sha256sum -c - pacman -U mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst - # hidapi and libusb were removed from the msys2 mingw64 repo, install the archived packages - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst - echo "89fb61549a437d65076a092a367fcb264fc7045ee9ebef48cd81f5a76a5588f8 mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst" | sha256sum -c - echo "124edc26b009678a4c5e759481a364abf63ed7f2f49a53092e8bd0a47c0c0740 mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst" | sha256sum -c - pacman -U mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst - echo "241919a5885a9270d0e0937e71cf5e146c2a8cabecc2714f09248350181cb1b6 mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst" | sha256sum -c - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst - echo "5fd77df3e5983ba0d9c689f46151119e0e59b3da6850c5a0ba54068eeae7749c mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst" | sha256sum -c - pacman -U mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst - wget https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst - echo "5b56ad87bc544336b2d51edb8bb0614f46542571362daf4d7e53bf23120175a1 mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst" | sha256sum -c - pacman -U mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst ``` - These are the versions CI uses. CI also pins the MinGW toolchain, see [build-windows.yml](.github/workflows/build-windows.yml). - 5. Clone repo: `git clone --recurse-submodules https://github.com/libmonero/monero-python.git` -6. Build monero-cpp, located as a submodule at ./external/monero-cpp: +6. Install the packages at the versions that CI uses (ICU, hidapi and libusb, boost and pybind11). [bin/msys2_pins.txt](bin/msys2_pins.txt) has them with their sha256, and the script downloads, checks and installs a group of them: + + ``` + cd monero-python + for group in icu hidapi-libusb boost pybind11; do bin/install_msys2_pins.sh $group; done + ``` + + hidapi and libusb were removed from the msys2 mingw64 repo, the script installs the archived packages. CI also pins the MinGW toolchain (`bin/install_msys2_pins.sh toolchain`), see [build-windows.yml](.github/workflows/build-windows.yml). + +7. Build monero-cpp, located as a submodule at ./external/monero-cpp: ``` cd external/monero-cpp @@ -208,7 +197,7 @@ The package is built from [debian/](debian) with `dpkg-buildpackage`, the way CI cd ../.. ``` -7. Build monero-python: +8. Build monero-python: ``` mkdir -p build && @@ -217,7 +206,7 @@ The package is built from [debian/](debian) with `dpkg-buildpackage`, the way CI cmake --build . ``` -8. Or run build script: `./bin/build_libmonero_python.sh` +9. Or run build script: `./bin/build_libmonero_python.sh` ## Use RPC servers in your project: diff --git a/bin/install_dependencies_linux.sh b/bin/install_dependencies_linux.sh index af30650..7d8fbeb 100755 --- a/bin/install_dependencies_linux.sh +++ b/bin/install_dependencies_linux.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# The packages of apt for the Linux build of build-linux.yml, and for the tests of build.yml: the module is linked to the shared libraries +# The packages of apt for the Linux build of build-linux.yml, and for the tests of ci.yml: the module is linked to the shared libraries # of some of them (the boost, zeromq, sodium and protobuf ones), so a job that only runs the module needs them too. The development packages # bring the libraries, and are the same ones in both so that the module finds at run time what it was linked to. set -euo pipefail diff --git a/bin/install_msys2_pins.sh b/bin/install_msys2_pins.sh new file mode 100755 index 0000000..7561a91 --- /dev/null +++ b/bin/install_msys2_pins.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Installs a group of the packages that bin/msys2_pins.txt pins, in the MSYS2 of the Windows build (build-windows.yml): it downloads +# each one, checks it against its sha256 and installs the group with a single pacman -U, since the packages of a group depend on each other. +# +# bin/install_msys2_pins.sh +set -euo pipefail + +group=${1:?usage: $0 } +pins="$(dirname "${BASH_SOURCE[0]}")/msys2_pins.txt" +repo=${MSYS2_REPO:-https://repo.msys2.org/mingw/mingw64} + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +files=() +while read -r pin_group sha256 file; do + [ "$pin_group" = "$group" ] || continue + wget -nv -P "$work" "$repo/$file" + echo "$sha256 $work/$file" | sha256sum -c + files+=("$work/$file") +done < <(grep -v '^#' "$pins") + +if [ ${#files[@]} -eq 0 ]; then + echo "No package of the group $group in $pins" >&2 + exit 1 +fi + +pacman -U --noconfirm "${files[@]}" diff --git a/bin/job_summary.sh b/bin/job_summary.sh new file mode 100755 index 0000000..6193c36 --- /dev/null +++ b/bin/job_summary.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Prints what it reads and also adds it to the summary of the job (the page of the run) as a block of code, with a title. It does nothing +# more than print when there is no summary, as outside of GitHub Actions, and it never fails because of the summary. +# +# bin/show_versions_linux.sh | bin/job_summary.sh "apt libraries" +set -uo pipefail + +title=${1:-} +text=$(cat) +printf '%s\n' "$text" + +summary=${GITHUB_STEP_SUMMARY:-} +if [ -n "$summary" ]; then + # in MSYS2 the file of the summary has a path of Windows + if command -v cygpath > /dev/null 2>&1; then + summary=$(cygpath -u "$summary") + fi + { + if [ -n "$title" ]; then + printf '### %s\n\n' "$title" + fi + printf '```\n%s\n```\n\n' "$text" + } >> "$summary" || true +fi +exit 0 diff --git a/bin/msys2_pins.txt b/bin/msys2_pins.txt new file mode 100644 index 0000000..21ee42f --- /dev/null +++ b/bin/msys2_pins.txt @@ -0,0 +1,22 @@ +# The packages of the MSYS2 mingw64 repository that the Windows build installs at an exact version, for bin/install_msys2_pins.sh. The +# versions of the repository change, and some of these have been dropped from it or need a gcc runtime that is not the one pinned here. +# A line is ` `: a group is installed with one pacman -U. To update, put the new file and its hash, and the +# version in the name of the step of the group in .github/workflows/build-windows.yml. +toolchain acad5748942953f91a0f2bba03e20437b909366e55949f478f1b2f2d30c726a7 mingw-w64-x86_64-headers-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain d24b4e7614756414ae3ef58793265448a3ca7d5fd447a991e9b2facde7b59d75 mingw-w64-x86_64-crt-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 827363748ce3320683319d860ee4fcfcdbb36baf66aac6dab0ec3822be2d4ff7 mingw-w64-x86_64-binutils-2.47-3-any.pkg.tar.zst +toolchain 481babe8c2fdfa4fc7f51bf13939701719af9f8166ef6a91a9a209cc25961b03 mingw-w64-x86_64-libmangle-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 7d86a267b6069198d28f3e77314282ff7b4e3c7bb8fe482299ca58f7a5b439db mingw-w64-x86_64-winpthreads-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 1d0ed7fda332a5c879de210f3d72c75fa1f3df7cc63dd951e763dec8eb02c1ee mingw-w64-x86_64-libwinpthread-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 7fcafd7ba30bd47000263e76a06bd893406c64f88955825f0a9188407a35c3d3 mingw-w64-x86_64-winstorecompat-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain c17557d09371e39db38deb56c7da6ae54e2c65b6f3466ca02f4da26b616ca8c3 mingw-w64-x86_64-tools-14.0.0.r375.g9c1abbbf5-1-any.pkg.tar.zst +toolchain 37b97f372409fa7cf54d709bab892f34e9c62d2658230a09f783a89a55ae590e mingw-w64-x86_64-pkgconf-1~3.0.7-1-any.pkg.tar.zst +toolchain c19e7caf09bbc89b2730556b2da73004118d4e1f967a685cc48524c7ff80d864 mingw-w64-x86_64-make-4.4.1-5-any.pkg.tar.zst +toolchain f8e25ea67bb796e7f65550f0dca9fce4cdde8aaa3dadafe4d13c6a8233c8de26 mingw-w64-x86_64-gcc-libs-16.2.0-3-any.pkg.tar.zst +toolchain 8b37c953ae12f3d8fa05c854236463231ea0a294dff423007cbd93d4d8089f9f mingw-w64-x86_64-gcc-16.2.0-3-any.pkg.tar.zst +icu 8486a018aca2e56d1fcd2eed2069011f2e1da02e03af72b38916ee06cbe7203f mingw-w64-x86_64-icu-78.3-4-any.pkg.tar.zst +hidapi-libusb 89fb61549a437d65076a092a367fcb264fc7045ee9ebef48cd81f5a76a5588f8 mingw-w64-x86_64-hidapi-0.15.0-2-any.pkg.tar.zst +hidapi-libusb 124edc26b009678a4c5e759481a364abf63ed7f2f49a53092e8bd0a47c0c0740 mingw-w64-x86_64-libusb-1.0.30-1-any.pkg.tar.zst +boost 241919a5885a9270d0e0937e71cf5e146c2a8cabecc2714f09248350181cb1b6 mingw-w64-x86_64-boost-1.87.0-3-any.pkg.tar.zst +boost 5fd77df3e5983ba0d9c689f46151119e0e59b3da6850c5a0ba54068eeae7749c mingw-w64-x86_64-boost-libs-1.87.0-3-any.pkg.tar.zst +pybind11 5b56ad87bc544336b2d51edb8bb0614f46542571362daf4d7e53bf23120175a1 mingw-w64-x86_64-pybind11-2.13.6-2-any.pkg.tar.zst diff --git a/bin/show_versions_linux.sh b/bin/show_versions_linux.sh new file mode 100755 index 0000000..c48e255 --- /dev/null +++ b/bin/show_versions_linux.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Prints the versions of the tools and of the libraries that bin/install_dependencies_linux.sh installs with apt, for the log of a job. They are the ones that the +# distribution of the runner has, not pinned ones, so what a run built and tested with is only known from its log. +set -uo pipefail + +# only the installed ones: a pattern also matches the packages of other versions that the distribution has, which are not installed +dpkg-query -W -f='${db:Status-Abbrev} ${Package} ${Version}\n' \ + libssl-dev libzmq3-dev libunbound-dev libsodium-dev libunwind8-dev liblzma-dev libexpat1-dev libpgm-dev \ + libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler 'libboost1.*-dev' \ + cmake ccache python3 lcov 2> /dev/null | awk '$1 == "ii" { print $2, $3 }' +g++ --version | head -n 1 diff --git a/debian/set-version.sh b/debian/set-version.sh index 1401b2b..8c0cfa0 100755 --- a/debian/set-version.sh +++ b/debian/set-version.sh @@ -1,8 +1,10 @@ #!/bin/sh # Adds an entry to debian/changelog for the version declared in pyproject.toml, so the # version is not repeated here. The package is named like python3-monero_0.9.10-1noble1_amd64.deb. +# With a snapshot, such as the commit that is built, it is python3-monero_0.9.10~git1a2b3c4-1noble1_amd64.deb: the tilde sorts it +# before the release, so a release is always newer than the snapshots that came before it. # -# usage: debian/set-version.sh [distribution] the distribution is the one of the system by default +# usage: debian/set-version.sh [distribution [snapshot]] the distribution is the one of the system by default set -eu cd "$(dirname "$0")/.." @@ -14,6 +16,8 @@ if [ -z "$version" ]; then fi dist=${1:-$(. /etc/os-release && echo "$VERSION_CODENAME")} +snapshot=${2:-} +version=${version}${snapshot:+~$snapshot} full_version="${version}-1${dist}1" if [ "$(dpkg-parsechangelog -S Version)" = "$full_version" ]; then diff --git a/docs/ci-ccache.md b/docs/ci-ccache.md index 2da26d7..bdf99c6 100644 --- a/docs/ci-ccache.md +++ b/docs/ci-ccache.md @@ -11,10 +11,11 @@ The workflows keep a ccache directory (`.ccache` in the workspace) between runs ### Where the cache lives -- Key is `ccache-_`, restore falls back to the newest `ccache-_`. The `_` is there because `macos-15` is a prefix of `macos-15-intel`. -- Each build has its own cache: Linux, Windows and macOS (one for each architecture) in `build.yml`, and the deb builds one per distro and arch. The tests do not compile, they use what the builds uploaded. +- Key is `ccache-_`, restore falls back to the newest `ccache-_`. The `_` is there because `macos-15` is a prefix of `macos-15-intel`. The local actions `.github/actions/restore-ccache` and `.github/actions/save-ccache` do both, the builds only give them the key. +- Each build has its own cache: Linux, Windows and macOS (one for each architecture) in `ci.yml`, and the deb builds one per distro and arch. The tests do not compile, they use what the builds uploaded. - Pull requests only restore. Pushes to main and manual runs (`workflow_dispatch`) save. This is on purpose, see quota below. The deb workflow also runs on pull requests that touch its own files, and follows the same rule. - A cache saved on a branch is only visible to that branch. Main's caches are visible to everyone, so a pull request starts from main's. +- Saving deletes only the entry of the same key and of the ref that is running (the API call has a `ref`): deleting by key alone removes the entries of every ref, and a manual run on a branch would take away main's. A tag does not save in the deb workflow, its entry would only be readable by that tag, and the release builds from the caches of main (the legs that main does not build, armhf and arm64 on Ubuntu, start with no cache). - The save step runs after the build and before the tests, so a flaky test doesn't throw the cache away. Failed builds save too, whatever got compiled is still good. ### What isn't cached @@ -44,6 +45,6 @@ Measured on `build monero` for Windows: about 9 minutes with a cold cache, about ### Checking it works -Run `build.yml` twice on the same branch with `workflow_dispatch`. Second run should show mostly hits in the "Show ccache statistics" step (for deb it's the end of the build step, printed by `debian/rules`). Stats are zeroed at the start of each job, so they're only for that run. +Run `ci.yml` twice on the same branch with `workflow_dispatch`. Second run should show mostly hits in the "Show ccache statistics" step (for deb it's the end of the build step, printed by `debian/rules`). Stats are zeroed at the start of each job, so they're only for that run. To start over: `gh cache list`, then `gh cache delete `. diff --git a/tests/requirements.txt b/tests/requirements.txt index 4caa5f3..4f0b49a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,4 @@ -# The versions that the tests run with, in CI (.github/workflows/build.yml) and locally: pip install -r tests/requirements.txt +# The versions that the tests run with, in CI (.github/workflows/ci.yml) and locally: pip install -r tests/requirements.txt # Dependabot proposes the updates. coverage==7.17.3 pytest==9.2.0