test-and-publish.yaml

name: 'Test and Publish Suite'

on:
  push:
    branches:
      - main
    tags:
      - 'v*'
  workflow_call:
  workflow_dispatch:
    inputs:
      debug_enabled_os:
        type: choice
        description: Optionally, choose an OS to run the build with SSH debugging on (https://github.com/fawazahmed0/action-debug)
        required: false
        options:
        - ''
        - 'ubuntu-22.04'
        - 'macos-13'
        - 'macos-14'
        - 'windows-2022'
      debug_enabled_python:
        type: choice
        description: Choose a Python version to run the build with SSH debugging on
        required: false
        options:
        - ''
        - '3.8'
        - '3.9'
        - '3.10'
        - '3.11'
        - '3.12'
        - '3.13'
        - '3.14-dev'
      build_type:
        type: choice
        description: 'Choose the build type to use'
        required: false
        default: 'Debug'
        options:
        - 'Debug'
        - 'Profile'
        - 'Sanitize'
        - 'DRelease'
        - 'Release'
        - 'None'
  pull_request:

env:
  # don't upgrade outdated brew packages because the process is too slow
  HOMEBREW_NO_INSTALL_UPGRADE: 1
  HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1
  # apt-get should run in noninteractive mode
  DEBIAN_FRONTEND: noninteractive

defaults:
  run:
    # run with Git Bash on Windows
    shell: bash

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-spidermonkey-unix:
    strategy:
      fail-fast: false
      matrix:
        # Use Ubuntu 22.04 / macOS 13 x86_64 / macOS 14 arm64 + Python 3.10 to build SpiderMonkey
        os: [ 'ubuntu-22.04', 'macos-13', 'macos-14', 'ubuntu-22.04-arm' ] # macOS 14 runner exclusively runs on M1 hardwares
                                                                           # see https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available
        python_version: [ '3.10' ]
    runs-on: ${{ matrix.os }}
    container: ${{ (startsWith(matrix.os, 'ubuntu') && 'ubuntu:20.04') || null }} # Use the Ubuntu 20.04 container inside Ubuntu 22.04 runner to build
    steps:
      - uses: actions/checkout@v4
      - name: Read the mozilla-central commit hash to be used
        run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
      - name: Cache spidermonkey build
        id: cache-spidermonkey
        uses: actions/cache@v4
        with:
          path: |
            ./_spidermonkey_install/*
          key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
      - name: Setup container
        if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        run: |
          apt-get update -y
          apt-get install -y sudo libnss3-dev libssl-dev
          apt-get install -y curl make git build-essential
          apt-get install -y zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv
          DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata
          echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04
      - name: Setup LLVM
        if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        run: |
          apt-get install -y llvm clang
          apt-get install -y lsb-release wget software-properties-common gnupg
          wget https://apt.llvm.org/llvm.sh
          chmod +x llvm.sh
          ./llvm.sh 18 # install LLVM version 18
          update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-18 18
          update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 18
          update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 18
          clang --version
          clang++ --version
      - name: Setup Python
        if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        run: |
          # Use pyenv to install Python version that is not available via `actions/setup-python`
          unset PYENV_ROOT
          curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
          echo "$HOME/.pyenv/bin" >> $GITHUB_PATH # ~/.bashrc file is not read, so we need to add to GITHUB_PATH manually
          echo "$HOME/.pyenv/shims" >> $GITHUB_PATH
          echo "PYENV_ROOT=$HOME/.pyenv" >> $GITHUB_ENV
          export PATH="$HOME/.pyenv/bin:$PATH"
          pyenv install $PYTHON_VERSION
          pyenv global $PYTHON_VERSION
        env:
          PYTHON_VERSION: ${{ matrix.python_version }}
      - uses: actions/setup-python@v5
        if: ${{ !startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        with:
          python-version: ${{ matrix.python_version }}
      - name: Setup XCode
        if: ${{ matrix.os == 'macos-13' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        # SpiderMonkey requires XCode SDK version at least 13.3
        run: sudo xcode-select -switch /Applications/Xcode_14.3.app
      - name: Build spidermonkey
        if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        run: ./setup.sh
      - name: Upload spidermonkey build as CI artifacts
        uses: actions/upload-artifact@v4
        with:
          name: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
          path: ./_spidermonkey_install/
  build-spidermonkey-win:
    runs-on: windows-2022
      # SpiderMonkey requires Visual Studio 2022 or newer.
      # The Windows 2019 runner only has Visual Studio Enterprise 2019 installed.
    steps:
      - uses: actions/checkout@v4
      - name: Read the mozilla-central commit hash to be used
        run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
      - name: Cache spidermonkey build
        id: cache-spidermonkey
        uses: actions/cache@v4
        with:
          path: |
            ./_spidermonkey_install/*
          key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
      - name: Install dependencies
        if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        shell: powershell
        run: |
          # Already installed in Github Actions runner
          # choco install -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' # add CMake to system PATH
          # choco install -y llvm gnuwin32-m4
          choco install -y wget make unzip
      - name: Install MozillaBuild
        if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        run: |
          wget -c -q https://ftp.mozilla.org/pub/mozilla/libraries/win32/MozillaBuildSetup-Latest.exe
          powershell -command 'Start-Process -Wait -FilePath "./MozillaBuildSetup-Latest.exe" -ArgumentList "/S"'
      - name: Build spidermonkey in MozillaBuild environment
        if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }}
        env:
          # Preserve MozillaBuild v4.0.x behaviour
          # see https://groups.google.com/u/1/a/mozilla.org/g/dev-platform/c/hF51Q3j6ca8
          USE_MINTTY: 0
        run: /c/mozilla-build/start-shell.bat -use-full-path -here ./setup.sh
      - name: Upload spidermonkey build as CI artifacts
        uses: actions/upload-artifact@v4
        with:
          name: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
          path: ./_spidermonkey_install/
  build-and-test:
    needs: [build-spidermonkey-unix, build-spidermonkey-win]
    strategy:
      fail-fast: false
      matrix:
        os: [ 'ubuntu-22.04', 'macos-13', 'macos-14', 'windows-2022', 'ubuntu-22.04-arm' ]
        python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13', '3.14-dev' ]
    runs-on: ${{ matrix.os }}
    container: ${{ (startsWith(matrix.os, 'ubuntu') && 'ubuntu:20.04') || null }}
    steps:
      - name: Setup container
        if: ${{ startsWith(matrix.os, 'ubuntu') }}
        run: |
          apt-get update -y
          apt-get install -y sudo libnss3-dev libssl-dev
          apt-get install -y curl zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv
          apt-get install -y git # required for `actions/checkout`
          apt-get install -y nodejs npm # required for pminit to build
          apt-get install -y build-essential
          apt-get install -y strace # required to run JS tests
          DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata # tzdata may ask for user interaction if not explicitly installed here
          echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04
          git config --global --add safe.directory '*' # silence "git failed because of dubious ownership"
          
          # CMake 3.25 or higher is required
          apt-get install -y ca-certificates gpg wget
          wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
          echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null
          apt-get update -y && apt-get install -y cmake
      - uses: actions/checkout@v4
        with:
          submodules: recursive
          fetch-depth: 0 # fetch all history for all branches and tags
          # poetry-dynamic-versioning needs git tags to produce the correct version number 
      - name: Setup Python
        if: ${{ startsWith(matrix.os, 'ubuntu') }}
        run: |
          # Use pyenv to install Python version that is not available via `actions/setup-python`
          unset PYENV_ROOT
          curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash
          echo "$HOME/.pyenv/bin" >> $GITHUB_PATH # ~/.bashrc file is not read, so we need to add to GITHUB_PATH manually
          echo "$HOME/.pyenv/shims" >> $GITHUB_PATH
          echo "PYENV_ROOT=$HOME/.pyenv" >> $GITHUB_ENV
          export PATH="$HOME/.pyenv/bin:$PATH"
          if [ "$PYTHON_VERSION" = "3.14-dev" ]; then
            # Replacing 3.14-dev with a pinned prerelease version for fast installation
            PYTHON_VERSION="3.14.0rc3"
          fi
          pyenv install $PYTHON_VERSION
          pyenv global $PYTHON_VERSION
        env:
          PYTHON_VERSION: ${{ matrix.python_version }}
      - uses: actions/setup-python@v5
        if: ${{ !startsWith(matrix.os, 'ubuntu') }}
        with:
          python-version: ${{ matrix.python_version }}
      - name: Setup Poetry
        uses: snok/install-poetry@v1
        with:
          version: 1.5.1
      - name: Install Dependencies
        run: |
          echo "Installing Dependencies"
          if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux
            sudo apt-get update -y
            sudo apt-get install -y cmake llvm
          elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS
            brew update || true # allow failure
            brew install cmake pkg-config wget unzip coreutils # `coreutils` installs the `realpath` command
          fi
          echo "Installing python deps"
          poetry self add "poetry-dynamic-versioning[plugin]"
          echo "Use the correct Python version we've set up"
          poetry env use python$PYTHON_VERSION || poetry env use python3 # use the correct Python version we've set up
          poetry install --no-root --only=dev
          echo "Installed Dependencies"
        env:
          PYTHON_VERSION: ${{ matrix.python_version }}
      - name: Build Docs # only build docs once
        if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python_version == '3.11' }}
        run: |
            sudo apt-get install -y graphviz 
            # the newest version in Ubuntu 20.04 repository is 1.8.17, but we need Doxygen 1.9 series
            wget -c -q https://www.doxygen.nl/files/doxygen-1.9.7.linux.bin.tar.gz
            tar xf doxygen-1.9.7.linux.bin.tar.gz
            cd doxygen-1.9.7 && sudo make install && cd -
            rm -rf doxygen-1.9.7 doxygen-1.9.7.linux.bin.tar.gz
            BUILD_DOCS=1 BUILD_TYPE=None poetry install
      - name: Upload Doxygen-generated docs as CI artifacts
        if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python_version == '3.11' }}
        uses: actions/upload-artifact@v4
        with:
          name: docs-${{ github.run_id }}-${{ github.sha }}
          path: ./build/docs/html/
      - name: Read the mozilla-central commit hash to be used
        run: echo "MOZCENTRAL_VERSION=$(cat mozcentral.version)" >> $GITHUB_ENV
      - name: Use cached spidermonkey build
        uses: actions/cache@v4
        with:
          path: |
            ./_spidermonkey_install/*
          key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }}
          fail-on-cache-miss: true # SpiderMonkey is expected to be cached in its dedicated job
      - name: Build pminit
        run: |
          cd python/pminit
          poetry build --format=sdist
          cd -
          mkdir -p ./dist/
          mv -v python/pminit/dist/* ./dist/
      - name: Build wheel
        run: |
          echo $(poetry run python --version)
          WORKFLOW_BUILD_TYPE=${{ inputs.build_type }}
          BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry build --format=wheel
          ls -lah ./dist/
      - name: Make the wheels we build also support lower versions of macOS
        if: ${{ matrix.os == 'macos-13' || matrix.os == 'macos-14' }}
        # Change the platform tag part of the wheel filename to `macosx_11_0_xxx` (means to support macOS 11.0 and above)
        # See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-format
        # A wheel package file will only be selected by pip to install if the platform tag satisfies, regardless of whether the binary compatibility actually is.
        # Otherwise, pip would fallback to compile from the source distribution.
        run: |
          cd ./dist/
          for file in *.whl; do
            mv "$file" "$(echo "$file" | sed -E 's/macosx_[0-9]+_[0-9]+/macosx_11_0/')";
          done
      - name: Upload wheel as CI artifacts
        uses: actions/upload-artifact@v4
        with:
          name: wheel-${{ github.run_id }}-${{ github.sha }}-${{ runner.os }}_${{ runner.arch }}_Python${{ matrix.python_version }}
          path: ./dist/
      - name: Run Python tests (pytest)
        run: |
          WORKFLOW_BUILD_TYPE=${{ inputs.build_type }}
          BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry run python -m pip install --force-reinstall --verbose ./dist/*
          poetry run python -m pytest tests/python
      - name: Run JS tests (peter-jr)
        if: ${{ (success() || failure()) }}
        run: |
          poetry run bash ./peter-jr ./tests/js/
      - name: SSH debug session
        if: ${{ (success() || failure()) && github.event_name == 'workflow_dispatch' && inputs.debug_enabled_os == matrix.os && inputs.debug_enabled_python == matrix.python_version}}
        uses: fawazahmed0/action-debug@main
        with:
          credentials: "admin:admin"
  sdist:
    runs-on: ubuntu-22.04
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - name: Setup Poetry
        uses: snok/install-poetry@v1
        with:
          version: 1.5.1
      - name: Build source distribution (sdist) file
        run: |
          poetry self add "poetry-dynamic-versioning[plugin]"
          poetry build --format=sdist
          ls -lah ./dist/
      - name: Upload sdist as CI artifacts
        uses: actions/upload-artifact@v4
        with:
          name: wheel-${{ github.run_id }}-${{ github.sha }}-sdist
          path: ./dist/
  publish:
    needs: [build-and-test, sdist]
    runs-on: ubuntu-22.04
    if: ${{ success() && github.event_name == 'push' && github.ref_type == 'tag' }}
    steps:
      # no need to checkout
      - uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      - run: pip install twine
      - name: Download wheels built
        uses: actions/download-artifact@v4
        with:
          pattern: wheel-${{ github.run_id }}-${{ github.sha }}-*
          merge-multiple: true
          path: ./dist/
      - run: ls -lah ./dist/
      - name: Publish package
        run: |
          twine upload dist/* \
            --non-interactive --skip-existing \
            --username __token__ --password ${{ secrets.PYPI_API_TOKEN }}
  publish-nightly:
    # Implement a very basic Python package repository (https://peps.python.org/pep-0503/)
    # and deploy the static files to GitHub Pages
    needs: [build-and-test, sdist]
    runs-on: ubuntu-22.04
    if: ${{ (success() || failure()) && (github.ref_name == 'main' || github.ref_type == 'tag') }} # publish nightly builds regardless of tests failure
    permissions: # grant GITHUB_TOKEN the permissions required to make a Pages deployment
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    steps:
      # don't checkout
      - name: Download wheels built
        uses: actions/download-artifact@v4
        with:
          pattern: wheel-${{ github.run_id }}-${{ github.sha }}-*
          merge-multiple: true
          path: ./dist/
      - name: Download docs html generated by Doxygen
        uses: actions/download-artifact@v4
        with:
          name: docs-${{ github.run_id }}-${{ github.sha }}
          path: ./docs/
      - name: Move wheels to the correct repository project paths
        run: |
          mkdir -p ./pythonmonkey/ ./pminit/
          mv ./dist/pythonmonkey-* ./pythonmonkey/
          mv ./dist/pminit-* ./pminit/
      - name: Generate index page for the project
        run: |
          generate_index () {
            cd $1
            html="<html><head><title>PythonMonkey Nightly Builds</title></head><body>"
            for file in ./*; do # generate <a> tags for each file
              html+="<a href=\"$file\">$file</a><br/>"
            done
            html+="</body></html>"
            echo "$html" > ./index.html
            cd -
          }
          generate_index ./pythonmonkey/
          generate_index ./pminit/
      - name: Generate repository root page
        run: |
          html="<html><head><title>PythonMonkey Nightly Builds</title></head><body>"
          html+="<h1>PythonMonkey Nightly Builds</h1>"
          html+="<p>To install nightly builds, run</p>"
          html+="<pre>pip install --extra-index-url https://nightly.pythonmonkey.io/ --pre pythonmonkey</pre>"
          html+="<h3>Browse files:</h3>"
          html+="<li><a href="pythonmonkey/">pythonmonkey</a></li>"
          html+="<li><a href="pminit/">pminit</a></li>"
          html+="</body></html>"
          echo "$html" > ./index.html
      - uses: actions/upload-pages-artifact@v3
        with:
          path: ./
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4
  publish-archive:
    # Publish to ⊇istributive's archive server (https://archive.distributed.computer/releases/pythonmonkey/)
    needs: [build-and-test, sdist]
    runs-on: ubuntu-22.04
    if: ${{ (success() || failure()) && (github.ref_name == 'main' || github.ref_type == 'tag') }}
    environment:
      name: archive
      url: https://archive.distributed.computer/releases/pythonmonkey/${{ steps.get_path.outputs.ARCHIVE_PATH }}
    steps:
      # no need to checkout
      - name: Download wheels built
        uses: actions/download-artifact@v4
        with:
          pattern: wheel-${{ github.run_id }}-${{ github.sha }}-*
          merge-multiple: true
          path: ./
      - name: Download docs html generated by Doxygen
        uses: actions/download-artifact@v4
        with:
          name: docs-${{ github.run_id }}-${{ github.sha }}
          path: ./docs/
      - name: Get the pythonmonkey/pminit version number
        run: |
          file=$(ls ./pminit*.tar.gz | head -1)
          pm_version=$(basename "${file%.tar.gz}" | cut -d- -f2) # match /pminit-([^-]+).tar.gz/
          echo "PM_VERSION=$pm_version" >> $GITHUB_ENV
      - name: Get the archive type (nightly or releases) and path
        id: get_path
        run: |
          path="$ARCHIVE_TYPE/$PM_VERSION/"
          echo "$path"
          echo "ARCHIVE_PATH=$path" >> $GITHUB_OUTPUT
        env:
          ARCHIVE_TYPE: ${{ (github.ref_type == 'tag' && 'releases') || 'nightly' }}
      - name: SCP to the archive server
        uses: appleboy/scp-action@v0.1.7
        with:
          host: ${{ secrets.ARCHIVE_HOST }}
          username: pythonmonkey
          key: ${{ secrets.ARCHIVE_KEY }}
          source: ./*
          target: archive/${{ steps.get_path.outputs.ARCHIVE_PATH }}
          overwrite: true