command.yaml

name: "ondemand test"

on:
  issue_comment:
    types: [created]

permissions:
  pull-requests: write
  checks: read
  contents: read

jobs:
  preprocess:
    runs-on: ubuntu-latest
    outputs:
      continue: ${{ steps.check.outputs.continue }}
      op: ${{ steps.parse-command.outputs.OP_ID}}
      runner: ${{ steps.parse-command.outputs.RUNNER }}
      pull: ${{ steps.parse-command.outputs.PR_NUMBER }}
      existing: ${{ steps.parse-command.outputs.EXISTING_OP }}
    steps:
      - id: check
        uses: github/command@3442f3fa1efe01bdb024b157083c337902d17372  # v2.0.3
        with:
          command: "/test"
          allowed_contexts: pull_request
          allow_forks: "true"
          skip_reviews: "true"
          fork_review_bypass: "true"
          permissions: "write,admin"

      - id: parse-command
        if: ${{ steps.check.outputs.continue == 'true' }}
        env:
          PULL_NUMBER: ${{ github.event.issue.number }}
        run: |
          # Parse command arguments
          params=${{ steps.check.outputs.params }}
          op=$(echo $params | cut -d ':' -f 1)
          runner=$(echo $params | cut -d ':' -f 2)
          echo "OP_ID=${op}" >> $GITHUB_OUTPUT
          echo "RUNNER=${runner}" >> $GITHUB_OUTPUT
          echo "PR_NUMBER=${PULL_NUMBER}" >> $GITHUB_OUTPUT

          # Detect if operator exists before
          num=$(grep "pytest.mark.${{ needs.preprocess.outputs.op }}" -r tests | wc -l)
          if [ "$num" -eq "0" ]; then
            echo "EXISTING_OP=0" >> $GITHUB_OUTPUT
          else
            echo "EXISTING_OP=1" >> $GITHUB_OUTPUT
          fi

  test-operator:
    needs: preprocess
    if: needs.preprocess.outputs.continue == 'true'
    runs-on: ${{ needs.preprocess.outputs.runner }}
    steps:
      - id: resolve-vendor
        shell: bash
        env:
          RUNNER_LABEL: ${{ needs.preprocess.outputs.runner }}
        run: |
          case "${RUNNER_LABEL}" in
            h20|a100|h100)
              VENDOR="nvidia"
              BACKEND="nvidia"
              ;;
            cann850)
              VENDOR="ascend"
              BACKEND="ascend-cann850"
              ;;
            cann9)
              VENDOR="ascend"
              BACKEND="ascend-cann900"
              ;;
            *)
              VENDOR="${RUNNER_LABEL}"
              BACKEND="${RUNNER_LABEL}"
              ;;
          esac
          echo "VENDOR=${VENDOR}" >> $GITHUB_ENV
          echo "BACKEND=${BACKEND}" >> $GITHUB_ENV

      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        continue-on-error: true
      - uses: ./.github/actions/checkout-retry
      - name: setup-gh
        shell: bash
        env:
          GH_VERSION: "2.95.0"
        run: |
          GH_DIR="${HOME}/.local/bin"
          if command -v gh &>/dev/null; then
            echo "gh already available: $(gh --version | head -1)"
          elif [ -x "${GH_DIR}/gh" ]; then
            echo "gh found at ${GH_DIR}: $("${GH_DIR}/gh" --version | head -1)"
            echo "${GH_DIR}" >> $GITHUB_PATH
          else
            mkdir -p "${GH_DIR}"
            curl -fsSL "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \
              | tar -xz --strip-components=2 -C "${GH_DIR}" "gh_${GH_VERSION}_linux_amd64/bin/gh"
            echo "Installed gh ${GH_VERSION} to ${GH_DIR}"
            echo "${GH_DIR}" >> $GITHUB_PATH
          fi
      - id: checkout-pr
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          echo "checking out ${{ needs.preprocess.outputs.pull }}"
          gh pr checkout ${{ needs.preprocess.outputs.pull }}

      - id: setup-flaggems
        shell: bash
        run: |
          ./setup.sh ${BACKEND}
          if [ "${BACKEND}" == "nvidia" ]; then
            source .venv/bin/activate
            uv pip install vllm==0.21.0 --torch-backend=cu130
          fi

      - id: check-gpu
        shell: bash
        run: |
          source tools/env.sh ${BACKEND}
          bash tools/gpu_check_${VENDOR}.sh

      - id: run-tests
        shell: bash
        env:
          EXISTING_OP: ${{ needs.preprocess.outputs.existing }}
        run: |
          source .venv/bin/activate
          source tools/env.sh ${BACKEND}
          tools/run_tests.py --ops ${{ needs.preprocess.outputs.op }} \
            --dump-output \
            --output-dir results_new

          if [ "$EXISTING_OP" == "0" ]; then
            tools/psum_text --format markdown \
              --output report.md \
              --single \
              results_new
            mv results_new results
          else
            git checkout master
            tools/run_tests.py --ops ${{ needs.preprocess.outputs.op }} \
              --dump-output \
              --output-dir results_old

            # merge two data sets into one directory
            mkdir results
            mv results_old results/before
            mv results_new results/after

            tools/psum_text --format markdown \
              --single \
              --compare \
              --output report.md \
              results
          fi

      - id: upload-results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: ondemand-test
          path: results/

      - id: attach-result-link
        run: |
          echo -e "\n---\n" >> report.md
          echo -e "📎 Download the full log [here](${{ steps.upload-results.outputs.artifact-url }})" >> report.md

      - id: comment
        uses: mshick/add-pr-comment@8e4927817251f1ff60c001f04568532b38e0b4a0  # 3.11.0
        with:
          message-path: report.md
          message-id: ${{ needs.preprocess.outputs.op }}-${{ needs.preprocess.outputs.runner }}
          allow-repeats: True