Schedule nightly torchbench runs

.github/workflows/v2-nightly.yml

@@ -0,0 +1,145 @@
+name: TorchBench V2 nightly
+on:
+  workflow_dispatch:
+  pull_request: # to remove
+  schedule:
+    - cron: '0 5 * * *' # run at 5 AM UTC (10 PM PDT)
+
+jobs:
+  run-benchmark:
+    env:
+      TORCHBENCH_VER: "v2"
+      CONFIG_VER: "v2"
+      PYTHON_VER: "3.8"
+      CONDA_ENV_NAME:  "torchbench-v2-nightly-ci"
+      OUTPUT_DIR: ".torchbench/v2-nightly-ci"
+      BISECTION_ROOT: ".torchbench/v2-bisection-ci"
+      IS_GHA: 1
+      BUILD_ENVIRONMENT: benchmark-nightly
+    runs-on: [macos-torchbench]
+    steps:
+      - name: Clean up disk space before running MacOS workflow
+        uses: pytorch/test-infra/.github/actions/check-disk-space@main
+
+      - name: Setup miniconda
+        uses: pytorch/test-infra/.github/actions/setup-miniconda@main
+        with:
+          python-version: 3.8
+          environment-file: .github/requirements/conda-env-${{ runner.os }}-${{ runner.arch }}
+
+      - name: Create conda environment and install deps
+        env:
+          ENV_NAME: conda-test-env-${{ github.run_id }}
+          PY_VERS: 3.8
+        shell: arch -arch arm64 bash {0}
+        run: |
+          # shellcheck disable=SC1090
+          set -ex
+          ${CONDA_RUN} pip install requests bs4 argparse gitpython boto3 regex pyyaml
+          # Check if nightly builds are available
+          # Install PyTorch nightly from pip
+          ${CONDA_RUN} pip3 install --pre torch torchvision torchtext torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu
+
+      - name: Checkout TorchBench
+        uses: malfet/checkout@silent-checkout
+        with:
+          repository: razarmehr/benchmark
+          path: benchmark
+          lfs: false
+          ref: ${{ env.TORCHBENCH_BRANCH }}
+
+      - name: Install other TorchBench dependencies
+        env:
+          ENV_NAME: conda-test-env-${{ github.run_id }}
+          PY_VERS: 3.8
+        shell: arch -arch arm64 bash {0}
+        run: |
+          # shellcheck disable=SC1090
+          set -ex
+          ${CONDA_RUN} conda install -y git-lfs
+          pushd benchmark
+          ${CONDA_RUN} python install.py --continue_on_fail
+          popd
+
+      - name: Run benchmark
+        env:
+          ENV_NAME: conda-test-env-${{ github.run_id }}
+          PY_VERS: 3.8
+        shell: arch -arch arm64 bash {0}
+        run: |
+          pushd benchmark
+          WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}"
+          ${CONDA_RUN} bash .github/scripts/run.sh "${WORKFLOW_HOME}"
+          popd
+      - name: Generate the bisection config
+        env:
+          ENV_NAME: conda-test-env-${{ github.run_id }}
+          PY_VERS: 3.8
+        shell: arch -arch arm64 bash {0}
+        run: |
+          set -x
+          WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}"
+          mkdir -p benchmark-output/
+          # Update the self-hosted pytorch version
+          pushd "${HOME}/pytorch"
+          git fetch origin
+          popd
+          pip install gitpython pyyaml dataclasses argparse
+          # Compare the result from yesterday and report any perf signals
+          ${CONDA_RUN} python ./.github/scripts/generate-abtest-config.py \
+                 --pytorch-dir "${HOME}/pytorch" \
+                 --github-issue "${WORKFLOW_HOME}/gh-issue.md" \
+                 --benchmark-dir "${WORKFLOW_HOME}" \
+                 --out "${WORKFLOW_HOME}/bisection.yaml"
+          # Include in the GitHub artifact
+          if [ -f "${WORKFLOW_HOME}/gh-issue.md" ]; then
+            cp "${WORKFLOW_HOME}/bisection.yaml" ./benchmark-output/
+            cp "${WORKFLOW_HOME}/gh-issue.md" ./benchmark-output/
+            # Setup the bisection environment
+            BISECTION_HOME="${HOME}/${{ env.BISECTION_ROOT }}/bisection-gh${GITHUB_RUN_ID}"
+            mkdir -p "${BISECTION_HOME}"
+            mv ./benchmark-output/gh-issue.md "${BISECTION_HOME}/gh-issue.md"
+            cp ./benchmark-output/bisection.yaml "${BISECTION_HOME}/config.yaml"
+          fi
+      # - name: Dispatch the bisection workflow
+      #   if: env.TORCHBENCH_PERF_SIGNAL
+      #   run: |
+      #     # Get the workflow ID from
+      #     # https://api.github.com/repos/pytorch/benchmark/actions/workflows
+      #     curl -u xuzhao9:${{ secrets.TORCHBENCH_ACCESS_TOKEN }} \
+      #       -X POST \
+      #       -H "Accept: application/vnd.github.v3+json" \
+      #       https://api.github.com/repos/pytorch/benchmark/actions/workflows/16176850/dispatches \
+      #       -d '{"ref": "main", "inputs": {"issue_name": "bisection-gh'"${GITHUB_RUN_ID}"'" } }'
+      # - name: Copy artifact and upload to scribe
+      #   run: |
+      #     . activate "${CONDA_ENV_NAME}"
+      #     TODAY=$(date "+%Y%m%d%H%M%S")
+      #     LATEST_RESULT=$(find ${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID} -name "*.json" | sort -r | head -1)
+      #     echo "Benchmark result file: ${LATEST_RESULT}"
+      #     mkdir -p benchmark-output/
+      #     cp "${LATEST_RESULT}" ./benchmark-output/benchmark-result-${CONFIG_VER}-${TODAY}.json
+      #     # Load environment variables
+      #     CONFIG_DIR=torchbenchmark/score/configs/${CONFIG_VER}
+      #     CONFIG_ENV=${CONFIG_DIR}/config-${CONFIG_VER}.env
+      #     # Load environment variables
+      #     set -a; source "${CONFIG_ENV}"; set +a
+      #     SCORE_FILE="./benchmark-result-${CONFIG_VER}-score-${TODAY}.json"
+      #     # Generate score file
+      #     python compute_score.py --score_version "${CONFIG_VER}" --benchmark_data_file "${LATEST_RESULT}" > "${SCORE_FILE}"
+      #     # Upload result to Scribe
+      #     python scripts/upload_scribe_${CONFIG_VER}.py --pytest_bench_json "${LATEST_RESULT}" --torchbench_score_file "${SCORE_FILE}"
+      - name: Upload artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: Benchmark result
+          path: benchmark-output/
+
+      - name: Clean up disk space
+        if: always()
+        continue-on-error: true
+        uses: pytorch/test-infra/.github/actions/check-disk-space@main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
+  cancel-in-progress: true