From 599e93930e0aa9bade0333659ad4989d00d9e6b8 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:08:31 -0700 Subject: [PATCH 01/13] Schedule nightly torchbench runs --- .github/workflows/v2-nightly.yml | 114 +++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 .github/workflows/v2-nightly.yml diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml new file mode 100644 index 0000000000000..d5e299123971d --- /dev/null +++ b/.github/workflows/v2-nightly.yml @@ -0,0 +1,114 @@ +name: TorchBench V2 nightly +on: + workflow_dispatch: + schedule: + - cron: '0 5 * * *' # run at 5 AM UTC (10 PM PDT) + +jobs: + run-benchmark: + env: + TORCHBENCH_VER: "v2" + CONFIG_VER: "v2" + PYTHON_VER: "3.8" + CONDA_ENV_NAME: "torchbench-v2-nightly-ci" + OUTPUT_DIR: ".torchbench/v2-nightly-ci" + BISECTION_ROOT: ".torchbench/v2-bisection-ci" + IS_GHA: 1 + BUILD_ENVIRONMENT: benchmark-nightly + runs-on: [macos-nightly] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + repository: razarmehr/benchmark + path: benchmark + ref: ${{ env.TORCHBENCH_BRANCH }} + - name: Create conda env + run: | + conda create -y -q --name "${CONDA_ENV_NAME}" python=${{ env.PYTHON_VER }} + - name: Install PyTorch nightly + run: | + . activate "${CONDA_ENV_NAME}" + # Install dependencies + pip install requests bs4 argparse gitpython boto3 regex + # Check if nightly builds are available + # Install PyTorch nightly from pip + pip install --pre torch torchtext torchvision torchaudio \ + -f --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install other TorchBench dependencies + run: | + . activate "${CONDA_ENV_NAME}" + conda install -y git-lfs + python install.py --continue_on_fail + - name: Run benchmark + run: | + . activate "${CONDA_ENV_NAME}" + WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}" + bash ./.github/scripts/run.sh "${WORKFLOW_HOME}" + - name: Generate the bisection config + run: | + set -x + . activate "${CONDA_ENV_NAME}" + WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}" + mkdir -p benchmark-output/ + # Update the self-hosted pytorch version + pushd "${HOME}/pytorch" + git fetch origin + popd + pip install gitpython pyyaml dataclasses argparse + # Compare the result from yesterday and report any perf signals + python ./.github/scripts/generate-abtest-config.py \ + --pytorch-dir "${HOME}/pytorch" \ + --github-issue "${WORKFLOW_HOME}/gh-issue.md" \ + --benchmark-dir "${WORKFLOW_HOME}" \ + --out "${WORKFLOW_HOME}/bisection.yaml" + # Include in the GitHub artifact + if [ -f "${WORKFLOW_HOME}/gh-issue.md" ]; then + cp "${WORKFLOW_HOME}/bisection.yaml" ./benchmark-output/ + cp "${WORKFLOW_HOME}/gh-issue.md" ./benchmark-output/ + # Setup the bisection environment + BISECTION_HOME="${HOME}/${{ env.BISECTION_ROOT }}/bisection-gh${GITHUB_RUN_ID}" + mkdir -p "${BISECTION_HOME}" + mv ./benchmark-output/gh-issue.md "${BISECTION_HOME}/gh-issue.md" + cp ./benchmark-output/bisection.yaml "${BISECTION_HOME}/config.yaml" + fi + # - name: Dispatch the bisection workflow + # if: env.TORCHBENCH_PERF_SIGNAL + # run: | + # # Get the workflow ID from + # # https://api.github.com/repos/pytorch/benchmark/actions/workflows + # curl -u xuzhao9:${{ secrets.TORCHBENCH_ACCESS_TOKEN }} \ + # -X POST \ + # -H "Accept: application/vnd.github.v3+json" \ + # https://api.github.com/repos/pytorch/benchmark/actions/workflows/16176850/dispatches \ + # -d '{"ref": "main", "inputs": {"issue_name": "bisection-gh'"${GITHUB_RUN_ID}"'" } }' + # - name: Copy artifact and upload to scribe + # run: | + # . activate "${CONDA_ENV_NAME}" + # TODAY=$(date "+%Y%m%d%H%M%S") + # LATEST_RESULT=$(find ${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID} -name "*.json" | sort -r | head -1) + # echo "Benchmark result file: ${LATEST_RESULT}" + # mkdir -p benchmark-output/ + # cp "${LATEST_RESULT}" ./benchmark-output/benchmark-result-${CONFIG_VER}-${TODAY}.json + # # Load environment variables + # CONFIG_DIR=torchbenchmark/score/configs/${CONFIG_VER} + # CONFIG_ENV=${CONFIG_DIR}/config-${CONFIG_VER}.env + # # Load environment variables + # set -a; source "${CONFIG_ENV}"; set +a + # SCORE_FILE="./benchmark-result-${CONFIG_VER}-score-${TODAY}.json" + # # Generate score file + # python compute_score.py --score_version "${CONFIG_VER}" --benchmark_data_file "${LATEST_RESULT}" > "${SCORE_FILE}" + # # Upload result to Scribe + # python scripts/upload_scribe_${CONFIG_VER}.py --pytest_bench_json "${LATEST_RESULT}" --torchbench_score_file "${SCORE_FILE}" + - name: Upload artifact + uses: actions/upload-artifact@v3 + with: + name: Benchmark result + path: benchmark-output/ + - name: Destroy conda env + run: | + conda env remove --name "${CONDA_ENV_NAME}" + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true From 13e8b9e3ad08a20715a5fa34aec658579356103a Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:10:00 -0700 Subject: [PATCH 02/13] more changes --- .github/workflows/v2-nightly.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index d5e299123971d..3eca63da44de5 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -1,6 +1,7 @@ name: TorchBench V2 nightly on: workflow_dispatch: + pull_request: # to remove schedule: - cron: '0 5 * * *' # run at 5 AM UTC (10 PM PDT) From a1a539158541fd568cf471e2fb9a4ea1ecd5ebbb Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:13:35 -0700 Subject: [PATCH 03/13] updates --- .github/workflows/v2-nightly.yml | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index 3eca63da44de5..bc493178e0a87 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -24,32 +24,33 @@ jobs: repository: razarmehr/benchmark path: benchmark ref: ${{ env.TORCHBENCH_BRANCH }} - - name: Create conda env + - name: Create conda environment and install deps + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: 3.8 + shell: arch -arch arm64 bash {0} run: | - conda create -y -q --name "${CONDA_ENV_NAME}" python=${{ env.PYTHON_VER }} + # shellcheck disable=SC1090 + set -ex - name: Install PyTorch nightly run: | - . activate "${CONDA_ENV_NAME}" # Install dependencies - pip install requests bs4 argparse gitpython boto3 regex + ${CONDA_RUN} pip install requests bs4 argparse gitpython boto3 regex # Check if nightly builds are available # Install PyTorch nightly from pip - pip install --pre torch torchtext torchvision torchaudio \ + ${CONDA_RUN} pip install --pre torch torchtext torchvision torchaudio \ -f --index-url https://download.pytorch.org/whl/nightly/cpu - name: Install other TorchBench dependencies run: | - . activate "${CONDA_ENV_NAME}" - conda install -y git-lfs - python install.py --continue_on_fail + ${CONDA_RUN} conda install -y git-lfs + ${CONDA_RUN} python install.py --continue_on_fail - name: Run benchmark run: | - . activate "${CONDA_ENV_NAME}" WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}" - bash ./.github/scripts/run.sh "${WORKFLOW_HOME}" + ${CONDA_RUN} bash ./.github/scripts/run.sh "${WORKFLOW_HOME}" - name: Generate the bisection config run: | set -x - . activate "${CONDA_ENV_NAME}" WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}" mkdir -p benchmark-output/ # Update the self-hosted pytorch version @@ -58,7 +59,7 @@ jobs: popd pip install gitpython pyyaml dataclasses argparse # Compare the result from yesterday and report any perf signals - python ./.github/scripts/generate-abtest-config.py \ + ${CONDA_RUN} python ./.github/scripts/generate-abtest-config.py \ --pytorch-dir "${HOME}/pytorch" \ --github-issue "${WORKFLOW_HOME}/gh-issue.md" \ --benchmark-dir "${WORKFLOW_HOME}" \ From 75383a885384cc67dfa0f9e30e7f413567fefb81 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:16:13 -0700 Subject: [PATCH 04/13] updates --- .github/workflows/v2-nightly.yml | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index bc493178e0a87..4f738fc95da79 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -18,12 +18,16 @@ jobs: BUILD_ENVIRONMENT: benchmark-nightly runs-on: [macos-nightly] steps: + - name: Clean up disk space before running MacOS workflow + uses: pytorch/test-infra/.github/actions/check-disk-space@main + - name: Checkout uses: actions/checkout@v3 with: repository: razarmehr/benchmark path: benchmark ref: ${{ env.TORCHBENCH_BRANCH }} + - name: Create conda environment and install deps env: ENV_NAME: conda-test-env-${{ github.run_id }} @@ -32,7 +36,12 @@ jobs: run: | # shellcheck disable=SC1090 set -ex + - name: Install PyTorch nightly + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: 3.8 + shell: arch -arch arm64 bash {0} run: | # Install dependencies ${CONDA_RUN} pip install requests bs4 argparse gitpython boto3 regex @@ -40,15 +49,30 @@ jobs: # Install PyTorch nightly from pip ${CONDA_RUN} pip install --pre torch torchtext torchvision torchaudio \ -f --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Install other TorchBench dependencies + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: 3.8 + shell: arch -arch arm64 bash {0} run: | ${CONDA_RUN} conda install -y git-lfs ${CONDA_RUN} python install.py --continue_on_fail + - name: Run benchmark + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: 3.8 + shell: arch -arch arm64 bash {0} run: | WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}" ${CONDA_RUN} bash ./.github/scripts/run.sh "${WORKFLOW_HOME}" + - name: Generate the bisection config + env: + ENV_NAME: conda-test-env-${{ github.run_id }} + PY_VERS: 3.8 + shell: arch -arch arm64 bash {0} run: | set -x WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}" @@ -107,9 +131,11 @@ jobs: with: name: Benchmark result path: benchmark-output/ - - name: Destroy conda env - run: | - conda env remove --name "${CONDA_ENV_NAME}" + + - name: Clean up disk space + if: always() + continue-on-error: true + uses: pytorch/test-infra/.github/actions/check-disk-space@main concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} From 7caf3d800588e14fce9139e54e2f03c6c0cdb05b Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:18:03 -0700 Subject: [PATCH 05/13] updates --- .github/workflows/v2-nightly.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index 4f738fc95da79..559363854ccec 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -36,14 +36,6 @@ jobs: run: | # shellcheck disable=SC1090 set -ex - - - name: Install PyTorch nightly - env: - ENV_NAME: conda-test-env-${{ github.run_id }} - PY_VERS: 3.8 - shell: arch -arch arm64 bash {0} - run: | - # Install dependencies ${CONDA_RUN} pip install requests bs4 argparse gitpython boto3 regex # Check if nightly builds are available # Install PyTorch nightly from pip @@ -56,6 +48,8 @@ jobs: PY_VERS: 3.8 shell: arch -arch arm64 bash {0} run: | + # shellcheck disable=SC1090 + set -ex ${CONDA_RUN} conda install -y git-lfs ${CONDA_RUN} python install.py --continue_on_fail From 1fc7240afa2b49ff07d7cf6d10f3ed5d4250ede1 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:20:07 -0700 Subject: [PATCH 06/13] updates --- .github/workflows/v2-nightly.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index 559363854ccec..e6025660d3beb 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -28,6 +28,12 @@ jobs: path: benchmark ref: ${{ env.TORCHBENCH_BRANCH }} + - name: Setup miniconda + uses: pytorch/test-infra/.github/actions/setup-miniconda@main + with: + python-version: 3.9 + environment-file: .github/requirements/conda-env-${{ runner.os }}-${{ runner.arch }} + - name: Create conda environment and install deps env: ENV_NAME: conda-test-env-${{ github.run_id }} From e866ce805b942591481ed860108a64de1e98dfea Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:23:45 -0700 Subject: [PATCH 07/13] updates --- .github/workflows/v2-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index e6025660d3beb..783888181d158 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -31,7 +31,7 @@ jobs: - name: Setup miniconda uses: pytorch/test-infra/.github/actions/setup-miniconda@main with: - python-version: 3.9 + python-version: 3.8 environment-file: .github/requirements/conda-env-${{ runner.os }}-${{ runner.arch }} - name: Create conda environment and install deps From b4419008f0b473b5aae481d2799a7fa95fe4ac6c Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:36:55 -0700 Subject: [PATCH 08/13] updates --- .github/workflows/v2-nightly.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index 783888181d158..52143800f2c1a 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -45,8 +45,7 @@ jobs: ${CONDA_RUN} pip install requests bs4 argparse gitpython boto3 regex # Check if nightly builds are available # Install PyTorch nightly from pip - ${CONDA_RUN} pip install --pre torch torchtext torchvision torchaudio \ - -f --index-url https://download.pytorch.org/whl/nightly/cpu + ${CONDA_RUN} pip3 install --pre torch torchvision torchtext torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu - name: Install other TorchBench dependencies env: From 2f53967af9f11fd35986bae40be24f149ed98a47 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:42:14 -0700 Subject: [PATCH 09/13] updates --- .github/workflows/v2-nightly.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index 52143800f2c1a..b0be1a911dd03 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -21,11 +21,12 @@ jobs: - name: Clean up disk space before running MacOS workflow uses: pytorch/test-infra/.github/actions/check-disk-space@main - - name: Checkout - uses: actions/checkout@v3 + - name: Checkout TorchBench + uses: malfet/checkout@silent-checkout with: repository: razarmehr/benchmark path: benchmark + lfs: false ref: ${{ env.TORCHBENCH_BRANCH }} - name: Setup miniconda From 63935e0f5a54e306b77ef45758722534dbcfe8ae Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:47:26 -0700 Subject: [PATCH 10/13] updates --- .github/workflows/v2-nightly.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index b0be1a911dd03..9da3fb075e3a9 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -21,14 +21,6 @@ jobs: - name: Clean up disk space before running MacOS workflow uses: pytorch/test-infra/.github/actions/check-disk-space@main - - name: Checkout TorchBench - uses: malfet/checkout@silent-checkout - with: - repository: razarmehr/benchmark - path: benchmark - lfs: false - ref: ${{ env.TORCHBENCH_BRANCH }} - - name: Setup miniconda uses: pytorch/test-infra/.github/actions/setup-miniconda@main with: @@ -48,6 +40,14 @@ jobs: # Install PyTorch nightly from pip ${CONDA_RUN} pip3 install --pre torch torchvision torchtext torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu + - name: Checkout TorchBench + uses: malfet/checkout@silent-checkout + with: + repository: razarmehr/benchmark + path: benchmark + lfs: false + ref: ${{ env.TORCHBENCH_BRANCH }} + - name: Install other TorchBench dependencies env: ENV_NAME: conda-test-env-${{ github.run_id }} From 7da8ff55d9acebfcf53abfe10546b723cd44d7ae Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 24 Mar 2023 10:59:23 -0700 Subject: [PATCH 11/13] updates --- .github/workflows/v2-nightly.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index 9da3fb075e3a9..8fecefcc0906c 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -57,7 +57,9 @@ jobs: # shellcheck disable=SC1090 set -ex ${CONDA_RUN} conda install -y git-lfs + pushd benchmark ${CONDA_RUN} python install.py --continue_on_fail + popd - name: Run benchmark env: @@ -65,9 +67,10 @@ jobs: PY_VERS: 3.8 shell: arch -arch arm64 bash {0} run: | + pushd benchmark WORKFLOW_HOME="${HOME}/${{ env.OUTPUT_DIR }}/gh${GITHUB_RUN_ID}" - ${CONDA_RUN} bash ./.github/scripts/run.sh "${WORKFLOW_HOME}" - + ${CONDA_RUN} bash .github/scripts/run.sh "${WORKFLOW_HOME}" + popd - name: Generate the bisection config env: ENV_NAME: conda-test-env-${{ github.run_id }} From 83a6959aa3cbe51c61ba437a222be8c83b88f2d5 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 19 May 2023 15:58:11 -0700 Subject: [PATCH 12/13] updates --- .github/workflows/v2-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index 8fecefcc0906c..988e0a3a98c32 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -16,7 +16,7 @@ jobs: BISECTION_ROOT: ".torchbench/v2-bisection-ci" IS_GHA: 1 BUILD_ENVIRONMENT: benchmark-nightly - runs-on: [macos-nightly] + runs-on: [macos-torchbench] steps: - name: Clean up disk space before running MacOS workflow uses: pytorch/test-infra/.github/actions/check-disk-space@main From e77f2efd71cd36196ad59f9229325808c977ee00 Mon Sep 17 00:00:00 2001 From: Denis Vieriu Date: Fri, 19 May 2023 16:03:30 -0700 Subject: [PATCH 13/13] updates --- .github/workflows/v2-nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/v2-nightly.yml b/.github/workflows/v2-nightly.yml index 988e0a3a98c32..3fa3f83be56ac 100644 --- a/.github/workflows/v2-nightly.yml +++ b/.github/workflows/v2-nightly.yml @@ -35,7 +35,7 @@ jobs: run: | # shellcheck disable=SC1090 set -ex - ${CONDA_RUN} pip install requests bs4 argparse gitpython boto3 regex + ${CONDA_RUN} pip install requests bs4 argparse gitpython boto3 regex pyyaml # Check if nightly builds are available # Install PyTorch nightly from pip ${CONDA_RUN} pip3 install --pre torch torchvision torchtext torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu