From f7b34fcc0782a8d3aeb212bdbe850bbcc1e8712b Mon Sep 17 00:00:00 2001 From: yolo2themoon Date: Mon, 18 Oct 2021 21:35:33 +0800 Subject: [PATCH 1/5] [ci] Add benchmark to postsubmit workflow --- .github/workflows/postsubmit.yml | 23 +++++++++++++++++++++++ benchmarks/misc/membound.py | 2 +- benchmarks/misc/run.py | 20 ++++++++++++++++---- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/postsubmit.yml b/.github/workflows/postsubmit.yml index e7ad3b5bebc28..84c846e8ba73a 100644 --- a/.github/workflows/postsubmit.yml +++ b/.github/workflows/postsubmit.yml @@ -231,3 +231,26 @@ jobs: TI_LIB_DIR=`python3 -c "import taichi;print(taichi.__path__[0])" | tail -1` TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests ti test -vr2 -t4 -x + + performance_monitoring: + name: Performance monitoring (NVGPU) + needs: check_code_format + timeout-minutes: 60 + runs-on: [self-hosted, x64, cuda, linux, ubuntu20.04, rtx2060, benchmark] + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + + - name: Build + run: | + REPO_DIR=`pwd` + export TAICHI_REPO_DIR=$REPO_DIR + export PYTHONPATH=$TAICHI_REPO_DIR/python:$PYTHONPATH + export PATH=$TAICHI_REPO_DIR/bin:$PATH + git submodule update --init --recursive --depth=1 + TAICHI_CMAKE_ARGS="-DTI_WITH_CUDA_TOOLKIT:BOOL=ON" python3 setup.py develop --user + + - name: Run benchmark + run: | + python3 benchmarks/misc/run.py /home/benchmarkbot/benchmark/ diff --git a/benchmarks/misc/membound.py b/benchmarks/misc/membound.py index 55bfd3c9942e3..c2aedab9d1432 100644 --- a/benchmarks/misc/membound.py +++ b/benchmarks/misc/membound.py @@ -8,7 +8,7 @@ test_cases = [fill, saxpy, reduction] test_archs = [ti.cuda] test_dtype = [ti.i32, ti.i64, ti.f32, ti.f64] -test_dsize = [(4**i) * kibibyte for i in range(1, 11)] #[4KB,16KB...1GB] +test_dsize = [(4**i) * kibibyte for i in range(1, 10)] #[4KB,16KB...256MB] test_repeat = 10 results_evaluation = [geometric_mean] diff --git a/benchmarks/misc/run.py b/benchmarks/misc/run.py index 69fff53d82a5c..4ff0ef0ebff21 100644 --- a/benchmarks/misc/run.py +++ b/benchmarks/misc/run.py @@ -1,4 +1,8 @@ +import datetime +import sys + from membound import Membound +from taichi.core import ti_core as _ti_core import taichi as ti @@ -18,16 +22,24 @@ def run(self): for s in self.suites: s.run() - def write_md(self): - filename = f'performance_result.md' - with open(filename, 'w') as f: + def write_to_path(self, path_with_file_name='./performance_result.md'): + with open(path_with_file_name, 'w') as f: for arch in test_archs: for s in self.suites: lines = s.mdlines(arch) for line in lines: print(line, file=f) + def store_with_date_and_commit_hash(self, path='./'): + current_time = datetime.datetime.now().strftime("%Y%m%dd%Hh%Mm%Ss") + commit_hash = _ti_core.get_commit_hash()[:8] + filename = f'perfresult_{current_time}_{commit_hash}.md' + print('store to: ' + path + filename) + self.write_to_path(path + filename) + +path_to_store = sys.argv[1] if len(sys.argv) > 1 else './' p = PerformanceMonitoring() p.run() -p.write_md() +p.write_to_path() # for /benchamark +p.store_with_date_and_commit_hash(path_to_store) #for post-submit From 17a924f679e07c6e325b41e03a3ecb92ba0330ad Mon Sep 17 00:00:00 2001 From: yolo2themoon Date: Tue, 19 Oct 2021 12:50:10 +0800 Subject: [PATCH 2/5] remove label 'rtx2060' --- .github/workflows/postsubmit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/postsubmit.yml b/.github/workflows/postsubmit.yml index 84c846e8ba73a..ce5e7b34d28e2 100644 --- a/.github/workflows/postsubmit.yml +++ b/.github/workflows/postsubmit.yml @@ -236,7 +236,7 @@ jobs: name: Performance monitoring (NVGPU) needs: check_code_format timeout-minutes: 60 - runs-on: [self-hosted, x64, cuda, linux, ubuntu20.04, rtx2060, benchmark] + runs-on: [self-hosted, x64, cuda, linux, ubuntu20.04, benchmark] steps: - uses: actions/checkout@v2 with: From f2c710e93c53a011da94f9ec5c03c02e6e5b48f8 Mon Sep 17 00:00:00 2001 From: yolo2themoon Date: Thu, 21 Oct 2021 18:15:32 +0800 Subject: [PATCH 3/5] using buildbot's installation process --- .github/workflows/postsubmit.yml | 22 +++++++++++++--------- benchmarks/misc/run.py | 27 +++++++++++++++++---------- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/workflows/postsubmit.yml b/.github/workflows/postsubmit.yml index ce5e7b34d28e2..6fb97e41a0a95 100644 --- a/.github/workflows/postsubmit.yml +++ b/.github/workflows/postsubmit.yml @@ -236,21 +236,25 @@ jobs: name: Performance monitoring (NVGPU) needs: check_code_format timeout-minutes: 60 - runs-on: [self-hosted, x64, cuda, linux, ubuntu20.04, benchmark] + runs-on: [self-hosted, x64, cuda, linux, benchmark] steps: - uses: actions/checkout@v2 with: submodules: 'recursive' - - name: Build + - name: Build & Install run: | - REPO_DIR=`pwd` - export TAICHI_REPO_DIR=$REPO_DIR - export PYTHONPATH=$TAICHI_REPO_DIR/python:$PYTHONPATH - export PATH=$TAICHI_REPO_DIR/bin:$PATH - git submodule update --init --recursive --depth=1 - TAICHI_CMAKE_ARGS="-DTI_WITH_CUDA_TOOLKIT:BOOL=ON" python3 setup.py develop --user - + export PATH=$PATH:/usr/local/cuda/bin + .github/workflows/scripts/unix_build.sh + env: + LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0 + LLVM_PATH: /opt/taichi-llvm-10.0.0/bin + LLVM_DIR: /opt/taichi-llvm-10.0.0/lib/cmake/llvm + CUDA_TOOLKIT_ROOT_DIR: /usr/local/cuda/ + CI_SETUP_CMAKE_ARGS: -DTI_WITH_CUDA_TOOLKIT:BOOL=ON + BUILD_NUM_THREADS: 8 + CXX: clang++-10 + - name: Run benchmark run: | python3 benchmarks/misc/run.py /home/benchmarkbot/benchmark/ diff --git a/benchmarks/misc/run.py b/benchmarks/misc/run.py index 4ff0ef0ebff21..06406c9fd41fa 100644 --- a/benchmarks/misc/run.py +++ b/benchmarks/misc/run.py @@ -1,4 +1,5 @@ import datetime +import os import sys from membound import Membound @@ -22,7 +23,7 @@ def run(self): for s in self.suites: s.run() - def write_to_path(self, path_with_file_name='./performance_result.md'): + def store_to_path(self, path_with_file_name='./performance_result.md'): with open(path_with_file_name, 'w') as f: for arch in test_archs: for s in self.suites: @@ -30,16 +31,22 @@ def write_to_path(self, path_with_file_name='./performance_result.md'): for line in lines: print(line, file=f) - def store_with_date_and_commit_hash(self, path='./'): + def store_with_date_and_commit_id(self, file_dir='./'): current_time = datetime.datetime.now().strftime("%Y%m%dd%Hh%Mm%Ss") commit_hash = _ti_core.get_commit_hash()[:8] - filename = f'perfresult_{current_time}_{commit_hash}.md' - print('store to: ' + path + filename) - self.write_to_path(path + filename) + file_name = f'perfresult_{current_time}_{commit_hash}.md' + path = os.path.join(file_dir, file_name) + print('Storing benchmark result to: ' + path) + self.store_to_path(path) -path_to_store = sys.argv[1] if len(sys.argv) > 1 else './' -p = PerformanceMonitoring() -p.run() -p.write_to_path() # for /benchamark -p.store_with_date_and_commit_hash(path_to_store) #for post-submit +def main(): + file_dir = sys.argv[1] if len(sys.argv) > 1 else './' + p = PerformanceMonitoring() + p.run() + p.store_to_path() # for /benchmark + p.store_with_date_and_commit_id(file_dir) #for postsubmit + + +if __name__ == '__main__': + main() From 3b66a0d1adb5cc514095f5e917b99a75b487e593 Mon Sep 17 00:00:00 2001 From: yolo2themoon Date: Mon, 25 Oct 2021 12:09:26 +0800 Subject: [PATCH 4/5] remove needs --- .github/workflows/postsubmit.yml | 1 - performance_result.md | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 performance_result.md diff --git a/.github/workflows/postsubmit.yml b/.github/workflows/postsubmit.yml index 6fb97e41a0a95..dc5d42d3e821f 100644 --- a/.github/workflows/postsubmit.yml +++ b/.github/workflows/postsubmit.yml @@ -234,7 +234,6 @@ jobs: performance_monitoring: name: Performance monitoring (NVGPU) - needs: check_code_format timeout-minutes: 60 runs-on: [self-hosted, x64, cuda, linux, benchmark] steps: diff --git a/performance_result.md b/performance_result.md new file mode 100644 index 0000000000000..c1854231cf3ba --- /dev/null +++ b/performance_result.md @@ -0,0 +1,20 @@ +|Membound.cuda||||||||||geometric_mean| +|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:| +|**data size**|4.0KB|16.0KB|64.0KB|256.0KB|1.0MB|4.0MB|16.0MB|64.0MB|256.0MB|| +|**repeat**|1000|1000|1000|1000|1000|1000|100|100|100|| +|kernel elapsed time(ms)|||||||||| +|fill.i32|0.004|0.0047|0.0048|0.005|0.0058|0.0162|0.0651|0.3556|1.4612|0.0224| +|fill.i64|0.0035|0.0047|0.0047|0.0049|0.0053|0.0158|0.0576|0.2512|1.08|0.0199| +|fill.f32|0.0035|0.0047|0.0048|0.005|0.0058|0.0163|0.0656|0.3549|1.4161|0.0221| +|fill.f64|0.0034|0.0047|0.0047|0.0049|0.0053|0.0156|0.0575|0.2492|1.036|0.0197| +|kernel elapsed time(ms)|||||||||| +|saxpy.i32|0.0041|0.0047|0.0047|0.005|0.0051|0.0165|0.0597|0.233|0.9935|0.0201| +|saxpy.i64|0.0033|0.0047|0.0047|0.0048|0.0056|0.0152|0.0587|0.2252|0.894|0.0191| +|saxpy.f32|0.0037|0.0047|0.0047|0.005|0.0057|0.0166|0.0599|0.2338|0.9946|0.0201| +|saxpy.f64|0.004|0.0047|0.0047|0.0048|0.0056|0.0154|0.0586|0.2245|0.8875|0.0196| +|kernel elapsed time(ms)|||||||||| +|reduction.i32|0.0063|0.0068|0.0067|0.0072|0.0082|0.0206|0.0638|0.2448|1.023|0.0261| +|reduction.i64|0.0741|0.0979|0.0979|0.0982|0.099|0.1053|0.1268|0.2502|0.8801|0.1397| +|reduction.f32|0.01|0.0126|0.0126|0.0128|0.0129|0.022|0.0653|0.247|0.9988|0.0356| +|reduction.f64|0.2129|0.2862|0.2862|0.2864|0.2864|0.2895|0.2989|0.3636|0.9322|0.3263| + From 1266ce10e764528f70fd19ea0dc271ef945920c3 Mon Sep 17 00:00:00 2001 From: yolo2themoon Date: Mon, 25 Oct 2021 13:07:27 +0800 Subject: [PATCH 5/5] remove benchmark result --- performance_result.md | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100644 performance_result.md diff --git a/performance_result.md b/performance_result.md deleted file mode 100644 index c1854231cf3ba..0000000000000 --- a/performance_result.md +++ /dev/null @@ -1,20 +0,0 @@ -|Membound.cuda||||||||||geometric_mean| -|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:| -|**data size**|4.0KB|16.0KB|64.0KB|256.0KB|1.0MB|4.0MB|16.0MB|64.0MB|256.0MB|| -|**repeat**|1000|1000|1000|1000|1000|1000|100|100|100|| -|kernel elapsed time(ms)|||||||||| -|fill.i32|0.004|0.0047|0.0048|0.005|0.0058|0.0162|0.0651|0.3556|1.4612|0.0224| -|fill.i64|0.0035|0.0047|0.0047|0.0049|0.0053|0.0158|0.0576|0.2512|1.08|0.0199| -|fill.f32|0.0035|0.0047|0.0048|0.005|0.0058|0.0163|0.0656|0.3549|1.4161|0.0221| -|fill.f64|0.0034|0.0047|0.0047|0.0049|0.0053|0.0156|0.0575|0.2492|1.036|0.0197| -|kernel elapsed time(ms)|||||||||| -|saxpy.i32|0.0041|0.0047|0.0047|0.005|0.0051|0.0165|0.0597|0.233|0.9935|0.0201| -|saxpy.i64|0.0033|0.0047|0.0047|0.0048|0.0056|0.0152|0.0587|0.2252|0.894|0.0191| -|saxpy.f32|0.0037|0.0047|0.0047|0.005|0.0057|0.0166|0.0599|0.2338|0.9946|0.0201| -|saxpy.f64|0.004|0.0047|0.0047|0.0048|0.0056|0.0154|0.0586|0.2245|0.8875|0.0196| -|kernel elapsed time(ms)|||||||||| -|reduction.i32|0.0063|0.0068|0.0067|0.0072|0.0082|0.0206|0.0638|0.2448|1.023|0.0261| -|reduction.i64|0.0741|0.0979|0.0979|0.0982|0.099|0.1053|0.1268|0.2502|0.8801|0.1397| -|reduction.f32|0.01|0.0126|0.0126|0.0128|0.0129|0.022|0.0653|0.247|0.9988|0.0356| -|reduction.f64|0.2129|0.2862|0.2862|0.2864|0.2864|0.2895|0.2989|0.3636|0.9322|0.3263| -