From 033a00cc3d01f33e07f1b8760312a41f818b60d9 Mon Sep 17 00:00:00 2001 From: sdcosmo <43455470+sdcosmo@users.noreply.github.com> Date: Mon, 20 Jul 2026 19:25:32 -0400 Subject: [PATCH] Add Tesla M10 stress test support --- CMakeLists.txt | 16 ++++++++++------ README.md | 39 +++++++++++++++++++++++++++++++++++++++ main.cu | 38 +++++++++++++++++++++++--------------- util/GST.h | 43 +++++++++++++++++++++++++++++++++---------- 4 files changed, 105 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index dc70fcc..4f8f1fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,10 +1,14 @@ cmake_minimum_required(VERSION 3.8.0) -set(CUDA_HOME "/usr/local/cuda" CACHE STRING "" FORCE) -set(GPUStressTest_HOME "/home/josephw/GPUStressTest" CACHE STRING "" FORCE) +find_program(NVCC_EXECUTABLE nvcc HINTS /usr/local/cuda/bin /usr/bin REQUIRED) +get_filename_component(CUDA_BIN_DIR "${NVCC_EXECUTABLE}" DIRECTORY) +get_filename_component(CUDA_HOME_DEFAULT "${CUDA_BIN_DIR}" DIRECTORY) + +set(CUDA_HOME "${CUDA_HOME_DEFAULT}" CACHE PATH "CUDA toolkit root") +set(GPUStressTest_HOME "${CMAKE_CURRENT_SOURCE_DIR}" CACHE PATH "GPUStressTest source root") set(LD_LIBRARY_PATH "${CUDA_HOME}/lib64;${LD_LIBRARY_PATH}") set(CMAKE_VERBOSE_MAKEFILE "ON" CACHE STRING "" FORCE) -set(CMAKE_CUDA_COMPILER "/usr/local/cuda/bin/nvcc" CACHE STRING "" FORCE) +set(CMAKE_CUDA_COMPILER "${NVCC_EXECUTABLE}" CACHE FILEPATH "CUDA compiler" FORCE) project(gst LANGUAGES CXX CUDA C) @@ -25,13 +29,13 @@ endif() set(FLAGS "") set(CMAKE_CUDA_FLAGS "" CACHE STRING "" FORCE) # Note that cmake will insert semicolons between these item automatically... -set(SM_LIST_IN "70,75,80,90" CACHE STRING "Please input compute capability") +set(SM_LIST_IN "52" CACHE STRING "Please input compute capability") MARK_AS_ADVANCED(SM_LIST) string(REPLACE "\," "\ " DISPLAY_SMS_LIST ${SM_LIST_IN}) string(REPLACE "\," "\;" SM_LIST ${SM_LIST_IN}) -message("SMS_LIST:" ${SMS_LIST}) -foreach(SM ${SMS_LIST}) +message("SM_LIST:" ${SM_LIST}) +foreach(SM ${SM_LIST}) string(APPEND CMAKE_CUDA_FLAGS " -gencode arch=compute_${SM},code=sm_${SM}") endforeach(SM) message("-- GPU SM = " ${DISPLAY_SMS_LIST}) diff --git a/README.md b/README.md index 0baebbb..77e797d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,44 @@ # GST - GPU Stress Test +This fork adds a validated NVIDIA Tesla M10 profile to NVIDIA's GPUStressTest. +It targets Maxwell compute capability 5.2 and keeps the upstream MIT license. + +## Tesla M10 Support + +The M10 profile runs five supported workloads: FP32, complex FP32, FP64, +complex FP64, and an alternate FP32 matrix layout. The two principal FP32 +tests allocate approximately 6.75 GiB on each 8 GiB GPU. + +The Maxwell path uses zero-filled matrices because the upstream matrix-fill +kernel does not provide a runnable Maxwell image. This still executes the full +cuBLASLt GEMM workload. The fork also fixes watchdog completion and indexing +bugs that could report successful multi-GPU runs as hung. + +Validated configuration: + +- Four NVIDIA Tesla M10 GPUs +- Ubuntu 24.04 +- NVIDIA driver 535.309.01 +- CUDA Toolkit 12.0 +- GCC/G++ 12 + +Build and run on Ubuntu: + +```bash +sudo apt install build-essential cmake gcc-12 g++-12 nvidia-cuda-toolkit +git clone https://github.com/sdcosmo/GPUStressTest-M10.git +cd GPUStressTest-M10 +cmake -S . -B build -DCMAKE_C_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12 +cmake --build build -j"$(nproc)" +./build/gst -T=1 +``` + +`-T=1` performs one full validation loop. Increase the value for a sustained +stress run. Use `CUDA_VISIBLE_DEVICES=` to test one physical GPU. + +This is an unofficial compatibility fork and is not an NVIDIA-supported +release. + # Build GST ## Prerequisites diff --git a/main.cu b/main.cu index 5b94d32..769566d 100644 --- a/main.cu +++ b/main.cu @@ -155,8 +155,8 @@ void* watchdog(void* in) pthread_exit(NULL); } sem_post(&go); - i++; - } while ((tests_done != true) || (test_hung != true)); + i = (i + 1) % NUM_TESTS; + } while ((tests_done != true) && (test_hung != true)); printf("WATCHDOG thread exiting....\n"); pthread_exit(NULL); @@ -1120,6 +1120,12 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: % memgb = 8; break; } + if (gpu_name.find("M10", 0) != string::npos) { + cout << "Initializing M10 based test suite" << endl; + gst = GST(GST::M10); + memgb = 8; + break; + } if (gpu_name.find("P40", 0) != string::npos) { cout << "Initilizing P40 based test suite" << endl; gst = GST(GST::P40); @@ -1201,6 +1207,11 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: % // cout << "DEBUG:" << "set opts" << endl; reset_blas_opts(command_line, blas_opts); + if (devprops[dev].major == 5) { + // The upstream fill kernel has no runnable Maxwell image. + // Zero-filled operands still exercise the full GEMM workload. + blas_opts.filling_zero = true; + } gst.dump_test_args(t_num); if (gst.stress_tests[t_num].P_arg.compare(0, 3, "sss") == 0 ) { @@ -1329,19 +1340,16 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: % test_cublasLt(blas_opts); printf("***** TEST %s On Device %d %s\n", gst.stress_tests[t_num].test_name, dev, devprops[dev].name); - if (!test_ran) + bool test_failed = has_error == true || test_hung == true; + if (!test_ran) { printf("***** TEST DID NOT EXECUTE *****\n\n"); - else { - if (has_error == true || test_hung == true) { - printf("***** TEST FAILED ****\n\n"); - ret = -1; - break; - } - else + } else if (test_failed) { + printf("***** TEST FAILED ****\n\n"); + ret = -1; + } else { #ifndef DEBUG_MATRIX_SIZES printf("***** TEST PASSED ****\n"); #endif - continue; } tstate[t_num].end_time = time(NULL); tstate[t_num].test_state = 0; @@ -1349,7 +1357,7 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: % cudaDeviceSynchronize(); cudaDeviceReset(); - if (t_num == NUM_TESTS) + if (dev == deviceCount - 1 && t_num == NUM_TESTS - 1) tests_done = true; /* Signal watchdog test finished*/ @@ -1357,6 +1365,9 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: % /* wait for watchdog to signal next next test */ sem_wait(&go); + + if (test_failed) + break; } } #ifdef DEBUG_MATRIX_SIZES @@ -1369,6 +1380,3 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: % - - - diff --git a/util/GST.h b/util/GST.h index dd57dca..c9a1066 100644 --- a/util/GST.h +++ b/util/GST.h @@ -106,7 +106,7 @@ class GST { struct stress_test_args stress_tests[NUM_TESTS]; - enum test_suite {T4, A100_40, A100_80, K80, M60, P40, P100, H100, H200, V100_16, V100_32, Generic}; + enum test_suite {T4, A100_40, A100_80, K80, M10, M60, P40, P100, H100, H200, V100_16, V100_32, Generic}; GST(const test_suite gpu) { switch (gpu) { @@ -125,9 +125,12 @@ class GST { case H200: init_h200(); break; - case K80: - init_k80(); - break; + case K80: + init_k80(); + break; + case M10: + init_m10(); + break; case P40: init_p40(); break; @@ -231,7 +234,7 @@ class GST { } void init_v100_16() { - stress_tests[0].test_name = "FP16"; + stress_tests[0].test_name = "FP16"; stress_tests[0].test_state = 0; stress_tests[0].P_arg = "hsh"; stress_tests[0].m_arg = 39864; @@ -602,13 +605,33 @@ class GST { stress_tests[4].B_arg = 0; } - void init_generic() { + void init_m10() { + const char *names[NUM_TESTS] = {"FP32", "C32", "FP64", "C64", "FP32-NT"}; + const char *types[NUM_TESTS] = {"sss", "ccc", "ddd", "zzz", "sss"}; + const int matrix_m[NUM_TESTS] = {16384, 8192, 4096, 3072, 16384}; + const int matrix_n[NUM_TESTS] = {16384, 8192, 4096, 3072, 16384}; + const int matrix_k[NUM_TESTS] = {47104, 45056, 8192, 6144, 47104}; + + for (int i = 0; i < NUM_TESTS; ++i) { + stress_tests[i].test_name = names[i]; + stress_tests[i].test_state = 0; + stress_tests[i].P_arg = types[i]; + stress_tests[i].m_arg = matrix_m[i]; + stress_tests[i].n_arg = matrix_n[i]; + stress_tests[i].k_arg = matrix_k[i]; + stress_tests[i].ta_arg = 0; + stress_tests[i].tb_arg = i == 4 ? 0 : 1; + stress_tests[i].B_arg = 0; + } + } + + void init_generic() { stress_tests[0].test_name = "FP16"; stress_tests[0].test_state = 0; - stress_tests[0].P_arg = "hsh"; - stress_tests[0].m_arg = 18432; - stress_tests[0].n_arg = 18824; - stress_tests[0].k_arg = 100152; + stress_tests[0].P_arg = "hsh"; + stress_tests[0].m_arg = 18432; + stress_tests[0].n_arg = 18824; + stress_tests[0].k_arg = 100152; stress_tests[0].ta_arg = 0; stress_tests[0].tb_arg = 1; stress_tests[0].B_arg = 0;