Skip to content

Add Tesla M10 stress test support #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
sdcosmo wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Add Tesla M10 stress test support #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 10 additions & 6 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
cmake_minimum_required(VERSION 3.8.0)

set(CUDA_HOME "/usr/local/cuda" CACHE STRING "" FORCE)
set(GPUStressTest_HOME "/home/josephw/GPUStressTest" CACHE STRING "" FORCE)
find_program(NVCC_EXECUTABLE nvcc HINTS /usr/local/cuda/bin /usr/bin REQUIRED)
get_filename_component(CUDA_BIN_DIR "${NVCC_EXECUTABLE}" DIRECTORY)
get_filename_component(CUDA_HOME_DEFAULT "${CUDA_BIN_DIR}" DIRECTORY)

set(CUDA_HOME "${CUDA_HOME_DEFAULT}" CACHE PATH "CUDA toolkit root")
set(GPUStressTest_HOME "${CMAKE_CURRENT_SOURCE_DIR}" CACHE PATH "GPUStressTest source root")
set(LD_LIBRARY_PATH "${CUDA_HOME}/lib64;${LD_LIBRARY_PATH}")
set(CMAKE_VERBOSE_MAKEFILE "ON" CACHE STRING "" FORCE)
set(CMAKE_CUDA_COMPILER "/usr/local/cuda/bin/nvcc" CACHE STRING "" FORCE)
set(CMAKE_CUDA_COMPILER "${NVCC_EXECUTABLE}" CACHE FILEPATH "CUDA compiler" FORCE)


project(gst LANGUAGES CXX CUDA C)
Expand All @@ -25,13 +29,13 @@ endif()
set(FLAGS "")
set(CMAKE_CUDA_FLAGS "" CACHE STRING "" FORCE)
# Note that cmake will insert semicolons between these item automatically...
set(SM_LIST_IN "70,75,80,90" CACHE STRING "Please input compute capability")
set(SM_LIST_IN "52" CACHE STRING "Please input compute capability")
MARK_AS_ADVANCED(SM_LIST)
string(REPLACE "\," "\ " DISPLAY_SMS_LIST ${SM_LIST_IN})
string(REPLACE "\," "\;" SM_LIST ${SM_LIST_IN})

message("SMS_LIST:" ${SMS_LIST})
foreach(SM ${SMS_LIST})
message("SM_LIST:" ${SM_LIST})
foreach(SM ${SM_LIST})
string(APPEND CMAKE_CUDA_FLAGS " -gencode arch=compute_${SM},code=sm_${SM}")
endforeach(SM)
message("-- GPU SM = " ${DISPLAY_SMS_LIST})
Expand Down
39 changes: 39 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,44 @@
# GST - GPU Stress Test

This fork adds a validated NVIDIA Tesla M10 profile to NVIDIA's GPUStressTest.
It targets Maxwell compute capability 5.2 and keeps the upstream MIT license.

## Tesla M10 Support

The M10 profile runs five supported workloads: FP32, complex FP32, FP64,
complex FP64, and an alternate FP32 matrix layout. The two principal FP32
tests allocate approximately 6.75 GiB on each 8 GiB GPU.

The Maxwell path uses zero-filled matrices because the upstream matrix-fill
kernel does not provide a runnable Maxwell image. This still executes the full
cuBLASLt GEMM workload. The fork also fixes watchdog completion and indexing
bugs that could report successful multi-GPU runs as hung.

Validated configuration:

- Four NVIDIA Tesla M10 GPUs
- Ubuntu 24.04
- NVIDIA driver 535.309.01
- CUDA Toolkit 12.0
- GCC/G++ 12

Build and run on Ubuntu:

```bash
sudo apt install build-essential cmake gcc-12 g++-12 nvidia-cuda-toolkit
git clone https://github.com/sdcosmo/GPUStressTest-M10.git
cd GPUStressTest-M10
cmake -S . -B build -DCMAKE_C_COMPILER=gcc-12 -DCMAKE_CXX_COMPILER=g++-12
cmake --build build -j"$(nproc)"
./build/gst -T=1
```

`-T=1` performs one full validation loop. Increase the value for a sustained
stress run. Use `CUDA_VISIBLE_DEVICES=<index>` to test one physical GPU.

This is an unofficial compatibility fork and is not an NVIDIA-supported
release.

# Build GST

## Prerequisites
Expand Down
38 changes: 23 additions & 15 deletions main.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,8 @@ void* watchdog(void* in)
pthread_exit(NULL);
}
sem_post(&go);
i++;
} while ((tests_done != true) || (test_hung != true));
i = (i + 1) % NUM_TESTS;
} while ((tests_done != true) && (test_hung != true));

printf("WATCHDOG thread exiting....\n");
pthread_exit(NULL);
Expand Down Expand Up @@ -1120,6 +1120,12 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: %
memgb = 8;
break;
}
if (gpu_name.find("M10", 0) != string::npos) {
cout << "Initializing M10 based test suite" << endl;
gst = GST(GST::M10);
memgb = 8;
break;
}
if (gpu_name.find("P40", 0) != string::npos) {
cout << "Initilizing P40 based test suite" << endl;
gst = GST(GST::P40);
Expand Down Expand Up @@ -1201,6 +1207,11 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: %
// cout << "DEBUG:" << "set opts" << endl;

reset_blas_opts(command_line, blas_opts);
if (devprops[dev].major == 5) {
// The upstream fill kernel has no runnable Maxwell image.
// Zero-filled operands still exercise the full GEMM workload.
blas_opts.filling_zero = true;
}
gst.dump_test_args(t_num);

if (gst.stress_tests[t_num].P_arg.compare(0, 3, "sss") == 0 ) {
Expand Down Expand Up @@ -1329,34 +1340,34 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: %
test_cublasLt(blas_opts);
printf("***** TEST %s On Device %d %s\n", gst.stress_tests[t_num].test_name, dev, devprops[dev].name);

if (!test_ran)
bool test_failed = has_error == true || test_hung == true;
if (!test_ran) {
printf("***** TEST DID NOT EXECUTE *****\n\n");
else {
if (has_error == true || test_hung == true) {
printf("***** TEST FAILED ****\n\n");
ret = -1;
break;
}
else
} else if (test_failed) {
printf("***** TEST FAILED ****\n\n");
ret = -1;
} else {
#ifndef DEBUG_MATRIX_SIZES
printf("***** TEST PASSED ****\n");
#endif
continue;
}
tstate[t_num].end_time = time(NULL);
tstate[t_num].test_state = 0;
printf("TEST TIME: %d seconds\n",(int)(tstate[t_num].end_time - tstate[t_num].start_time));
cudaDeviceSynchronize();
cudaDeviceReset();

if (t_num == NUM_TESTS)
if (dev == deviceCount - 1 && t_num == NUM_TESTS - 1)
tests_done = true;

/* Signal watchdog test finished*/
sem_post(&done);

/* wait for watchdog to signal next next test */
sem_wait(&go);

if (test_failed)
break;
}
}
#ifdef DEBUG_MATRIX_SIZES
Expand All @@ -1369,6 +1380,3 @@ printf("DEBUG_MATRIX_SIZES: Checking matrix size only (no CUDA execution) for: %






43 changes: 33 additions & 10 deletions util/GST.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ class GST {

struct stress_test_args stress_tests[NUM_TESTS];

enum test_suite {T4, A100_40, A100_80, K80, M60, P40, P100, H100, H200, V100_16, V100_32, Generic};
enum test_suite {T4, A100_40, A100_80, K80, M10, M60, P40, P100, H100, H200, V100_16, V100_32, Generic};

GST(const test_suite gpu) {
switch (gpu) {
Expand All @@ -125,9 +125,12 @@ class GST {
case H200:
init_h200();
break;
case K80:
init_k80();
break;
case K80:
init_k80();
break;
case M10:
init_m10();
break;
case P40:
init_p40();
break;
Expand Down Expand Up @@ -231,7 +234,7 @@ class GST {
}

void init_v100_16() {
stress_tests[0].test_name = "FP16";
stress_tests[0].test_name = "FP16";
stress_tests[0].test_state = 0;
stress_tests[0].P_arg = "hsh";
stress_tests[0].m_arg = 39864;
Expand Down Expand Up @@ -602,13 +605,33 @@ class GST {
stress_tests[4].B_arg = 0;
}

void init_generic() {
void init_m10() {
const char *names[NUM_TESTS] = {"FP32", "C32", "FP64", "C64", "FP32-NT"};
const char *types[NUM_TESTS] = {"sss", "ccc", "ddd", "zzz", "sss"};
const int matrix_m[NUM_TESTS] = {16384, 8192, 4096, 3072, 16384};
const int matrix_n[NUM_TESTS] = {16384, 8192, 4096, 3072, 16384};
const int matrix_k[NUM_TESTS] = {47104, 45056, 8192, 6144, 47104};

for (int i = 0; i < NUM_TESTS; ++i) {
stress_tests[i].test_name = names[i];
stress_tests[i].test_state = 0;
stress_tests[i].P_arg = types[i];
stress_tests[i].m_arg = matrix_m[i];
stress_tests[i].n_arg = matrix_n[i];
stress_tests[i].k_arg = matrix_k[i];
stress_tests[i].ta_arg = 0;
stress_tests[i].tb_arg = i == 4 ? 0 : 1;
stress_tests[i].B_arg = 0;
}
}

void init_generic() {
stress_tests[0].test_name = "FP16";
stress_tests[0].test_state = 0;
stress_tests[0].P_arg = "hsh";
stress_tests[0].m_arg = 18432;
stress_tests[0].n_arg = 18824;
stress_tests[0].k_arg = 100152;
stress_tests[0].P_arg = "hsh";
stress_tests[0].m_arg = 18432;
stress_tests[0].n_arg = 18824;
stress_tests[0].k_arg = 100152;
stress_tests[0].ta_arg = 0;
stress_tests[0].tb_arg = 1;
stress_tests[0].B_arg = 0;
Expand Down