diff --git a/pyproject.toml b/pyproject.toml index 0508875..67b82ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,7 +80,7 @@ rocm = [ "torch==2.12.0+rocm7.1", "torchaudio==2.11.0+rocm7.1", "torchvision==0.27.0+rocm7.1", - "mpi4py==4.1.1", + "mpi4py==4.1.1+mpich.9.1.0", ] rocmwci = [ "torch==2.10.0+rocm710", diff --git a/scripts/install-tuolumne-torchpypi.sh b/scripts/install-tuolumne-torchpypi.sh index 0b7c597..71de5df 100644 --- a/scripts/install-tuolumne-torchpypi.sh +++ b/scripts/install-tuolumne-torchpypi.sh @@ -1,5 +1,5 @@ -ml load python/3.13.2 && python3 -m venv .venvs/scaffoldvenv-tuo-pypi && source .venvs/scaffoldvenv-tuo-pypi/bin/activate && pip install --upgrade pip +ml load python/3.11.5 && python3 -m venv .venvs/scaffoldvenv-tuo-pypi && source .venvs/scaffoldvenv-tuo-pypi/bin/activate && pip install --upgrade pip ml cce/21.0.1 cray-mpich/9.1.0 rocm/7.1.1 rccl/fast-env-slows-mpi pip install -e .[rocm] --find-links https://download.pytorch.org/whl/torch/ --find-links https://download.pytorch.org/whl/torchaudio/ --find-links https://download.pytorch.org/whl/torchvision/ --find-links https://download.pytorch.org/whl/triton-rocm/ 2>&1 | tee install.log # libmpi.so.12 does not exist => ls /opt/cray/pe/lib64/ | grep libmpi -patchelf --replace-needed libmpi.so.12 libmpi_gnu.so.12 .venvs/scaffoldvenv-tuo-pypi/lib/python3.13/site-packages/mpi4py/MPI.mpich.cpython-313-x86_64-linux-gnu.so +patchelf --replace-needed libmpi.so.12 libmpi_gnu.so.12 .venvs/scaffoldvenv-tuo-pypi/lib/python3.11/site-packages/mpi4py/MPI.cpython-311-x86_64-linux-gnu.so diff --git a/scripts/install-tuolumne.sh b/scripts/install-tuolumne.sh deleted file mode 100644 index 4b026f8..0000000 --- a/scripts/install-tuolumne.sh +++ /dev/null @@ -1,26 +0,0 @@ -ml load python/3.11.5 && python3 -m venv .venvs/scaffoldvenv-tuo && source .venvs/scaffoldvenv-tuo/bin/activate && pip install --upgrade pip -ml cce/21.0.0 cray-mpich/9.1.0 rocm/7.1.1 rccl/fast-env-slows-mpi -pip install -e .[rocmwci] 2>&1 | tee install.log -# Needed until new wheel exists for torch using mpich 9.1.0 -TORCH_LIB_DIR=".venvs/scaffoldvenv-tuo/lib/python3.11/site-packages/torch/lib" -OLD="libmpi_gnu_112.so.12" -NEW="libmpi_gnu.so.12" -cd "$TORCH_LIB_DIR" || exit 1 -# Patch every file that has OLD in its DT_NEEDED -for f in *.so*; do - [ -f "$f" ] || continue - - if patchelf --print-needed "$f" 2>/dev/null | grep -Fxq "$OLD"; then - echo "Patching $f" - patchelf --replace-needed "$OLD" "$NEW" "$f" - fi -done -echo -echo "Verification (should show no $OLD):" -for f in *.so*; do - [ -f "$f" ] || continue - if patchelf --print-needed "$f" 2>/dev/null | grep -Fxq "$OLD"; then - echo "STILL NEEDS $OLD -> $f" - fi -done -cd - diff --git a/scripts/scaffold-tuolumne.job b/scripts/scaffold-tuolumne.job deleted file mode 100644 index 1ae88b0..0000000 --- a/scripts/scaffold-tuolumne.job +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# flux: --exclusive -# flux: -N 1 -# flux: -g=1 -# flux: -t 60m -# flux: -qpdebug -# flux: -B flask - -ml cce/21.0.0 cray-mpich/9.1.0 rocm/7.1.1 rccl/fast-env-slows-mpi - -. .venvs/scaffoldvenv-tuo/bin/activate - -# (1) Avoid libmagma error -# (2) Removing libmpi may cause segfault on mpi4py import -# (3-5) undefined symbol: cblas_gemm_f16f16f32 -export LD_PRELOAD="/opt/rocm-7.1.1/llvm/lib/libomp.so /opt/cray/pe/mpich/9.1.0/ofi/gnu/11.2/lib/libmpi_gnu.so.12 /opt/intel/oneapi/mkl/2024.2/lib/libmkl_core.so.2 /opt/intel/oneapi/mkl/2024.2/lib/libmkl_gnu_thread.so.2 /opt/intel/oneapi/mkl/2024.2/lib/libmkl_intel_lp64.so.2" - -# Disable direct convolution benchmarking (should speedup warmup by a significant amount, does the below three options together) -# export MIOPEN_DEBUG_CONV_DIRECT=0 -# Disable direct naive convolution benchmarking (naive_conv_ab_nonpacked_fwd_ndhwc_half_double_half.kd) -export MIOPEN_DEBUG_CONV_DIRECT_NAIVE_CONV_FWD=0 -# Disable naive_conv_ab_nonpacked_bwd_ndhwc_half_double_half.kd -export MIOPEN_DEBUG_CONV_DIRECT_NAIVE_CONV_BWD=0 -# Disable naive_conv_ab_nonpacked_wrw_ndhwc_half_double_half.kd -export MIOPEN_DEBUG_CONV_DIRECT_NAIVE_CONV_WRW=0 - -CONFIG_PATH="$(pwd)/ScaFFold/configs/benchmark_default.yml" -FRACT_BASE_DIR="${FRACT_BASE_DIR:-$(pwd)/ScaFFold/fractals}" - -torchrun-hpc -N 1 -n 1 $(which scaffold) generate_fractals \ - -c "$CONFIG_PATH" \ - --fract-base-dir "$FRACT_BASE_DIR" - -# Uncomment if you want torch profiling -#export PROFILE_TORCH=ON - -torchrun-hpc -N 1 -n 4 --gpus-per-proc 1 $(which scaffold) benchmark \ - -c "$CONFIG_PATH" \ - --fract-base-dir "$FRACT_BASE_DIR" -# torchrun-hpc -N 2 -n 4 --gpus-per-proc 1 $(which scaffold) benchmark -c "$CONFIG_PATH" --fract-base-dir "$FRACT_BASE_DIR"