diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..1d6a8a7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,31 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-ast + - id: check-case-conflict + - id: check-json + - id: check-merge-conflict + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: fix-encoding-pragma + - id: mixed-line-ending + - id: requirements-txt-fixer + - id: trailing-whitespace +- repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: python-check-blanket-noqa + - id: python-check-blanket-type-ignore + - id: python-use-type-annotations + - id: rst-backticks +- repo: https://github.com/psf/black + rev: 24.2.0 + hooks: + - id: black +- repo: https://github.com/PyCQA/autoflake + rev: v2.2.1 + hooks: + - id: autoflake + args: [--in-place, --remove-all-unused-imports, --ignore-init-module-imports] diff --git a/daint/.bashrc b/daint/.bashrc index 07e81fb..873f3d6 100644 --- a/daint/.bashrc +++ b/daint/.bashrc @@ -121,6 +121,7 @@ export LC_ALL=C.UTF-8 export LANG=C.UTF-8 # project + project_id=$(sacct --format=Account --noheader | head -n 1 | awk '{$1=$1}1') export PROJECT=/project/"$project_id"/"$USER" @@ -140,6 +141,7 @@ export SQUEUE_FORMAT="%.9i %.50j %.15u %.5q %.15T %.10M %.10l %.5D %.13f %R" # hdf5 and netcdf export HDF5_ROOT=/users/"$USER"/hdf5/1.14.2/build/gnu export NETCDF_ROOT=/users/"$USER"/netcdf-c/4.9.2/build/gnu +export PATH="$HOME"/autoconf/2.72/build/gnu/bin/:"$HOME"/help2man/1.49.3/build/gnu/bin/:$PATH # get node id of a salloc function get_node_id() { diff --git a/daint/generate_build_autoconf.py b/daint/generate_build_autoconf.py index f5d7584..0384c16 100755 --- a/daint/generate_build_autoconf.py +++ b/daint/generate_build_autoconf.py @@ -35,6 +35,7 @@ def core(env: defs.ProgrammingEnvironment, partition: defs.Partition, root_dir: utils.run("CC=cc", "CXX=CC", "./configure", f"--prefix={build_dir}") utils.run("make -j 8 install") utils.export_variable("AUTOCONF_ROOT", build_dir) + utils.append_to_path("PATH", f"{build_dir}/bin") if __name__ == "__main__": diff --git a/daint/generate_build_help2man.py b/daint/generate_build_help2man.py new file mode 100755 index 0000000..86f34e7 --- /dev/null +++ b/daint/generate_build_help2man.py @@ -0,0 +1,45 @@ +#!/opt/python/3.9.4.1/bin/python +# -*- coding: utf-8 -*- +from __future__ import annotations +import argparse +import os + +import defs +import utils + + +# >>> config: start +ENV: defs.ProgrammingEnvironment = "gnu" +PARTITION: defs.Partition = "gpu" +ROOT_DIR: str = f"/users/{os.getlogin()}" +VERSION: str = "1.49.3" +# >>> config: endi + + +def core(env: defs.ProgrammingEnvironment, partition: defs.Partition, root_dir: str, version: str): + with utils.batch_file(prefix="build_help2man"): + utils.module_purge(force=True) + utils.load_partition(partition) + utils.load_env(env) + root_dir = os.path.abspath(root_dir) + with utils.chdir(root_dir): + utils.run("mkdir -p help2man") + branch = f"master" + utils.run( + f"git clone --branch={branch} " + f"https://github.com/Distrotech/help2man.git help2man/{version}" + ) + with utils.chdir(f"help2man/{version}"): + build_dir = os.path.join(root_dir, f"help2man/{version}/build/{env}") + utils.run("CC=cc", "CXX=CC", "./configure", f"--prefix={build_dir}") + utils.run("make -j 8 install") + utils.append_to_path("PATH", f"{build_dir}/bin") + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--env", type=str, default=ENV) + parser.add_argument("--partition", type=str, default=PARTITION) + parser.add_argument("--root-dir", type=str, default=ROOT_DIR) + parser.add_argument("--version", type=str, default=VERSION) + args = parser.parse_args() + core(**args.__dict__) diff --git a/daint/generate_prepare_mpi.py b/daint/generate_prepare_mpi.py old mode 100644 new mode 100755 diff --git a/daint/generate_prepare_pmapl.py b/daint/generate_prepare_pmapl.py old mode 100644 new mode 100755 index 876af0d..5ec1728 --- a/daint/generate_prepare_pmapl.py +++ b/daint/generate_prepare_pmapl.py @@ -10,7 +10,7 @@ # >>> config: start -BRANCH: str = "cloudsc-cy49r1" +BRANCH: str = "main" ENV: defs.ProgrammingEnvironment = "gnu" PARTITION: defs.Partition = "gpu" # >>> config: end diff --git a/daint/salloc.py b/daint/salloc.py index af8d8c0..9552ab7 100755 --- a/daint/salloc.py +++ b/daint/salloc.py @@ -7,6 +7,7 @@ # >>> config: start + project_id="$(sacct --format=Account --noheader | head -n 1 | awk '{$1=$1}1')" ACCOUNT: str = project_id NUM_NODES: int = 1 diff --git a/daint/sbatch.py b/daint/sbatch.py old mode 100644 new mode 100755 diff --git a/daint/sbatch_pmapl.py b/daint/sbatch_pmapl.py old mode 100644 new mode 100755 index 7b643e0..8e92787 --- a/daint/sbatch_pmapl.py +++ b/daint/sbatch_pmapl.py @@ -12,7 +12,7 @@ # >>> config: start project_id="$(sacct --format=Account --noheader | head -n 1 | awk '{$1=$1}1')" account: str = project_id -branch_l: list[str] = ["cloudsc-cy49r1"] +branch_l: list[str] = ["main"] partition: defs.Partition = "gpu" env_l: list[defs.ProgrammingEnvironment] = ["gnu"] ghex_aggregate_fields: bool = False diff --git a/daint/utils.py b/daint/utils.py index a3fd50b..bd17863 100644 --- a/daint/utils.py +++ b/daint/utils.py @@ -103,6 +103,8 @@ def load_env(env: str) -> None: def export_variable(name: str, value: typing.Any) -> None: run(f"export {name}={str(value)}") +def append_to_path( name: str, value: typing.Any) -> None: + run(f"{name}={str(value)}:${name}") def setup_cuda(): run("NVCC_PATH=$(which nvcc)") diff --git a/hpc2020/.bashrc b/hpc2020/.bashrc index d554cd1..f061062 100644 --- a/hpc2020/.bashrc +++ b/hpc2020/.bashrc @@ -95,9 +95,16 @@ alias l='ls -CF' export LC_ALL=C.UTF-8 export LANG=C.UTF-8 +# nvim +alias vim='/home/"$USER"/neovim/0.9.5/install/bin/nvim' +export VIMRUNTIME=/home/"$USER"/neovim/0.9.5/install/share/nvim/runtime + # slurm shortcuts and settings alias sb='sbatch' alias sc='scancel' alias sq='squeue -u $USER' alias sr='srun' -export SQUEUE_FORMAT="%.9i %.50j %.15u %.15q %.15T %.10M %.10l %.5D %.13f %R" +export SQUEUE_FORMAT="%.9i %.60j %.10u %.10q %.15T %.10M %.10l %.5D %.13f %R" + +# update path with manually built software +export PATH="$HPCPERM"/autoconf/2.72/build/gnu/bin/:"$HPCPERM"/help2man/build/gnu/bin/:$PATH diff --git a/hpc2020/defs.py b/hpc2020/defs.py new file mode 100644 index 0000000..d993e4a --- /dev/null +++ b/hpc2020/defs.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +import os +import typing + +FloatingPointPrecision = typing.Literal["double", "single"] +MPI = typing.Literal["hpcx", "intelmpi", "openmpi"] +Partition = typing.Literal["gpu", "par"] +ProgrammingEnvironment = typing.Literal["gnu", "intel"] + +valid_mpi_libraries = typing.get_args(MPI) +valid_partitions = typing.get_args(Partition) +valid_programming_environments = typing.get_args(ProgrammingEnvironment) + +root_dir = os.environ.get("HPCPERM", f"/home/{os.getlogin()}") diff --git a/hpc2020/generate_build_autoconf.py b/hpc2020/generate_build_autoconf.py new file mode 100644 index 0000000..f3ebca6 --- /dev/null +++ b/hpc2020/generate_build_autoconf.py @@ -0,0 +1,72 @@ +#!/usr/local/apps/python3/3.11.8-01/bin/python3 +# -*- coding: utf-8 -*- +from __future__ import annotations +import argparse +import os + +import defs +import generate_build_help2man +import utils + + +# >>> config: start +ENV: defs.ProgrammingEnvironment = "gnu" +COMPILER_VERSION: str = "11.2.0" +ROOT_DIR: str = defs.root_dir +VERSION: str = "2.72" +# >>> config: end + + +def _setup(build_dir: str) -> None: + utils.export_variable("AUTOCONF_ROOT", build_dir) + utils.update_path(f"{build_dir}/bin") + + +def core( + env: defs.ProgrammingEnvironment, + compiler_version: str, + root_dir: str, + version: str, + _build: bool = True, +) -> str: + with utils.batch_file(prefix="build_autoconf"): + utils.module_purge(force=True) + utils.load_env(env) + env_id = utils.load_compiler(env, compiler_version) + root_dir = os.path.abspath(root_dir) + build_dir = os.path.join(root_dir, f"autoconf/{version}/build/{env_id}") + + generate_build_help2man.setup(env, compiler_version, root_dir) + + if _build: + with utils.chdir(root_dir): + utils.run("mkdir -p autoconf") + utils.run( + f"git clone --branch=v{version} " + f"http://git.sv.gnu.org/r/autoconf.git autoconf/{version}" + ) + with utils.chdir(f"autoconf/{version}"): + utils.run("./bootstrap") + utils.run("./configure", f"--prefix={build_dir}") + utils.run("make -j 8 install") + + _setup(build_dir) + + return build_dir + + +def setup( + env: defs.ProgrammingEnvironment, compiler_version: str, root_dir: str, version: str +) -> None: + build_dir = core(env, compiler_version, root_dir, version, _build=False) + _setup(build_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--env", type=str, default=ENV) + parser.add_argument("--compiler-version", type=str, default=COMPILER_VERSION) + parser.add_argument("--root-dir", type=str, default=ROOT_DIR) + parser.add_argument("--version", type=str, default=VERSION) + args = parser.parse_args() + core(**args.__dict__) diff --git a/hpc2020/generate_build_hdf5.py b/hpc2020/generate_build_hdf5.py new file mode 100755 index 0000000..e669b30 --- /dev/null +++ b/hpc2020/generate_build_hdf5.py @@ -0,0 +1,106 @@ +#!/usr/local/apps/python3/3.11.8-01/bin/python3 +# -*- coding: utf-8 -*- +from __future__ import annotations +import argparse +import os + +import defs +import generate_build_autoconf +import generate_build_help2man +import utils + + +# >>> config: start +AUTOCONF_VERSION: str = "2.72" +ENV: defs.ProgrammingEnvironment = "gnu" +COMPILER_VERSION: str = "11.2.0" +MPI: defs.MPI = "hpcx" +PARTITION: defs.Partition = "gpu" +ROOT_DIR: str = defs.root_dir +VERSION: str = "1.14.4.2" +# >>> config: end + + +def _setup(build_dir: str) -> None: + utils.export_variable("HDF5_DIR", build_dir) + utils.export_variable("HDF5_ROOT", build_dir) + + +def core( + autoconf_version: str, + env: defs.ProgrammingEnvironment, + compiler_version: str, + mpi: defs.MPI, + partition: defs.Partition, + root_dir: str, + version: str, + _build: bool = True, +) -> str: + with utils.batch_file(prefix="build_hdf5"): + utils.module_purge(force=True) + utils.load_env(env) + env_id = utils.load_compiler(env, compiler_version) + mpi_id = utils.load_mpi(mpi, env, compiler_version, partition) + root_dir = os.path.abspath(root_dir) + build_dir = os.path.join(root_dir, f"hdf5/{version}/build/{env_id}/{mpi_id}") + + generate_build_help2man.setup(env, compiler_version, root_dir) + generate_build_autoconf.setup(env, compiler_version, root_dir, autoconf_version) + + if _build: + with utils.chdir(root_dir): + utils.run("mkdir -p hdf5") + branch = ( + f"hdf5-{version.replace('.', '_')}" if version < "1.14.4" else f"hdf5_{version}" + ) + utils.run( + f"git clone --branch={branch} --depth=1 " + f"https://github.com/HDFGroup/hdf5.git hdf5/{version}" + ) + with utils.chdir(f"hdf5/{version}"): + utils.run("chmod +x autogen.sh") + utils.run("./autogen.sh") + utils.run( + "CFLAGS='-fPIC'", + "./configure", + f"--prefix={build_dir}", + "--enable-build-mode=production", + "--enable-parallel", + "--enable-shared=no", + "--enable-tests", + "--enable-tools", + ) + utils.run("make -j 8 install") + + _setup(build_dir) + + return build_dir + + +def setup( + autoconf_version: str, + env: defs.ProgrammingEnvironment, + compiler_version: str, + mpi: defs.MPI, + partition: defs.Partition, + root_dir: str, + version: str, +) -> str: + build_dir = core( + autoconf_version, env, compiler_version, mpi, partition, root_dir, version, _build=False + ) + _setup(build_dir) + return build_dir + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--autoconf-version", type=str, default=AUTOCONF_VERSION) + parser.add_argument("--env", type=str, default=ENV) + parser.add_argument("--compiler-version", type=str, default=COMPILER_VERSION) + parser.add_argument("--mpi", type=str, default=MPI) + parser.add_argument("--partition", type=str, default=PARTITION) + parser.add_argument("--root-dir", type=str, default=ROOT_DIR) + parser.add_argument("--version", type=str, default=VERSION) + args = parser.parse_args() + core(**args.__dict__) diff --git a/hpc2020/generate_build_help2man.py b/hpc2020/generate_build_help2man.py new file mode 100644 index 0000000..3aff0da --- /dev/null +++ b/hpc2020/generate_build_help2man.py @@ -0,0 +1,58 @@ +#!/usr/local/apps/python3/3.11.8-01/bin/python3 +# -*- coding: utf-8 -*- +from __future__ import annotations +import argparse +import os + +import defs +import utils + +# >>> config: start +ENV: defs.ProgrammingEnvironment = "gnu" +COMPILER_VERSION: str = "11.2.0" +ROOT_DIR: str = defs.root_dir +# >>> config: end + + +def _setup(build_dir: str) -> None: + utils.update_path(f"{build_dir}/bin") + + +def core( + env: defs.ProgrammingEnvironment, compiler_version: str, root_dir: str, _build: bool = True +) -> str: + with utils.batch_file(prefix="build_help2man"): + utils.module_purge(force=True) + utils.load_env(env) + env_id = utils.load_compiler(env, compiler_version) + root_dir = os.path.abspath(root_dir) + build_dir = os.path.join(root_dir, f"help2man/build/{env_id}") + + if _build: + with utils.chdir(root_dir): + utils.run("mkdir -p help2man") + utils.run( + f"git clone --branch=master " + f"https://github.com/Distrotech/help2man.git help2man" + ) + with utils.chdir(f"help2man"): + utils.run("./configure", f"--prefix={build_dir}") + utils.run("make -j 8 install") + + _setup(build_dir) + + return build_dir + + +def setup(env: defs.ProgrammingEnvironment, compiler_version: str, root_dir: str) -> None: + build_dir = core(env, compiler_version, root_dir, _build=False) + _setup(build_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--env", type=str, default=ENV) + parser.add_argument("--compiler-version", type=str, default=COMPILER_VERSION) + parser.add_argument("--root-dir", type=str, default=ROOT_DIR) + args = parser.parse_args() + core(**args.__dict__) diff --git a/hpc2020/generate_build_netcdf.py b/hpc2020/generate_build_netcdf.py new file mode 100755 index 0000000..ef4d737 --- /dev/null +++ b/hpc2020/generate_build_netcdf.py @@ -0,0 +1,121 @@ +#!/usr/local/apps/python3/3.11.8-01/bin/python3 +# -*- coding: utf-8 -*- +from __future__ import annotations +import argparse +import os + +import defs +import generate_build_autoconf +import generate_build_hdf5 +import generate_build_help2man +import utils + + +# >>> config: start +AUTOCONF_VERSION: str = "2.72" +ENV: defs.ProgrammingEnvironment = "gnu" +COMPILER_VERSION: str = "11.2.0" +HDF5_VERSION: str = "1.14.4.2" +MPI: defs.MPI = "hpcx" +PARTITION: defs.Partition = "gpu" +ROOT_DIR: str = defs.root_dir +VERSION: str = "4.9.2" +# >>> config: end + + +def _setup(build_dir: str) -> None: + utils.export_variable("NETCDF4_DIR", build_dir) + utils.export_variable("NETCDF_ROOT", build_dir) + + +def core( + autoconf_version: str, + env: defs.ProgrammingEnvironment, + compiler_version: str, + hdf5_version: str, + mpi: defs.MPI, + partition: defs.Partition, + root_dir: str, + version: str, + _build: bool = True, +): + with utils.batch_file(prefix="build_netcdf"): + utils.module_purge(force=True) + utils.load_env(env) + env_id = utils.load_compiler(env, compiler_version) + mpi_id = utils.load_mpi(mpi, env, compiler_version, partition) + root_dir = os.path.abspath(root_dir) + build_dir = os.path.join(root_dir, f"netcdf-c/{version}/build/{env_id}/{mpi_id}") + + generate_build_help2man.setup(env, compiler_version, root_dir) + generate_build_autoconf.setup(env, compiler_version, root_dir, autoconf_version) + hdf5_build_dir = generate_build_hdf5.setup( + autoconf_version, env, compiler_version, mpi, partition, root_dir, hdf5_version + ) + + with utils.chdir(root_dir): + os.makedirs("netcdf-c", exist_ok=True) + utils.run( + f"git clone --branch=v{version} --depth=1 " + f"https://github.com/Unidata/netcdf-c.git netcdf-c/{version}" + ) + with utils.chdir(f"netcdf-c/{version}"): + utils.run("autoupdate") + utils.run("autoreconf -if") + utils.run("rm -rf build") + hdf5_include_dir = os.path.join(hdf5_build_dir, "include") + hdf5_lib_dir = os.path.join(hdf5_build_dir, "lib") + utils.run( + f"CFLAGS='-fPIC -I{hdf5_include_dir}'", + f"CPPFLAGS='-fPIC -I{hdf5_include_dir}'", + f"LDFLAGS='-fPIC -L{hdf5_lib_dir}'", + "LIBS=-ldl", + "./configure", + f"--prefix={build_dir}", + "--disable-shared", + "--enable-parallel-tests", + ) + utils.run("make -j 8 install") + + _setup(build_dir) + + return build_dir + + +def setup( + autoconf_version: str, + env: defs.ProgrammingEnvironment, + compiler_version: str, + hdf5_version: str, + mpi: defs.MPI, + partition: defs.Partition, + root_dir: str, + version: str, +) -> str: + build_dir = core( + autoconf_version, + env, + compiler_version, + hdf5_version, + mpi, + partition, + root_dir, + version, + _build=False, + ) + _setup(build_dir) + return build_dir + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--autoconf-version", type=str, default=AUTOCONF_VERSION) + parser.add_argument("--env", type=str, default=ENV) + parser.add_argument("--compiler-version", type=str, default=COMPILER_VERSION) + parser.add_argument("--hdf5-version", type=str, default=HDF5_VERSION) + parser.add_argument("--mpi", type=str, default=MPI) + parser.add_argument("--partition", type=str, default=PARTITION) + parser.add_argument("--root-dir", type=str, default=ROOT_DIR) + parser.add_argument("--version", type=str, default=VERSION) + args = parser.parse_args() + core(**args.__dict__) diff --git a/hpc2020/generate_prepare_pmapl.py b/hpc2020/generate_prepare_pmapl.py new file mode 100755 index 0000000..c7517a3 --- /dev/null +++ b/hpc2020/generate_prepare_pmapl.py @@ -0,0 +1,101 @@ +#!/usr/local/apps/python3/3.11.8-01/bin/python3 +# -*- coding: utf-8 -*- +from __future__ import annotations +import argparse +import os + +import defs +import generate_build_autoconf +import generate_build_hdf5 +import generate_build_help2man +import generate_build_netcdf +import utils + + +# >>> config: start +AUTOCONF_VERSION: str = "2.72" +BRANCH: str = "main" +ENV: defs.ProgrammingEnvironment = "gnu" +COMPILER_VERSION: str = "11.2.0" +HDF5_VERSION: str = "1.14.4.2" +MPI: defs.MPI = "hpcx" +NETCDF_VERSION: str = "4.9.2" +PARTITION: defs.Partition = "gpu" +ROOT_DIR: str = defs.root_dir +# >>> config: end + + +def core( + autoconf_version: str, + branch: str, + env: defs.ProgrammingEnvironment, + compiler_version: str, + hdf5_version: str, + mpi: defs.MPI, + netcdf_version: str, + partition: defs.Partition, + root_dir: str, +) -> str: + with utils.batch_file(prefix="prepare_pmapl") as (f, fname): + # clear environment + utils.module_purge(force=True) + + # load relevant modules + utils.load_env(env) + env_id = utils.load_compiler(env, compiler_version) + mpi_id = utils.load_mpi(mpi, env, compiler_version, partition) + utils.module_load("boost", "cmake", "python3/3.11.8-01") + if partition == "gpu": + utils.load_gpu_libraries(env, compiler_version) + + # set path to custom build of external dependencies + generate_build_help2man.setup(env, compiler_version, root_dir) + generate_build_autoconf.setup(env, compiler_version, root_dir, autoconf_version) + generate_build_hdf5.setup( + autoconf_version, env, compiler_version, mpi, partition, root_dir, hdf5_version + ) + generate_build_netcdf.setup( + autoconf_version, + env, + compiler_version, + hdf5_version, + mpi, + partition, + root_dir, + netcdf_version, + ) + + # set path to PMAP code + pmapl_dir = os.path.join(defs.root_dir, "pmapl", branch) + assert os.path.exists(pmapl_dir) + utils.export_variable("PMAPL", pmapl_dir) + pmapl_venv_dir = os.path.join(pmapl_dir, "venv", partition, env_id, mpi_id) + utils.export_variable("PMAPL_VENV", pmapl_venv_dir) + + # low-level GT4Py, DaCe and GHEX config + gt_cache_root = os.path.join(defs.root_dir, "pmapl", "gt_cache", env_id) + utils.export_variable("GT_CACHE_ROOT", gt_cache_root) + utils.export_variable("GT_CACHE_DIR_NAME", ".gt_cache") + utils.export_variable("DACE_CONFIG", os.path.join(gt_cache_root, ".dace.conf")) + + # jump into project source directory and activate virtual environment (if it already exists) + with utils.chdir(pmapl_dir, restore=False): + if os.path.exists(pmapl_venv_dir): + utils.run(f"source {pmapl_venv_dir}/bin/activate") + + return fname + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--autoconf-version", type=str, default=AUTOCONF_VERSION) + parser.add_argument("--branch", type=str, default=BRANCH) + parser.add_argument("--env", type=str, default=ENV) + parser.add_argument("--compiler-version", type=str, default=COMPILER_VERSION) + parser.add_argument("--hdf5-version", type=str, default=HDF5_VERSION) + parser.add_argument("--mpi", type=str, default=MPI) + parser.add_argument("--netcdf-version", type=str, default=NETCDF_VERSION) + parser.add_argument("--partition", type=str, default=PARTITION) + parser.add_argument("--root-dir", type=str, default=ROOT_DIR) + args = parser.parse_args() + core(**args.__dict__) diff --git a/hpc2020/salloc.py b/hpc2020/salloc.py new file mode 100755 index 0000000..861a89a --- /dev/null +++ b/hpc2020/salloc.py @@ -0,0 +1,39 @@ +#!/usr/local/apps/python3/3.11.8-01/bin/python3 +# -*- coding: utf-8 -*- +import argparse +import os + +import defs +import utils + + +# >>> config: start + +ACCOUNT: str = os.environ.get("ECACCOUNT", "") +NUM_NODES: int = 1 +PARTITION: defs.Partition = "gpu" +TIME: str = "01:00:00" +# >>> config: end + + +def core(account: int, num_nodes: int, partition: defs.Partition, time: str) -> None: + command = [ + f"salloc", + f"--account={account}", + "--cpus-per-task=256", + f"--nodes={num_nodes}", + f"--partition={partition}", + f"--time={time}", + ] + command += ["--qos=ng", "--gpus=4"] if partition == "gpu" else ["--qos=np"] + utils.run(*command) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Get an allocation on the compute nodes.") + parser.add_argument("--account", type=str, default=ACCOUNT) + parser.add_argument("--num-nodes", type=int, default=NUM_NODES) + parser.add_argument("--partition", type=str, default=PARTITION) + parser.add_argument("--time", type=str, default=TIME) + args = parser.parse_args() + core(**args.__dict__) diff --git a/hpc2020/utils.py b/hpc2020/utils.py new file mode 100644 index 0000000..d5b2f11 --- /dev/null +++ b/hpc2020/utils.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- +from __future__ import annotations +import contextlib +import dataclasses +import os +import subprocess +import tempfile +import typing + +import defs + + +BATCH_DIRECTORY_REGISTRY = [] +BATCH_FILE_REGISTRY = [] + + +@contextlib.contextmanager +def batch_directory(): + os.makedirs("_tmp", exist_ok=True) + try: + if len(BATCH_DIRECTORY_REGISTRY) > 0: + final_cleanup = False + yield BATCH_DIRECTORY_REGISTRY[-1] + else: + final_cleanup = True + dirname = os.path.abspath(tempfile.mkdtemp(dir="_tmp")) + BATCH_DIRECTORY_REGISTRY.append(dirname) + os.makedirs(dirname, exist_ok=True) + print(f"py-hpc-scripts: create {dirname}") + yield dirname + finally: + if final_cleanup: + BATCH_DIRECTORY_REGISTRY.pop() + + +@contextlib.contextmanager +def batch_file(prefix: typing.Optional[str] = None): + if len(BATCH_DIRECTORY_REGISTRY) > 0: + fname = os.path.abspath(os.path.join(BATCH_DIRECTORY_REGISTRY[-1], prefix + ".sh")) + else: + # os.makedirs("_tmp", exist_ok=True) + # fname = os.path.abspath(tempfile.mktemp(prefix=prefix + "_", suffix=".sh", dir="_tmp")) + fname = os.path.abspath(prefix + ".sh") + + try: + with open(fname, "w") as f: + BATCH_FILE_REGISTRY.append(f) + f.write("#!/bin/bash -l\n\n") + yield f, fname + finally: + print(f"py-hpc-scripts: write {fname}") + BATCH_FILE_REGISTRY.pop() + + +def run(*args: str) -> None: + split_args = [item for arg in args for item in arg.split(" ")] + command = " ".join(split_args) + if len(BATCH_FILE_REGISTRY) > 0: + BATCH_FILE_REGISTRY[-1].write(command + "\n") + else: + subprocess.run(command, capture_output=False, shell=True) + + +def module_purge(force: bool = False) -> None: + run(f"module{' --force ' if force else ' '}purge") + + +def module_load(*module_names: str) -> None: + for module_name in module_names: + run(f"module load {module_name}") + + +class InvalidArgumentError(Exception): + def __init__(self, parameter: str, token: str, options: list[str]): + options = [f"`{opt}`" for opt in options] + msg = ( + f"Invalid value `{token}` for parameter `{parameter}`. " + f"Available options: {', '.join(options)}." + ) + super().__init__(msg) + + +@contextlib.contextmanager +def check_argument(parameter, token, options): + if token not in options: + raise InvalidArgumentError(parameter, token, options) + try: + yield token + finally: + pass + + +def load_env(env: str) -> None: + with check_argument("env", env, defs.valid_programming_environments): + module_load(f"prgenv/{env}") + + +def load_compiler(env: str, compiler_version: str) -> str: + with check_argument("env", env, defs.valid_programming_environments): + if env == "gnu": + module_load(f"gcc/{compiler_version}") + cc, cxx, fc = "gcc", "g++", "gfortran" + env_id = f"gnu-{compiler_version}" + export_variable( + "LD_LIBRARY_PATH", + f"/usr/local/apps/gcc/{compiler_version}/lib64", + prepend_value=True, + ) + else: + module_load(f"intel/{compiler_version}") + cc, cxx, fc = "icc", "icpc", "ifort" + env_id = f"intel-{compiler_version}" + export_variable("CC", cc) + export_variable("CXX", cxx) + export_variable("FC", fc) + return env_id + + +def load_mpi(mpi: str, env: str, compiler_version: str, partition: str) -> str: + with check_argument("mpi", mpi, defs.valid_mpi_libraries): + with check_argument("env", env, defs.valid_programming_environments): + with check_argument("partition", partition, defs.valid_partitions): + cc, cxx, fc = "mpicc", "mpicxx", "mpifort" + if mpi == "hpcx": + if env == "gnu" and compiler_version == "13.2.0": + module_name = "hpcx-openmpi/2.17.1" + else: + module_name = "hpcx-openmpi/2.10.0" + elif mpi == "intel-mpi": + module_name = "intel-mpi/2023.2.0" + if env == "intel": + cc, cxx, fc = "mpiicc", "mpiicpc", "mpiifort" + else: + cc, cxx, fc = "mpigcc", "mpigxx", "mpif90" + else: + module_name = "openmpi/4.1.5.4" if partition == "gpu" else "openmpi/4.1.1.1" + module_load(module_name) + export_variable("CC", cc) + export_variable("MPICC", cc) + export_variable("CXX", cxx) + export_variable("MPICXX", cxx) + export_variable("FC", fc) + export_variable("MPIFC", fc) + return module_name.replace("/", "-") + + +def load_gpu_libraries(env: str, compiler_version: str) -> None: + with check_argument("env", env, defs.valid_programming_environments): + if env == "gnu" and compiler_version == "13.2.0": + module_load("nvidia/24.1", "cuda/11.6") + else: + module_load("nvidia/22.11", "cuda/11.6") + export_variable("GHEX_USE_GPU", 1) + export_variable("GHEX_GPU_TYPE", "NVIDIA") + export_variable("GHEX_GPU_ARCH", 80) + + +def export_variable(name: str, value: typing.Any, prepend_value: bool = False) -> None: + cmd = f"export {name}={str(value)}" + if prepend_value: + cmd += f":${name}" + run(cmd) + + +def update_path(value: str): + export_variable("PATH", value, prepend_value=True) + + +@contextlib.contextmanager +def chdir(dirname: str, restore: bool = True) -> None: + try: + run(f"pushd {dirname}") + yield None + finally: + if restore: + run("popd") + + +@dataclasses.dataclass +class ThreadsLayout: + num_nodes: int + num_tasks_per_node: int + num_threads_per_task: int diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..251d2d2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +black==24.4.0 +pre-commit