Add Python scripts for HPC2020 and for help2man

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+    -   id: check-ast
+    -   id: check-case-conflict
+    -   id: check-json
+    -   id: check-merge-conflict
+    -   id: check-yaml
+    -   id: debug-statements
+    -   id: end-of-file-fixer
+    -   id: fix-encoding-pragma
+    -   id: mixed-line-ending
+    -   id: requirements-txt-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/pre-commit/pygrep-hooks
+    rev: v1.10.0
+    hooks:
+    -   id: python-check-blanket-noqa
+    -   id: python-check-blanket-type-ignore
+    -   id: python-use-type-annotations
+    -   id: rst-backticks
+-   repo: https://github.com/psf/black
+    rev: 24.2.0
+    hooks:
+    -   id: black
+-   repo: https://github.com/PyCQA/autoflake
+    rev: v2.2.1
+    hooks:
+    -   id: autoflake
+        args: [--in-place, --remove-all-unused-imports, --ignore-init-module-imports]

daint/.bashrc

@@ -121,6 +121,7 @@ export LC_ALL=C.UTF-8
 export LANG=C.UTF-8
 
 # project
+
 project_id=$(sacct --format=Account --noheader | head -n 1 | awk '{$1=$1}1')
 export PROJECT=/project/"$project_id"/"$USER"
 
@@ -140,6 +141,7 @@ export SQUEUE_FORMAT="%.9i %.50j %.15u %.5q %.15T %.10M %.10l %.5D %.13f %R"
 # hdf5 and netcdf
 export HDF5_ROOT=/users/"$USER"/hdf5/1.14.2/build/gnu
 export NETCDF_ROOT=/users/"$USER"/netcdf-c/4.9.2/build/gnu
+export PATH="$HOME"/autoconf/2.72/build/gnu/bin/:"$HOME"/help2man/1.49.3/build/gnu/bin/:$PATH
 
 # get node id of a salloc
 function get_node_id() {

daint/generate_build_autoconf.py

@@ -35,6 +35,7 @@ def core(env: defs.ProgrammingEnvironment, partition: defs.Partition, root_dir:
                 utils.run("CC=cc", "CXX=CC", "./configure", f"--prefix={build_dir}")
                 utils.run("make -j 8 install")
                 utils.export_variable("AUTOCONF_ROOT", build_dir)
+                utils.append_to_path("PATH", f"{build_dir}/bin")
 
 
 if __name__ == "__main__":

daint/generate_build_help2man.py

@@ -0,0 +1,45 @@
+#!/opt/python/3.9.4.1/bin/python
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+import argparse
+import os
+
+import defs
+import utils
+
+
+# >>> config: start
+ENV: defs.ProgrammingEnvironment = "gnu"
+PARTITION: defs.Partition = "gpu"
+ROOT_DIR: str = f"/users/{os.getlogin()}"
+VERSION: str = "1.49.3"
+# >>> config: endi
+
+
+def core(env: defs.ProgrammingEnvironment, partition: defs.Partition, root_dir: str, version: str):
+    with utils.batch_file(prefix="build_help2man"):
+        utils.module_purge(force=True)
+        utils.load_partition(partition)
+        utils.load_env(env)
+        root_dir = os.path.abspath(root_dir)
+        with utils.chdir(root_dir):
+            utils.run("mkdir -p help2man")
+            branch = f"master"
+            utils.run(
+                f"git clone --branch={branch} "
+                f"https://github.com/Distrotech/help2man.git help2man/{version}"
+            )
+            with utils.chdir(f"help2man/{version}"):
+                build_dir = os.path.join(root_dir, f"help2man/{version}/build/{env}")
+                utils.run("CC=cc", "CXX=CC", "./configure", f"--prefix={build_dir}")
+                utils.run("make -j 8 install")
+                utils.append_to_path("PATH", f"{build_dir}/bin")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--env", type=str, default=ENV)
+    parser.add_argument("--partition", type=str, default=PARTITION)
+    parser.add_argument("--root-dir", type=str, default=ROOT_DIR)
+    parser.add_argument("--version", type=str, default=VERSION)
+    args = parser.parse_args()
+    core(**args.__dict__)

daint/generate_prepare_pmapl.py

@@ -10,7 +10,7 @@
 
 
 # >>> config: start
-BRANCH: str = "cloudsc-cy49r1"
+BRANCH: str = "main"
 ENV: defs.ProgrammingEnvironment = "gnu"
 PARTITION: defs.Partition = "gpu"
 # >>> config: end

daint/salloc.py

@@ -7,6 +7,7 @@
 
 
 # >>> config: start
+
 project_id="$(sacct --format=Account --noheader | head -n 1 | awk '{$1=$1}1')"
 ACCOUNT: str = project_id
 NUM_NODES: int = 1

daint/sbatch_pmapl.py

@@ -12,7 +12,7 @@
 # >>> config: start
 project_id="$(sacct --format=Account --noheader | head -n 1 | awk '{$1=$1}1')"
 account: str = project_id
-branch_l: list[str] = ["cloudsc-cy49r1"]
+branch_l: list[str] = ["main"]
 partition: defs.Partition = "gpu"
 env_l: list[defs.ProgrammingEnvironment] = ["gnu"]
 ghex_aggregate_fields: bool = False

daint/utils.py

@@ -103,6 +103,8 @@ def load_env(env: str) -> None:
 def export_variable(name: str, value: typing.Any) -> None:
     run(f"export {name}={str(value)}")
 
+def append_to_path( name: str, value: typing.Any) -> None:
+    run(f"{name}={str(value)}:${name}")
 
 def setup_cuda():
     run("NVCC_PATH=$(which nvcc)")

hpc2020/.bashrc

@@ -95,9 +95,16 @@ alias l='ls -CF'
 export LC_ALL=C.UTF-8
 export LANG=C.UTF-8
 
+# nvim
+alias vim='/home/"$USER"/neovim/0.9.5/install/bin/nvim'
+export VIMRUNTIME=/home/"$USER"/neovim/0.9.5/install/share/nvim/runtime
+
 # slurm shortcuts and settings
 alias sb='sbatch'
 alias sc='scancel'
 alias sq='squeue -u $USER'
 alias sr='srun'
-export SQUEUE_FORMAT="%.9i %.50j %.15u %.15q %.15T %.10M %.10l %.5D %.13f %R"
+export SQUEUE_FORMAT="%.9i %.60j %.10u %.10q %.15T %.10M %.10l %.5D %.13f %R"
+
+# update path with manually built software
+export PATH="$HPCPERM"/autoconf/2.72/build/gnu/bin/:"$HPCPERM"/help2man/build/gnu/bin/:$PATH

hpc2020/defs.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+import os
+import typing
+
+FloatingPointPrecision = typing.Literal["double", "single"]
+MPI = typing.Literal["hpcx", "intelmpi", "openmpi"]
+Partition = typing.Literal["gpu", "par"]
+ProgrammingEnvironment = typing.Literal["gnu", "intel"]
+
+valid_mpi_libraries = typing.get_args(MPI)
+valid_partitions = typing.get_args(Partition)
+valid_programming_environments = typing.get_args(ProgrammingEnvironment)
+
+root_dir = os.environ.get("HPCPERM", f"/home/{os.getlogin()}")

hpc2020/generate_build_autoconf.py

@@ -0,0 +1,72 @@
+#!/usr/local/apps/python3/3.11.8-01/bin/python3
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+import argparse
+import os
+
+import defs
+import generate_build_help2man
+import utils
+
+
+# >>> config: start
+ENV: defs.ProgrammingEnvironment = "gnu"
+COMPILER_VERSION: str = "11.2.0"
+ROOT_DIR: str = defs.root_dir
+VERSION: str = "2.72"
+# >>> config: end
+
+
+def _setup(build_dir: str) -> None:
+    utils.export_variable("AUTOCONF_ROOT", build_dir)
+    utils.update_path(f"{build_dir}/bin")
+
+
+def core(
+    env: defs.ProgrammingEnvironment,
+    compiler_version: str,
+    root_dir: str,
+    version: str,
+    _build: bool = True,
+) -> str:
+    with utils.batch_file(prefix="build_autoconf"):
+        utils.module_purge(force=True)
+        utils.load_env(env)
+        env_id = utils.load_compiler(env, compiler_version)
+        root_dir = os.path.abspath(root_dir)
+        build_dir = os.path.join(root_dir, f"autoconf/{version}/build/{env_id}")
+
+        generate_build_help2man.setup(env, compiler_version, root_dir)
+
+        if _build:
+            with utils.chdir(root_dir):
+                utils.run("mkdir -p autoconf")
+                utils.run(
+                    f"git clone --branch=v{version} "
+                    f"http://git.sv.gnu.org/r/autoconf.git autoconf/{version}"
+                )
+                with utils.chdir(f"autoconf/{version}"):
+                    utils.run("./bootstrap")
+                    utils.run("./configure", f"--prefix={build_dir}")
+                    utils.run("make -j 8 install")
+
+        _setup(build_dir)
+
+    return build_dir
+
+
+def setup(
+    env: defs.ProgrammingEnvironment, compiler_version: str, root_dir: str, version: str
+) -> None:
+    build_dir = core(env, compiler_version, root_dir, version, _build=False)
+    _setup(build_dir)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--env", type=str, default=ENV)
+    parser.add_argument("--compiler-version", type=str, default=COMPILER_VERSION)
+    parser.add_argument("--root-dir", type=str, default=ROOT_DIR)
+    parser.add_argument("--version", type=str, default=VERSION)
+    args = parser.parse_args()
+    core(**args.__dict__)

hpc2020/generate_build_hdf5.py

@@ -0,0 +1,106 @@
+#!/usr/local/apps/python3/3.11.8-01/bin/python3
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+import argparse
+import os
+
+import defs
+import generate_build_autoconf
+import generate_build_help2man
+import utils
+
+
+# >>> config: start
+AUTOCONF_VERSION: str = "2.72"
+ENV: defs.ProgrammingEnvironment = "gnu"
+COMPILER_VERSION: str = "11.2.0"
+MPI: defs.MPI = "hpcx"
+PARTITION: defs.Partition = "gpu"
+ROOT_DIR: str = defs.root_dir
+VERSION: str = "1.14.4.2"
+# >>> config: end
+
+
+def _setup(build_dir: str) -> None:
+    utils.export_variable("HDF5_DIR", build_dir)
+    utils.export_variable("HDF5_ROOT", build_dir)
+
+
+def core(
+    autoconf_version: str,
+    env: defs.ProgrammingEnvironment,
+    compiler_version: str,
+    mpi: defs.MPI,
+    partition: defs.Partition,
+    root_dir: str,
+    version: str,
+    _build: bool = True,
+) -> str:
+    with utils.batch_file(prefix="build_hdf5"):
+        utils.module_purge(force=True)
+        utils.load_env(env)
+        env_id = utils.load_compiler(env, compiler_version)
+        mpi_id = utils.load_mpi(mpi, env, compiler_version, partition)
+        root_dir = os.path.abspath(root_dir)
+        build_dir = os.path.join(root_dir, f"hdf5/{version}/build/{env_id}/{mpi_id}")
+
+        generate_build_help2man.setup(env, compiler_version, root_dir)
+        generate_build_autoconf.setup(env, compiler_version, root_dir, autoconf_version)
+
+        if _build:
+            with utils.chdir(root_dir):
+                utils.run("mkdir -p hdf5")
+                branch = (
+                    f"hdf5-{version.replace('.', '_')}" if version < "1.14.4" else f"hdf5_{version}"
+                )
+                utils.run(
+                    f"git clone --branch={branch} --depth=1 "
+                    f"https://github.com/HDFGroup/hdf5.git hdf5/{version}"
+                )
+                with utils.chdir(f"hdf5/{version}"):
+                    utils.run("chmod +x autogen.sh")
+                    utils.run("./autogen.sh")
+                    utils.run(
+                        "CFLAGS='-fPIC'",
+                        "./configure",
+                        f"--prefix={build_dir}",
+                        "--enable-build-mode=production",
+                        "--enable-parallel",
+                        "--enable-shared=no",
+                        "--enable-tests",
+                        "--enable-tools",
+                    )
+                    utils.run("make -j 8 install")
+
+        _setup(build_dir)
+
+    return build_dir
+
+
+def setup(
+    autoconf_version: str,
+    env: defs.ProgrammingEnvironment,
+    compiler_version: str,
+    mpi: defs.MPI,
+    partition: defs.Partition,
+    root_dir: str,
+    version: str,
+) -> str:
+    build_dir = core(
+        autoconf_version, env, compiler_version, mpi, partition, root_dir, version, _build=False
+    )
+    _setup(build_dir)
+    return build_dir
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--autoconf-version", type=str, default=AUTOCONF_VERSION)
+    parser.add_argument("--env", type=str, default=ENV)
+    parser.add_argument("--compiler-version", type=str, default=COMPILER_VERSION)
+    parser.add_argument("--mpi", type=str, default=MPI)
+    parser.add_argument("--partition", type=str, default=PARTITION)
+    parser.add_argument("--root-dir", type=str, default=ROOT_DIR)
+    parser.add_argument("--version", type=str, default=VERSION)
+    args = parser.parse_args()
+    core(**args.__dict__)