Feature/estimate workflow execution time

ConnectionManager/machines.py

@@ -84,3 +84,23 @@
         }
     ]
 }
+
+
+machines["HPDA"] = {
+    "host": "fe-store01.sc.bs.dlr.de",
+    "username": "holk_jo",
+    "SSHkey": "",
+    "basedir": "/home/holk_jo/VESTEC/VESTEC-INSTALL",
+    "available_nodes": 4,
+    "cores_per_node": 56,
+    "scheduler": "slurm",
+    "main_queue": "cpu",
+    "queues": [
+        {
+            "queue_name": 'cpu',
+            "max_nodes": 4,
+            "min_time": "00:01:00",
+            "max_time": "168:00:00"
+        }
+    ]
+}

ConnectionManager/machines.yaml

@@ -29,3 +29,15 @@ CIRRUS:
   scheduler: pbs
   username: gpsgibb
 
+HPDA:
+  SSHHkey: ""
+  available_nodes: 4
+  basedir: /home/holk_jo/VESTEC/VESTEC-install
+  cores_per_node: 56
+  host: fe-store01.sc.bs.dlr.de
+  main_queue: cpu
+  queues:
+  - {max_nodes: 4, max_time: '168:00:00', min_time: '00:01:00', queue_name: cpu}
+  scheduler: slurm
+  username: holk_jo
+

Database/generate_db.py

@@ -14,6 +14,9 @@ def initialiseStaticInformation():
     cirrus = Machine(machine_id=str(uuid4()), machine_name="CIRRUS", host_name="cirrus.epcc.ed.ac.uk", 
                      scheduler="PBS", num_nodes=280, cores_per_node=36, base_work_dir="/lustre/home/????")
 
+    hpda = Machine(machine_id=str(uuid4()), machine_name="HPDA", host_name="fe-store01.sc.bs.dlr.de",
+                   scheduler="SLURM", num_nodes=4, cores_per_node=56, base_work_dir="/home/holk_jo/VESTEC/VESTEC-install")
+
     pny.commit()
 
     archer.queues.create(queue_id=str(uuid4()), queue_name="standard", max_nodes=4920, min_walltime=60,
@@ -43,5 +46,9 @@ def initialiseStaticInformation():
     cirrus.queues.create(queue_id=str(uuid4()), queue_name="large", max_nodes=280, min_walltime=60,
                          max_walltime=172800, default=0)
 
+
+    hpda.queues.create(queue_id=str(uuid4()), queue_name="cpu", max_nodes=4, min_walltime=60,
+                         max_walltime=604800, default=0)
+
     pny.commit()
 

DockerFiles/docker-compose-local.yml

@@ -0,0 +1,181 @@
+version: '3'
+services:
+  nginx:
+    container_name: nginx
+    #network_mode: "bridge"
+    build:
+      context: ../Nginx
+      dockerfile: Dockerfile-local
+    ports:
+      - 80:80
+    volumes:
+      - ../Nginx/static/:/var/www/
+    links:
+      - externalservices:externalservices
+    networks:
+      - vestec
+
+  externalservices:
+    container_name: externalservices
+    build:
+      context: ../
+      dockerfile: ExternalServices/Dockerfile
+    ports:
+      - 8000:8000
+    environment:
+      VESTEC_SM_URI: "http://simmanager:5500/SM"
+      VESTEC_EDI_URI: "http://edi:5501/EDImanager"
+      VESTEC_MSM_URI: "http://msm:5502/MSM"
+      VESTEC_DM_URI : "http://datamanager:5503/DM"
+      VESTEC_SHARED_FILE_LOCATION : "/shared_data"
+      JWT_PASSWD: "SECRET"  # to be replaced by docker secrets
+      PYTHONUNBUFFERED: 1
+    env_file:
+      - environment_variables.env
+    volumes:
+      - "/shared_data:/shared_data"
+    networks:
+      - vestec
+
+  simmanager:
+    container_name: simmanager
+    depends_on:
+      - "maria"
+    build:
+      context: ../
+      dockerfile: SimulationManager/Dockerfile
+    #WE ONLY NEED THIS IF WE ARE USING SQLITE
+    # volumes:
+    #   - "../Database:/database"
+    environment:
+      VESTEC_MSM_URI: "http://msm:5502/MSM"
+      VESTEC_SHARED_FILE_LOCATION : "/shared_data"
+      PYTHONUNBUFFERED: 1
+    env_file:
+      - environment_variables.env
+    volumes:
+      - "/shared_data:/shared_data"
+    networks:
+      - vestec
+
+  msm:
+    container_name: msm
+    depends_on:
+      - "maria"
+    build:
+      context: ../
+      dockerfile: MachineStatusManager/Dockerfile
+    #WE ONLY NEED THIS IF WE ARE USING SQLITE
+    # volumes:
+    #   - "../Database:/database"
+    environment:
+      VESTEC_SHARED_FILE_LOCATION : "/shared_data"
+      PYTHONUNBUFFERED: 1
+    env_file:
+      - environment_variables.env
+    volumes:
+      - "/shared_data:/shared_data"
+    networks:
+      - vestec
+
+  machineinterface:
+    container_name: machineinterface
+    depends_on:
+      - "rmq"
+      - "maria"
+    build:
+      context: ../
+      dockerfile: MachineInterface/Dockerfile-local
+    #WE ONLY NEED THIS IF WE ARE USING SQLITE
+    # volumes:
+    #   - "../Database:/database"
+    environment:
+      VESTEC_SHARED_FILE_LOCATION : "/shared_data"    
+      PYTHONUNBUFFERED: 1
+    env_file:
+      - environment_variables.env
+    volumes:
+      - "/shared_data:/shared_data"
+    networks:
+      - vestec
+
+  edi:
+    container_name: edi
+    depends_on:
+      - "maria"
+    build:
+      context: ../
+      dockerfile: ExternalDataInterface/Dockerfile
+    #WE ONLY NEED THIS IF WE ARE USING SQLITE
+    # volumes:
+    #   - "../Database:/database"
+    environment:
+      VESTEC_SHARED_FILE_LOCATION : "/shared_data"    
+      PYTHONUNBUFFERED: 1
+    env_file:
+      - environment_variables.env
+    volumes:
+      - "/shared_data:/shared_data"
+    networks:
+      - vestec
+
+  maria:
+    container_name: mariadb
+    image: mariadb
+    environment:
+      MYSQL_ROOT_PASSWORD: "changeme"
+    volumes:
+      - "../../vestec-wp5-db:/var/lib/mysql"
+    networks:
+      - vestec
+
+
+  rmq:
+    container_name: rabbitmq
+    image: rabbitmq
+    networks:
+      - vestec
+
+  workflow:
+    container_name: workflowManager
+    depends_on:
+      - "rmq"
+      - "maria"
+    build:
+      context: ../
+      dockerfile: WorkflowManager/Dockerfile
+    environment:
+      VESTEC_SM_URI: "http://simmanager:5500/SM"
+      VESTEC_EDI_URI: "http://edi:5501/EDImanager"
+      VESTEC_MSM_URI: "http://msm:5502/MSM"
+      VESTEC_DM_URI : "http://datamanager:5503/DM"
+      VESTEC_SHARED_FILE_LOCATION : "/shared_data"
+      PYTHONUNBUFFERED: 1
+    env_file:
+      - environment_variables.env
+    volumes:
+      - "/shared_data:/shared_data"
+    networks:
+      - vestec
+
+  datamanager:
+    container_name: DataManager
+    depends_on:
+      - "maria"
+    build:
+      context: ../
+      dockerfile: DataManager/Dockerfile
+    environment:
+      VESTEC_SHARED_FILE_LOCATION : "/shared_data"
+      PYTHONUNBUFFERED: 1
+    env_file:
+      - environment_variables.env
+    volumes:
+      - "/shared_data:/shared_data"
+    networks:
+      - vestec
+
+
+networks:
+  vestec:
+    driver: "bridge"

MachineInterface/Dockerfile

@@ -4,5 +4,7 @@ COPY MachineInterface/requirements.txt /app/MachineInterface/requirements.txt
 RUN pip install -r /app/MachineInterface/requirements.txt
 COPY . /app
 WORKDIR /app/MachineInterface
-RUN cat /app/MachineInterface/misc/ssh_archer_config >> /etc/ssh/ssh_config
+#RUN cat /app/MachineInterface/misc/ssh_archer_config >> /etc/ssh/ssh_config
+#RUN cat /app/MachineInterface/misc/ssh_hpda_config >> /etc/ssh/ssh_config
+RUN cat /app/MachineInterface/misc/ssh_hpda_config >> $HOME/.ssh/ssh_config
 CMD python interface.py

MachineInterface/Dockerfile-local

@@ -0,0 +1,9 @@
+FROM python:3.8-slim
+RUN apt-get update && apt-get install -y openssh-client vim
+COPY MachineInterface/requirements.txt /app/MachineInterface/requirements.txt
+RUN pip install -r /app/MachineInterface/requirements.txt
+COPY . /app
+WORKDIR /app/MachineInterface
+RUN cat /app/MachineInterface/misc/ssh_hpda_config >> /etc/ssh/ssh_config
+RUN mkdir -p /root/.ssh/controlmasters
+CMD python interface.py

MachineInterface/hpda_test.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Nov  9 09:55:05 2020
+
+@author: kont_ma
+"""
+
+import asyncio
+from mproxy.client import Client
+
+async def get_job_directory_listing(machine_name, directory_name):
+    client = await Client.create(machine_name)
+    return await client.ls(directory_name)
+
+if __name__=='__main__':
+    print(asyncio.run(get_job_directory_listing("HPDA", ".")))

MachineInterface/misc/ssh_hpda_config

@@ -0,0 +1,14 @@
+Host hpda
+  ControlMaster auto
+  ControlPath /root/.ssh/controlmasters/%r@%h:%p
+  ControlPersist 24h
+  HostName fe-store01.sc.bs.dlr.de
+  User holk_jo
+
+Host fe-store01.sc.bs.dlr.de
+  ControlMaster auto
+  ControlPath /root/.ssh/controlmasters/%r@%h:%p
+  ControlPersist 24h
+  HostName fe-store01.sc.bs.dlr.de
+  User holk_jo
+

Nginx/Dockerfile-local

@@ -0,0 +1,4 @@
+FROM nginx
+EXPOSE 80
+COPY default-local.conf /etc/nginx/conf.d/
+CMD ["nginx", "-g", "daemon off;"]