BraggNet/generate_training_data_parallel.py at master · sullivanbt/BraggNet · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
from __future__ import (absolute_import, division, print_function)
import os
import sys
import threading
import time
import pickle


python = '/home/ntv/workspace/mantid/release/bin/mantidpython'
"""
# Beta lac second xtal july 2018
reduce_one_run_script = '/home/ntv/ml_peak_integration/generate_training_data_v2_betalacsecondxtal.py'
run_nums = [9113, 9114, 9115, 9116, 9117]
max_processes = 3
"""
reduce_one_run_script = '/home/ntv/ml_peak_integration/generate_training_data_v2_dna.py'
run_nums = range(9291, 9304+1)
max_processes = 3


"""
# beta lac first xtal july 2018
reduce_one_run_script = '/home/ntv/ml_peak_integration/generate_training_data_v2_betalacfirstxtal.py'
run_nums = [9078, 9079, 9080, 9082, 9083, 9084]
max_processes = 6
"""

#Define a class for threading (taken from ReduceSCD_Parallel.py) and set up parallel runs
class ProcessThread ( threading.Thread ):
    command = ""

    def setCommand( self, command="" ):
        self.command = command

    def run ( self ):
        print('STARTING PROCESS: ' + self.command)
        os.system( self.command )

procList=[]
index = 0
for r_num in run_nums:
    procList.append( ProcessThread() )
    cmd = '%s %s %s' % (python, reduce_one_run_script, r_num)
    procList[index].setCommand( cmd )
    index = index + 1

all_done = False
active_list=[]
while not all_done:
    if  len(procList) > 0 and len(active_list) < max_processes :
        thread = procList[0]
        procList.remove(thread)
        active_list.append( thread )
        thread.start()
    time.sleep(2)
    for thread in active_list:
        if not thread.isAlive():
            active_list.remove( thread )
    if len(procList) == 0 and len(active_list) == 0 :
        all_done = True