From 10f5f9bb25c2d62194245fe8355dd3f3cc733833 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 15 Dec 2016 17:01:33 -0800 Subject: [PATCH 01/24] some code for multiprocessing. It doesn't work, but might be made to work in Python >=3.4. --- src/python/frontend/multidiags.py | 50 ++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index 5af22684..155e91a4 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -4,6 +4,9 @@ import logging, pdb, importlib, time, cProfile from pprint import pprint from itertools import groupby +from multiprocessing import Pool, current_process, Process, Queue +import multiprocessing +from os import getpid from metrics.frontend.multimaster import * # this file is just a demo import metrics.frontend.multimaster as multimaster import metrics.frontend.defines as defines @@ -253,7 +256,14 @@ def expand_lists_collections( opt, okeys=None ): flattened = [ o for ops in expanded_opts for o in ops ] return flattened -def multidiags1( opt ): +def md_run_diags( opt, queue1=None ): + #curr_proc = current_process() + #curr_proc.daemon = False + opt['modobs_names'] = run_diags(opt.clone()) # run_diags messes with its input arg, though it shouldn't + #queue1.put(opt) + return opt + +def multidiags1( opt, mp_pool ): """The input opt is a single Options instance, possibly with lists or collection names in place of real option values. This function expands out such lists or collection names, giving us a list of simple Options instances. @@ -269,12 +279,30 @@ def multidiags1( opt ): o = finalize_modelobs(o) # copies obspath to obs['path'], changes {} to [{}] o['runby'] = 'multi' # in case someone needs to know that multidiags is running it print "jfp about to run_diags on",o['vars'],len(o.get('obs',[])),o.get('obs',[]) - t0 = time.time() o.finalizeOpts() - o['modobs_names'] = run_diags(o.clone()) - # re o.clone(): diags.py should leave the options alone, but it doesn't: it replaces o['varopts'], at least. - trun = time.time() - t0 - print "jfp run_diags took",trun,"seconds" + + # single-processing: + newopts = map( md_run_diags, newopts ) + + # multi-processing: + # doesn't work, and I tried it two ways. I think VCS, Qt, or the like crashes when you + # try to make a plot in a forked process. The multiprocessing module won't spawn for + # Unix-like systems except in Python >=3.4, which has set_start_method('spawn'). +# newopts = mp_pool.map( md_run_diags, newopts ) +# thse two lines aren't necessary, but I was desperate enough to try them: +# mp_pool.close() +# mp_pool.join() +## +# queues = [] +# procs = [] +# multiprocessing.set_start_method('spawn') +# for i,o in enumerate(newopts): +# queues.append( Queue() ) +# procs.append( Process( target=md_run_diags, args=(o,queues[i]) ) ) +# procs[i].start() +# for i,o in enumerate(newopts): +# procs[i].join() + setup_viewer( newopts ) def multidiags( opts ): @@ -303,8 +331,16 @@ def multidiags( opts ): nopts.extend(opt) else: logger.error("cannot understand opt=%s",opt) + mp_pool = Pool(processes = 6) for opt in nopts: - multidiags1(opt) + multidiags1( opt, mp_pool ) + +def p_run_diags( opt ): + """run_diags() but run as a separate process. returns the process, the caller should + join it.""" + p = Process( target=run_diags, args=(opt,) ) + p.start() + return p def finalize_modelobs(opt): """For the model and obs options in opt, copies any modelpath or obspath option into its model or obs From b212adb59f30a0cde8d573eed6fb4b6c285a30c4 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Fri, 16 Dec 2016 13:10:05 -0800 Subject: [PATCH 02/24] Clean up a path with ~ in it before checking whether a file exists (for model,obs,output) - because os.path.exists() doesn't understand '~' in a path. --- src/python/frontend/options.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/frontend/options.py b/src/python/frontend/options.py index 6e1cb1fa..9ced9555 100644 --- a/src/python/frontend/options.py +++ b/src/python/frontend/options.py @@ -515,7 +515,7 @@ def verifyOptions(self): logging.critical('Each dataset must have a path provided') quit() # check if the path exists - if not os.path.exists(self._opts['model'][i]['path']): + if not os.path.exists(os.path.expanduser(self._opts['model'][i]['path'])): logging.critical('Path - %s - does not exist', self._opts['model'][i]['path']) quit() if len(self._opts['obs']) != 0: @@ -523,7 +523,7 @@ def verifyOptions(self): if self._opts['obs'][i]['path'] == None or self._opts['obs'][i]['path'] == '': logging.critical('Each dataset must have a path provided') quit() - if not os.path.exists(self._opts['obs'][i]['path']): + if not os.path.exists(os.path.expanduser(self._opts['obs'][i]['path'])): logging.critical('Obs Path - %s - does not exist', self._opts['obs'][i]['path']) quit() # if(self._opts['package'] == None): @@ -533,7 +533,7 @@ def verifyOptions(self): # A path is guaranteed, even if it is just /tmp. So check for it # We shouldn't get here anyway. This is primarily in case something gets postpended to the user-specified outputdir # in options(). Currently that happens elsewhere, but seems like a raesonable check to keep here anyway. - if not os.path.exists(self._opts['output']['outputdir']): + if not os.path.exists(os.path.expanduser(self._opts['output']['outputdir'])): logging.critical('output directory %s does not exist', self._opts['output']['outputdir']) quit() From a8f58667b34ef35140e44fd10de2fdef971e0e5d Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Fri, 16 Dec 2016 13:15:55 -0800 Subject: [PATCH 03/24] Implement multiprocessing. I need to spawn a process, so the subprocesses have to be run in a shell with Popen(). Communication is done with a temporary file and through the stdout pipe. There remains some non-functional commented-out code to do this other ways (which would be better is spawning, rather than forking, were allowed those ways.) --- src/python/frontend/diags.py | 25 +++++++++++++++-------- src/python/frontend/multidiags.py | 34 ++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/src/python/frontend/diags.py b/src/python/frontend/diags.py index 1db88310..2ebbe3e5 100755 --- a/src/python/frontend/diags.py +++ b/src/python/frontend/diags.py @@ -83,6 +83,8 @@ def setManualColormap(canvas=None, level=0): def getNames(opts, model, obs): """ The purpose of this kludge is to get the names of the model and obs specified on the command line for output on the graphic. + These particular names should be sufficiently unique and human-readable, and + reasonably short. Each of model and obs should be a list of instances of basic_filetable. This function returns a dictionary of the form {'model':modelname, 'obs':obsname} where modelname and obsname are strings.""" @@ -470,6 +472,7 @@ def run_diags( opts ): # If this were called from multidiags, the names dictionary would be helpful. In particular, # it will help to not have to re-open a file to re-compute the case name for the model. + print "multidiags start here names=", names, "multidiags stop here" return names @@ -889,14 +892,20 @@ def makeplots(res, vcanvas, vcanvas2, varid, frname, plot, package, opts, displa if __name__ == '__main__': print "UV-CDAT Diagnostics, command-line version" print ' '.join(sys.argv) - try: - irunby = sys.argv.index('--runby') - runby = sys.argv[irunby+1] - o = Options(runby=runby) - except ValueError: - o = Options() + if sys.argv[1][:10]=='--optfile=': + # special case. All options are described in a file containing a pickled Options object. + optfile = open( sys.argv[1][10:], 'rb' ) + o = pickle.load(optfile) + optfile.close() + else: + # Normal case, with command-line options. Their interpretation differs slightly according + # to how this was run, as specified in the '--runby' option. + try: + irunby = sys.argv.index('--runby') + runby = sys.argv[irunby+1] + o = Options(runby=runby) + except ValueError: + o = Options() o.parseCmdLine() o.verifyOptions() - #print o._opts['levels'] - #print o._opts['displayunits'] run_diags(o) diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index 155e91a4..c7aecd13 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -1,12 +1,14 @@ # Run multiple diagnostics. The user only specifies paths and a collection name. # The collection name is a key into multimaster.py, or (>>>> TO DO >>>>) another file provided by the user. -import logging, pdb, importlib, time, cProfile +import logging, os, pdb, importlib, time, cProfile, pickle, tempfile, subprocess from pprint import pprint from itertools import groupby from multiprocessing import Pool, current_process, Process, Queue import multiprocessing +from tempfile import NamedTemporaryFile from os import getpid +from subprocess import PIPE from metrics.frontend.multimaster import * # this file is just a demo import metrics.frontend.multimaster as multimaster import metrics.frontend.defines as defines @@ -263,7 +265,7 @@ def md_run_diags( opt, queue1=None ): #queue1.put(opt) return opt -def multidiags1( opt, mp_pool ): +def multidiags1( opt ): """The input opt is a single Options instance, possibly with lists or collection names in place of real option values. This function expands out such lists or collection names, giving us a list of simple Options instances. @@ -282,20 +284,39 @@ def multidiags1( opt, mp_pool ): o.finalizeOpts() # single-processing: - newopts = map( md_run_diags, newopts ) + #newopts = map( md_run_diags, newopts ) + #for i,o in enumerate(newopts): + # newopts[i] = md_run_diags(o) # multi-processing: # doesn't work, and I tried it two ways. I think VCS, Qt, or the like crashes when you # try to make a plot in a forked process. The multiprocessing module won't spawn for # Unix-like systems except in Python >=3.4, which has set_start_method('spawn'). +#mp_pool = Pool(processes = 6) # normally done in multidiags() and passed to multidiags1() # newopts = mp_pool.map( md_run_diags, newopts ) # thse two lines aren't necessary, but I was desperate enough to try them: # mp_pool.close() # mp_pool.join() ## # queues = [] -# procs = [] -# multiprocessing.set_start_method('spawn') + procs = [] + tempfilens = [] + for i,o in enumerate(newopts): + f = NamedTemporaryFile(delete=False) + fpath = os.path.realpath(f.name) + pickle.dump( o, f ) # easier than generating a full-length command line + f.close() + cmd = "diags --optfile=%s" % fpath # don't change this syntax, c.f. bottom of diags.py + p = subprocess.Popen([cmd],shell=True,stdout=PIPE,stderr=PIPE) + tempfilens.append(fpath) + procs.append(p) + for i,o in enumerate(newopts): + stout,sterr = procs[i].communicate() # waits for process to finish, then reads pipes + namstr = stout[ stout.find("multidiags start here names=")+28 : + stout.find("multidiags stop here") ] + os.remove(tempfilens[i]) + o['modobs_names'] = eval( namstr ) + # for i,o in enumerate(newopts): # queues.append( Queue() ) # procs.append( Process( target=md_run_diags, args=(o,queues[i]) ) ) @@ -331,9 +352,8 @@ def multidiags( opts ): nopts.extend(opt) else: logger.error("cannot understand opt=%s",opt) - mp_pool = Pool(processes = 6) for opt in nopts: - multidiags1( opt, mp_pool ) + multidiags1( opt ) def p_run_diags( opt ): """run_diags() but run as a separate process. returns the process, the caller should From b6e26a56a27cecc5f45f7cfb8cf90780273b241d Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Fri, 16 Dec 2016 16:27:42 -0800 Subject: [PATCH 04/24] Minor bug fixes affecting only multidiags. --- src/python/frontend/diags.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/frontend/diags.py b/src/python/frontend/diags.py index 2ebbe3e5..c13030f6 100755 --- a/src/python/frontend/diags.py +++ b/src/python/frontend/diags.py @@ -158,7 +158,7 @@ def run_diags( opts ): # Setup filetable arrays modelfts = [] obsfts = [] - names = {'model':'', 'obs':''} + names = {} for i in range(len(opts['model'])): ft = path2filetable(opts, modelid=i) if ft._id.nickname=='': @@ -327,7 +327,8 @@ def run_diags( opts ): logger.warning('Could not find any of the requested variables %s among %s', opts['vars'], pclass.list_variables(modelfts,obsfts,sname) ) logger.warning("among %s", variables) - return {} + print "multidiags start here names=", names, "multidiags stop here" + return names # Ok, start the next layer of work - seasons and regions # loop over the seasons for this plot From e1086f15e183179c0b88a763431b1aa6563e004e Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Fri, 16 Dec 2016 16:28:34 -0800 Subject: [PATCH 05/24] Re-organize the Popen-based multiprocessing so as to put a maximum on the number of processes to be run simultaneously. For now, the maximum is hard-coded. --- src/python/frontend/multidiags.py | 144 +++++++++++++++++------------- 1 file changed, 82 insertions(+), 62 deletions(-) diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index c7aecd13..b358188c 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -259,18 +259,68 @@ def expand_lists_collections( opt, okeys=None ): return flattened def md_run_diags( opt, queue1=None ): - #curr_proc = current_process() - #curr_proc.daemon = False opt['modobs_names'] = run_diags(opt.clone()) # run_diags messes with its input arg, though it shouldn't - #queue1.put(opt) return opt +def run_next(): + """Runs a diags process on the next option in the newopts list.""" + global newopts + global optidx + global procs + global active + global tempfilens + if None in active: + active[active.index(None)] = optidx + o = newopts[optidx] + f = NamedTemporaryFile(delete=False) + fpath = os.path.realpath(f.name) + pickle.dump( o, f ) # easier than generating a full-length command line + f.close() + cmd = "diags --optfile=%s" % fpath # don't change this syntax, c.f. bottom of diags.py + p = subprocess.Popen([cmd],shell=True,stdout=PIPE,stderr=PIPE) + procs[optidx] = p + tempfilens[optidx] = fpath + optidx += 1 + elif optidx=3.4, which has set_start_method('spawn'). -#mp_pool = Pool(processes = 6) # normally done in multidiags() and passed to multidiags1() -# newopts = mp_pool.map( md_run_diags, newopts ) -# thse two lines aren't necessary, but I was desperate enough to try them: -# mp_pool.close() -# mp_pool.join() -## -# queues = [] - procs = [] - tempfilens = [] - for i,o in enumerate(newopts): - f = NamedTemporaryFile(delete=False) - fpath = os.path.realpath(f.name) - pickle.dump( o, f ) # easier than generating a full-length command line - f.close() - cmd = "diags --optfile=%s" % fpath # don't change this syntax, c.f. bottom of diags.py - p = subprocess.Popen([cmd],shell=True,stdout=PIPE,stderr=PIPE) - tempfilens.append(fpath) - procs.append(p) - for i,o in enumerate(newopts): - stout,sterr = procs[i].communicate() # waits for process to finish, then reads pipes - namstr = stout[ stout.find("multidiags start here names=")+28 : - stout.find("multidiags stop here") ] - os.remove(tempfilens[i]) - o['modobs_names'] = eval( namstr ) - -# for i,o in enumerate(newopts): -# queues.append( Queue() ) -# procs.append( Process( target=md_run_diags, args=(o,queues[i]) ) ) -# procs[i].start() -# for i,o in enumerate(newopts): -# procs[i].join() + procs = [None] * len(newopts) + active = [None] * 2 # For testing, this is MAX_N_PROCS, the maximimum number of simultaneous processes + tempfilens = [None] * len(newopts) + optidx = 0 + while optidx Date: Mon, 19 Dec 2016 15:04:03 -0800 Subject: [PATCH 06/24] No more use of a temporary file in multidiags. In issuing a command for Popen, it pickles its Options object to a string, which is passed to diags.py in the shell command line. --- src/python/frontend/diags.py | 12 ++++---- src/python/frontend/multidiags.py | 46 +++++++++++-------------------- 2 files changed, 22 insertions(+), 36 deletions(-) diff --git a/src/python/frontend/diags.py b/src/python/frontend/diags.py index c13030f6..63e78c3f 100755 --- a/src/python/frontend/diags.py +++ b/src/python/frontend/diags.py @@ -892,12 +892,12 @@ def makeplots(res, vcanvas, vcanvas2, varid, frname, plot, package, opts, displa if __name__ == '__main__': print "UV-CDAT Diagnostics, command-line version" - print ' '.join(sys.argv) - if sys.argv[1][:10]=='--optfile=': - # special case. All options are described in a file containing a pickled Options object. - optfile = open( sys.argv[1][10:], 'rb' ) - o = pickle.load(optfile) - optfile.close() + print repr(repr(' '.join(sys.argv))) + if sys.argv[1][:10]=='--pickopt=': + # special case. All options are described in the argument, a pickled Options object. + pickopt = sys.argv[1][10:] + pickopt = pickopt.replace('jfpsingleq','\'') + o = pickle.loads(pickopt) else: # Normal case, with command-line options. Their interpretation differs slightly according # to how this was run, as specified in the '--runby' option. diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index b358188c..1c309ec4 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -18,6 +18,9 @@ from output_viewer.index import OutputIndex, OutputPage, OutputGroup, OutputRow, OutputFile, OutputMenu logger = logging.getLogger(__name__) +MAX_N_PROCS = 2 # maximum number of simultaneous processes. + # These are run with Popen or the multiprocessing module. +ARG_MAX = subprocess.check_output(['getconf', 'ARG_MAX']) # max length of a command line def merge_option_with_its_defaults( opt, diags_collection ): """If opt, an Options instance, has a default_opts option set, this will import the @@ -268,18 +271,19 @@ def run_next(): global optidx global procs global active - global tempfilens + #old global tempfilens if None in active: active[active.index(None)] = optidx o = newopts[optidx] - f = NamedTemporaryFile(delete=False) - fpath = os.path.realpath(f.name) - pickle.dump( o, f ) # easier than generating a full-length command line - f.close() - cmd = "diags --optfile=%s" % fpath # don't change this syntax, c.f. bottom of diags.py + ostr = pickle.dumps( o, 0 ) # easier than generating a full-length command line + ostr = ostr.replace('\'','jfpsingleq') + ostr = '\'' + ostr + '\'' + cmd = r"diags --pickopt=%s" % ostr # don't change this syntax, c.f. bottom of diags.py + if len(cmd)>=ARG_MAX: + logger.critical("generated command line is too long for the operating sytem!") + quit() p = subprocess.Popen([cmd],shell=True,stdout=PIPE,stderr=PIPE) procs[optidx] = p - tempfilens[optidx] = fpath optidx += 1 elif optidx Date: Mon, 19 Dec 2016 16:11:47 -0800 Subject: [PATCH 07/24] When there will be only one process, use the simpler single-processing code, which is no more than a function call. --- src/python/frontend/multidiags.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index 1c309ec4..aa86ec72 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -337,20 +337,21 @@ def multidiags1( opt ): print "jfp about to run_diags on",o['vars'],len(o.get('obs',[])),o.get('obs',[]) o.finalizeOpts() - # single-processing: - #for i,o in enumerate(newopts): - # o['modobs_names'] = run_diags(o.clone()) # run_diags messes with its input arg, though it shouldn't - - # multi-processing: - procs = [None] * len(newopts) - active = [None] * MAX_N_PROCS # MAX_N_PROCS is defined at the top of this file. - optidx = 0 - while optidx Date: Tue, 20 Dec 2016 14:36:48 -0800 Subject: [PATCH 08/24] Minor changes for better names and ad-hoc time units. --- src/python/computation/reductions.py | 3 ++- src/python/fileio/findfiles.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/python/computation/reductions.py b/src/python/computation/reductions.py index 9b03456d..d0957472 100644 --- a/src/python/computation/reductions.py +++ b/src/python/computation/reductions.py @@ -2856,7 +2856,8 @@ def run_cdscan( fam, famfiles, cache_path=None ): # get here. # Another problem is that units stuck in the long_name sometimes are # nonstandard. So fix them! - if hasattr(f['time'],'long_name'): + if hasattr(f['time'],'long_name') and\ + getattr(f['time'],'long_name',None) not in ['time',None,'']: time_units = f['time'].long_name else: time_units = 'days' # probably wrong but we can't go on without something diff --git a/src/python/fileio/findfiles.py b/src/python/fileio/findfiles.py index 931b63eb..743acdb1 100755 --- a/src/python/fileio/findfiles.py +++ b/src/python/fileio/findfiles.py @@ -209,7 +209,9 @@ def _getdatafiles( self, root, filt ): def shortest_name(self): if len(self._filt.mystr())>3: - return self._filt.mystr() + shortname = self._filt.mystr() + shortname = shortname.strip('-_') + return shortname else: return self.short_name() @@ -226,6 +228,7 @@ def short_name(self): else: shortname = ','.join([os.path.basename(str(r)) for r in self._root]) # + shortname = shortname.strip('-_') return shortname def long_name(self): From 6633d58bcf7ca5d3138aea78381214015560d0dd Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Tue, 20 Dec 2016 14:39:04 -0800 Subject: [PATCH 09/24] clean up and improve naming --- src/python/frontend/diags.py | 4 +- src/python/frontend/multidiags.py | 74 ++++++------------------------- 2 files changed, 16 insertions(+), 62 deletions(-) diff --git a/src/python/frontend/diags.py b/src/python/frontend/diags.py index 63e78c3f..997f584b 100755 --- a/src/python/frontend/diags.py +++ b/src/python/frontend/diags.py @@ -109,7 +109,8 @@ def get_model_case(filetable): nicknames['obs'] = opts['obs'][0]['name'] elif ft2 is not None: #nicknames['obs'] = ft2.source().split('_')[-1] - nicknames['obs'] = ft2.source().split('_')[0] + #nicknames['obs'] = ft2.source().split('_')[0] #doesn't work on HadISST_CL, which isn't HadISST + nicknames['obs'] = ft2.source() return nicknames def setnum( setname ): @@ -871,6 +872,7 @@ def makeplots(res, vcanvas, vcanvas2, varid, frname, plot, package, opts, displa descr=descr, vname=vname, more_id='combined' ) else: source_descr2 = list(set(source_descr2)) + source_descr2.sort() fnamepng,fnamesvg,fnamepdf = form_filename( frnamebase, ('png','svg','pdf'), modobs=list(source_descr2), more_id='combined' ) diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index aa86ec72..a0d7db98 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -1,6 +1,8 @@ # Run multiple diagnostics. The user only specifies paths and a collection name. # The collection name is a key into multimaster.py, or (>>>> TO DO >>>>) another file provided by the user. +# ---------------- section 0: Imports and global initializations ---------------- + import logging, os, pdb, importlib, time, cProfile, pickle, tempfile, subprocess from pprint import pprint from itertools import groupby @@ -18,10 +20,12 @@ from output_viewer.index import OutputIndex, OutputPage, OutputGroup, OutputRow, OutputFile, OutputMenu logger = logging.getLogger(__name__) -MAX_N_PROCS = 2 # maximum number of simultaneous processes. +MAX_N_PROCS = 4 # maximum number of simultaneous processes. # These are run with Popen or the multiprocessing module. ARG_MAX = subprocess.check_output(['getconf', 'ARG_MAX']) # max length of a command line +# ---------------- section 1: Expand names and merge options. ---------------- + def merge_option_with_its_defaults( opt, diags_collection ): """If opt, an Options instance, has a default_opts option set, this will import the other Options instance specified by default_opts and present in diags_collection, @@ -66,54 +70,6 @@ def expand_and_merge_with_defaults( opt ): opts.append(newopt) return opts -def merge_all_options_old( opt ): - """Merge the supplied Options object with other Options instances. Conflicts are resolved by - ordering them in this priority: - opt, opt['default_opts'], my_opts, my_opts['default_opts'], diags_opts, options_defaults - At present the use of 'default_opts' cannot be recursive.""" - try: - import my_opts # user can put my_opts.py in his PYTHONPATH - myopts = my_opts.my_opts - except: - my_opts = None - myopts = None - try: - import metrics.frontend.diags_opts - diagsopts = diags_opts.diags_opts - except: - diags_opts = None - diagsopts = None - # The 'sets' or 'colls' option identifies a member (or more) of diags_collection, whose values - # are Options instances. We have to expand it now, to a list 'opts' of real Options instances, - # in order to perform the merger. - # Similarly, if any 'model' or 'obs' option be a key into a collection, we have to expand it - # now in order to perform a proper merge. - if opt.get('sets') is None: # later use a specialized option name, e.g. 'collec' <<<< - opts = [opt] - else: - opts = [] - for dset in opt['sets']: - # Each set in opt['sets'] may be a list. And opt['sets'] may be a list of length>1. - # Either way, we end out with a list of Options instances which should be merged with - # opt (opt having priority) but should not be merged together. - if isinstance(diags_collection[dset],Options): - merge_option_with_its_defaults(diags_collection[dset],diags_collection) - opts.append(opt) - elif type(diags_collection[dset]) is list: - for optset in diags_collection[dset]: - merge_option_with_its_defaults(optset,diags_collection) - newopt = opt.clone() - opts.append(newopt) - # (done in the newer version of this function): This should also be done for the myopts, diagsopts, etc. - if my_opts is not None: - merge_option_with_its_defaults( myopts, my_opts.diags_collection ) - for op in opts: - merge_option_with_its_defaults( op, diags_collection ) - op.merge( myopts ) - op.merge( diagsopts ) - op.merge( options_defaults ) - return opts - def merge_all_options( opt ): """Merge the supplied Options object with other Options instances. When two Options instances are merged and only one has a value for an option, then that @@ -261,9 +217,7 @@ def expand_lists_collections( opt, okeys=None ): flattened = [ o for ops in expanded_opts for o in ops ] return flattened -def md_run_diags( opt, queue1=None ): - opt['modobs_names'] = run_diags(opt.clone()) # run_diags messes with its input arg, though it shouldn't - return opt +# ---------------- section 2: Higher-level run functions, including multiprocessing ---------------- def run_next(): """Runs a diags process on the next option in the newopts list.""" @@ -271,7 +225,6 @@ def run_next(): global optidx global procs global active - #old global tempfilens if None in active: active[active.index(None)] = optidx o = newopts[optidx] @@ -384,13 +337,6 @@ def multidiags( opts ): for opt in nopts: multidiags1( opt ) -def p_run_diags( opt ): - """run_diags() but run as a separate process. returns the process, the caller should - join it.""" - p = Process( target=run_diags, args=(opt,) ) - p.start() - return p - def finalize_modelobs(opt): """For the model and obs options in opt, copies any modelpath or obspath option into its model or obs option dictionary's 'path' value. Also if the model or obs option is a dictionary, we change that to @@ -408,6 +354,8 @@ def finalize_modelobs(opt): opt[key][i]['path'] = opt[pathkey][i] return opt +# ---------------- section 3: Interface to viewer ---------------- + def organize_opts_for_viewer( opts ): """Basically we have to re-organize the opts list into something structured by variable, season, obs set, etc. More precisely, the organization we need is lists nested as follows: @@ -547,7 +495,9 @@ def setup_viewer_2( optsnested, vardesc ): #...Note that this only supports [model,obs] not [model1,model2]. Doing both shouldn't be much harder. modelname_path = os.path.basename(os.path.normpath(modelpath)) modelname = opt['modobs_names'].get( 'model', modelname_path ) - fname = form_filename( rootname, 'png', modobs=[modelname,obsname], more_id='combined' ) + modobs = [modelname,obsname] + modobs.sort() + fname = form_filename( rootname, 'png', modobs=modobs, more_id='combined' ) path = os.path.join( opt['output']['outputdir'], fname ) cols.append( OutputFile(path, title="{season}".format(season=season)) ) rowtitle = vname+regname+varoptname @@ -559,6 +509,8 @@ def setup_viewer_2( optsnested, vardesc ): index.toJSON(os.path.join(opt['output']['outputdir'], opt['package'].lower(), "index.json")) +# ---------------- section 4: Shell (command-line) driver ---------------- + if __name__ == '__main__': print "UV-CDAT Diagnostics, Experimental Multi-Diags version" print ' '.join(sys.argv) From 0db55e447789bd4251412638e0565f887df6ae43 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Tue, 20 Dec 2016 14:39:38 -0800 Subject: [PATCH 10/24] Add a full-scale diagnostic collection '7' like that in amwgmaster.py. --- src/python/frontend/multimaster.py | 70 ++++++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/src/python/frontend/multimaster.py b/src/python/frontend/multimaster.py index bb16eb8b..b26a4cc0 100644 --- a/src/python/frontend/multimaster.py +++ b/src/python/frontend/multimaster.py @@ -26,14 +26,22 @@ var_collection['MyVars'] = ['FLUT', 'LWCF'] obs_collection['MyObs'] = ['ISCCP', 'CERES', 'NCEP'] -obs_collection['ISCCP'] = {'filter':"f_startswith('ISCCP_')",'climos':'yes','name':None} -obs_collection['CERES'] = {'filter':"f_and( f_startswith('CERES_'), f_not(f_contains('EBAF')))",'climos':'yes','name':None} -obs_collection['ECMWF'] = {'filter':"f_startswith('ECMWF')",'climos':'yes','name':None} -obs_collection['NCEP'] = {'filter':"f_startswith('NCEP')",'climos':'yes','name':None} -obs_collection['ERA40'] = {'filter':"f_startswith('ERA40')",'climos':'yes','name':None} -obs_collection['AIRS'] = {'filter':"f_startswith('AIRS')",'climos':'yes','name':None} -obs_collection['JRA25'] = {'filter':"f_startswith('JRA25')",'climos':'yes','name':None} -obs_collection['WILLMOTT'] = {'filter':"f_startswith('WILLMOTT')", 'climos':'yes', 'name':None} +obs_collection['AIRS'] = {'filter':"f_startswith('AIRS')",'climos':'yes','name':None} +obs_collection['CERES'] = {'filter':"f_and( f_startswith('CERES_'), f_not(f_contains('EBAF')))",'climos':'yes','name':None} +obs_collection['CERES2'] = {'filter':"f_startswith('CERES2')",'climos':'yes','name':None} +obs_collection['ECMWF'] = {'filter':"f_startswith('ECMWF')",'climos':'yes','name':None} +obs_collection['ERA40'] = {'filter':"f_startswith('ERA40')",'climos':'yes','name':None} +obs_collection['ERBE'] = {'filter':"f_startswith('ERBE')",'climos':'yes','name':None} +obs_collection['GPCP'] = {'filter':"f_startswith('GPCP')",'climos':'yes','name':None} +obs_collection['HadISST_CL'] = {'filter':"f_startswith('HadISST_CL_')",'climos':'yes','name':None} +obs_collection['HadISST_PD'] = {'filter':"f_startswith('HadISST_PD_')",'climos':'yes','name':None} +obs_collection['HadISST_PI'] = {'filter':"f_startswith('HadISST_PI_')",'climos':'yes','name':None} +obs_collection['ISCCP'] = {'filter':"f_startswith('ISCCP_')",'climos':'yes','name':None} +obs_collection['JRA25'] = {'filter':"f_startswith('JRA25')",'climos':'yes','name':None} +obs_collection['LARYEA'] = {'filter':"f_startswith('LARYEA')",'climos':'yes','name':None} +obs_collection['NCEP'] = {'filter':"f_startswith('NCEP')",'climos':'yes','name':None} +obs_collection['SSMI'] = {'filter':"f_and( f_startswith('SSMI_'), f_not(f_contains('SEAICE')))",'climos':'yes','name':None} +obs_collection['WILLMOTT']={'filter':"f_startswith('WILLMOTT')", 'climos':'yes', 'name':None} model_collection['generic'] = {'climos':'yes', 'name':None} diags_collection['MyDefaults'] = Options( desc = 'default options to be incorporated into other collections', @@ -79,6 +87,11 @@ Options( vars=['U','V'], obs=['ERA40'], seasons=['ANN', 'DJF', 'JJA'], default_opts='4base' ) ] +diags_collection['7base'] = Options( + sets=['7'], desc='Polar contour and vector plots of DJF, JJA and ANN means', + seasons=['DJF', 'JJA', 'ANN'], regions=['N_Hemisphere_Land', 'S_Hemisphere_Land'], + package='AMWG', model='generic', default_opts='MyDefaults' ) + diags_collection['7s'] = [ Options( sets=['7'], vars=['TREFHT'], obs='WILLMOTT', model=['generic'], seasons=['DJF', 'JJA', 'ANN'], package='AMWG', default_opts='MyDefaults', @@ -90,6 +103,47 @@ desc='Polar contour plots of DJF, JJA, and ANN means' ) ] +diags_collection['7'] = [ + Options( + vars=['TREFHT'], obs=['WILLMOTT'], default_opts='7base' ), + Options( + vars=['PS','PSL', 'TS', 'SURF_WIND'], obs=['NCEP'], default_opts='7base' ), + Options( + vars=['PSL'], obs=['JRA25'], default_opts='7base' ), + Options( + vars=['Z3'], obs=['ECMWF', 'JRA25', 'NCEP', 'ERA40'], varopts=['500'], default_opts='7base' ), + Options( + vars=['SHFLX', 'QFLX', 'FLNS', 'FSNS'], obs=['LARYEA'], default_opts='7base' ), + Options( + vars=['ALBEDO', 'ALBEDOC', 'FLUT', 'FLUTC', 'FSNTOA', 'FSNTOAC'], obs=['CERES2', 'CERES'], default_opts='7base' ), + Options( + vars=['FLUT', 'FLUTC', 'FSNTOA', 'FSNTOAC'], obs=['ERBE'], default_opts='7base' ), + Options( + vars=['FLNS', 'FLDS', 'FLDSC', 'FLNSC', 'FSDS', 'FSDSC', 'FSNS', 'FSNSC'], obs=['ISCCP'], default_opts='7base' ), + Options( + vars=['CLDLOW_VISIR','CLDMED_VISIR','CLDHGH_VISIR','CLDTOT_VISR'], obs=['ISCCP'], regions=['N_Hemisphere_Land'], + default_opts='7base' ), +# ... 'modelvar':'CLDLOW'} +# ... 'modelvar':'CLDMED'} +# ... 'modelvar':'CLDHGH'} +# ... 'modelvar':'CLDTOT'} + Options( + vars=['CLDMED','CLDHGH'], obs=['ISCCP', 'CLOUDSAT'], default_opts='7base' ), + Options( + vars=['CLDLOW','CLDTOT'], obs=['ISCCP', 'WARREN', 'CLOUDSAT'], default_opts='7base' ), + Options( + vars=['ICEFRAC'], obs=['SSMI', 'HadISST_CL', 'HadISST_PD', 'HadISST_PI'], default_opts='7base' ), + Options( + vars=['SST'], obs=['HadISST_CL', 'HadISST_PD', 'HadISST_PI'], default_opts='7base' ), + Options( + vars=['PRECT'], obs=['GPCP'], default_opts='7base' ), +] + +diags_collection['7t'] = [ + Options( + vars=['ICEFRAC'], obs=['SSMI', 'HadISST_CL', 'HadISST_PD', 'HadISST_PI'], default_opts='7base' ), +] + # Given a keyword, if it identifies a member of one of the collections, this will tell you From ae725a52405feb529e806fc6058ba886aa24ad34 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Tue, 27 Dec 2016 15:41:22 -0800 Subject: [PATCH 11/24] rearrange diags_collection['7']; no functional change --- src/python/frontend/multimaster.py | 43 ++++++++++-------------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/src/python/frontend/multimaster.py b/src/python/frontend/multimaster.py index b26a4cc0..83f32a88 100644 --- a/src/python/frontend/multimaster.py +++ b/src/python/frontend/multimaster.py @@ -104,40 +104,25 @@ ] diags_collection['7'] = [ - Options( - vars=['TREFHT'], obs=['WILLMOTT'], default_opts='7base' ), - Options( - vars=['PS','PSL', 'TS', 'SURF_WIND'], obs=['NCEP'], default_opts='7base' ), - Options( - vars=['PSL'], obs=['JRA25'], default_opts='7base' ), - Options( - vars=['Z3'], obs=['ECMWF', 'JRA25', 'NCEP', 'ERA40'], varopts=['500'], default_opts='7base' ), - Options( - vars=['SHFLX', 'QFLX', 'FLNS', 'FSNS'], obs=['LARYEA'], default_opts='7base' ), - Options( - vars=['ALBEDO', 'ALBEDOC', 'FLUT', 'FLUTC', 'FSNTOA', 'FSNTOAC'], obs=['CERES2', 'CERES'], default_opts='7base' ), - Options( - vars=['FLUT', 'FLUTC', 'FSNTOA', 'FSNTOAC'], obs=['ERBE'], default_opts='7base' ), - Options( - vars=['FLNS', 'FLDS', 'FLDSC', 'FLNSC', 'FSDS', 'FSDSC', 'FSNS', 'FSNSC'], obs=['ISCCP'], default_opts='7base' ), - Options( - vars=['CLDLOW_VISIR','CLDMED_VISIR','CLDHGH_VISIR','CLDTOT_VISR'], obs=['ISCCP'], regions=['N_Hemisphere_Land'], + Options( vars=['TREFHT'], obs=['WILLMOTT'], default_opts='7base' ), + Options( vars=['PS','PSL', 'TS', 'SURF_WIND'], obs=['NCEP'], default_opts='7base' ), + Options( vars=['PSL'], obs=['JRA25'], default_opts='7base' ), + Options( vars=['Z3'], obs=['ECMWF', 'JRA25', 'NCEP', 'ERA40'], varopts=['500'], default_opts='7base' ), + Options( vars=['SHFLX', 'QFLX', 'FLNS', 'FSNS'], obs=['LARYEA'], default_opts='7base' ), + Options( vars=['ALBEDO', 'ALBEDOC', 'FLUT', 'FLUTC', 'FSNTOA', 'FSNTOAC'], obs=['CERES2', 'CERES'], default_opts='7base' ), + Options( vars=['FLUT', 'FLUTC', 'FSNTOA', 'FSNTOAC'], obs=['ERBE'], default_opts='7base' ), + Options( vars=['FLNS', 'FLDS', 'FLDSC', 'FLNSC', 'FSDS', 'FSDSC', 'FSNS', 'FSNSC'], obs=['ISCCP'], default_opts='7base' ), + Options( vars=['CLDLOW_VISIR','CLDMED_VISIR','CLDHGH_VISIR','CLDTOT_VISR'], obs=['ISCCP'], regions=['N_Hemisphere_Land'], default_opts='7base' ), + Options( vars=['CLDMED','CLDHGH'], obs=['ISCCP', 'CLOUDSAT'], default_opts='7base' ), + Options( vars=['CLDLOW','CLDTOT'], obs=['ISCCP', 'WARREN', 'CLOUDSAT'], default_opts='7base' ), + Options( vars=['ICEFRAC'], obs=['SSMI', 'HadISST_CL', 'HadISST_PD', 'HadISST_PI'], default_opts='7base' ), + Options( vars=['SST'], obs=['HadISST_CL', 'HadISST_PD', 'HadISST_PI'], default_opts='7base' ), + Options( vars=['PRECT'], obs=['GPCP'], default_opts='7base' ) ] # ... 'modelvar':'CLDLOW'} # ... 'modelvar':'CLDMED'} # ... 'modelvar':'CLDHGH'} # ... 'modelvar':'CLDTOT'} - Options( - vars=['CLDMED','CLDHGH'], obs=['ISCCP', 'CLOUDSAT'], default_opts='7base' ), - Options( - vars=['CLDLOW','CLDTOT'], obs=['ISCCP', 'WARREN', 'CLOUDSAT'], default_opts='7base' ), - Options( - vars=['ICEFRAC'], obs=['SSMI', 'HadISST_CL', 'HadISST_PD', 'HadISST_PI'], default_opts='7base' ), - Options( - vars=['SST'], obs=['HadISST_CL', 'HadISST_PD', 'HadISST_PI'], default_opts='7base' ), - Options( - vars=['PRECT'], obs=['GPCP'], default_opts='7base' ), -] diags_collection['7t'] = [ Options( From d99473be3b24842d9f1ff0c0d76c2f964520a806 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 12 Jan 2017 09:06:15 -0800 Subject: [PATCH 12/24] When adding a file to a filetable, be more flexible in determining whether its file format matches the rest. --- src/python/fileio/filetable.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/python/fileio/filetable.py b/src/python/fileio/filetable.py index 28a55ddb..13499bf8 100755 --- a/src/python/fileio/filetable.py +++ b/src/python/fileio/filetable.py @@ -266,7 +266,8 @@ def addfile( self, filep, options): filesupp = get_datafile_filefmt( dfile, options ) if self.filefmt is None: self.filefmt = filesupp.name - elif self.filefmt!= filesupp.name: + elif self.filefmt!= filesupp.name and self.filefmt.find(filesupp.name)<0 and\ + filesupp.name.find(self.filefmt)<0: self.filefmt = "various" vars = filesupp.interesting_variables() if len(vars)>0: From 9c9b61cf48f5fb3dfbf5f0ffc658c4380a74a926 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 12 Jan 2017 09:09:04 -0800 Subject: [PATCH 13/24] Stop sorting the source information used for the compbined plot filenames. It makes the filenames more repeatable, but would require us to change the present ctest baselines. --- src/python/frontend/diags.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/python/frontend/diags.py b/src/python/frontend/diags.py index 997f584b..9881ab2c 100755 --- a/src/python/frontend/diags.py +++ b/src/python/frontend/diags.py @@ -328,7 +328,6 @@ def run_diags( opts ): logger.warning('Could not find any of the requested variables %s among %s', opts['vars'], pclass.list_variables(modelfts,obsfts,sname) ) logger.warning("among %s", variables) - print "multidiags start here names=", names, "multidiags stop here" return names # Ok, start the next layer of work - seasons and regions @@ -474,7 +473,6 @@ def run_diags( opts ): # If this were called from multidiags, the names dictionary would be helpful. In particular, # it will help to not have to re-open a file to re-compute the case name for the model. - print "multidiags start here names=", names, "multidiags stop here" return names @@ -872,7 +870,7 @@ def makeplots(res, vcanvas, vcanvas2, varid, frname, plot, package, opts, displa descr=descr, vname=vname, more_id='combined' ) else: source_descr2 = list(set(source_descr2)) - source_descr2.sort() + #source_descr2.sort() fnamepng,fnamesvg,fnamepdf = form_filename( frnamebase, ('png','svg','pdf'), modobs=list(source_descr2), more_id='combined' ) From 928d0a542790658dc72305e5e19d3555d76161a9 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 12 Jan 2017 09:31:29 -0800 Subject: [PATCH 14/24] In the outfile method, be more flexible about finding the filetable id for use in the NetCDF file's name. In the write_plot_data method, getting the axis of zax, do it whether zax is a variable or tuple of variables. --- src/python/frontend/multidiags.py | 3 --- src/python/frontend/uvcdat.py | 29 +++++++++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index a0d7db98..47bb32f4 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -261,8 +261,6 @@ def wait( idx ): stout,sterr = procs[idx].communicate() # waits for process to finish, then reads pipes namstr = stout[ stout.find("multidiags start here names=")+28 : stout.find("multidiags stop here") ] - #print "jfp stout=",stout - #print "jfp sterr=",sterr if len(namstr)>0: newopts[idx]['modobs_names'] = eval( namstr ) active[active.index(idx)] = None @@ -287,7 +285,6 @@ def multidiags1( opt ): merge_defaults(o) o = finalize_modelobs(o) # copies obspath to obs['path'], changes {} to [{}] o['runby'] = 'multi' # in case someone needs to know that multidiags is running it - print "jfp about to run_diags on",o['vars'],len(o.get('obs',[])),o.get('obs',[]) o.finalizeOpts() if len(newopts)==1 or MAX_N_PROCS==1: diff --git a/src/python/frontend/uvcdat.py b/src/python/frontend/uvcdat.py index e6992f8a..9f2b22c6 100755 --- a/src/python/frontend/uvcdat.py +++ b/src/python/frontend/uvcdat.py @@ -819,6 +819,22 @@ def synchronize_axes( self, pset ): self.axmin[vids][aid] = axmins[aid] pset.axmin[vidp][aid] = axmins[aid] + def var_ftid( self, var ): + """from a variable (normally TransientVariable) var, returns the variable id and the ftid + field of the id of the corresponding filetable.""" + if type(var) is tuple: # happens for plot set 6 + var0 = var[0] + varid = var[0].id+var[1].id + else: + var0 = var + varid = var.id + if hasattr(var0,'_filetableid'): + return varid, var0._filetableid.ftid + if hasattr(var0,'filetableid'): + return varid, var0.filetableid.ftid + if hasattr(var0,'filetable'): + return varid, var0.filetable.id().ftid + def outfile( self, format="", where="" ): """returns a filename for writing out this plot""" if not os.path.isdir(where): @@ -846,10 +862,7 @@ def outfile( self, format="", where="" ): if file_descr[0:3]=='obs': file_descr='obs' if file_descr[0:4]=='diff': file_descr='diff' var = self.vars[0] - if hasattr(var,'_filetableid'): - ft1id = var._filetableid.ftid - else: - ft1id = var.filetableid.ftid + varid, ft1id = self.var_ftid( var ) if hasattr(var,'_filetable2id'): ft2id = var._filetable2id.ftid elif hasattr(var,'filetable2id'): @@ -858,7 +871,7 @@ def outfile( self, format="", where="" ): ft2id = '' descr = underscore_join([ft1id,ft2id,file_descr]) if len(self.vars)>1: where = where+'-combined' - return form_filename( where, 'nc', descr=descr, vname=self.vars[0].id, more_id=self.more_id ) + return form_filename( where, 'nc', descr=descr, vname=varid, more_id=self.more_id ) elif len(self.title)<=0: fname = 'foo'+''.join([random.choice('0123456789') for _ in range(4)])+'.nc' else: @@ -917,7 +930,11 @@ def write_plot_data( self, format="", where="" ): zax.filetable2id= str(zax.filetable2id) # and the named tuple ids aren't writeable as such except: pass - for ax in zax.getAxisList(): + if type(zax) is tuple: + axes = zax[0].getAxisList() + else: + axes = zax.getAxisList() + for ax in axes: try: del ax.filetable except: From 2c9fe7408347552c50f2b7144e8bee84992d7239 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 12 Jan 2017 12:18:27 -0800 Subject: [PATCH 15/24] For tables (set 1) the obs filter can have " inside ' or vice versa. Do both; previously only one was supported. --- src/python/frontend/diags.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/python/frontend/diags.py b/src/python/frontend/diags.py index 9881ab2c..1b7292f9 100755 --- a/src/python/frontend/diags.py +++ b/src/python/frontend/diags.py @@ -295,7 +295,11 @@ def run_diags( opts ): filter = opts._opts['obs'][0]['filter'] obsfilter = None if filter != None: - obsfilter = filter.split('"')[1] + # Assume the filter is something simple like 'f_startswith("NCEP")', or "f_startswith('NCEP')" + splitfilt = filter.split('"') + if len(splitfilt)<=1: + splitfilt = filter.split("'") + obsfilter = splitfilt[1] computeall = not opts['output']['table'] table = sclass( modelfts, obsfts, varid=varid, obsfilter=obsfilter, dryrun=opts['dryrun'], sbatch=opts["sbatch"], computeall=computeall, outdir=outdir) From f73c2c87e17b2baf8324ea12c20b99c101090a11 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 12 Jan 2017 12:19:36 -0800 Subject: [PATCH 16/24] Form a path with os.path.join rather than a simple concatenation. --- src/python/packages/amwg/amwg1.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/python/packages/amwg/amwg1.py b/src/python/packages/amwg/amwg1.py index 02d87158..2b2bffd1 100644 --- a/src/python/packages/amwg/amwg1.py +++ b/src/python/packages/amwg/amwg1.py @@ -21,7 +21,7 @@ from metrics.common.utilities import * from metrics.computation.region import * from unidata import udunits -import cdutil.times, numpy, pdb +import cdutil.times, numpy, pdb, os import logging logger = logging.getLogger(__name__) @@ -491,7 +491,7 @@ def write_plot_data( self, format="text", where="", fname="" ): self.ptype = "text" if fname != "": logger.debug('filename was: %s', fname) - filename = where + fname + filename = os.path.join( where, fname ) else: filename = self.outfile( format, where ) writer = open( filename, 'w' ) From fd03cf4473f17c733bc3fe7ced60a9e8ebdb73d8 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 12 Jan 2017 16:24:05 -0800 Subject: [PATCH 17/24] Restore two lines deleted by mistake: they print to stdout for use in multiprocessing. --- src/python/frontend/diags.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python/frontend/diags.py b/src/python/frontend/diags.py index 1b7292f9..740675d8 100755 --- a/src/python/frontend/diags.py +++ b/src/python/frontend/diags.py @@ -332,6 +332,7 @@ def run_diags( opts ): logger.warning('Could not find any of the requested variables %s among %s', opts['vars'], pclass.list_variables(modelfts,obsfts,sname) ) logger.warning("among %s", variables) + print "multidiags start here names=", names, "multidiags stop here" # do not mess with this! return names # Ok, start the next layer of work - seasons and regions @@ -477,6 +478,7 @@ def run_diags( opts ): # If this were called from multidiags, the names dictionary would be helpful. In particular, # it will help to not have to re-open a file to re-compute the case name for the model. + print "multidiags start here names=", names, "multidiags stop here" # do not mess with this! return names From 9c758122f51d31dbc6e042c36e8e2e42c14acc13 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 12 Jan 2017 16:25:38 -0800 Subject: [PATCH 18/24] Add support for a --dryrun option. --- src/python/frontend/multidiags.py | 81 ++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 24 deletions(-) diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index 47bb32f4..35885e4d 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -3,6 +3,8 @@ # ---------------- section 0: Imports and global initializations ---------------- +from __future__ import print_function # for new-style print( string, file=outputfile ) + import logging, os, pdb, importlib, time, cProfile, pickle, tempfile, subprocess from pprint import pprint from itertools import groupby @@ -259,12 +261,58 @@ def wait( idx ): global procs global active stout,sterr = procs[idx].communicate() # waits for process to finish, then reads pipes - namstr = stout[ stout.find("multidiags start here names=")+28 : - stout.find("multidiags stop here") ] + multifirst = stout.find("multidiags start here names=")+28 + multilast = stout.find("multidiags stop here") + if multifirst <0 or multilast<0: + namstr = '' + else: + namstr = stout[ multifirst : multilast ] if len(namstr)>0: newopts[idx]['modobs_names'] = eval( namstr ) active[active.index(idx)] = None +def multidiags_dryrun( newopts, dryrun ): + """A list of Options objects newopts is set up, but we don'e want to actually run the + diagnostics. Just print out something about what we have to do.""" + try: + outfile = open(dryrun,'w') + except TypeError: # happens in the common case, dryrun==True + outfile = None + for opt in newopts: + print( "Options instance", file=outfile ) + for key in ['outpath', 'obspath','vars']: + try: + print( " ", key, "=", opt[key], file=outfile ) + except KeyError: + print( " ", key, "undefined" ) + if outfile is not None: + outfile.close() + +def multidiags2( optslist ): + """Now that a list of Options objects newopts is set up, actually run the diagnostics. Then + set up for running the viewer.""" + global optidx + global procs + global active + + if len(optslist)==1 or MAX_N_PROCS==1: + # single-processing: + for i,o in enumerate(optslist): + o['modobs_names'] = run_diags(o.clone()) # run_diags messes with its input arg, though it shouldn't + else: + # multi-processing: + procs = [None] * len(optslist) + active = [None] * MAX_N_PROCS # MAX_N_PROCS is defined at the top of this file. + optidx = 0 + while optidx Date: Thu, 12 Jan 2017 16:26:11 -0800 Subject: [PATCH 19/24] A little more output when cdscan fails. --- src/python/computation/reductions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/python/computation/reductions.py b/src/python/computation/reductions.py index 2e45143e..42821679 100644 --- a/src/python/computation/reductions.py +++ b/src/python/computation/reductions.py @@ -2882,8 +2882,8 @@ def run_cdscan( fam, famfiles, cache_path=None ): import shlex logger.info('cdscan command line: %s', cdscan_line) try: - cdscan_line = shlex.split(cdscan_line) - cdscan.main(cdscan_line) + cdscan_lineargs = shlex.split(cdscan_line) + cdscan.main(cdscan_lineargs) except Exception,err: print "CDSCAN RUN ERROR",err import traceback,sys @@ -2891,7 +2891,8 @@ def run_cdscan( fam, famfiles, cache_path=None ): traceback.print_tb(exc_traceback) print "END ERROR LOG" logger.error( 'ERROR: cdscan terminated. This is usually fatal. The arguments were:%s\n', - cdscan_line ) + cdscan_lineargs ) + print "original cdscan_line:",cdscan_line except Exception,err: print "CDSCAN IMPORT ERROR",err import traceback From d5d9a4260e14564880b3069388de35ae5cfa51bd Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 12 Jan 2017 16:47:29 -0800 Subject: [PATCH 20/24] For dry run, print out everything in each Options object which is essential for testing. --- src/python/frontend/multidiags.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index 35885e4d..7484fe4e 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -279,10 +279,18 @@ def multidiags_dryrun( newopts, dryrun ): except TypeError: # happens in the common case, dryrun==True outfile = None for opt in newopts: - print( "Options instance", file=outfile ) - for key in ['outpath', 'obspath','vars']: + print( "Options instance", file=outfile ) # also want obs filter>>>>> + #print( "keys=", sorted(opt.keys()) ) + for key in [('output','outputdir'), 'modelpath', 'obspath', ('obs','filter'), 'package', + 'sets', 'seasons', 'vars' ]: try: - print( " ", key, "=", opt[key], file=outfile ) + if type(key) is str: + print( " ", key, "=", opt[key], file=outfile ) + else: # nested dictionaries, two keys in a tuple + if type(opt[key[0]]) is list: # we also have to drill through a list! (happens with model,obs) + print( " ", key[0], "[0][", key[1], "] =", opt[key[0]][0][key[1]], file=outfile ) + else: + print( " ", key[0], "[", key[1], "] =", opt[key[0]][key[1]], file=outfile ) except KeyError: print( " ", key, "undefined" ) if outfile is not None: From 85058131b72c238ab5a90ecc005d520da32014da Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Mon, 23 Jan 2017 14:04:09 -0800 Subject: [PATCH 21/24] Test of multidiags: make sure that it would invoke run_diags with correct values of some essential options. --- setup.py | 2 + src/python/frontend/multidiags | 1 + src/python/frontend/multidiags.py | 4 +- test/CMakeLists.txt | 7 ++++ test/baselines/diagsmulti/dryrunopts | 24 +++++++++++ test/diagsmulti.py | 61 ++++++++++++++++++++++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) create mode 120000 src/python/frontend/multidiags create mode 100644 test/baselines/diagsmulti/dryrunopts create mode 100644 test/diagsmulti.py diff --git a/setup.py b/setup.py index 9913499c..590dd0c8 100755 --- a/setup.py +++ b/setup.py @@ -75,6 +75,8 @@ "src/python/frontend/climatology.py", "src/python/frontend/metadiags.py", "src/python/frontend/metadiags", + "src/python/frontend/multidiags.py", + "src/python/frontend/multidiags", "src/python/devel/update_metrics_baselines.py", "src/python/devel/update_metrics_baselines", "src/python/packages/atm_tier1b/u850.ncl", diff --git a/src/python/frontend/multidiags b/src/python/frontend/multidiags new file mode 120000 index 00000000..f349c823 --- /dev/null +++ b/src/python/frontend/multidiags @@ -0,0 +1 @@ +multidiags.py \ No newline at end of file diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index 7484fe4e..42339823 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python + # Run multiple diagnostics. The user only specifies paths and a collection name. # The collection name is a key into multimaster.py, or (>>>> TO DO >>>>) another file provided by the user. @@ -281,7 +283,7 @@ def multidiags_dryrun( newopts, dryrun ): for opt in newopts: print( "Options instance", file=outfile ) # also want obs filter>>>>> #print( "keys=", sorted(opt.keys()) ) - for key in [('output','outputdir'), 'modelpath', 'obspath', ('obs','filter'), 'package', + for key in ['modelpath', 'obspath', ('obs','filter'), 'package', 'sets', 'seasons', 'vars' ]: try: if type(key) is str: diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index f90ee209..c9e380d1 100755 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -123,3 +123,10 @@ ${metrics_SOURCE_DIR}/test/diagsmeta.py --datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/ --baseline=${BASELINE_DIR}/ ) #set_tests_properties(diags_meta PROPERTIES DEPENDS diags_test_15) + +add_test("diags_multi" +"python" +${metrics_SOURCE_DIR}/test/diagsmulti.py +--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/ +--baseline=${BASELINE_DIR}/ ) +#set_tests_properties(diags_meta PROPERTIES DEPENDS diags_test_15) diff --git a/test/baselines/diagsmulti/dryrunopts b/test/baselines/diagsmulti/dryrunopts new file mode 100644 index 00000000..1b9570b2 --- /dev/null +++ b/test/baselines/diagsmulti/dryrunopts @@ -0,0 +1,24 @@ +Options instance + modelpath = ['/test/data//cam35_data_smaller/'] + obspath = ['/test/data/obs_data_5.6/'] + obs [0][ filter ] = f_startswith('ISCCP_') + package = AMWG + sets = ['5'] + seasons = ['DJF', 'JJA', 'ANN'] + vars = ['FLUT', 'LWCF'] +Options instance + modelpath = ['/test/data//cam35_data_smaller/'] + obspath = ['/test/data/obs_data_5.6/'] + obs [0][ filter ] = f_and( f_startswith('CERES_'), f_not(f_contains('EBAF'))) + package = AMWG + sets = ['5'] + seasons = ['DJF', 'JJA', 'ANN'] + vars = ['FLUT', 'LWCF'] +Options instance + modelpath = ['/test/data//cam35_data_smaller/'] + obspath = ['/test/data/obs_data_5.6/'] + obs [0][ filter ] = f_startswith('NCEP') + package = AMWG + sets = ['5'] + seasons = ['DJF', 'JJA', 'ANN'] + vars = ['FLUT', 'LWCF'] diff --git a/test/diagsmulti.py b/test/diagsmulti.py new file mode 100644 index 00000000..c6707a3e --- /dev/null +++ b/test/diagsmulti.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +"""" This is a test of multidiags""" +import sys, os, shutil, tempfile, subprocess, filecmp +import argparse, pdb +import diags_test + +#to run this app use +#uvcmetrics/test/diagsmulti.py --datadir $HOME/uvcmetrics_test_data/ --baseline $HOME/uvcmetrics/test/baselines/ + +#get commmand line args +p = argparse.ArgumentParser(description="Basic multidiags test") +p.add_argument("--datadir", dest="datadir", help="root directory for model and obs data") +p.add_argument("--baseline", dest="baseline", help="directory with baseline files for comparing results") +p.add_argument("--rebaseline", dest="rebaseline", help="instructions to rebaseline the output") + +args = p.parse_args(sys.argv[1:]) +datadir = args.datadir +print 'datadir = ', datadir +baselinepath = args.baseline + '/diagsmulti/' +print "baselinepath = ", baselinepath +outpath = tempfile.mkdtemp() + "/" +print "outpath=", outpath +rebaseline = False +if 'rebaseline' in dir(args): + # Actually this never happens. + rebaseline = args.rebaseline + +test_str = 'multidiags\n' +#run this from command line to get the files required +command = "multidiags.py --package AMWG --set 5 --modelpath=%s/cam35_data_smaller/ --obspath=%sobs_data_5.6/ --dryrun=%s/dryrunopts --outputdir=%s "%(datadir, datadir, outpath, outpath) +print command +#+ "--outputdir " + outpath + +#run the command +proc = subprocess.Popen([command], shell=True) +proc_status = proc.wait() +if proc_status!=0: + raise Exception("multidiags test failed") + +new = open(os.path.join(outpath,"dryrunopts")) +old = open(os.path.join(baselinepath,"dryrunopts")) + +print "Comparing: %s %s" % (new.name, old.name) + +new = new.readlines() +old = old.readlines() + +diff = False +for OLD, NEW in zip(old, new): + # Get rid of root part of model and obs paths: + # This may not work if there be more than one item in the path list... + testfind = NEW.find('/test/') + if testfind>0: + NEW = NEW[0:NEW.find("path = [")+9] + "" + NEW[testfind:] + if not rebaseline: + if not (OLD==NEW): + print "OLD=",OLD + print "NEW=",NEW + raise Exception("multidiags.sh generated different files") +if rebaseline: + print 'new baseline is',os.path.join(baselinepath,'dryrunopts') From 5dd7b361a6c5e304460f973163c822357dc6af90 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Mon, 23 Jan 2017 14:06:46 -0800 Subject: [PATCH 22/24] Make it possible to use the --dryrun option specify an output file to use instead of stdout. --- src/python/frontend/options.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/python/frontend/options.py b/src/python/frontend/options.py index 9ced9555..ed991b99 100644 --- a/src/python/frontend/options.py +++ b/src/python/frontend/options.py @@ -802,7 +802,11 @@ def processCmdLine(self, progname, parser): help="Enable translation for obs sets to datasets. Optional provide a colon separated input to output list e.g. DSVAR1:OBSVAR1") runopts.add_argument('--dryrun', help="Do not run anything simply store list of commands in file table_commands.sh for diags or metadiags_commands.sh for metadiags", - action="store_true") + nargs='?', default=False, const=True, + action="store") + # ... Thus there are three normal ways to specify dryrun: (1) leave the option out - means self['dryrun']==False, + # (2) --dryrun (with no value provided) - means self['dryrun']==True, print dryrun output to stdout, + # (3) --dryrun path - means print dryrun output to the specified path runopts.add_argument('--sbatch', help="Run sbatch with the specified number of nodes", default=0, From f9e3c7679b7fc58051ded193635efdf6ff120914 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 26 Jan 2017 12:21:19 -0800 Subject: [PATCH 23/24] New test to check whether multidiags and metadiags do the same thing. --- test/CMakeLists.txt | 7 ++++- test/diagsmultimeta.py | 61 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 test/diagsmultimeta.py diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c9e380d1..7bb49f4e 100755 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -129,4 +129,9 @@ add_test("diags_multi" ${metrics_SOURCE_DIR}/test/diagsmulti.py --datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/ --baseline=${BASELINE_DIR}/ ) -#set_tests_properties(diags_meta PROPERTIES DEPENDS diags_test_15) + +add_test("diags_multimeta" +"python" +${metrics_SOURCE_DIR}/test/diagsmultimeta.py +--datadir=${UVCMETRICS_TEST_DATA_DIRECTORY}/ +--baseline=${BASELINE_DIR}/ ) diff --git a/test/diagsmultimeta.py b/test/diagsmultimeta.py new file mode 100644 index 00000000..97aa76d9 --- /dev/null +++ b/test/diagsmultimeta.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +"""" This is a test of whether multidiags and metadiags do the same thing""" +import sys, os, shutil, tempfile, subprocess, filecmp +import argparse, pdb +import diags_test + +#to run this app use +#uvcmetrics/test/diagsmultmetai.py --datadir $HOME/uvcmetrics_test_data/ --baseline $HOME/uvcmetrics/test/baselines/ + +#get commmand line args +p = argparse.ArgumentParser(description="Basic multidiags test") +p.add_argument("--datadir", dest="datadir", help="root directory for model and obs data") +p.add_argument("--baseline", dest="baseline", help="directory with baseline files for comparing results") +p.add_argument("--rebaseline", dest="rebaseline", help="instructions to rebaseline the output") + +args = p.parse_args(sys.argv[1:]) +datadir = args.datadir +print 'datadir = ', datadir +baselinepath = args.baseline + '/diagsmultimeta/' +print "baselinepath = ", baselinepath +outpath = tempfile.mkdtemp() + "/" +print "outpath=", outpath +rebaseline = False +if 'rebaseline' in dir(args): + # Actually this never happens. + rebaseline = args.rebaseline + +test_str = 'multidiags\n' +#run this from command line to get the files required +multi_command = "multidiags.py --package AMWG --set 7s --modelpath=%s/cam35_data_smaller/ --obspath=%sobs_data_5.6/ --dryrun=%s/multi_dryrunopts --outputdir=%s "%(datadir, datadir, outpath, outpath) +print multi_command +#+ "--outputdir " + outpath + +test_str = 'metadiags\n' +#run this from command line to get the files required +meta_command = "metadiags.py --package AMWG --set 7s --model path=%s/cam35_data_smaller/,climos=yes --obs path=%sobs_data_5.6/,climos=yes --dryrun=%s/meta_dryrunopts --outputdir=%s"%(datadir, datadir, outpath, outpath) +print meta_command +#+ "--outputdir " + outpath + +#run the multidiags command +multi_proc = subprocess.Popen([multi_command], shell=True) + +#run the metadiags command +meta_proc = subprocess.Popen([meta_command], shell=True) + +# Wait for both processes to finish. +multi_proc_status = multi_proc.wait() +if multi_proc_status!=0: + raise Exception("multidiags-metadiags test failed in multidiags") +meta_proc_status = meta_proc.wait() +if meta_proc_status!=0: + raise Exception("multidiags-metadiags test failed in metadiags") + +metafile = open(os.path.join(outpath,"meta_dryrunopts")) +multifile = open(os.path.join(outpath,"multi_dryrunopts")) +print "Comparing: %s %s" % (metafile.name, multifile.name) + +if filecmp.cmp( metafile.name, multifile.name ): + pass +else: + raise Exception("metadiags and multidiags produced different files") From 9e1a538d9f401fcb2067eed1053a48bf4ed43243 Mon Sep 17 00:00:00 2001 From: Jeff Painter Date: Thu, 26 Jan 2017 12:23:32 -0800 Subject: [PATCH 24/24] Changes in metadiags.py, options.py, and elsewhere to allow a clean comparison of an Options instance for collection 7s as generated by metadiags and multidiags. --- src/python/frontend/amwgmaster.py | 5 +- src/python/frontend/metadiags.py | 75 +++++++++++++++++++++++++--- src/python/frontend/multidiags.py | 2 +- src/python/frontend/options.py | 13 ++++- test/baselines/diagsmulti/dryrunopts | 12 ++--- 5 files changed, 91 insertions(+), 16 deletions(-) diff --git a/src/python/frontend/amwgmaster.py b/src/python/frontend/amwgmaster.py index a6a8f052..aa569154 100644 --- a/src/python/frontend/amwgmaster.py +++ b/src/python/frontend/amwgmaster.py @@ -361,8 +361,9 @@ diags_collection['7s']['options'] = {'logo':'no'} diags_collection['7s']['combined'] = True diags_collection['7s']['TREFHT'] = {'plottype': '7', 'obs': ['WILLMOTT_1']} -diags_collection['7s']['PS'] = {'plottype': '7', 'obs': ['NCEP_1']} -diags_collection['7s']['PSL'] = {'plottype': '7', 'obs': ['JRA25_1', 'NCEP_1']} +diags_collection['7s']['PSL'] = {'plottype': '7', 'obs': ['JRA25_1']} +diags_collection['7s']['Z3'] = {'plottype': '7', 'obs': ['JRA25_1']} + # These 4 are northern only. diff --git a/src/python/frontend/metadiags.py b/src/python/frontend/metadiags.py index 29ca2145..13d836d8 100644 --- a/src/python/frontend/metadiags.py +++ b/src/python/frontend/metadiags.py @@ -200,7 +200,8 @@ def makeTables(collnum, model_dict, obspath, outpath, pname, outlogdir, dryrun=F runcmdline(cmdline, outlogdir, dryrun) -def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls=None, dryrun=False): +def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls=None, + dryrun=False, opts_dryrun=False): import os # Did the user specify a single collection? If not find out what collections we have if colls == None: @@ -469,7 +470,7 @@ def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls pstr2 = '' cmdline = (def_executable, pstr1, pstr2, obsstr, optionsstr, packagestr, setstr, seasonstr, varstr, outstr, xmlstr, prestr, poststr, regionstr) if collnum != 'dontrun': - runcmdline(cmdline, outlogdir, dryrun) + runcmdline(cmdline, outlogdir, dryrun, opts_dryrun) else: message = cmdline logger.debug('DONTRUN: %s', cmdline) @@ -532,7 +533,7 @@ def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls if varopts: cmdline += [varopts] if collnum != 'dontrun': - runcmdline(cmdline, outlogdir, dryrun) + runcmdline(cmdline, outlogdir, dryrun, opts_dryrun) else: logger.debug('DONTRUN: %s', cmdline) else: # different executable; just pass all option key:values as command line options. @@ -569,7 +570,7 @@ def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls execstr = execstr+' --figurebase '+ fnamebase if execstr != def_executable: - runcmdline([execstr], outlogdir, dryrun) + runcmdline([execstr], outlogdir, dryrun, opts_dryrun) # VIEWER Code # Build rows for this group in the index... @@ -688,16 +689,75 @@ def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls pid_to_tmpfile = {} active_processes = [] DIAG_TOTAL = 0 +optsout = None + + +def dequote(s): # from stackoverflow, and a little better than the way I would have done it: + """ + If a string has single or double quotes around it, remove them. + Make sure the pair of quotes match. + If a matching pair of quotes is not found, return the string unchanged. + """ + if (s[0] == s[-1]) and s.startswith(("'", '"')): + return s[1:-1] + return s + +def cmd2opts( cmdline ): + """Input is a metadiags command line, a tuple or list of strings. + This function computes and returns the corresponding Options object.""" + opts = Options() + # Some strings in cmdline contain blanks, e.g. '--seasons DJF JJA'. + # We have to convert this to a list of strings like sys.argv when cmdline is run, i.e. '--seasons','DJF','JJA'. + cmdl = ' '.join(cmdline) + sys.argv = [item for item in cmdl.split(' ') if item!=''] + opts.parseCmdLine() + return opts + +def cmd2print( cmdline, opts_dryrun=True ): + """Input is a metadiags command line, and optionally the dryrun option. This dryrun option may + be the name of a file, or True or False. This function will print values of some key options. + If dryrun be True, they will be printed to stdout. If dryrun be the name of a file, they will + be written to the file. Nothing will happen if dryrun be False.""" + global optsout + if opts_dryrun==False: + return + opts = cmd2opts( cmdline ) + if optsout is None: + try: + optsout = open(opts_dryrun,'w') + except TypeError: + # happens in the common case, opts_dryrun==True + optsout = None # print to this means print to stdout + if 'modelpath' not in opts.keys(): + opts['modelpath'] = [os.path.abspath(opts['model'][0]['path'])] + if 'obspath' not in opts.keys(): + opts['obspath'] = [os.path.abspath(opts['obs'][0]['path'])] + + print >>optsout, "Options instance" + for key in ['modelpath', 'obspath', ('obs','filter'), 'package', + 'sets', 'seasons', 'vars' ]: + try: + if type(key) is str: + print >>optsout, " ", key, "=", opts[key] + else: # nested dictionaries, two keys in a tuple + if type(opts[key[0]]) is list: # we also have to drill through a list! (happens with model,obs) + print >>optsout, " ", key[0], "[0][", key[1], "] =", dequote( opts[key[0]][0][key[1]] ) + else: + print >>optsout, " ", key[0], "[", key[1], "] =", opts[key[0]][key[1]] + except KeyError: + print( " ", key, "undefined" ) def cmderr(popened): logfile = pid_to_cmd[popened.pid].split(" ")[-1] logger.error("Command \n%s\n failed with code of %d. Log file is at %s.", pid_to_cmd[popened.pid], popened.returncode, logfile) -def runcmdline(cmdline, outlogdir, dryrun=False): +def runcmdline(cmdline, outlogdir, dryrun=False, opts_dryrun=False): global DIAG_TOTAL + cmd2print( cmdline, opts_dryrun ) + # the following is a total KLUDGE. It's more of a KLUDGE than last time. # I'm not proud of this but I feel threatned if I don't do it. # there is some sort of memory leak in vcs. @@ -928,6 +988,7 @@ def postDB(fts, dsname, package, host=None): cmaps = opts._opts["colormaps"] tmpDict["colormaps"] = " ".join(["%s=%s" % (k, cmaps[k]) for k in cmaps]) diags_collection[K]["options"] = tmpDict + print "jfp opts['dryrun']=",opts['dryrun'] if opts["dryrun"]: fnm = os.path.join(outpath, "metadiags_commands.sh") dryrun = open(fnm, "w") @@ -945,6 +1006,7 @@ def postDB(fts, dsname, package, host=None): module use /usr/common/contrib/acme/modulefiles module load uvcdat/batch """ % (opts["sbatch"]) + else: dryrun = False @@ -960,7 +1022,8 @@ def postDB(fts, dsname, package, host=None): data_path_hmac.update(obspath) data_hash = data_path_hmac.hexdigest() - menus, pages = generatePlots(model_dict, obspath, outpath, package, xmlflag, data_hash, colls=colls,dryrun=dryrun) + menus, pages = generatePlots(model_dict, obspath, outpath, package, xmlflag, data_hash, + colls=colls,dryrun=dryrun,opts_dryrun=opts['dryrun']) for page in pages: # Grab file metadata for every image that exists. diff --git a/src/python/frontend/multidiags.py b/src/python/frontend/multidiags.py index 42339823..b2108317 100644 --- a/src/python/frontend/multidiags.py +++ b/src/python/frontend/multidiags.py @@ -281,7 +281,7 @@ def multidiags_dryrun( newopts, dryrun ): except TypeError: # happens in the common case, dryrun==True outfile = None for opt in newopts: - print( "Options instance", file=outfile ) # also want obs filter>>>>> + print( "Options instance", file=outfile ) #print( "keys=", sorted(opt.keys()) ) for key in ['modelpath', 'obspath', ('obs','filter'), 'package', 'sets', 'seasons', 'vars' ]: diff --git a/src/python/frontend/options.py b/src/python/frontend/options.py index ed991b99..e1f03227 100644 --- a/src/python/frontend/options.py +++ b/src/python/frontend/options.py @@ -508,6 +508,9 @@ def verifyOptions(self): self.merge(options_defaults) # jfp not really a "verify" step, but has to be done now that defaults are out of __init__ if len(self._opts['model']) == 0 and len(self._opts['obs']) == 0: logging.critical('At least one model or obs set needs describted') + import traceback + tb = traceback.format_exc() + logger.debug("traceback:\n%s", tb) quit() if len(self._opts['model']) != 0: for i in range(len(self._opts['model'])): @@ -996,8 +999,16 @@ def parseCmdLine(self): raise e if args.modelpath != None: self['modelpath'] = args.modelpath[0] + if type(self['modelpath']) is str: + self['modelpath'] = os.path.abspath( self['modelpath'] ) + else: + self['modelpath'] = [ os.path.abspath(pth) for pth in self['modelpath'] ] if args.obspath != None: self['obspath'] = args.obspath[0] + if type(self['obspath']) is str: + self['obspath'] = os.path.abspath( self['obspath'] ) + else: + self['obspath'] = [ os.path.abspath(pth) for pth in self['obspath'] ] if (args.levels) != None: self.processLevels('levels', args.levels, extras) @@ -1189,7 +1200,7 @@ def finalizeOpts( self ): self['times'] = self['times']+slist else: self['times'] = slist - logger.debug('seasons: %s', self._opts['times']) + #logger.debug('seasons: %s', self._opts['times']) def listOpts(self, args): diff --git a/test/baselines/diagsmulti/dryrunopts b/test/baselines/diagsmulti/dryrunopts index 1b9570b2..4ec9e060 100644 --- a/test/baselines/diagsmulti/dryrunopts +++ b/test/baselines/diagsmulti/dryrunopts @@ -1,22 +1,22 @@ Options instance - modelpath = ['/test/data//cam35_data_smaller/'] - obspath = ['/test/data/obs_data_5.6/'] + modelpath = ['/test/data/cam35_data_smaller'] + obspath = ['/test/data/obs_data_5.6'] obs [0][ filter ] = f_startswith('ISCCP_') package = AMWG sets = ['5'] seasons = ['DJF', 'JJA', 'ANN'] vars = ['FLUT', 'LWCF'] Options instance - modelpath = ['/test/data//cam35_data_smaller/'] - obspath = ['/test/data/obs_data_5.6/'] + modelpath = ['/test/data/cam35_data_smaller'] + obspath = ['/test/data/obs_data_5.6'] obs [0][ filter ] = f_and( f_startswith('CERES_'), f_not(f_contains('EBAF'))) package = AMWG sets = ['5'] seasons = ['DJF', 'JJA', 'ANN'] vars = ['FLUT', 'LWCF'] Options instance - modelpath = ['/test/data//cam35_data_smaller/'] - obspath = ['/test/data/obs_data_5.6/'] + modelpath = ['/test/data/cam35_data_smaller'] + obspath = ['/test/data/obs_data_5.6'] obs [0][ filter ] = f_startswith('NCEP') package = AMWG sets = ['5']