Multidiags mp

setup.py

@@ -75,6 +75,8 @@
                   "src/python/frontend/climatology.py",
                   "src/python/frontend/metadiags.py",
                   "src/python/frontend/metadiags",
+                  "src/python/frontend/multidiags.py",
+                  "src/python/frontend/multidiags",
                   "src/python/devel/update_metrics_baselines.py",
                   "src/python/devel/update_metrics_baselines",
                   "src/python/packages/atm_tier1b/u850.ncl",

src/python/computation/reductions.py

@@ -2856,7 +2856,8 @@ def run_cdscan( fam, famfiles, cache_path=None ):
             # get here.
             # Another problem is that units stuck in the long_name sometimes are
             # nonstandard.  So fix them!
-            if hasattr(f['time'],'long_name'):
+            if hasattr(f['time'],'long_name') and\
+                    getattr(f['time'],'long_name',None) not in ['time',None,'']:
                 time_units = f['time'].long_name
             else:
                 time_units = 'days'  # probably wrong but we can't go on without something
@@ -2881,16 +2882,17 @@ def run_cdscan( fam, famfiles, cache_path=None ):
         import shlex
         logger.info('cdscan command line: %s', cdscan_line)
         try:
-            cdscan_line = shlex.split(cdscan_line)
-            cdscan.main(cdscan_line)
+            cdscan_lineargs = shlex.split(cdscan_line)
+            cdscan.main(cdscan_lineargs)
         except Exception,err:
             print "CDSCAN RUN ERROR",err
             import traceback,sys
             exc_type, exc_value, exc_traceback = sys.exc_info()
             traceback.print_tb(exc_traceback)
             print "END ERROR LOG"
             logger.error( 'ERROR: cdscan terminated. This is usually fatal. The arguments were:%s\n',
-                           cdscan_line )
+                           cdscan_lineargs )
+            print "original cdscan_line:",cdscan_line
     except Exception,err:
         print "CDSCAN IMPORT ERROR",err
         import traceback

src/python/fileio/filetable.py

@@ -266,7 +266,8 @@ def addfile( self, filep, options):
         filesupp = get_datafile_filefmt( dfile, options )
         if self.filefmt is None:
            self.filefmt = filesupp.name
-        elif self.filefmt!= filesupp.name:
+        elif self.filefmt!= filesupp.name and self.filefmt.find(filesupp.name)<0 and\
+                filesupp.name.find(self.filefmt)<0:
            self.filefmt = "various"
         vars = filesupp.interesting_variables()
         if len(vars)>0:

src/python/fileio/findfiles.py

@@ -209,7 +209,9 @@ def _getdatafiles( self, root, filt ):
 
     def shortest_name(self):
         if len(self._filt.mystr())>3:
-            return self._filt.mystr()
+            shortname = self._filt.mystr()
+            shortname = shortname.strip('-_')
+            return shortname
         else:
             return self.short_name()
 
@@ -226,6 +228,7 @@ def short_name(self):
         else:
             shortname = ','.join([os.path.basename(str(r))
                                   for r in self._root])   # <base directory> <filter name>
+        shortname = shortname.strip('-_')
         return shortname
 
     def long_name(self):

src/python/frontend/amwgmaster.py

@@ -361,8 +361,9 @@
 diags_collection['7s']['options'] = {'logo':'no'}
 diags_collection['7s']['combined'] = True
 diags_collection['7s']['TREFHT'] = {'plottype': '7', 'obs': ['WILLMOTT_1']}
-diags_collection['7s']['PS'] = {'plottype': '7', 'obs': ['NCEP_1']}
-diags_collection['7s']['PSL'] = {'plottype': '7', 'obs': ['JRA25_1', 'NCEP_1']}
+diags_collection['7s']['PSL'] = {'plottype': '7', 'obs': ['JRA25_1']}
+diags_collection['7s']['Z3'] = {'plottype': '7', 'obs': ['JRA25_1']}
+
 
 # These 4 are northern only.
 

src/python/frontend/diags.py

@@ -83,6 +83,8 @@ def setManualColormap(canvas=None, level=0):
 def getNames(opts, model, obs):
     """ The purpose of this kludge is to get the names of the model and obs 
     specified on the command line for output on the graphic.
+    These particular names should be sufficiently unique and human-readable, and
+    reasonably short.
     Each of model and obs should be a list of instances of basic_filetable.
     This function returns a dictionary of the form {'model':modelname, 'obs':obsname}
     where modelname and obsname are strings."""
@@ -107,7 +109,8 @@ def get_model_case(filetable):
         nicknames['obs'] = opts['obs'][0]['name']
     elif ft2 is not None:
         #nicknames['obs'] = ft2.source().split('_')[-1]
-        nicknames['obs'] = ft2.source().split('_')[0]
+        #nicknames['obs'] = ft2.source().split('_')[0] #doesn't work on HadISST_CL, which isn't HadISST
+        nicknames['obs'] = ft2.source()
     return nicknames
 
 def setnum( setname ):
@@ -156,7 +159,7 @@ def run_diags( opts ):
     # Setup filetable arrays
     modelfts = []
     obsfts = []
-    names = {'model':'', 'obs':''}
+    names = {}
     for i in range(len(opts['model'])):
         ft = path2filetable(opts, modelid=i)
         if ft._id.nickname=='':
@@ -292,7 +295,11 @@ def run_diags( opts ):
             filter = opts._opts['obs'][0]['filter']
             obsfilter = None
             if filter != None:
-                obsfilter  = filter.split('"')[1]
+                # Assume the filter is something simple like 'f_startswith("NCEP")', or "f_startswith('NCEP')"
+                splitfilt =  filter.split('"')
+                if len(splitfilt)<=1:
+                    splitfilt =  filter.split("'")
+                obsfilter  = splitfilt[1]
 
             computeall = not opts['output']['table']
             table = sclass( modelfts, obsfts, varid=varid, obsfilter=obsfilter, dryrun=opts['dryrun'], sbatch=opts["sbatch"], computeall=computeall, outdir=outdir)
@@ -325,7 +332,8 @@ def run_diags( opts ):
                 logger.warning('Could not find any of the requested variables %s among %s', opts['vars'],
                                 pclass.list_variables(modelfts,obsfts,sname) )
                 logger.warning("among %s", variables)
-                return {}
+                print "multidiags start here names=", names, "multidiags stop here"  # do not mess with this!
+                return names
 
         # Ok, start the next layer of work - seasons and regions
         # loop over the seasons for this plot
@@ -470,6 +478,7 @@ def run_diags( opts ):
 
     # If this were called from multidiags, the names dictionary would be helpful.  In particular,
     # it will help to not have to re-open a file to re-compute the case name for the model.
+    print "multidiags start here names=", names, "multidiags stop here"  # do not mess with this!
     return names
 
 
@@ -867,6 +876,7 @@ def makeplots(res, vcanvas, vcanvas2, varid, frname, plot, package, opts, displa
                                                     descr=descr, vname=vname, more_id='combined' )
     else:
         source_descr2 = list(set(source_descr2))
+        #source_descr2.sort()
         fnamepng,fnamesvg,fnamepdf = form_filename( frnamebase, ('png','svg','pdf'),
                                                     modobs=list(source_descr2), more_id='combined' )
 
@@ -888,15 +898,21 @@ def makeplots(res, vcanvas, vcanvas2, varid, frname, plot, package, opts, displa
 
 if __name__ == '__main__':
     print "UV-CDAT Diagnostics, command-line version"
-    print ' '.join(sys.argv)
-    try:
-        irunby = sys.argv.index('--runby')
-        runby = sys.argv[irunby+1]
-        o = Options(runby=runby)
-    except ValueError:
-        o = Options()
+    print repr(repr(' '.join(sys.argv)))
+    if sys.argv[1][:10]=='--pickopt=':
+        # special case.  All options are described in the argument, a pickled Options object.
+        pickopt = sys.argv[1][10:]
+        pickopt = pickopt.replace('jfpsingleq','\'')
+        o = pickle.loads(pickopt)
+    else:
+        # Normal case, with command-line options.  Their interpretation differs slightly according
+        # to how this was run, as specified in the '--runby' option.
+        try:
+            irunby = sys.argv.index('--runby')
+            runby = sys.argv[irunby+1]
+            o = Options(runby=runby)
+        except ValueError:
+            o = Options()
     o.parseCmdLine()
     o.verifyOptions()
-    #print o._opts['levels']
-    #print o._opts['displayunits']
     run_diags(o)

src/python/frontend/metadiags.py

@@ -200,7 +200,8 @@ def makeTables(collnum, model_dict, obspath, outpath, pname, outlogdir, dryrun=F
                 runcmdline(cmdline, outlogdir, dryrun)
 
 
-def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls=None, dryrun=False):
+def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls=None,
+                  dryrun=False, opts_dryrun=False):
     import os
     # Did the user specify a single collection? If not find out what collections we have
     if colls == None:
@@ -469,7 +470,7 @@ def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls
                         pstr2 = ''
                     cmdline = (def_executable, pstr1, pstr2, obsstr, optionsstr, packagestr, setstr, seasonstr, varstr, outstr, xmlstr, prestr, poststr, regionstr)
                     if collnum != 'dontrun':
-                        runcmdline(cmdline, outlogdir, dryrun)
+                        runcmdline(cmdline, outlogdir, dryrun, opts_dryrun)
                     else:
                         message = cmdline
                         logger.debug('DONTRUN: %s', cmdline)
@@ -532,7 +533,7 @@ def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls
                         if varopts:
                             cmdline +=  [varopts]
                         if collnum != 'dontrun':
-                            runcmdline(cmdline, outlogdir, dryrun)
+                            runcmdline(cmdline, outlogdir, dryrun, opts_dryrun)
                         else:
                             logger.debug('DONTRUN: %s', cmdline)
                 else: # different executable; just pass all option key:values as command line options.
@@ -569,7 +570,7 @@ def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls
                             execstr = execstr+' --figurebase '+ fnamebase
 
                     if execstr != def_executable:
-                        runcmdline([execstr], outlogdir, dryrun)
+                        runcmdline([execstr], outlogdir, dryrun, opts_dryrun)
 
                 # VIEWER Code
                 # Build rows for this group in the index...
@@ -688,16 +689,75 @@ def generatePlots(model_dict, obspath, outpath, pname, xmlflag, data_hash, colls
 pid_to_tmpfile = {}
 active_processes = []
 DIAG_TOTAL = 0
+optsout = None
+
+
+def dequote(s):  # from stackoverflow, and a little better than the way I would have done it:
+    """
+    If a string has single or double quotes around it, remove them.
+    Make sure the pair of quotes match.
+    If a matching pair of quotes is not found, return the string unchanged.
+    """
+    if (s[0] == s[-1]) and s.startswith(("'", '"')):
+        return s[1:-1]
+    return s
+
+def cmd2opts( cmdline ):
+    """Input is a metadiags command line, a tuple or list of strings.
+    This function computes and returns the corresponding Options object."""
+    opts = Options()
+    # Some strings in cmdline contain blanks, e.g. '--seasons DJF JJA'.
+    # We have to convert this to a list of strings like sys.argv when cmdline is run, i.e. '--seasons','DJF','JJA'.
+    cmdl = ' '.join(cmdline)
+    sys.argv = [item for item in cmdl.split(' ') if item!='']
+    opts.parseCmdLine()
+    return opts
+
+def cmd2print( cmdline, opts_dryrun=True ):
+    """Input is a metadiags command line, and optionally the dryrun option.  This dryrun option may
+    be the name of a file, or True or False.  This function will print values of some key options.
+    If dryrun be True, they will be printed to stdout.  If dryrun be the name of a file, they will
+    be written to the file.  Nothing will happen if dryrun be False."""
+    global optsout
+    if opts_dryrun==False:
+        return
+    opts = cmd2opts( cmdline )
 
+    if optsout is None:
+        try:
+            optsout = open(opts_dryrun,'w')
+        except TypeError:
+            # happens in the common case, opts_dryrun==True
+            optsout = None   # print to this means print to stdout
+    if 'modelpath' not in opts.keys():
+        opts['modelpath'] = [os.path.abspath(opts['model'][0]['path'])]
+    if 'obspath' not in opts.keys():
+        opts['obspath'] = [os.path.abspath(opts['obs'][0]['path'])]
+
+    print >>optsout, "Options instance"
+    for key in ['modelpath', 'obspath', ('obs','filter'), 'package',
+                'sets', 'seasons', 'vars' ]:
+        try:
+            if type(key) is str:
+                print >>optsout, "  ", key, "=", opts[key]
+            else: # nested dictionaries, two keys in a tuple
+                if type(opts[key[0]]) is list:  # we also have to drill through a list! (happens with model,obs)
+                    print >>optsout, "  ", key[0], "[0][", key[1], "] =", dequote( opts[key[0]][0][key[1]] )
+                else:
+                    print >>optsout, "  ", key[0], "[", key[1], "] =", opts[key[0]][key[1]]
+        except KeyError:
+            print( "  ", key, "undefined" )
 
 def cmderr(popened):
     logfile = pid_to_cmd[popened.pid].split(" ")[-1]
     logger.error("Command \n%s\n failed with code of %d. Log file is at %s.", pid_to_cmd[popened.pid], popened.returncode, logfile)
 
 
-def runcmdline(cmdline, outlogdir, dryrun=False):
+def runcmdline(cmdline, outlogdir, dryrun=False, opts_dryrun=False):
     global DIAG_TOTAL
 
+    cmd2print( cmdline, opts_dryrun )
+
     # the following is a total KLUDGE. It's more of a KLUDGE than last time.
     # I'm not proud of this but I feel threatned if I don't do it.
     # there is some sort of memory leak in vcs.
@@ -928,6 +988,7 @@ def postDB(fts, dsname, package, host=None):
         cmaps = opts._opts["colormaps"]
         tmpDict["colormaps"] = " ".join(["%s=%s" % (k, cmaps[k]) for k in cmaps])
         diags_collection[K]["options"] = tmpDict
+    print "jfp opts['dryrun']=",opts['dryrun']
     if opts["dryrun"]:
         fnm = os.path.join(outpath, "metadiags_commands.sh")
         dryrun = open(fnm, "w")
@@ -945,6 +1006,7 @@ def postDB(fts, dsname, package, host=None):
 module use /usr/common/contrib/acme/modulefiles
 module load uvcdat/batch
 """ % (opts["sbatch"])
+
     else:
         dryrun = False
 
@@ -960,7 +1022,8 @@ def postDB(fts, dsname, package, host=None):
     data_path_hmac.update(obspath)
     data_hash = data_path_hmac.hexdigest()
 
-    menus, pages = generatePlots(model_dict, obspath, outpath, package, xmlflag, data_hash, colls=colls,dryrun=dryrun)
+    menus, pages = generatePlots(model_dict, obspath, outpath, package, xmlflag, data_hash,
+                                 colls=colls,dryrun=dryrun,opts_dryrun=opts['dryrun'])
 
     for page in pages:
         # Grab file metadata for every image that exists.

src/python/frontend/multidiags

@@ -0,0 +1 @@
+multidiags.py