1dvars cmtb v2

.gitmodules

@@ -1,9 +1,9 @@
 [submodule "getdatatestbed"]
 	path = getdatatestbed
-	url = git@github.com:SBFRF/getdatatestbed.git
+	url = https://github.com/SBFRF/getdatatestbed.git
 [submodule "testbedutils"]
 	path = testbedutils
-	url = git@github.com:SBFRF/testbedutils.git
+	url = https://github.com/SBFRF/testbedutils.git
 [submodule "prepdata"]
 	path = prepdata
-	url = git@github.com:SBFRF/prepdata.git
+	url = https://github.com/collins-frf/prepdata.git

frontback/frontBackNEW.py

@@ -14,6 +14,7 @@
 from prepdata import py2netCDF as p2nc
 from prepdata import postData
 
+
 def ww3simSetup(startTimeString, inputDict, allWind , allWL, allWave, wrr):
     """This Function is the master call for the  data preparation for the Coastal Model
     Test Bed (CMTB) for ww3.
@@ -108,6 +109,7 @@ def ww3simSetup(startTimeString, inputDict, allWind , allWL, allWave, wrr):
 
     return wavepacket, windpacket, wlpacket, bathy, gridFname, wrr
 
+
 def swashSimSetup(startTimeString, inputDict, allWind, allWL, allWave, wrr):
     """This Function is the master call for the  data preparation for the Coastal Model
     Test Bed (CMTB) and the Swash wave/FLow model
@@ -166,6 +168,21 @@ def swashSimSetup(startTimeString, inputDict, allWind, allWL, allWave, wrr):
 
     return wavepacket, None, WLpacket, gridDict, None, wrr
 
+
+def var1dsSimSetup(startTimeString, inputDict, allWind, allWL, allWave, allBathy, allCurrent, wrr):
+    endTime = inputDict['endTime']
+    startTime = inputDict['startTime']
+    dateStartList, wrr.dateStringList, projectStart, projectEnd = fileHandling.createTimeInfo(startTime, endTime,
+                                                                                  simulationDuration=3)
+    wrr.timestep = inputDict['modelSettings']['timestep']
+    gridFname = ''
+    ## ___________WATER LEVEL__________________
+    prepdata = PrepDataTools()
+    WLpacket = prepdata.prep_WL(allWL, allWind['epochtime'])
+
+    return allWave, allWind, WLpacket, allBathy, allCurrent, gridFname, wrr
+
+
 def genericPostProcess(startTime, inputDict, spatialData, pointData, wrr):
     """This runs the post process script for Wave Watch 3.
 
@@ -274,10 +291,6 @@ class objects as listed below:
         print("is this necessary??")
 
 
-
-
-
-
 def cshoreSimSetup(startTimeString, inputDict, allWave, allBathy, allWL, allWind, allCTD, wrr):
     """Author: David Young
     Association: USACE CHL Field Research Facility

genericWorkFlow.py

@@ -1,11 +1,11 @@
 # -*- coding: utf-8 -*-
 import matplotlib
-matplotlib.use('Agg')
+matplotlib.use('TkAgg')
 import os, getopt, sys, shutil, glob, logging, yaml, re, pickle
 import datetime as DT
 import numpy as np
 from frontback import frontBackNEW
-from getdatatestbed.getDataFRF import getObs
+from getdatatestbed.getDataFRF import getObs, getDataTestBed
 from testbedutils import fileHandling
 from prepdata import writeRunRead as wrrClass
 import glob
@@ -57,15 +57,29 @@ def Master_workFlow(inputDict):
     if generateFlag == True:
         go = getObs(projectStart-DT.timedelta(hours=3), projectEnd+DT.timedelta(hours=3)) # initialize get observation
         if modelName in ['ww3']:
-            gauge = 'waverider-26m'
+            wave_gauge = 'waverider-26m'
+            current_gauge = None
         elif modelName.lower() in ['swash', 'funwave']:
-            gauge = '8m-array'
+            wave_gauge = '8m-array'
+            current_gauge = None
         elif modelName.lower() in ['cshore']:
-            gauge = 'awac-6m'
+            wave_gauge = 'awac-6m'
+            current_gauge = None
+        elif modelName.lower() in ['1dvars']:
+            wave_gauge = ['xp100m', 'xp125m', 'xp150m', 'xp200m', '8m-array', 'paros-200-940m', 'paros-250-940m',
+                       'paros-340x-940y-top', 'sig940-300']
+            current_gauge = ['adop-3.5m', 'awac-4.5m', 'awac-6m', 'awac-8m', 'awac-11m']
+        else:
+            print("Model name: " + modelName + "not found.")
+            wave_gauge = None
+            current_gauge = None
 
-        rawspec = go.getWaveData(gaugenumber=gauge, spec=True)
+        if wave_gauge != None:
+            rawspec = go.getWaveData(gaugenumber=wave_gauge, spec=True)
         rawWL = go.getWL()
         rawwind = go.getWind(gaugenumber=0)
+        if current_gauge != None:
+            rawcurrent = go.getCurrents(gaugenumber=current_gauge, roundto=1)
 
     # ________________________________________________ RUN LOOP ________________________________________________
     # run the process through each of the above dates
@@ -105,7 +119,6 @@ def Master_workFlow(inputDict):
                                                                                                      allWL=rawWL,
                                                                                                      allWave=rawspec,
                                                                                                      wrr=wrr)
-
             elif modelName in ['swash']:
 
                 if generateFlag is True:
@@ -144,7 +157,28 @@ def Master_workFlow(inputDict):
                                                                                                      allCTD=rawctd,
                                                                                                      wrr=wrr)
                     gridFname = None
-
+            elif modelName in ['1dvars']:
+                wrr = wrrClass.var1dsIO('1dvars', workingDirectory=workingDirectory, testName=testName,
+                                        versionPrefix=version_prefix,
+                                        startTime=DT.datetime.strptime(time, '%Y-%m-%dT%H:%M:%SZ'),
+                                        simulatedRunTime=inputDict['simulationDuration'],
+                                        endTime=DT.datetime.strptime(time, '%Y-%m-%dT%H:%M:%SZ') + DT.timedelta(
+                                            hours=inputDict['simulationDuration']), runFlag=runFlag,
+                                        generateFlag=generateFlag, readFlag=analyzeFlag, pbsFlag=pbsFlag)
+                if generateFlag is True:
+                    bathygo = getDataTestBed(projectStart, projectEnd)
+                    rawbathy = bathygo.getBathyIntegratedTransect(method=1)  # grab bathymetry transects
+                    wavePacket, windPacket, wlPacket, bathyPacket, currentPacket, gridFname, wrr = frontBackNEW.var1dsSimSetup(time,
+                                                                                                                    inputDict=inputDict,
+                                                                                                                    allWind=rawwind,
+                                                                                                                    allWL=rawWL,
+                                                                                                                    allWave=rawspec,
+                                                                                                                    allBathy=rawbathy,
+                                                                                                                    allCurrent=rawcurrent,
+                                                                                                                    wrr=wrr)
+                    ctdPacket = None
+                    updateBathy = None
+
             if generateFlag is True:
                 print(" TODO: TY you're handing me back the same prepdata packets from all frontBacks")
                 print('TODO: document Packets coming from sim-setup')
@@ -162,8 +196,9 @@ def Master_workFlow(inputDict):
                     wrr.hpcNodes = inputDict['hpcSettings']['hpcNodes']
                 # write simulation files (if assigned)
                 wrr.writeAllFiles(wavePacket=wavePacket, windPacket=windPacket, wlPacket=wlPacket,
-                                  bathyPacket=bathyPacket, gridfname=gridFname,
-                                  ctdPacket=ctdPacket, updateBathy=updateBathy)
+                                  bathyPacket=bathyPacket, gridFname=gridFname,
+                                  ctdPacket=ctdPacket, currentPacket=currentPacket,
+                                  updateBathy=updateBathy)
 
             # run simulation (as appropriate)
             if runFlag is True:
@@ -175,25 +210,27 @@ def Master_workFlow(inputDict):
                 frontBackNEW.genericPostProcess(time, inputDict, spatialData=spatialData, pointData=savePointData,
                                                 wrr=wrr)
 
+            #TODO: As seen by the print statement above, the following if statement evaluates as True no matter the input
             # if it's a live run, move the plots to the output directory
-            if plotFlag is True and DT.date.today() == projectEnd or inputDict['slack'] is not None:
+            if plotFlag is True and DT.date.today() == projectEnd:
                 from testbedutils import cmtbSlack
                 moveFnames = glob.glob(wrr.plottingDirectory + '/CMTB*.png')
                 moveFnames.extend(glob.glob(wrr.plottingDirectory + '/CMTB*.gif'))
+            else:
+                # move files
+                moveFnames = glob.glob(wrr.plottingDirectory + 'CMTB*.png')
+                moveFnames.extend(glob.glob(wrr.plottingDirectory + '/CMTB*.gif'))
+                liveFileMoveToDirectory = '/mnt/gaia/cmtb'
+                for file in moveFnames:
+                    shutil.move(file,  liveFileMoveToDirectory)
+                    print('moved {} to {} '.format(file, liveFileMoveToDirectory))
+
+            if inputDict['slack'] is True:
+                myslack = cmtbSlack.slack('testbedutils/slackSettings.yml')  # initialize
+                myslack.postMessageWithFiles(f"checkout {wrr.modelName} simulations from {wrr.dateString}",
+                                             moveFnames)
+
 
-                if inputDict['slack'] is not None:
-                    myslack = cmtbSlack.slack('testbedutils/slackSettings.yml')  # initialize
-                    myslack.postMessageWithFiles(f"checkout {wrr.modelName} simulations from {wrr.dateString}",
-                                                 moveFnames)
-
-                else:
-                    # move files
-                    moveFnames = glob.glob(wrr.plottingDirectory + 'CMTB*.png')
-                    moveFnames.extend(glob.glob(wrr.plottingDirectory + '/CMTB*.gif'))
-                    liveFileMoveToDirectory = '/mnt/gaia/cmtb'
-                    for file in moveFnames:
-                        shutil.move(file,  liveFileMoveToDirectory)
-                        print('moved {} to {} '.format(file, liveFileMoveToDirectory))
             print('------------------SUCCESS-----------------------------------------')
 
         except Exception as e:
@@ -222,5 +259,4 @@ def Master_workFlow(inputDict):
     except:
         raise IOError('Input YAML file required. See yaml_files/TestBedExampleInputs/[model]_Input_example for example '
                       'yaml file.')
-
     Master_workFlow(inputDict=inputDict)

yaml_files/TestBedExampleInputs/1DVarS_Input_example.yml

@@ -0,0 +1,32 @@
+# # # # # # # # # # # #
+# Data control        #
+# # # # # # # # # # # #
+plotFlag: False                           # turn plotting on
+generateFlag: True                     # generate simulation input files (go get data, process, and write files)
+runFlag: True                          # run the simulation
+analyzeFlag: False                      # post process simulations (read files, post process data, make netCDF files, plot if desired)
+bathyLoc: integrated_bathy                  # bathymetry source.  OPTIONAL: defaults to FRF
+logging: False
+slack: None
+########################
+#simulations parameters#
+########################
+startTime: '2021-09-09T00:00:00Z'           # project start time
+endTime:   '2021-09-09T06:00:00Z'           # project End time not inclusive
+
+simulationDuration: 144               #number of hours between startTime and endTime, only used for metadata
+modelExecutable: D:/cmtb/bin/1DVarS   # path to the model    REQUIRED# this is the directory where simulation files and QA/QC plots are made and existing architecture takes over
+########################
+# path stuff           #
+########################
+modelSettings:
+  modelName: 1dvars
+  version_prefix: Base                        # controls switched with int the code
+  grid: None
+  timestep: 3                                 # duration in hours how frequently to get new data, size of individual simulations
+
+# this is the sim files, and plots
+workingDirectory: D:/cmtb/data/waveModels/1DVar                                  # REQUIRED usually [cmtbRoot]/data/waveModels
+mainDirectory: D:/cmtb/  # Set to main filepath
+# this is the base directory where netCDF files are output to,
+netCDFdir: ./data/netCDF_files/                       #optional will default to home/[whoami]/thredds_data -- this needs /

yaml_files/TestBedExampleInputs/1DVar_Input_example.yml

@@ -0,0 +1,29 @@
+# # # # # # # # # # # #
+# Data control        #
+# # # # # # # # # # # #
+pFlag: False                           # turn plotting on
+generateFlag: True                     # generate simulation input files (go get data, process, and write files)
+runFlag: True                          # run the simulation
+analyzeFlag: False                      # post process simulations (read files, post process data, make netCDF files, plot if desired)
+bathyLoc: integrated_bathy                  # bathymetry source.  OPTIONAL: defaults to FRF
+########################
+#simulations parameters#
+########################
+startTime: '2016-11-05T00:00:00Z'           # project start time
+endTime:   '2016-11-08T00:00:00Z'           # project End time not inclusive
+
+simulationDuration: 3                 # duration in hours how frequently to get new data, size of individual simulations
+modelExecutable: /home/wilsongr/work/funded_projects/USCRP_DA/project/cmtb/bin/1DVar   # path to the model    REQUIRED# this is the directory where simulation files and QA/QC plots are made and existing architecture takes over
+########################
+# path stuff           #
+########################
+modelSettings:
+  model: 1DVar
+  version_prefix: Base                        # controls switched with int the code
+  grid: None
+
+# this is the sim files, and plots
+workingDirectory: /home/wilsongr/work/funded_projects/USCRP_DA/project/cmtb/data/waveModels/1DVar                                  # REQUIRED usually [cmtbRoot]/data/waveModels
+mainDirectory: /home/wilsongr/work/funded_projects/USCRP_DA/project/cmtb/  # Set to main filepath
+# this is the base directory where netCDF files are output to,
+netCDFdir: ./netCDF                        #optional will default to home/[whoami]/thredds_data -- this needs /

yaml_files/morphModels/1dvars/1dvars_global.yml

@@ -0,0 +1,58 @@
+featureType: grid
+title:  Observations
+summary: >
+  Observations coinciding with the model run.
+  This component of the CMTB is composed of a 1D Variotional Morphological model developed by
+  Dr. Greg Wilson @ Oregon State University:
+history: posterior created as output of assimilated data for CMTB
+source: CMTB 1DVar simulations
+platform: FRF Cluster - Crunchy
+sourceUrl: (local files)
+standard_name_vocabulary: CFv25
+Metadata_Conventions: Unidata Dataset Discovery v1.0, CF-1.6
+metadata_link: N/A
+Conventions: CF-1.6
+creator_name: USACE/CHL/COAB
+creator_url: http://frf.usace.army.mil
+creator_email: frfwebmaster@usace.army.mil
+license: These data may be redistributed and used without restriction. Data are intended for scholarly use by the research community, with the express agreement that users will properly acknowledge the USACE Field Research Facility and the supporting investigator(s). Use or reproduction of these data for commercial purposes is prohibited without prior written permission.
+keywords_vocabulary: FGlobal Change Master Directory (GCMD) Earth Science Keywords; CF Standard Name Table (v23, 23 March 2013)
+keywords: Oceans > Ocean Waves > Wave Frequency, Oceans > Ocean Waves > Wave Height, Oceans > Ocean Waves > Wave Period, Oceans > Ocean Waves > Wave Spectra, Oceans > Ocean Waves > Wave Direction, DOD > DOD/USARMY/USACE/CHL/FRF > Field Research Facility, Coastal And Hydraulics Laboratory,U. S. Army Corps Of Engineers, U.S. Army, U. S. Department Of Defense, Oregon State University, College of Earth and Ocean Sciences, sea_surface_wave_variance_spectral_density, sea_surface_wave_significant_height, sea_surface_wave_from_direction,  sea_surface_wave_directional_variance_spectral_density
+processing: realtime
+organization: USACE/CHL/COAB/OSU
+publisher_url: http://frf.usace.army.mil
+infoUrl: http://frf.usace.army.mil
+publisher_email: frfwebmaster@usace.army.mil
+publisher_name: USACE/CHL/COAB
+format_version: v1.0
+institution: USACE/CHL/COAB
+contact: USACE/CHL/COAB
+contact_info: USACE/CHL/COAB
+contact_role: Owner
+contributor_name: USACE/CHL/COAB
+contributor_role: USACE/CHL/COAB
+naming_authority: FRF
+origin: USACE/CHL/COAB
+acknowledgement: Data are provided by the Field Research Facility; Coastal Observations & Analysis Branch; US Army Corps of Engineers, Duck, North Carolina.
+project: USACE/COAB observations
+id: FRF_CMTB_CSHORE_Fixed
+processing_level: L1
+geospatial_vertical_units: m
+geospatial_vertical_resolution: 0
+geospatial_vertical_min: 0
+geospatial_vertical_max: 0
+geospatial_vertical_origin: sea surface
+geospatial_lat_min: to be updated
+geospatial_lat_max: to be updated
+geospatial_lat_units: degrees_north
+geospatial_lon_min: to be updated
+geospatial_lon_max: to be updated
+geospatial_lon_units: degrees_east
+geospatial_vertical_positive: up
+time_coverage_start: '2017-05-01T22:00:00Z'
+time_coverage_end: '2017-05-01T22:00:00Z'
+deployment_start: '2017-05-01T22:00:00Z'
+instrument: N/A
+cross_shore_angle_units: degrees
+cross_shore_angle_description: cross shore angle at the USACE FRF site in DUCK, NC, clockwise from true north
+cross_shore_angle: 71.8